Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions app/controllers/artifacts_controller.rb
Original file line number Diff line number Diff line change
@@ -1,10 +1,58 @@
# Downloads one artifact. The file comes from Active Storage through a short-lived signed URL,
# so the storage service is never public. Inline for images and PDFs, a download for the rest.
#
# A `view` artifact (the page as one HTML file) also has a viewer: `view` is the Stash page
# around it, `frame` the document itself, served from Stash with a policy that lets it run
# nothing. The frame carries the archived page's own HTML, so it is treated as hostile: no
# scripts, no same-origin rights, no network except the data URLs the file holds.
class ArtifactsController < ApplicationController
# What the archived document may do inside the frame. Styles and the inlined resources
# (data URLs) render; scripts, fetches, forms, frames and every network request do not, so
# the site under review never learns that someone opened the copy. A font or an image the
# MHTML lacks falls back; the screenshot shows the page as it looked. `sandbox` without
# `allow-same-origin` keeps it off Stash's origin and its cookies.
FRAME_CSP = [
"default-src 'none'",
"img-src data:",
"media-src data:",
"font-src data:",
"style-src 'unsafe-inline' data:",
"form-action 'none'",
"base-uri *",
"sandbox allow-popups allow-popups-to-escape-sandbox"
].join("; ").freeze

def show
artifact = Artifact.includes(:capture).find_by_public_param!(params[:id])
authorize artifact.archive, :show?
disposition = artifact.image? || artifact.pdf? ? :inline : :attachment
redirect_to artifact.file.url(expires_in: 10.minutes, disposition: disposition), allow_other_host: true
end

def view
@artifact = viewable_artifact
@archive = @artifact.archive
# A parent's frame-src governs every navigation of its frames, the ones the frame starts
# too, so whatever the rewrite missed, the frame cannot leave Stash.
response.headers["Content-Security-Policy"] = "frame-src 'self'"
end

def frame
artifact = viewable_artifact
response.headers["Content-Security-Policy"] = FRAME_CSP
response.headers["X-Frame-Options"] = "SAMEORIGIN"
response.headers["Referrer-Policy"] = "no-referrer"
response.headers["Cache-Control"] = "private, max-age=600"
send_data artifact.file.download, type: "text/html; charset=utf-8", disposition: "inline"
end

private

def viewable_artifact
artifact = Artifact.includes(capture: :archive).find_by_public_param!(params[:id])
authorize artifact.archive, :show?
raise ActiveRecord::RecordNotFound unless artifact.view?

artifact
end
end
8 changes: 7 additions & 1 deletion app/models/artifact.rb
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
# rendition a derived copy in a viewable form: the print PDF of a page, the PDF of a deck
# screenshot a PNG of the page or a poster of a video
# metadata JSON or text about the thing: repo metadata, refs, yt-dlp info, a normalized post
# view a copy the browser shows on the archive page: the page as one HTML file, with
# its styles and images inlined (Capturers::Mhtml), in a sandboxed frame
class Artifact < CentralRecord
# The prefix "art" is registered in internal-docs (encoded-id-prefixes.md).
include EncodedIds::UuidIdentifiable
set_public_id_prefix "art"

ROLES = %w[original rendition screenshot metadata].freeze
ROLES = %w[original rendition screenshot metadata view].freeze

belongs_to :capture, inverse_of: :artifacts
has_one :archive, through: :capture
Expand All @@ -33,4 +35,8 @@ def pdf?
def text?
content_type.start_with?("text/") || content_type == "application/json"
end

def view?
role == "view"
end
end
218 changes: 218 additions & 0 deletions app/services/capturers/mhtml.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,218 @@
require "mail"
require "base64"
require "uri"
require "nokogiri"

module Capturers
# Turns the MHTML that Chromium saved into one HTML file that a browser can show: every
# resource the MHTML carries (styles, images) is inlined as a data URL, every link is made
# absolute and opens in a new tab, and everything that could reach the network or run is
# taken out. The result is the `view` artifact, shown on the archive page in a sandboxed frame
# with scripts off and no network. The document is parsed (Nokogiri's HTML5 parser) and
# rewritten node by node, never with regular expressions over the markup, so an odd attribute
# cannot slip a tag past the rewrite.
#
# Chromium's MHTML leaves two things out: web fonts, and what was drawn on a <canvas>. The
# capturer passes those in: `resources` are the responses it saw on the network (fonts and
# images the MHTML lacks), `canvases` the PNG data URL of each canvas in document order, which
# replaces the canvas with an <img>.
#
# MHTML is MIME multipart/related: the first part is the document, the others its resources,
# each with a Content-Location (its URL) and base64 or quoted-printable content.
module Mhtml
# Attributes whose value is one URL to inline, on any element but a link.
URL_ATTRIBUTES = %w[src href poster background data].freeze
# <link rel> values that make the browser touch the network for nothing the copy needs.
NETWORK_HINTS = %w[preconnect dns-prefetch prefetch prerender preload modulepreload].freeze
# <meta http-equiv> values that navigate, loosen or leak.
META_EQUIVS = %w[refresh content-security-policy x-dns-prefetch-control].freeze
# The resources the view inlines once, however often the page references them.
MAX_PART_BYTES = 10.megabytes

Part = Data.define(:location, :content_type, :body)

class Inliner
def initialize(location, resources)
@resources = resources
@data_urls = {}
@base = location
end

def call(html, canvases)
doc = Nokogiri::HTML5.parse(html, @base, "UTF-8")
take_base(doc)
remove_active_parts(doc)
replace_canvases(doc, canvases)
rewrite_links(doc)
rewrite_resources(doc)
rewrite_styles(doc)
note(doc)
doc.to_html
end

private

# The page's own <base> moves the resolution base; the view gets one of its own.
def take_base(doc)
if (own = doc.at_css("base[href]"))
@base = absolute(own["href"], @base)
end
doc.css("base").each(&:remove)
end

# Scripts never run in the frame, so they only add weight. The metas and link hints could
# navigate the frame or make the browser call the archived site's hosts; the copy has to be
# silent. Event handler attributes are blocked by the policy too; drop them anyway.
def remove_active_parts(doc)
doc.css("script, noscript > script").each(&:remove)
doc.css("meta").each do |meta|
equiv = meta["http-equiv"].to_s.strip.downcase
name = meta["name"].to_s.strip.downcase
meta.remove if META_EQUIVS.include?(equiv) || name == "referrer"
end
doc.css("link").each do |link|
rels = link["rel"].to_s.downcase.split
link.remove if rels.intersect?(NETWORK_HINTS)
end
doc.css("*").each do |node|
node.attribute_nodes.each { |attribute| attribute.remove if attribute.name.start_with?("on") }
node.remove_attribute("ping")
end
end

# Each <canvas>, in document order, becomes an <img> of what was drawn on it. A canvas
# with no image (tainted, or WebGL without a kept buffer) stays as it is, empty.
def replace_canvases(doc, canvases)
doc.css("canvas").each_with_index do |canvas, index|
data = canvases[index].to_s
next unless data.start_with?("data:image/")

img = doc.create_element("img", src: data, alt: "")
%w[id class style width height].each { |name| img[name] = canvas[name] if canvas[name] }
canvas.attribute_nodes.each { |a| img[a.name] = a.value if a.name.start_with?("aria-") }
canvas.replace(img)
end
end

# A person's link: absolute, in a new tab, no referrer, never a data URL. A fragment stays
# a fragment, so it scrolls the copy. The sandboxed frame has no scripts and no same-origin
# rights, so these links are the one way out of it.
def rewrite_links(doc)
doc.css("a, area").each do |link|
href = link["href"].to_s.strip
unless href.empty? || href.start_with?("#")
link["href"] = absolute(href, @base)
link.remove_attribute("href") if link["href"].match?(/\A(javascript|data|blob|vbscript):/i)
end
link["target"] = "_blank"
link["rel"] = "noopener noreferrer nofollow"
end
end

def rewrite_resources(doc)
doc.css("*").each do |node|
next if %w[a area base].include?(node.name)

URL_ATTRIBUTES.each do |name|
node[name] = resolve(node[name]) if node[name]
end
node["srcset"] = rewrite_srcset(node["srcset"]) if node["srcset"]
node["style"] = rewrite_css(node["style"]) if node["style"]
end
end

def rewrite_styles(doc)
doc.css("style").each { |style| style.content = rewrite_css(style.content) }
end

# The srcset grammar: candidates separated by a comma and whitespace, each a URL and an
# optional descriptor. A data URL carries commas of its own, so the split is on ", " only
# after a descriptor or at a candidate boundary, never inside a URL.
def rewrite_srcset(value)
value.strip.split(/,\s+(?=\S)/).map do |candidate|
url, descriptor = candidate.strip.split(/\s+/, 2)
[ resolve(url.to_s), descriptor ].compact.join(" ")
end.join(", ")
end

def rewrite_css(css, base: @base, depth: 0)
return css.to_s if depth > 3 || css.nil?

# Linear patterns: a hostile stylesheet full of `url(` must not stall the capture. The value
# may not hold a quote or a parenthesis, so a second `url(` ends the attempt at once.
css = css.gsub(/url\(\s*(["']?)([^"'()]*)\1\s*\)/i) do
quote, value = Regexp.last_match(1), Regexp.last_match(2)
"url(#{quote}#{resolve(value, base: base, depth: depth)}#{quote})"
end
css.gsub(/@import\s+(["'])([^"']*)\1/i) do
quote, value = Regexp.last_match(1), Regexp.last_match(2)
"@import #{quote}#{resolve(value, base: base, depth: depth)}#{quote}"
end
end

# The data URL of the resource at `value`, or its absolute URL when nothing was captured
# (the policy then blocks it, and the element falls back). HTML attributes carry `&amp;`
# where the URL has `&`; Nokogiri gives the decoded value, so nothing to undo here.
def resolve(value, base: @base, depth: 0)
value = value.to_s.strip
return value if value.empty? || value.start_with?("data:", "#", "about:", "javascript:", "blob:", "cid:")

url = absolute(value, base)
part = @resources[url] || @resources[value]
return url unless part
return url if part.body.bytesize > MAX_PART_BYTES

@data_urls[part] ||= begin
body = part.body
body = rewrite_css(body.dup.force_encoding("UTF-8").scrub, base: part.location, depth: depth + 1) if part.content_type == "text/css"
"data:#{part.content_type};base64,#{Base64.strict_encode64(body)}"
end
end

def absolute(value, base)
return value if base.to_s.empty?

URI.join(base, URI::RFC2396_PARSER.escape(value.to_s.strip)).to_s
rescue URI::Error
value
end

def note(doc)
text = " Archived by Stash. A copy of #{@base.to_s.gsub("--", "%2D%2D")}; styles and images are inlined, scripts are removed and do not run. "
comment = Nokogiri::XML::Comment.new(doc, text)
if (head = doc.at_css("head"))
head.prepend_child(comment)
else
doc.root&.prepend_child(comment)
end
end
end

module_function

# Reads the MHTML file and returns the HTML string. Raises ToolFailed when the file is not
# MHTML or has no document part. `resources` is { url => Part } for what the MHTML lacks;
# a part in the MHTML wins. `canvases` is the data URL (or nil) of each <canvas>, in order.
def inline(path, resources: {}, canvases: [])
message = Mail.read_from_string(::File.binread(path))
parts = message.multipart? ? message.parts : [ message ]
parts = parts.map { |part| part_from(part) }.compact
document = parts.find { |part| part.content_type.start_with?("text/html") }
raise ToolFailed, "the MHTML has no HTML part" unless document

known = resources.merge(parts.reject { |part| part.equal?(document) }.to_h { |part| [ part.location, part ] })
html = document.body.dup.force_encoding("UTF-8").scrub
Inliner.new(document.location, known).call(html, canvases)
rescue Mail::Field::ParseError, ArgumentError, EncodingError => e
raise ToolFailed, "the MHTML could not be read: #{e.class}"
end

def part_from(part)
location = part.header["Content-Location"]&.value.to_s.strip
type = part.content_type.to_s.split(";").first.to_s.strip.downcase
return nil if location.empty? && !type.start_with?("text/html")

Part.new(location: location, content_type: type, body: part.decoded.to_s)
end
end
end
68 changes: 67 additions & 1 deletion app/services/capturers/page.rb
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,28 @@ def call
raise
end

[
outputs = [
output("original", "page.mhtml", content_type: "multipart/related"),
output("screenshot", "page.png", content_type: "image/png"),
output("rendition", "page.pdf", content_type: "application/pdf")
]
outputs.insert(1, output("view", "page.html", content_type: "text/html")) if ::File.exist?(path("page.html"))
outputs
rescue Ferrum::Error => e
raise ToolFailed, "Chromium: #{e.class}: #{e.message.to_s.first(300)}"
end

# One response from the network is kept for the view up to this size, and all of them up
# to the total. Fonts and images only; the MHTML already carries the HTML and the CSS.
VIEW_RESOURCE_BYTES = 5.megabytes
VIEW_RESOURCES_TOTAL = 60.megabytes
# No view for a page whose MHTML is larger than this: the view is the MHTML's resources
# as base64 and the frame loads the whole file into memory to serve it.
VIEW_MAX_MHTML_BYTES = 50.megabytes
# The largest canvas drawing kept; a page can make toDataURL answer anything.
VIEW_CANVAS_BYTES = 5.megabytes
VIEW_TYPES = %r{\A(font/|image/|application/(x-)?font|application/vnd\.ms-fontobject)}

private

def capture_in(browser)
Expand All @@ -72,6 +85,8 @@ def capture_in(browser)
screenshot(page, height)
log("saving the PDF (#{elapsed}s)")
page.pdf(path: path("page.pdf"), format: :A4, print_background: true)
log("saving the view (#{elapsed}s)")
write_view(page)
log("done (#{elapsed}s)")
end

Expand Down Expand Up @@ -183,6 +198,57 @@ def screenshot(page, height)
::File.binwrite(path("page.png"), Base64.decode64(result.fetch("data")))
end

# The copy a browser can show (Capturers::Mhtml): the MHTML as one HTML file, with the
# fonts and images Chromium's MHTML leaves out taken from the network traffic of this load,
# and each <canvas> replaced by what was drawn on it. A view that cannot be made is logged
# and skipped; the MHTML, PNG and PDF stand on their own.
def write_view(page)
mhtml_bytes = ::File.size(path("page.mhtml"))
if mhtml_bytes > VIEW_MAX_MHTML_BYTES
log("no view: the MHTML is #{mhtml_bytes} bytes, more than #{VIEW_MAX_MHTML_BYTES}")
return
end

html = Mhtml.inline(path("page.mhtml"), resources: view_resources(page), canvases: canvas_images(page))
::File.write(path("page.html"), html)
log("view: #{html.bytesize} bytes as one HTML file")
rescue StandardError => e
# Any failure here, a hostile stylesheet included, costs the view and nothing else.
log("no view: #{e.class}: #{e.message.to_s.first(200)}")
end

def view_resources(page)
total = 0
page.network.traffic.each_with_object({}) do |exchange, resources|
response = exchange.response
next unless response&.status == 200 && response.content_type.to_s.match?(VIEW_TYPES)
next if resources.key?(response.url)

body = response.body
next if body.nil? || body.bytesize > VIEW_RESOURCE_BYTES || (total += body.bytesize) > VIEW_RESOURCES_TOTAL

part = Mhtml::Part.new(location: response.url, content_type: response.content_type, body: body)
resources[response.url] = part
# The page names the URL it asked for; a redirect answers from another.
resources[exchange.request.url] ||= part if exchange.request && exchange.request.url != response.url
rescue Ferrum::Error
next
end
end

# What each canvas shows, as a PNG data URL, in document order; nil where it cannot be read
# (a tainted canvas, or WebGL without a kept drawing buffer).
def canvas_images(page)
images = Array(page.evaluate(<<~JS))
Array.from(document.querySelectorAll("canvas")).map((canvas) => {
try { return canvas.width && canvas.height ? canvas.toDataURL("image/png") : null; } catch (_) { return null; }
})
JS
images.map { |data| data.is_a?(String) && data.start_with?("data:image/") && data.bytesize <= VIEW_CANVAS_BYTES ? data : nil }
rescue Ferrum::Error
[]
end

def with_browser
browser = self.class.new_browser
yield browser
Expand Down
Loading
Loading