diff --git a/app/controllers/artifacts_controller.rb b/app/controllers/artifacts_controller.rb index aebb770..b522032 100644 --- a/app/controllers/artifacts_controller.rb +++ b/app/controllers/artifacts_controller.rb @@ -1,10 +1,58 @@ # Downloads one artifact. The file comes from Active Storage through a short-lived signed URL, # so the storage service is never public. Inline for images and PDFs, a download for the rest. +# +# A `view` artifact (the page as one HTML file) also has a viewer: `view` is the Stash page +# around it, `frame` the document itself, served from Stash with a policy that lets it run +# nothing. The frame carries the archived page's own HTML, so it is treated as hostile: no +# scripts, no same-origin rights, no network except the data URLs the file holds. class ArtifactsController < ApplicationController + # What the archived document may do inside the frame. Styles and the inlined resources + # (data URLs) render; scripts, fetches, forms, frames and every network request do not, so + # the site under review never learns that someone opened the copy. A font or an image the + # MHTML lacks falls back; the screenshot shows the page as it looked. `sandbox` without + # `allow-same-origin` keeps it off Stash's origin and its cookies. + FRAME_CSP = [ + "default-src 'none'", + "img-src data:", + "media-src data:", + "font-src data:", + "style-src 'unsafe-inline' data:", + "form-action 'none'", + "base-uri *", + "sandbox allow-popups allow-popups-to-escape-sandbox" + ].join("; ").freeze + def show artifact = Artifact.includes(:capture).find_by_public_param!(params[:id]) authorize artifact.archive, :show? disposition = artifact.image? || artifact.pdf? ? :inline : :attachment redirect_to artifact.file.url(expires_in: 10.minutes, disposition: disposition), allow_other_host: true end + + def view + @artifact = viewable_artifact + @archive = @artifact.archive + # A parent's frame-src governs every navigation of its frames, the ones the frame starts + # too, so whatever the rewrite missed, the frame cannot leave Stash. + response.headers["Content-Security-Policy"] = "frame-src 'self'" + end + + def frame + artifact = viewable_artifact + response.headers["Content-Security-Policy"] = FRAME_CSP + response.headers["X-Frame-Options"] = "SAMEORIGIN" + response.headers["Referrer-Policy"] = "no-referrer" + response.headers["Cache-Control"] = "private, max-age=600" + send_data artifact.file.download, type: "text/html; charset=utf-8", disposition: "inline" + end + + private + + def viewable_artifact + artifact = Artifact.includes(capture: :archive).find_by_public_param!(params[:id]) + authorize artifact.archive, :show? + raise ActiveRecord::RecordNotFound unless artifact.view? + + artifact + end end diff --git a/app/models/artifact.rb b/app/models/artifact.rb index a39969f..bc3c479 100644 --- a/app/models/artifact.rb +++ b/app/models/artifact.rb @@ -5,12 +5,14 @@ # rendition a derived copy in a viewable form: the print PDF of a page, the PDF of a deck # screenshot a PNG of the page or a poster of a video # metadata JSON or text about the thing: repo metadata, refs, yt-dlp info, a normalized post +# view a copy the browser shows on the archive page: the page as one HTML file, with +# its styles and images inlined (Capturers::Mhtml), in a sandboxed frame class Artifact < CentralRecord # The prefix "art" is registered in internal-docs (encoded-id-prefixes.md). include EncodedIds::UuidIdentifiable set_public_id_prefix "art" - ROLES = %w[original rendition screenshot metadata].freeze + ROLES = %w[original rendition screenshot metadata view].freeze belongs_to :capture, inverse_of: :artifacts has_one :archive, through: :capture @@ -33,4 +35,8 @@ def pdf? def text? content_type.start_with?("text/") || content_type == "application/json" end + + def view? + role == "view" + end end diff --git a/app/services/capturers/mhtml.rb b/app/services/capturers/mhtml.rb new file mode 100644 index 0000000..e7a1661 --- /dev/null +++ b/app/services/capturers/mhtml.rb @@ -0,0 +1,218 @@ +require "mail" +require "base64" +require "uri" +require "nokogiri" + +module Capturers + # Turns the MHTML that Chromium saved into one HTML file that a browser can show: every + # resource the MHTML carries (styles, images) is inlined as a data URL, every link is made + # absolute and opens in a new tab, and everything that could reach the network or run is + # taken out. The result is the `view` artifact, shown on the archive page in a sandboxed frame + # with scripts off and no network. The document is parsed (Nokogiri's HTML5 parser) and + # rewritten node by node, never with regular expressions over the markup, so an odd attribute + # cannot slip a tag past the rewrite. + # + # Chromium's MHTML leaves two things out: web fonts, and what was drawn on a . The + # capturer passes those in: `resources` are the responses it saw on the network (fonts and + # images the MHTML lacks), `canvases` the PNG data URL of each canvas in document order, which + # replaces the canvas with an . + # + # MHTML is MIME multipart/related: the first part is the document, the others its resources, + # each with a Content-Location (its URL) and base64 or quoted-printable content. + module Mhtml + # Attributes whose value is one URL to inline, on any element but a link. + URL_ATTRIBUTES = %w[src href poster background data].freeze + # values that make the browser touch the network for nothing the copy needs. + NETWORK_HINTS = %w[preconnect dns-prefetch prefetch prerender preload modulepreload].freeze + # values that navigate, loosen or leak. + META_EQUIVS = %w[refresh content-security-policy x-dns-prefetch-control].freeze + # The resources the view inlines once, however often the page references them. + MAX_PART_BYTES = 10.megabytes + + Part = Data.define(:location, :content_type, :body) + + class Inliner + def initialize(location, resources) + @resources = resources + @data_urls = {} + @base = location + end + + def call(html, canvases) + doc = Nokogiri::HTML5.parse(html, @base, "UTF-8") + take_base(doc) + remove_active_parts(doc) + replace_canvases(doc, canvases) + rewrite_links(doc) + rewrite_resources(doc) + rewrite_styles(doc) + note(doc) + doc.to_html + end + + private + + # The page's own moves the resolution base; the view gets one of its own. + def take_base(doc) + if (own = doc.at_css("base[href]")) + @base = absolute(own["href"], @base) + end + doc.css("base").each(&:remove) + end + + # Scripts never run in the frame, so they only add weight. The metas and link hints could + # navigate the frame or make the browser call the archived site's hosts; the copy has to be + # silent. Event handler attributes are blocked by the policy too; drop them anyway. + def remove_active_parts(doc) + doc.css("script, noscript > script").each(&:remove) + doc.css("meta").each do |meta| + equiv = meta["http-equiv"].to_s.strip.downcase + name = meta["name"].to_s.strip.downcase + meta.remove if META_EQUIVS.include?(equiv) || name == "referrer" + end + doc.css("link").each do |link| + rels = link["rel"].to_s.downcase.split + link.remove if rels.intersect?(NETWORK_HINTS) + end + doc.css("*").each do |node| + node.attribute_nodes.each { |attribute| attribute.remove if attribute.name.start_with?("on") } + node.remove_attribute("ping") + end + end + + # Each , in document order, becomes an of what was drawn on it. A canvas + # with no image (tainted, or WebGL without a kept buffer) stays as it is, empty. + def replace_canvases(doc, canvases) + doc.css("canvas").each_with_index do |canvas, index| + data = canvases[index].to_s + next unless data.start_with?("data:image/") + + img = doc.create_element("img", src: data, alt: "") + %w[id class style width height].each { |name| img[name] = canvas[name] if canvas[name] } + canvas.attribute_nodes.each { |a| img[a.name] = a.value if a.name.start_with?("aria-") } + canvas.replace(img) + end + end + + # A person's link: absolute, in a new tab, no referrer, never a data URL. A fragment stays + # a fragment, so it scrolls the copy. The sandboxed frame has no scripts and no same-origin + # rights, so these links are the one way out of it. + def rewrite_links(doc) + doc.css("a, area").each do |link| + href = link["href"].to_s.strip + unless href.empty? || href.start_with?("#") + link["href"] = absolute(href, @base) + link.remove_attribute("href") if link["href"].match?(/\A(javascript|data|blob|vbscript):/i) + end + link["target"] = "_blank" + link["rel"] = "noopener noreferrer nofollow" + end + end + + def rewrite_resources(doc) + doc.css("*").each do |node| + next if %w[a area base].include?(node.name) + + URL_ATTRIBUTES.each do |name| + node[name] = resolve(node[name]) if node[name] + end + node["srcset"] = rewrite_srcset(node["srcset"]) if node["srcset"] + node["style"] = rewrite_css(node["style"]) if node["style"] + end + end + + def rewrite_styles(doc) + doc.css("style").each { |style| style.content = rewrite_css(style.content) } + end + + # The srcset grammar: candidates separated by a comma and whitespace, each a URL and an + # optional descriptor. A data URL carries commas of its own, so the split is on ", " only + # after a descriptor or at a candidate boundary, never inside a URL. + def rewrite_srcset(value) + value.strip.split(/,\s+(?=\S)/).map do |candidate| + url, descriptor = candidate.strip.split(/\s+/, 2) + [ resolve(url.to_s), descriptor ].compact.join(" ") + end.join(", ") + end + + def rewrite_css(css, base: @base, depth: 0) + return css.to_s if depth > 3 || css.nil? + + # Linear patterns: a hostile stylesheet full of `url(` must not stall the capture. The value + # may not hold a quote or a parenthesis, so a second `url(` ends the attempt at once. + css = css.gsub(/url\(\s*(["']?)([^"'()]*)\1\s*\)/i) do + quote, value = Regexp.last_match(1), Regexp.last_match(2) + "url(#{quote}#{resolve(value, base: base, depth: depth)}#{quote})" + end + css.gsub(/@import\s+(["'])([^"']*)\1/i) do + quote, value = Regexp.last_match(1), Regexp.last_match(2) + "@import #{quote}#{resolve(value, base: base, depth: depth)}#{quote}" + end + end + + # The data URL of the resource at `value`, or its absolute URL when nothing was captured + # (the policy then blocks it, and the element falls back). HTML attributes carry `&` + # where the URL has `&`; Nokogiri gives the decoded value, so nothing to undo here. + def resolve(value, base: @base, depth: 0) + value = value.to_s.strip + return value if value.empty? || value.start_with?("data:", "#", "about:", "javascript:", "blob:", "cid:") + + url = absolute(value, base) + part = @resources[url] || @resources[value] + return url unless part + return url if part.body.bytesize > MAX_PART_BYTES + + @data_urls[part] ||= begin + body = part.body + body = rewrite_css(body.dup.force_encoding("UTF-8").scrub, base: part.location, depth: depth + 1) if part.content_type == "text/css" + "data:#{part.content_type};base64,#{Base64.strict_encode64(body)}" + end + end + + def absolute(value, base) + return value if base.to_s.empty? + + URI.join(base, URI::RFC2396_PARSER.escape(value.to_s.strip)).to_s + rescue URI::Error + value + end + + def note(doc) + text = " Archived by Stash. A copy of #{@base.to_s.gsub("--", "%2D%2D")}; styles and images are inlined, scripts are removed and do not run. " + comment = Nokogiri::XML::Comment.new(doc, text) + if (head = doc.at_css("head")) + head.prepend_child(comment) + else + doc.root&.prepend_child(comment) + end + end + end + + module_function + + # Reads the MHTML file and returns the HTML string. Raises ToolFailed when the file is not + # MHTML or has no document part. `resources` is { url => Part } for what the MHTML lacks; + # a part in the MHTML wins. `canvases` is the data URL (or nil) of each , in order. + def inline(path, resources: {}, canvases: []) + message = Mail.read_from_string(::File.binread(path)) + parts = message.multipart? ? message.parts : [ message ] + parts = parts.map { |part| part_from(part) }.compact + document = parts.find { |part| part.content_type.start_with?("text/html") } + raise ToolFailed, "the MHTML has no HTML part" unless document + + known = resources.merge(parts.reject { |part| part.equal?(document) }.to_h { |part| [ part.location, part ] }) + html = document.body.dup.force_encoding("UTF-8").scrub + Inliner.new(document.location, known).call(html, canvases) + rescue Mail::Field::ParseError, ArgumentError, EncodingError => e + raise ToolFailed, "the MHTML could not be read: #{e.class}" + end + + def part_from(part) + location = part.header["Content-Location"]&.value.to_s.strip + type = part.content_type.to_s.split(";").first.to_s.strip.downcase + return nil if location.empty? && !type.start_with?("text/html") + + Part.new(location: location, content_type: type, body: part.decoded.to_s) + end + end +end diff --git a/app/services/capturers/page.rb b/app/services/capturers/page.rb index 8ccb661..a0a13a3 100644 --- a/app/services/capturers/page.rb +++ b/app/services/capturers/page.rb @@ -43,15 +43,28 @@ def call raise end - [ + outputs = [ output("original", "page.mhtml", content_type: "multipart/related"), output("screenshot", "page.png", content_type: "image/png"), output("rendition", "page.pdf", content_type: "application/pdf") ] + outputs.insert(1, output("view", "page.html", content_type: "text/html")) if ::File.exist?(path("page.html")) + outputs rescue Ferrum::Error => e raise ToolFailed, "Chromium: #{e.class}: #{e.message.to_s.first(300)}" end + # One response from the network is kept for the view up to this size, and all of them up + # to the total. Fonts and images only; the MHTML already carries the HTML and the CSS. + VIEW_RESOURCE_BYTES = 5.megabytes + VIEW_RESOURCES_TOTAL = 60.megabytes + # No view for a page whose MHTML is larger than this: the view is the MHTML's resources + # as base64 and the frame loads the whole file into memory to serve it. + VIEW_MAX_MHTML_BYTES = 50.megabytes + # The largest canvas drawing kept; a page can make toDataURL answer anything. + VIEW_CANVAS_BYTES = 5.megabytes + VIEW_TYPES = %r{\A(font/|image/|application/(x-)?font|application/vnd\.ms-fontobject)} + private def capture_in(browser) @@ -72,6 +85,8 @@ def capture_in(browser) screenshot(page, height) log("saving the PDF (#{elapsed}s)") page.pdf(path: path("page.pdf"), format: :A4, print_background: true) + log("saving the view (#{elapsed}s)") + write_view(page) log("done (#{elapsed}s)") end @@ -183,6 +198,57 @@ def screenshot(page, height) ::File.binwrite(path("page.png"), Base64.decode64(result.fetch("data"))) end + # The copy a browser can show (Capturers::Mhtml): the MHTML as one HTML file, with the + # fonts and images Chromium's MHTML leaves out taken from the network traffic of this load, + # and each replaced by what was drawn on it. A view that cannot be made is logged + # and skipped; the MHTML, PNG and PDF stand on their own. + def write_view(page) + mhtml_bytes = ::File.size(path("page.mhtml")) + if mhtml_bytes > VIEW_MAX_MHTML_BYTES + log("no view: the MHTML is #{mhtml_bytes} bytes, more than #{VIEW_MAX_MHTML_BYTES}") + return + end + + html = Mhtml.inline(path("page.mhtml"), resources: view_resources(page), canvases: canvas_images(page)) + ::File.write(path("page.html"), html) + log("view: #{html.bytesize} bytes as one HTML file") + rescue StandardError => e + # Any failure here, a hostile stylesheet included, costs the view and nothing else. + log("no view: #{e.class}: #{e.message.to_s.first(200)}") + end + + def view_resources(page) + total = 0 + page.network.traffic.each_with_object({}) do |exchange, resources| + response = exchange.response + next unless response&.status == 200 && response.content_type.to_s.match?(VIEW_TYPES) + next if resources.key?(response.url) + + body = response.body + next if body.nil? || body.bytesize > VIEW_RESOURCE_BYTES || (total += body.bytesize) > VIEW_RESOURCES_TOTAL + + part = Mhtml::Part.new(location: response.url, content_type: response.content_type, body: body) + resources[response.url] = part + # The page names the URL it asked for; a redirect answers from another. + resources[exchange.request.url] ||= part if exchange.request && exchange.request.url != response.url + rescue Ferrum::Error + next + end + end + + # What each canvas shows, as a PNG data URL, in document order; nil where it cannot be read + # (a tainted canvas, or WebGL without a kept drawing buffer). + def canvas_images(page) + images = Array(page.evaluate(<<~JS)) + Array.from(document.querySelectorAll("canvas")).map((canvas) => { + try { return canvas.width && canvas.height ? canvas.toDataURL("image/png") : null; } catch (_) { return null; } + }) + JS + images.map { |data| data.is_a?(String) && data.start_with?("data:image/") && data.bytesize <= VIEW_CANVAS_BYTES ? data : nil } + rescue Ferrum::Error + [] + end + def with_browser browser = self.class.new_browser yield browser diff --git a/app/views/archives/show.html.erb b/app/views/archives/show.html.erb index cc26aa0..f1c836b 100644 --- a/app/views/archives/show.html.erb +++ b/app/views/archives/show.html.erb @@ -134,9 +134,16 @@ <% end %> <% screenshot = capture.artifacts.find { |a| a.role == "screenshot" } %> + <% view = capture.artifacts.find(&:view?) %> + + <% if view %> + <%= link_to view_artifact_path(view), class: "btn btn-primary" do %> + <%= icon(:eye) %>View the archived page + <% end %> + <% end %> <% if screenshot %> - <%= link_to artifact_path(screenshot), class: "block" do %> + <%= link_to (view ? view_artifact_path(view) : artifact_path(screenshot)), class: "block" do %> <%= image_tag artifact_path(screenshot), alt: "Screenshot of #{@archive.url}", class: "max-h-96 w-full rounded-[var(--radius-control)] border border-hairline object-cover object-top", loading: "lazy" %> <% end %> <% end %> diff --git a/app/views/artifacts/view.html.erb b/app/views/artifacts/view.html.erb new file mode 100644 index 0000000..62d42ab --- /dev/null +++ b/app/views/artifacts/view.html.erb @@ -0,0 +1,35 @@ +<% content_for :title, "View · #{@archive.host}" %> + +
+
+ <%= link_to archive_path(@archive), class: "link-quiet inline-flex items-center gap-1" do %> + <%= icon(:arrow_left) %>Back to the archive + <% end %> + + + Archived copy of <%= @archive.url %> from + <%= time_tag @artifact.created_at, @artifact.created_at.utc.strftime("%Y-%m-%d %H:%M UTC") %>. Scripts do not run, + and nothing is loaded from the live site, so it never learns that you looked. Links open the live site in a new + tab. + + + + <%= link_to artifact_path(@artifact), class: "btn btn-quiet btn-sm" do %> + <%= icon(:download) %>Download the HTML + <% end %> + + <%= link_to @archive.url, class: "btn btn-secondary btn-sm", rel: "noopener noreferrer nofollow", target: "_blank" do %> + <%= icon(:external_link) %>Live site + <% end %> + +
+ + <%# The frame has no scripts, no same-origin rights and no network (ArtifactsController::FRAME_CSP). %> + +
diff --git a/config/routes.rb b/config/routes.rb index 3ac25b8..0e1ffa4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -25,7 +25,11 @@ end # One artifact's file, by public id (art_...), through a signed storage URL. - resources :artifacts, only: :show + resources :artifacts, only: :show do + # The viewer for a `view` artifact: the page around the frame, and the frame's document. + get :view, on: :member + get :frame, on: :member + end # The archive API for patch apps, such as Krater. Bearer tokens from Weave, no session. namespace :api do diff --git a/docs/api.md b/docs/api.md index d5aa8af..00a0bd5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -123,7 +123,7 @@ Newest first, at most 50. The URL is normalized before the lookup, so `?utm_sour | Method | What it keeps | |---|---| -| `page` | the rendered page as MHTML, a full-page PNG, a print PDF | +| `page` | the rendered page as MHTML, the same page as one HTML file (`view`, shown in Stash's viewer), a full-page PNG, a print PDF | | `github` | the repository record, README, releases, last 100 commits and contributors, from the GitHub API | | `git` | a mirror clone as a git bundle, and the list of refs | | `document` | the document as served (`.pptx`, `.docx`, `.pdf`, a Google export) and a PDF rendition | diff --git a/docs/design.md b/docs/design.md index e0023a0..c3e42b8 100644 --- a/docs/design.md +++ b/docs/design.md @@ -64,7 +64,7 @@ Each capturer is one class in `app/services/capturers/`, with one public method | Method | Tool | Applies to | Artifacts | |---|---|---|---| -| `page` | headless Chromium (Ferrum), one load | every `http(s)` URL that serves HTML | `original`: MHTML of the rendered page, `screenshot`: full-page PNG, `rendition`: print PDF | +| `page` | headless Chromium (Ferrum), one load | every `http(s)` URL that serves HTML | `original`: MHTML of the rendered page, `view`: the same page as one HTML file for the viewer, `screenshot`: full-page PNG, `rendition`: print PDF | | `git` | `git clone --mirror`, `git bundle` | GitHub, GitLab, Codeberg, Bitbucket, sr.ht, any `.git` URL | `original`: `.bundle`, `metadata`: refs and HEAD | | `github` | GitHub REST API | `github.com//` | `metadata`: repo JSON, releases, last 100 commits, contributors; README as served | | `document` | HTTP download, Google export URLs, LibreOffice | Google Docs, Slides, Sheets, Drawings; direct `.pdf`, `.pptx`, `.docx`, `.xlsx`, `.odp`, `.odt`, `.key` and others | `original`: the file as served; `rendition`: PDF | @@ -169,8 +169,17 @@ Stash uses `patchwork-central` (not the restricted database), schema `stash`. Ar - `staging`: deployed at `stash-staging.patchworklabs.org`. Weave staging as IdP, the staging central database on `hem`, its own bucket. The construction border and the STAGING label on every page. - `production`: deployed at `stash.patchworklabs.org`. +## Viewing a page + +A browser does not render an MHTML served over HTTP, so the `page` capturer also writes a `view` artifact: the MHTML turned into one HTML file by `Capturers::Mhtml`. The document is parsed with Nokogiri's HTML5 parser and rewritten node by node: every resource the MHTML carries (styles, images) is inlined as a data URL, once however often the page uses it; the web fonts and images Chromium's MHTML leaves out come from the network traffic of the load (fonts and images only, 5 MB each, 60 MB in all); each `` becomes a PNG of what it drew; every link is made absolute, opens in a new tab with `rel="noopener noreferrer nofollow"`, and is never a `javascript:` or `data:` URL. Scripts, ``, the referrer meta, `preconnect` and the other link hints, `ping` and `on*` attributes are removed, so the copy makes no request of its own. The page's own `` is used for resolution, then dropped. + +The archive page offers "View the archived page", which shows that file in an iframe served from Stash (`ArtifactsController#frame`) with a policy that runs nothing: `default-src 'none'`, inline styles and data URLs allowed, no network at all, `sandbox` without `allow-same-origin`, so the archived page never touches Stash's origin or cookies, and the site under review never learns that someone opened the copy. The page around the frame carries `frame-src 'self'`, so the frame cannot navigate away from Stash whatever the rewrite missed. A font or an image the capture did not keep falls back; the screenshot shows the page as it looked. A page that draws itself with JavaScript shows the DOM as captured. There is no view for an MHTML over 50 MB; the MHTML, PNG and PDF stand on their own, and a view that cannot be made is logged and skipped. The MHTML, PNG and PDF stay as the simple copies that need no software. + +The next step is replay the way the Internet Archive does it: capture into WACZ with Browsertrix Crawler (in the image, or a sidecar container on hank), store the WACZ as an artifact served with range requests, and embed ReplayWeb.page with its service worker on the Stash origin, so the captured page runs its own JavaScript against the recorded responses and links stay inside the archive. Decided 2026-11-08; it follows the Krater error loop. + ## Later +- Wayback-style replay: WACZ through Browsertrix Crawler and ReplayWeb.page (see "Viewing a page"). - Share links with an expiry, for one archive. - A callback URL on `POST /api/v1/archives`. - Figma files, itch.io games, Notion pages. diff --git a/test/controllers/artifact_views_test.rb b/test/controllers/artifact_views_test.rb new file mode 100644 index 0000000..c1edbf7 --- /dev/null +++ b/test/controllers/artifact_views_test.rb @@ -0,0 +1,75 @@ +require "test_helper" + +# The viewer for a `view` artifact: the page around the frame, and the frame itself with the +# policy that keeps the archived document from running anything. +class ArtifactViewsTest < ActionDispatch::IntegrationTest + setup do + @archive = create_archive_with_captures({ "page" => "done" }, url: "https://example.org/story/") + capture = @archive.captures.first + html = "A story

A story

" + @view = capture.artifacts.create!(role: "view", filename: "page.html", content_type: "text/html", + byte_size: html.bytesize, sha256: Digest::SHA256.hexdigest(html)) + @view.file.attach(io: StringIO.new(html), filename: "page.html", content_type: "text/html") + @screenshot = capture.artifacts.create!(role: "screenshot", filename: "page.png", content_type: "image/png", + byte_size: 3, sha256: "a" * 64) + @screenshot.file.attach(io: StringIO.new("png"), filename: "page.png", content_type: "image/png") + end + + test "signed out goes to sign-in" do + get view_artifact_path(@view) + assert_redirected_to sign_in_path + + get frame_artifact_path(@view) + assert_redirected_to sign_in_path + end + + test "a viewer sees the page around the frame" do + sign_in_as(:viewer) + + get view_artifact_path(@view) + + assert_response :success + assert_select "iframe[src=?][sandbox=?]", frame_artifact_path(@view), "allow-popups allow-popups-to-escape-sandbox" + assert_select "a[href=?]", archive_path(@archive) + assert_includes response.body, "Scripts do not run" + assert_equal "frame-src 'self'", response.headers["Content-Security-Policy"] + end + + test "the frame serves the document inline with a policy that runs nothing" do + sign_in_as(:viewer) + + get frame_artifact_path(@view) + + assert_response :success + assert_equal "text/html; charset=utf-8", response.headers["Content-Type"] + assert_includes response.body, "

A story

" + csp = response.headers["Content-Security-Policy"] + assert_includes csp, "default-src 'none'" + assert_includes csp, "img-src data:;" + assert_includes csp, "sandbox allow-popups allow-popups-to-escape-sandbox" + assert_not_includes csp, "script-src" + assert_not_includes csp, "https:" + assert_equal "SAMEORIGIN", response.headers["X-Frame-Options"] + assert_equal "no-referrer", response.headers["Referrer-Policy"] + assert_match(/\Ainline/, response.headers["Content-Disposition"]) + end + + test "an artifact that is not a view has no viewer" do + sign_in_as(:viewer) + + get view_artifact_path(@screenshot) + assert_response :not_found + + get frame_artifact_path(@screenshot) + assert_response :not_found + end + + test "the archive page offers the viewer when a capture has a view" do + sign_in_as(:viewer) + + get archive_path(@archive) + + assert_response :success + assert_select "a[href=?]", view_artifact_path(@view), text: /View the archived page/ + end +end diff --git a/test/fixtures/files/page.mhtml b/test/fixtures/files/page.mhtml new file mode 100644 index 0000000..52107fe --- /dev/null +++ b/test/fixtures/files/page.mhtml @@ -0,0 +1,45 @@ +From: +Snapshot-Content-Location: https://example.org/story/ +Subject: A story +Date: Sun, 9 Nov 2026 01:00:00 -0000 +MIME-Version: 1.0 +Content-Type: multipart/related; + type="text/html"; + boundary="----MultipartBoundary--stashtest----" + +------MultipartBoundary--stashtest---- +Content-Type: text/html +Content-ID: +Content-Transfer-Encoding: quoted-printable +Content-Location: https://example.org/story/ + +A story + + + +

A story

+3D"photo" +3D"two" +Next +Other + + +------MultipartBoundary--stashtest---- +Content-Type: text/css +Content-Transfer-Encoding: quoted-printable +Content-Location: https://example.org/css/site.css + +h1 { color: rebeccapurple; background: url("../img/bg.png"); } +------MultipartBoundary--stashtest---- +Content-Type: image/png +Content-Transfer-Encoding: base64 +Content-Location: https://example.org/img/photo.png + +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg== +------MultipartBoundary--stashtest---- +Content-Type: image/png +Content-Transfer-Encoding: base64 +Content-Location: https://example.org/img/bg.png + +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg== +------MultipartBoundary--stashtest------ diff --git a/test/services/capturers/mhtml_test.rb b/test/services/capturers/mhtml_test.rb new file mode 100644 index 0000000..7e3b262 --- /dev/null +++ b/test/services/capturers/mhtml_test.rb @@ -0,0 +1,165 @@ +require "test_helper" + +class Capturers::MhtmlTest < ActiveSupport::TestCase + PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==".freeze + DATA_PNG = "data:image/png;base64,#{PNG}".freeze + + setup do + @html = Capturers::Mhtml.inline(file_fixture("page.mhtml").to_s) + @doc = Nokogiri::HTML5.parse(@html) + end + + # An MHTML like the fixture, with the document body replaced. + def mhtml_with(body) + path = File.join(Dir.mktmpdir, "x.mhtml") + source = File.read(file_fixture("page.mhtml")) + document = source[/.*?<\/html>/m] + File.write(path, source.sub(document, body.gsub("=", "=3D"))) + path + end + + def inline_body(body, **options) + Nokogiri::HTML5.parse(Capturers::Mhtml.inline(mhtml_with(body), **options)) + end + + test "keeps the document and inlines the images it carries" do + assert_equal "A story", @doc.at_css("h1").text + assert_equal DATA_PNG, @doc.at_css("img[alt=photo]")["src"] + end + + test "inlines the stylesheet, with the urls inside it resolved" do + href = @doc.at_css("link[rel=stylesheet]")["href"] + assert_match %r{\Adata:text/css;base64,}, href + css = Base64.decode64(href.split(",", 2).last) + assert_includes css, "rebeccapurple" + assert_includes css, "url(\"#{DATA_PNG}\")" + end + + test "resolves urls in inline styles and in srcset" do + assert_includes @doc.at_css("style").content, "background: url(#{DATA_PNG})" + assert_equal "#{DATA_PNG} 1x, #{DATA_PNG} 2x", @doc.at_css("img[alt=two]")["srcset"] + end + + test "links are absolute, open in a new tab and send no referrer" do + links = @doc.css("a") + assert_equal [ "https://example.org/next", "https://other.example/x" ], links.map { |a| a["href"] } + assert_equal [ "_blank" ], links.map { |a| a["target"] }.uniq + assert_equal [ "noopener noreferrer nofollow" ], links.map { |a| a["rel"] }.uniq + end + + test "scripts are removed, and a resource the MHTML lacks keeps its absolute url" do + assert_empty @doc.css("script") + doc = inline_body(%()) + assert_equal "https://example.org/img/missing.png", doc.at_css("img")["src"] + end + + test "adds a note and drops the page's own base after using it" do + assert_includes @html, "Archived by Stash" + assert_empty @doc.css("base") + + doc = inline_body(%(x)) + assert_equal "https://cdn.example/root/x", doc.at_css("a")["href"] + assert_equal "https://cdn.example/root/i.png", doc.at_css("img")["src"] + assert_empty doc.css("base") + end + + test "a resource from the network fills what the MHTML lacks, and the MHTML wins when both have it" do + font = Capturers::Mhtml::Part.new(location: "https://example.org/fonts/a.woff", content_type: "font/woff", body: "WOFF") + other = Capturers::Mhtml::Part.new(location: "https://example.org/img/photo.png", content_type: "image/gif", body: "GIF") + doc = inline_body(%(), + resources: { font.location => font, other.location => other }) + + assert_includes doc.at_css("style").content, %(url("data:font/woff;base64,#{Base64.strict_encode64("WOFF")}")) + assert_equal DATA_PNG, doc.at_css("img")["src"] + end + + test "each canvas becomes an image of what it showed, and one without an image stays" do + doc = inline_body(%(), + canvases: [ DATA_PNG, nil ]) + + img = doc.at_css("img") + assert_equal [ DATA_PNG, "chart", "wide", "300", "150", "true", nil ], + [ img["src"], img["id"], img["class"], img["width"], img["height"], img["aria-hidden"], img["data-x"] ] + assert_equal 1, doc.css("canvas").size + end + + test "the copy makes no network request of its own: refresh, referrer and link hints are removed" do + doc = inline_body(<<~HTML) + + + + + + + + + + + + x + HTML + + assert_equal [ "viewport" ], doc.css("meta").map { |m| m["name"] || m["http-equiv"] } + assert_equal [ "icon" ], doc.css("link").map { |l| l["rel"] } + assert_nil doc.at_css("a")["ping"] + assert_nil doc.at_css("a")["onclick"] + end + + test "link attributes in any spelling end up safe, and a link is never a data or javascript url" do + doc = inline_body(<<~HTML) + + one + two + three + four + five + six + + HTML + + links = doc.css("a") + assert_equal [ "_blank" ], links.map { |a| a["target"] }.uniq + assert_equal [ "noopener noreferrer nofollow" ], links.map { |a| a["rel"] }.uniq + # /next is also a text/html part in a real MHTML (an iframe document); the link stays a URL, never a data: URL. + assert_equal "https://example.org/next", links[2]["href"] + assert_equal "#top", links[3]["href"] + assert_nil links[4]["href"] + assert_nil links[5]["href"] + end + + test "a data url in srcset and a non-ascii path both survive" do + part = Capturers::Mhtml::Part.new(location: "https://example.org/img/caf%C3%A9.png", content_type: "image/png", body: "cafe") + doc = inline_body(%(c), + resources: { part.location => part }) + + assert_equal "#{DATA_PNG} 1x, #{DATA_PNG} 2x", doc.at_css("img")["srcset"] + assert_equal "data:image/png;base64,#{Base64.strict_encode64("cafe")}", doc.at_css("img[alt=c]")["src"] + end + + test "a part is encoded once however often the page uses it, and a huge part is left as a url" do + big = Capturers::Mhtml::Part.new(location: "https://example.org/big.png", content_type: "image/png", body: "x" * (Capturers::Mhtml::MAX_PART_BYTES + 1)) + doc = inline_body(%(), + resources: { big.location => big }) + + assert_equal "https://example.org/big.png", doc.css("img")[0]["src"] + assert_equal [ DATA_PNG, DATA_PNG ], doc.css("img")[1..].map { |i| i["src"] } + end + + test "a hostile stylesheet does not stall the rewrite" do + css = "a { background: " + ("url(" * 20_000) + " }" + doc = nil + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + doc = inline_body(%()) + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + + assert_operator elapsed, :<, 2 + assert_includes doc.at_css("style").content, "url(" + end + + test "a file that is not MHTML fails as a tool failure" do + path = File.join(Dir.mktmpdir, "x.mhtml") + File.write(path, "not mhtml at all") + + assert_raises(Capturers::ToolFailed) { Capturers::Mhtml.inline(path) } + end +end diff --git a/test/services/capturers/page_test.rb b/test/services/capturers/page_test.rb index 6790676..285c252 100644 --- a/test/services/capturers/page_test.rb +++ b/test/services/capturers/page_test.rb @@ -11,6 +11,9 @@ def initialize(idle) # Ferrum answers false at the deadline; it does not raise. def wait_for_idle(timeout:) = @idle + + # No responses recorded: the view gets nothing beyond the MHTML. + def traffic = [] end class FakeFrame