From 3b0313c8b097de50a0dc8d39b9f5acca17b1d735 Mon Sep 17 00:00:00 2001 From: Jasper Mayone Date: Sun, 8 Nov 2026 23:00:31 -0500 Subject: [PATCH 1/3] feat(view): show an archived page in a sandboxed viewer, from the mhtml as one html file --- app/controllers/artifacts_controller.rb | 45 ++++++++ app/models/artifact.rb | 8 +- app/services/capturers/mhtml.rb | 133 ++++++++++++++++++++++++ app/services/capturers/page.rb | 3 + app/views/archives/show.html.erb | 9 +- app/views/artifacts/view.html.erb | 35 +++++++ config/routes.rb | 6 +- docs/api.md | 2 +- docs/design.md | 9 +- test/controllers/artifact_views_test.rb | 74 +++++++++++++ test/fixtures/files/page.mhtml | 45 ++++++++ test/services/capturers/mhtml_test.rb | 48 +++++++++ 12 files changed, 412 insertions(+), 5 deletions(-) create mode 100644 app/services/capturers/mhtml.rb create mode 100644 app/views/artifacts/view.html.erb create mode 100644 test/controllers/artifact_views_test.rb create mode 100644 test/fixtures/files/page.mhtml create mode 100644 test/services/capturers/mhtml_test.rb diff --git a/app/controllers/artifacts_controller.rb b/app/controllers/artifacts_controller.rb index aebb770..a621b2c 100644 --- a/app/controllers/artifacts_controller.rb +++ b/app/controllers/artifacts_controller.rb @@ -1,10 +1,55 @@ # Downloads one artifact. The file comes from Active Storage through a short-lived signed URL, # so the storage service is never public. Inline for images and PDFs, a download for the rest. +# +# A `view` artifact (the page as one HTML file) also has a viewer: `view` is the Stash page +# around it, `frame` the document itself, served from Stash with a policy that lets it run +# nothing. The frame carries the archived page's own HTML, so it is treated as hostile: no +# scripts, no same-origin rights, no network except the data URLs the file holds. class ArtifactsController < ApplicationController + # What the archived document may do inside the frame. Styles and the inlined resources + # (data URLs) render; scripts, fetches, forms, frames and every network request do not, so + # the site under review never learns that someone opened the copy. A font or an image the + # MHTML lacks falls back; the screenshot shows the page as it looked. `sandbox` without + # `allow-same-origin` keeps it off Stash's origin and its cookies. + FRAME_CSP = [ + "default-src 'none'", + "img-src data:", + "media-src data:", + "font-src data:", + "style-src 'unsafe-inline' data:", + "form-action 'none'", + "base-uri *", + "sandbox allow-popups allow-popups-to-escape-sandbox" + ].join("; ").freeze + def show artifact = Artifact.includes(:capture).find_by_public_param!(params[:id]) authorize artifact.archive, :show? disposition = artifact.image? || artifact.pdf? ? :inline : :attachment redirect_to artifact.file.url(expires_in: 10.minutes, disposition: disposition), allow_other_host: true end + + def view + @artifact = viewable_artifact + @archive = @artifact.archive + end + + def frame + artifact = viewable_artifact + response.headers["Content-Security-Policy"] = FRAME_CSP + response.headers["X-Frame-Options"] = "SAMEORIGIN" + response.headers["Referrer-Policy"] = "no-referrer" + response.headers["Cache-Control"] = "private, max-age=600" + send_data artifact.file.download, type: "text/html; charset=utf-8", disposition: "inline" + end + + private + + def viewable_artifact + artifact = Artifact.includes(capture: :archive).find_by_public_param!(params[:id]) + authorize artifact.archive, :show? + raise ActiveRecord::RecordNotFound unless artifact.view? + + artifact + end end diff --git a/app/models/artifact.rb b/app/models/artifact.rb index a39969f..bc3c479 100644 --- a/app/models/artifact.rb +++ b/app/models/artifact.rb @@ -5,12 +5,14 @@ # rendition a derived copy in a viewable form: the print PDF of a page, the PDF of a deck # screenshot a PNG of the page or a poster of a video # metadata JSON or text about the thing: repo metadata, refs, yt-dlp info, a normalized post +# view a copy the browser shows on the archive page: the page as one HTML file, with +# its styles and images inlined (Capturers::Mhtml), in a sandboxed frame class Artifact < CentralRecord # The prefix "art" is registered in internal-docs (encoded-id-prefixes.md). include EncodedIds::UuidIdentifiable set_public_id_prefix "art" - ROLES = %w[original rendition screenshot metadata].freeze + ROLES = %w[original rendition screenshot metadata view].freeze belongs_to :capture, inverse_of: :artifacts has_one :archive, through: :capture @@ -33,4 +35,8 @@ def pdf? def text? content_type.start_with?("text/") || content_type == "application/json" end + + def view? + role == "view" + end end diff --git a/app/services/capturers/mhtml.rb b/app/services/capturers/mhtml.rb new file mode 100644 index 0000000..dd28a0e --- /dev/null +++ b/app/services/capturers/mhtml.rb @@ -0,0 +1,133 @@ +require "mail" +require "base64" +require "uri" + +module Capturers + # Turns the MHTML that Chromium saved into one HTML file that a browser can show: every + # resource the MHTML carries (styles, images, fonts) is inlined as a data URL, and every link + # is made absolute and opens in a new tab. The result is the `view` artifact, shown on the + # archive page in a sandboxed frame with scripts off. Scripts are left in place but never run + # there, so a page that draws itself with JavaScript shows the DOM as captured. + # + # MHTML is MIME multipart/related: the first part is the document, the others its resources, + # each with a Content-Location (its URL) and base64 or quoted-printable content. + module Mhtml + # Attributes whose value is one URL to inline. + URL_ATTRIBUTES = %w[src href poster background].freeze + # Elements whose href is a link for a person, not a resource. + LINK_TAGS = %w[a area].freeze + + Part = Data.define(:location, :content_type, :body) + + module_function + + # Reads the MHTML file and returns the HTML string. Raises ToolFailed when the file is not + # MHTML or has no document part. + def inline(path) + message = Mail.read_from_string(::File.binread(path)) + parts = message.multipart? ? message.parts : [ message ] + parts = parts.map { |part| part_from(part) }.compact + document = parts.find { |part| part.content_type.start_with?("text/html") } + raise ToolFailed, "the MHTML has no HTML part" unless document + + resources = parts.reject { |part| part.equal?(document) }.to_h { |part| [ part.location, part ] } + base = document.location + html = rewrite_html(document.body.dup.force_encoding("UTF-8").scrub, base, resources) + inject_base_and_meta(html, base) + rescue Mail::Field::ParseError, ArgumentError, EncodingError => e + raise ToolFailed, "the MHTML could not be read: #{e.class}" + end + + def part_from(part) + location = part.header["Content-Location"]&.value.to_s.strip + type = part.content_type.to_s.split(";").first.to_s.strip.downcase + return nil if location.empty? && !type.start_with?("text/html") + + Part.new(location: location, content_type: type, body: part.decoded.to_s) + end + + # Resolves every resource URL in the document and in its inline styles to a data URL. + def rewrite_html(html, base, resources) + html = html.gsub(/<(a|area)\b([^>]*)>/i) do + tag, attributes = Regexp.last_match(1), Regexp.last_match(2) + "<#{tag}#{link_attributes(attributes, base)}>" + end + html = html.gsub(/\b(#{URL_ATTRIBUTES.join("|")})=("|')(.*?)\2/i) do + attribute, quote, value = Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3) + "#{attribute}=#{quote}#{resolve(value, base, resources)}#{quote}" + end + html = html.gsub(/\bsrcset=("|')(.*?)\1/i) do + quote, value = Regexp.last_match(1), Regexp.last_match(2) + candidates = value.split(",").map do |candidate| + url, descriptor = candidate.strip.split(/\s+/, 2) + [ resolve(url.to_s, base, resources), descriptor ].compact.join(" ") + end + "srcset=#{quote}#{candidates.join(", ")}#{quote}" + end + html.gsub(/]*>(.*?)<\/style>/mi) do + tag, css = Regexp.last_match(0), Regexp.last_match(1) + css.empty? ? tag : tag.sub(css) { rewrite_css(css, base, resources) } + end + end + + # A person's link: absolute, in a new tab, no referrer. The sandboxed frame has no scripts + # and no same-origin rights, so this is the one way out of it. + def link_attributes(attributes, base) + attributes = attributes.gsub(/\s(target|rel)=("|')(.*?)\2/i, "") + attributes = attributes.sub(/\bhref=("|')(.*?)\1/i) do + quote, value = Regexp.last_match(1), Regexp.last_match(2) + "href=#{quote}#{absolute(value, base)}#{quote}" + end + "#{attributes} target=\"_blank\" rel=\"noopener noreferrer nofollow\"" + end + + def rewrite_css(css, base, resources, depth: 0) + return css if depth > 3 + + # The quote group always takes part (it may be empty), so the backreference matches an + # unquoted url() too. + css.gsub(/url\(\s*(["']?)(.*?)\1\s*\)/i) do + quote, value = Regexp.last_match(1), Regexp.last_match(2) + "url(#{quote}#{resolve(value, base, resources, depth: depth)}#{quote})" + end.gsub(/@import\s+("|')(.*?)\1/i) do + quote, value = Regexp.last_match(1), Regexp.last_match(2) + "@import #{quote}#{resolve(value, base, resources, depth: depth)}#{quote}" + end + end + + # The data URL of the resource at `value`, or the absolute URL when the MHTML lacks it. + def resolve(value, base, resources, depth: 0) + value = value.to_s.strip + return value if value.empty? || value.start_with?("data:", "#", "about:", "javascript:") + + url = absolute(value, base) + part = resources[url] || resources[value] + return url unless part + + body = part.body + body = rewrite_css(body.dup.force_encoding("UTF-8").scrub, part.location, resources, depth: depth + 1) if part.content_type == "text/css" + "data:#{part.content_type};base64,#{Base64.strict_encode64(body)}" + end + + def absolute(value, base) + return value if base.to_s.empty? + + URI.join(base, value.to_s.strip).to_s + rescue URI::Error + value + end + + # A for the links the rewrite missed, and a note of what this is. + def inject_base_and_meta(html, base) + head = "\n" + if base.present? && html !~ /\n" + end + if (index = html =~ /]*>/i) + html.dup.insert(index + Regexp.last_match(0).length, "\n#{head}") + else + head + html + end + end + end +end diff --git a/app/services/capturers/page.rb b/app/services/capturers/page.rb index 8ccb661..61db8cc 100644 --- a/app/services/capturers/page.rb +++ b/app/services/capturers/page.rb @@ -42,9 +42,12 @@ def call retry if crashed?(e) && attempts < 2 && elapsed < Capturers.timeout / 2 raise end + # The copy a browser can show: the MHTML as one HTML file (Capturers::Mhtml). + ::File.write(path("page.html"), Mhtml.inline(path("page.mhtml"))) [ output("original", "page.mhtml", content_type: "multipart/related"), + output("view", "page.html", content_type: "text/html"), output("screenshot", "page.png", content_type: "image/png"), output("rendition", "page.pdf", content_type: "application/pdf") ] diff --git a/app/views/archives/show.html.erb b/app/views/archives/show.html.erb index cc26aa0..f1c836b 100644 --- a/app/views/archives/show.html.erb +++ b/app/views/archives/show.html.erb @@ -134,9 +134,16 @@ <% end %> <% screenshot = capture.artifacts.find { |a| a.role == "screenshot" } %> + <% view = capture.artifacts.find(&:view?) %> + + <% if view %> + <%= link_to view_artifact_path(view), class: "btn btn-primary" do %> + <%= icon(:eye) %>View the archived page + <% end %> + <% end %> <% if screenshot %> - <%= link_to artifact_path(screenshot), class: "block" do %> + <%= link_to (view ? view_artifact_path(view) : artifact_path(screenshot)), class: "block" do %> <%= image_tag artifact_path(screenshot), alt: "Screenshot of #{@archive.url}", class: "max-h-96 w-full rounded-[var(--radius-control)] border border-hairline object-cover object-top", loading: "lazy" %> <% end %> <% end %> diff --git a/app/views/artifacts/view.html.erb b/app/views/artifacts/view.html.erb new file mode 100644 index 0000000..62d42ab --- /dev/null +++ b/app/views/artifacts/view.html.erb @@ -0,0 +1,35 @@ +<% content_for :title, "View · #{@archive.host}" %> + +
+
+ <%= link_to archive_path(@archive), class: "link-quiet inline-flex items-center gap-1" do %> + <%= icon(:arrow_left) %>Back to the archive + <% end %> + + + Archived copy of <%= @archive.url %> from + <%= time_tag @artifact.created_at, @artifact.created_at.utc.strftime("%Y-%m-%d %H:%M UTC") %>. Scripts do not run, + and nothing is loaded from the live site, so it never learns that you looked. Links open the live site in a new + tab. + + + + <%= link_to artifact_path(@artifact), class: "btn btn-quiet btn-sm" do %> + <%= icon(:download) %>Download the HTML + <% end %> + + <%= link_to @archive.url, class: "btn btn-secondary btn-sm", rel: "noopener noreferrer nofollow", target: "_blank" do %> + <%= icon(:external_link) %>Live site + <% end %> + +
+ + <%# The frame has no scripts, no same-origin rights and no network (ArtifactsController::FRAME_CSP). %> + +
diff --git a/config/routes.rb b/config/routes.rb index 3ac25b8..0e1ffa4 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -25,7 +25,11 @@ end # One artifact's file, by public id (art_...), through a signed storage URL. - resources :artifacts, only: :show + resources :artifacts, only: :show do + # The viewer for a `view` artifact: the page around the frame, and the frame's document. + get :view, on: :member + get :frame, on: :member + end # The archive API for patch apps, such as Krater. Bearer tokens from Weave, no session. namespace :api do diff --git a/docs/api.md b/docs/api.md index d5aa8af..00a0bd5 100644 --- a/docs/api.md +++ b/docs/api.md @@ -123,7 +123,7 @@ Newest first, at most 50. The URL is normalized before the lookup, so `?utm_sour | Method | What it keeps | |---|---| -| `page` | the rendered page as MHTML, a full-page PNG, a print PDF | +| `page` | the rendered page as MHTML, the same page as one HTML file (`view`, shown in Stash's viewer), a full-page PNG, a print PDF | | `github` | the repository record, README, releases, last 100 commits and contributors, from the GitHub API | | `git` | a mirror clone as a git bundle, and the list of refs | | `document` | the document as served (`.pptx`, `.docx`, `.pdf`, a Google export) and a PDF rendition | diff --git a/docs/design.md b/docs/design.md index e0023a0..a0f9f5c 100644 --- a/docs/design.md +++ b/docs/design.md @@ -64,7 +64,7 @@ Each capturer is one class in `app/services/capturers/`, with one public method | Method | Tool | Applies to | Artifacts | |---|---|---|---| -| `page` | headless Chromium (Ferrum), one load | every `http(s)` URL that serves HTML | `original`: MHTML of the rendered page, `screenshot`: full-page PNG, `rendition`: print PDF | +| `page` | headless Chromium (Ferrum), one load | every `http(s)` URL that serves HTML | `original`: MHTML of the rendered page, `view`: the same page as one HTML file for the viewer, `screenshot`: full-page PNG, `rendition`: print PDF | | `git` | `git clone --mirror`, `git bundle` | GitHub, GitLab, Codeberg, Bitbucket, sr.ht, any `.git` URL | `original`: `.bundle`, `metadata`: refs and HEAD | | `github` | GitHub REST API | `github.com//` | `metadata`: repo JSON, releases, last 100 commits, contributors; README as served | | `document` | HTTP download, Google export URLs, LibreOffice | Google Docs, Slides, Sheets, Drawings; direct `.pdf`, `.pptx`, `.docx`, `.xlsx`, `.odp`, `.odt`, `.key` and others | `original`: the file as served; `rendition`: PDF | @@ -169,8 +169,15 @@ Stash uses `patchwork-central` (not the restricted database), schema `stash`. Ar - `staging`: deployed at `stash-staging.patchworklabs.org`. Weave staging as IdP, the staging central database on `hem`, its own bucket. The construction border and the STAGING label on every page. - `production`: deployed at `stash.patchworklabs.org`. +## Viewing a page + +A browser does not render an MHTML served over HTTP, so the `page` capturer also writes a `view` artifact: the MHTML turned into one HTML file by `Capturers::Mhtml`. Every resource the MHTML carries (styles, images, fonts) is inlined as a data URL, every link is made absolute and opens in a new tab, and a `` covers the rest. The archive page offers "View the archived page", which shows that file in an iframe served from Stash (`ArtifactsController#frame`) with a policy that runs nothing: `default-src 'none'`, inline styles and data URLs allowed, no network at all, `sandbox` without `allow-same-origin`, so the archived page never touches Stash's origin or cookies, and the site under review never learns that someone opened the copy. A font or an image the MHTML did not carry falls back; the screenshot shows the page as it looked. Scripts are left in the file but never run, so a page that draws itself with JavaScript shows the DOM as captured. The MHTML, PNG and PDF stay as the simple copies that need no software. + +The next step is replay the way the Internet Archive does it: capture into WACZ with Browsertrix Crawler (in the image, or a sidecar container on hank), store the WACZ as an artifact served with range requests, and embed ReplayWeb.page with its service worker on the Stash origin, so the captured page runs its own JavaScript against the recorded responses and links stay inside the archive. Decided 2026-11-08; it follows the Krater error loop. + ## Later +- Wayback-style replay: WACZ through Browsertrix Crawler and ReplayWeb.page (see "Viewing a page"). - Share links with an expiry, for one archive. - A callback URL on `POST /api/v1/archives`. - Figma files, itch.io games, Notion pages. diff --git a/test/controllers/artifact_views_test.rb b/test/controllers/artifact_views_test.rb new file mode 100644 index 0000000..7b7a14a --- /dev/null +++ b/test/controllers/artifact_views_test.rb @@ -0,0 +1,74 @@ +require "test_helper" + +# The viewer for a `view` artifact: the page around the frame, and the frame itself with the +# policy that keeps the archived document from running anything. +class ArtifactViewsTest < ActionDispatch::IntegrationTest + setup do + @archive = create_archive_with_captures({ "page" => "done" }, url: "https://example.org/story/") + capture = @archive.captures.first + html = "A story

A story

" + @view = capture.artifacts.create!(role: "view", filename: "page.html", content_type: "text/html", + byte_size: html.bytesize, sha256: Digest::SHA256.hexdigest(html)) + @view.file.attach(io: StringIO.new(html), filename: "page.html", content_type: "text/html") + @screenshot = capture.artifacts.create!(role: "screenshot", filename: "page.png", content_type: "image/png", + byte_size: 3, sha256: "a" * 64) + @screenshot.file.attach(io: StringIO.new("png"), filename: "page.png", content_type: "image/png") + end + + test "signed out goes to sign-in" do + get view_artifact_path(@view) + assert_redirected_to sign_in_path + + get frame_artifact_path(@view) + assert_redirected_to sign_in_path + end + + test "a viewer sees the page around the frame" do + sign_in_as(:viewer) + + get view_artifact_path(@view) + + assert_response :success + assert_select "iframe[src=?][sandbox=?]", frame_artifact_path(@view), "allow-popups allow-popups-to-escape-sandbox" + assert_select "a[href=?]", archive_path(@archive) + assert_includes response.body, "Scripts do not run" + end + + test "the frame serves the document inline with a policy that runs nothing" do + sign_in_as(:viewer) + + get frame_artifact_path(@view) + + assert_response :success + assert_equal "text/html; charset=utf-8", response.media_type + "; charset=utf-8" + assert_includes response.body, "

A story

" + csp = response.headers["Content-Security-Policy"] + assert_includes csp, "default-src 'none'" + assert_includes csp, "img-src data:;" + assert_includes csp, "sandbox allow-popups allow-popups-to-escape-sandbox" + assert_not_includes csp, "script-src" + assert_not_includes csp, "https:" + assert_equal "SAMEORIGIN", response.headers["X-Frame-Options"] + assert_equal "no-referrer", response.headers["Referrer-Policy"] + assert_match(/\Ainline/, response.headers["Content-Disposition"]) + end + + test "an artifact that is not a view has no viewer" do + sign_in_as(:viewer) + + get view_artifact_path(@screenshot) + assert_response :not_found + + get frame_artifact_path(@screenshot) + assert_response :not_found + end + + test "the archive page offers the viewer when a capture has a view" do + sign_in_as(:viewer) + + get archive_path(@archive) + + assert_response :success + assert_select "a[href=?]", view_artifact_path(@view), text: /View the archived page/ + end +end diff --git a/test/fixtures/files/page.mhtml b/test/fixtures/files/page.mhtml new file mode 100644 index 0000000..52107fe --- /dev/null +++ b/test/fixtures/files/page.mhtml @@ -0,0 +1,45 @@ +From: +Snapshot-Content-Location: https://example.org/story/ +Subject: A story +Date: Sun, 9 Nov 2026 01:00:00 -0000 +MIME-Version: 1.0 +Content-Type: multipart/related; + type="text/html"; + boundary="----MultipartBoundary--stashtest----" + +------MultipartBoundary--stashtest---- +Content-Type: text/html +Content-ID: +Content-Transfer-Encoding: quoted-printable +Content-Location: https://example.org/story/ + +A story + + + +

A story

+3D"photo" +3D"two" +Next +Other + + +------MultipartBoundary--stashtest---- +Content-Type: text/css +Content-Transfer-Encoding: quoted-printable +Content-Location: https://example.org/css/site.css + +h1 { color: rebeccapurple; background: url("../img/bg.png"); } +------MultipartBoundary--stashtest---- +Content-Type: image/png +Content-Transfer-Encoding: base64 +Content-Location: https://example.org/img/photo.png + +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg== +------MultipartBoundary--stashtest---- +Content-Type: image/png +Content-Transfer-Encoding: base64 +Content-Location: https://example.org/img/bg.png + +iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg== +------MultipartBoundary--stashtest------ diff --git a/test/services/capturers/mhtml_test.rb b/test/services/capturers/mhtml_test.rb new file mode 100644 index 0000000..c7e40d7 --- /dev/null +++ b/test/services/capturers/mhtml_test.rb @@ -0,0 +1,48 @@ +require "test_helper" + +class Capturers::MhtmlTest < ActiveSupport::TestCase + PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==".freeze + + setup do + @html = Capturers::Mhtml.inline(file_fixture("page.mhtml").to_s) + end + + test "keeps the document and inlines the images it carries" do + assert_includes @html, "

A story

" + assert_includes @html, %(src="data:image/png;base64,#{PNG}") + assert_not_includes @html, %(src="/img/photo.png") + end + + test "inlines the stylesheet, with the urls inside it resolved" do + assert_match %r{href="data:text/css;base64,[A-Za-z0-9+/=]+"}, @html + css = Base64.decode64(@html[%r{href="data:text/css;base64,([A-Za-z0-9+/=]+)"}, 1]) + assert_includes css, "rebeccapurple" + assert_includes css, "url(\"data:image/png;base64,#{PNG}\")" + end + + test "resolves urls in inline styles and in srcset" do + assert_includes @html, "background: url(data:image/png;base64,#{PNG})" + assert_includes @html, %(srcset="data:image/png;base64,#{PNG} 1x, data:image/png;base64,#{PNG} 2x") + end + + test "links are absolute, open in a new tab and send no referrer" do + assert_includes @html, %(Next) + assert_includes @html, %(Other) + end + + test "a resource the MHTML lacks keeps its absolute url, and scripts are left in place" do + assert_includes @html, %(src="https://example.org/js/app.js") + end + + test "adds a base and a note" do + assert_includes @html, %() + assert_includes @html, "Archived by Stash" + end + + test "a file that is not MHTML fails as a tool failure" do + path = File.join(Dir.mktmpdir, "x.mhtml") + File.write(path, "not mhtml at all") + + assert_raises(Capturers::ToolFailed) { Capturers::Mhtml.inline(path) } + end +end From 26d5524e6bb2471f60d44f3efbb21033c2310b84 Mon Sep 17 00:00:00 2001 From: Jasper Mayone Date: Sun, 8 Nov 2026 23:05:55 -0500 Subject: [PATCH 2/3] feat(view): inline the fonts from the network and the drawing of each canvas --- app/services/capturers/mhtml.rb | 60 +++++++++++++++++++-------- app/services/capturers/page.rb | 55 ++++++++++++++++++++++-- test/services/capturers/mhtml_test.rb | 23 ++++++++++ test/services/capturers/page_test.rb | 3 ++ 4 files changed, 120 insertions(+), 21 deletions(-) diff --git a/app/services/capturers/mhtml.rb b/app/services/capturers/mhtml.rb index dd28a0e..4f09031 100644 --- a/app/services/capturers/mhtml.rb +++ b/app/services/capturers/mhtml.rb @@ -4,35 +4,41 @@ module Capturers # Turns the MHTML that Chromium saved into one HTML file that a browser can show: every - # resource the MHTML carries (styles, images, fonts) is inlined as a data URL, and every link - # is made absolute and opens in a new tab. The result is the `view` artifact, shown on the - # archive page in a sandboxed frame with scripts off. Scripts are left in place but never run + # resource the MHTML carries (styles, images) is inlined as a data URL, and every link is made + # absolute and opens in a new tab. The result is the `view` artifact, shown on the archive page + # in a sandboxed frame with scripts off and no network. Scripts are left in place but never run # there, so a page that draws itself with JavaScript shows the DOM as captured. # + # Chromium's MHTML leaves two things out: web fonts, and what was drawn on a . The + # capturer passes those in: `resources` are the responses it saw on the network (fonts and + # images the MHTML lacks), `canvases` the PNG data URL of each canvas in document order, which + # replaces the canvas with an . + # # MHTML is MIME multipart/related: the first part is the document, the others its resources, # each with a Content-Location (its URL) and base64 or quoted-printable content. module Mhtml # Attributes whose value is one URL to inline. URL_ATTRIBUTES = %w[src href poster background].freeze - # Elements whose href is a link for a person, not a resource. - LINK_TAGS = %w[a area].freeze Part = Data.define(:location, :content_type, :body) module_function # Reads the MHTML file and returns the HTML string. Raises ToolFailed when the file is not - # MHTML or has no document part. - def inline(path) + # MHTML or has no document part. `resources` is { url => Part } for what the MHTML lacks; + # a part in the MHTML wins. `canvases` is the data URL (or nil) of each , in order. + def inline(path, resources: {}, canvases: []) message = Mail.read_from_string(::File.binread(path)) parts = message.multipart? ? message.parts : [ message ] parts = parts.map { |part| part_from(part) }.compact document = parts.find { |part| part.content_type.start_with?("text/html") } raise ToolFailed, "the MHTML has no HTML part" unless document - resources = parts.reject { |part| part.equal?(document) }.to_h { |part| [ part.location, part ] } + known = resources.merge(parts.reject { |part| part.equal?(document) }.to_h { |part| [ part.location, part ] }) base = document.location - html = rewrite_html(document.body.dup.force_encoding("UTF-8").scrub, base, resources) + html = document.body.dup.force_encoding("UTF-8").scrub + html = replace_canvases(html, canvases) + html = rewrite_html(html, base, known) inject_base_and_meta(html, base) rescue Mail::Field::ParseError, ArgumentError, EncodingError => e raise ToolFailed, "the MHTML could not be read: #{e.class}" @@ -46,6 +52,22 @@ def part_from(part) Part.new(location: location, content_type: type, body: part.decoded.to_s) end + # Each , in document order, becomes an of what was drawn on it. A canvas + # with no image (tainted, or WebGL without a kept buffer) stays as it is, empty. + def replace_canvases(html, canvases) + index = -1 + html.gsub(%r{]*)>.*?}mi) do + index += 1 + attributes, data = Regexp.last_match(1), canvases[index] + next Regexp.last_match(0) unless data.to_s.start_with?("data:image/") + + # Keep the attributes that place and size the element; drop the rest. + kept = attributes.to_enum(:scan, /\s(?:id|class|style|width|height|aria-[a-z-]+)=(["'])(?:(?!\1).)*\1/i) + .map { Regexp.last_match(0) }.join + "\"\"#{kept}" + end + end + # Resolves every resource URL in the document and in its inline styles to a data URL. def rewrite_html(html, base, resources) html = html.gsub(/<(a|area)\b([^>]*)>/i) do @@ -64,7 +86,11 @@ def rewrite_html(html, base, resources) end "srcset=#{quote}#{candidates.join(", ")}#{quote}" end - html.gsub(/]*>(.*?)<\/style>/mi) do + html = html.gsub(/\bstyle=("|')(.*?)\1/i) do + quote, css = Regexp.last_match(1), Regexp.last_match(2) + "style=#{quote}#{rewrite_css(css, base, resources)}#{quote}" + end + html.gsub(%r{]*>(.*?)}mi) do tag, css = Regexp.last_match(0), Regexp.last_match(1) css.empty? ? tag : tag.sub(css) { rewrite_css(css, base, resources) } end @@ -95,12 +121,13 @@ def rewrite_css(css, base, resources, depth: 0) end end - # The data URL of the resource at `value`, or the absolute URL when the MHTML lacks it. + # The data URL of the resource at `value`, or the absolute URL when nothing was captured. + # HTML attributes carry `&` where the URL has `&`. def resolve(value, base, resources, depth: 0) value = value.to_s.strip - return value if value.empty? || value.start_with?("data:", "#", "about:", "javascript:") + return value if value.empty? || value.start_with?("data:", "#", "about:", "javascript:", "blob:") - url = absolute(value, base) + url = absolute(value.gsub("&", "&"), base) part = resources[url] || resources[value] return url unless part @@ -119,10 +146,9 @@ def absolute(value, base) # A for the links the rewrite missed, and a note of what this is. def inject_base_and_meta(html, base) - head = "\n" - if base.present? && html !~ /\n" - end + safe_base = base.to_s.gsub(/["<>]/, '"' => """, "<" => "%3C", ">" => "%3E").gsub("--", "%2D%2D") + head = "\n" + head += "\n" if base.present? && html !~ /]*>/i) html.dup.insert(index + Regexp.last_match(0).length, "\n#{head}") else diff --git a/app/services/capturers/page.rb b/app/services/capturers/page.rb index 61db8cc..dbfa4f1 100644 --- a/app/services/capturers/page.rb +++ b/app/services/capturers/page.rb @@ -42,19 +42,24 @@ def call retry if crashed?(e) && attempts < 2 && elapsed < Capturers.timeout / 2 raise end - # The copy a browser can show: the MHTML as one HTML file (Capturers::Mhtml). - ::File.write(path("page.html"), Mhtml.inline(path("page.mhtml"))) - [ + outputs = [ output("original", "page.mhtml", content_type: "multipart/related"), - output("view", "page.html", content_type: "text/html"), output("screenshot", "page.png", content_type: "image/png"), output("rendition", "page.pdf", content_type: "application/pdf") ] + outputs.insert(1, output("view", "page.html", content_type: "text/html")) if ::File.exist?(path("page.html")) + outputs rescue Ferrum::Error => e raise ToolFailed, "Chromium: #{e.class}: #{e.message.to_s.first(300)}" end + # One response from the network is kept for the view up to this size, and all of them up + # to the total. Fonts and images only; the MHTML already carries the HTML and the CSS. + VIEW_RESOURCE_BYTES = 5.megabytes + VIEW_RESOURCES_TOTAL = 60.megabytes + VIEW_TYPES = %r{\A(font/|image/|application/(x-)?font|application/vnd\.ms-fontobject)} + private def capture_in(browser) @@ -75,6 +80,8 @@ def capture_in(browser) screenshot(page, height) log("saving the PDF (#{elapsed}s)") page.pdf(path: path("page.pdf"), format: :A4, print_background: true) + log("saving the view (#{elapsed}s)") + write_view(page) log("done (#{elapsed}s)") end @@ -186,6 +193,46 @@ def screenshot(page, height) ::File.binwrite(path("page.png"), Base64.decode64(result.fetch("data"))) end + # The copy a browser can show (Capturers::Mhtml): the MHTML as one HTML file, with the + # fonts and images Chromium's MHTML leaves out taken from the network traffic of this load, + # and each replaced by what was drawn on it. A view that cannot be made is logged + # and skipped; the MHTML, PNG and PDF stand on their own. + def write_view(page) + html = Mhtml.inline(path("page.mhtml"), resources: view_resources(page), canvases: canvas_images(page)) + ::File.write(path("page.html"), html) + log("view: #{html.bytesize} bytes as one HTML file") + rescue Error, Ferrum::Error => e + log("no view: #{e.message.to_s.first(200)}") + end + + def view_resources(page) + total = 0 + page.network.traffic.each_with_object({}) do |exchange, resources| + response = exchange.response + next unless response&.status == 200 && response.content_type.to_s.match?(VIEW_TYPES) + next if resources.key?(response.url) + + body = response.body + next if body.nil? || body.bytesize > VIEW_RESOURCE_BYTES || (total += body.bytesize) > VIEW_RESOURCES_TOTAL + + resources[response.url] = Mhtml::Part.new(location: response.url, content_type: response.content_type, body: body) + rescue Ferrum::Error + next + end + end + + # What each canvas shows, as a PNG data URL, in document order; nil where it cannot be read + # (a tainted canvas, or WebGL without a kept drawing buffer). + def canvas_images(page) + Array(page.evaluate(<<~JS)) + Array.from(document.querySelectorAll("canvas")).map((canvas) => { + try { return canvas.width && canvas.height ? canvas.toDataURL("image/png") : null; } catch (_) { return null; } + }) + JS + rescue Ferrum::Error + [] + end + def with_browser browser = self.class.new_browser yield browser diff --git a/test/services/capturers/mhtml_test.rb b/test/services/capturers/mhtml_test.rb index c7e40d7..931b10f 100644 --- a/test/services/capturers/mhtml_test.rb +++ b/test/services/capturers/mhtml_test.rb @@ -39,6 +39,29 @@ class Capturers::MhtmlTest < ActiveSupport::TestCase assert_includes @html, "Archived by Stash" end + test "a resource from the network fills what the MHTML lacks, and the MHTML wins when both have it" do + font = Capturers::Mhtml::Part.new(location: "https://example.org/js/app.js", content_type: "font/woff", body: "WOFF") + other = Capturers::Mhtml::Part.new(location: "https://example.org/img/photo.png", content_type: "image/gif", body: "GIF") + html = Capturers::Mhtml.inline(file_fixture("page.mhtml").to_s, resources: { font.location => font, other.location => other }) + + assert_includes html, %(src="data:font/woff;base64,#{Base64.strict_encode64("WOFF")}") + assert_includes html, %(src="data:image/png;base64,#{PNG}") + assert_not_includes html, "image/gif" + end + + test "each canvas becomes an image of what it showed, and one without an image stays" do + path = File.join(Dir.mktmpdir, "c.mhtml") + File.write(path, File.read(file_fixture("page.mhtml")).sub("

A story

", + "

A story

")) + data = "data:image/png;base64,#{PNG}" + + html = Capturers::Mhtml.inline(path, canvases: [ data, nil ]) + + assert_includes html, %() + assert_includes html, "" + assert_not_includes html, "data-x" + end + test "a file that is not MHTML fails as a tool failure" do path = File.join(Dir.mktmpdir, "x.mhtml") File.write(path, "not mhtml at all") diff --git a/test/services/capturers/page_test.rb b/test/services/capturers/page_test.rb index 6790676..285c252 100644 --- a/test/services/capturers/page_test.rb +++ b/test/services/capturers/page_test.rb @@ -11,6 +11,9 @@ def initialize(idle) # Ferrum answers false at the deadline; it does not raise. def wait_for_idle(timeout:) = @idle + + # No responses recorded: the view gets nothing beyond the MHTML. + def traffic = [] end class FakeFrame From 885ce4c82f59be6fd6ef2eeedd2cf50ec0f51246 Mon Sep 17 00:00:00 2001 From: Jasper Mayone Date: Sun, 8 Nov 2026 23:23:58 -0500 Subject: [PATCH 3/3] fix(view): rewrite with the html5 parser, strip what reaches the network, cap the view --- app/controllers/artifacts_controller.rb | 3 + app/services/capturers/mhtml.rb | 287 ++++++++++++++---------- app/services/capturers/page.rb | 24 +- docs/design.md | 4 +- test/controllers/artifact_views_test.rb | 3 +- test/services/capturers/mhtml_test.rb | 148 +++++++++--- 6 files changed, 322 insertions(+), 147 deletions(-) diff --git a/app/controllers/artifacts_controller.rb b/app/controllers/artifacts_controller.rb index a621b2c..b522032 100644 --- a/app/controllers/artifacts_controller.rb +++ b/app/controllers/artifacts_controller.rb @@ -32,6 +32,9 @@ def show def view @artifact = viewable_artifact @archive = @artifact.archive + # A parent's frame-src governs every navigation of its frames, the ones the frame starts + # too, so whatever the rewrite missed, the frame cannot leave Stash. + response.headers["Content-Security-Policy"] = "frame-src 'self'" end def frame diff --git a/app/services/capturers/mhtml.rb b/app/services/capturers/mhtml.rb index 4f09031..e7a1661 100644 --- a/app/services/capturers/mhtml.rb +++ b/app/services/capturers/mhtml.rb @@ -1,13 +1,16 @@ require "mail" require "base64" require "uri" +require "nokogiri" module Capturers # Turns the MHTML that Chromium saved into one HTML file that a browser can show: every - # resource the MHTML carries (styles, images) is inlined as a data URL, and every link is made - # absolute and opens in a new tab. The result is the `view` artifact, shown on the archive page - # in a sandboxed frame with scripts off and no network. Scripts are left in place but never run - # there, so a page that draws itself with JavaScript shows the DOM as captured. + # resource the MHTML carries (styles, images) is inlined as a data URL, every link is made + # absolute and opens in a new tab, and everything that could reach the network or run is + # taken out. The result is the `view` artifact, shown on the archive page in a sandboxed frame + # with scripts off and no network. The document is parsed (Nokogiri's HTML5 parser) and + # rewritten node by node, never with regular expressions over the markup, so an odd attribute + # cannot slip a tag past the rewrite. # # Chromium's MHTML leaves two things out: web fonts, and what was drawn on a . The # capturer passes those in: `resources` are the responses it saw on the network (fonts and @@ -17,143 +20,199 @@ module Capturers # MHTML is MIME multipart/related: the first part is the document, the others its resources, # each with a Content-Location (its URL) and base64 or quoted-printable content. module Mhtml - # Attributes whose value is one URL to inline. - URL_ATTRIBUTES = %w[src href poster background].freeze + # Attributes whose value is one URL to inline, on any element but a link. + URL_ATTRIBUTES = %w[src href poster background data].freeze + # values that make the browser touch the network for nothing the copy needs. + NETWORK_HINTS = %w[preconnect dns-prefetch prefetch prerender preload modulepreload].freeze + # values that navigate, loosen or leak. + META_EQUIVS = %w[refresh content-security-policy x-dns-prefetch-control].freeze + # The resources the view inlines once, however often the page references them. + MAX_PART_BYTES = 10.megabytes Part = Data.define(:location, :content_type, :body) - module_function - - # Reads the MHTML file and returns the HTML string. Raises ToolFailed when the file is not - # MHTML or has no document part. `resources` is { url => Part } for what the MHTML lacks; - # a part in the MHTML wins. `canvases` is the data URL (or nil) of each , in order. - def inline(path, resources: {}, canvases: []) - message = Mail.read_from_string(::File.binread(path)) - parts = message.multipart? ? message.parts : [ message ] - parts = parts.map { |part| part_from(part) }.compact - document = parts.find { |part| part.content_type.start_with?("text/html") } - raise ToolFailed, "the MHTML has no HTML part" unless document + class Inliner + def initialize(location, resources) + @resources = resources + @data_urls = {} + @base = location + end - known = resources.merge(parts.reject { |part| part.equal?(document) }.to_h { |part| [ part.location, part ] }) - base = document.location - html = document.body.dup.force_encoding("UTF-8").scrub - html = replace_canvases(html, canvases) - html = rewrite_html(html, base, known) - inject_base_and_meta(html, base) - rescue Mail::Field::ParseError, ArgumentError, EncodingError => e - raise ToolFailed, "the MHTML could not be read: #{e.class}" - end + def call(html, canvases) + doc = Nokogiri::HTML5.parse(html, @base, "UTF-8") + take_base(doc) + remove_active_parts(doc) + replace_canvases(doc, canvases) + rewrite_links(doc) + rewrite_resources(doc) + rewrite_styles(doc) + note(doc) + doc.to_html + end - def part_from(part) - location = part.header["Content-Location"]&.value.to_s.strip - type = part.content_type.to_s.split(";").first.to_s.strip.downcase - return nil if location.empty? && !type.start_with?("text/html") + private - Part.new(location: location, content_type: type, body: part.decoded.to_s) - end + # The page's own moves the resolution base; the view gets one of its own. + def take_base(doc) + if (own = doc.at_css("base[href]")) + @base = absolute(own["href"], @base) + end + doc.css("base").each(&:remove) + end - # Each , in document order, becomes an of what was drawn on it. A canvas - # with no image (tainted, or WebGL without a kept buffer) stays as it is, empty. - def replace_canvases(html, canvases) - index = -1 - html.gsub(%r{]*)>.*?}mi) do - index += 1 - attributes, data = Regexp.last_match(1), canvases[index] - next Regexp.last_match(0) unless data.to_s.start_with?("data:image/") - - # Keep the attributes that place and size the element; drop the rest. - kept = attributes.to_enum(:scan, /\s(?:id|class|style|width|height|aria-[a-z-]+)=(["'])(?:(?!\1).)*\1/i) - .map { Regexp.last_match(0) }.join - "\"\"#{kept}" + # Scripts never run in the frame, so they only add weight. The metas and link hints could + # navigate the frame or make the browser call the archived site's hosts; the copy has to be + # silent. Event handler attributes are blocked by the policy too; drop them anyway. + def remove_active_parts(doc) + doc.css("script, noscript > script").each(&:remove) + doc.css("meta").each do |meta| + equiv = meta["http-equiv"].to_s.strip.downcase + name = meta["name"].to_s.strip.downcase + meta.remove if META_EQUIVS.include?(equiv) || name == "referrer" + end + doc.css("link").each do |link| + rels = link["rel"].to_s.downcase.split + link.remove if rels.intersect?(NETWORK_HINTS) + end + doc.css("*").each do |node| + node.attribute_nodes.each { |attribute| attribute.remove if attribute.name.start_with?("on") } + node.remove_attribute("ping") + end end - end - # Resolves every resource URL in the document and in its inline styles to a data URL. - def rewrite_html(html, base, resources) - html = html.gsub(/<(a|area)\b([^>]*)>/i) do - tag, attributes = Regexp.last_match(1), Regexp.last_match(2) - "<#{tag}#{link_attributes(attributes, base)}>" + # Each , in document order, becomes an of what was drawn on it. A canvas + # with no image (tainted, or WebGL without a kept buffer) stays as it is, empty. + def replace_canvases(doc, canvases) + doc.css("canvas").each_with_index do |canvas, index| + data = canvases[index].to_s + next unless data.start_with?("data:image/") + + img = doc.create_element("img", src: data, alt: "") + %w[id class style width height].each { |name| img[name] = canvas[name] if canvas[name] } + canvas.attribute_nodes.each { |a| img[a.name] = a.value if a.name.start_with?("aria-") } + canvas.replace(img) + end end - html = html.gsub(/\b(#{URL_ATTRIBUTES.join("|")})=("|')(.*?)\2/i) do - attribute, quote, value = Regexp.last_match(1), Regexp.last_match(2), Regexp.last_match(3) - "#{attribute}=#{quote}#{resolve(value, base, resources)}#{quote}" + + # A person's link: absolute, in a new tab, no referrer, never a data URL. A fragment stays + # a fragment, so it scrolls the copy. The sandboxed frame has no scripts and no same-origin + # rights, so these links are the one way out of it. + def rewrite_links(doc) + doc.css("a, area").each do |link| + href = link["href"].to_s.strip + unless href.empty? || href.start_with?("#") + link["href"] = absolute(href, @base) + link.remove_attribute("href") if link["href"].match?(/\A(javascript|data|blob|vbscript):/i) + end + link["target"] = "_blank" + link["rel"] = "noopener noreferrer nofollow" + end end - html = html.gsub(/\bsrcset=("|')(.*?)\1/i) do - quote, value = Regexp.last_match(1), Regexp.last_match(2) - candidates = value.split(",").map do |candidate| - url, descriptor = candidate.strip.split(/\s+/, 2) - [ resolve(url.to_s, base, resources), descriptor ].compact.join(" ") + + def rewrite_resources(doc) + doc.css("*").each do |node| + next if %w[a area base].include?(node.name) + + URL_ATTRIBUTES.each do |name| + node[name] = resolve(node[name]) if node[name] + end + node["srcset"] = rewrite_srcset(node["srcset"]) if node["srcset"] + node["style"] = rewrite_css(node["style"]) if node["style"] end - "srcset=#{quote}#{candidates.join(", ")}#{quote}" end - html = html.gsub(/\bstyle=("|')(.*?)\1/i) do - quote, css = Regexp.last_match(1), Regexp.last_match(2) - "style=#{quote}#{rewrite_css(css, base, resources)}#{quote}" + + def rewrite_styles(doc) + doc.css("style").each { |style| style.content = rewrite_css(style.content) } end - html.gsub(%r{]*>(.*?)}mi) do - tag, css = Regexp.last_match(0), Regexp.last_match(1) - css.empty? ? tag : tag.sub(css) { rewrite_css(css, base, resources) } + + # The srcset grammar: candidates separated by a comma and whitespace, each a URL and an + # optional descriptor. A data URL carries commas of its own, so the split is on ", " only + # after a descriptor or at a candidate boundary, never inside a URL. + def rewrite_srcset(value) + value.strip.split(/,\s+(?=\S)/).map do |candidate| + url, descriptor = candidate.strip.split(/\s+/, 2) + [ resolve(url.to_s), descriptor ].compact.join(" ") + end.join(", ") end - end - # A person's link: absolute, in a new tab, no referrer. The sandboxed frame has no scripts - # and no same-origin rights, so this is the one way out of it. - def link_attributes(attributes, base) - attributes = attributes.gsub(/\s(target|rel)=("|')(.*?)\2/i, "") - attributes = attributes.sub(/\bhref=("|')(.*?)\1/i) do - quote, value = Regexp.last_match(1), Regexp.last_match(2) - "href=#{quote}#{absolute(value, base)}#{quote}" + def rewrite_css(css, base: @base, depth: 0) + return css.to_s if depth > 3 || css.nil? + + # Linear patterns: a hostile stylesheet full of `url(` must not stall the capture. The value + # may not hold a quote or a parenthesis, so a second `url(` ends the attempt at once. + css = css.gsub(/url\(\s*(["']?)([^"'()]*)\1\s*\)/i) do + quote, value = Regexp.last_match(1), Regexp.last_match(2) + "url(#{quote}#{resolve(value, base: base, depth: depth)}#{quote})" + end + css.gsub(/@import\s+(["'])([^"']*)\1/i) do + quote, value = Regexp.last_match(1), Regexp.last_match(2) + "@import #{quote}#{resolve(value, base: base, depth: depth)}#{quote}" + end end - "#{attributes} target=\"_blank\" rel=\"noopener noreferrer nofollow\"" - end - def rewrite_css(css, base, resources, depth: 0) - return css if depth > 3 - - # The quote group always takes part (it may be empty), so the backreference matches an - # unquoted url() too. - css.gsub(/url\(\s*(["']?)(.*?)\1\s*\)/i) do - quote, value = Regexp.last_match(1), Regexp.last_match(2) - "url(#{quote}#{resolve(value, base, resources, depth: depth)}#{quote})" - end.gsub(/@import\s+("|')(.*?)\1/i) do - quote, value = Regexp.last_match(1), Regexp.last_match(2) - "@import #{quote}#{resolve(value, base, resources, depth: depth)}#{quote}" + # The data URL of the resource at `value`, or its absolute URL when nothing was captured + # (the policy then blocks it, and the element falls back). HTML attributes carry `&` + # where the URL has `&`; Nokogiri gives the decoded value, so nothing to undo here. + def resolve(value, base: @base, depth: 0) + value = value.to_s.strip + return value if value.empty? || value.start_with?("data:", "#", "about:", "javascript:", "blob:", "cid:") + + url = absolute(value, base) + part = @resources[url] || @resources[value] + return url unless part + return url if part.body.bytesize > MAX_PART_BYTES + + @data_urls[part] ||= begin + body = part.body + body = rewrite_css(body.dup.force_encoding("UTF-8").scrub, base: part.location, depth: depth + 1) if part.content_type == "text/css" + "data:#{part.content_type};base64,#{Base64.strict_encode64(body)}" + end end - end - # The data URL of the resource at `value`, or the absolute URL when nothing was captured. - # HTML attributes carry `&` where the URL has `&`. - def resolve(value, base, resources, depth: 0) - value = value.to_s.strip - return value if value.empty? || value.start_with?("data:", "#", "about:", "javascript:", "blob:") + def absolute(value, base) + return value if base.to_s.empty? - url = absolute(value.gsub("&", "&"), base) - part = resources[url] || resources[value] - return url unless part + URI.join(base, URI::RFC2396_PARSER.escape(value.to_s.strip)).to_s + rescue URI::Error + value + end - body = part.body - body = rewrite_css(body.dup.force_encoding("UTF-8").scrub, part.location, resources, depth: depth + 1) if part.content_type == "text/css" - "data:#{part.content_type};base64,#{Base64.strict_encode64(body)}" + def note(doc) + text = " Archived by Stash. A copy of #{@base.to_s.gsub("--", "%2D%2D")}; styles and images are inlined, scripts are removed and do not run. " + comment = Nokogiri::XML::Comment.new(doc, text) + if (head = doc.at_css("head")) + head.prepend_child(comment) + else + doc.root&.prepend_child(comment) + end + end end - def absolute(value, base) - return value if base.to_s.empty? + module_function - URI.join(base, value.to_s.strip).to_s - rescue URI::Error - value + # Reads the MHTML file and returns the HTML string. Raises ToolFailed when the file is not + # MHTML or has no document part. `resources` is { url => Part } for what the MHTML lacks; + # a part in the MHTML wins. `canvases` is the data URL (or nil) of each , in order. + def inline(path, resources: {}, canvases: []) + message = Mail.read_from_string(::File.binread(path)) + parts = message.multipart? ? message.parts : [ message ] + parts = parts.map { |part| part_from(part) }.compact + document = parts.find { |part| part.content_type.start_with?("text/html") } + raise ToolFailed, "the MHTML has no HTML part" unless document + + known = resources.merge(parts.reject { |part| part.equal?(document) }.to_h { |part| [ part.location, part ] }) + html = document.body.dup.force_encoding("UTF-8").scrub + Inliner.new(document.location, known).call(html, canvases) + rescue Mail::Field::ParseError, ArgumentError, EncodingError => e + raise ToolFailed, "the MHTML could not be read: #{e.class}" end - # A for the links the rewrite missed, and a note of what this is. - def inject_base_and_meta(html, base) - safe_base = base.to_s.gsub(/["<>]/, '"' => """, "<" => "%3C", ">" => "%3E").gsub("--", "%2D%2D") - head = "\n" - head += "\n" if base.present? && html !~ /]*>/i) - html.dup.insert(index + Regexp.last_match(0).length, "\n#{head}") - else - head + html - end + def part_from(part) + location = part.header["Content-Location"]&.value.to_s.strip + type = part.content_type.to_s.split(";").first.to_s.strip.downcase + return nil if location.empty? && !type.start_with?("text/html") + + Part.new(location: location, content_type: type, body: part.decoded.to_s) end end end diff --git a/app/services/capturers/page.rb b/app/services/capturers/page.rb index dbfa4f1..a0a13a3 100644 --- a/app/services/capturers/page.rb +++ b/app/services/capturers/page.rb @@ -58,6 +58,11 @@ def call # to the total. Fonts and images only; the MHTML already carries the HTML and the CSS. VIEW_RESOURCE_BYTES = 5.megabytes VIEW_RESOURCES_TOTAL = 60.megabytes + # No view for a page whose MHTML is larger than this: the view is the MHTML's resources + # as base64 and the frame loads the whole file into memory to serve it. + VIEW_MAX_MHTML_BYTES = 50.megabytes + # The largest canvas drawing kept; a page can make toDataURL answer anything. + VIEW_CANVAS_BYTES = 5.megabytes VIEW_TYPES = %r{\A(font/|image/|application/(x-)?font|application/vnd\.ms-fontobject)} private @@ -198,11 +203,18 @@ def screenshot(page, height) # and each replaced by what was drawn on it. A view that cannot be made is logged # and skipped; the MHTML, PNG and PDF stand on their own. def write_view(page) + mhtml_bytes = ::File.size(path("page.mhtml")) + if mhtml_bytes > VIEW_MAX_MHTML_BYTES + log("no view: the MHTML is #{mhtml_bytes} bytes, more than #{VIEW_MAX_MHTML_BYTES}") + return + end + html = Mhtml.inline(path("page.mhtml"), resources: view_resources(page), canvases: canvas_images(page)) ::File.write(path("page.html"), html) log("view: #{html.bytesize} bytes as one HTML file") - rescue Error, Ferrum::Error => e - log("no view: #{e.message.to_s.first(200)}") + rescue StandardError => e + # Any failure here, a hostile stylesheet included, costs the view and nothing else. + log("no view: #{e.class}: #{e.message.to_s.first(200)}") end def view_resources(page) @@ -215,7 +227,10 @@ def view_resources(page) body = response.body next if body.nil? || body.bytesize > VIEW_RESOURCE_BYTES || (total += body.bytesize) > VIEW_RESOURCES_TOTAL - resources[response.url] = Mhtml::Part.new(location: response.url, content_type: response.content_type, body: body) + part = Mhtml::Part.new(location: response.url, content_type: response.content_type, body: body) + resources[response.url] = part + # The page names the URL it asked for; a redirect answers from another. + resources[exchange.request.url] ||= part if exchange.request && exchange.request.url != response.url rescue Ferrum::Error next end @@ -224,11 +239,12 @@ def view_resources(page) # What each canvas shows, as a PNG data URL, in document order; nil where it cannot be read # (a tainted canvas, or WebGL without a kept drawing buffer). def canvas_images(page) - Array(page.evaluate(<<~JS)) + images = Array(page.evaluate(<<~JS)) Array.from(document.querySelectorAll("canvas")).map((canvas) => { try { return canvas.width && canvas.height ? canvas.toDataURL("image/png") : null; } catch (_) { return null; } }) JS + images.map { |data| data.is_a?(String) && data.start_with?("data:image/") && data.bytesize <= VIEW_CANVAS_BYTES ? data : nil } rescue Ferrum::Error [] end diff --git a/docs/design.md b/docs/design.md index a0f9f5c..c3e42b8 100644 --- a/docs/design.md +++ b/docs/design.md @@ -171,7 +171,9 @@ Stash uses `patchwork-central` (not the restricted database), schema `stash`. Ar ## Viewing a page -A browser does not render an MHTML served over HTTP, so the `page` capturer also writes a `view` artifact: the MHTML turned into one HTML file by `Capturers::Mhtml`. Every resource the MHTML carries (styles, images, fonts) is inlined as a data URL, every link is made absolute and opens in a new tab, and a `` covers the rest. The archive page offers "View the archived page", which shows that file in an iframe served from Stash (`ArtifactsController#frame`) with a policy that runs nothing: `default-src 'none'`, inline styles and data URLs allowed, no network at all, `sandbox` without `allow-same-origin`, so the archived page never touches Stash's origin or cookies, and the site under review never learns that someone opened the copy. A font or an image the MHTML did not carry falls back; the screenshot shows the page as it looked. Scripts are left in the file but never run, so a page that draws itself with JavaScript shows the DOM as captured. The MHTML, PNG and PDF stay as the simple copies that need no software. +A browser does not render an MHTML served over HTTP, so the `page` capturer also writes a `view` artifact: the MHTML turned into one HTML file by `Capturers::Mhtml`. The document is parsed with Nokogiri's HTML5 parser and rewritten node by node: every resource the MHTML carries (styles, images) is inlined as a data URL, once however often the page uses it; the web fonts and images Chromium's MHTML leaves out come from the network traffic of the load (fonts and images only, 5 MB each, 60 MB in all); each `` becomes a PNG of what it drew; every link is made absolute, opens in a new tab with `rel="noopener noreferrer nofollow"`, and is never a `javascript:` or `data:` URL. Scripts, ``, the referrer meta, `preconnect` and the other link hints, `ping` and `on*` attributes are removed, so the copy makes no request of its own. The page's own `` is used for resolution, then dropped. + +The archive page offers "View the archived page", which shows that file in an iframe served from Stash (`ArtifactsController#frame`) with a policy that runs nothing: `default-src 'none'`, inline styles and data URLs allowed, no network at all, `sandbox` without `allow-same-origin`, so the archived page never touches Stash's origin or cookies, and the site under review never learns that someone opened the copy. The page around the frame carries `frame-src 'self'`, so the frame cannot navigate away from Stash whatever the rewrite missed. A font or an image the capture did not keep falls back; the screenshot shows the page as it looked. A page that draws itself with JavaScript shows the DOM as captured. There is no view for an MHTML over 50 MB; the MHTML, PNG and PDF stand on their own, and a view that cannot be made is logged and skipped. The MHTML, PNG and PDF stay as the simple copies that need no software. The next step is replay the way the Internet Archive does it: capture into WACZ with Browsertrix Crawler (in the image, or a sidecar container on hank), store the WACZ as an artifact served with range requests, and embed ReplayWeb.page with its service worker on the Stash origin, so the captured page runs its own JavaScript against the recorded responses and links stay inside the archive. Decided 2026-11-08; it follows the Krater error loop. diff --git a/test/controllers/artifact_views_test.rb b/test/controllers/artifact_views_test.rb index 7b7a14a..c1edbf7 100644 --- a/test/controllers/artifact_views_test.rb +++ b/test/controllers/artifact_views_test.rb @@ -32,6 +32,7 @@ class ArtifactViewsTest < ActionDispatch::IntegrationTest assert_select "iframe[src=?][sandbox=?]", frame_artifact_path(@view), "allow-popups allow-popups-to-escape-sandbox" assert_select "a[href=?]", archive_path(@archive) assert_includes response.body, "Scripts do not run" + assert_equal "frame-src 'self'", response.headers["Content-Security-Policy"] end test "the frame serves the document inline with a policy that runs nothing" do @@ -40,7 +41,7 @@ class ArtifactViewsTest < ActionDispatch::IntegrationTest get frame_artifact_path(@view) assert_response :success - assert_equal "text/html; charset=utf-8", response.media_type + "; charset=utf-8" + assert_equal "text/html; charset=utf-8", response.headers["Content-Type"] assert_includes response.body, "

A story

" csp = response.headers["Content-Security-Policy"] assert_includes csp, "default-src 'none'" diff --git a/test/services/capturers/mhtml_test.rb b/test/services/capturers/mhtml_test.rb index 931b10f..7e3b262 100644 --- a/test/services/capturers/mhtml_test.rb +++ b/test/services/capturers/mhtml_test.rb @@ -2,64 +2,158 @@ class Capturers::MhtmlTest < ActiveSupport::TestCase PNG = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==".freeze + DATA_PNG = "data:image/png;base64,#{PNG}".freeze setup do @html = Capturers::Mhtml.inline(file_fixture("page.mhtml").to_s) + @doc = Nokogiri::HTML5.parse(@html) + end + + # An MHTML like the fixture, with the document body replaced. + def mhtml_with(body) + path = File.join(Dir.mktmpdir, "x.mhtml") + source = File.read(file_fixture("page.mhtml")) + document = source[/.*?<\/html>/m] + File.write(path, source.sub(document, body.gsub("=", "=3D"))) + path + end + + def inline_body(body, **options) + Nokogiri::HTML5.parse(Capturers::Mhtml.inline(mhtml_with(body), **options)) end test "keeps the document and inlines the images it carries" do - assert_includes @html, "

A story

" - assert_includes @html, %(src="data:image/png;base64,#{PNG}") - assert_not_includes @html, %(src="/img/photo.png") + assert_equal "A story", @doc.at_css("h1").text + assert_equal DATA_PNG, @doc.at_css("img[alt=photo]")["src"] end test "inlines the stylesheet, with the urls inside it resolved" do - assert_match %r{href="data:text/css;base64,[A-Za-z0-9+/=]+"}, @html - css = Base64.decode64(@html[%r{href="data:text/css;base64,([A-Za-z0-9+/=]+)"}, 1]) + href = @doc.at_css("link[rel=stylesheet]")["href"] + assert_match %r{\Adata:text/css;base64,}, href + css = Base64.decode64(href.split(",", 2).last) assert_includes css, "rebeccapurple" - assert_includes css, "url(\"data:image/png;base64,#{PNG}\")" + assert_includes css, "url(\"#{DATA_PNG}\")" end test "resolves urls in inline styles and in srcset" do - assert_includes @html, "background: url(data:image/png;base64,#{PNG})" - assert_includes @html, %(srcset="data:image/png;base64,#{PNG} 1x, data:image/png;base64,#{PNG} 2x") + assert_includes @doc.at_css("style").content, "background: url(#{DATA_PNG})" + assert_equal "#{DATA_PNG} 1x, #{DATA_PNG} 2x", @doc.at_css("img[alt=two]")["srcset"] end test "links are absolute, open in a new tab and send no referrer" do - assert_includes @html, %(Next) - assert_includes @html, %(Other) + links = @doc.css("a") + assert_equal [ "https://example.org/next", "https://other.example/x" ], links.map { |a| a["href"] } + assert_equal [ "_blank" ], links.map { |a| a["target"] }.uniq + assert_equal [ "noopener noreferrer nofollow" ], links.map { |a| a["rel"] }.uniq end - test "a resource the MHTML lacks keeps its absolute url, and scripts are left in place" do - assert_includes @html, %(src="https://example.org/js/app.js") + test "scripts are removed, and a resource the MHTML lacks keeps its absolute url" do + assert_empty @doc.css("script") + doc = inline_body(%()) + assert_equal "https://example.org/img/missing.png", doc.at_css("img")["src"] end - test "adds a base and a note" do - assert_includes @html, %() + test "adds a note and drops the page's own base after using it" do assert_includes @html, "Archived by Stash" + assert_empty @doc.css("base") + + doc = inline_body(%(x)) + assert_equal "https://cdn.example/root/x", doc.at_css("a")["href"] + assert_equal "https://cdn.example/root/i.png", doc.at_css("img")["src"] + assert_empty doc.css("base") end test "a resource from the network fills what the MHTML lacks, and the MHTML wins when both have it" do - font = Capturers::Mhtml::Part.new(location: "https://example.org/js/app.js", content_type: "font/woff", body: "WOFF") + font = Capturers::Mhtml::Part.new(location: "https://example.org/fonts/a.woff", content_type: "font/woff", body: "WOFF") other = Capturers::Mhtml::Part.new(location: "https://example.org/img/photo.png", content_type: "image/gif", body: "GIF") - html = Capturers::Mhtml.inline(file_fixture("page.mhtml").to_s, resources: { font.location => font, other.location => other }) + doc = inline_body(%(), + resources: { font.location => font, other.location => other }) - assert_includes html, %(src="data:font/woff;base64,#{Base64.strict_encode64("WOFF")}") - assert_includes html, %(src="data:image/png;base64,#{PNG}") - assert_not_includes html, "image/gif" + assert_includes doc.at_css("style").content, %(url("data:font/woff;base64,#{Base64.strict_encode64("WOFF")}")) + assert_equal DATA_PNG, doc.at_css("img")["src"] end test "each canvas becomes an image of what it showed, and one without an image stays" do - path = File.join(Dir.mktmpdir, "c.mhtml") - File.write(path, File.read(file_fixture("page.mhtml")).sub("

A story

", - "

A story

")) - data = "data:image/png;base64,#{PNG}" + doc = inline_body(%(), + canvases: [ DATA_PNG, nil ]) + + img = doc.at_css("img") + assert_equal [ DATA_PNG, "chart", "wide", "300", "150", "true", nil ], + [ img["src"], img["id"], img["class"], img["width"], img["height"], img["aria-hidden"], img["data-x"] ] + assert_equal 1, doc.css("canvas").size + end + + test "the copy makes no network request of its own: refresh, referrer and link hints are removed" do + doc = inline_body(<<~HTML) + + + + + + + + + + + + x + HTML + + assert_equal [ "viewport" ], doc.css("meta").map { |m| m["name"] || m["http-equiv"] } + assert_equal [ "icon" ], doc.css("link").map { |l| l["rel"] } + assert_nil doc.at_css("a")["ping"] + assert_nil doc.at_css("a")["onclick"] + end + + test "link attributes in any spelling end up safe, and a link is never a data or javascript url" do + doc = inline_body(<<~HTML) + + one + two + three + four + five + six + + HTML + + links = doc.css("a") + assert_equal [ "_blank" ], links.map { |a| a["target"] }.uniq + assert_equal [ "noopener noreferrer nofollow" ], links.map { |a| a["rel"] }.uniq + # /next is also a text/html part in a real MHTML (an iframe document); the link stays a URL, never a data: URL. + assert_equal "https://example.org/next", links[2]["href"] + assert_equal "#top", links[3]["href"] + assert_nil links[4]["href"] + assert_nil links[5]["href"] + end + + test "a data url in srcset and a non-ascii path both survive" do + part = Capturers::Mhtml::Part.new(location: "https://example.org/img/caf%C3%A9.png", content_type: "image/png", body: "cafe") + doc = inline_body(%(c), + resources: { part.location => part }) + + assert_equal "#{DATA_PNG} 1x, #{DATA_PNG} 2x", doc.at_css("img")["srcset"] + assert_equal "data:image/png;base64,#{Base64.strict_encode64("cafe")}", doc.at_css("img[alt=c]")["src"] + end + + test "a part is encoded once however often the page uses it, and a huge part is left as a url" do + big = Capturers::Mhtml::Part.new(location: "https://example.org/big.png", content_type: "image/png", body: "x" * (Capturers::Mhtml::MAX_PART_BYTES + 1)) + doc = inline_body(%(), + resources: { big.location => big }) + + assert_equal "https://example.org/big.png", doc.css("img")[0]["src"] + assert_equal [ DATA_PNG, DATA_PNG ], doc.css("img")[1..].map { |i| i["src"] } + end - html = Capturers::Mhtml.inline(path, canvases: [ data, nil ]) + test "a hostile stylesheet does not stall the rewrite" do + css = "a { background: " + ("url(" * 20_000) + " }" + doc = nil + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + doc = inline_body(%()) + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started - assert_includes html, %() - assert_includes html, "" - assert_not_includes html, "data-x" + assert_operator elapsed, :<, 2 + assert_includes doc.at_css("style").content, "url(" end test "a file that is not MHTML fails as a tool failure" do