diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6888b32..8a60f23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,14 +37,7 @@ jobs: - name: Install StrykerJS run: npm install --no-save --no-package-lock @stryker-mutator/core - name: Build executable - run: | - touch persist.dat - deno compile \ - --allow-read \ - --allow-write \ - --allow-net \ - --allow-env=HOST,PORT,PERSIST,QUEUE_API_TOKEN,QUEUE_DEPTH_LIMIT,QUEUE_COUNT_LIMIT,RATE_LIMIT_REQUESTS \ - main.ts + run: deno run --frozen --allow-run scripts/compile.ts - name: Run tests run: deno test --allow-read --allow-write --allow-net --allow-env --allow-run diff --git a/Dockerfile b/Dockerfile index 2ee9012..f3b9242 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ COPY . /queue WORKDIR /queue -RUN deno compile --allow-read --allow-write --allow-net --allow-env=HOST,PORT,PERSIST,QUEUE_API_TOKEN,QUEUE_DEPTH_LIMIT,QUEUE_COUNT_LIMIT,RATE_LIMIT_REQUESTS --allow-sys main.ts && cp ./queue /usr/bin/ +RUN deno run --frozen --allow-run scripts/compile.ts /usr/bin/queue RUN mkdir -p /data && chown -R deno:deno /queue /usr/bin/queue /data diff --git a/README.md b/README.md index 14bbeed..037d464 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,23 @@ the `Authorization` header when making requests. Note the use of environment variables to change the listening address - these also work for the executable. +## Building from source + +With Deno 2.7.6, build using the locked production compile operation: + +```sh +deno run --frozen --allow-run scripts/compile.ts +# Optional isolated output path: +deno run --frozen --allow-run scripts/compile.ts /tmp/queue-test +``` + +Docker, CI, and the compiled-binary regression use this same operation. +`scripts/compile.ts` owns the permissions; environment names are shared with +runtime discovery in `src/config.ts`. Write and network permissions intentionally +remain unrestricted to support runtime-selected persistence directories and +HOST/PORT. Run `deno test --frozen --allow-read --allow-write --allow-net --allow-env --allow-run tests/compiled_binary_test.ts` +to verify startup and persistence across a restart with the shipped permissions. + ## Usage Once started, you interact with the server using HTTP requests. diff --git a/main.ts b/main.ts index 400ed45..f85fc31 100644 --- a/main.ts +++ b/main.ts @@ -1,7 +1,7 @@ import * as Persistency from "./src/persist.ts"; import QueueManager from "./src/manager.ts"; import { createHandler } from "./src/handler.ts"; -import { parseConfig } from "./src/config.ts"; +import { ENV_VAR_NAMES, parseConfig } from "./src/config.ts"; import type { Payload } from "./src/payload.ts"; const LOG_ENCODER = new TextEncoder(); @@ -13,16 +13,7 @@ function writeLog(message: string): void { // Read each config var by name rather than Deno.env.toObject(), which // enumerates the entire process environment and therefore requires // unrestricted env access. A compiled binary with a scoped --allow-env -// allowlist (see Dockerfile/CI) can only grant per-name access. -const ENV_VAR_NAMES = [ - "HOST", - "PORT", - "PERSIST", - "QUEUE_API_TOKEN", - "QUEUE_DEPTH_LIMIT", - "QUEUE_COUNT_LIMIT", - "RATE_LIMIT_REQUESTS", -] as const; +// allowlist (see scripts/compile.ts) can only grant per-name access. function readEnv(): Record { const env: Record = {}; diff --git a/scripts/compile.ts b/scripts/compile.ts new file mode 100644 index 0000000..16b03fb --- /dev/null +++ b/scripts/compile.ts @@ -0,0 +1,37 @@ +// The production binary's permissions. Docker, CI, and startup regressions all +// build here; unrestricted write/network access supports runtime configuration. +import { ENV_VAR_NAMES } from "../src/config.ts"; + +export async function compileBinary( + output = "queue", + overrides: { env?: true } = {}, +): Promise { + const permissions = [ + "--allow-read", + "--allow-write", + "--allow-net", + overrides.env ? "--allow-env" : `--allow-env=${ENV_VAR_NAMES.join(",")}`, + "--allow-sys", + ]; + const result = await new Deno.Command(Deno.execPath(), { + args: [ + "compile", + "--frozen", + ...permissions, + "--output", + output, + "main.ts", + ], + cwd: new URL("../", import.meta.url), + stdout: "inherit", + stderr: "inherit", + }).output(); + if (!result.success) { + throw new Error(`Compilation failed (exit ${result.code})`); + } +} + +if (import.meta.main) { + if (Deno.args.length > 1) throw new Error("Usage: compile.ts [output-path]"); + await compileBinary(Deno.args[0]); +} diff --git a/src/config.ts b/src/config.ts index c2d3e80..55a9f5d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,16 @@ import { parseArgs } from "jsr:@std/cli@1.0/parse-args"; +// Shared by runtime environment discovery and the production compiler. +export const ENV_VAR_NAMES = [ + "HOST", + "PORT", + "PERSIST", + "QUEUE_API_TOKEN", + "QUEUE_DEPTH_LIMIT", + "QUEUE_COUNT_LIMIT", + "RATE_LIMIT_REQUESTS", +] as const; + export class ConfigError extends Error { constructor(message: string) { super(message); diff --git a/tests/compiled_binary_test.ts b/tests/compiled_binary_test.ts index 6a7a995..21b24ac 100644 --- a/tests/compiled_binary_test.ts +++ b/tests/compiled_binary_test.ts @@ -1,188 +1,147 @@ -// Regression coverage for the class of bug where `deno compile` permission -// flags don't match what main.ts actually needs at runtime. No other CI job -// executes the compiled artifact (the "test" job compiles it but then runs -// `deno test`; mutation/quality jobs use `deno run`), so a broken binary can -// ship green without these tests. -// -// Each test isolates one permission dimension by leaving the others -// unrestricted, mirroring how each bug was originally diagnosed. - +// Exercise real binaries: the shipped build goes through the same CLI as Docker +// and CI. Per-dimension probes widen only env access, never copy the flag list. import { assertEquals } from "jsr:@std/assert@1.0"; +import { compileBinary } from "../scripts/compile.ts"; -async function compile(permFlags: string[], outPath: string): Promise { - const cmd = new Deno.Command(Deno.execPath(), { - args: ["compile", ...permFlags, "-o", outPath, "main.ts"], - cwd: ".", +async function compile(output: string): Promise { + const result = await new Deno.Command(Deno.execPath(), { + args: ["run", "--frozen", "--allow-run", "scripts/compile.ts", output], stdout: "piped", stderr: "piped", - }); - const { code, stderr } = await cmd.output(); - if (code !== 0) { - throw new Error(`compile failed: ${new TextDecoder().decode(stderr)}`); - } + }).output(); + assertEquals(result.code, 0, new TextDecoder().decode(result.stderr)); } -// Spawns a compiled binary and waits for it to either announce it's -// listening, or exit/crash. Returns the outcome so callers can assert on it. -async function probeStartup( - binPath: string, - args: string[], +async function withServer( + binary: string, env: Record, -): Promise<{ started: boolean; output: string }> { - const decoder = new TextDecoder(); - const child = new Deno.Command(binPath, { + args: string[], + check: (url: string) => Promise, +): Promise { + const child = new Deno.Command(binary, { args, - env, + env: { + QUEUE_API_TOKEN: "compile-test-token", + HOST: "127.0.0.1", + PORT: "0", + ...env, + }, stdout: "piped", stderr: "piped", }).spawn(); - - let buf = ""; - let started = false; - - const stdoutReader = child.stdout.getReader(); - const stderrReader = child.stderr.getReader(); - - const readAll = async (reader: ReadableStreamDefaultReader) => { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buf += decoder.decode(value, { stream: true }); - if (buf.includes("Listening on")) { - started = true; - return; + let output = ""; + let announce!: (address: { url: string; hostname: string }) => void; + const listening = new Promise<{ url: string; hostname: string }>((resolve) => + announce = resolve + ); + const read = async (stream: ReadableStream) => { + for await (const chunk of stream) { + output += new TextDecoder().decode(chunk); + const match = output.match(/Listening on ([^\s]+):(\d+)/); + if (match) { + announce({ url: `http://127.0.0.1:${match[2]}`, hostname: match[1] }); } } }; - - const timeout = new Promise((resolve) => setTimeout(resolve, 3000)); - - await Promise.race([ - Promise.all([readAll(stdoutReader), readAll(stderrReader)]), - timeout, - ]); - - try { - stdoutReader.releaseLock(); - } catch { /* ignore */ } + const readers = Promise.all([read(child.stdout), read(child.stderr)]); + let timer: number | undefined; try { - stderrReader.releaseLock(); - } catch { /* ignore */ } - try { - child.kill("SIGKILL"); - } catch { /* ignore */ } - try { - await child.status; - } catch { /* ignore */ } + const address = await Promise.race([ + listening, + child.status.then(() => { + throw new Error(`Binary exited: ${output}`); + }), + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`Startup timed out: ${output}`)), + 10000, + ); + }), + ]); + assertEquals(address.hostname, env.HOST ?? "127.0.0.1"); + await check(address.url); + } finally { + clearTimeout(timer); + try { + child.kill("SIGTERM"); + } catch { /* already exited */ } + const killTimer = setTimeout(() => { + try { + child.kill("SIGKILL"); + } catch { /* already exited */ } + }, 5000); + const status = await child.status; + clearTimeout(killTimer); + await readers; + assertEquals(status.code, 0, output); + } +} + +const headers = { + Authorization: "Bearer compile-test-token", + "Content-Type": "application/json", +}; - return { started, output: buf }; +async function checkPersistence( + binary: string, + persistDir: string, +): Promise { + const env = { HOST: "0.0.0.0", PERSIST: persistDir }; + await withServer(binary, env, ["--persist"], async (url) => { + const response = await fetch(`${url}/enqueue/compiled`, { + method: "POST", + headers, + body: JSON.stringify({ payload: { evidence: "survives-restart" } }), + }); + assertEquals(response.status, 200, await response.text()); + }); + await withServer(binary, env, ["--persist"], async (url) => { + const response = await fetch(`${url}/dequeue/compiled`, { headers }); + assertEquals(response.status, 200); + assertEquals(await response.json(), { evidence: "survives-restart" }); + }); } -Deno.test({ - name: "compiled binary: starts with a scoped --allow-env allowlist (#64)", - fn: async () => { - const tempDir = await Deno.makeTempDir({ prefix: "queue-compile-test-" }); - const outPath = `${tempDir}/queue`; - try { - await compile( - [ - "--allow-read", - "--allow-write", - "--allow-net", - "--allow-env=HOST,PORT,PERSIST,QUEUE_API_TOKEN,QUEUE_DEPTH_LIMIT,QUEUE_COUNT_LIMIT,RATE_LIMIT_REQUESTS", - "--allow-sys", - ], - outPath, - ); - const { started, output } = await probeStartup(outPath, [], { - QUEUE_API_TOKEN: "compile-test-token", - HOST: "127.0.0.1", - PORT: "0", - }); - assertEquals( - started, - true, - `binary did not report listening. Output:\n${output}`, - ); - } finally { - await Deno.remove(tempDir, { recursive: true }).catch(() => {}); - } - }, - // Compiling a 100MB+ binary is slow; this is an integration test, not a unit test. - sanitizeResources: false, - sanitizeOps: false, +Deno.test("compiled binary: shipped permissions support runtime config and persistence (#64)", async () => { + const dir = await Deno.makeTempDir({ prefix: "queue-compile-test-" }); + try { + await compile(`${dir}/queue`); + await checkPersistence(`${dir}/queue`, `${dir}/non-default-persist`); + } finally { + await Deno.remove(dir, { recursive: true }); + } }); -Deno.test({ - name: - "compiled binary: starts and persists with an arbitrary --persist dir (#65)", - fn: async () => { - const tempDir = await Deno.makeTempDir({ prefix: "queue-compile-test-" }); - const outPath = `${tempDir}/queue`; - const persistDir = `${tempDir}/persist`; - try { - await compile( - [ - "--allow-read", - "--allow-write", - "--allow-net", - "--allow-env", - "--allow-sys", - ], - outPath, - ); - const { started, output } = await probeStartup(outPath, ["--persist"], { - QUEUE_API_TOKEN: "compile-test-token", - HOST: "127.0.0.1", - PORT: "0", - PERSIST: persistDir, - }); - assertEquals( - started, - true, - `binary did not report listening. Output:\n${output}`, - ); - } finally { - await Deno.remove(tempDir, { recursive: true }).catch(() => {}); - } - }, - // Compiling a 100MB+ binary is slow; this is an integration test, not a unit test. - sanitizeResources: false, - sanitizeOps: false, +Deno.test("compiled binary: arbitrary persistence directory with unrestricted env (#65)", async () => { + const dir = await Deno.makeTempDir({ prefix: "queue-compile-test-" }); + try { + await compileBinary(`${dir}/queue`, { env: true }); + await checkPersistence(`${dir}/queue`, `${dir}/custom-persist`); + } finally { + await Deno.remove(dir, { recursive: true }); + } }); -Deno.test({ - name: - "compiled binary: binds to a runtime PORT/HOST different from build-time defaults (#68)", - fn: async () => { - const tempDir = await Deno.makeTempDir({ prefix: "queue-compile-test-" }); - const outPath = `${tempDir}/queue`; - try { - await compile( - [ - "--allow-read", - "--allow-write", - "--allow-net", - "--allow-env", - "--allow-sys", - ], - outPath, - ); - const { started, output } = await probeStartup(outPath, [], { - QUEUE_API_TOKEN: "compile-test-token", - HOST: "0.0.0.0", - PORT: "1991", - }); - assertEquals( - started, - true, - `binary did not report listening. Output:\n${output}`, - ); - } finally { - await Deno.remove(tempDir, { recursive: true }).catch(() => {}); - } - }, - // Compiling a 100MB+ binary is slow; this is an integration test, not a unit test. - sanitizeResources: false, - sanitizeOps: false, +Deno.test("compiled binary: runtime HOST/PORT with unrestricted env (#68)", async () => { + const dir = await Deno.makeTempDir({ prefix: "queue-compile-test-" }); + // Reserve an available non-default port without depending on a fixed free port. + const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 }); + const port = listener.addr.port; + listener.close(); + try { + await compileBinary(`${dir}/queue`, { env: true }); + await withServer( + `${dir}/queue`, + { HOST: "0.0.0.0", PORT: String(port) }, + [], + async (url) => { + assertEquals(new URL(url).port, String(port)); + const response = await fetch(`${url}/health`); + assertEquals(response.status, 200); + await response.body?.cancel(); + }, + ); + } finally { + await Deno.remove(dir, { recursive: true }); + } }); diff --git a/verify_dockerfile_hardening.sh b/verify_dockerfile_hardening.sh deleted file mode 100755 index 492499f..0000000 --- a/verify_dockerfile_hardening.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -set -e - -echo "=== Verifying Dockerfile Hardening ===" - -# Color output -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' # No Color - -FAILED=0 - -# Test 1: Check for non-root USER directive -echo "" -echo "Test 1: Checking for non-root USER directive..." -if grep -q "^USER " Dockerfile; then - USER_LINE=$(grep "^USER " Dockerfile) - if [[ "$USER_LINE" != "USER root" ]]; then - echo -e "${GREEN}✓ PASS${NC}: Found non-root USER directive: $USER_LINE" - else - echo -e "${RED}✗ FAIL${NC}: USER is set to root" - FAILED=1 - fi -else - echo -e "${RED}✗ FAIL${NC}: No USER directive found in Dockerfile" - FAILED=1 -fi - -# Test 2: Check for COPY instead of ADD -echo "" -echo "Test 2: Checking for COPY instead of ADD..." -if ! grep -q "^ADD " Dockerfile; then - echo -e "${GREEN}✓ PASS${NC}: No ADD directive found (COPY is used instead)" -else - echo -e "${RED}✗ FAIL${NC}: ADD directive found (should use COPY)" - FAILED=1 -fi - -# Test 3: Check for scoped Deno permissions -echo "" -echo "Test 3: Checking for scoped Deno permissions..." -if grep -q "\-\-allow-write=" Dockerfile && grep -q "\-\-allow-net=" Dockerfile && grep -q "\-\-allow-env=" Dockerfile; then - echo -e "${GREEN}✓ PASS${NC}: Found scoped Deno permission flags" -else - echo -e "${RED}✗ FAIL${NC}: Missing scoped permission flags" - FAILED=1 -fi - -# Test 4: Check .dockerignore is expanded -echo "" -echo "Test 4: Checking .dockerignore has required entries..." -if grep -q "tests/" .dockerignore && grep -q "CLAUDE.md" .dockerignore && grep -q "README.md" .dockerignore; then - echo -e "${GREEN}✓ PASS${NC}: .dockerignore contains tests/, CLAUDE.md, README.md" -else - echo -e "${RED}✗ FAIL${NC}: .dockerignore missing required entries" - FAILED=1 -fi - -# Test 5: Build the image -echo "" -echo "Test 5: Building Docker image..." -if docker build -t queue-hardened:test . > /dev/null 2>&1; then - echo -e "${GREEN}✓ PASS${NC}: Docker image built successfully" -else - echo -e "${RED}✗ FAIL${NC}: Docker image build failed" - FAILED=1 -fi - -# Test 6: Verify running process is non-root -if [ $FAILED -eq 0 ]; then - echo "" - echo "Test 6: Verifying running process is non-root..." - - # Start container in background - CONTAINER_ID=$(docker run -d --rm queue-hardened:test) - sleep 2 - - # Get the UID of the process - PROC_UID=$(docker exec "$CONTAINER_ID" id -u 2>/dev/null || echo "999") - - # Stop the container - docker stop "$CONTAINER_ID" > /dev/null 2>&1 || true - - if [ "$PROC_UID" != "0" ]; then - echo -e "${GREEN}✓ PASS${NC}: Process running as UID $PROC_UID (non-root)" - else - echo -e "${RED}✗ FAIL${NC}: Process running as UID 0 (root)" - FAILED=1 - fi -fi - -echo "" -echo "=== Verification Summary ===" -if [ $FAILED -eq 0 ]; then - echo -e "${GREEN}✓ All hardening checks passed!${NC}" - exit 0 -else - echo -e "${RED}✗ Some checks failed${NC}" - exit 1 -fi