From 71e25af50511bd4713851bb8d1d9a22cf407263e Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Thu, 29 Oct 2026 12:23:41 +0800 Subject: [PATCH 1/2] fix: strengthen verification evidence and context checks Signed-off-by: Kent Huang --- .github/workflows/ci.yml | 9 +- docs/architecture.md | 58 ++- .../references/scenario-and-test-data.md | 100 +++++- .../behavior-diff/scripts/behavior-diff.sh | 52 ++- .../skills/behavior-diff/scripts/decisions.py | 39 +- .../behavior-diff/scripts/native_context.py | 337 ++++++++++++++++++ .../scripts/preparation_evidence.py | 127 +++++++ .../behavior-diff/scripts/prepare_shared.py | 49 ++- .../scripts/reporting/context_boundary.py | 59 +++ .../scripts/reporting/evidence.py | 132 ++++++- .../behavior-diff/scripts/reporting/load.py | 26 +- .../scripts/reporting/scenario.py | 62 +++- .../behavior-diff/scripts/reporting/schema.py | 5 +- .../behavior-diff/scripts/reporting/target.py | 67 +++- .../skills/behavior-diff/scripts/run-trial.sh | 37 +- .../skills/behavior-diff/scripts/scenario.py | 12 +- tests/native-context-boundary-test.py | 276 ++++++++++++++ tests/preparation-evidence-test.py | 135 +++++++ tests/recorded-evidence-test.py | 180 ++++++++++ tests/report-schema-test.py | 1 + tests/shared-data-test.py | 66 ++++ 21 files changed, 1760 insertions(+), 69 deletions(-) create mode 100755 plugin/skills/behavior-diff/scripts/native_context.py create mode 100644 plugin/skills/behavior-diff/scripts/preparation_evidence.py create mode 100644 plugin/skills/behavior-diff/scripts/reporting/context_boundary.py create mode 100755 tests/native-context-boundary-test.py create mode 100755 tests/preparation-evidence-test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49370ea..c681553 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,6 +89,8 @@ jobs: plugin/skills/behavior-diff/scripts/source.py \ plugin/skills/behavior-diff/scripts/ci_context.py \ plugin/skills/behavior-diff/scripts/prepare_shared.py \ + plugin/skills/behavior-diff/scripts/native_context.py \ + plugin/skills/behavior-diff/scripts/preparation_evidence.py \ plugin/skills/behavior-diff/scripts/reporting/*.py - name: Run hook unit checks @@ -106,8 +108,11 @@ jobs: python3 tests/ci-context-test.py python3 tests/report-ci-context-test.py - - name: Check shared-data preparation boundaries - run: python3 tests/shared-data-test.py + - name: Check preparation and evidence boundaries + run: | + python3 tests/shared-data-test.py + python3 tests/preparation-evidence-test.py + python3 tests/native-context-boundary-test.py - name: Check isolated Codex launcher without models run: python3 tests/codex-isolation-test.py diff --git a/docs/architecture.md b/docs/architecture.md index 0fe8da4..4c09725 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -396,8 +396,8 @@ shared-data stage. The private helper [`prepare_shared.py`](../plugin/skills/behavior-diff/scripts/prepare_shared.py) delivers a reviewed data directory to every trial without changing the user's -checkout or adding runner options. It only delivers and verifies: the skill -designs the data, and the helper does not generate data, launch models, or infer +checkout. It only delivers and verifies: the skill designs the data, and the helper +does not generate data, launch models, or infer provenance. The optional private `--handoff` descriptor names task, purpose, scenario, optional source, notes, required references, custom tools and settings. It reuses the existing input readers and freezes those inputs plus shipped @@ -455,6 +455,31 @@ existing runner only after re-verification in the same environment. Semantic capability probes, source closure and privacy review remain with the agent; digests cannot prove that a probe ran or succeeded. +Optional private `preparation_evidence` binds native preparer records and the +reviewed input file to this inventory. `preparation_evidence.py` checks the +supported Codex native export for one complete, non-inherited legacy-history +turn with matching plain-text user input. Changed or missing files fail verify. +Unsupported or absent records remain unavailable/unverified. The native thread's +model field is not per-turn execution telemetry: neither a resolved model nor the +entire historical context is claimed verified. These private records never become +trial inputs or assessment content; validation does not authenticate a fabricated +export. + +Optional `context_requirements_file` freezes a declared native filesystem +restriction separately from preparation provenance. The existing runner prepares +trial snapshots and checks the selected native policy before starting any model +worker. Unsupported hosts, conflicting configuration or failed access canaries +stop launch rather than silently changing routes. The ordinary route remains +unchanged when this contract is absent; it provides no new isolation claim. + +The per-trial private marker records enforcement and observed forbidden-path +attempts. `reporting/context_boundary.py` is the shared eligibility rule: +violations or unavailable required markers exclude raw trial evidence from the +assessment and eligible counts. Successful filesystem canaries do not establish +clean injected, connector, browser or cloud context; an unobserved read attempt +is not proof of its absence. +A prepared trial remains ineligible until post-trial observation finishes. + The local [`audit_evidence.py`](../plugin/skills/behavior-diff/scripts/audit_evidence.py) indexes every expected trial and every original normalized/native JSONL event with exact line, byte and tool-part pointers, preserving duplicate identities, @@ -473,11 +498,14 @@ One trial is shown below. The coordinator repeats this sequence for all ```text behavior-diff.sh run-trial.sh Agent CLI | | | + | stage all snapshots | | + | optional native filesystem preflight | |-- start ----------->| | | |-- task + model ---->| | |< host events -------| | | | | | capture/normalize trace.jsonl + | | optional boundary marker |< process ends ------| | | wait for all 2 x N trials @@ -491,8 +519,8 @@ behavior-diff.sh run-trial.sh Agent CLI The modules below live in the skill's [`scripts/`](../plugin/skills/behavior-diff/scripts/) directory. - **Model-based interpretation:** `decisions.py` reads the task, trial actions, - final answers, numbered instruction-diff hunks, frozen purpose, and bounded - recorded returns. Extractor prompts travel over stdin, not command-line + final answers, numbered instruction-diff hunks, frozen purpose, bounded recorded + returns and explicitly declared decision artifacts. Extractor prompts travel over stdin, not command-line arguments, so large comparisons do not hit the operating system's per-argument size limit. The same extraction call defines observable target criteria, required evidence, and per-trial outcomes (`met`, `not_met`, `uncertain`, or @@ -556,9 +584,10 @@ The modules below live in the skill's [`scripts/`](../plugin/skills/behavior-dif compares recorded command sequences. `reporting/instruction.py` supplies the instruction diff. `reporting/content.py` derives shared wording and evidence limits. `reporting/target.py` validates purpose, criteria, exact completed-trial - membership, trial-local source references, and contiguous final-answer excerpts. - Source consistency needs a final-answer excerpt and readable recorded sources - matching every frozen `source_requirements` group for that check. Alternatives + membership, trial-local source references, and contiguous output excerpts. + Output can come from the final answer or declared captured decision artifacts. + Source consistency also requires readable recorded sources matching every frozen + `source_requirements` group for that check. Alternatives within a group are OR; groups are AND. Descriptor matching constrains provenance, not semantic truth. A trustworthy captured call can establish that a call was made even when its return was omitted; it cannot establish source agreement. @@ -616,8 +645,9 @@ distinct. Links between decisions and edits do not prove causality. ### Recorded evidence bounds -`reporting/evidence.py` selects only recorded read/search/list returns and -conservatively recognized read-only shell commands. Stable IDs use the trial +`reporting/evidence.py` selects recorded read/search/list returns, +conservatively recognized read-only shell commands, and explicitly declared saved +decision artifacts. Stable tool IDs use the trial name and recorded tool-call ordinal. Source descriptors retain supplied ranges; excerpt ranges are character offsets in returned text, **not file line numbers**. Text blocks are joined with newlines; images are not sent as text evidence. @@ -632,6 +662,16 @@ tools. API request-body flags, mutating methods, compound commands, errors and sensitive content remain excluded. This selects already-recorded text; it never invokes `gh` or fetches a source. +Optional frozen scenario `artifact_requirements` bind output checks to at most +sixteen exact project-relative paths. Only those retained trial-project files are +read; there is no general filesystem scrape. No-follow directory-relative opens +reject symlinks, hardlinks and nonregular files. Files over 64 KiB are omitted; +bounded UTF-8 content is filtered before sharing the existing evidence budgets. +Artifact records identify captured bytes by SHA-256, not proposed write input, +authorship or a proven read action. Missing required artifacts prevent determinate +judgments even when the final answer claims completion. Source-consistency checks +still need their separately captured authoritative source returns. + Limits are 4,096 Unicode characters per excerpt, 16,384 per trial, 131,072 per run, and 128 tool records per trial. Selection uses the saved `grades.tsv` inventory (directory fallback only without that inventory) and reserves a share diff --git a/plugin/skills/behavior-diff/references/scenario-and-test-data.md b/plugin/skills/behavior-diff/references/scenario-and-test-data.md index 7b11b72..6dfef0e 100644 --- a/plugin/skills/behavior-diff/references/scenario-and-test-data.md +++ b/plugin/skills/behavior-diff/references/scenario-and-test-data.md @@ -26,6 +26,34 @@ report the missing isolation prerequisite before deriving or freezing purpose. If case selection or tooling was informed by hindsight, still disclose that limit: isolation does not remove historical selection bias. +Before deriving historical purpose, check whether the chosen fresh host can +retain native supplied-input and completion records and expose the required +context controls. Do not discover missing provenance only after making a claim +of compliant preparation, or replace missing fresh preparation with a session +that already knows the outcomes. + +After preparation, the private handoff may include `preparation_evidence` with +`host`, `record_file`, `input_file`, `thread_id`, and `turn_id`. The files must +be private and outside trial inputs. `input_file` is the reviewed plain-text +preparer prompt; `record_file` is the host-native export, not an agent-written +summary. Freeze binds both files by digest and verify rejects changed or missing +records. + +The Codex adapter validates a native `ThreadReadResponse` containing one complete, +non-inherited legacy-history turn and one matching plain-text user input. +Partial, paginated, forked, compacted, ambiguous or failed records do not establish +those facts. Unsupported host formats remain explicitly unavailable. + +These checks establish only the supplied user input and native completion +represented in the provided record. They do not authenticate an arbitrary export +or enumerate injected system context. Codex's thread model is configured/latest +metadata, not per-turn execution telemetry: resolved execution model and the +complete historical context boundary remain unavailable. Missing references +remain unverified. Never turn those unavailable facets into a compliant-preparation +claim; when the requested claim requires them, report the missing prerequisite. +Native records and reviewed prompts stay out of trial copies, assessment input +and public reports. + 1. **Goals.** Start from the frozen purpose goals. Name new behavior and behavior that must survive the change. Mark preservation goals as inferred unless stated. @@ -91,8 +119,8 @@ limit: isolation does not remove historical selection bias. characters; `required_evidence` names what decides the check. - `evidence_kind` names exactly one kind of evidence: `recorded_action` for a recorded tool call such as a required code read, `output` for content - of the final answer, and `source_consistency` for a final answer that - must agree with recorded returned content. Split a compound requirement + of the final answer or a declared decision artifact, and `source_consistency` + for that output's agreement with recorded returned content. Split a compound requirement into separate checks. For example, "reads the code and reports the value it finds" becomes a `recorded_action` check for the read, an `output` check for the report, and a `source_consistency` check for agreement. @@ -134,6 +162,29 @@ limit: isolation does not remove historical selection bias. before incidental reads, so keep required sources small; many large required returns can still exhaust that budget. + - Optional `artifact_requirements` selects saved decision files that an + `output` or `source_consistency` check needs: + `{"check":"reports-total","paths":["deliverables/decision.md"]}`. + Each check appears at most once; all its listed paths are required. + Declare these paths before trials, include the requested outputs in the + neutral task, and obtain authority for sending their contents to assessment. + Select at most sixteen distinct exact project-relative paths, each at most + 120 ASCII characters. No globs, absolute paths, aliases, hidden paths, or + private evaluator, credential, or secret files are permitted. + + The collector reads only these declared files from each retained trial + project. It rejects symlinks, hardlinks and nonregular files. UTF-8 files + over 64 KiB are omitted; eligible contents pass the existing sensitive-text + filter and share the existing assessment budgets. Missing, omitted and + truncated evidence remains explicit. A final-answer completion claim cannot + substitute for a missing required artifact. + + Artifact records identify captured bytes and their SHA-256 digest, not who + wrote them or which tool read them. They cannot prove a `recorded_action` + or satisfy a required source-read return. For `source_consistency`, retain + the separate authoritative `source_requirements`. Select only outputs + reviewed as suitable for disclosure; filtering is not a secrets audit. + Use plain text: no Markdown, links, or control characters. Generalize private detail as for purpose: no names, credentials, private URLs, source excerpts, raw notes, or later history. Never record outcomes, per-trial @@ -168,11 +219,12 @@ attribute a difference solely to the selected rule. If the request explicitly requires a strict incremental-rule claim, identify the additional influences that must be excluded and establish those boundaries before -launch. The optional [macOS Codex helper](#isolated-codex-decision-comparisons-on-macos) -applies only when filesystem isolation is explicitly required and its documented -boundary fits the required capabilities. It is not the normal route and never an -automatic fallback for a runner or sandbox failure. A filesystem boundary alone -does not establish causal isolation from every model or host influence. +launch. For a required Codex local-filesystem boundary, use the existing runner's +native permission-profile contract below when its capabilities fit. The optional +[macOS Codex helper](#isolated-codex-decision-comparisons-on-macos) remains an +explicitly selected, separately scoped route, never an automatic fallback. +A filesystem boundary alone does not establish causal isolation from every +model or host influence. If a required boundary or capability is unavailable, report that precise blocker. Do not weaken host permissions, invent a launcher, or silently substitute a @@ -181,6 +233,40 @@ not block an ordinary supported-environment comparison solely because it cannot support a strict causal claim. Historical fresh-context preparation above remains required on either route; disclosing host exposure does not waive it. +### Required native filesystem boundary + +When the agreed comparison requires excluding historical files from trial tool +reads, freeze a private `context_requirements_file` in the handoff. Its JSON is: + +```json +{"version":1,"scope":"codex-local-filesystem","deny_read":["/private/synthetic/historical-context"]} +``` + +Use 1–64 exact canonical absolute file or subtree paths, not globs or relative +paths. Do not include private paths in public reports. The verified handoff +passes this file to the existing runner; it does not add another executor. +The supported route is native Codex permission profiles on macOS. It permits +the native minimal runtime and trial workspace, explicitly denies the nominated +paths, disables network, and never combines that profile with the legacy +`--sandbox` override or approval escalation. + +Before any model worker starts, the runner checks configuration and exercises +actual allowed workspace access and prohibited-read canaries in staged trial +workspaces. Unsupported hosts, configuration conflicts and failed canaries stop +launch. This is not approval to change global permissions or switch to Full +Access. The minimal runtime does not automatically expose Homebrew or other +host-installed tools: exercise the scenario's required commands under the same +policy during semantic preflight, or report the missing capability. + +The private `context-boundary.json` records enforcement and any observed attempt +to use a nominated forbidden path. Required missing/invalid markers and recorded +violations exclude the trial's raw content from assessment and eligible counts. +A denied attempt does not prove a successful read; conservative exclusion keeps +that ambiguity visible. No observed attempt is not proof of clean context. +Injected instructions, connectors, browser/cloud state and other non-filesystem +channels remain unverified. Establish any additional required boundary separately; +do not silently substitute detection-only checks for prevention. + ## Preflight diff --git a/plugin/skills/behavior-diff/scripts/behavior-diff.sh b/plugin/skills/behavior-diff/scripts/behavior-diff.sh index 2c4a53f..ebfd821 100755 --- a/plugin/skills/behavior-diff/scripts/behavior-diff.sh +++ b/plugin/skills/behavior-diff/scripts/behavior-diff.sh @@ -7,6 +7,9 @@ # behavior-diff.sh --file CLAUDE.md --task "one-line scenario" \ # [--trials 3 | --fast] [--model NAME] [--before-file ORIGINAL] # [--purpose-file JSON] [--scenario-file JSON] [--source-file JSON] +# [--context-requirements-file PRIVATE_JSON] +# Optional context requirements select a fail-closed native Codex filesystem +# policy, not proof of isolation from injected/non-filesystem context. # Purpose is frozen report/assessment context, never an added trial instruction. # Purpose and bounded recorded read/search returns go to the assessment provider # and local reports. Supply reviewed safe context; exclusions are not privacy guarantees. @@ -33,10 +36,10 @@ # ${BEHAVIOR_DIFF_HOME:-~/.behavior-diff}/runs. set -euo pipefail -file="" task="" trials=3 agent=claude model="" vocab=generic extract_agent="" extract_model="" before_file="" purpose_file="" scenario_file="" source_file="" +file="" task="" trials=3 agent=claude model="" vocab=generic extract_agent="" extract_model="" before_file="" purpose_file="" scenario_file="" source_file="" context_requirements_file="" while [ $# -gt 0 ]; do case "$1" in - --file | --before-file | --purpose-file | --scenario-file | --source-file | --task | --trials | --model | --agent | --vocab | --extract-agent | --extract-model) + --file | --before-file | --purpose-file | --scenario-file | --source-file | --context-requirements-file | --task | --trials | --model | --agent | --vocab | --extract-agent | --extract-model) if [ $# -lt 2 ] || [ -z "$2" ]; then echo "behavior-diff: $1 needs a value" >&2 exit 2 @@ -64,6 +67,10 @@ while [ $# -gt 0 ]; do source_file=$2 shift 2 ;; + --context-requirements-file) + context_requirements_file=$2 + shift 2 + ;; --task) task=$2 shift 2 @@ -153,6 +160,15 @@ command -v jq >/dev/null || { exit 3 } scripts=$(cd "$(dirname "$0")" && pwd) +context_args=() +context_required=false +if [ -n "$context_requirements_file" ]; then + context_requirements_file=$(cd "$(dirname "$context_requirements_file")" && pwd -P)/$(basename "$context_requirements_file") + python3 "$scripts/native_context.py" validate --requirements "$context_requirements_file" \ + --agent "$agent" --workspace "$PWD" || exit 2 + context_args=(--context-requirements-file "$context_requirements_file") + context_required=true +fi # Validate and freeze purpose before any model discovery or trial launch. # Never print rejected content: even a derived purpose can contain secrets. @@ -251,6 +267,14 @@ context_rel() { # $1 option name, $2 absolute context path -> repo-relative path fi case "$2" in "$repo"/*) printf '%s\n' "${2#"$repo"/}" ;; esac } +if [ -n "$context_requirements_file" ]; then + case "$context_requirements_file" in + "$repo" | "$repo"/*) + echo "behavior-diff: context requirements must remain outside trial source" >&2 + exit 2 + ;; + esac +fi purpose_rel="" scenario_rel="" source_rel="" if [ -n "$purpose_file" ]; then purpose_rel=$(context_rel --purpose-file "$purpose_file") || exit 2 @@ -305,6 +329,12 @@ mkdir -p "$runs_root/runs" runs_dir=$(cd "$runs_root/runs" && pwd -P) run=$(mktemp -d "$runs_dir/diff-$(date +%Y%m%d-%H%M%S)-XXXXXX") run_id=${run##*/} +if [ -n "$context_requirements_file" ]; then + python3 "$scripts/native_context.py" freeze --requirements "$context_requirements_file" \ + --output "$run/context-requirements.json" || exit 2 + context_requirements_file=$run/context-requirements.json + context_args=(--context-requirements-file "$context_requirements_file") +fi say() { # one console line; a closed console must never stop the run if ! (printf '%s\n' "$*") 2>/dev/null; then @@ -428,9 +458,10 @@ jq -n --arg f "$rel" --arg task "$task" --arg vocab "$vocab" \ --arg title "Behavior Diff — $rel" --argjson purpose "$purpose" \ --argjson scenario_record "$scenario_record" \ --argjson source "$source_json" --argjson ci_context "$ci_context" \ + --argjson context_required "$context_required" \ --arg before_label "$before_label" --arg after_label "$after_label" \ '{title:$title, scenario:$task, purpose:$purpose, - scenario_record:$scenario_record, + scenario_record:$scenario_record, context_boundary_required:$context_required, target_file:$f, mode:"review", vocab:$vocab, before_label:$before_label, after_label:$after_label} + (if $source == null then {} else {source:$source} end) @@ -471,7 +502,7 @@ run_trial_worker() { # $1 trial directory; runs as a background job if cd "$dir/project" && guarded_step snapshot_project; then guarded_step "$scripts/run-trial.sh" --agent "$agent" --model "$model" \ --dir "$dir/project" --task-file "$run/task.md" \ - --allowed "$ALLOWED" --trace-dir "$dir" --status-file "$trial_status" || + --allowed "$ALLOWED" --trace-dir "$dir" --status-file "$trial_status" ${context_args[@]+"${context_args[@]}"} || code=$? else code=$? @@ -558,7 +589,6 @@ launch() { # $1 variant, $2 trial index prepare_snapshot "$@" fi preparing_trial="" - run_trial_worker "$dir" & } say "behavior diff on $rel: $trials trial(s) per variant, agent $agent, model $model" @@ -567,6 +597,18 @@ milestone "starting $((2 * trials)) trials" for v in before after; do for t in $(seq 1 "$trials"); do launch "$v" "$t"; done done +if [ -n "$context_requirements_file" ]; then + for v in before after; do + for t in $(seq 1 "$trials"); do + python3 "$scripts/native_context.py" validate --requirements "$context_requirements_file" \ + --agent "$agent" --workspace "$run/$v-$t/project" \ + --policy-workspace "$run/$v-$t/project" || exit 2 + done + done +fi +for v in before after; do + for t in $(seq 1 "$trials"); do run_trial_worker "$run/$v-$t" & done +done wait || true if [ -z "$extract_agent" ]; then diff --git a/plugin/skills/behavior-diff/scripts/decisions.py b/plugin/skills/behavior-diff/scripts/decisions.py index 958ba0a..33324c6 100755 --- a/plugin/skills/behavior-diff/scripts/decisions.py +++ b/plugin/skills/behavior-diff/scripts/decisions.py @@ -62,6 +62,7 @@ from reporting.evidence import collect_run_evidence from reporting.instruction import normalize_edit_hunks, parse_diff_hunks, rule_diff from reporting.load import read_trial_trace, trial_interruption +from reporting.context_boundary import boundary_exclusion from reporting.summary import parse_intent, parse_narrative, parse_purpose_summary from reporting.scenario import parse_scenario from reporting.trial_summary import parse_trial_summaries, trial_group_names @@ -115,12 +116,12 @@ "before": [{{"trial": "", "outcome": "met"|"not_met"|"uncertain"|"unassessable", "refs": [""], - "output_excerpt": "", + "output_excerpt": "", "explanation": ""}}], "after": [{{"trial": "", "outcome": "met"|"not_met"|"uncertain"|"unassessable", "refs": [""], - "output_excerpt": "", + "output_excerpt": "", "explanation": ""}}] }} ] @@ -275,6 +276,14 @@ cannot substitute for a required code diff. Source patterns constrain provenance, not semantic agreement: judge the actual returned content as well. Do not add source_requirements or validator rejection fields to your criterion objects. +The optional artifact_requirements select exact output paths per check. Every +required path needs a cited readable artifact-N record for a met/not_met judgment; +missing, unsafe, sensitive or omitted artifacts make the check unassessable even +if the final answer claims completion. Truncated artifacts support only visible +content, never a claim about omitted sections. Artifact records capture saved bytes, +not authorship, a successful write, a read action or compliance with a source-read +requirement. A proposed write command/input is not captured output. Never infer +that unchanged initial files were authored by the trial. Preserve every supplied purpose goal without revising it to fit the results. If purpose is null, infer a fallback goal ONLY from the instruction diff and label it in intent; never derive purpose from a convenient observed difference. @@ -290,16 +299,17 @@ is absent). Keep each exact trial identity and minority outcome. Criterion mode supports correction, new behavior, behavior change, and preservation. References must resolve to that trial's recorded evidence id or trial-name:answer. -Evidence kinds: "output" judges this trial's final answer; cite trial-name:answer. -"source_consistency" judges whether that answer agrees with recorded returned -content; cite trial-name:answer and at least one record whose status is available -or truncated. For both, met/not_met needs a nonempty output_excerpt: choose ONE -exact contiguous span of that trial's final answer showing the judged claim and -copy it character for character. Never join fragments, add ellipses, paraphrase +Evidence kinds: "output" judges this trial's final answer or its required captured +artifact; cite trial-name:answer or the relevant artifact-N record. +"source_consistency" judges whether that output agrees with recorded returned +source content; cite output evidence AND at least one readable captured source call. +For both, met/not_met needs a nonempty output_excerpt: choose ONE exact contiguous +span of the cited trial final answer or declared captured artifact showing the claim +and copy it character for character. Never join fragments, add ellipses, paraphrase or repair it. "recorded_action" judges whether the agent made a call: a tool-N record proves only that its listed source call was made, even when its return was -omitted or unavailable, never what the call returned. Files the agent -wrote or published are recorded actions, not its final answer. Records ending in +omitted or unavailable, never what the call returned. Files the agent wrote or +published do not prove completion without captured output. Records ending in :trace or :overflow, sources shown as [sensitive source omitted], self-reported actions and duplicate call or result identities prove no outcome. Absence of a call is shown only by the trial's complete call list; an :overflow record means later @@ -677,7 +687,10 @@ def trials_of(run, finished_only=True): ) if variant is None: continue - cmds, final = read_trial_trace(run / name / "trace.jsonl") + if boundary_exclusion(run / name) is not None: + cmds, final = [], "" + else: + cmds, final = read_trial_trace(run / name / "trace.jsonl") interruption = trial_interruption(run / name) if interruption is not None: final = "" @@ -1198,6 +1211,7 @@ def build_prompt(run, *, include_style=True): Every extraction mode uses the same evidence, hunk numbering and frozen scenario record, read only from the run configuration. """ + config = read_config(run) trials = trials_of(run) completed_trial_names = { side: [trial["name"] for trial in trials.get(side, [])] @@ -1207,7 +1221,6 @@ def build_prompt(run, *, include_style=True): return None, completed_trial_names, "" all_trials = trials_of(run, finished_only=False) task = (run / "task.md").read_text().strip() - config = read_config(run) terms = SOURCE_TERMS[config.get("trace_source", "captured")] # Reject a malformed frozen record before any model call; keep its content private. try: @@ -2183,7 +2196,6 @@ def unreadable(path, *args, **kwargs): progress("Validate trace and instruction provenance") # Supplied but invalid provenance fails before extractor dispatch. - provenance_error = 'trace_source must be either "captured" or "self-reported"' invalid_configs = ( "{", "null", @@ -2196,7 +2208,6 @@ def unreadable(path, *args, **kwargs): (run / "config.json").write_text(invalid_config) p = cli(str(run), "--emit-prompt") assert p.returncode != 0, p.stdout - assert provenance_error in p.stderr, p.stderr # Live runs identify numbered entries as self-reported before emit. (run / "config.json").write_text(json.dumps({"trace_source": "self-reported"})) diff --git a/plugin/skills/behavior-diff/scripts/native_context.py b/plugin/skills/behavior-diff/scripts/native_context.py new file mode 100755 index 0000000..3376881 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/native_context.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +"""Native Codex filesystem policy construction and model-free preflight. + +This is not an agent launcher. The existing shell runner owns model execution. +Only macOS native local-command enforcement is supported here. Other context +surfaces and undetected/obfuscated access attempts are not isolation evidence. +""" + +import argparse +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +import tempfile +from purpose import unique_object + + +PROFILE = "behavior-diff-context" + + +def load_requirements(path): + source = Path(path) + if source.is_symlink() or not source.is_file(): + raise ValueError("context requirements must be a regular non-symlink file") + data = json.loads(source.read_text(), object_pairs_hook=unique_object) + if not isinstance(data, dict) or set(data) != {"version", "scope", "deny_read"}: + raise ValueError("invalid context requirements fields") + if type(data["version"]) is not int or data["version"] != 1: + raise ValueError("unsupported context requirements version") + if data["scope"] != "codex-local-filesystem": + raise ValueError("required context scope is unavailable") + paths = data["deny_read"] + if not isinstance(paths, list) or not paths or len(paths) > 64: + raise ValueError("deny_read requires 1 to 64 exact absolute paths") + for item in paths: + if not isinstance(item, str) or not item.startswith("/") or item == "/": + raise ValueError("deny_read requires exact absolute paths") + if any(c in item for c in "\x00\n\r*?[]") or ".." in Path(item).parts: + raise ValueError("deny_read does not accept globs or parent traversal") + if str(Path(item).resolve()) != item: + raise ValueError("deny_read paths must be canonical") + return data + + +def policy_args(workspace, denied): + # Whole-table override prevents additive profile roots/allow rules. + entries = [ + '":minimal"="read"', + '":workspace_roots"={"."="write",".git"="read",".agents"="read",".codex"="read",".aws"="read"}', + ] + entries.extend(f'{json.dumps(path)}="deny"' for path in denied) + profile = ( + "{" + + json.dumps(PROFILE) + + "={filesystem={" + + ",".join(entries) + + "},network={enabled=false}}}" + ) + args = [ + "-c", + f'default_permissions="{PROFILE}"', + "-c", + "permissions=" + profile, + "-c", + 'approval_policy="never"', + "-c", + 'web_search="disabled"', + "-c", + "mcp_servers={}", + ] + for feature in ( + "memories", + "apps", + "plugins", + "hooks", + "multi_agent", + "multi_agent_v2", + ): + args.extend(["-c", f"features.{feature}=false"]) + return args + + +def check_route(agent, workspace): + if agent != "codex" or platform.system() != "Darwin": + raise ValueError("required native filesystem route unavailable on this host") + if os.environ.get("BEHAVIOR_DIFF_CODEX_ISOLATION_ROOT") or os.environ.get( + "BEHAVIOR_DIFF_CODEX_DENY_READ" + ): + raise ValueError("native requirements cannot use the custom isolation helper") + # A loaded legacy sandbox setting overrides default_permissions. Do not + # modify any config or assume CLI precedence can remove that setting. + home = Path(os.environ.get("CODEX_HOME", str(Path.home() / ".codex"))) + configs = [ + home / "config.toml", + Path("/etc/codex/config.toml"), + Path("/etc/codex/managed_config.toml"), + ] + configs.extend( + parent / ".codex/config.toml" for parent in [workspace, *workspace.parents] + ) + for config in configs: + if config.exists(): + try: + import tomllib + except ImportError as error: + raise ValueError( + "native policy inspection requires Python 3.11 or later" + ) from error + data = tomllib.loads(config.read_text()) + if config != home / "config.toml": + raise ValueError( + "project or system config makes native route unavailable" + ) + if "sandbox_mode" in data or "sandbox_workspace_write" in data: + raise ValueError( + "legacy sandbox config conflicts with required permission profile" + ) + selected = data.get("profiles", {}).get(data.get("profile"), {}) + if "sandbox_mode" in selected or "sandbox_workspace_write" in selected: + raise ValueError( + "selected legacy sandbox profile conflicts with native permissions" + ) + help_text = subprocess.run( + ["codex", "exec", "--help"], capture_output=True, text=True, check=True + ).stdout + if "--strict-config" not in help_text: + raise ValueError( + "native Codex permission route unavailable in this CLI version" + ) + + +def preflight(workspace, denied): + # Synthetic canary checks the native effective sandbox, not a Python + # approximation. Never read prohibited real files or emit their contents. + with ( + tempfile.TemporaryDirectory(prefix="bd-outside-") as outside, + tempfile.TemporaryDirectory(prefix=".bd-policy-", dir=workspace) as root, + ): + root = Path(root).resolve() + forbidden = root / "denied" + forbidden.write_text("synthetic canary\n") + writable = root / "writable" + args = policy_args(workspace, [*denied, str(forbidden)]) + config = subprocess.run( + ["codex", "app-server", *args, "--print-startup-config", "--strict-config"], + cwd=workspace, + capture_output=True, + timeout=30, + ) + if config.returncode != 0: + failure = ValueError( + "native strict configuration check failed; trial unavailable" + ) + failure.native_stderr = config.stderr + failure.native_stdout = config.stdout + raise failure + outside_file = Path(outside).resolve() / "outside" + outside_file.write_text("synthetic outside canary\n") + code = 'printf "synthetic write\\n" >"$2" || exit 20; /bin/cat "$2" >/dev/null || exit 21; if /bin/cat "$1" >/dev/null 2>&1; then exit 19; fi; if /bin/cat "$3" >/dev/null 2>&1; then exit 19; fi' + result = subprocess.run( + [ + "codex", + "sandbox", + *args, + "--permission-profile", + PROFILE, + "--include-managed-config", + "-C", + str(workspace), + "--", + "/bin/sh", + "-c", + code, + "bd-native-canary", + str(forbidden), + str(writable), + str(outside_file), + ], + capture_output=True, + timeout=30, + ) + if result.returncode != 0 or not writable.is_file(): + failure = ValueError( + "native filesystem policy canary failed; trial unavailable" + ) + failure.native_stderr = result.stderr + failure.native_stdout = result.stdout + raise failure + return policy_args(workspace, denied) + + +def observe(trace, denied): + attempted = False + unattributed_denial = False + for line in Path(trace).read_text().splitlines(): + event = json.loads(line) + if not isinstance(event, dict): + raise ValueError("invalid trace event") + message = event.get("message", {}) + if not isinstance(message, dict) or not isinstance( + message.get("content", []), list + ): + raise ValueError("invalid trace message") + for block in message.get("content", []): + if isinstance(block, dict) and block.get("type") == "tool_use": + value = json.dumps(block.get("input", {}), ensure_ascii=False) + if any(path in value for path in denied): + attempted = True + elif ( + isinstance(block, dict) + and block.get("type") == "tool_result" + and block.get("is_error") + ): + content = json.dumps(block.get("content", "")) + if ( + "Operation not permitted" in content + or "Permission denied" in content + ): + unattributed_denial = True + status = ( + "violation" + if attempted + else "unavailable" + if unattributed_denial + else "enforced" + ) + reasons = ( + ["recorded_prohibited_path_reference"] + if attempted + else ["unattributed_permission_denial"] + if unattributed_denial + else [] + ) + return { + "version": 1, + "required": True, + "scope": "codex-local-filesystem", + "status": status, + "observation": "attempted_or_successful" + if attempted + else "unknown" + if unattributed_denial + else "not_observed", + "clean_context_proven": False, + "reason_codes": reasons, + "limits": [ + "sandboxed_local_commands_only", + "non_filesystem_context_unverified", + "attempt_detection_incomplete", + ], + } + + +def private_json(path, value): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w") as stream: + json.dump(value, stream) + stream.write("\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "operation", choices=["validate", "freeze", "prepare", "observe"] + ) + parser.add_argument("--requirements", required=True) + parser.add_argument("--agent", default="codex") + parser.add_argument("--workspace") + parser.add_argument("--output") + parser.add_argument("--policy-workspace") + parser.add_argument("--trace") + args = parser.parse_args() + try: + requirements = load_requirements(args.requirements) + if args.operation == "freeze": + private_json(args.output, requirements) + elif args.operation == "validate": + check_route(args.agent, Path(args.workspace or os.getcwd()).resolve()) + if args.policy_workspace: + preflight( + Path(args.policy_workspace).resolve(), requirements["deny_read"] + ) + else: + with tempfile.TemporaryDirectory(prefix="bd-native-preflight-") as root: + preflight(Path(root).resolve(), requirements["deny_read"]) + elif args.operation == "prepare": + workspace = Path(args.workspace).resolve() + check_route(args.agent, workspace) + native_args = preflight(workspace, requirements["deny_read"]) + private_json( + args.output, + { + "version": 1, + "required": True, + "status": "unavailable", + "scope": requirements["scope"], + "observation": "unknown", + "clean_context_proven": False, + "reason_codes": ["observation_pending"], + }, + ) + for arg in native_args: + sys.stdout.buffer.write(arg.encode() + b"\0") + else: + private_json(args.output, observe(args.trace, requirements["deny_read"])) + return 0 + except ( + ValueError, + OSError, + subprocess.SubprocessError, + json.JSONDecodeError, + ) as error: + # Never expose config values, forbidden paths or tool output. + if args.output: + private_json( + args.output, + { + "version": 1, + "required": True, + "status": "unavailable", + "observation": "unknown", + "clean_context_proven": False, + "reason_codes": ["native_context_unavailable"], + }, + ) + print( + "native context boundary unavailable: requirements, policy, or native capability check failed", + file=sys.stderr, + ) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/plugin/skills/behavior-diff/scripts/preparation_evidence.py b/plugin/skills/behavior-diff/scripts/preparation_evidence.py new file mode 100644 index 0000000..02c8254 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/preparation_evidence.py @@ -0,0 +1,127 @@ +"""Validate available native preparation facets, never asserted isolation or model proof. + +Codex 0.165 ThreadReadResponse exposes supplied user input and turn completion, +but Thread.model is explicitly not per-turn execution telemetry. Raw records +remain private; this module returns only facet availability and identities. +""" + +import json +from pathlib import Path + +from purpose import unique_object + + +FIELDS = {"host", "record_file", "input_file", "thread_id", "turn_id"} +MODEL_LIMIT = "Native thread model is not resolved per-turn execution telemetry." +CONTEXT_LIMIT = "Native user input does not enumerate all ambient or system context." + + +def validate_preparation_evidence(reference): + if type(reference) is not dict or reference.keys() != FIELDS: + raise ValueError("invalid native preparation evidence reference") + if any( + type(value) is not str or not value or "\0" in value + for value in reference.values() + ): + raise ValueError("native preparation evidence fields must be nonempty strings") + result = { + "host": reference["host"], + "status": "unverified", + "supplied_user_input": "unavailable", + "completion": "unavailable", + "resolved_model": "unavailable", + "historical_boundary": "unavailable", + "limitations": [MODEL_LIMIT, CONTEXT_LIMIT], + } + if reference["host"] != "codex": + result["limitations"] = [ + "No validated native preparation record format for this host." + ] + return result + raw = json.loads( + Path(reference["record_file"]).read_text(), object_pairs_hook=unique_object + ) + if type(raw) is not dict or not {"thread"} <= raw.keys() <= {"thread", "readState"}: + raise ValueError("expected a full native Codex ThreadReadResponse") + thread = raw["thread"] + if type(thread) is not dict: + raise ValueError("native preparation thread is missing") + if thread.get("id") != reference["thread_id"]: + raise ValueError("native preparation thread identity mismatch") + if type(thread.get("sessionId")) is not str or not thread["sessionId"]: + raise ValueError("native preparation session identity is unavailable") + if ( + "forkedFromId" not in thread + or thread["forkedFromId"] is not None + or "parentThreadId" not in thread + or thread["parentThreadId"] is not None + ): + raise ValueError("forked or inherited native preparation history is unverified") + if thread.get("historyMode", "legacy") != "legacy": + raise ValueError( + "paginated native preparation history is unverified; full legacy history required" + ) + turns = thread.get("turns") + if type(turns) is not list or len(turns) != 1: + raise ValueError("fresh native preparation requires one complete turn") + turn = turns[0] + if type(turn) is not dict or turn.get("id") != reference["turn_id"]: + raise ValueError("native preparation turn identity mismatch") + if turn.get("itemsView", "full") != "full": + raise ValueError("partial native preparation items are unverified") + if ( + turn.get("status") != "completed" + or "error" not in turn + or turn["error"] is not None + ): + raise ValueError("native preparation did not complete successfully") + items = turn.get("items") + if type(items) is not list or not items: + raise ValueError("native preparation items are unavailable") + identities = set() + messages = [] + for item in items: + if ( + type(item) is not dict + or type(item.get("id")) is not str + or not item["id"] + or item["id"] in identities + ): + raise ValueError("ambiguous native preparation item identities") + identities.add(item["id"]) + if item.get("type") in ("contextCompaction", "hookPrompt"): + raise ValueError( + "compacted or hook-augmented preparation input is unverified" + ) + if item.get("type") == "userMessage": + messages.append(item.get("content")) + if len(messages) != 1 or type(messages[0]) is not list or not messages[0]: + raise ValueError( + "native preparation supplied input is unavailable or ambiguous" + ) + if len(messages[0]) != 1: + raise ValueError("multi-part native preparation input is ambiguous") + text = [] + for content in messages[0]: + if ( + type(content) is not dict + or content.get("type") != "text" + or type(content.get("text")) is not str + or content.get("text_elements", []) != [] + ): + raise ValueError( + "native preparation non-text or annotated context is unverified" + ) + text.append(content["text"]) + expected = Path(reference["input_file"]).read_text() + if not expected.strip() or "".join(text) != expected: + raise ValueError( + "native preparation supplied input differs from reviewed input" + ) + result.update( + supplied_user_input="verified", + completion="verified", + thread_id=reference["thread_id"], + turn_id=reference["turn_id"], + ) + return result diff --git a/plugin/skills/behavior-diff/scripts/prepare_shared.py b/plugin/skills/behavior-diff/scripts/prepare_shared.py index ddfbf38..a537b64 100755 --- a/plugin/skills/behavior-diff/scripts/prepare_shared.py +++ b/plugin/skills/behavior-diff/scripts/prepare_shared.py @@ -18,6 +18,8 @@ import tarfile import unicodedata +from native_context import load_requirements +from preparation_evidence import validate_preparation_evidence from purpose import read_purpose, unique_object from scenario import read_scenario from source import read_source @@ -54,6 +56,8 @@ def handoff(path, project, working, root, data=None): if type(record) is not dict or not required <= record.keys() <= required | { "scenario_file", "source_file", + "preparation_evidence", + "context_requirements_file", }: raise PreparationError("invalid handoff fields") files = {str(path): regular_file(path)} @@ -63,6 +67,7 @@ def handoff(path, project, working, root, data=None): "notes_file", "scenario_file", "source_file", + "context_requirements_file", ): if key not in record: continue @@ -84,6 +89,35 @@ def handoff(path, project, working, root, data=None): raise PreparationError("a non-null purpose needs its reviewed scenario") if "source_file" in record: read_source(record["source_file"]) + if "context_requirements_file" in record: + load_requirements(record["context_requirements_file"]) + preparation = { + "facts": { + "status": "unverified", + "supplied_user_input": "unavailable", + "completion": "unavailable", + "resolved_model": "unavailable", + "historical_boundary": "unavailable", + "limitations": ["No native preparation record was supplied."], + } + } + if "preparation_evidence" in record: + reference = record["preparation_evidence"] + if type(reference) is not dict: + raise PreparationError("invalid native preparation evidence reference") + for key in ("record_file", "input_file"): + value = private_file(reference[key]) + if any(within(value, p) for p in trial_roots): + raise PreparationError( + "native preparation records must remain outside trial inputs" + ) + reference[key] = str(value) + private_paths.append(str(value)) + files[str(value)] = regular_file(value) + preparation = { + "reference": reference, + "facts": validate_preparation_evidence(reference), + } task = Path(record["task_file"]).read_text().rstrip("\n") if not task.strip() or "\0" in task: raise PreparationError("task must be nonempty text without NUL") @@ -131,7 +165,12 @@ def handoff(path, project, working, root, data=None): raise PreparationError("fast must be a boolean") if settings.get("fast"): argv.append("--fast") - for key in ("purpose_file", "scenario_file", "source_file"): + for key in ( + "purpose_file", + "scenario_file", + "source_file", + "context_requirements_file", + ): if key in record: argv += ["--" + key.replace("_", "-"), record[key]] # Freeze shipped mechanics as well as custom tools, without caches or data. @@ -142,6 +181,7 @@ def handoff(path, project, working, root, data=None): "files": inventory(files), "private": private_paths, "settings": settings, + "preparation_evidence": preparation, "codex_isolation": { name: os.environ[name] for name in ( @@ -647,6 +687,13 @@ def verify(root, as_json=False): raise PreparationError( "handoff context, settings or tools changed after freeze" ) + preparation = context.get("preparation_evidence", {}) + if "reference" in preparation: + if ( + validate_preparation_evidence(preparation["reference"]) + != preparation["facts"] + ): + raise PreparationError("native preparation facts changed after freeze") if frozen["before_mode"] == "absent": if (root / "before").exists() or (root / "before").is_symlink(): raise PreparationError("absent Before must not have a frozen content file") diff --git a/plugin/skills/behavior-diff/scripts/reporting/context_boundary.py b/plugin/skills/behavior-diff/scripts/reporting/context_boundary.py new file mode 100644 index 0000000..13a2168 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/reporting/context_boundary.py @@ -0,0 +1,59 @@ +"""Sanitized shared filesystem-boundary eligibility, never raw private paths.""" + +import json +from pathlib import Path + + +def boundary_status(trial_dir): + trial_dir = Path(trial_dir) + config_path = trial_dir.parent / "config.json" + if config_path.is_symlink(): + return "unavailable" + try: + config = json.loads(config_path.read_text()) + except FileNotFoundError: + config = {} + except (OSError, ValueError): + return "unavailable" + if not isinstance(config, dict): + return "unavailable" + if ( + "context_boundary_required" in config + and type(config["context_boundary_required"]) is not bool + ): + return "unavailable" + required = config.get("context_boundary_required") is True + marker = trial_dir / "context-boundary.json" + if not marker.exists(): + return "unavailable" if required else None + try: + if marker.is_symlink(): + return "unavailable" + value = json.loads(marker.read_text()) + except (OSError, ValueError): + return "unavailable" + if not isinstance(value, dict) or value.get("version") != 1: + return "unavailable" + status = value.get("status") + if status == "violation": + return status + if ( + status != "enforced" + or value.get("scope") != "codex-local-filesystem" + or value.get("clean_context_proven") is not False + ): + return "unavailable" + if value.get("observation") != "not_observed": + return "unavailable" + return "enforced" + + +def boundary_exclusion(trial_dir): + status = boundary_status(trial_dir) + if status == "violation": + return ( + "recorded filesystem context boundary violation; private evidence excluded" + ) + if status == "unavailable": + return "required filesystem context boundary unavailable; private evidence excluded" + return None diff --git a/plugin/skills/behavior-diff/scripts/reporting/evidence.py b/plugin/skills/behavior-diff/scripts/reporting/evidence.py index b12a9c5..d07431a 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/evidence.py +++ b/plugin/skills/behavior-diff/scripts/reporting/evidence.py @@ -1,13 +1,18 @@ -"""Bound recorded tool returns for assessment; never retrieve source files. +"""Bound recorded tool returns and explicitly declared captured output artifacts. Offsets describe the returned text, not file line numbers. Filtering is deliberately conservative and cannot certify arbitrary prose as free of private information. """ import json +import hashlib +import os +import stat +from contextlib import nullcontext import re import shlex from pathlib import Path +from reporting.context_boundary import boundary_exclusion from reporting.sources import ( DESCRIPTOR_TRUNCATED, @@ -21,6 +26,7 @@ RUN_LIMIT = 131072 RECORD_LIMIT = 128 SOURCE_LIMIT = 240 +ARTIFACT_BYTE_LIMIT = 65536 SELF_REPORTED = "self-reported action, not a captured return" DUPLICATE_CALL = "duplicate tool call identity" DUPLICATE_RESULT = "duplicate tool result identity" @@ -172,14 +178,91 @@ def _share(records, budget): return used -def _trial_evidence(path, name, remaining, self_reported=False, required=()): +def _artifact_evidence(trial, paths): + """Read only nominated regular files, using no-follow directory descriptors.""" records = [] + for index, relative in enumerate(paths, 1): + record = { + "id": f"{trial.name}:artifact-{index}", + "source": "artifact " + relative, + "range": {"start": 0, "end": 0, "unit": "characters"}, + "text": "", + "status": "unavailable", + "reason": "required artifact missing or unsafe", + } + records.append(record) + descriptors = [] + try: + # Opening each component relative to an already opened directory + # prevents both static links and path-swap escapes. + directory = os.open(trial, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + descriptors.append(directory) + for part in ("project", *relative.split("/")[:-1]): + directory = os.open( + part, + os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, + dir_fd=directory, + ) + descriptors.append(directory) + fd = os.open( + relative.split("/")[-1], + os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, + dir_fd=directory, + ) + descriptors.append(fd) + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise ValueError("unsafe file") + if info.st_size > ARTIFACT_BYTE_LIMIT: + record.update( + status="omitted", reason="artifact exceeds bounded privacy scan" + ) + continue + with os.fdopen(os.dup(fd), "rb") as stream: + data = stream.read(ARTIFACT_BYTE_LIMIT + 1) + after = os.fstat(fd) + if len(data) > ARTIFACT_BYTE_LIMIT or ( + info.st_size, + info.st_mtime_ns, + info.st_ctime_ns, + ) != (after.st_size, after.st_mtime_ns, after.st_ctime_ns): + record.update(reason="artifact changed or exceeded capture limit") + continue + text = data.decode("utf-8") + if "\0" in text: + raise ValueError("non-text artifact") + if sensitive_text(text): + record.update( + status="omitted", reason="potentially sensitive artifact content" + ) + continue + record["source"] += " sha256:" + hashlib.sha256(data).hexdigest() + record.update( + text=text[:EXCERPT_LIMIT], + status="truncated" if len(text) > EXCERPT_LIMIT else "available", + reason="artifact assessment truncation" + if len(text) > EXCERPT_LIMIT + else "", + ) + record["range"]["end"] = len(record["text"]) + except (OSError, ValueError, UnicodeError): + pass + finally: + for fd in reversed(descriptors): + os.close(fd) + return records + + +def _trial_evidence( + path, name, remaining, self_reported=False, required=(), artifacts=() +): + records = _artifact_evidence(path.parent, artifacts) pending = {} used = 0 call_count = 0 overflow = 0 if not path.is_file() or path.is_symlink(): - return [ + records.append( { "id": f"{name}:trace", "source": "trial recording", @@ -188,8 +271,13 @@ def _trial_evidence(path, name, remaining, self_reported=False, required=()): "status": "unavailable", "reason": "recording missing or symlink excluded", } - ], used - with path.open(encoding="utf-8", errors="replace") as stream: + ) + stream_context = ( + path.open(encoding="utf-8", errors="replace") + if path.is_file() and not path.is_symlink() + else nullcontext([]) + ) + with stream_context as stream: for line in stream: try: event = json.loads(line) @@ -318,7 +406,9 @@ def _trial_evidence(path, name, remaining, self_reported=False, required=()): priority, incidental = [], [] for record in records: if record["text"]: - matched = source_matches(record["source"], required) + matched = ":artifact-" in record["id"] or source_matches( + record["source"], required + ) (priority if matched else incidental).append(record) budget = max(0, min(TRIAL_LIMIT, remaining)) # Frozen required sources claim the same trial budget before incidental reads. @@ -327,10 +417,10 @@ def _trial_evidence(path, name, remaining, self_reported=False, required=()): return records, used -def _required_patterns(config): - """Return frozen source-requirement patterns; historical runs have none.""" +def _required_inputs(config): + """Return frozen source patterns and exact output paths; historical runs have none.""" if type(config) is not dict or config.get("scenario_record") is None: - return () + return (), () # Scenario validation imports target, which imports this module. from reporting.scenario import parse_scenario from reporting.target import parse_purpose @@ -343,13 +433,21 @@ def _required_patterns(config): raise ValueError( "invalid frozen scenario record in run configuration" ) from None - return tuple( + patterns = tuple( dict.fromkeys( pattern for requirement in scenario.source_requirements for pattern in requirement.any_of ) ) + artifacts = tuple( + dict.fromkeys( + path + for requirement in scenario.artifact_requirements + for path in requirement.paths + ) + ) + return patterns, artifacts def collect_run_evidence(run): @@ -362,7 +460,7 @@ def collect_run_evidence(run): self_reported = ( type(config) is dict and config.get("trace_source") == "self-reported" ) - required = _required_patterns(config) + required, artifacts = _required_inputs(config) trials = {} remaining = RUN_LIMIT grades = run / "grades.tsv" @@ -379,11 +477,14 @@ def collect_run_evidence(run): ] for index, name in enumerate(names): trial = run / name + if boundary_exclusion(trial) is not None: + trials[trial.name] = [] + continue trace = trial / "trace.jsonl" # Reserve a share for every remaining trial, including the other variant. allowance = remaining // (len(names) - index) records, used = _trial_evidence( - trace, trial.name, allowance, self_reported, required + trace, trial.name, allowance, self_reported, required, artifacts ) trials[trial.name] = records remaining -= used @@ -392,10 +493,11 @@ def collect_run_evidence(run): "per_excerpt_characters": EXCERPT_LIMIT, "per_trial_characters": TRIAL_LIMIT, "per_run_characters": RUN_LIMIT, - "per_trial_records": RECORD_LIMIT, - "selection": _REQUIRED_SELECTION if required else _SELECTION, + "per_trial_records": RECORD_LIMIT + len(artifacts), + "selection": ("declared artifacts and " if artifacts else "") + + (_REQUIRED_SELECTION if required else _SELECTION), "privacy": "Known sensitive sources and credential/email patterns are omitted; arbitrary private prose cannot be reliably detected.", - "provenance": "Ranges are offsets in recorded returned text, not source-file lines. Absence here does not prove the agent lacked information.", + "provenance": "Ranges are character offsets in captured returns or explicitly declared project artifacts, not source-file lines. Artifact SHA-256 identifies captured bytes, not authorship or a read/action. Absence does not prove the agent lacked information.", }, "trials": trials, } diff --git a/plugin/skills/behavior-diff/scripts/reporting/load.py b/plugin/skills/behavior-diff/scripts/reporting/load.py index 5eb3816..6f73e64 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/load.py +++ b/plugin/skills/behavior-diff/scripts/reporting/load.py @@ -10,6 +10,7 @@ import progress from reporting import content from reporting.attention import parse_attention +from reporting.context_boundary import boundary_exclusion, boundary_status from reporting.explanation import _text, parse_explanation from reporting.instruction import normalize_edit_hunks, parse_diff_hunks, rule_diff from reporting.provenance import config_ci_context, config_source @@ -182,6 +183,16 @@ def _variants(run, grades, metadata): def _trial(run, name, grade, metadata): verdict, actions = grade + boundary = boundary_exclusion(run / name) + if boundary is not None: + return TrialData( + name=name, + verdict="BLOCKED", + actions="-", + commands=(), + final="", + outcome="Blocked — " + boundary, + ) commands, final = read_trial_trace(run / name / "trace.jsonl") interruption = trial_interruption(run / name) if interruption is not None: @@ -199,7 +210,11 @@ def _trial(run, name, grade, metadata): actions=actions, commands=tuple(commands), final=final, - outcome=_outcome_label(verdict, actions, metadata.mode), + outcome=( + "Native filesystem policy enforced; no violation observed, not proof of clean context. Other context coverage unavailable." + if boundary_status(run / name) == "enforced" + else _outcome_label(verdict, actions, metadata.mode) + ), ) @@ -210,6 +225,9 @@ def trial_interruption(trial_dir): normalized final text; that text is not a completed answer. Runs without trial status records, such as older runs and authored fixtures, return None. """ + boundary = boundary_exclusion(trial_dir) + if boundary is not None: + return boundary try: record = progress.read_record(trial_dir / progress.TRIAL_RECORD, "trial") except progress.ProgressError: @@ -559,6 +577,12 @@ def _convert_decisions( goals = persisted_goals or parse_purpose(purpose) summarized_goals = persisted_goals canonical = json.loads(json.dumps(asdict(scenario))) if scenario else None + if canonical is not None and not scenario.artifact_requirements: + if ( + type(raw.get("scenario_record")) is dict + and "artifact_requirements" not in raw["scenario_record"] + ): + canonical.pop("artifact_requirements") mismatch = "scenario_record" in raw and raw["scenario_record"] != canonical if scenario is not None: # Frozen checks number the configured goals; persisted copies must agree. diff --git a/plugin/skills/behavior-diff/scripts/reporting/scenario.py b/plugin/skills/behavior-diff/scripts/reporting/scenario.py index 310e559..4962c89 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/scenario.py +++ b/plugin/skills/behavior-diff/scripts/reporting/scenario.py @@ -10,9 +10,11 @@ from dataclasses import dataclass from typing import Tuple +import re from reporting.explanation import _text from reporting.sources import SourceRequirementData, parse_source_requirements +from reporting.sources import sensitive_text from reporting.target import EVIDENCE_KINDS, MODES SUMMARY_LIMIT = 400 @@ -21,6 +23,15 @@ CHECK_TEXT_LIMIT = 600 ID_LIMIT = 80 +ARTIFACT_LIMIT = 16 +ARTIFACT_PATH_LIMIT = 120 + + +@dataclass(frozen=True) +class ArtifactRequirementData: + check: str + paths: Tuple[str, ...] + @dataclass(frozen=True) class ScenarioCheckData: @@ -40,6 +51,7 @@ class ScenarioData: limitations: Tuple[str, ...] checks: Tuple[ScenarioCheckData, ...] source_requirements: Tuple[SourceRequirementData, ...] + artifact_requirements: Tuple[ArtifactRequirementData, ...] = () def _texts(values, minimum, maximum): @@ -69,13 +81,60 @@ def _check(raw, purpose): ) +def _artifact_requirements(raw, checks): + allowed = {check.id for check in checks if check.evidence_kind != "recorded_action"} + if type(raw) is not list or len(raw) > ARTIFACT_LIMIT: + raise ValueError("invalid artifact requirements") + result, seen, paths, aliases = [], set(), set(), {} + for item in raw: + if ( + type(item) is not dict + or set(item) != {"check", "paths"} + or type(item["check"]) is not str + or item["check"] not in allowed + or item["check"] in seen + or type(item["paths"]) is not list + or not 1 <= len(item["paths"]) <= ARTIFACT_LIMIT + ): + raise ValueError("invalid artifact requirement") + selected = [] + for path in item["paths"]: + if ( + type(path) is not str + or not 1 <= len(path) <= ARTIFACT_PATH_LIMIT + or sensitive_text(path) + or (path.casefold() in aliases and aliases[path.casefold()] != path) + or any( + not re.fullmatch(r"[A-Za-z0-9_-][A-Za-z0-9_.-]*", part) + or re.match( + r"(?i)(?:private|eval|assessment|criteria|grader)", part + ) + for part in path.split("/") + ) + or path in selected + ): + raise ValueError("invalid or private artifact path") + selected.append(path) + aliases[path.casefold()] = path + paths.add(path) + seen.add(item["check"]) + result.append(ArtifactRequirementData(item["check"], tuple(selected))) + if len(paths) > ARTIFACT_LIMIT: + raise ValueError("too many required artifact paths") + return tuple(result) + + def parse_scenario(raw, purpose): """Return None for an unrecorded scenario, else a validated ScenarioData.""" if raw is None: return None if not purpose: raise ValueError("a scenario record needs frozen purpose goals") - if type(raw) is not dict or set(raw) != set(ScenarioData.__dataclass_fields__): + fields = set(ScenarioData.__dataclass_fields__) + if type(raw) is not dict or set(raw) not in ( + fields, + fields - {"artifact_requirements"}, + ): raise ValueError("invalid scenario fields") checks = raw["checks"] if type(checks) is not list or not 1 <= len(checks) <= 16: @@ -96,4 +155,5 @@ def parse_scenario(raw, purpose): _texts(raw["limitations"], 0, 8), parsed, requirements, + _artifact_requirements(raw.get("artifact_requirements", []), parsed), ) diff --git a/plugin/skills/behavior-diff/scripts/reporting/schema.py b/plugin/skills/behavior-diff/scripts/reporting/schema.py index bf1665c..c08b124 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/schema.py +++ b/plugin/skills/behavior-diff/scripts/reporting/schema.py @@ -597,7 +597,10 @@ def _decisions(value, path, hunk_count, groups=None, final_answers=None): evidence = persisted_evidence(value.get("recorded_evidence", [])) raw_scenario = value.get("scenario_record") scenario = parse_scenario(raw_scenario, purpose) - if raw_scenario is not None and _json_value(asdict(scenario)) != raw_scenario: + canonical_scenario = _json_value(asdict(scenario)) if scenario else None + if canonical_scenario is not None and "artifact_requirements" not in raw_scenario: + canonical_scenario.pop("artifact_requirements") + if raw_scenario is not None and canonical_scenario != raw_scenario: _invalid(path + ".scenario_record", "canonical scenario recorded before trials") raw_target = value.get("target_assessment") target = parse_target( diff --git a/plugin/skills/behavior-diff/scripts/reporting/target.py b/plugin/skills/behavior-diff/scripts/reporting/target.py index 94b4a79..a04f3df 100644 --- a/plugin/skills/behavior-diff/scripts/reporting/target.py +++ b/plugin/skills/behavior-diff/scripts/reporting/target.py @@ -179,10 +179,11 @@ def evidence_records(raw): if ( type(trial) is not str or type(entries) is not list - or len(entries) > RECORD_LIMIT + 1 + or len(entries) > RECORD_LIMIT + 17 ): raise ValueError("invalid recorded evidence trial") trial_characters = 0 + ordinary_records = 0 for entry in entries: if type(entry) is not dict or set(entry) != { "id", @@ -199,13 +200,24 @@ def evidence_records(raw): if ( type(identifier) is not str or re.fullmatch( - re.escape(trial) + r":(?:tool-[1-9][0-9]*|trace|overflow)", + re.escape(trial) + + r":(?:tool-[1-9][0-9]*|artifact-(?:[1-9]|1[0-6])|trace|overflow)", identifier, ) is None or identifier in seen ): raise ValueError("invalid recorded evidence identity") + if ":artifact-" not in identifier: + ordinary_records += 1 + if ordinary_records > RECORD_LIMIT + 1: + raise ValueError("invalid recorded evidence trial") + elif status in ("available", "truncated") and ( + type(entry.get("source")) is not str + or re.fullmatch(r"artifact .+ sha256:[a-f0-9]{64}", entry["source"]) + is None + ): + raise ValueError("invalid captured artifact descriptor") if status not in ("available", "truncated", "omitted", "unavailable"): raise ValueError("invalid recorded evidence status") if ( @@ -338,7 +350,14 @@ def _criterion_definition(item, purpose, seen, frozen): def _criterion_assessment( - item, definition, final_answers, records, positions, requirements, persisted + item, + definition, + final_answers, + records, + positions, + requirements, + persisted, + artifact_paths=(), ): fields = set(_DEFINITION) | {"decisions", "before", "after"} if set(item) not in (fields, fields | {"rejection"}): @@ -394,9 +413,26 @@ def _criterion_assessment( excerpt = trial["output_excerpt"] if type(excerpt) is not str or len(excerpt) > 1200: raise ValueError("invalid target output excerpt") - if excerpt and name + ":answer" not in refs: - raise ValueError("target output excerpt must cite its trial answer") - if excerpt and excerpt not in final_answers[side][name]: + output_sources = [ + final_answers[side][name] for ref in refs if ref == name + ":answer" + ] + output_sources.extend( + record.text + for ref in refs + if (record := records.get(ref)) is not None + and record.trial == name + and ":artifact-" in record.id + and _returned_content(record) + and any( + record.source.startswith("artifact " + path + " sha256:") + for path in artifact_paths + ) + ) + if excerpt and not output_sources: + raise ValueError( + "target output excerpt must cite its trial answer or required artifact" + ) + if excerpt and not any(excerpt in text for text in output_sources): raise ValueError("target output excerpt must be exact and trial-local") for ref in refs: if ref != name + ":answer" and ( @@ -418,7 +454,14 @@ def _criterion_assessment( raise ValueError( "determinate target outcome needs available evidence" ) - has_output = answer in refs + has_output = bool(output_sources) + for path in artifact_paths: + if not any( + records[ref].source.startswith("artifact " + path + " sha256:") + for ref in content + if ":artifact-" in ref + ): + raise ValueError("criterion lacks a required captured artifact") if ( item["evidence_kind"] == "source_consistency" and not (has_output and content) @@ -528,6 +571,14 @@ def check_target( positions, requirements, persisted, + tuple( + path + for requirement in ( + scenario.artifact_requirements if scenario else () + ) + if requirement.check == definition[0] + for path in requirement.paths + ), ) except ValueError as error: criterion = TargetCriterionData(*definition, (), (), (), str(error)) @@ -610,6 +661,6 @@ def evidence_limit_text(limits): return ( f"At most {limits.per_excerpt_characters} characters per excerpt, " f"{limits.per_trial_characters} per trial, {limits.per_run_characters} per run, " - f"and {limits.per_trial_records} tool records per trial. " + f"and {limits.per_trial_records} captured records per trial. " f"Selection: {limits.selection}. {limits.privacy} {limits.provenance}" ) diff --git a/plugin/skills/behavior-diff/scripts/run-trial.sh b/plugin/skills/behavior-diff/scripts/run-trial.sh index c01fb29..baeb7d2 100755 --- a/plugin/skills/behavior-diff/scripts/run-trial.sh +++ b/plugin/skills/behavior-diff/scripts/run-trial.sh @@ -6,12 +6,13 @@ # # Usage: run-trial.sh --agent claude|codex|pi|omp --model M --dir DIR \ # --task-file FILE [--allowed CLAUDE_TOOL_LIST] [--trace-dir DIR] \ -# [--status-file FILE] +# [--status-file FILE] [--context-requirements-file PRIVATE_JSON] # The agent runs with cwd DIR; trace.jsonl/stderr.log land in --trace-dir # (default: DIR). --status-file names the caller-owned trial status record; # this script merges only its phase and the host CLI exit code into it. set -euo pipefail -agent="" model="" dir="" task_file="" allowed="" trace_dir="" status_file="" +umask 077 +agent="" model="" dir="" task_file="" allowed="" trace_dir="" status_file="" context_requirements_file="" while [ $# -gt 0 ]; do case "$1" in --agent) @@ -42,6 +43,10 @@ while [ $# -gt 0 ]; do status_file=$2 shift 2 ;; + --context-requirements-file) + context_requirements_file=$2 + shift 2 + ;; *) echo "run-trial: unknown argument $1" >&2 exit 2 @@ -58,6 +63,12 @@ esac exit 2 } scripts=$(cd "$(dirname "$0")" && pwd) +native_context_args=() +if [ -n "$context_requirements_file" ]; then + context_requirements_file=$(cd "$(dirname "$context_requirements_file")" && pwd -P)/$(basename "$context_requirements_file") + python3 "$scripts/native_context.py" validate --requirements "$context_requirements_file" \ + --agent "$agent" --workspace "$dir" || exit 2 +fi if [ "$agent" = codex ]; then case "$model" in sol | luna) @@ -71,6 +82,18 @@ fi task=$(cat "$task_file") trace_dir=$(cd "${trace_dir:-$dir}" && pwd) cd "$dir" +if [ -n "$context_requirements_file" ]; then + python3 "$scripts/native_context.py" prepare --requirements "$context_requirements_file" \ + --agent "$agent" --workspace "$PWD" --output "$trace_dir/context-boundary.json" \ + >"$trace_dir/native-context-args" || { + rm -f "$trace_dir/native-context-args" + exit 2 + } + while IFS= read -r -d '' arg; do + native_context_args+=("$arg") + done <"$trace_dir/native-context-args" + rm "$trace_dir/native-context-args" +fi # BEHAVIOR_DIFF_TRIAL tells this plugin's own hooks to stay silent inside a # trial — otherwise a user-scope install would nudge itself recursively. @@ -129,7 +152,11 @@ if [ "$agent" = claude ]; then --output-format stream-json --verbose >"$trace_dir/trace.jsonl" 2>"$trace_dir/stderr.log" || cli_exit=$? record_status "{\"cli_exit_code\":$cli_exit}" elif [ "$agent" = codex ]; then - if [ -n "${BEHAVIOR_DIFF_CODEX_ISOLATION_ROOT:-}" ]; then + if [ -n "$context_requirements_file" ]; then + codex "${native_context_args[@]}" exec --strict-config \ + --ephemeral --skip-git-repo-check -m "$model" --json "$task" \ + "$trace_dir/codex-raw.jsonl" 2>"$trace_dir/stderr.log" || cli_exit=$? + elif [ -n "${BEHAVIOR_DIFF_CODEX_ISOLATION_ROOT:-}" ]; then isolation_args=(exec --workspace "$PWD" --root "$BEHAVIOR_DIFF_CODEX_ISOLATION_ROOT" --model "$model") if [ -n "${BEHAVIOR_DIFF_CODEX_DENY_READ:-}" ]; then isolation_args+=(--deny-read "$BEHAVIOR_DIFF_CODEX_DENY_READ") @@ -158,6 +185,10 @@ elif [ "$agent" = codex ]; then | .item.text] | {type: "result", result: (last // "")}' \ "$trace_dir/codex-raw.jsonl" >>"$trace_dir/trace.jsonl" + if [ -n "$context_requirements_file" ]; then + python3 "$scripts/native_context.py" observe --requirements "$context_requirements_file" \ + --trace "$trace_dir/trace.jsonl" --output "$trace_dir/context-boundary.json" || exit 2 + fi elif [ "$agent" = omp ]; then # PIPESTATUS separates the host CLI's exit from the task writer's. printf '%s\n' "$task" | omp -p \ diff --git a/plugin/skills/behavior-diff/scripts/scenario.py b/plugin/skills/behavior-diff/scripts/scenario.py index c5acc33..46d9dfb 100755 --- a/plugin/skills/behavior-diff/scripts/scenario.py +++ b/plugin/skills/behavior-diff/scripts/scenario.py @@ -28,7 +28,12 @@ def read_scenario(path, purpose_path): with path.open(encoding="utf-8") as stream: raw = json.load(stream, object_pairs_hook=unique_object) scenario = parse_scenario(raw, parse_purpose(read_purpose(purpose_path))) - return asdict(scenario) if scenario else None + if scenario is None: + return None + result = asdict(scenario) + if "artifact_requirements" not in raw: + result.pop("artifact_requirements") + return result def main(): @@ -56,7 +61,10 @@ def main(): "with 1..8 distinct fnmatch patterns (<=200, no brackets, no edge " "spaces) over recorded source descriptors; each pattern needs a " "literal path, endpoint, or identifier and must not name a " - "sensitive source", + "sensitive source. Optional artifact_requirements is 0..16 " + "{check,paths} groups naming output/source_consistency checks, " + "with at most 16 distinct exact project-relative text paths " + "(<=120 characters); no aliases, hidden, private, evaluator or secret paths", file=sys.stderr, ) return 2 diff --git a/tests/native-context-boundary-test.py b/tests/native-context-boundary-test.py new file mode 100755 index 0000000..aeb0f4a --- /dev/null +++ b/tests/native-context-boundary-test.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Synthetic, model-free native boundary contract tests.""" + +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest +import sys +from unittest.mock import patch +import subprocess + +SCRIPT = ( + Path(__file__).resolve().parents[1] + / "plugin/skills/behavior-diff/scripts/native_context.py" +) +sys.path.insert(0, str(SCRIPT.parent)) +from reporting.context_boundary import boundary_exclusion +from reporting.load import trial_interruption, _trial +from reporting.evidence import collect_run_evidence +from decisions import trials_of + + +class NativeContextTests(unittest.TestCase): + def setUp(self): + spec = importlib.util.spec_from_file_location("native_context", SCRIPT) + self.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.module) + + def test_explicit_scope_and_absolute_paths(self): + with tempfile.TemporaryDirectory() as root: + path = Path(root) / "requirements.json" + path.write_text( + json.dumps( + { + "version": 1, + "scope": "codex-local-filesystem", + "deny_read": ["relative"], + } + ) + ) + with self.assertRaises(ValueError): + self.module.load_requirements(path) + + def test_native_profile_never_uses_legacy_sandbox(self): + with tempfile.TemporaryDirectory() as root: + workspace = Path(root) / "workspace" + workspace.mkdir() + forbidden = Path(root) / "historical-memory" + forbidden.mkdir() + args = self.module.policy_args(workspace, [str(forbidden)]) + joined = " ".join(args) + self.assertNotIn("--sandbox", args) + self.assertNotIn("danger-full-access", joined) + self.assertIn('default_permissions="behavior-diff-context"', args) + self.assertIn('"deny"', joined) + self.assertIn("enabled=false", joined) + + def test_observed_attempt_is_not_clean(self): + with tempfile.TemporaryDirectory() as root: + trace = Path(root) / "trace.jsonl" + trace.write_text( + json.dumps( + { + "type": "assistant", + "message": { + "content": [ + { + "type": "tool_use", + "input": { + "command": "cat /synthetic/forbidden/memory" + }, + } + ] + }, + } + ) + + "\n" + ) + result = self.module.observe(trace, ["/synthetic/forbidden"]) + self.assertEqual(result["status"], "violation") + self.assertNotIn("memory", json.dumps(result)) + + def test_no_observation_is_not_clean_proof(self): + with tempfile.TemporaryDirectory() as root: + trace = Path(root) / "trace.jsonl" + trace.write_text("") + result = self.module.observe(trace, ["/synthetic/forbidden"]) + self.assertEqual(result["observation"], "not_observed") + self.assertFalse(result["clean_context_proven"]) + + def test_required_missing_marker_excluded_everywhere(self): + with tempfile.TemporaryDirectory() as root: + run = Path(root) + (run / "config.json").write_text( + json.dumps({"context_boundary_required": True}) + ) + trial = run / "before-1" + trial.mkdir() + (trial / "trace.jsonl").write_text( + json.dumps({"type": "result", "result": "synthetic private result"}) + + "\n" + ) + (run / "grades.tsv").write_text("before-1\tPASS\t-\n") + self.assertIsNotNone(boundary_exclusion(trial)) + self.assertEqual(trial_interruption(trial), boundary_exclusion(trial)) + self.assertFalse(trials_of(run, finished_only=True)) + self.assertEqual(collect_run_evidence(run)["trials"]["before-1"], []) + + def test_violation_keeps_raw_evidence_out_of_assessment(self): + with tempfile.TemporaryDirectory() as root: + run = Path(root) + trial = run / "after-1" + trial.mkdir() + (trial / "context-boundary.json").write_text( + json.dumps({"version": 1, "status": "violation"}) + ) + (trial / "trace.jsonl").write_text( + json.dumps({"type": "result", "result": "synthetic private result"}) + + "\n" + ) + (run / "grades.tsv").write_text("after-1\tPASS\t-\n") + self.assertFalse(trials_of(run, finished_only=True)) + all_trials = trials_of(run, finished_only=False) + self.assertNotIn("synthetic private result", json.dumps(all_trials)) + self.assertEqual(collect_run_evidence(run)["trials"]["after-1"], []) + report_trial = _trial( + run, "after-1", ("PASS", "synthetic private action"), None + ) + self.assertEqual(report_trial.verdict, "BLOCKED") + self.assertEqual(report_trial.commands, ()) + self.assertEqual(report_trial.final, "") + self.assertNotIn("synthetic private", report_trial.outcome) + + def test_prepared_trial_is_excluded_until_observation_finishes(self): + with tempfile.TemporaryDirectory() as root: + run = Path(root).resolve() + trial = run / "before-1" + project = trial / "project" + project.mkdir(parents=True) + requirements = run / "requirements.json" + requirements.write_text( + json.dumps( + { + "version": 1, + "scope": "codex-local-filesystem", + "deny_read": [str(run / "historical-memory")], + } + ) + ) + (run / "config.json").write_text( + json.dumps( + { + "context_boundary_required": True, + } + ) + ) + argv = [ + "native_context.py", + "prepare", + "--requirements", + str(requirements), + "--workspace", + str(project), + "--output", + str(trial / "context-boundary.json"), + ] + with ( + patch.object(sys, "argv", argv), + patch.object(self.module, "check_route"), + patch.object(self.module, "preflight", return_value=[]), + ): + self.assertEqual(self.module.main(), 0) + # A worker may leave partial evidence before normalization fails. + (trial / "trace.jsonl").write_text( + json.dumps( + { + "type": "result", + "result": "synthetic private result", + } + ) + + "\n" + ) + (run / "grades.tsv").write_text("before-1\tPASS\t-\n") + self.assertIsNotNone(boundary_exclusion(trial)) + self.assertFalse(trials_of(run, finished_only=True)) + self.assertNotIn( + "synthetic private result", + json.dumps(trials_of(run, finished_only=False)), + ) + self.assertEqual(collect_run_evidence(run)["trials"]["before-1"], []) + + def test_unsupported_host_fails_before_native_invocation(self): + with ( + tempfile.TemporaryDirectory() as root, + patch.object(self.module.subprocess, "run") as native, + ): + with self.assertRaises(ValueError): + self.module.check_route("claude", Path(root)) + native.assert_not_called() + + def test_runner_rejects_required_unsupported_route_before_agent(self): + with tempfile.TemporaryDirectory() as root: + work = Path(root).resolve() + (work / "SKILL.md").write_text("synthetic instruction") + requirements = work / "requirements.json" + requirements.write_text( + json.dumps( + { + "version": 1, + "scope": "codex-local-filesystem", + "deny_read": [str(work / "historical-memory")], + } + ) + ) + result = subprocess.run( + [ + "bash", + str(SCRIPT.parent / "behavior-diff.sh"), + "--file", + "SKILL.md", + "--task", + "synthetic task", + "--agent", + "claude", + "--context-requirements-file", + str(requirements), + ], + cwd=work, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 2) + self.assertIn("native context boundary unavailable", result.stderr) + self.assertNotIn("starting", result.stdout) + + def test_corrupt_config_excludes_missing_marker_evidence(self): + for raw in ( + '{"context_boundary_required":', + "[]", + '{"context_boundary_required":"false"}', + ): + with self.subTest(raw=raw), tempfile.TemporaryDirectory() as root: + run = Path(root) + trial = run / "before-1" + trial.mkdir() + (run / "config.json").write_text(raw) + (trial / "trace.jsonl").write_text( + json.dumps({"type": "result", "result": "synthetic private result"}) + + "\n" + ) + (run / "grades.tsv").write_text("before-1\tPASS\t-\n") + self.assertIsNotNone(boundary_exclusion(trial)) + self.assertFalse(trials_of(run)) + self.assertEqual(collect_run_evidence(run)["trials"]["before-1"], []) + report_trial = _trial(run, "before-1", ("PASS", "-"), None) + self.assertEqual(report_trial.verdict, "BLOCKED") + + def test_missing_legacy_config_does_not_claim_boundary(self): + with tempfile.TemporaryDirectory() as root: + trial = Path(root) / "before-1" + trial.mkdir() + self.assertIsNone(boundary_exclusion(trial)) + + def test_requirements_duplicate_keys_rejected(self): + with tempfile.TemporaryDirectory() as root: + path = Path(root) / "requirements.json" + path.write_text( + '{"version":1,"version":1,"scope":"codex-local-filesystem","deny_read":["/synthetic/history"]}' + ) + with self.assertRaises(ValueError): + self.module.load_requirements(path) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/preparation-evidence-test.py b/tests/preparation-evidence-test.py new file mode 100755 index 0000000..d49b617 --- /dev/null +++ b/tests/preparation-evidence-test.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Synthetic native preparation records only; no host processes or model calls.""" + +import copy +import json +from pathlib import Path +import sys +import tempfile +import unittest + +SCRIPTS = Path(__file__).resolve().parents[1] / "plugin/skills/behavior-diff/scripts" +sys.path.insert(0, str(SCRIPTS)) +from preparation_evidence import validate_preparation_evidence + + +class NativePreparationTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.input = self.root / "input.txt" + self.input.write_text("Synthetic historical sources only.\n") + self.record = self.root / "native.json" + self.reference = { + "host": "codex", + "record_file": str(self.record), + "input_file": str(self.input), + "thread_id": "synthetic-thread", + "turn_id": "synthetic-turn", + } + self.native = { + "thread": { + "id": "synthetic-thread", + "sessionId": "synthetic-session", + "forkedFromId": None, + "parentThreadId": None, + "model": "synthetic-current-model-not-execution-proof", + "historyMode": "legacy", + "turns": [ + { + "id": "synthetic-turn", + "status": "completed", + "error": None, + "itemsView": "full", + "items": [ + { + "id": "synthetic-input", + "type": "userMessage", + "content": [ + {"type": "text", "text": self.input.read_text()} + ], + } + ], + } + ], + }, + } + + def validate(self, native=None): + self.record.write_text(json.dumps(self.native if native is None else native)) + return validate_preparation_evidence(self.reference) + + def test_available_facets_never_prove_model_or_full_context(self): + facts = self.validate() + self.assertEqual(facts["supplied_user_input"], "verified") + self.assertEqual(facts["completion"], "verified") + self.assertEqual(facts["resolved_model"], "unavailable") + self.assertEqual(facts["historical_boundary"], "unavailable") + self.assertEqual(facts["status"], "unverified") + self.assertNotIn(self.input.read_text(), json.dumps(facts)) + self.assertNotIn(self.native["thread"]["model"], json.dumps(facts)) + + def test_current_model_never_changes_native_fact_availability(self): + original = self.validate() + self.native["thread"]["model"] = "another-requested-model" + self.assertEqual(self.validate(), original) + self.native["thread"].pop("model") + self.assertEqual(self.validate(), original) + + def test_truncated_and_duplicate_native_json_are_rejected(self): + for content in ( + '{"thread":', + '{"thread": {}, "thread": {}}', + ): + with self.subTest(content=content): + self.record.write_text(content) + with self.assertRaises(ValueError): + validate_preparation_evidence(self.reference) + + def test_missing_wrong_failed_partial_and_forked_records_rejected(self): + mutations = [ + lambda n: n["thread"].pop("turns"), + lambda n: n["thread"].pop("sessionId"), + lambda n: n["thread"].pop("forkedFromId"), + lambda n: n["thread"]["turns"][0].pop("error"), + lambda n: n["thread"].update(id="wrong-thread"), + lambda n: n["thread"]["turns"][0].update(id="wrong-turn"), + lambda n: n["thread"].update(forkedFromId="prior-thread"), + lambda n: n["thread"].update(parentThreadId="prior-thread"), + lambda n: n["thread"].update(historyMode="paginated"), + lambda n: n["thread"]["turns"].append( + copy.deepcopy(n["thread"]["turns"][0]) + ), + lambda n: n["thread"]["turns"][0].update(status="failed"), + lambda n: n["thread"]["turns"][0].update(error={"message": "synthetic"}), + lambda n: n["thread"]["turns"][0].update(itemsView="summary"), + lambda n: n["thread"]["turns"][0].update(items=[]), + lambda n: n["thread"]["turns"][0]["items"].append( + copy.deepcopy(n["thread"]["turns"][0]["items"][0]) + ), + lambda n: n["thread"]["turns"][0]["items"].append( + {"id": "compact", "type": "contextCompaction"} + ), + lambda n: n["thread"]["turns"][0]["items"][0]["content"][0].update( + text="altered" + ), + ] + for index, mutate in enumerate(mutations): + with self.subTest(index=index): + native = copy.deepcopy(self.native) + mutate(native) + with self.assertRaises(ValueError): + self.validate(native) + + def test_unsupported_host_explicitly_unavailable(self): + for host in ("claude", "pi", "omp"): + with self.subTest(host=host): + self.reference["host"] = host + facts = self.validate() + self.assertEqual(facts["supplied_user_input"], "unavailable") + self.assertEqual(facts["completion"], "unavailable") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/recorded-evidence-test.py b/tests/recorded-evidence-test.py index 3e2f271..3712018 100755 --- a/tests/recorded-evidence-test.py +++ b/tests/recorded-evidence-test.py @@ -623,6 +623,186 @@ def test_invalid_frozen_scenario_fails_closed(self): self.assertEqual(self.records()[0]["status"], "available") +class ArtifactEvidenceTests(unittest.TestCase): + """Only frozen, exact output paths admit captured project bytes.""" + + setUp = RecordedEvidenceTests.setUp + trace = RecordedEvidenceTests.trace + records = RecordedEvidenceTests.records + + def configure_artifacts(self, paths=("decision-record.md",)): + config = scenario_config(["*facts.txt*"]) + config["scenario_record"]["artifact_requirements"] = [ + {"check": "cap-agrees", "paths": list(paths)} + ] + (self.root / "config.json").write_text(json.dumps(config)) + return config + + def test_required_shell_written_artifact_reaches_assessment(self): + self.trace(command("write", "printf proposed > decision-record.md", "")) + project = self.root / "before-1/project" + project.mkdir() + (project / "decision-record.md").write_text("Synthetic actual saved decision.") + (project / "private-evaluator.md").write_text("Must never enter assessment.") + self.configure_artifacts() + records = self.records() + artifact = next(r for r in records if ":artifact-" in r["id"]) + self.assertEqual(artifact["text"], "Synthetic actual saved decision.") + self.assertEqual(artifact["status"], "available") + self.assertIn("decision-record.md", artifact["source"]) + self.assertNotIn("proposed", artifact["text"]) + self.assertNotIn("Must never", json.dumps(records)) + self.assertEqual( + next(r for r in records if ":tool-" in r["id"])["status"], "omitted" + ) + evidence_records(collect_run_evidence(self.root)) + + def test_missing_links_nontext_and_sensitive_artifacts_fail_closed(self): + self.trace([]) + project = self.root / "before-1/project" + project.mkdir() + outside = self.root / "evaluator.txt" + outside.write_text("Private evaluator sentinel") + (project / "linked.md").symlink_to(outside) + (project / "hardlink.md").hardlink_to(outside) + (project / "directory.md").mkdir() + (project / "binary.md").write_bytes(b"synthetic\0binary") + (project / "filtered.md").write_text("password=synthetic-value") + (project / "large.md").write_text("x" * 65537) + self.configure_artifacts( + ( + "missing.md", + "linked.md", + "hardlink.md", + "directory.md", + "binary.md", + "filtered.md", + "large.md", + ) + ) + records = self.records() + self.assertTrue(all(not record["text"] for record in records)) + self.assertTrue( + all(record["status"] in ("omitted", "unavailable") for record in records) + ) + self.assertNotIn("Private evaluator sentinel", json.dumps(records)) + + def test_artifact_excerpt_budget_and_full_privacy_scan(self): + self.trace([]) + project = self.root / "before-1/project" + project.mkdir() + for index in range(6): + (project / f"decision-{index}.md").write_text("s" * (EXCERPT_LIMIT + 1)) + (project / "filtered.md").write_text("s" * EXCERPT_LIMIT + " password=hidden") + self.configure_artifacts( + tuple(f"decision-{index}.md" for index in range(6)) + ("filtered.md",) + ) + records = self.records() + self.assertEqual(sum(len(record["text"]) for record in records), TRIAL_LIMIT) + self.assertTrue(all(record["status"] == "truncated" for record in records[:6])) + self.assertEqual(records[-1]["status"], "omitted") + self.assertEqual(records[-1]["text"], "") + evidence_records(collect_run_evidence(self.root)) + + def test_artifact_contract_bounds_and_case_aliases(self): + self.trace([]) + for paths in ( + (), + ("same.md", "same.md"), + ("Same.md", "same.md"), + ("a" * 121,), + tuple(f"decision-{index}.md" for index in range(17)), + ): + with self.subTest(paths=paths): + self.configure_artifacts(paths) + with self.assertRaises(ValueError): + collect_run_evidence(self.root) + + def test_missing_trace_keeps_artifact_and_project_symlinks_are_excluded(self): + project = self.root / "before-1/project" + project.mkdir(parents=True) + (project / "decision-record.md").write_text("Synthetic saved decision.") + self.configure_artifacts() + records = self.records() + self.assertEqual(records[0]["status"], "available") + self.assertEqual(records[1]["id"], "before-1:trace") + (project / "decision-record.md").unlink() + project.rmdir() + outside = self.root / "private-evaluator" + outside.mkdir() + (outside / "decision-record.md").write_text("Private sentinel") + project.symlink_to(outside, target_is_directory=True) + records = self.records() + self.assertEqual(records[0]["status"], "unavailable") + self.assertNotIn("Private sentinel", json.dumps(records)) + + def test_path_aliases_and_private_evaluator_selection_rejected(self): + self.trace([]) + for path in ( + "../decision.md", + "/tmp/decision.md", + "./decision.md", + "a//decision.md", + "a/../decision.md", + ".git/config", + ".env", + "private/decision.md", + "private-evaluator.md", + "criteria.json", + "assessment.md", + "eval-notes.md", + "a\\decision.md", + "decision*.md", + ): + with self.subTest(path=path): + self.configure_artifacts((path,)) + with self.assertRaises(ValueError): + collect_run_evidence(self.root) + + def test_artifact_is_output_not_a_source_read_or_recorded_action(self): + for trial in ("before-1", "after-1"): + self.trace(call("read-1"), trial) + project = self.root / trial / "project" + project.mkdir() + (project / "decision-record.md").write_text("Synthetic cap: 500.") + config = self.configure_artifacts() + scenario = parse_scenario(config["scenario_record"], PURPOSE) + definition = config["scenario_record"]["checks"][0] + raw = criterion( + "source_consistency", + excerpt="Synthetic cap: 500.", + before_outcome="unassessable", + ) + row = raw["criteria"][0] + row.update(definition) + row["after"][0]["refs"] = ["after-1:artifact-1", "after-1:tool-1"] + evidence = evidence_records(collect_run_evidence(self.root)) + target, rejection = check_target( + raw, PURPOSE, ANSWERS, evidence, scenario=scenario + ) + self.assertIsNone(rejection) + self.assertIsNone(target.criteria[0].rejection) + row["after"][0]["refs"] = ["after-1:artifact-1"] + target, _ = check_target(raw, PURPOSE, ANSWERS, evidence, scenario=scenario) + self.assertEqual( + target.criteria[0].rejection, "criterion lacks a required frozen source" + ) + row["after"][0]["refs"] = ["after-1:answer", "after-1:tool-1"] + row["after"][0]["output_excerpt"] = "Synthetic draft" + target, _ = check_target(raw, PURPOSE, ANSWERS, evidence, scenario=scenario) + self.assertEqual( + target.criteria[0].rejection, "criterion lacks a required captured artifact" + ) + row["evidence_kind"] = "recorded_action" + row["after"][0]["refs"] = ["after-1:artifact-1"] + row["after"][0]["output_excerpt"] = "" + target, _ = check_target(raw, PURPOSE, ANSWERS, evidence) + self.assertEqual( + target.criteria[0].rejection, + "criterion lacks required source or output evidence", + ) + + class TargetEvidenceTests(unittest.TestCase): """A captured call proves that the call ran, never what it returned.""" diff --git a/tests/report-schema-test.py b/tests/report-schema-test.py index 24155ec..12b686a 100755 --- a/tests/report-schema-test.py +++ b/tests/report-schema-test.py @@ -406,6 +406,7 @@ def scenario_record(payload=""): } ], "source_requirements": [], + "artifact_requirements": [], } diff --git a/tests/shared-data-test.py b/tests/shared-data-test.py index 957cb40..e241055 100755 --- a/tests/shared-data-test.py +++ b/tests/shared-data-test.py @@ -433,6 +433,72 @@ def test_non_null_purpose_rejects_null_scenario_before_freezing(self): self.assertTrue(result.stderr.strip()) self.assertFalse(root.exists()) + def test_native_preparation_files_are_private_frozen_and_drift_checked(self): + project, data = self.standard() + descriptor = self.handoff_file() + reference = { + "host": "codex", + "thread_id": "synthetic-thread", + "turn_id": "synthetic-turn", + "record_file": str(descriptor.parent / "native.json"), + "input_file": str(descriptor.parent / "input.txt"), + } + supplied = "Synthetic historical preparation input.\n" + native = { + "thread": { + "id": reference["thread_id"], + "sessionId": "synthetic-session", + "forkedFromId": None, + "parentThreadId": None, + "historyMode": "legacy", + "turns": [ + { + "id": reference["turn_id"], + "status": "completed", + "error": None, + "itemsView": "full", + "items": [ + { + "id": "synthetic-user-item", + "type": "userMessage", + "content": [{"type": "text", "text": supplied}], + } + ], + } + ], + }, + } + for key, content in ( + ("record_file", json.dumps(native)), + ("input_file", supplied), + ): + path = Path(reference[key]) + path.write_text(content) + path.chmod(0o600) + handoff = json.loads(descriptor.read_text()) + handoff["preparation_evidence"] = reference + descriptor.write_text(json.dumps(handoff)) + stage = self.staged(project, data, handoff=descriptor, exclude=PRIVATE) + frozen = json.loads((stage / "freeze.json").read_text()) + context = frozen["context"] + facts = context["preparation_evidence"]["facts"] + self.assertEqual(facts["supplied_user_input"], "verified") + self.assertEqual(facts["resolved_model"], "unavailable") + self.assertEqual(facts["historical_boundary"], "unavailable") + self.assertNotIn(supplied, json.dumps(frozen)) + self.assertNotIn(reference["record_file"], context["argv"]) + self.assertNotIn(reference["input_file"], context["argv"]) + for key in ("record_file", "input_file"): + path = Path(reference[key]) + original = path.read_bytes() + path.write_bytes(original + b" ") + self.assertNotEqual(self.verify(stage).returncode, 0) + path.unlink() + self.assertNotEqual(self.verify(stage).returncode, 0) + path.write_bytes(original) + path.chmod(0o600) + self.assertEqual(self.verify(stage).returncode, 0) + def test_complete_staged_handoff_rejects_every_context_and_settings_mutation(self): project, data = self.standard() descriptor = self.handoff_file() From 5808dbe9bd538ce4e806f7e9c237a5bb5e53bc04 Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Thu, 29 Oct 2026 12:28:23 +0800 Subject: [PATCH 2/2] test: keep native runner smoke out of CI Signed-off-by: Kent Huang --- tests/native-context-boundary-test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/native-context-boundary-test.py b/tests/native-context-boundary-test.py index aeb0f4a..bed138e 100755 --- a/tests/native-context-boundary-test.py +++ b/tests/native-context-boundary-test.py @@ -3,6 +3,7 @@ import importlib.util import json +import os from pathlib import Path import tempfile import unittest @@ -199,6 +200,10 @@ def test_unsupported_host_fails_before_native_invocation(self): self.module.check_route("claude", Path(root)) native.assert_not_called() + @unittest.skipIf( + os.environ.get("CI"), + "Local-only runner smoke; CI must not invoke behavior-diff.sh", + ) def test_runner_rejects_required_unsupported_route_before_agent(self): with tempfile.TemporaryDirectory() as root: work = Path(root).resolve()