diff --git a/.github/workflows/validation-catalog-tests.yml b/.github/workflows/validation-catalog-tests.yml index 17ce043..7a69242 100644 --- a/.github/workflows/validation-catalog-tests.yml +++ b/.github/workflows/validation-catalog-tests.yml @@ -74,6 +74,10 @@ on: - rustfs-replication-test.sh - rustfs-storage-test.sh - test_validation_scanner_usage.sh + - test_validation_scanner_usage.py + - rustfs-scanner-usage-test.sh + - scripts/validation_scanner_usage/** + - docs/scanner-usage-validation.md - .github/workflows/validation-catalog-tests.yml push: branches: [main] @@ -149,6 +153,10 @@ on: - rustfs-replication-test.sh - rustfs-storage-test.sh - test_validation_scanner_usage.sh + - test_validation_scanner_usage.py + - rustfs-scanner-usage-test.sh + - scripts/validation_scanner_usage/** + - docs/scanner-usage-validation.md - .github/workflows/validation-catalog-tests.yml permissions: @@ -187,3 +195,5 @@ jobs: - run: bash -n scripts/run_validation_case.sh scripts/validation_heal/run_case.sh scripts/validation_oidc/run_case.sh - run: bash -n rustfs-storage-test.sh rustfs_pool_expand.sh rustfs-replication-test.sh scripts/validation_pool/run_case.sh scripts/validation_replication/run_case.sh test_validation_scanner_usage.sh - run: bash test_validation_scanner_usage.sh + - run: python3 -B -m unittest -v test_validation_scanner_usage.py + - run: bash -n rustfs-scanner-usage-test.sh scripts/validation_scanner_usage/run_case.sh diff --git a/README.md b/README.md index 00b3e02..599f9d5 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,18 @@ cases use single-disk deployment. The scanner case uses the storage suite's single-node four-disk deployment, waits for a post-upload successful scanner cycle, and requires a complete usage snapshot with exactly 20 objects and 20,690 logical bytes. -Each requires a candidate manifest with `source_sha`, `package_url`, and -`package_sha256`. The installed binary -SHA-256 is measured on the test node and included in result provenance: +The separate `scanner-usage-regressions` module adds USAGE-01 through USAGE-20 +without changing that strict smoke test or the 158 source catalog cases. +Run it explicitly with `rustfs-scanner-usage-test.sh` against existing endpoints; +see [the usage regression contracts and evidence requirements](docs/scanner-usage-validation.md). +Public observations cannot prove internal ACK ordering or cross-cycle races, +so missing candidate-native hooks remain BLOCKED or UNSUPPORTED. +The endpoint-only usage runner records operator-supplied candidate and binary +identities; it does not independently attest the remote installed binary. +The canonical S3, KMS, IAM, and scanner smoke deployment cases above require a +candidate manifest with `source_sha`, `package_url`, and `package_sha256`. +Those deployment cases measure the installed binary SHA-256 on the test node +and include it in result provenance: ```bash ./rustfs-s3-compat-test.sh \ diff --git a/docs/scanner-usage-validation.md b/docs/scanner-usage-validation.md new file mode 100644 index 0000000..5c18d14 --- /dev/null +++ b/docs/scanner-usage-validation.md @@ -0,0 +1,235 @@ +# Scanner usage regression suite + +This suite registers the twenty USAGE acceptance contracts from backlog issue #3178 and implements public probes plus a strict adapter for candidate-native tests. Registration alone is not test implementation or acceptance: each native assertion needs an executable, reviewed candidate binding before the complete case is available. +It runs against **explicitly configured endpoints**. It never installs RustFS, +changes scanner settings, restarts a node, acquires a VM deployment lock, or +deletes system metadata. It creates random `usage--*` buckets and deletes +only buckets that this invocation successfully created. `owned-buckets.json` +retains the exact names and create intents if cleanup fails. A separate +`acknowledged-requests.jsonl` appends each acknowledged PUT and DELETE without +rewriting previous requests. An unacknowledged create remains in `unconfirmed_creates` and forces +`cleanup_status=FAIL`, even when the create may have committed remotely. +Ownership is unconfirmed, so the runner does not delete that bucket +automatically; the environment owner must inspect +the retained request and deployment before any scoped manual recovery. + +The existing `scanner.usage-convergence` 20-object smoke test is unchanged. +The new cases are a `local_modules` extension in `validation-cases.json`; the +158-case upstream source snapshot and its hash retain their original meaning. +They are available through explicit case/module selection and the canonical +runner, but do not enter a scheduled fleet rotation without endpoint admission. + +## Run the public API cases + +Requirements: Python 3.10+, AWS CLI, `rc` for the shared report's client +provenance, and credentials allowed to create/delete test buckets and read the +Admin data-usage API. Credentials are read from the environment and are not +written into reports. TLS verification is enabled and requests do not use an +ambient proxy. No endpoint is inferred from an old lab inventory. + +The candidate manifest must contain `package_url`, `package_sha256`, +`source_sha`, and `binary_sha256`. The operator must verify that this manifest +describes the running deployment before execution. This endpoint-only runner +does not attest a remote executable's identity; that limitation is recorded in +the result provenance. Preserve the VM deployment's binary hashes, topology, +features, configuration and base/candidate identities beside this report. + +```bash +export RUSTFS_ACCESS_KEY='your-test-access-key' +export RUSTFS_SECRET_KEY='your-test-secret-key' +bash rustfs-scanner-usage-test.sh \ + --endpoint http://test-node:9000 \ + --admin-endpoint http://test-node:9000 \ + --candidate-manifest /path/to/deployed-candidate.json \ + --artifacts-dir /path/to/new-usage-results \ + --cases blackbox +``` + +`blackbox` runs USAGE-01 and USAGE-20. These have complete public API assertions. +Object, prefix, historical version and marker accounting uses acknowledged +requests plus an independent, explicitly paginated S3 listing; payload readback +uses SHA-256. The version case counts keys with retained historical versions +even when their latest version is a delete marker. Marker-only keys are not +covered by this initial fixture. Missing counters remain unknown, and neither +pending nor observed values can pass an exact-convergence assertion. + +Use `--cases USAGE-03,USAGE-05,USAGE-07,USAGE-08,USAGE-09,USAGE-11,USAGE-18` +to execute additional public probes. Supply multiple `--endpoint` and +`--admin-endpoint` options for USAGE-07. These cases also require internal +assertions; successful public probes alone leave their overall verdict +`BLOCKED`, with the successful assertions and the missing evidence listed +separately. Endpoint count is never treated as physical set/source count. + +The C1 metadata adapter recognizes the existing +`GET /rustfs/admin/v3/datausageinfo` endpoint and these optional bucket fields: + +- `usage_value_state`: `certified`, `carried`, or `observed`. +- `usage_certification`: null or `{epoch, cycle, certified_at}`; + `certified_at` uses `{secs_since_epoch, nanos_since_epoch}`. +- Top-level pending, partial, observational and admin-projection flags veto a + false convergence claim. Unknown future state values remain unknown. + +Candidates without these optional fields can still run the legacy-compatible +static/version accounting probes. Metadata-specific probes return +`UNSUPPORTED`; a missing carried/observed sampling window is `BLOCKED`. +The adapter never derives a certificate from `last_update` or collection time. + +Default request timeout is 30 seconds, convergence timeout 600 seconds, and +poll interval 5 seconds. `--objects` defaults to 32 and is bounded at 100000. +These are test deadlines and fixture controls, not RustFS runtime changes. +Timeout, malformed JSON, missing evidence and cleanup failure cannot pass. +An S3 service error is retained with its error code; Admin evidence includes +the observed HTTP status and request ID. AWS CLI stderr does not always expose +an HTTP status or request ID, so a `ServiceUnavailable` code is not relabelled +as a directly observed HTTP 503. + +USAGE-17 is disabled unless `--load-seconds` is at least 3600. Its public probe +performs bounded sequential PUT/GET/DELETE operations and retains each command's +duration and error. That duration includes AWS CLI startup. It is **not** a +saturation benchmark or a matched A/B throughput claim; its comparison/native +assertion remains blocked without the corresponding evidence. + +## Native capability adapter + +Public APIs cannot prove a scanned frontier, a budget-stop reason, an epoch +fence, source membership, an ACK race, producer-watch coverage or quota/cache +projection isolation. Those assertions call a candidate-native runner. +No native manifest means `BLOCKED`; a missing case/test capability is +`UNSUPPORTED`. An exact test that passes without the required assertion +milestones is `INFRA_ERROR`, not PASS. A failed test is FAIL. + +Provide `--native-manifest /path/to/native-manifest.json`, or set +`RUSTFS_USAGE_NATIVE_MANIFEST`. To run native tests without endpoint requests, +use `--execution native`. The resulting profile is `native-test`, not a VM run. +This adapter does not compile RustFS or modify its checkout. Obtain reviewed +test binaries from the candidate build, retaining their build provenance. + +```json +{ + "schema_version": 1, + "source_commit": "the_same_40_hex_commit_as_the_candidate_manifest", + "cases": { + "USAGE-13": { + "tests": [ + { + "binary": "/absolute/path/to/candidate-scanner-test-binary", + "binary_sha256": "the_actual_64_hex_binary_sha256", + "test": "scanner::tests::exact_reviewed_regression_name", + "assertions": ["USAGE-13-A01"] + } + ] + } + } +} +``` + +The manifest is an explicit integration contract, **not a bundled claim that +these hooks already exist**. The suite ships no fabricated native pass fixture. +The adapter verifies the binary hash, matching candidate source identity fields, exact libtest +inventory, exactly one executed and non-ignored test, every declared assertion, +and fixture cleanup. Multiple exact tests may contribute to one case, but all +required assertion IDs must be covered. The case shares one native timeout; +inventory and individual tests cannot reset it. +Each assertion ID has exactly one complete test owner; duplicate claims across +tests are rejected instead of treating one subcondition as complete coverage. +Each exact test belongs to one case. A milestone for a different case rejects +the execution; separate case wrappers are required for shared fixture logic. +If only some native assertions have reviewed hooks, their real PASS evidence +is retained and the uncovered assertions remain UNSUPPORTED; the case cannot pass. + +### Preparing and using the manifest + +1. Build the reviewed native tests from the candidate's committed checkout. Retain the source SHA, clean checkout status, lockfile digest, toolchain, features and build command beside the executables. A dirty working tree is development evidence and must not be labelled as a build of its unchanged HEAD. +2. Obtain executable paths from the build's structured output, enumerate each binding with `BINARY EXACT_TEST --exact --list`, and calculate its SHA-256. Review each test's assertions against `contracts.py`; similar names or related unit tests do not establish complete coverage. +3. Write the manifest using those exact paths, hashes, source SHA and reviewed assertion IDs. Include only available bindings. The adapter compares provenance fields but cannot reconstruct or attest the compilation, so the build receipt remains part of the acceptance evidence. +4. Run `--execution native --cases all` to produce the full twenty-row capability report. Keep partial PASS assertions, BLOCKED/UNSUPPORTED rows and cleanup failures in the report. This profile proves candidate-local regressions only; it does not establish S3 endpoint or distributed VM acceptance. +5. After the environment owner verifies deployment identity and admits the shared environment, run the relevant public and combined cases with that same candidate identity. Preserve the native report and VM report as separate results. Rebuild and regenerate hashes whenever the candidate source or test binaries change. + +Native-only execution still requires the candidate manifest and `rc` for the shared report's client provenance. It does not require endpoints, AWS credentials or a deployment. Without a native manifest, all twenty native-profile rows remain BLOCKED and no public fixture is created. A partial manifest never supplies missing capabilities by implication. + +After the real assertions and durable fences have been checked, the native +test emits one line for each claimed assertion and one cleanup line: + +```text +usage_assertion: {"case_id":"USAGE-13","assertion_id":"USAGE-13-A01","status":"PASS","evidence":{"observed_budget_reason":"objects","measured_drain_ms":30}} +usage_cleanup: {"case_id":"USAGE-13","status":"PASS"} +``` + +Evidence values must come from that test execution. A test name or a process +exit code does not demonstrate the broader assertion by itself. Keep native +test scopes aligned with the exact assertion texts in `contracts.py`. +If a test proves only one subcondition, do not use it to mark a broader row +complete. Missing producer, mutation or remote hooks remain visible gaps. +USAGE-13 separates the count-stop drain, remote parent limits, drain completion, +lease/epoch loss, root CAS failure, companion failure, sealed-candidate +immutability and durable-root ACK into eight assertion IDs. The example above +covers only A01; A02 through A08 still need their own reviewed tests and markers. + +## Twenty-row acceptance mapping + +The public-probe column identifies executable probes in this repository. The additional-evidence column states the remaining contract, not a claim that a native test already implements it. USAGE-01 and USAGE-20 have complete public probes; every other case requires candidate-native implementation and bindings for all of its native assertions. A missing binding is an implementation/integration gap even when the report correctly records BLOCKED or UNSUPPORTED. + +| Design case | Public probe | Additional native or controlled-fleet evidence | +| --- | --- | --- | +| USAGE-01 | Create/write/delete/empty and absent values | None for the stated public contract | +| USAGE-02 | None | Three-cycle long sweep plus unrelated remote writes and watch proof | +| USAGE-03 | Insert/overwrite/delete after a measured baseline | Already-scanned mutation boundary, mixed non-certification and ACK | +| USAGE-04 | Nested/prefix-shaped keys and final exactness | Continuous hot prefix, cold durable frontier and no prefix ACK | +| USAGE-05 | Hot writes plus three cold buckets | Actual shared budget, quantum cancellation and durable fairness | +| USAGE-06 | None | 100000-entry wide/deep directories and bounded checkpoints | +| USAGE-07 | Writes/readback/admin reads through multiple endpoints | Actual pools/sets beyond concurrency limit and all-source coverage | +| USAGE-08 | Unique absent bucket and eventual exactness | Cold cluster, no LKG, legacy primary/quota isolation | +| USAGE-09 | Carried/observed metadata sampling | Multi-round timestamps, source mixing and certificate provenance | +| USAGE-10 | None | Crash before/after EOF CAS and successor fencing | +| USAGE-11 | Same-name owned bucket recreation and exact successor | Old-incarnation carry/overlay/ACK rejection | +| USAGE-12 | None | Watch gaps, install race, expiry, restart, overflow and ABA | +| USAGE-13 | None | Lease/CAS/companion failure and budget/terminal/drain races | +| USAGE-14 | None | Lost/reordered/duplicate ACK and post-walk mutation | +| USAGE-15 | None | Movement, topology and tier-generation guardrails | +| USAGE-16 | None | Old readers/peers, independent capabilities and rollback | +| USAGE-17 | Explicit 60-minute sequential workload and exact convergence | Same-condition base/candidate metrics and workload comparison | +| USAGE-18 | Per-bucket metadata with separate collection timestamps | Metrics failures and admin/quota/cache/writer separation | +| USAGE-19 | None | Quota v1/v2 and legacy fallback decisions | +| USAGE-20 | Object/prefix collision, versions, markers and payloads | Marker-only keys remain an explicit follow-up fixture | + +USAGE-02 remains open until C5 and VM acceptance. Default-disabled quantum is +not completion. Independent prefix certification/replace/ACK is deferred. +No test in this suite promises ten-minute convergence for arbitrary data size. + +## Reports and canonical execution + +Each run writes `run-configuration.json`, `summary.json`, `report.md`, a result +under `cases/` for each selected row, and evidence under `evidence//`. +Each assertion has its own verdict and concrete artifact paths. Native output, +S3 acknowledgements, independent inventories, raw Admin samples, collection +times, exact commands/exit codes, cleanup and ownership journals are retained. +Every non-PASS case makes the runner exit nonzero. Reports never convert a +capability gap or a successful public subset into a complete PASS. + +The canonical four-argument runner supports these IDs: + +```bash +export RUSTFS_USAGE_ENDPOINT=http://test-node:9000 +export RUSTFS_USAGE_ADMIN_ENDPOINT=http://test-node:9000 +bash scripts/run_validation_case.sh \ + scanner.usage-01-static-accounting \ + /path/to/deployed-candidate.json \ + "$(git rev-parse HEAD)" \ + /path/to/new-canonical-artifacts +``` + +The existing host/shared-lock procedure must be applied by the environment +owner before using this endpoint-only adapter on a shared VM lab. The adapter +does not take ownership of deployments or other users' buckets. + +Tool-only verification, with no endpoint or RustFS binary: + +```bash +python3 -B -m unittest -v test_validation_scanner_usage.py +python3 -B -m unittest -v test_validation_migration.py +python3 -B scripts/validation_catalog.py check +bash test_validation_scanner_usage.sh +``` + +The local test doubles verify error handling and assertion gates; their PASS +results are not product or VM acceptance results. diff --git a/rustfs-scanner-usage-test.sh b/rustfs-scanner-usage-test.sh new file mode 100755 index 0000000..ddb1001 --- /dev/null +++ b/rustfs-scanner-usage-test.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${ROOT}" +exec python3 -B -m scripts.validation_scanner_usage.runner "$@" diff --git a/scripts/run_validation_case.sh b/scripts/run_validation_case.sh index 87f3674..5efe6db 100755 --- a/scripts/run_validation_case.sh +++ b/scripts/run_validation_case.sh @@ -13,6 +13,8 @@ TESTING_COMMIT="$3" ARTIFACTS_DIR="$4" case "${CASE_ID}" in + scanner.usage-[0-9][0-9]-*) + SUITE=scripts/validation_scanner_usage/run_case.sh; EXTRA_ARGS=() ;; pool.*) SUITE=scripts/validation_pool/run_case.sh; EXTRA_ARGS=() ;; bucket-replication.*|site-replication.*) @@ -36,8 +38,8 @@ case "${CASE_ID}" in exit 2 ;; esac -if [[ "${CASE_ID}" == pool.* || "${CASE_ID}" == bucket-replication.* || "${CASE_ID}" == site-replication.* ]]; then - python3 -B -c 'import json,sys; catalog=json.load(open(sys.argv[1])); valid={case["id"] for module in catalog["modules"] for case in module["cases"]}; sys.exit(0 if sys.argv[2] in valid else 2)' \ +if [[ "${CASE_ID}" == pool.* || "${CASE_ID}" == bucket-replication.* || "${CASE_ID}" == site-replication.* || "${CASE_ID}" == scanner.usage-[0-9][0-9]-* ]]; then + python3 -B -c 'import json,sys; catalog=json.load(open(sys.argv[1])); valid={case["id"] for module in catalog["modules"]+catalog.get("local_modules",[]) for case in module["cases"]}; sys.exit(0 if sys.argv[2] in valid else 2)' \ "${SCRIPT_DIR}/validation-cases.json" "${CASE_ID}" || { printf 'unknown canonical validation case: %s\n' "${CASE_ID}" >&2 exit 2 diff --git a/scripts/select_validation_cases.py b/scripts/select_validation_cases.py index d19c23a..94b5dd1 100644 --- a/scripts/select_validation_cases.py +++ b/scripts/select_validation_cases.py @@ -45,6 +45,11 @@ def select_validation_cases(catalog, *, case="", module="", limit=None, rotate=F selected = sweepable_cases(catalog) one_case = (case or "").strip() module_filter = (module or "").strip() + # Configured-endpoint procedures have their own admission and never enter + # nightly rotation implicitly. Explicit selection can still execute them. + if one_case or module_filter: + selected.extend((item["id"], local["id"]) for local in catalog.get("local_modules", []) + for item in local["cases"]) known = {case_id: module_id for case_id, module_id in selected} if one_case: if one_case not in known: diff --git a/scripts/validation_catalog.py b/scripts/validation_catalog.py index 436760f..916e4b9 100644 --- a/scripts/validation_catalog.py +++ b/scripts/validation_catalog.py @@ -64,6 +64,11 @@ } +def catalog_modules(snapshot: dict) -> list[dict]: + """Include local executors without rewriting the immutable source snapshot.""" + return snapshot["modules"] + snapshot.get("local_modules", []) + + def validate_snapshot(snapshot: dict) -> None: if snapshot.get("schema") != 1: raise ValueError("unsupported validation catalog schema") @@ -100,10 +105,28 @@ def validate_snapshot(snapshot: dict) -> None: raise ValueError(f"invalid reuse case IDs: {case_id}") if dict(statuses) != EXPECTED_STATUS_COUNTS: raise ValueError(f"unexpected case counts: {dict(statuses)}") + local_modules = snapshot.get("local_modules", []) + if not isinstance(local_modules, list): + raise ValueError("local_modules must be a list") + for module in local_modules: + if module.get("origin") != "auto-testing" or not module.get("id") or not isinstance(module.get("cases"), list): + raise ValueError("invalid local validation module") + for case in module["cases"]: + case_id = case.get("id") + if not isinstance(case_id, str) or not case_id or case_id in ids: + raise ValueError(f"duplicate or invalid local case ID: {case_id}") + ids.add(case_id) + if case.get("source_status") not in EXPECTED_STATUS_COUNTS or not case.get("assertions"): + raise ValueError(f"invalid local assertion contract: {case_id}") + if case.get("probes", {}).get("executionMode") != "configured-endpoint": + raise ValueError(f"local executor must declare explicit endpoint admission: {case_id}") + candidates = case.get("reuse_candidates") + if not isinstance(candidates, dict) or not isinstance(candidates.get("suites"), list) or not isinstance(candidates.get("case_ids"), list): + raise ValueError(f"missing local reuse assessment: {case_id}") def validate_reuse_candidates(snapshot: dict, checkout: Path) -> None: - for module in snapshot["modules"]: + for module in catalog_modules(snapshot): for case in module["cases"]: candidates = case["reuse_candidates"] suites = candidates["suites"] @@ -179,6 +202,11 @@ def main() -> int: if args.source is None: parser.error("sync requires --source") snapshot = build_snapshot(args.source) + if args.catalog.is_file(): + local_modules = json.loads(args.catalog.read_text()).get("local_modules") + if local_modules is not None: + snapshot["local_modules"] = local_modules + validate_snapshot(snapshot) args.catalog.write_text(json.dumps(snapshot, ensure_ascii=False, indent=2) + "\n") print(f"wrote {args.catalog}") return 0 @@ -196,7 +224,8 @@ def main() -> int: print(f"{module['id']}: {len(module['cases'])} {dict(statuses)}") else: source_note = "source hash matched" if args.source is not None else "source checkout not supplied" - print(f"validation catalog: 158 unique cases, statuses 66/84/8, {source_note}") + local_count = sum(len(module["cases"]) for module in snapshot.get("local_modules", [])) + print(f"validation catalog: 158 source cases, statuses 66/84/8, {local_count} local cases, {source_note}") return 0 diff --git a/scripts/validation_report.py b/scripts/validation_report.py index 810417b..1359e55 100644 --- a/scripts/validation_report.py +++ b/scripts/validation_report.py @@ -11,9 +11,9 @@ import sys try: - from .validation_catalog import validate_snapshot + from .validation_catalog import catalog_modules, validate_snapshot except ImportError: - from validation_catalog import validate_snapshot + from validation_catalog import catalog_modules, validate_snapshot TERMINAL = {"PASS", "FAIL", "INFRA_ERROR", "BLOCKED", "UNSUPPORTED"} @@ -129,7 +129,7 @@ def validate_results( validate_snapshot(catalog) cases = { case["id"]: (case, module["target_environments"]) - for module in catalog["modules"] for case in module["cases"] + for module in catalog_modules(catalog) for case in module["cases"] } gate_cases = gate_cases or set() unknown = gate_cases - set(cases) diff --git a/scripts/validation_result.py b/scripts/validation_result.py index 0cdfe29..847f1f8 100644 --- a/scripts/validation_result.py +++ b/scripts/validation_result.py @@ -11,7 +11,7 @@ import sys import tempfile -from validation_catalog import validate_snapshot +from validation_catalog import catalog_modules, validate_snapshot def write_result(path: Path, value: dict) -> None: @@ -25,7 +25,7 @@ def write_result(path: Path, value: dict) -> None: def case_from_catalog(catalog: dict, case_id: str) -> dict: validate_snapshot(catalog) - for module in catalog["modules"]: + for module in catalog_modules(catalog): for case in module["cases"]: if case["id"] == case_id: return case diff --git a/scripts/validation_scanner_usage/__init__.py b/scripts/validation_scanner_usage/__init__.py new file mode 100644 index 0000000..9dafec4 --- /dev/null +++ b/scripts/validation_scanner_usage/__init__.py @@ -0,0 +1 @@ +"""Configured-endpoint scanner usage regressions and native evidence adapters.""" diff --git a/scripts/validation_scanner_usage/client.py b/scripts/validation_scanner_usage/client.py new file mode 100644 index 0000000..c3fff11 --- /dev/null +++ b/scripts/validation_scanner_usage/client.py @@ -0,0 +1,266 @@ +"""Bounded S3 CLI and signed Admin GET requests; never deploy or restart a node.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import hashlib +import hmac +import json +import os +import re +from pathlib import Path +import signal +import subprocess +import time +from urllib import error, parse, request + + +class ProbeError(RuntimeError): + def __init__(self, message, verdict="INFRA_ERROR", cleanup_status=None): + super().__init__(message) + self.verdict = verdict + self.cleanup_status = cleanup_status + + +def utc_now(): + return datetime.now(timezone.utc).isoformat() + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def run_command(command, timeout, env=None): + """Kill the process group on timeout, including a native test's children.""" + started = time.monotonic() + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, env=env, start_new_session=True) + timed_out = False + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + os.killpg(process.pid, signal.SIGKILL) + stdout, stderr = process.communicate() + except BaseException: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.communicate() + raise + return {"command": [str(part) for part in command], "exit_code": process.returncode, + "timed_out": timed_out, "duration_seconds": time.monotonic() - started, + "stdout": stdout, "stderr": stderr} + + +class Evidence: + def __init__(self, root): + self.root = Path(root) + self.root.mkdir(parents=True, exist_ok=True) + self.sequence = 0 + + def save(self, stem, value): + self.sequence += 1 + name = f"{self.sequence:05d}-{stem}.json" + (self.root / name).write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + return name + + +def endpoint_url(value): + message = "endpoint must be an HTTP(S) origin without credentials, query or path" + try: + if not isinstance(value, str) or any(character.isspace() or ord(character) < 32 for character in value): + raise ValueError(message) + url = parse.urlsplit(value) + if (url.scheme not in {"http", "https"} or not url.hostname + or url.username is not None or url.password is not None + or url.query or url.fragment or url.path not in {"", "/"}): + raise ValueError(message) + # Force port validation here; urllib otherwise defers it until I/O. + _ = url.port + except ValueError: + # Parser errors can echo a malformed port or authority containing a + # secret. Only this fixed diagnostic may reach logs and artifacts. + raise ValueError(message) from None + return parse.urlunsplit((url.scheme, url.netloc, "", "", "")) + + +class NoSignedRedirects(request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + # A signature authorizes only the admitted origin and request path. + # Preserve every 3xx as HTTPError for the ordinary failure evidence. + return None + + +class Client: + def __init__(self, endpoints, admin_endpoints, evidence, *, aws_bin="aws", timeout=30, + region="us-east-1", environment=None): + self.endpoints = [endpoint_url(value) for value in endpoints] + self.admin_endpoints = [endpoint_url(value) for value in admin_endpoints] or self.endpoints + self.evidence = evidence + self.aws_bin = aws_bin + self.timeout = timeout + self.region = region + self.environment = dict(os.environ if environment is None else environment) + self.access = self.environment.get("RUSTFS_ACCESS_KEY") or self.environment.get("AWS_ACCESS_KEY_ID") + self.secret = self.environment.get("RUSTFS_SECRET_KEY") or self.environment.get("AWS_SECRET_ACCESS_KEY") + self.session = self.environment.get("AWS_SESSION_TOKEN") + if not self.access or not self.secret: + raise ProbeError("RUSTFS_ACCESS_KEY/RUSTFS_SECRET_KEY or AWS credentials are required", "BLOCKED") + for name in ("AWS_PROFILE", "AWS_DEFAULT_PROFILE", "AWS_SECURITY_TOKEN"): + self.environment.pop(name, None) + for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + self.environment.pop(name, None) + self.environment.update(AWS_ACCESS_KEY_ID=self.access, AWS_SECRET_ACCESS_KEY=self.secret, + AWS_DEFAULT_REGION=region, AWS_EC2_METADATA_DISABLED="true", + AWS_CONFIG_FILE=os.devnull, AWS_SHARED_CREDENTIALS_FILE=os.devnull, + AWS_MAX_ATTEMPTS="1", AWS_PAGER="", NO_PROXY="*", no_proxy="*") + self.http = request.build_opener(request.ProxyHandler({}), NoSignedRedirects()) + self.last_evidence = None + + def _redact(self, text): + for value in (self.access, self.secret, self.session): + if value: + text = text.replace(value, "[REDACTED]") + return text + + def s3(self, operation, *arguments, endpoint_index=0): + command = [self.aws_bin, "--endpoint-url", self.endpoints[endpoint_index], "--no-cli-pager", + "--region", self.region, "--cli-connect-timeout", str(max(1, int(self.timeout))), + "--cli-read-timeout", str(max(1, int(self.timeout))), "s3api", operation, *map(str, arguments)] + result = run_command(command, self.timeout, self.environment) + result["collected_at"] = utc_now() + result["stdout"] = self._redact(result["stdout"]) + result["stderr"] = self._redact(result["stderr"]) + failure_code = re.search(r"An error occurred \(([^)]+)\)", result["stderr"]) + result["service_error_code"] = failure_code.group(1) if failure_code else None + self.last_evidence = self.evidence.save("s3-" + operation, result) + if result["timed_out"]: + raise ProbeError(f"S3 {operation} exceeded the request deadline; inspect {self.last_evidence}") + if result["exit_code"]: + # The response is retained so a service 503 is not relabelled as a + # fixture success. Request transport failures remain distinct. + product = "An error occurred (" in result["stderr"] + raise ProbeError(f"S3 {operation} failed; inspect {self.last_evidence}", "FAIL" if product else "INFRA_ERROR") + try: + value = json.loads(result["stdout"] or "{}") + except ValueError as exc: + raise ProbeError(f"S3 {operation} returned invalid JSON") from exc + if not isinstance(value, dict): + raise ProbeError(f"S3 {operation} returned a non-object response") + return value + + def admin(self, path="/rustfs/admin/v3/datausageinfo", endpoint_index=0): + url = self.admin_endpoints[endpoint_index] + path + now = datetime.now(timezone.utc) + stamp, day = now.strftime("%Y%m%dT%H%M%SZ"), now.strftime("%Y%m%d") + host = parse.urlsplit(url).netloc + payload = hashlib.sha256(b"").hexdigest() + headers = {"host": host, "x-amz-content-sha256": payload, "x-amz-date": stamp} + if self.session: + headers["x-amz-security-token"] = self.session + signed = ";".join(sorted(headers)) + canonical = "GET\n" + path + "\n\n" + "".join(f"{key}:{headers[key]}\n" for key in sorted(headers)) + canonical += "\n" + signed + "\n" + payload + scope = f"{day}/{self.region}/s3/aws4_request" + message = "AWS4-HMAC-SHA256\n" + stamp + "\n" + scope + "\n" + hashlib.sha256(canonical.encode()).hexdigest() + key = ("AWS4" + self.secret).encode() + for part in (day, self.region, "s3", "aws4_request"): + key = hmac.new(key, part.encode(), hashlib.sha256).digest() + signature = hmac.new(key, message.encode(), hashlib.sha256).hexdigest() + headers["Authorization"] = f"AWS4-HMAC-SHA256 Credential={self.access}/{scope}, SignedHeaders={signed}, Signature={signature}" + started = time.monotonic() + try: + try: + response = self.http.open(request.Request(url, headers=headers), timeout=self.timeout) + except error.HTTPError as exc: + response = exc + with response: + status = response.code + raw = response.read(8 * 1024 * 1024 + 1) + request_id = response.headers.get("x-amz-request-id") + except OSError as exc: + self.last_evidence = self.evidence.save("admin-error", { + "url": url, "collected_at": utc_now(), "duration_seconds": time.monotonic() - started, + "error": self._redact(str(exc))}) + raise ProbeError("Admin request failed or timed out; inspect " + self.last_evidence) from exc + record = {"url": url, "status": status, "request_id": request_id, + "collected_at": utc_now(), "duration_seconds": time.monotonic() - started} + if len(raw) > 8 * 1024 * 1024: + record["error"] = "response exceeded the 8 MiB evidence limit" + self.last_evidence = self.evidence.save("admin", record) + raise ProbeError(record["error"]) + try: + value = json.loads(raw) + except ValueError: + value = None + record["response"] = json.loads(self._redact(json.dumps(value))) if isinstance(value, dict) else self._redact(raw.decode(errors="replace")) + self.last_evidence = self.evidence.save("admin", record) + if status in (401, 403): + raise ProbeError("Admin credentials lack the required read permissions", "BLOCKED") + if status in (404, 405): + raise ProbeError("Candidate does not expose the required Admin endpoint", "UNSUPPORTED") + if status != 200: + raise ProbeError(f"Admin returned HTTP {status}; inspect {self.last_evidence}", "FAIL") + if not isinstance(value, dict): + raise ProbeError("Admin returned invalid JSON") + return value + + +def integer(value): + return type(value) is int and value >= 0 + + +def bucket_view(snapshot, bucket): + """Missing fields remain None. Never turn unknown into an empty bucket.""" + mapping = snapshot.get("buckets_usage") + if not isinstance(mapping, dict): + raise ProbeError("Admin response has no buckets_usage object") + entry = mapping.get(bucket) + if entry is None: + return {"present": False, "state": "unknown", "count": None, "size": None, + "versions": None, "markers": None, "certificate": None, "converged": False} + if not isinstance(entry, dict): + raise ProbeError("Bucket usage is not an object") + state = entry.get("usage_value_state") + if state is not None and not isinstance(state, str): + raise ProbeError("Bucket usage_value_state must be a string or null", "FAIL") + known = state in {"certified", "carried", "observed"} + certificate = entry.get("usage_certification") + if known and certificate is not None: + if (not isinstance(certificate, dict) or not integer(certificate.get("epoch")) + or not integer(certificate.get("cycle"))): + raise ProbeError("Malformed per-bucket certification", "FAIL") + clock = certificate.get("certified_at") + if (not isinstance(clock, dict) or not integer(clock.get("secs_since_epoch")) + or not integer(clock.get("nanos_since_epoch")) or clock["nanos_since_epoch"] >= 10**9): + raise ProbeError("Malformed per-bucket certification time", "FAIL") + if state == "observed": + raise ProbeError("Observed value incorrectly carries a certificate", "FAIL") + if state == "certified" and certificate is None: + raise ProbeError("Certified value has no per-bucket certificate", "FAIL") + fields = {"count": "objects_count", "size": "size", "versions": "versions_count", "markers": "delete_markers_count"} + values = {name: entry.get(field) for name, field in fields.items()} + if any(value is not None and not integer(value) for value in values.values()): + raise ProbeError("Bucket usage contains a negative or non-integer counter", "FAIL") + pending = snapshot.get("usage_snapshot_pending_buckets", []) + if not isinstance(pending, list) or any(not isinstance(item, str) for item in pending): + raise ProbeError("Malformed usage_snapshot_pending_buckets") + # Legacy fields must affirm completeness and convergence; absent booleans + # are not a positive signal. New carried/observed metadata always vetoes it. + converged = (snapshot.get("usage_snapshot_complete") is True + and snapshot.get("usage_snapshot_converged") is True + and snapshot.get("usage_snapshot_partial") is not True + and snapshot.get("usage_snapshot_observational") is not True + and snapshot.get("usage_snapshot_admin_projection") is not True + and bucket not in pending and state not in {"carried", "observed"} + and (state is None or state == "certified")) + return {"present": True, "state": state if known else "unknown", **values, + "certificate": certificate if known else None, "converged": converged, + "incarnation": entry.get("bucket_incarnation"), "metadata_available": known, + "pending": bucket in pending, "observational": snapshot.get("usage_snapshot_observational") is True, + "partial": snapshot.get("usage_snapshot_partial") is True, + "complete": snapshot.get("usage_snapshot_complete") is True} diff --git a/scripts/validation_scanner_usage/contracts.py b/scripts/validation_scanner_usage/contracts.py new file mode 100644 index 0000000..68d35d4 --- /dev/null +++ b/scripts/validation_scanner_usage/contracts.py @@ -0,0 +1,139 @@ +"""Keep the twenty design acceptance rows distinct from black-box probes.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Case: + number: int + slug: str + title: str + scenario: str | None + assertions: tuple[tuple[str, str], ...] + + @property + def id(self): + return f"scanner.usage-{self.number:02d}-{self.slug}" + + @property + def design_id(self): + return f"USAGE-{self.number:02d}" + + +# Each assertion's first item is its required evidence source. Native assertions +# require an exact candidate test and emitted assertion evidence, not a shell exit. +CASES = ( + Case(1, "static-accounting", "Static creation, writes, deletion and empty buckets", "static", ( + ("public", "Committed objects and an independent paginated S3 inventory have identical keys and bytes."), + ("public", "Usage converges to exact object counts and logical bytes after writes and deletes."), + ("public", "A measured empty bucket is distinct from an absent or unknown bucket value."))), + Case(2, "unrelated-remote-mutations", "Long sweep with unrelated remote mutations", None, ( + ("native", "A long bucket makes durable progress across at least three cycles while another node mutates an unrelated bucket."), + ("native", "The complete producer watch proof prevents unrelated writes from invalidating that bucket's certification."))), + Case(3, "cross-cycle-mutations", "Mutations of an already scanned range", "mutations", ( + ("public", "Acknowledged insert, overwrite and delete operations converge to an independent final S3 inventory."), + ("native", "A mutation occurs after the affected range was scanned and before a later cycle whose start and post generations agree."), + ("native", "The mixed result is not certified or acknowledged, and a later fresh sweep is exact."))), + Case(4, "hot-prefix-progress", "Hot and cold ranges in one bucket", "mutations", ( + ("public", "Nested and prefix-shaped keys retain exact final contents after controlled mutations stop."), + ("native", "The cold range obtains durable traversal progress while a hot prefix is continuously modified."), + ("native", "A whole-bucket observation never claims independent prefix certification or prefix ACK."))), + Case(5, "shared-budget-fairness", "One large bucket and three cold buckets", "multi-bucket", ( + ("public", "Three cold buckets converge exactly while requests continue against a separate hot bucket."), + ("native", "A real shared cycle budget is exhausted without a bucket quantum cancelling its parent."), + ("native", "All cold buckets receive service and a zero-progress turn cannot busy-loop."))), + Case(6, "wide-deep-directories", "Wide and deep directory progress", None, ( + ("native", "A prefix containing at least 100000 entries and nested wide directories advances its durable frontier."), + ("native", "Checkpoint memory and bytes remain bounded across rounds without sacrificing exact counts."))), + Case(7, "multi-source-coverage", "Coverage across pools and sets", "multi-endpoint", ( + ("public", "Acknowledged writes through every configured endpoint remain readable and converge to the same exact bucket usage."), + ("native", "The fixture has more physical sources than the set concurrency limit and every expected source contributes."), + ("native", "A missing or non-fresh source prevents complete totals and whole-bucket ACK."))), + Case(8, "no-baseline", "Cold startup and a new bucket without LKG", "no-baseline", ( + ("public", "An absent bucket value stays unknown in the test oracle until a real measured value is available."), + ("native", "A cold-start partial or mixed bucket has no legal LKG and cannot enter the legacy primary map or quota fallback."))), + Case(9, "value-freshness", "Carried values and complete mixed observations", "freshness", ( + ("public", "Returned carried or observed values obey the per-bucket certification contract without using last_update as a certificate."), + ("native", "Multiple PartialCarried and mixed rounds preserve the original value certificate or observation completion time."), + ("native", "A mixture of current and old source values does not inherit an unrelated whole-bucket certificate."))), + Case(10, "checkpoint-recovery", "Checkpoint and mixed EOF crash recovery", None, ( + ("native", "Crashes before and after mixed EOF CAS keep the observation, cleared cursor and fresh-sweep debt atomic."), + ("native", "A successor rejects late old-leader writes and never clears responsibility after losing an observation."))), + Case(11, "bucket-recreation", "Same-name bucket recreation", "recreation", ( + ("public", "Deleting and recreating the suite-owned bucket converges to the successor's exact keys and bytes."), + ("native", "An old incarnation cannot contribute to carry, overlay, certification or ACK after recreation."))), + Case(12, "watch-proof", "Watch capability, gaps, replay and ABA", None, ( + ("native", "A missing producer watch proof stays unknown and retains the strict mutation fallback."), + ("native", "A peer without watch capability cannot authorize cross-cycle reuse."), + ("native", "A mutation racing watch installation cannot establish a falsely clean interval."), + ("native", "Watch capacity overflow invalidates proof instead of retaining a clean revision."), + ("native", "An expired watch cannot prove an unchanged bucket."), + ("native", "A producer restart invalidates watches from its previous incarnation."), + ("native", "A stale or replayed watch response cannot authorize the current request."), + ("native", "An ACK ABA race cannot resurrect an old clean watch or clear newer work."))), + Case(13, "publication-failures", "Lease, CAS and deadline publication failures", None, ( + ("native", "Object and directory count stops share one drain deadline that does not restart after a later budget or lease phase."), + ("native", "A bucket quantum's remote request duration and work limits never exceed its parent's remaining limits."), + ("native", "A remote completion received during the bounded drain retains the owning cycle and reaches the publication decision."), + ("native", "Losing the lease or epoch during drain rejects publication and cannot restart the drain deadline."), + ("native", "A failed canonical root CAS cannot authorize ACK."), + ("native", "A failed companion persistence operation cannot authorize ACK or clear pending responsibility."), + ("native", "A sealed publication candidate cannot be changed by late collector results."), + ("native", "A successful durable canonical root for the still-owned epoch permits only its all-source-fresh bucket ACKs."))), + Case(14, "ack-races", "Duplicate, lost and reordered acknowledgements", None, ( + ("native", "Duplicate, lost or reordered ACK cannot clear a newer generation or a post-walk mutation."), + ("native", "Only a durable exact canonical root permits all-source-fresh bucket ACK; pending never falls back to a whole-generation ACK."))), + Case(15, "structural-changes", "Movement, topology and tier changes", None, ( + ("native", "Movement, expansion, decommission and tier-generation changes reject obsolete source and structure identities."), + ("native", "Structural changes cannot bypass publication fences or permit stale overlay and certification."))), + Case(16, "mixed-version", "Mixed readers, workers, peers and rollback", None, ( + ("native", "An old reader cannot treat a new mixed observation as an LKG primary value."), + ("native", "Terminal, progress and watch capabilities negotiate independently and rolling rollback preserves system data and responsibility."))), + Case(17, "load-and-convergence", "Sixty-minute workload and stop-write convergence", "load", ( + ("public", "A configured sixty-minute sequential workload retains per-request timings and available service error codes before exact stop-write convergence."), + ("native", "Matched base and candidate load runs retain comparable throughput, latency, observed HTTP 503 counts, per-bucket freshness and binary identities."))), + Case(18, "diagnostic-boundaries", "Metrics and admin projection boundaries", "diagnostics", ( + ("public", "Admin samples retain collection times separately from returned per-bucket certification times and reject malformed metadata."), + ("native", "Collector delay or failure cannot invent a per-bucket measurement or certification time."), + ("native", "The admin-only companion overlay cannot flow into quota, cache or durable writer inputs."))), + Case(19, "quota-compatibility", "Quota and legacy snapshot compatibility", None, ( + ("native", "No-quota, durable quota v1/v2 and legacy snapshot paths retain their original admission decisions."), + ("native", "Companion observations cannot expand legacy quota fallback or durable-ledger admission."))), + Case(20, "versions-and-prefix-keys", "Object/prefix collisions, versions and markers", "versions", ( + ("public", "An object key that is also a prefix and nested encoding fixtures retain every acknowledged version and payload."), + ("public", "Usage matches the independent version inventory's object, version, marker and logical-byte accounting exactly."))), +) + + +def assertion_id(case: Case, index: int) -> str: + return f"{case.design_id}-A{index + 1:02d}" + + +def select_cases(value: str) -> list[Case]: + if value == "all": + return list(CASES) + if value == "blackbox": + return [case for case in CASES if all(kind == "public" for kind, _ in case.assertions)] + names = value.split(",") + selected = [case for case in CASES if case.id in names or case.design_id in names] + if not selected or len(selected) != len(names): + raise ValueError("unknown or duplicate scanner usage case selection") + return selected + + +def catalog_module() -> dict: + return {"id": "scanner-usage-regressions", "origin": "auto-testing", + "target_environments": ["configured-endpoint", "native-test"], + "required_capabilities": ["explicit-endpoint", "candidate-provenance"], + "cases": [{"id": case.id, "source_status": "procedure-ready", "priority": "P1", + "channels": ["admin-api", "s3-api", "native-test"], + "assertions": [text for _, text in case.assertions], + "probes": {"designCode": case.design_id, + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": case.scenario}, + "reuse_candidates": {"suites": ["rustfs-scanner-usage-test.sh"], "case_ids": []}} + for case in CASES]} diff --git a/scripts/validation_scanner_usage/native.py b/scripts/validation_scanner_usage/native.py new file mode 100644 index 0000000..349c701 --- /dev/null +++ b/scripts/validation_scanner_usage/native.py @@ -0,0 +1,143 @@ +"""Run exact candidate-native regressions and validate assertion milestones.""" + +from __future__ import annotations + +import json +from pathlib import Path +import re +import time + +from .client import ProbeError, run_command, sha256 +from .contracts import assertion_id + + +def run_native(case, manifest, source_commit, evidence, timeout, required_ids): + """Return native assertion records; no manifest means a capability gap.""" + if manifest is None: + raise ProbeError("Candidate-native assertion adapter was not supplied; no deterministic hook evidence is available", "BLOCKED") + if not isinstance(manifest, dict) or manifest.get("schema_version") != 1 or manifest.get("source_commit") != source_commit: + raise ProbeError("Native manifest schema or candidate source identity does not match") + cases = manifest.get("cases", {}) + if not isinstance(cases, dict): + raise ProbeError("Native manifest cases must be an object") + entry = cases.get(case.design_id) + if entry is None: + raise ProbeError("Native manifest does not provide this case's capability", "UNSUPPORTED") + specs = entry.get("tests") if isinstance(entry, dict) else None + if not isinstance(specs, list) or not specs: + raise ProbeError("Native case manifest requires a nonempty tests list") + claimed = set() + for spec in specs: + ids = spec.get("assertions") if isinstance(spec, dict) else None + if not isinstance(ids, list) or not all(isinstance(item, str) for item in ids): + raise ProbeError("Native test assertions must be a list of IDs") + if claimed.intersection(ids): + raise ProbeError("Each native assertion needs one complete test owner; duplicate coverage is ambiguous") + claimed.update(ids) + declared = set() + records = {} + deadline = time.monotonic() + timeout + for index, spec in enumerate(specs): + remaining = deadline - time.monotonic() + if remaining <= 0: + error = ProbeError("Native case exhausted its total execution deadline") + error.native_records = {identifier: row for identifier, row in records.items() if identifier in required_ids} + raise error + try: + current = _run_native_test(case, spec, evidence, remaining) + except (ProbeError, OSError, ValueError) as exc: + failure = exc if isinstance(exc, ProbeError) else ProbeError(f"Native test could not run: {exc}") + failure.native_records = {identifier: row for identifier, row in records.items() if identifier in required_ids} + failure.native_not_run = {identifier for pending in specs[index + 1:] for identifier in pending["assertions"]} + if failure is exc: + raise + raise failure from exc + declared.update(current) + for identifier, row in current.items(): + records[identifier] = {**row, "native_evidence": [row["native_evidence"]]} + if not set(required_ids) <= declared: + error = ProbeError("Native manifest does not cover every required assertion", "UNSUPPORTED") + error.native_records = {identifier: row for identifier, row in records.items() if identifier in required_ids} + raise error + return {identifier: row for identifier, row in records.items() if identifier in required_ids} + + +def _run_native_test(case, spec, evidence, timeout): + started = time.monotonic() + if not isinstance(spec, dict): + raise ProbeError("Native test specification must be an object") + if not isinstance(spec.get("binary"), str): + raise ProbeError("Native executable path must be a string") + binary = Path(spec["binary"]) + if (not binary.is_absolute() or not binary.is_file() or binary.is_symlink() + or sha256(binary) != spec.get("binary_sha256")): + raise ProbeError("Native executable is missing or its SHA-256 does not match") + name = spec.get("test") + if not isinstance(name, str) or not re.fullmatch(r"[A-Za-z0-9_:]+", name): + raise ProbeError("Native manifest must name one exact libtest test") + known = {assertion_id(case, index) for index in range(len(case.assertions))} + declared = spec.get("assertions") + if (not isinstance(declared, list) or not all(isinstance(item, str) for item in declared) or len(set(declared)) != len(declared) + or not declared or not set(declared) <= known): + raise ProbeError("Native manifest declares empty or unknown assertion IDs") + inventory = run_command([str(binary), name, "--exact", "--list"], min(timeout, 30)) + inventory_path = evidence.save("native-inventory", inventory) + tests = [line[:-6] for line in inventory["stdout"].splitlines() if line.endswith(": test")] + if inventory["timed_out"] or inventory["exit_code"]: + raise ProbeError("Native inventory command failed or timed out") + if tests != [name]: + raise ProbeError("Candidate binary does not expose the exact requested native test", "UNSUPPORTED") + remaining = timeout - (time.monotonic() - started) + if remaining <= 0: + raise ProbeError("Native inventory exhausted the execution deadline") + try: + return _execute_native_test(case, binary, name, declared, evidence, remaining, inventory_path) + except (ProbeError, OSError, ValueError) as exc: + failure = exc if isinstance(exc, ProbeError) else ProbeError(f"Native execution was interrupted: {exc}") + failure.cleanup_status = "FAIL" + if failure is exc: + raise + raise failure from exc + + +def _execute_native_test(case, binary, name, declared, evidence, remaining, inventory_path): + """Cleanup stays unverified until both execution and its milestones settle.""" + result = run_command([str(binary), name, "--exact", "--show-output", "--test-threads=1"], remaining) + execution_path = evidence.save("native-execution", result) + if result["timed_out"]: + raise ProbeError("Native regression exceeded its deadline", cleanup_status="FAIL") + if result["exit_code"]: + raise ProbeError("Native regression failed; inspect " + execution_path, "FAIL", cleanup_status="FAIL") + output = result["stdout"] + if (re.findall(r"(?m)^running .*", output) != ["running 1 test"] + or re.findall(r"(?m)^test (?!result:).*", output) != [f"test {name} ... ok"] + or len(re.findall(r"(?m)^test result:", output)) != 1 + or not re.search(r"(?m)^test result: ok\. 1 passed; 0 failed; 0 ignored;", output)): + raise ProbeError("Native output does not prove one real passing test", cleanup_status="FAIL") + records = {} + cleanups = [] + for line in output.splitlines(): + prefix = next((prefix for prefix in ("usage_assertion: ", "usage_cleanup: ") if line.startswith(prefix)), None) + if prefix is None: + continue + try: + item = json.loads(line[len(prefix):]) + except ValueError as exc: + raise ProbeError("Malformed native assertion milestone", cleanup_status="FAIL") from exc + if not isinstance(item, dict) or item.get("case_id") != case.design_id: + raise ProbeError("Native milestone belongs to a different case", cleanup_status="FAIL") + if prefix == "usage_cleanup: ": + cleanups.append(item) + continue + identifier = item.get("assertion_id") + if identifier not in declared or identifier in records: + raise ProbeError("Native assertion milestone is unknown or repeated", cleanup_status="FAIL") + if item.get("status") != "PASS" or not isinstance(item.get("evidence"), dict) or not item["evidence"]: + raise ProbeError("Native assertion lacks positive structured evidence", cleanup_status="FAIL") + records[identifier] = {"status": "PASS", "evidence": [inventory_path, execution_path], + "native_evidence": item["evidence"]} + if set(records) != set(declared): + raise ProbeError("Native test passed but omitted required assertion evidence", cleanup_status="FAIL") + if len(cleanups) != 1 or cleanups[0].get("status") != "PASS": + raise ProbeError("Native fixture cleanup was not confirmed", cleanup_status="FAIL") + return records diff --git a/scripts/validation_scanner_usage/run_case.sh b/scripts/validation_scanner_usage/run_case.sh new file mode 100755 index 0000000..016a7ec --- /dev/null +++ b/scripts/validation_scanner_usage/run_case.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +exec bash "${ROOT}/rustfs-scanner-usage-test.sh" "$@" diff --git a/scripts/validation_scanner_usage/runner.py b/scripts/validation_scanner_usage/runner.py new file mode 100644 index 0000000..2822780 --- /dev/null +++ b/scripts/validation_scanner_usage/runner.py @@ -0,0 +1,583 @@ +"""Run usage probes on explicitly configured endpoints and retain every verdict.""" + +from __future__ import annotations + +import argparse +from collections import Counter +import hashlib +import json +import math +import os +from pathlib import Path +import secrets +import shutil +import signal +import sys +import time + +from .client import Client, Evidence, ProbeError, bucket_view, endpoint_url, integer, run_command, sha256, utc_now +from .contracts import CASES, assertion_id, select_cases +from .native import run_native + +ROOT = Path(__file__).resolve().parents[2] + + +def require(condition, message, verdict="FAIL"): + if not condition: + raise ProbeError(message, verdict) + + +class Fixture: + """Only buckets successfully created by this invocation are mutable.""" + + def __init__(self, client, directory, evidence): + self.client, self.directory, self.evidence = client, Path(directory), evidence + self.owned = set() + self.unconfirmed_creates = set() + self.objects = {} + self.versions = {} + self.request_count = 0 + self.journal_path = self.directory / "acknowledged-requests.jsonl" + self.prefix = "usage-" + secrets.token_hex(10) + + def persist_ownership(self): + (self.directory / "owned-buckets.json").write_text(json.dumps({ + "prefix": self.prefix, "buckets": sorted(self.owned), + "unconfirmed_creates": sorted(self.unconfirmed_creates), + "request_journal": self.journal_path.name}, indent=2) + "\n") + + def record_acknowledgement(self, entry): + with self.journal_path.open("a", encoding="utf-8") as stream: + stream.write(json.dumps({"sequence": self.request_count + 1, **entry}) + "\n") + self.request_count += 1 + + def create(self, suffix, *, name=None, versioned=False): + bucket = name or self.prefix + "-" + suffix + require(bucket.startswith(self.prefix + "-"), "refusing a foreign bucket name", "INFRA_ERROR") + # Record the intent before the request: a lost response cannot prove + # either ownership or absence, so cleanup must expose that uncertainty. + self.unconfirmed_creates.add(bucket) + self.persist_ownership() + self.client.s3("create-bucket", "--bucket", bucket) + self.owned.add(bucket) + self.unconfirmed_creates.remove(bucket) + self.objects[bucket] = {} + self.versions[bucket] = [] if versioned else None + self.persist_ownership() + if versioned: + self.client.s3("put-bucket-versioning", "--bucket", bucket, + "--versioning-configuration", '{"Status":"Enabled"}') + return bucket + + def put(self, bucket, key, size, *, endpoint_index=0): + require(bucket in self.owned, "refusing writes to an unowned bucket", "INFRA_ERROR") + payload = (hashlib.sha256((key + str(self.request_count)).encode()).digest() * ((size + 31) // 32))[:size] + path = self.directory / "request-payload.bin" + path.write_bytes(payload) + response = self.client.s3("put-object", "--bucket", bucket, "--key", key, "--body", path, + endpoint_index=endpoint_index) + entry = {"key": key, "size": size, "sha256": hashlib.sha256(payload).hexdigest(), + "version_id": response.get("VersionId"), "marker": False} + if self.versions[bucket] is not None: + require(bool(entry["version_id"]), "versioned PUT did not return a VersionId") + self.versions[bucket].append(entry) + self.objects[bucket][key] = entry + self.record_acknowledgement({"operation": "put", "bucket": bucket, "endpoint_index": endpoint_index, + "acknowledged_at": utc_now(), **entry}) + return entry + + def delete(self, bucket, key): + require(bucket in self.owned, "refusing deletion from an unowned bucket", "INFRA_ERROR") + response = self.client.s3("delete-object", "--bucket", bucket, "--key", key) + if self.versions[bucket] is not None: + require(response.get("DeleteMarker") is True and bool(response.get("VersionId")), + "versioned DELETE did not create the expected marker") + self.versions[bucket].append({"key": key, "version_id": response["VersionId"], "marker": True, "size": 0}) + self.objects[bucket].pop(key, None) + self.record_acknowledgement({"operation": "delete", "bucket": bucket, "key": key, "response": response, + "acknowledged_at": utc_now()}) + + def inventory(self, bucket, *, versioned=False): + require(bucket in self.owned, "refusing inventory of an unowned bucket", "INFRA_ERROR") + records, continuation, seen = [], [], set() + for _ in range(10000): + operation = "list-object-versions" if versioned else "list-objects-v2" + value = self.client.s3(operation, "--bucket", bucket, "--max-keys", "100", "--no-paginate", *continuation) + if versioned: + for field, marker in (("Versions", False), ("DeleteMarkers", True)): + for row in value.get(field, []): + records.append({"key": row["Key"], "version_id": row["VersionId"], + "size": 0 if marker else row["Size"], "marker": marker}) + else: + records.extend({"key": row["Key"], "size": row["Size"]} for row in value.get("Contents", [])) + if value.get("IsTruncated") is not True: + break + token = ((value.get("NextKeyMarker"), value.get("NextVersionIdMarker")) if versioned + else (value.get("NextContinuationToken"),)) + require(token[0] is not None and token not in seen, "S3 listing repeated or omitted its continuation", "INFRA_ERROR") + seen.add(token) + continuation = (["--key-marker", token[0]] + (["--version-id-marker", token[1]] if token[1] else []) + if versioned else ["--continuation-token", token[0]]) + else: + raise ProbeError("S3 listing exceeded the bounded pagination limit") + identities = [(row["key"], row.get("version_id")) for row in records] + require(len(identities) == len(set(identities)), "S3 inventory duplicated an object/version") + return records + + def oracle(self, bucket, *, verify_payloads=False, endpoint_index=0): + versioned = self.versions[bucket] is not None + actual = self.inventory(bucket, versioned=versioned) + expected = self.versions[bucket] if versioned else list(self.objects[bucket].values()) + fields = ("key", "version_id", "size", "marker") if versioned else ("key", "size") + normalize = lambda rows: sorted(tuple(row[field] for field in fields) for row in rows) + require(normalize(actual) == normalize(expected), "Independent S3 inventory differs from acknowledged operations") + if verify_payloads: + target = self.directory / "readback.bin" + for row in expected: + if row.get("marker"): + continue + arguments = ["--bucket", bucket, "--key", row["key"]] + if row.get("version_id"): + arguments += ["--version-id", row["version_id"]] + self.client.s3("get-object", *arguments, target, endpoint_index=endpoint_index) + require(target.is_file() and sha256(target) == row["sha256"], "GET payload hash differs from the acknowledged fixture") + # RustFS scanner counts a key with retained object versions once even + # when its latest version is a delete marker. Fixtures avoid marker-only + # keys so the oracle does not silently choose a disputed empty-key rule. + values = {"count": len({row["key"] for row in actual}), "size": sum(row["size"] for row in actual)} + if versioned: + values.update(versions=sum(not row["marker"] for row in actual), markers=sum(row["marker"] for row in actual)) + path = self.evidence.save("independent-oracle", {"bucket": bucket, "versioned": versioned, + "inventory": actual, "expected": values, + "payloads_verified": verify_payloads}) + return values, path + + def remove(self, bucket): + require(bucket in self.owned, "refusing cleanup of an unowned bucket", "INFRA_ERROR") + # Always enumerate versions: this also removes markers if a setup + # operation succeeded but its response was lost later in the case. + for row in self.inventory(bucket, versioned=True): + self.client.s3("delete-object", "--bucket", bucket, "--key", row["key"], "--version-id", row["version_id"]) + self.client.s3("delete-bucket", "--bucket", bucket) + self.owned.remove(bucket) + self.persist_ownership() + + def cleanup(self): + failures = [{"bucket": bucket, "error": "CreateBucket did not confirm ownership; remote cleanup is unverified"} + for bucket in sorted(self.unconfirmed_creates)] + for bucket in sorted(self.owned): + try: + self.remove(bucket) + except (OSError, KeyError, ProbeError) as exc: + failures.append({"bucket": bucket, "error": str(exc)}) + path = self.evidence.save("cleanup", {"status": "FAIL" if failures else "PASS", + "failures": failures, "remaining_owned_buckets": sorted(self.owned), + "unconfirmed_creates": sorted(self.unconfirmed_creates)}) + return ("FAIL" if failures else "PASS"), path + + +class PublicProbe: + def __init__(self, client, fixture, evidence, options): + self.client, self.fixture, self.evidence, self.options = client, fixture, evidence, options + self.passed = {} + self.observed = [] + + def check(self, index, facts): + self.passed[index] = self.evidence.save("assertion", facts) + + def sample(self, bucket, endpoint_index=0): + snapshot = self.client.admin(endpoint_index=endpoint_index) + view = bucket_view(snapshot, bucket) + row = {"collected_at": utc_now(), "bucket": bucket, "view": view, + "scanner_cycle": snapshot.get("scanner_cycle"), "scanner_epoch": snapshot.get("scanner_epoch"), + "raw_evidence": self.client.last_evidence} + self.observed.append(row) + return view + + def wait_usage(self, bucket, expected, endpoint_index=0): + deadline = time.monotonic() + self.options.convergence_timeout + samples = 0 + while True: + view = self.sample(bucket, endpoint_index) + samples += 1 + if view["converged"] and all(view[field] == value for field, value in expected.items()): + return self.evidence.save("convergence", {"bucket": bucket, "expected": expected, + "actual": view, "samples": samples}) + if time.monotonic() >= deadline: + self.evidence.save("convergence-timeout", {"bucket": bucket, "expected": expected, + "actual": view, "samples": samples}) + raise ProbeError("Usage did not converge to exact independently listed values within the configured deadline", "FAIL") + time.sleep(min(self.options.poll_interval, max(0, deadline - time.monotonic()))) + + def seed(self, bucket, count=None, *, endpoints=False): + for index in range(count if count is not None else self.options.objects): + self.fixture.put(bucket, f"range-{index % 4}/nested/key-{index:06d}", 1024 + index, + endpoint_index=index % len(self.client.endpoints) if endpoints else 0) + + def static(self): + bucket = self.fixture.create("static") + absent = self.sample(self.fixture.prefix + "-never-created") + require(not absent["present"] and absent["count"] is None and absent["size"] is None, + "An absent bucket was represented as a measured empty bucket") + self.seed(bucket) + values, oracle = self.fixture.oracle(bucket, verify_payloads=True) + self.check(0, {"oracle": oracle, "owned_bucket": bucket}) + writes = self.wait_usage(bucket, values) + for key in list(self.fixture.objects[bucket]): + self.fixture.delete(bucket, key) + empty, empty_oracle = self.fixture.oracle(bucket) + deletes = self.wait_usage(bucket, empty) + self.check(1, {"write_convergence": writes, "delete_convergence": deletes, "empty_oracle": empty_oracle}) + self.check(2, {"absent": absent, "measured_empty": empty, "convergence": deletes}) + + def mutations(self): + bucket = self.fixture.create("mutation") + self.seed(bucket) + values, _ = self.fixture.oracle(bucket) + before = self.wait_usage(bucket, values) + keys = list(self.fixture.objects[bucket]) + self.fixture.put(bucket, keys[0], 8193) + self.fixture.delete(bucket, keys[1]) + self.fixture.put(bucket, "range-0/nested/new-after-baseline", 4099) + values, oracle = self.fixture.oracle(bucket, verify_payloads=True) + after = self.wait_usage(bucket, values) + self.check(0, {"baseline": before, "final_oracle": oracle, "final_convergence": after, + "limits": "Public cycle observations do not prove which internal range was scanned before mutation."}) + + def multi_bucket(self): + hot = self.fixture.create("hot") + cold = [self.fixture.create(f"cold-{index}") for index in range(3)] + self.seed(hot) + targets = {} + for bucket in cold: + self.seed(bucket, 3) + targets[bucket], _ = self.fixture.oracle(bucket) + deadline = time.monotonic() + self.options.convergence_timeout + done = set() + operations = 0 + while time.monotonic() < deadline and len(done) != len(cold): + self.fixture.put(hot, "hot-prefix/mutating", 2048 + operations % 64) + operations += 1 + for bucket in cold: + view = self.sample(bucket) + # A cold bucket can be certified while another bucket remains + # pending. New per-bucket metadata is required for this claim. + if (view["state"] == "certified" and view["certificate"] is not None + and view.get("complete") and not view.get("pending") + and not view.get("partial") and not view.get("observational") + and all(view[field] == value for field, value in targets[bucket].items())): + done.add(bucket) + if len(done) < len(cold): + time.sleep(self.options.poll_interval) + require(len(done) == len(cold), "Cold-bucket certification was not observed during the hot workload", + "UNSUPPORTED" if not any(row["view"].get("metadata_available") for row in self.observed) else "FAIL") + self.check(0, {"cold_buckets": sorted(done), "hot_puts": operations, + "limits": "This public probe does not prove that a cycle budget actually expired."}) + + def multi_endpoint(self): + require(len(self.client.endpoints) >= 2, "Multiple explicit S3 endpoints are required", "BLOCKED") + require(len(self.client.admin_endpoints) >= 2, "Multiple explicit Admin endpoints are required", "BLOCKED") + bucket = self.fixture.create("sources") + self.seed(bucket, max(self.options.objects, len(self.client.endpoints)), endpoints=True) + evidence = [] + for index in range(len(self.client.endpoints)): + values, oracle = self.fixture.oracle(bucket, verify_payloads=True, endpoint_index=index) + evidence.append(oracle) + for index in range(len(self.client.admin_endpoints)): + evidence.append(self.wait_usage(bucket, values, index)) + self.check(0, {"observations": evidence, "limits": "Endpoint count is not physical source/set coverage."}) + + def no_baseline(self): + bucket = self.fixture.prefix + "-bootstrap" + before = self.sample(bucket) + require(not before["present"] and before["count"] is None and before["size"] is None, + "The unique absent fixture bucket did not remain unknown") + self.fixture.create("bootstrap") + self.seed(bucket) + values, oracle = self.fixture.oracle(bucket) + convergence = self.wait_usage(bucket, values) + self.check(0, {"before": before, "oracle": oracle, "convergence": convergence, + "limits": "A new bucket is not proof of cluster cold startup or missing internal LKG."}) + + def freshness(self): + bucket = self.fixture.create("freshness") + self.seed(bucket) + values, _ = self.fixture.oracle(bucket) + self.wait_usage(bucket, values) + baseline = self.sample(bucket) + require(baseline.get("metadata_available"), "Candidate lacks per-bucket value metadata", "UNSUPPORTED") + self.fixture.put(bucket, "range-0/nested/key-000000", 8291) + carried_or_observed = [] + for _ in range(self.options.freshness_samples): + view = self.sample(bucket) + if view["state"] == "carried": + require(view["certificate"] == baseline["certificate"], "Carried value replaced its original certificate") + carried_or_observed.append(view) + if view["state"] == "observed": + require(view["certificate"] is None, "Observed value inherited a certificate") + carried_or_observed.append(view) + time.sleep(self.options.poll_interval) + require(carried_or_observed, "No carried or observed value was exposed during this public sampling window", "BLOCKED") + self.check(0, {"baseline": baseline, "carried_or_observed": carried_or_observed}) + + def recreation(self): + bucket = self.fixture.create("recreated") + self.seed(bucket, 3) + old, _ = self.fixture.oracle(bucket) + self.wait_usage(bucket, old) + self.fixture.remove(bucket) + self.fixture.create("recreated", name=bucket) + self.fixture.put(bucket, "successor-only", 7193) + new, oracle = self.fixture.oracle(bucket, verify_payloads=True) + result = self.wait_usage(bucket, new) + self.check(0, {"old": old, "successor_oracle": oracle, "successor_convergence": result}) + + def versions(self): + bucket = self.fixture.create("versions", versioned=True) + for index, key in enumerate(("a", "a/child", "a/child/deeper", "space key", "unicode-测试", "percent%+key")): + self.fixture.put(bucket, key, 1027 + index) + self.fixture.put(bucket, "a", 4097) + self.fixture.put(bucket, "a/child", 5099) + self.fixture.delete(bucket, "a") + values, oracle = self.fixture.oracle(bucket, verify_payloads=True) + self.check(0, {"oracle": oracle}) + result = self.wait_usage(bucket, values) + self.check(1, {"oracle": oracle, "usage_convergence": result}) + + def diagnostics(self): + bucket = self.fixture.create("diagnostics") + self.seed(bucket, 3) + values, _ = self.fixture.oracle(bucket) + self.wait_usage(bucket, values) + for _ in range(self.options.freshness_samples): + view = self.sample(bucket) + require(view.get("metadata_available"), "Candidate lacks per-bucket value metadata", "UNSUPPORTED") + time.sleep(self.options.poll_interval) + self.check(0, {"samples": self.observed, "collection_clock": "client UTC", + "certification_clock": "unmodified server SystemTime; no last_update synthesis"}) + + def load(self): + require(self.options.load_seconds >= 3600, "USAGE-17 requires an explicit workload of at least 3600 seconds", "BLOCKED") + bucket = self.fixture.create("load") + end = time.monotonic() + self.options.load_seconds + index = 0 + while time.monotonic() < end: + self.fixture.put(bucket, f"bounded-key-{index % 32}", 4096 + index % 257) + if index % 4 == 0: + self.fixture.oracle(bucket, verify_payloads=True) + if index % 7 == 0: + self.fixture.delete(bucket, f"bounded-key-{index % 32}") + self.sample(bucket) + index += 1 + values, oracle = self.fixture.oracle(bucket, verify_payloads=True) + result = self.wait_usage(bucket, values) + self.check(0, {"workload_seconds": self.options.load_seconds, "rounds": index, "oracle": oracle, + "convergence": result, "latency_scope": "AWS CLI end-to-end, including subprocess startup", + "limits": "This bounded sequential probe is not a saturation benchmark or an A/B comparison."}) + + +def execute_case(case, directory, options, provenance, native_manifest=None, client_factory=Client): + directory = Path(directory) + evidence = Evidence(directory / "evidence" / case.id) + fixture = None + result = {"schema_version": 1, "case_id": case.id, "design_id": case.design_id, + "source_status": "procedure-ready", "profile": "native-test" if options.execution == "native" else "configured-endpoint", + "execution_mode": options.execution, "migration_status": "unverified", + "verdict": "RUNNING", "started_at": utc_now(), "cleanup_status": "PENDING", + "provenance": provenance, "assertions": [], "capabilities": {}, "hit_count": 0} + records, errors = {}, [] + try: + if case.scenario and options.execution != "native": + require(options.endpoints, "Explicit --endpoint is required for public probes", "BLOCKED") + client = client_factory(options.endpoints, options.admin_endpoints, evidence, + aws_bin=options.aws_bin, timeout=options.request_timeout, region=options.region) + fixture = Fixture(client, evidence.root, evidence) + probe = PublicProbe(client, fixture, evidence, options) + try: + getattr(probe, case.scenario.replace("-", "_"))() + except (ProbeError, OSError, KeyError, ValueError) as exc: + failure = exc if isinstance(exc, ProbeError) else ProbeError(str(exc)) + errors.append(failure) + attempted = next((index for index, (kind, _) in enumerate(case.assertions) + if kind == "public" and index not in probe.passed), None) + if attempted is not None: + path = evidence.save("public-probe-error", {"reason": str(failure), "verdict": failure.verdict, + "last_request_evidence": client.last_evidence}) + records[assertion_id(case, attempted)] = {"status": failure.verdict, "reason": str(failure), "evidence": [path]} + finally: + samples = evidence.save("usage-samples", probe.observed) + for index, path in probe.passed.items(): + records[assertion_id(case, index)] = {"status": "PASS", "evidence": [path, samples]} + missing = [assertion_id(case, index) for index in range(len(case.assertions)) + if assertion_id(case, index) not in records] + native_needed = ([identifier for index, identifier in enumerate( + [assertion_id(case, i) for i in range(len(case.assertions))]) + if case.assertions[index][0] == "native" and identifier in missing] + if options.execution != "native" else missing) + if native_needed and not getattr(options, "cancelled", False): + try: + records.update(run_native(case, native_manifest, provenance.get("source_commit"), evidence, + options.native_timeout, native_needed)) + result["capabilities"]["native_assertions"] = "executed" + except (ProbeError, OSError, ValueError) as exc: + failure = exc if isinstance(exc, ProbeError) else ProbeError(str(exc)) + errors.append(failure) + records.update(getattr(failure, "native_records", {})) + path = None + try: + path = evidence.save("native-adapter-error", {"reason": str(failure), "verdict": failure.verdict, + "cleanup_status": failure.cleanup_status}) + except (OSError, ValueError) as diagnostic_error: + errors.append(ProbeError(f"Native adapter diagnostic could not be saved: {diagnostic_error}")) + for identifier in native_needed: + if identifier not in records: + not_run = identifier in getattr(failure, "native_not_run", set()) + records[identifier] = {"status": "BLOCKED" if not_run else failure.verdict, + "reason": "Not executed after an earlier native test failed" if not_run else str(failure), + "evidence": [path] if path is not None else []} + result["capabilities"]["native_assertions"] = "unavailable or rejected" + except (ProbeError, OSError, ValueError) as exc: + errors.append(exc if isinstance(exc, ProbeError) else ProbeError(str(exc))) + finally: + if fixture: + cleanup, cleanup_path = fixture.cleanup() + else: + cleanup = "PASS" + cleanup_path = evidence.save("cleanup", {"status": "PASS", "owned_buckets": [], "reason": "no public fixture created"}) + result["cleanup_status"] = cleanup + if any(error.cleanup_status == "FAIL" for error in errors): + result["cleanup_status"] = "FAIL" + if cleanup != "PASS": + errors.append(ProbeError("Fixture cleanup failed; retain owned-buckets.json for scoped recovery")) + for index, (_, text) in enumerate(case.assertions): + identifier = assertion_id(case, index) + item = records.get(identifier, {"status": "BLOCKED", "evidence": [], "reason": "Required assertion was not demonstrated"}) + result["assertions"].append({"assertion_id": identifier, "source_assertion": text, **item}) + if errors: + priorities = {"FAIL": 0, "INFRA_ERROR": 1, "BLOCKED": 2, "UNSUPPORTED": 3} + result["verdict"] = min((error.verdict for error in errors), key=priorities.__getitem__) + result["reason"] = "; ".join(dict.fromkeys(str(error) for error in errors)) + elif len(records) != len(case.assertions): + result["verdict"] = "BLOCKED" + result["reason"] = "Required assertion evidence is missing" + else: + result["verdict"] = "PASS" + result["migration_status"] = "verified" + result["hit_count"] = 1 + prefix = f"evidence/{case.id}/" + for item in result["assertions"]: + item["evidence"] = [prefix + path for path in item["evidence"]] + result["evidence"] = [prefix + path.name for path in sorted(evidence.root.iterdir()) if path.is_file()] + result["cleanup_evidence"] = prefix + cleanup_path + result["finished_at"] = utc_now() + result["limits"] = ["Public observations cannot certify private ACK, checkpoint, source-turn or watch race boundaries."] + (directory / "cases").mkdir(exist_ok=True) + (directory / "cases" / (case.id + ".json")).write_text(json.dumps(result, indent=2) + "\n") + return result + + +def render_report(results): + lines = ["# Scanner usage regression report", "", "PASS requires every listed assertion and successful cleanup.", "", + "| Design case | Verdict | Assertions passed | Cleanup |", "| --- | --- | --- | --- |"] + for result in results: + passed = sum(item["status"] == "PASS" for item in result["assertions"]) + lines.append(f"| {result['design_id']} | {result['verdict']} | {passed}/{len(result['assertions'])} | {result['cleanup_status']} |") + for result in results: + lines += ["", f"## {result['design_id']} — {result['case_id']}", "", result.get("reason", "Every required assertion passed."), "", + f"Started: {result['started_at']} Finished: {result['finished_at']}", "", + "| Assertion | Status | Evidence |", "| --- | --- | --- |"] + for item in result["assertions"]: + links = ", ".join(f"[artifact]({path})" for path in item["evidence"]) + lines.append(f"| {item['assertion_id']}: {item['source_assertion']} | {item['status']} | {links or item.get('reason', '')} |") + lines += ["", f"[Cleanup evidence]({result['cleanup_evidence']})", ""] + return "\n".join(lines) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cases", default="blackbox", help="blackbox, all, USAGE-NN or canonical IDs, comma-separated") + parser.add_argument("--validation-case", help="one canonical validation ID") + parser.add_argument("--endpoint", action="append", default=[]) + parser.add_argument("--admin-endpoint", action="append", default=[]) + parser.add_argument("--artifacts-dir", "--validation-artifacts-dir", dest="artifacts", type=Path, required=True) + parser.add_argument("--candidate-manifest", type=Path, required=True) + parser.add_argument("--testing-commit") + parser.add_argument("--native-manifest", type=Path, default=os.environ.get("RUSTFS_USAGE_NATIVE_MANIFEST")) + parser.add_argument("--execution", choices=("combined", "native"), default="combined") + parser.add_argument("--aws-bin", default="aws") + parser.add_argument("--rc-bin", default="rc", help="client provenance required by the shared validation report") + parser.add_argument("--region", default="us-east-1") + parser.add_argument("--request-timeout", type=float, default=30) + parser.add_argument("--convergence-timeout", type=float, default=600) + parser.add_argument("--native-timeout", type=float, default=900) + parser.add_argument("--poll-interval", type=float, default=5) + parser.add_argument("--objects", type=int, default=32) + parser.add_argument("--freshness-samples", type=int, default=6) + parser.add_argument("--load-seconds", type=float, default=0, help="USAGE-17 requires at least 3600; disabled by default") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("-y", "--yes", action="store_true", help="accepted for the canonical runner; no node changes are performed") + args = parser.parse_args(argv) + args.endpoints = args.endpoint or ([os.environ["RUSTFS_USAGE_ENDPOINT"]] if os.environ.get("RUSTFS_USAGE_ENDPOINT") else []) + args.admin_endpoints = args.admin_endpoint or ([os.environ["RUSTFS_USAGE_ADMIN_ENDPOINT"]] if os.environ.get("RUSTFS_USAGE_ADMIN_ENDPOINT") else []) + try: + args.endpoints = [endpoint_url(value) for value in args.endpoints] + args.admin_endpoints = [endpoint_url(value) for value in args.admin_endpoints] + selected = select_cases(args.validation_case or args.cases) + require(all(math.isfinite(value) and value > 0 for value in (args.request_timeout, args.convergence_timeout, args.native_timeout, + args.poll_interval, args.freshness_samples)), "Timeouts and sample counts must be positive", "INFRA_ERROR") + require(2 <= args.objects <= 100000 and math.isfinite(args.load_seconds) and args.load_seconds >= 0, "Fixture counts or load duration are outside the supported bounds", "INFRA_ERROR") + if args.dry_run: + print(json.dumps({"cases": [case.id for case in selected], "mutations": "suite-owned buckets only", + "node_actions": [], "native_capability_required": [case.id for case in selected + if any(kind == "native" for kind, _ in case.assertions)]}, indent=2)) + return 0 + require(not args.artifacts.exists(), "Artifacts directory must not already exist", "INFRA_ERROR") + candidate = json.loads(args.candidate_manifest.read_text()) + for field, length in (("source_sha", 40), ("package_sha256", 64), ("binary_sha256", 64)): + value = candidate.get(field) + require(isinstance(value, str) and len(value) == length and all(c in "0123456789abcdef" for c in value), + f"Candidate manifest requires {field}", "INFRA_ERROR") + require(bool(candidate.get("package_url")), "Candidate manifest requires package_url", "INFRA_ERROR") + actual_commit = run_command(["git", "-C", str(ROOT), "rev-parse", "HEAD"], 15)["stdout"].strip() + require(not args.testing_commit or args.testing_commit == actual_commit, "Testing commit differs from checkout", "INFRA_ERROR") + rc_path = shutil.which(args.rc_bin) + require(rc_path is not None, "rc executable is required for shared report provenance", "BLOCKED") + provenance = {"candidate_url": candidate["package_url"], "candidate_sha256": candidate["package_sha256"], + "binary_sha256": candidate["binary_sha256"], "source_commit": candidate["source_sha"], + "testing_commit": actual_commit, "rc_path": str(Path(rc_path).resolve()), "rc_sha256": sha256(rc_path), + "binary_identity_source": "operator-supplied candidate manifest; suite does not deploy or inspect host binaries"} + native = json.loads(args.native_manifest.read_text()) if args.native_manifest else None + args.artifacts.mkdir(parents=True) + (args.artifacts / "run-configuration.json").write_text(json.dumps({ + "selected_cases": [case.id for case in selected], "endpoints": args.endpoints, + "admin_endpoints": args.admin_endpoints, "objects": args.objects, + "request_timeout": args.request_timeout, "convergence_timeout": args.convergence_timeout, + "poll_interval": args.poll_interval, "load_seconds": args.load_seconds, + "provenance": provenance, "node_actions": []}, indent=2) + "\n") + args.cancelled = False + def cancel(signum, _frame): + if args.cancelled: + return + args.cancelled = True + raise ProbeError(f"Runner interrupted by signal {signum}") + prior_handlers = {number: signal.signal(number, cancel) for number in (signal.SIGINT, signal.SIGTERM)} + results = [] + try: + for case in selected: + results.append(execute_case(case, args.artifacts, args, provenance, native)) + if args.cancelled: + break + finally: + for number, handler in prior_handlers.items(): + signal.signal(number, handler) + (args.artifacts / "report.md").write_text(render_report(results) + "\n") + (args.artifacts / "summary.json").write_text(json.dumps({"verdicts": dict(Counter(result["verdict"] for result in results)), + "results": results}, indent=2) + "\n") + return 0 if all(result["verdict"] == "PASS" for result in results) else 1 + except (OSError, ValueError, ProbeError) as exc: + print(f"scanner usage runner: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test_validation_migration.py b/test_validation_migration.py index d82c279..f5ef462 100644 --- a/test_validation_migration.py +++ b/test_validation_migration.py @@ -24,7 +24,7 @@ def test_snapshot_contains_all_source_cases(self) -> None: validate_reuse_candidates(snapshot, ROOT) self.assertEqual(158, sum(len(module["cases"]) for module in snapshot["modules"])) if SOURCE.is_file(): - self.assertEqual(build_snapshot(SOURCE), snapshot) + self.assertEqual(build_snapshot(SOURCE), {key: value for key, value in snapshot.items() if key != "local_modules"}) def test_duplicate_case_is_rejected(self) -> None: snapshot = json.loads((ROOT / "validation-cases.json").read_text(encoding="utf-8")) diff --git a/test_validation_scanner_usage.py b/test_validation_scanner_usage.py new file mode 100644 index 0000000..387593f --- /dev/null +++ b/test_validation_scanner_usage.py @@ -0,0 +1,723 @@ +#!/usr/bin/env python3 +"""Test the usage runner, not the RustFS product, with explicit local doubles.""" + +from __future__ import annotations + +import copy +from contextlib import redirect_stderr +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from scripts.validation_catalog import catalog_modules, validate_snapshot +from scripts.validation_report import check_result +from scripts.select_validation_cases import select_validation_cases, sweepable_cases +from scripts.validation_scanner_usage.client import Client, Evidence, ProbeError, bucket_view, endpoint_url, run_command, sha256 +from scripts.validation_scanner_usage.contracts import CASES, assertion_id, catalog_module, select_cases +from scripts.validation_scanner_usage.native import run_native +from scripts.validation_scanner_usage.runner import execute_case, Fixture, main, render_report + +ROOT = Path(__file__).resolve().parent + + +def certificate(cycle=1): + return {"epoch": 1, "cycle": cycle, "certified_at": {"secs_since_epoch": 1800000000, "nanos_since_epoch": 0}} + + +def snapshot(entry=None, **changes): + value = {"usage_snapshot_complete": True, "usage_snapshot_converged": True, + "buckets_usage": {"bucket": entry or {"objects_count": 1, "size": 8}}} + value.update(changes) + return value + + +def options(**changes): + value = dict(endpoints=["http://example.invalid:9000"], admin_endpoints=[], + execution="combined", aws_bin="aws", region="us-east-1", request_timeout=1, + convergence_timeout=0.002, native_timeout=2, poll_interval=0.001, + objects=3, freshness_samples=2, load_seconds=0) + value.update(changes) + return SimpleNamespace(**value) + + +PROVENANCE = {"candidate_url": "https://example.invalid/rustfs.deb", "candidate_sha256": "a" * 64, + "binary_sha256": "b" * 64, "source_commit": "c" * 40, "testing_commit": "d" * 40, + "rc_path": "/bin/true", "rc_sha256": "e" * 64} + + +class FakeClient: + """Acknowledgement/inventory double used only to exercise runner branches.""" + def __init__(self, endpoints, admin_endpoints, evidence, **_): + self.endpoints = endpoints + self.admin_endpoints = admin_endpoints or endpoints + self.evidence = evidence + self.buckets = {} + self.versioned = set() + self.serial = 0 + self.last_evidence = None + self.fail_cleanup = False + self.fail_put = False + self.pending = False + self.observed = False + self.malformed = False + + def s3(self, operation, *arguments, endpoint_index=0): + self.last_evidence = self.evidence.save("fake-s3", {"operation": operation, "arguments": list(map(str, arguments))}) + def arg(name, default=None): + return arguments[arguments.index(name) + 1] if name in arguments else default + bucket, key = arg("--bucket"), arg("--key") + if operation == "create-bucket": + if bucket in self.buckets: + raise ProbeError("already exists", "FAIL") + self.buckets[bucket] = [] + elif operation == "put-bucket-versioning": + self.versioned.add(bucket) + elif operation == "put-object": + if self.fail_put: + raise ProbeError("The service is unavailable (HTTP 503)", "FAIL") + self.serial += 1 + version = str(self.serial) if bucket in self.versioned else "null" + if bucket not in self.versioned: + self.buckets[bucket] = [row for row in self.buckets[bucket] if row["Key"] != key] + self.buckets[bucket].append({"Key": key, "VersionId": version, "payload": Path(arg("--body")).read_bytes(), "marker": False}) + return {"VersionId": version, "ETag": "test"} + elif operation == "delete-object": + version = arg("--version-id") + if version is not None or bucket not in self.versioned: + self.buckets[bucket] = [row for row in self.buckets[bucket] if not (row["Key"] == key and (version is None or row["VersionId"] == version))] + else: + self.serial += 1 + self.buckets[bucket].append({"Key": key, "VersionId": str(self.serial), "payload": b"", "marker": True}) + return {"VersionId": str(self.serial), "DeleteMarker": True} + elif operation == "list-objects-v2": + return {"Contents": [{"Key": row["Key"], "Size": len(row["payload"])} for row in self.buckets[bucket] if not row["marker"]], "IsTruncated": False} + elif operation == "list-object-versions": + return {"Versions": [{"Key": row["Key"], "VersionId": row["VersionId"], "Size": len(row["payload"])} for row in self.buckets[bucket] if not row["marker"]], + "DeleteMarkers": [{"Key": row["Key"], "VersionId": row["VersionId"]} for row in self.buckets[bucket] if row["marker"]], "IsTruncated": False} + elif operation == "get-object": + version = arg("--version-id") + row = next(row for row in reversed(self.buckets[bucket]) if row["Key"] == key and (version is None or row["VersionId"] == version)) + Path(arguments[-1]).write_bytes(row["payload"]) + elif operation == "delete-bucket": + if self.fail_cleanup: + raise ProbeError("cleanup failed") + assert not self.buckets[bucket] + del self.buckets[bucket] + else: + raise AssertionError(operation) + return {} + + def admin(self, path="/rustfs/admin/v3/datausageinfo", endpoint_index=0): + if self.malformed: + raise ProbeError("Admin returned invalid JSON") + value = {"scanner_cycle": self.serial, "usage_snapshot_complete": True, + "usage_snapshot_converged": not self.pending, "buckets_usage": {}} + for bucket, rows in self.buckets.items(): + value["buckets_usage"][bucket] = {"objects_count": len({row["Key"] for row in rows}), + "size": sum(len(row["payload"]) for row in rows), "versions_count": sum(not row["marker"] for row in rows), + "delete_markers_count": sum(row["marker"] for row in rows), + "usage_value_state": "observed" if self.observed else "certified", + "usage_certification": None if self.observed else certificate(self.serial)} + self.last_evidence = self.evidence.save("fake-admin", value) + return value + + +class ValueContractTests(unittest.TestCase): + def test_absence_and_missing_counters_do_not_become_zero(self): + view = bucket_view({"buckets_usage": {}}, "bucket") + self.assertEqual((view["count"], view["size"], view["state"]), (None, None, "unknown")) + view = bucket_view(snapshot({"size": 0}), "bucket") + self.assertIsNone(view["count"]) + self.assertEqual(view["size"], 0) + + def test_pending_partial_observed_and_unknown_cannot_converge(self): + for changes in ({"usage_snapshot_converged": False}, {"usage_snapshot_partial": True}, + {"usage_snapshot_observational": True}, {"usage_snapshot_pending_buckets": ["bucket"]}, + {"usage_snapshot_admin_projection": True}, + {"usage_snapshot_complete": False}): + self.assertFalse(bucket_view(snapshot(**changes), "bucket")["converged"]) + for state in ("carried", "observed", "future-state"): + entry = {"objects_count": 1, "size": 8, "usage_value_state": state} + self.assertFalse(bucket_view(snapshot(entry), "bucket")["converged"]) + value = snapshot() + del value["usage_snapshot_converged"] + self.assertFalse(bucket_view(value, "bucket")["converged"]) + + def test_diagnostic_contract_and_no_last_update_synthesis(self): + value = snapshot({"objects_count": 1, "size": 8, "usage_value_state": "observed", "usage_certification": certificate()}) + with self.assertRaisesRegex(ProbeError, "Observed"): + bucket_view(value, "bucket") + value = snapshot({"objects_count": 1, "size": 8, "usage_value_state": "certified"}, last_update=certificate()["certified_at"]) + with self.assertRaisesRegex(ProbeError, "no per-bucket"): + bucket_view(value, "bucket") + value = snapshot({"objects_count": 1, "size": 8, "usage_value_state": "future", "usage_certification": {"future": True}}) + self.assertIsNone(bucket_view(value, "bucket")["certificate"]) + + def test_invalid_numeric_and_timestamp_types(self): + for bad in (-1, True, "1"): + with self.assertRaises(ProbeError): + bucket_view(snapshot({"objects_count": bad, "size": 8}), "bucket") + cert = certificate() + cert["certified_at"]["nanos_since_epoch"] = 10**9 + with self.assertRaises(ProbeError): + bucket_view(snapshot({"usage_value_state": "certified", "usage_certification": cert}), "bucket") + + def test_endpoint_rejects_embedded_secrets_and_non_origins(self): + for bad in ("http://user:pass@host", "file:///etc/passwd", "http://host/path", "http://host?token=a", + "http://@host", "http://host:65536", "http://host:secret-port", "http://host\n"): + with self.assertRaises(ValueError): + endpoint_url(bad) + self.assertEqual(endpoint_url("https://host:9001/"), "https://host:9001") + + def test_usage_state_rejects_non_string_json_values(self): + for state in ([], {}, 1, True): + with self.subTest(state=state), self.assertRaisesRegex(ProbeError, "usage_value_state") as error: + bucket_view(snapshot({"objects_count": 1, "size": 8, "usage_value_state": state}), "bucket") + self.assertEqual(error.exception.verdict, "FAIL") + + def test_signed_admin_requests_never_follow_redirects(self): + received = [] + class RedirectHandler(BaseHTTPRequestHandler): + def do_GET(self): + received.append((self.server.server_port, self.path)) + self.send_response(self.server.response_status) + self.send_header("Location", self.server.destination) + self.end_headers() + self.wfile.write(b'{"redirect":true}') + + def log_message(self, *_): + pass + + servers = [ThreadingHTTPServer(("127.0.0.1", 0), RedirectHandler) for _ in range(2)] + for server in servers: + thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + self.addCleanup(thread.join, 2) + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + server.response_status = 200 + server.destination = "http://127.0.0.1/unused" + origin = f"http://127.0.0.1:{servers[0].server_port}" + with tempfile.TemporaryDirectory() as directory: + client = Client([origin], [], Evidence(directory), environment={ + "RUSTFS_ACCESS_KEY": "unit-access", "RUSTFS_SECRET_KEY": "unit-secret", "AWS_SESSION_TOKEN": "unit-token"}) + for status in (301, 302, 303, 307, 308): + for destination in (origin + "/unexpected", f"http://127.0.0.1:{servers[1].server_port}/unexpected"): + with self.subTest(status=status, destination=destination): + received.clear() + servers[0].response_status, servers[0].destination = status, destination + with self.assertRaisesRegex(ProbeError, f"HTTP {status}") as error: + client.admin() + self.assertEqual(error.exception.verdict, "FAIL") + self.assertEqual(received, [(servers[0].server_port, "/rustfs/admin/v3/datausageinfo")]) + record = json.loads((Path(directory) / client.last_evidence).read_text()) + self.assertEqual((record["status"], record["url"]), (status, origin + "/rustfs/admin/v3/datausageinfo")) + + +class RunnerTests(unittest.TestCase): + def run_case(self, case=CASES[0], *, configure=lambda _: None, **changes): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.directory = Path(self.tmp.name) + def factory(*args, **kwargs): + self.client = FakeClient(*args, **kwargs) + configure(self.client) + return self.client + return execute_case(case, self.directory, options(**changes), PROVENANCE, client_factory=factory) + + def test_static_scenario_runs_oracle_convergence_and_cleanup(self): + result = self.run_case() + self.assertEqual(result["verdict"], "PASS", result) + self.assertEqual(self.client.buckets, {}) + self.assertEqual(len(result["assertions"]), 3) + self.assertFalse(check_result(catalog_module()["cases"][0], result, gate=True, + artifacts_root=self.directory, target_environments=["configured-endpoint"])) + self.assertIn("USAGE-01-A03", render_report([result])) + + def test_versioned_scenario_preserves_historical_payloads_and_marker_counts(self): + result = self.run_case(CASES[19]) + self.assertEqual(result["verdict"], "PASS", result) + oracles = list((self.directory / "evidence" / CASES[19].id).glob("*-independent-oracle.json")) + expected = json.loads(oracles[0].read_text())["expected"] + self.assertEqual((expected["count"], expected["versions"], expected["markers"]), (6, 8, 1)) + + def test_product_error_and_cleanup_failure_cannot_pass(self): + for attr, verdict in (("fail_put", "FAIL"), ("fail_cleanup", "INFRA_ERROR"), ("malformed", "INFRA_ERROR")): + with self.subTest(attr=attr): + result = self.run_case(configure=lambda client: setattr(client, attr, True)) + self.assertEqual(result["verdict"], verdict, result) + if attr == "fail_cleanup": + self.assertEqual(result["cleanup_status"], "FAIL") + self.assertTrue(self.client.buckets) + + def test_pending_and_observed_exact_values_are_not_converged(self): + for attr in ("pending", "observed"): + result = self.run_case(configure=lambda client: setattr(client, attr, True)) + self.assertEqual(result["verdict"], "FAIL") + self.assertIn("did not converge", result["reason"]) + + def test_mutation_probe_does_not_claim_internal_cycle_or_ack_evidence(self): + result = self.run_case(CASES[2]) + self.assertEqual(result["verdict"], "BLOCKED", result) + self.assertEqual([item["status"] for item in result["assertions"]], ["PASS", "BLOCKED", "BLOCKED"]) + + def test_no_endpoint_is_blocked_without_creating_a_fixture(self): + result = self.run_case(endpoints=[]) + self.assertEqual(result["verdict"], "BLOCKED") + self.assertEqual(result["cleanup_status"], "PASS") + + def test_one_endpoint_cannot_prove_multiple_sources(self): + result = self.run_case(CASES[6]) + self.assertEqual(result["verdict"], "BLOCKED") + self.assertIn("Multiple explicit", result["reason"]) + + def test_minimum_fixture_still_writes_through_every_endpoint(self): + endpoints = [f"http://endpoint-{index}.invalid:9000" for index in range(3)] + result = self.run_case(CASES[6], endpoints=endpoints, admin_endpoints=endpoints, objects=2) + self.assertEqual(result["assertions"][0]["status"], "PASS", result) + directory = self.directory / "evidence" / CASES[6].id + acknowledged = [json.loads(line) for line in (directory / "acknowledged-requests.jsonl").read_text().splitlines()] + puts = [row for row in acknowledged if row["operation"] == "put"] + self.assertEqual({row["endpoint_index"] for row in puts}, {0, 1, 2}) + self.assertEqual(len(puts), 3) + + def test_malformed_usage_state_preserves_results_and_allows_the_next_case(self): + class MalformedStateClient(FakeClient): + def admin(self, *args, **kwargs): + result = super().admin(*args, **kwargs) + for entry in result["buckets_usage"].values(): + entry["usage_value_state"] = [] + return result + with tempfile.TemporaryDirectory() as directory: + first = execute_case(CASES[0], directory, options(), PROVENANCE, client_factory=MalformedStateClient) + second = execute_case(CASES[19], directory, options(), PROVENANCE, client_factory=FakeClient) + self.assertEqual(first["verdict"], "FAIL") + self.assertEqual([row["status"] for row in first["assertions"]], ["PASS", "FAIL", "BLOCKED"]) + self.assertEqual(first["cleanup_status"], "PASS") + self.assertEqual(second["verdict"], "PASS") + self.assertEqual(len(list((Path(directory) / "cases").glob("*.json"))), 2) + + def test_request_journal_appends_without_rewriting_ownership_or_prior_entries(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Evidence(directory) + client = FakeClient(["http://example.invalid"], [], evidence) + fixture = Fixture(client, directory, evidence) + bucket = fixture.create("append-only") + ownership_path = Path(directory) / "owned-buckets.json" + original_ownership = ownership_path.read_bytes() + with patch.object(fixture, "persist_ownership", wraps=fixture.persist_ownership) as ownership: + for _ in range(512): + fixture.put(bucket, "replaced", 1) + fixture.delete(bucket, "replaced") + ownership.assert_not_called() + self.assertEqual(ownership_path.read_bytes(), original_ownership) + journal = fixture.journal_path.read_bytes() + rows = [json.loads(line) for line in journal.splitlines()] + self.assertEqual([row["sequence"] for row in rows], list(range(1, 1025))) + self.assertEqual([row["operation"] for row in rows], ["put", "delete"] * 512) + self.assertLess(len(journal), 1024 * 512, "journal bytes must be linear in acknowledgement count") + fixture.put(bucket, "later", 1) + self.assertTrue(fixture.journal_path.read_bytes().startswith(journal)) + self.assertEqual(fixture.cleanup()[0], "PASS") + + def test_cleanup_never_accepts_a_foreign_bucket(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Evidence(directory) + client = FakeClient(["http://example.invalid"], [], evidence) + fixture = Fixture(client, directory, evidence) + client.buckets["desktop"] = [] + with self.assertRaisesRegex(ProbeError, "unowned"): + fixture.remove("desktop") + self.assertEqual(fixture.cleanup()[0], "PASS") + self.assertIn("desktop", client.buckets) + + def test_lost_create_response_cannot_report_cleanup_success(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Evidence(directory) + client = FakeClient(["http://example.invalid"], [], evidence) + fixture = Fixture(client, directory, evidence) + operation = client.s3 + def lose_response(*args, **kwargs): + operation(*args, **kwargs) + raise ProbeError("CreateBucket response was lost") + with patch.object(client, "s3", side_effect=lose_response), self.assertRaises(ProbeError): + fixture.create("lost-response") + ownership = json.loads((Path(directory) / "owned-buckets.json").read_text()) + self.assertEqual(ownership["buckets"], []) + self.assertEqual(ownership["unconfirmed_creates"], sorted(client.buckets)) + status, cleanup_path = fixture.cleanup() + self.assertEqual(status, "FAIL") + self.assertTrue(json.loads((Path(directory) / cleanup_path).read_text())["unconfirmed_creates"]) + self.assertTrue(client.buckets, "unconfirmed ownership cannot authorize deletion") + + def test_inventory_follows_pages_and_rejects_repeated_continuations(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Evidence(directory) + client = FakeClient(["http://example.invalid"], [], evidence) + fixture = Fixture(client, directory, evidence) + bucket = fixture.create("pages") + pages = [{"Contents": [{"Key": "first", "Size": 1}], "IsTruncated": True, + "NextContinuationToken": "page-two"}, + {"Contents": [{"Key": "second", "Size": 2}], "IsTruncated": False}] + with patch.object(client, "s3", side_effect=pages) as request: + self.assertEqual(fixture.inventory(bucket), [{"key": "first", "size": 1}, {"key": "second", "size": 2}]) + self.assertEqual(request.call_args_list[1].args[-2:], ("--continuation-token", "page-two")) + with patch.object(client, "s3", return_value=pages[0]), self.assertRaisesRegex(ProbeError, "continuation"): + fixture.inventory(bucket) + + def test_version_inventory_preserves_both_markers_across_pages(self): + with tempfile.TemporaryDirectory() as directory: + evidence = Evidence(directory) + client = FakeClient(["http://example.invalid"], [], evidence) + fixture = Fixture(client, directory, evidence) + bucket = fixture.create("version-pages", versioned=True) + pages = [{"Versions": [{"Key": "key", "VersionId": "v1", "Size": 3}], "IsTruncated": True, + "NextKeyMarker": "key", "NextVersionIdMarker": "v1"}, + {"DeleteMarkers": [{"Key": "key", "VersionId": "v2"}], "IsTruncated": False}] + with patch.object(client, "s3", side_effect=pages) as request: + rows = fixture.inventory(bucket, versioned=True) + self.assertEqual([(row["version_id"], row["marker"]) for row in rows], [("v1", False), ("v2", True)]) + self.assertEqual(request.call_args_list[1].args[-4:], ("--key-marker", "key", "--version-id-marker", "v1")) + + def test_command_timeout_is_recorded(self): + result = run_command([sys.executable, "-c", "import time;time.sleep(10)"], 0.02) + self.assertTrue(result["timed_out"]) + self.assertNotEqual(result["exit_code"], 0) + + def test_s3_subprocess_cannot_inherit_ambient_proxy_routes(self): + with tempfile.TemporaryDirectory() as directory: + environment = {"RUSTFS_ACCESS_KEY": "unit-access", "RUSTFS_SECRET_KEY": "unit-secret", + "HTTPS_PROXY": "http://proxy.invalid", "http_proxy": "http://proxy.invalid"} + client = Client(["http://example.invalid"], [], Evidence(directory), environment=environment) + self.assertNotIn("HTTPS_PROXY", client.environment) + self.assertNotIn("http_proxy", client.environment) + self.assertEqual(client.environment["NO_PROXY"], "*") + + def test_failed_request_retains_service_code_without_credentials(self): + with tempfile.TemporaryDirectory() as directory: + client = Client(["http://example.invalid"], [], Evidence(directory), environment={ + "RUSTFS_ACCESS_KEY": "unit-access-secret", "RUSTFS_SECRET_KEY": "unit-signing-secret"}) + response = {"command": ["aws"], "exit_code": 1, "timed_out": False, "duration_seconds": 1, + "stdout": "", "stderr": "An error occurred (ServiceUnavailable): unit-signing-secret unit-access-secret"} + with patch("scripts.validation_scanner_usage.client.run_command", return_value=response), self.assertRaises(ProbeError) as error: + client.s3("put-object", "--bucket", "unit-fixture") + self.assertEqual(error.exception.verdict, "FAIL") + text = (Path(directory) / client.last_evidence).read_text() + self.assertNotIn("unit-signing-secret", text) + self.assertNotIn("unit-access-secret", text) + self.assertEqual(json.loads(text)["service_error_code"], "ServiceUnavailable") + + def test_admin_timeout_is_infra_error_and_retains_evidence(self): + with tempfile.TemporaryDirectory() as directory: + client = Client(["http://example.invalid"], [], Evidence(directory), environment={ + "RUSTFS_ACCESS_KEY": "unit-access", "RUSTFS_SECRET_KEY": "unit-secret"}) + with patch.object(client.http, "open", side_effect=TimeoutError("socket deadline")), self.assertRaises(ProbeError) as error: + client.admin() + self.assertEqual(error.exception.verdict, "INFRA_ERROR") + self.assertTrue((Path(directory) / client.last_evidence).is_file()) + + +class NativeTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.case = CASES[13] + self.name = "scanner::tests::usage_exact_fixture" + self.ids = [assertion_id(self.case, index) for index in range(len(self.case.assertions))] + + def manifest(self, *, missing=False, cleanup="PASS", zero=False, fail=False): + binary = self.root / "native-test" + output = ["running 1 test", f"test {self.name} ... ok"] + output += ["usage_assertion: " + json.dumps({"case_id": self.case.design_id, "assertion_id": identifier, + "status": "PASS", "evidence": {"test_double": True}}) for identifier in self.ids if not missing] + output += ["usage_cleanup: " + json.dumps({"case_id": self.case.design_id, "status": cleanup}), + "test result: ok. 1 passed; 0 failed; 0 ignored; 20 filtered out;"] + if zero: + output = ["running 0 tests", "test result: ok. 0 passed; 0 failed; 0 ignored;"] + script = "#!" + sys.executable + "\nimport sys\n" + script += f"print({(self.name + ': test')!r} if '--list' in sys.argv else {chr(10).join(output)!r})\n" + if fail: + script += "sys.exit(0 if '--list' in sys.argv else 1)\n" + binary.write_text(script) + binary.chmod(0o700) + return {"schema_version": 1, "source_commit": PROVENANCE["source_commit"], "cases": { + self.case.design_id: {"tests": [{"binary": str(binary), "binary_sha256": sha256(binary), + "test": self.name, "assertions": self.ids}]}}} + + def run_native(self, manifest): + return run_native(self.case, manifest, PROVENANCE["source_commit"], Evidence(self.root / "evidence"), 2, self.ids) + + def test_exact_test_and_each_marker_are_required(self): + records = self.run_native(self.manifest()) + self.assertEqual(set(records), set(self.ids)) + for values in ({"missing": True}, {"zero": True}, {"cleanup": "FAIL"}, {"fail": True}): + with self.subTest(values=values), self.assertRaises(ProbeError) as error: + self.run_native(self.manifest(**values)) + self.assertEqual(error.exception.cleanup_status, "FAIL") + + def test_native_identity_and_capability_fail_closed(self): + with self.assertRaises(ProbeError) as error: + self.run_native(None) + self.assertEqual(error.exception.verdict, "BLOCKED") + manifest = self.manifest() + manifest["source_commit"] = "0" * 40 + with self.assertRaisesRegex(ProbeError, "identity"): + self.run_native(manifest) + manifest = self.manifest() + manifest["cases"][self.case.design_id]["tests"][0]["binary_sha256"] = "f" * 64 + with self.assertRaisesRegex(ProbeError, "SHA-256"): + self.run_native(manifest) + + def test_two_tests_cannot_claim_the_same_atomic_assertion(self): + manifest = self.manifest() + specs = manifest["cases"][self.case.design_id]["tests"] + specs.append(copy.deepcopy(specs[0])) + with patch("scripts.validation_scanner_usage.native.run_command") as execute: + with self.assertRaisesRegex(ProbeError, "duplicate coverage"): + self.run_native(manifest) + execute.assert_not_called() + + def test_another_cases_milestone_cannot_be_reused(self): + manifest = self.manifest() + spec = manifest["cases"][self.case.design_id]["tests"][0] + binary = Path(spec["binary"]) + binary.write_text(binary.read_text().replace(self.case.design_id, "USAGE-01")) + spec["binary_sha256"] = sha256(binary) + with self.assertRaisesRegex(ProbeError, "different case") as error: + self.run_native(manifest) + self.assertEqual(error.exception.cleanup_status, "FAIL") + + def test_malformed_native_manifest_produces_a_terminal_adapter_error(self): + for invalid in ([], {"schema_version": 1, "source_commit": PROVENANCE["source_commit"], "cases": []}): + with self.subTest(manifest=invalid), self.assertRaises(ProbeError): + self.run_native(invalid) + for field, invalid in (("binary", None), ("assertions", [{}])): + manifest = self.manifest() + manifest["cases"][self.case.design_id]["tests"][0][field] = invalid + with self.subTest(field=field), self.assertRaises(ProbeError): + self.run_native(manifest) + + def test_missing_native_cleanup_cannot_be_hidden_by_public_cleanup(self): + result = execute_case(self.case, self.root, options(execution="native"), PROVENANCE, + native_manifest=self.manifest(cleanup="FAIL")) + self.assertEqual(result["verdict"], "INFRA_ERROR") + self.assertEqual(result["cleanup_status"], "FAIL") + + def test_partial_native_coverage_retains_true_assertions_without_passing_case(self): + all_ids = self.ids + self.ids = all_ids[:1] + manifest = self.manifest() + self.ids = all_ids + result = execute_case(self.case, self.root, options(execution="native"), PROVENANCE, + native_manifest=manifest) + self.assertEqual(result["verdict"], "UNSUPPORTED") + self.assertEqual(result["assertions"][0]["status"], "PASS") + self.assertTrue(all(row["status"] == "UNSUPPORTED" for row in result["assertions"][1:])) + self.assertEqual(result["migration_status"], "unverified") + + def test_unexecuted_tests_after_failure_are_blocked_not_product_failures(self): + case = CASES[12] + ids = [assertion_id(case, index) for index in range(len(case.assertions))] + manifest = {"schema_version": 1, "source_commit": PROVENANCE["source_commit"], + "cases": {case.design_id: {"tests": [{"assertions": [identifier]} for identifier in ids]}}} + with patch("scripts.validation_scanner_usage.native._run_native_test", side_effect=ProbeError("native regression failed", "FAIL")) as execute: + result = execute_case(case, self.root, options(execution="native"), PROVENANCE, native_manifest=manifest) + execute.assert_called_once() + self.assertEqual(result["verdict"], "FAIL") + self.assertEqual(result["assertions"][0]["status"], "FAIL") + self.assertTrue(all(row["status"] == "BLOCKED" for row in result["assertions"][1:]), result) + + def test_nonexecutable_later_binary_keeps_prior_pass_and_later_blocked_records(self): + self.case = CASES[12] + all_ids = [assertion_id(self.case, index) for index in range(len(self.case.assertions))] + self.ids = all_ids[:1] + manifest = self.manifest() + specs = manifest["cases"][self.case.design_id]["tests"] + blocked_binary = self.root / "not-executable" + blocked_binary.write_text(Path(specs[0]["binary"]).read_text()) + blocked_binary.chmod(0o600) + specs.extend({**specs[0], "binary": str(blocked_binary), "binary_sha256": sha256(blocked_binary), + "assertions": [identifier]} for identifier in all_ids[1:]) + result = execute_case(self.case, self.root, options(execution="native"), PROVENANCE, native_manifest=manifest) + self.assertEqual(result["verdict"], "INFRA_ERROR") + self.assertEqual([row["status"] for row in result["assertions"]], ["PASS", "INFRA_ERROR"] + ["BLOCKED"] * 6) + self.assertTrue(result["assertions"][0]["native_evidence"]) + self.assertIn("Permission denied", result["reason"]) + + def test_diagnostic_write_failure_keeps_completed_native_records_and_unexecuted_states(self): + self.case = CASES[12] + all_ids = [assertion_id(self.case, index) for index in range(len(self.case.assertions))] + self.ids = all_ids[:1] + manifest = self.manifest() + specs = manifest["cases"][self.case.design_id]["tests"] + blocked_binary = self.root / "not-executable" + blocked_binary.write_text(Path(specs[0]["binary"]).read_text()) + blocked_binary.chmod(0o600) + specs.extend({**specs[0], "binary": str(blocked_binary), "binary_sha256": sha256(blocked_binary), + "assertions": [identifier]} for identifier in all_ids[1:]) + original_save = Evidence.save + def fail_diagnostic(evidence, stem, value): + if stem == "native-adapter-error": + raise OSError("injected diagnostic write failure") + return original_save(evidence, stem, value) + with patch.object(Evidence, "save", fail_diagnostic): + result = execute_case(self.case, self.root, options(execution="native"), PROVENANCE, native_manifest=manifest) + self.assertEqual(result["verdict"], "INFRA_ERROR") + self.assertEqual([row["status"] for row in result["assertions"]], ["PASS", "INFRA_ERROR"] + ["BLOCKED"] * 6) + for path in result["assertions"][0]["evidence"]: + self.assertTrue((self.root / path).is_file()) + self.assertEqual(result["assertions"][1]["evidence"], []) + self.assertIn("diagnostic could not be saved", result["reason"]) + + def test_real_signal_interrupts_native_process_and_leaves_cleanup_unverified(self): + manifest = self.manifest() + spec = manifest["cases"][self.case.design_id]["tests"][0] + binary, ready = Path(spec["binary"]), self.root / "native-ready.json" + binary.write_text("#!" + sys.executable + "\nimport json, os, pathlib, sys, time\n" + + f"if '--list' in sys.argv:\n print({(self.name + ': test')!r})\n" + + "else:\n" + + f" ready = pathlib.Path({str(ready)!r})\n" + + " ready.with_suffix('.tmp').write_text(json.dumps({'pid': os.getpid()}))\n" + + " ready.with_suffix('.tmp').replace(ready)\n" + + " time.sleep(30)\n") + spec["binary_sha256"] = sha256(binary) + manifest_path, candidate_path = self.root / "native.json", self.root / "candidate.json" + manifest_path.write_text(json.dumps(manifest)) + candidate_path.write_text(json.dumps({"source_sha": PROVENANCE["source_commit"], + "package_sha256": "a" * 64, "binary_sha256": "b" * 64, "package_url": "https://example.invalid/package"})) + artifacts = self.root / "interrupted-run" + command = [sys.executable, "-B", "-m", "scripts.validation_scanner_usage.runner", "--execution", "native", + "--cases", self.case.design_id, "--native-manifest", str(manifest_path), + "--candidate-manifest", str(candidate_path), "--artifacts-dir", str(artifacts), + "--rc-bin", sys.executable, "--native-timeout", "20"] + process = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not ready.exists() and process.poll() is None and time.monotonic() < deadline: + time.sleep(0.01) + self.assertTrue(ready.exists(), "native command did not start") + pid = json.loads(ready.read_text())["pid"] + process.send_signal(signal.SIGTERM) + stdout, stderr = process.communicate(timeout=10) + self.assertEqual(process.returncode, 1, (stdout, stderr)) + result = json.loads((artifacts / "cases" / (self.case.id + ".json")).read_text()) + self.assertEqual((result["verdict"], result["cleanup_status"]), ("INFRA_ERROR", "FAIL")) + self.assertIn("interrupted by signal", result["reason"]) + errors = list((artifacts / "evidence" / self.case.id).glob("*-native-adapter-error.json")) + self.assertEqual(json.loads(errors[0].read_text())["cleanup_status"], "FAIL") + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + finally: + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.communicate(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + process.communicate() + + def test_interruption_before_native_milestone_validation_cannot_report_cleanup_pass(self): + manifest = self.manifest() + original_save = Evidence.save + def interrupted_save(evidence, stem, value): + if stem == "native-execution": + raise ProbeError("Runner interrupted before native milestones were validated") + return original_save(evidence, stem, value) + with patch.object(Evidence, "save", interrupted_save): + result = execute_case(self.case, self.root, options(execution="native"), PROVENANCE, native_manifest=manifest) + self.assertEqual((result["verdict"], result["cleanup_status"]), ("INFRA_ERROR", "FAIL")) + + +class RegistrationTests(unittest.TestCase): + def test_cli_rejects_endpoint_secrets_before_artifacts_in_every_mode(self): + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, {}, clear=True): + artifacts = Path(directory) / "must-not-exist" + for execution in ("combined", "native"): + for flag in ("--endpoint", "--admin-endpoint"): + for endpoint in ("http://user:private-password@host", "http://host?token=private-token", + "http://host:private-port"): + with self.subTest(execution=execution, flag=flag, endpoint=endpoint): + stderr = io.StringIO() + with redirect_stderr(stderr): + result = main(["--execution", execution, flag, endpoint, "--cases", "USAGE-14", + "--artifacts-dir", str(artifacts), "--candidate-manifest", "not-read.json"]) + self.assertEqual(result, 2) + self.assertIn("endpoint must be", stderr.getvalue()) + self.assertNotIn("private-", stderr.getvalue()) + self.assertFalse(artifacts.exists()) + + def test_native_cli_records_only_normalized_endpoints(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + candidate = root / "candidate.json" + candidate.write_text(json.dumps({"source_sha": "c" * 40, "package_sha256": "a" * 64, + "binary_sha256": "b" * 64, "package_url": "https://example.invalid/package"})) + artifacts = root / "artifacts" + result = main(["--execution", "native", "--cases", "USAGE-14", "--endpoint", "http://host:9000/", + "--admin-endpoint", "http://host:9001/", "--artifacts-dir", str(artifacts), + "--candidate-manifest", str(candidate), "--rc-bin", sys.executable]) + self.assertEqual(result, 1, "missing native capability stays blocked") + config = json.loads((artifacts / "run-configuration.json").read_text()) + self.assertEqual(config["endpoints"], ["http://host:9000"]) + self.assertEqual(config["admin_endpoints"], ["http://host:9001"]) + + def test_twenty_native_rows_without_capabilities_remain_blocked(self): + with tempfile.TemporaryDirectory() as directory: + with patch("scripts.validation_scanner_usage.runner.Client", side_effect=AssertionError("no endpoint access")) as client: + results = [execute_case(case, directory, options(execution="native", endpoints=[]), PROVENANCE, client_factory=client) + for case in select_cases("all")] + client.assert_not_called() + self.assertEqual(len(results), 20) + self.assertTrue(all(result["verdict"] == "BLOCKED" for result in results)) + self.assertTrue(all(result["profile"] == "native-test" and result["hit_count"] == 0 for result in results)) + self.assertTrue(all(result["cleanup_status"] == "PASS" for result in results)) + self.assertTrue(all(assertion["status"] == "BLOCKED" for result in results for assertion in result["assertions"])) + self.assertEqual(len(list((Path(directory) / "cases").glob("*.json"))), 20) + + def test_twenty_rows_are_registered_without_rewriting_source_counts(self): + catalog = json.loads((ROOT / "validation-cases.json").read_text()) + validate_snapshot(catalog) + self.assertEqual(catalog["local_modules"], [catalog_module()]) + self.assertEqual(sum(len(module["cases"]) for module in catalog["modules"]), 158) + self.assertEqual(sum(len(module["cases"]) for module in catalog_modules(catalog)), 178) + self.assertEqual(len(select_cases("all")), 20) + self.assertEqual([case.number for case in select_cases("blackbox")], [1, 20]) + self.assertNotIn(CASES[0].id, [case for case, _ in sweepable_cases(catalog)]) + self.assertEqual(select_validation_cases(catalog, case=CASES[0].id), [(CASES[0].id, "scanner-usage-regressions")]) + + def test_missing_assertion_artifact_fails_the_shared_report(self): + with tempfile.TemporaryDirectory() as directory: + result = {"schema_version": 1, "case_id": CASES[0].id, "source_status": "procedure-ready", + "profile": "configured-endpoint", "verdict": "PASS", "migration_status": "verified", + "cleanup_status": "PASS", "provenance": PROVENANCE, + "assertions": [{"source_assertion": text, "status": "PASS", "evidence": ["missing.json"]} + for _, text in CASES[0].assertions]} + errors = check_result(catalog_module()["cases"][0], result, gate=True, + artifacts_root=Path(directory), target_environments=["configured-endpoint"]) + self.assertTrue(any("missing or unsafe evidence" in error for error in errors)) + + def test_dry_run_neither_reads_candidate_nor_creates_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + destination = Path(directory) / "not-created" + result = run_command(["bash", str(ROOT / "rustfs-scanner-usage-test.sh"), "--dry-run", + "--cases", "all", "--artifacts-dir", str(destination), + "--candidate-manifest", str(Path(directory) / "not-a-manifest")], 10) + self.assertEqual(result["exit_code"], 0, result) + self.assertEqual(len(json.loads(result["stdout"])["cases"]), 20) + self.assertFalse(destination.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/validation-cases.json b/validation-cases.json index ff57fa0..0bdadd8 100644 --- a/validation-cases.json +++ b/validation-cases.json @@ -4146,5 +4146,580 @@ } ] } + ], + "local_modules": [ + { + "id": "scanner-usage-regressions", + "origin": "auto-testing", + "target_environments": [ + "configured-endpoint", + "native-test" + ], + "required_capabilities": [ + "explicit-endpoint", + "candidate-provenance" + ], + "cases": [ + { + "id": "scanner.usage-01-static-accounting", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Committed objects and an independent paginated S3 inventory have identical keys and bytes.", + "Usage converges to exact object counts and logical bytes after writes and deletes.", + "A measured empty bucket is distinct from an absent or unknown bucket value." + ], + "probes": { + "designCode": "USAGE-01", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "static" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-02-unrelated-remote-mutations", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "A long bucket makes durable progress across at least three cycles while another node mutates an unrelated bucket.", + "The complete producer watch proof prevents unrelated writes from invalidating that bucket's certification." + ], + "probes": { + "designCode": "USAGE-02", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": null + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-03-cross-cycle-mutations", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Acknowledged insert, overwrite and delete operations converge to an independent final S3 inventory.", + "A mutation occurs after the affected range was scanned and before a later cycle whose start and post generations agree.", + "The mixed result is not certified or acknowledged, and a later fresh sweep is exact." + ], + "probes": { + "designCode": "USAGE-03", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "mutations" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-04-hot-prefix-progress", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Nested and prefix-shaped keys retain exact final contents after controlled mutations stop.", + "The cold range obtains durable traversal progress while a hot prefix is continuously modified.", + "A whole-bucket observation never claims independent prefix certification or prefix ACK." + ], + "probes": { + "designCode": "USAGE-04", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "mutations" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-05-shared-budget-fairness", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Three cold buckets converge exactly while requests continue against a separate hot bucket.", + "A real shared cycle budget is exhausted without a bucket quantum cancelling its parent.", + "All cold buckets receive service and a zero-progress turn cannot busy-loop." + ], + "probes": { + "designCode": "USAGE-05", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "multi-bucket" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-06-wide-deep-directories", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "A prefix containing at least 100000 entries and nested wide directories advances its durable frontier.", + "Checkpoint memory and bytes remain bounded across rounds without sacrificing exact counts." + ], + "probes": { + "designCode": "USAGE-06", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": null + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-07-multi-source-coverage", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Acknowledged writes through every configured endpoint remain readable and converge to the same exact bucket usage.", + "The fixture has more physical sources than the set concurrency limit and every expected source contributes.", + "A missing or non-fresh source prevents complete totals and whole-bucket ACK." + ], + "probes": { + "designCode": "USAGE-07", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "multi-endpoint" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-08-no-baseline", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "An absent bucket value stays unknown in the test oracle until a real measured value is available.", + "A cold-start partial or mixed bucket has no legal LKG and cannot enter the legacy primary map or quota fallback." + ], + "probes": { + "designCode": "USAGE-08", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "no-baseline" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-09-value-freshness", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Returned carried or observed values obey the per-bucket certification contract without using last_update as a certificate.", + "Multiple PartialCarried and mixed rounds preserve the original value certificate or observation completion time.", + "A mixture of current and old source values does not inherit an unrelated whole-bucket certificate." + ], + "probes": { + "designCode": "USAGE-09", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "freshness" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-10-checkpoint-recovery", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Crashes before and after mixed EOF CAS keep the observation, cleared cursor and fresh-sweep debt atomic.", + "A successor rejects late old-leader writes and never clears responsibility after losing an observation." + ], + "probes": { + "designCode": "USAGE-10", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": null + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-11-bucket-recreation", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Deleting and recreating the suite-owned bucket converges to the successor's exact keys and bytes.", + "An old incarnation cannot contribute to carry, overlay, certification or ACK after recreation." + ], + "probes": { + "designCode": "USAGE-11", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "recreation" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-12-watch-proof", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "A missing producer watch proof stays unknown and retains the strict mutation fallback.", + "A peer without watch capability cannot authorize cross-cycle reuse.", + "A mutation racing watch installation cannot establish a falsely clean interval.", + "Watch capacity overflow invalidates proof instead of retaining a clean revision.", + "An expired watch cannot prove an unchanged bucket.", + "A producer restart invalidates watches from its previous incarnation.", + "A stale or replayed watch response cannot authorize the current request.", + "An ACK ABA race cannot resurrect an old clean watch or clear newer work." + ], + "probes": { + "designCode": "USAGE-12", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": null + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-13-publication-failures", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Object and directory count stops share one drain deadline that does not restart after a later budget or lease phase.", + "A bucket quantum's remote request duration and work limits never exceed its parent's remaining limits.", + "A remote completion received during the bounded drain retains the owning cycle and reaches the publication decision.", + "Losing the lease or epoch during drain rejects publication and cannot restart the drain deadline.", + "A failed canonical root CAS cannot authorize ACK.", + "A failed companion persistence operation cannot authorize ACK or clear pending responsibility.", + "A sealed publication candidate cannot be changed by late collector results.", + "A successful durable canonical root for the still-owned epoch permits only its all-source-fresh bucket ACKs." + ], + "probes": { + "designCode": "USAGE-13", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": null + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-14-ack-races", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Duplicate, lost or reordered ACK cannot clear a newer generation or a post-walk mutation.", + "Only a durable exact canonical root permits all-source-fresh bucket ACK; pending never falls back to a whole-generation ACK." + ], + "probes": { + "designCode": "USAGE-14", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": null + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-15-structural-changes", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Movement, expansion, decommission and tier-generation changes reject obsolete source and structure identities.", + "Structural changes cannot bypass publication fences or permit stale overlay and certification." + ], + "probes": { + "designCode": "USAGE-15", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": null + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-16-mixed-version", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "An old reader cannot treat a new mixed observation as an LKG primary value.", + "Terminal, progress and watch capabilities negotiate independently and rolling rollback preserves system data and responsibility." + ], + "probes": { + "designCode": "USAGE-16", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": null + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-17-load-and-convergence", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "A configured sixty-minute sequential workload retains per-request timings and available service error codes before exact stop-write convergence.", + "Matched base and candidate load runs retain comparable throughput, latency, observed HTTP 503 counts, per-bucket freshness and binary identities." + ], + "probes": { + "designCode": "USAGE-17", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "load" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-18-diagnostic-boundaries", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "Admin samples retain collection times separately from returned per-bucket certification times and reject malformed metadata.", + "Collector delay or failure cannot invent a per-bucket measurement or certification time.", + "The admin-only companion overlay cannot flow into quota, cache or durable writer inputs." + ], + "probes": { + "designCode": "USAGE-18", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "diagnostics" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-19-quota-compatibility", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "No-quota, durable quota v1/v2 and legacy snapshot paths retain their original admission decisions.", + "Companion observations cannot expand legacy quota fallback or durable-ledger admission." + ], + "probes": { + "designCode": "USAGE-19", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": null + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + }, + { + "id": "scanner.usage-20-versions-and-prefix-keys", + "source_status": "procedure-ready", + "priority": "P1", + "channels": [ + "admin-api", + "s3-api", + "native-test" + ], + "assertions": [ + "An object key that is also a prefix and nested encoding fixtures retain every acknowledged version and payload.", + "Usage matches the independent version inventory's object, version, marker and logical-byte accounting exactly." + ], + "probes": { + "designCode": "USAGE-20", + "procedure": "docs/scanner-usage-validation.md", + "executionMode": "configured-endpoint", + "runner": "scripts/validation_scanner_usage/run_case.sh", + "publicScenario": "versions" + }, + "reuse_candidates": { + "suites": [ + "rustfs-scanner-usage-test.sh" + ], + "case_ids": [] + } + } + ] + } ] } diff --git a/validation-migration-review.json b/validation-migration-review.json index 7967f44..ea4e1f0 100644 --- a/validation-migration-review.json +++ b/validation-migration-review.json @@ -7,7 +7,7 @@ "scope": "Test implementation migration and candidate live acceptance are independent. This review does not rewrite original results, change migration_status semantics or relax the canonical gate. Environment procedures remain disabled by default under the accepted delivery scope.", "evidence_basis": "20261026-canonical-migration-ledger-7094.json plus later G01/G02 attempts and source review on 2026-10-28; historical result bytes and audit reports remain in the separate evidence store. 2026-11-10: a scoped re-review of the Rebalance activity fixture, capacity admission, Started non-zero progress, and the early-completion diagnosis; this local regression adds no product live acceptance.", "reviewed_files": { - "scripts/run_validation_case.sh": "ac51328ff860074f285999236a07ec8d2274d56607dc7c6c6389747fd7a0c498", + "scripts/run_validation_case.sh": "1c4afb02fc9cfba124239cd57f9fc8cfc40cea164d8093e5f41d25d7575aafef", "scripts/validation_heal/authority.py": "8f2225f184427674de1f45aff4c4976c812f9d70fd4bc927ce460726a4ff74a0", "scripts/validation_heal/authority_runner.py": "d1dce07019a37318f9fbfc9c4d7383fa259d8d6da881ebacf083a67d04fc2545", "scripts/validation_heal/bounded_volume.py": "56f57d2bef882af81a7bf05365fa8b5ae01f9fcdd515b3bc930e1f36cf98e944",