diff --git a/scripts/validation_scanner_usage/client.py b/scripts/validation_scanner_usage/client.py index c3fff11..b168458 100644 --- a/scripts/validation_scanner_usage/client.py +++ b/scripts/validation_scanner_usage/client.py @@ -263,4 +263,5 @@ def bucket_view(snapshot, bucket): "incarnation": entry.get("bucket_incarnation"), "metadata_available": known, "pending": bucket in pending, "observational": snapshot.get("usage_snapshot_observational") is True, "partial": snapshot.get("usage_snapshot_partial") is True, + "admin_projection": snapshot.get("usage_snapshot_admin_projection") is True, "complete": snapshot.get("usage_snapshot_complete") is True} diff --git a/scripts/validation_scanner_usage/runner.py b/scripts/validation_scanner_usage/runner.py index 2822780..c2e063b 100644 --- a/scripts/validation_scanner_usage/runner.py +++ b/scripts/validation_scanner_usage/runner.py @@ -14,6 +14,7 @@ import signal import sys import time +from uuid import UUID from .client import Client, Evidence, ProbeError, bucket_view, endpoint_url, integer, run_command, sha256, utc_now from .contracts import CASES, assertion_id, select_cases @@ -97,12 +98,13 @@ def delete(self, bucket, key): self.record_acknowledgement({"operation": "delete", "bucket": bucket, "key": key, "response": response, "acknowledged_at": utc_now()}) - def inventory(self, bucket, *, versioned=False): + def inventory(self, bucket, *, versioned=False, endpoint_index=0): require(bucket in self.owned, "refusing inventory of an unowned bucket", "INFRA_ERROR") records, continuation, seen = [], [], set() for _ in range(10000): operation = "list-object-versions" if versioned else "list-objects-v2" - value = self.client.s3(operation, "--bucket", bucket, "--max-keys", "100", "--no-paginate", *continuation) + value = self.client.s3(operation, "--bucket", bucket, "--max-keys", "100", "--no-paginate", *continuation, + endpoint_index=endpoint_index) if versioned: for field, marker in (("Versions", False), ("DeleteMarkers", True)): for row in value.get(field, []): @@ -126,7 +128,7 @@ def inventory(self, bucket, *, versioned=False): def oracle(self, bucket, *, verify_payloads=False, endpoint_index=0): versioned = self.versions[bucket] is not None - actual = self.inventory(bucket, versioned=versioned) + actual = self.inventory(bucket, versioned=versioned, endpoint_index=endpoint_index) expected = self.versions[bucket] if versioned else list(self.objects[bucket].values()) fields = ("key", "version_id", "size", "marker") if versioned else ("key", "size") normalize = lambda rows: sorted(tuple(row[field] for field in fields) for row in rows) @@ -148,6 +150,8 @@ def oracle(self, bucket, *, verify_payloads=False, endpoint_index=0): if versioned: values.update(versions=sum(not row["marker"] for row in actual), markers=sum(row["marker"] for row in actual)) path = self.evidence.save("independent-oracle", {"bucket": bucket, "versioned": versioned, + "endpoint_index": endpoint_index, + "endpoint": self.client.endpoints[endpoint_index], "inventory": actual, "expected": values, "payloads_verified": verify_payloads}) return values, path @@ -249,9 +253,13 @@ def multi_bucket(self): cold = [self.fixture.create(f"cold-{index}") for index in range(3)] self.seed(hot) targets = {} + oracles, incarnations, accepted = {}, {}, {} for bucket in cold: self.seed(bucket, 3) - targets[bucket], _ = self.fixture.oracle(bucket) + targets[bucket], oracles[bucket] = self.fixture.oracle(bucket, verify_payloads=True) + initial = self.sample(bucket) + if initial.get("incarnation") is not None: + incarnations[bucket] = initial["incarnation"] deadline = time.monotonic() + self.options.convergence_timeout done = set() operations = 0 @@ -260,18 +268,32 @@ def multi_bucket(self): operations += 1 for bucket in cold: view = self.sample(bucket) - # A cold bucket can be certified while another bucket remains - # pending. New per-bucket metadata is required for this claim. - if (view["state"] == "certified" and view["certificate"] is not None + incarnation = view.get("incarnation") + if incarnation is not None: + require(isinstance(incarnation, str), "Cold-bucket incarnation is not a UUID") + try: + valid_identity = UUID(incarnation).int != 0 + except ValueError: + valid_identity = False + require(valid_identity, "Cold-bucket incarnation is not a non-nil UUID") + require(incarnations.setdefault(bucket, incarnation) == incarnation, + "Cold-bucket incarnation changed during the hot workload") + # A carried cold value retains its original certificate. It can + # satisfy this bucket probe while the hot bucket is still pending. + if (view["state"] in {"certified", "carried"} and view["certificate"] is not None + and incarnation is not None and view.get("complete") and not view.get("pending") and not view.get("partial") and not view.get("observational") + and not view.get("admin_projection") and all(view[field] == value for field, value in targets[bucket].items())): done.add(bucket) + accepted[bucket] = view if len(done) < len(cold): time.sleep(self.options.poll_interval) require(len(done) == len(cold), "Cold-bucket certification was not observed during the hot workload", "UNSUPPORTED" if not any(row["view"].get("metadata_available") for row in self.observed) else "FAIL") self.check(0, {"cold_buckets": sorted(done), "hot_puts": operations, + "independent_oracles": oracles, "incarnations": incarnations, "accepted_values": accepted, "limits": "This public probe does not prove that a cycle budget actually expired."}) def multi_endpoint(self): diff --git a/test_validation_scanner_usage.py b/test_validation_scanner_usage.py index 387593f..c79324b 100644 --- a/test_validation_scanner_usage.py +++ b/test_validation_scanner_usage.py @@ -18,6 +18,7 @@ import time from types import SimpleNamespace import unittest +from uuid import NAMESPACE_DNS, uuid5 from unittest.mock import patch from scripts.validation_catalog import catalog_modules, validate_snapshot @@ -73,7 +74,8 @@ def __init__(self, endpoints, admin_endpoints, evidence, **_): self.malformed = False def s3(self, operation, *arguments, endpoint_index=0): - self.last_evidence = self.evidence.save("fake-s3", {"operation": operation, "arguments": list(map(str, arguments))}) + self.last_evidence = self.evidence.save("fake-s3", {"operation": operation, "arguments": list(map(str, arguments)), + "endpoint_index": endpoint_index}) def arg(name, default=None): return arguments[arguments.index(name) + 1] if name in arguments else default bucket, key = arg("--bucket"), arg("--key") @@ -127,6 +129,7 @@ def admin(self, path="/rustfs/admin/v3/datausageinfo", endpoint_index=0): value["buckets_usage"][bucket] = {"objects_count": len({row["Key"] for row in rows}), "size": sum(len(row["payload"]) for row in rows), "versions_count": sum(not row["marker"] for row in rows), "delete_markers_count": sum(row["marker"] for row in rows), + "bucket_incarnation": str(uuid5(NAMESPACE_DNS, bucket)), "usage_value_state": "observed" if self.observed else "certified", "usage_certification": None if self.observed else certificate(self.serial)} self.last_evidence = self.evidence.save("fake-admin", value) @@ -291,6 +294,124 @@ def test_minimum_fixture_still_writes_through_every_endpoint(self): puts = [row for row in acknowledged if row["operation"] == "put"] self.assertEqual({row["endpoint_index"] for row in puts}, {0, 1, 2}) self.assertEqual(len(puts), 3) + oracles = [json.loads(path.read_text()) for path in directory.glob("*-independent-oracle.json")] + self.assertEqual({(row["endpoint_index"], row["endpoint"]) for row in oracles}, set(enumerate(endpoints))) + requests = [json.loads(path.read_text()) for path in directory.glob("*-fake-s3.json")] + for operation in ("list-objects-v2", "get-object"): + self.assertEqual({row["endpoint_index"] for row in requests if row["operation"] == operation}, {0, 1, 2}) + + def test_certified_and_carried_cold_values_with_exact_oracles_can_pass(self): + for state in ("certified", "carried"): + def configure(client): + original = client.admin + def admin(*args, **kwargs): + value = original(*args, **kwargs) + value["usage_snapshot_converged"] = False + for bucket, entry in value["buckets_usage"].items(): + if "-cold-" in bucket: + entry["usage_value_state"] = state + entry["usage_certification"] = certificate(1) + return value + client.admin = admin + with self.subTest(state=state): + result = self.run_case(CASES[4], configure=configure, convergence_timeout=0.1) + self.assertEqual(result["assertions"][0]["status"], "PASS", result) + self.assertTrue(all(row["status"] == "BLOCKED" for row in result["assertions"][1:])) + + def test_oracle_propagates_endpoint_and_markers_on_every_inventory_page(self): + for versioned in (False, True): + with self.subTest(versioned=versioned), tempfile.TemporaryDirectory() as directory: + evidence = Evidence(directory) + endpoints = [f"http://endpoint-{index}.invalid:9000" for index in range(3)] + client = FakeClient(endpoints, endpoints, evidence) + fixture = Fixture(client, directory, evidence) + bucket = fixture.create("pages", versioned=versioned) + fixture.put(bucket, "first", 11) + fixture.put(bucket, "second", 17) + original = client.s3 + requests = [] + def paginated(operation, *arguments, endpoint_index=0): + if operation not in ("list-objects-v2", "list-object-versions"): + return original(operation, *arguments, endpoint_index=endpoint_index) + requests.append((endpoint_index, list(arguments))) + page = original(operation, *arguments, endpoint_index=endpoint_index) + marker = "--key-marker" if versioned else "--continuation-token" + second = marker in arguments + field = "Versions" if versioned else "Contents" + page[field] = [page[field][1 if second else 0]] + page["IsTruncated"] = not second + if not second: + if versioned: + page.update(NextKeyMarker="first", NextVersionIdMarker="1") + else: + page["NextContinuationToken"] = "opaque-token" + return page + with patch.object(client, "s3", side_effect=paginated): + for index in range(3): + values, _ = fixture.oracle(bucket, verify_payloads=True, endpoint_index=index) + self.assertEqual((values["count"], values["size"]), (2, 28)) + self.assertEqual([index for index, _ in requests], [0, 0, 1, 1, 2, 2]) + for _, arguments in requests[1::2]: + marker = "--key-marker" if versioned else "--continuation-token" + self.assertEqual(arguments[arguments.index(marker) + 1], "first" if versioned else "opaque-token") + if versioned: + self.assertEqual(arguments[arguments.index("--version-id-marker") + 1], "1") + self.assertEqual(fixture.cleanup()[0], "PASS") + + def test_one_endpoint_with_a_different_inventory_cannot_pass(self): + def configure(client): + original = client.s3 + def divergent(operation, *arguments, endpoint_index=0): + value = original(operation, *arguments, endpoint_index=endpoint_index) + if operation == "list-objects-v2" and endpoint_index == 1: + value["Contents"] = value["Contents"][:-1] + return value + client.s3 = divergent + endpoints = ["http://first.invalid:9000", "http://second.invalid:9000"] + result = self.run_case(CASES[6], configure=configure, endpoints=endpoints, admin_endpoints=endpoints) + self.assertEqual(result["assertions"][0]["status"], "FAIL", result) + self.assertIn("Independent S3 inventory", result["reason"]) + self.assertEqual(result["cleanup_status"], "PASS") + + def test_invalid_cold_values_cannot_pass_even_with_exact_counts(self): + for defect in ("certificate", "observed", "identity", "changed_identity", "count", "size", "pending", + "partial", "observational", "admin_projection", "complete", "missing_identity", "unknown"): + def configure(client): + original = client.admin + calls = 0 + def admin(*args, **kwargs): + nonlocal calls + calls += 1 + value = original(*args, **kwargs) + for bucket, entry in value["buckets_usage"].items(): + if "-cold-" not in bucket: + continue + entry["usage_value_state"] = "carried" + if defect == "certificate": + entry["usage_certification"] = None + elif defect in ("observed", "unknown"): + entry["usage_value_state"] = "observed" if defect == "observed" else "future" + entry["usage_certification"] = None + elif defect == "identity": + entry["bucket_incarnation"] = "00000000-0000-0000-0000-000000000000" + elif defect == "changed_identity" and calls > 3: + entry["bucket_incarnation"] = str(uuid5(NAMESPACE_DNS, bucket + "-successor")) + elif defect == "missing_identity": + entry.pop("bucket_incarnation") + elif defect in ("count", "size"): + entry["objects_count" if defect == "count" else "size"] += 1 + elif defect == "pending": + value.setdefault("usage_snapshot_pending_buckets", []).append(bucket) + elif defect == "complete": + value["usage_snapshot_complete"] = False + else: + value["usage_snapshot_" + defect] = True + return value + client.admin = admin + with self.subTest(defect=defect): + result = self.run_case(CASES[4], configure=configure, convergence_timeout=0.03) + self.assertEqual(result["assertions"][0]["status"], "UNSUPPORTED" if defect == "unknown" else "FAIL", result) + self.assertEqual(result["cleanup_status"], "PASS", result) def test_malformed_usage_state_preserves_results_and_allows_the_next_case(self): class MalformedStateClient(FakeClient):