Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions scripts/validation_scanner_usage/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,4 +263,5 @@ def bucket_view(snapshot, bucket):
"incarnation": entry.get("bucket_incarnation"), "metadata_available": known,
"pending": bucket in pending, "observational": snapshot.get("usage_snapshot_observational") is True,
"partial": snapshot.get("usage_snapshot_partial") is True,
"admin_projection": snapshot.get("usage_snapshot_admin_projection") is True,
"complete": snapshot.get("usage_snapshot_complete") is True}
36 changes: 29 additions & 7 deletions scripts/validation_scanner_usage/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import signal
import sys
import time
from uuid import UUID

from .client import Client, Evidence, ProbeError, bucket_view, endpoint_url, integer, run_command, sha256, utc_now
from .contracts import CASES, assertion_id, select_cases
Expand Down Expand Up @@ -97,12 +98,13 @@ def delete(self, bucket, key):
self.record_acknowledgement({"operation": "delete", "bucket": bucket, "key": key, "response": response,
"acknowledged_at": utc_now()})

def inventory(self, bucket, *, versioned=False):
def inventory(self, bucket, *, versioned=False, endpoint_index=0):
require(bucket in self.owned, "refusing inventory of an unowned bucket", "INFRA_ERROR")
records, continuation, seen = [], [], set()
for _ in range(10000):
operation = "list-object-versions" if versioned else "list-objects-v2"
value = self.client.s3(operation, "--bucket", bucket, "--max-keys", "100", "--no-paginate", *continuation)
value = self.client.s3(operation, "--bucket", bucket, "--max-keys", "100", "--no-paginate", *continuation,
endpoint_index=endpoint_index)
if versioned:
for field, marker in (("Versions", False), ("DeleteMarkers", True)):
for row in value.get(field, []):
Expand All @@ -126,7 +128,7 @@ def inventory(self, bucket, *, versioned=False):

def oracle(self, bucket, *, verify_payloads=False, endpoint_index=0):
versioned = self.versions[bucket] is not None
actual = self.inventory(bucket, versioned=versioned)
actual = self.inventory(bucket, versioned=versioned, endpoint_index=endpoint_index)
expected = self.versions[bucket] if versioned else list(self.objects[bucket].values())
fields = ("key", "version_id", "size", "marker") if versioned else ("key", "size")
normalize = lambda rows: sorted(tuple(row[field] for field in fields) for row in rows)
Expand All @@ -148,6 +150,8 @@ def oracle(self, bucket, *, verify_payloads=False, endpoint_index=0):
if versioned:
values.update(versions=sum(not row["marker"] for row in actual), markers=sum(row["marker"] for row in actual))
path = self.evidence.save("independent-oracle", {"bucket": bucket, "versioned": versioned,
"endpoint_index": endpoint_index,
"endpoint": self.client.endpoints[endpoint_index],
"inventory": actual, "expected": values,
"payloads_verified": verify_payloads})
return values, path
Expand Down Expand Up @@ -249,9 +253,13 @@ def multi_bucket(self):
cold = [self.fixture.create(f"cold-{index}") for index in range(3)]
self.seed(hot)
targets = {}
oracles, incarnations, accepted = {}, {}, {}
for bucket in cold:
self.seed(bucket, 3)
targets[bucket], _ = self.fixture.oracle(bucket)
targets[bucket], oracles[bucket] = self.fixture.oracle(bucket, verify_payloads=True)
initial = self.sample(bucket)
if initial.get("incarnation") is not None:
incarnations[bucket] = initial["incarnation"]
deadline = time.monotonic() + self.options.convergence_timeout
done = set()
operations = 0
Expand All @@ -260,18 +268,32 @@ def multi_bucket(self):
operations += 1
for bucket in cold:
view = self.sample(bucket)
# A cold bucket can be certified while another bucket remains
# pending. New per-bucket metadata is required for this claim.
if (view["state"] == "certified" and view["certificate"] is not None
incarnation = view.get("incarnation")
if incarnation is not None:
require(isinstance(incarnation, str), "Cold-bucket incarnation is not a UUID")
try:
valid_identity = UUID(incarnation).int != 0
except ValueError:
valid_identity = False
require(valid_identity, "Cold-bucket incarnation is not a non-nil UUID")
require(incarnations.setdefault(bucket, incarnation) == incarnation,
"Cold-bucket incarnation changed during the hot workload")
# A carried cold value retains its original certificate. It can
# satisfy this bucket probe while the hot bucket is still pending.
if (view["state"] in {"certified", "carried"} and view["certificate"] is not None
and incarnation is not None
and view.get("complete") and not view.get("pending")
and not view.get("partial") and not view.get("observational")
and not view.get("admin_projection")
and all(view[field] == value for field, value in targets[bucket].items())):
done.add(bucket)
accepted[bucket] = view
if len(done) < len(cold):
time.sleep(self.options.poll_interval)
require(len(done) == len(cold), "Cold-bucket certification was not observed during the hot workload",
"UNSUPPORTED" if not any(row["view"].get("metadata_available") for row in self.observed) else "FAIL")
self.check(0, {"cold_buckets": sorted(done), "hot_puts": operations,
"independent_oracles": oracles, "incarnations": incarnations, "accepted_values": accepted,
"limits": "This public probe does not prove that a cycle budget actually expired."})

def multi_endpoint(self):
Expand Down
123 changes: 122 additions & 1 deletion test_validation_scanner_usage.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import time
from types import SimpleNamespace
import unittest
from uuid import NAMESPACE_DNS, uuid5
from unittest.mock import patch

from scripts.validation_catalog import catalog_modules, validate_snapshot
Expand Down Expand Up @@ -73,7 +74,8 @@ def __init__(self, endpoints, admin_endpoints, evidence, **_):
self.malformed = False

def s3(self, operation, *arguments, endpoint_index=0):
self.last_evidence = self.evidence.save("fake-s3", {"operation": operation, "arguments": list(map(str, arguments))})
self.last_evidence = self.evidence.save("fake-s3", {"operation": operation, "arguments": list(map(str, arguments)),
"endpoint_index": endpoint_index})
def arg(name, default=None):
return arguments[arguments.index(name) + 1] if name in arguments else default
bucket, key = arg("--bucket"), arg("--key")
Expand Down Expand Up @@ -127,6 +129,7 @@ def admin(self, path="/rustfs/admin/v3/datausageinfo", endpoint_index=0):
value["buckets_usage"][bucket] = {"objects_count": len({row["Key"] for row in rows}),
"size": sum(len(row["payload"]) for row in rows), "versions_count": sum(not row["marker"] for row in rows),
"delete_markers_count": sum(row["marker"] for row in rows),
"bucket_incarnation": str(uuid5(NAMESPACE_DNS, bucket)),
"usage_value_state": "observed" if self.observed else "certified",
"usage_certification": None if self.observed else certificate(self.serial)}
self.last_evidence = self.evidence.save("fake-admin", value)
Expand Down Expand Up @@ -291,6 +294,124 @@ def test_minimum_fixture_still_writes_through_every_endpoint(self):
puts = [row for row in acknowledged if row["operation"] == "put"]
self.assertEqual({row["endpoint_index"] for row in puts}, {0, 1, 2})
self.assertEqual(len(puts), 3)
oracles = [json.loads(path.read_text()) for path in directory.glob("*-independent-oracle.json")]
self.assertEqual({(row["endpoint_index"], row["endpoint"]) for row in oracles}, set(enumerate(endpoints)))
requests = [json.loads(path.read_text()) for path in directory.glob("*-fake-s3.json")]
for operation in ("list-objects-v2", "get-object"):
self.assertEqual({row["endpoint_index"] for row in requests if row["operation"] == operation}, {0, 1, 2})

def test_certified_and_carried_cold_values_with_exact_oracles_can_pass(self):
for state in ("certified", "carried"):
def configure(client):
original = client.admin
def admin(*args, **kwargs):
value = original(*args, **kwargs)
value["usage_snapshot_converged"] = False
for bucket, entry in value["buckets_usage"].items():
if "-cold-" in bucket:
entry["usage_value_state"] = state
entry["usage_certification"] = certificate(1)
return value
client.admin = admin
with self.subTest(state=state):
result = self.run_case(CASES[4], configure=configure, convergence_timeout=0.1)
self.assertEqual(result["assertions"][0]["status"], "PASS", result)
self.assertTrue(all(row["status"] == "BLOCKED" for row in result["assertions"][1:]))

def test_oracle_propagates_endpoint_and_markers_on_every_inventory_page(self):
for versioned in (False, True):
with self.subTest(versioned=versioned), tempfile.TemporaryDirectory() as directory:
evidence = Evidence(directory)
endpoints = [f"http://endpoint-{index}.invalid:9000" for index in range(3)]
client = FakeClient(endpoints, endpoints, evidence)
fixture = Fixture(client, directory, evidence)
bucket = fixture.create("pages", versioned=versioned)
fixture.put(bucket, "first", 11)
fixture.put(bucket, "second", 17)
original = client.s3
requests = []
def paginated(operation, *arguments, endpoint_index=0):
if operation not in ("list-objects-v2", "list-object-versions"):
return original(operation, *arguments, endpoint_index=endpoint_index)
requests.append((endpoint_index, list(arguments)))
page = original(operation, *arguments, endpoint_index=endpoint_index)
marker = "--key-marker" if versioned else "--continuation-token"
second = marker in arguments
field = "Versions" if versioned else "Contents"
page[field] = [page[field][1 if second else 0]]
page["IsTruncated"] = not second
if not second:
if versioned:
page.update(NextKeyMarker="first", NextVersionIdMarker="1")
else:
page["NextContinuationToken"] = "opaque-token"
return page
with patch.object(client, "s3", side_effect=paginated):
for index in range(3):
values, _ = fixture.oracle(bucket, verify_payloads=True, endpoint_index=index)
self.assertEqual((values["count"], values["size"]), (2, 28))
self.assertEqual([index for index, _ in requests], [0, 0, 1, 1, 2, 2])
for _, arguments in requests[1::2]:
marker = "--key-marker" if versioned else "--continuation-token"
self.assertEqual(arguments[arguments.index(marker) + 1], "first" if versioned else "opaque-token")
if versioned:
self.assertEqual(arguments[arguments.index("--version-id-marker") + 1], "1")
self.assertEqual(fixture.cleanup()[0], "PASS")

def test_one_endpoint_with_a_different_inventory_cannot_pass(self):
def configure(client):
original = client.s3
def divergent(operation, *arguments, endpoint_index=0):
value = original(operation, *arguments, endpoint_index=endpoint_index)
if operation == "list-objects-v2" and endpoint_index == 1:
value["Contents"] = value["Contents"][:-1]
return value
client.s3 = divergent
endpoints = ["http://first.invalid:9000", "http://second.invalid:9000"]
result = self.run_case(CASES[6], configure=configure, endpoints=endpoints, admin_endpoints=endpoints)
self.assertEqual(result["assertions"][0]["status"], "FAIL", result)
self.assertIn("Independent S3 inventory", result["reason"])
self.assertEqual(result["cleanup_status"], "PASS")

def test_invalid_cold_values_cannot_pass_even_with_exact_counts(self):
for defect in ("certificate", "observed", "identity", "changed_identity", "count", "size", "pending",
"partial", "observational", "admin_projection", "complete", "missing_identity", "unknown"):
def configure(client):
original = client.admin
calls = 0
def admin(*args, **kwargs):
nonlocal calls
calls += 1
value = original(*args, **kwargs)
for bucket, entry in value["buckets_usage"].items():
if "-cold-" not in bucket:
continue
entry["usage_value_state"] = "carried"
if defect == "certificate":
entry["usage_certification"] = None
elif defect in ("observed", "unknown"):
entry["usage_value_state"] = "observed" if defect == "observed" else "future"
entry["usage_certification"] = None
elif defect == "identity":
entry["bucket_incarnation"] = "00000000-0000-0000-0000-000000000000"
elif defect == "changed_identity" and calls > 3:
entry["bucket_incarnation"] = str(uuid5(NAMESPACE_DNS, bucket + "-successor"))
elif defect == "missing_identity":
entry.pop("bucket_incarnation")
elif defect in ("count", "size"):
entry["objects_count" if defect == "count" else "size"] += 1
elif defect == "pending":
value.setdefault("usage_snapshot_pending_buckets", []).append(bucket)
elif defect == "complete":
value["usage_snapshot_complete"] = False
else:
value["usage_snapshot_" + defect] = True
return value
client.admin = admin
with self.subTest(defect=defect):
result = self.run_case(CASES[4], configure=configure, convergence_timeout=0.03)
self.assertEqual(result["assertions"][0]["status"], "UNSUPPORTED" if defect == "unknown" else "FAIL", result)
self.assertEqual(result["cleanup_status"], "PASS", result)

def test_malformed_usage_state_preserves_results_and_allows_the_next_case(self):
class MalformedStateClient(FakeClient):
Expand Down
Loading