diff --git a/.github/workflows/fault-tolerance-script-tests.yml b/.github/workflows/fault-tolerance-script-tests.yml index e19c94c..67c6a43 100644 --- a/.github/workflows/fault-tolerance-script-tests.yml +++ b/.github/workflows/fault-tolerance-script-tests.yml @@ -6,6 +6,12 @@ on: - rustfs-fault-tolerance-test.sh - test_rustfs_fault_tolerance.py - scripts/ft_s3_diagnostics.py + - scripts/node_replacement.py + - scripts/node_replacement_node.py + - scripts/prepare_node_replacement.py + - scripts/node-replacement.Dockerfile + - test_node_replacement.py + - .github/workflows/rustfs-node-replacement-test.yml - .github/workflows/fault-tolerance-script-tests.yml push: branches: [main] @@ -13,6 +19,12 @@ on: - rustfs-fault-tolerance-test.sh - test_rustfs_fault_tolerance.py - scripts/ft_s3_diagnostics.py + - scripts/node_replacement.py + - scripts/node_replacement_node.py + - scripts/prepare_node_replacement.py + - scripts/node-replacement.Dockerfile + - test_node_replacement.py + - .github/workflows/rustfs-node-replacement-test.yml - .github/workflows/fault-tolerance-script-tests.yml permissions: @@ -29,4 +41,4 @@ jobs: - name: Check shell syntax run: bash -n rustfs-fault-tolerance-test.sh - name: Run contract tests - run: python3 -B -m unittest -v test_rustfs_fault_tolerance.py + run: python3 -B -m unittest -v test_rustfs_fault_tolerance.py test_node_replacement.py diff --git a/.github/workflows/rustfs-node-replacement-test.yml b/.github/workflows/rustfs-node-replacement-test.yml new file mode 100644 index 0000000..f9368c1 --- /dev/null +++ b/.github/workflows/rustfs-node-replacement-test.yml @@ -0,0 +1,66 @@ +name: RustFS XFS Node Replacement Test + +on: + workflow_dispatch: + inputs: + release: + description: Official GNU release to test on mounted XFS data disks + required: true + default: '1.0.5' + type: string + drives_per_node: + description: Data disks per node in one pool + default: '4' + type: choice + options: ['4', '1'] + +concurrency: + group: rustfs-node-replacement-xfs + cancel-in-progress: false + +permissions: + contents: read + +jobs: + node-replacement: + runs-on: ubuntu-latest + timeout-minutes: 40 + env: + REPLACEMENT_RELEASE: ${{ inputs.release }} + REPLACEMENT_DRIVES: ${{ inputs.drives_per_node }} + REPLACEMENT_IMAGE: rustfs-node-replacement-xfs:ci + REPLACEMENT_ARTIFACTS: ${{ github.workspace }}/node-replacement-evidence + steps: + - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + persist-credentials: false + - name: Install S3 client + run: python3 -m pip install 'boto3>=1.35.76,<2' + - name: Prepare digest-verified official GNU binary and XFS tools + env: + GH_TOKEN: ${{ github.token }} + run: >- + python3 -B scripts/prepare_node_replacement.py + --release "${REPLACEMENT_RELEASE}" + --out "${RUNNER_TEMP}/node-replacement-build" + --tag "${REPLACEMENT_IMAGE}" + - name: Replace failed node and verify reconstructed shards + run: >- + python3 -B scripts/node_replacement.py + --image "${REPLACEMENT_IMAGE}" + --artifacts-dir "${REPLACEMENT_ARTIFACTS}" + --drives-per-node "${REPLACEMENT_DRIVES}" + - name: Recover run resources after interruption + if: ${{ always() }} + run: | + if [ -f "${REPLACEMENT_ARTIFACTS}/resources.json" ]; then + python3 -B scripts/node_replacement.py --cleanup --artifacts-dir "${REPLACEMENT_ARTIFACTS}" + fi + - name: Upload replacement evidence + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: node-replacement-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.REPLACEMENT_ARTIFACTS }} + if-no-files-found: warn + retention-days: 7 diff --git a/README.md b/README.md index 599f9d5..4dc5021 100644 --- a/README.md +++ b/README.md @@ -241,6 +241,72 @@ bash -n rustfs-fault-tolerance-test.sh python3 -B -m unittest -v test_rustfs_fault_tolerance.py ``` +## XFS node replacement regression + +`scripts/node_replacement.py` exercises the XFS data-directory layout from +backlog issue #3283. The default is four nodes with four XFS disks each, +one 16-drive pool/set and `EC:4`. `--drives-per-node 1` selects a four-drive +pool with `EC:2`. + +Each disposable container mounts independent 512 MiB XFS image files on loop +block devices at `/data/rustfs1` through `/data/rustfs4`. RustFS uses the +`mnmd` subdirectory under each mount. Both +`RUSTFS_UNSAFE_BYPASS_DISK_CHECK=false` and `MINIO_CI=false` are enforced in +every service process. Filesystem UUIDs, block device identities, mount +locations, endpoint mount status, process identities and the official binary +SHA-256 are retained in the evidence. + +The runner seeds 100 one-KiB objects and three large objects. It stops node 2, +acknowledges three additional large PUTs on the survivors, archives the stopped +node's data outside its service endpoints, and starts a fresh process with +empty `mnmd` directories on the same XFS filesystems. This reproduces the +report's directory-reset procedure while retaining its original disk evidence. + +Readability and disk recovery are independent assertions. Every replacement +disk must recover `format.json`, `.rustfs.sys/pool.bin/xl.meta`, +`.rustfs.sys/pool.bin.identity/xl.meta`, and the acknowledged objects' metadata +and external parts. A ready process with missing system metadata fails the +case. After physical repair, the runner disables healing and quarantines +survivor external parts and inline object directories. Readback of every +acknowledged object must then use replacement shards. A separate cold-start +control removes survivor copies of both system objects, witnesses new-process +metadata failures while the replacement is offline, and requires restored +replacement metadata to participate in the startup quorum. Every recovered +payload is checked against its acknowledged SHA-256. + +Run the **RustFS XFS Node Replacement Test** workflow, or build the helper +image and run locally. Preparation requires Docker, `gh`, and `curl`; the +runner requires Python and `boto3>=1.35.76,<2`. Docker must support privileged +containers, loop devices and the XFS kernel driver. + +```bash +python3 -B scripts/prepare_node_replacement.py --release 1.0.5 --out /tmp/node-replacement-build --tag rustfs-node-replacement-xfs:local +python3 -B scripts/node_replacement.py --image rustfs-node-replacement-xfs:local --artifacts-dir /tmp/node-replacement-evidence +``` + +Preparation verifies the official GNU release archive's published SHA-256 +before building the helper image. Its binary digest is rechecked before every +service start. The helper formats exclusively created, run-owned regular +image files and mounts actual XFS filesystems in private container mount +namespaces. The test uses virtual block devices on one Docker host. + +`--endpoint-layout mountpoint` adds an endpoint bind mount for a control run +with exact endpoint mountpoints. The default subdirectory layout keeps the +same mount relationship as #3283. Each run records its layout and bypass flags; +a control run has its own result and evidence directory. + +Artifacts include PUT acknowledgments, health responses, independent system +metadata and object-shard results, stopped physical archives, service logs, +filesystem identities and `result.json`. An unrepaired replacement retains a +FAIL even when S3 reads succeed through surviving nodes. Cleanup stops owned +services, unmounts their private XFS filesystems, and removes the run's labeled +resources. An interrupted run can be recovered from its saved manifest. + +```bash +python3 -B scripts/node_replacement.py --cleanup --artifacts-dir /tmp/node-replacement-evidence +python3 -B -m unittest -v test_node_replacement.py +``` + ## Security suite diagnostics `rustfs-security-test.sh` includes STS-105 request timing, the full revoke diff --git a/scripts/node-replacement.Dockerfile b/scripts/node-replacement.Dockerfile new file mode 100644 index 0000000..7d17d87 --- /dev/null +++ b/scripts/node-replacement.Dockerfile @@ -0,0 +1,7 @@ +FROM debian:bookworm-slim +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 xfsprogs util-linux ca-certificates && rm -rf /var/lib/apt/lists/* +COPY rustfs /usr/bin/rustfs +COPY candidate.json /opt/node_replacement_candidate.json +COPY node_replacement_node.py /opt/node_replacement_node.py +ENTRYPOINT ["python3", "/opt/node_replacement_node.py", "serve"] diff --git a/scripts/node_replacement.py b/scripts/node_replacement.py new file mode 100644 index 0000000..e32946e --- /dev/null +++ b/scripts/node_replacement.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""Rebuild a member of a real XFS-backed four-node pool with disk bypass disabled.""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import os +from pathlib import Path, PurePosixPath +import secrets +import signal +import subprocess +import tarfile +import time +from urllib.error import HTTPError, URLError +from urllib.request import ProxyHandler, build_opener + +try: + from .validation_heal.heal_probe import payload, require, save +except ImportError: + from validation_heal.heal_probe import payload, require, save + + +LABEL = 'org.rustfs.auto-testing.node-replacement' +SIZES = (1024 * 1024 + 47, 3 * 1024 * 1024 + 19, 8 * 1024 * 1024 + 7) +SYSTEM_KEYS = ('pool.bin', 'pool.bin.identity') +ARCHIVE_LIMIT = 128 * 1024 * 1024 +NODE_TOOL = '/opt/node_replacement_node.py' + + +def archive_files(raw): + require(len(raw) <= ARCHIVE_LIMIT, 'physical archive exceeds the fixture budget') + result = {} + with tarfile.open(fileobj=io.BytesIO(raw)) as archive: + for entry in archive: + path = PurePosixPath(entry.name) + require(not path.is_absolute() and '..' not in path.parts, 'unsafe physical archive path') + if entry.isdir(): + continue + require(entry.isfile() and entry.size <= ARCHIVE_LIMIT, 'unexpected physical file type or size') + name = str(path) + require(name not in result, 'duplicate physical file') + result[name] = archive.extractfile(entry).read() + return result + + +def file_manifest(files): + return {name: {'size': len(data), 'sha256': hashlib.sha256(data).hexdigest()} + for name, data in sorted(files.items())} + + +def topology(disks, *, original=None, replaced_disks=None): + require(len(disks) in (4, 16), 'expected a four-node 4x1 or 4x4 pool') + formats = [json.loads(files['.rustfs.sys/format.json']) for files in disks] + sets = formats[0]['xl']['sets'] + require(len(sets) == 1 and len(sets[0]) == len(set(sets[0])) == len(disks), + 'expected one complete cross-node erasure set') + require(all(item['format'] == 'xl' and item['id'] == formats[0]['id'] and item['xl']['sets'] == sets + for item in formats), 'pool or set membership differs') + members = [item['xl']['this'] for item in formats] + require(len(set(members)) == len(disks) and set(members) == set(sets[0]), 'incomplete physical membership') + if original is not None: + require(formats[0]['id'] == original['poolId'], 'replacement changed the pool identity') + replaced_disks = {len(disks) - 1} if replaced_disks is None else set(replaced_disks) + require(all(member == original['members'][index] for index, member in enumerate(members) + if index not in replaced_disks), 'replacement changed a surviving disk identity') + require(all(sets[0][original['set'].index(old)] == members[index] + for index, old in enumerate(original['members'])), 'replacement moved another disk set slot') + # Per-slot identity changes are recorded; a fresh process and the empty + # endpoint census establish replacement even when the slot UUID is inherited. + return {'poolId': formats[0]['id'], 'set': sets[0], 'members': members, + 'nodes': 4, 'drivesPerNode': len(disks) // 4} + + +def system_metadata_manifest(files): + missing = [] + if not files.get('.rustfs.sys/format.json'): + missing.append('format.json') + result = {} + for key in SYSTEM_KEYS: + path = f'.rustfs.sys/{key}/xl.meta' + metadata = files.get(path, b'') + if metadata.startswith(b'XL2 ') and len(metadata) > 32: + result[key] = file_manifest({path: metadata})[path] + else: + missing.append(key + '/xl.meta') + require(not missing, 'replacement system metadata is missing or invalid: ' + ', '.join(missing)) + return result + + +def shard_manifest(files, bucket, objects): + result = {} + for key in objects: + prefix = f'{bucket}/{key}/' + metadata = files.get(prefix + 'xl.meta', b'') + parts = [name for name in files if name.startswith(prefix) and name.endswith('/part.1')] + require(metadata and len(parts) == 1, f'missing or ambiguous replacement metadata/part: {key}') + part = parts[0] + require(len(PurePosixPath(part).relative_to(prefix).parts) == 2 and len(files[part]) > 0, + f'invalid external shard: {key}') + result[key] = {'path': part, **file_manifest({part: files[part]})[part], + 'metadataSha256': hashlib.sha256(metadata).hexdigest()} + return result + + +def reconstruction_sources(drives, parity): + data = 4 * drives - parity + keep = (data + drives - 1) // drives - 1 + require(1 <= keep < 3 and keep * drives < data <= (keep + 1) * drives, + 'unsupported reconstruction quorum geometry') + return keep + + +class Case: + def __init__(self, image, artifacts, timeout, drives=4, target=2, layout='subdirectory'): + self.image_ref, self.artifacts, self.timeout = image, artifacts, timeout + self.drives, self.target, self.layout = drives, target - 1, layout + self.parity = 2 if drives == 1 else 4 + self.run_id = 'nr-' + secrets.token_hex(6) + self.network = self.run_id + self.containers, self.volumes = [], [] + self.access, self.secret = secrets.token_hex(12), secrets.token_hex(24) + self.bucket = 'node-replacement-' + self.run_id + self.objects = {} + self.sequence, self.image = 0, None + self.env_file = artifacts / '.runtime.env' + self.assertions = {} + self.phase = 'preparation' + + def record(self, label, value): + self.sequence += 1 + save(self.artifacts / f'{self.sequence:04d}-{label}.json', value) + if label not in {'put-ack', 'degraded-read', 'recovered-read', 'replacement-required-read'}: + print(label, flush=True) + return value + + def docker(self, *arguments, timeout=90, binary=False, input=None, live_archive=False): + deadline = time.monotonic() + timeout + while True: + result = subprocess.run(['docker', *arguments], capture_output=True, timeout=timeout, input=input) + if not result.returncode: + return result.stdout if binary else result.stdout.decode().strip() + message = result.stderr.decode(errors='replace') + lines = [line for line in message.splitlines() if line] + changing = lines and all(line.endswith(('file changed as we read it', 'File removed before we read it')) for line in lines) + if live_archive and changing and time.monotonic() < deadline: + time.sleep(.2) + continue + for value in (self.access, self.secret): + message = message.replace(value, '[REDACTED]') + raise RuntimeError(f'docker {arguments[0]} failed: {message[:2000]}') + + def inspect(self, kind, name): + return json.loads(self.docker(kind, 'inspect', name))[0] + + def node(self, name, action, **params): + return json.loads(self.docker('exec', '-i', name, 'python3', NODE_TOOL, action, + input=json.dumps(params).encode(), timeout=60)) + + def resources(self): + value = {'runId': self.run_id, 'network': self.network, 'containers': self.containers, 'volumes': self.volumes} + path = self.artifacts / 'resources.tmp' + path.write_text(json.dumps(value, indent=2) + '\n') + path.replace(self.artifacts / 'resources.json') + + def create(self, slot): + volume = f'{self.run_id}-backing{slot}' + self.volumes.append(volume) + self.resources() + self.docker('volume', 'create', '--label', f'{LABEL}={self.run_id}', volume) + name = f'{self.run_id}-node{slot}' + self.containers.append(name) + self.resources() + self.docker('create', '--platform', self.platform, '--name', name, '--privileged', '--label', f'{LABEL}={self.run_id}', + '--hostname', f'node{slot}', '--network', self.network, '--network-alias', f'node{slot}', + '--publish', '127.0.0.1::9000', '--mount', f'type=volume,source={volume},target=/backing,volume-nocopy', + '--env-file', str(self.env_file), '--env', f'NR_DRIVES={self.drives}', + '--env', f'NR_SLOT={slot}', '--env', f'NR_RUN_ID={self.run_id}', '--env', f'NR_LAYOUT={self.layout}', + '--entrypoint', 'python3', self.image, NODE_TOOL, 'serve', timeout=180) + require(self.inspect('container', name)['Image'] == self.image, 'candidate image changed') + self.docker('start', name) + return name + + def endpoint(self, name): + info = self.inspect('container', name) + require(info['State']['Running'], 'XFS test container exited; see bootstrap log') + ports = info['NetworkSettings']['Ports']['9000/tcp'] + require(len(ports) == 1 and ports[0]['HostIp'] == '127.0.0.1', 'unexpected published endpoint') + return 'http://127.0.0.1:' + ports[0]['HostPort'] + + def wait_ready(self, names, *, writable=False): + opener = build_opener(ProxyHandler({})) + deadline = time.monotonic() + self.timeout + endpoints = [self.endpoint(name) for name in names] + while True: + require(all(self.inspect('container', name)['State']['Running'] for name in names), + 'an XFS test container exited before readiness') + try: + for endpoint in endpoints: + with opener.open(endpoint + '/health/ready', timeout=3) as response: + require(response.status == 200, 'node is not ready') + if writable: + with opener.open(endpoint + '/minio/health/cluster', timeout=3) as response: + require(response.status == 200 and json.load(response).get('ready') is True, + 'cluster has not admitted writes') + return + except (OSError, ValueError, URLError): + require(time.monotonic() < deadline, 'node readiness deadline exceeded') + time.sleep(1) + + def s3(self, name): + import boto3 + from botocore.config import Config + return boto3.client('s3', endpoint_url=self.endpoint(name), region_name='us-east-1', + aws_access_key_id=self.access, aws_secret_access_key=self.secret, + config=Config(s3={'addressing_style': 'path'}, proxies={}, connect_timeout=3, read_timeout=15, + retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', + response_checksum_validation='when_required')) + + def write(self, name, phase, *, small=0): + client = self.s3(name) + sizes = [1024] * small + list(SIZES) + for index, size in enumerate(sizes): + key = f'{phase}-{index}.bin' + body = payload(self.run_id + key, size) + response = client.put_object(Bucket=self.bucket, Key=key, Body=body) + require(response['ResponseMetadata']['HTTPStatusCode'] == 200, 'PUT was not acknowledged') + self.objects[key] = {'size': size, 'sha256': hashlib.sha256(body).hexdigest(), 'phase': phase} + self.record('put-ack', {'key': key, **self.objects[key], + 'requestId': response['ResponseMetadata'].get('RequestId')}) + + def read(self, names, label, *, objects=None): + for name in names: + client = self.s3(name) + for key, expected in (self.objects if objects is None else objects).items(): + response = client.get_object(Bucket=self.bucket, Key=key) + with response['Body'] as stream: + body = stream.read() + require(len(body) == expected['size'] and hashlib.sha256(body).hexdigest() == expected['sha256'], + f'acknowledged object content differs: {key}') + self.record(label, {'node': name, 'key': key, **expected, 'contentVerified': True}) + + def stop(self, names): + for name in names: + state = self.node(name, 'stop') + require(state['running'] is False, 'RustFS process did not exit') + + def start(self, names, *, repair=False): + for name in names: + state = self.node(name, 'start', repair=repair) + require(state['unsafeBypassDiskCheck'] is False and state['minioCI'] is False, + 'disk-check bypass was enabled') + observed = state['observedEnvironment'] + require(observed.get('RUSTFS_UNSAFE_BYPASS_DISK_CHECK') == 'false' and observed.get('MINIO_CI') == 'false', + 'the actual service process enabled disk-check bypass') + + def read_without_replacement(self, name, *, objects=None): + client = self.s3(name) + for key in (self.objects if objects is None else objects): + try: + response = client.get_object(Bucket=self.bucket, Key=key) + except Exception as exc: + error = getattr(exc, 'response', {}) + code = error.get('Error', {}).get('Code') + require(error.get('ResponseMetadata', {}).get('HTTPStatusCode') == 503 + and code in {'SlowDownRead', 'InsufficientReadQuorum', 'XMinioReadQuorum'}, + f'negative reconstruction control did not reach a read-quorum failure: {code}') + self.record('without-replacement-read', {'key': key, 'httpStatus': 503, 'code': code}) + else: + response['Body'].close() + raise ValueError('GET succeeded without enough surviving shards; reconstruction oracle is invalid') + + def scenes(self, name, *, view='data'): + directory = 'mnmd' if view == 'data' else 'retired' + return [archive_files(self.docker('exec', name, 'tar', '--exclude=./.rustfs.sys/tmp', + '-C', f'/data/rustfs{disk}/{directory}', '-cf', '-', '.', binary=True, live_archive=True)) + for disk in range(1, self.drives + 1)] + + def snapshot(self, name, label, *, view='data', quiescent=True): + state = self.node(name, 'state') + require(not quiescent or state['running'] is False, 'snapshot requires a stopped RustFS process') + directory = 'mnmd' if view == 'data' else 'retired' + raw = [self.docker('exec', name, 'tar', *(['--exclude=./.rustfs.sys/tmp'] if not quiescent else []), + '-C', f'/data/rustfs{disk}/{directory}', '-cf', '-', '.', binary=True, live_archive=not quiescent) + for disk in range(1, self.drives + 1)] + scenes = [archive_files(value) for value in raw] + self.record(label, {'node': name, 'view': view, 'process': state, 'disks': [file_manifest(value) for value in scenes]}) + for disk, value in enumerate(raw, 1): + (self.artifacts / f'{self.sequence:04d}-{label}-disk{disk}.tar').write_bytes(value) + return scenes + + def wait_repair(self, name, large, reference=None): + deadline, previous, last = time.monotonic() + self.timeout, None, None + while True: + scenes = self.scenes(name) + observed, missing = [], [] + for disk, files in enumerate(scenes, 1): + checks = {} + for label, check in (('systemMetadata', lambda: system_metadata_manifest(files)), + ('objectShards', lambda: shard_manifest(files, self.bucket, large))): + try: + checks[label] = check() + if label == 'objectShards' and reference is not None: + require(all(row['path'] == reference[disk-1][key]['path'] + and row['size'] == reference[disk-1][key]['size'] + for key, row in checks[label].items()), 'replacement shard geometry is incomplete') + except (KeyError, ValueError) as exc: + checks.pop(label, None) + missing.append({'disk': disk, 'assertion': label, 'error': str(exc)}) + observed.append(checks) + if not missing and observed == previous: + self.assertions.update(systemMetadataRecovered=True, replacementShardsPhysicallyPresent=True) + self.record('automatic-repair', {'disks': observed}) + return + previous = observed if not missing else None + if missing != last: + self.record('repair-missing', {'missing': missing}) + last = missing + if time.monotonic() >= deadline: + self.assertions.update(systemMetadataRecovered=all('systemMetadata' in row for row in observed), + replacementShardsPhysicallyPresent=all('objectShards' in row for row in observed)) + self.record('repair-deadline', {'missing': missing, 'disks': observed}) + raise ValueError('automatic replacement repair deadline exceeded; see repair-deadline evidence') + time.sleep(2) + + def health_snapshot(self, names, label): + opener = build_opener(ProxyHandler({})) + for name in names: + for path in ('/health/ready', '/minio/health/cluster'): + try: + with opener.open(self.endpoint(name) + path, timeout=3) as response: + body = response.read(65536).decode(errors='replace') + self.record(label, {'node': name, 'path': path, 'httpStatus': response.status, 'body': body}) + except (URLError, OSError) as exc: + self.record(label, {'node': name, 'path': path, 'error': str(exc)}) + + def forced_reconstruction(self, names, large, repaired): + replacement = names[self.target] + survivors = [name for name in names if name != replacement] + keep = reconstruction_sources(self.drives, self.parity) + quarantined = survivors[keep:] + replacement_proofs = [shard_manifest(files, self.bucket, large) for files in repaired[self.target]] + small = {key: value for key, value in self.objects.items() if value['size'] <= 1024} + for name in quarantined: + proofs = [shard_manifest(files, self.bucket, large) for files in repaired[names.index(name)]] + self.node(name, 'quarantine', paths=[[str(PurePosixPath(row['path']).parent) for row in disk.values()] + + [f'{self.bucket}/{key}' for key in small] + for disk in proofs]) + self.start(survivors) + self.wait_ready(survivors) + self.read_without_replacement(survivors[0], objects=large) + self.start([replacement]) + self.wait_ready(names) + self.read(names, 'replacement-required-read') + self.stop(names) + for name in quarantined: + scenes = self.snapshot(name, 'quarantined-source-after-reads') + original = repaired[names.index(name)] + for files, old in zip(scenes, original, strict=True): + proof = shard_manifest(old, self.bucket, large) + require(all(row['path'] not in files for row in proof.values()), 'quarantined source was repaired during verification') + require(all(f'{self.bucket}/{key}/xl.meta' not in files for key in small), + 'quarantined inline metadata was repaired during verification') + final = self.snapshot(replacement, 'replacement-after-forced-reads') + require([shard_manifest(files, self.bucket, large) for files in final] == replacement_proofs, + 'replacement shards changed during reconstruction verification') + for before, after in zip(repaired[self.target], final, strict=True): + require(all(before[f'{self.bucket}/{key}/xl.meta'] == after[f'{self.bucket}/{key}/xl.meta'] for key in small), + 'replacement inline data changed during reconstruction verification') + self.assertions['replacementRequiredForReconstruction'] = True + # Force a subsequent cold start to read both pool metadata objects from + # a quorum that includes the replacement's restored system shards. + # The system metadata writer uses balanced data/parity, independently + # of the bucket's EC:4 object policy. Keep one old node's system shards. + for name in survivors[1:]: + self.node(name, 'quarantine', paths=[[f'.rustfs.sys/{key}' for key in SYSTEM_KEYS]] * self.drives) + cursors = {name: self.node(name, 'log-cursors') for name in survivors} + self.start(survivors) + opener = build_opener(ProxyHandler({})) + deadline = time.monotonic() + self.timeout + while True: + failures = {} + for name in survivors: + try: + with opener.open(self.endpoint(name) + '/health/ready', timeout=3) as response: + require(response.status != 200, 'cold-start pool metadata control unexpectedly became ready') + except (HTTPError, URLError, OSError): + pass + failures[name] = self.node(name, 'metadata-failure', cursors=cursors[name]) + if all(value['observed'] for value in failures.values()): + break + require(time.monotonic() < deadline, 'cold-start metadata failure was not witnessed in the new processes') + time.sleep(1) + self.record('pool-metadata-negative-control', {'replacementOffline': True, 'noSurvivorReady': True, 'failures': failures}) + self.stop(survivors) + self.start(names) + self.wait_ready(names) + self.read(names, 'recovered-read', objects=large) + self.assertions['replacementPoolMetadataRequiredForColdStart'] = True + self.stop(names) + + def run(self): + info = self.inspect('image', self.image_ref) + self.image = info['Id'] + self.platform = 'linux/' + info['Architecture'] + self.record('candidate-image', {'imageRef': self.image_ref, 'imageId': self.image, 'architecture': info['Architecture']}) + descriptor = os.open(self.env_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, 'w') as output: + config = {'RUSTFS_ACCESS_KEY': self.access, 'RUSTFS_SECRET_KEY': self.secret, + 'RUSTFS_ADDRESS': ':9000', 'RUSTFS_CONSOLE_ENABLE': 'false', + 'RUSTFS_STORAGE_CLASS_STANDARD': f'EC:{self.parity}', 'RUSTFS_ERASURE_SET_DRIVE_COUNT': str(4 * self.drives), + 'RUSTFS_SCANNER_ENABLED': 'true', 'RUSTFS_HEAL_INTERVAL_SECS': '5', + 'RUSTFS_UNSAFE_BYPASS_DISK_CHECK': 'false', 'MINIO_CI': 'false', + 'RUSTFS_OBS_LOGGER_LEVEL': 'info', 'RUST_LOG': 'info', 'RUSTFS_OBS_LOG_DIRECTORY': '/backing/logs'} + output.write('\n'.join(f'{key}={value}' for key, value in config.items()) + '\n') + self.resources() + self.docker('network', 'create', '--label', f'{LABEL}={self.run_id}', self.network) + names = [self.create(slot) for slot in range(1, 5)] + self.wait_ready(names, writable=True) + mounts = {name: self.record('xfs-mounts', {'node': name, 'mounts': self.node(name, 'mounts')})['mounts'] for name in names} + require(len({row['uuid'] for rows in mounts.values() for row in rows}) == self.drives * 4, + 'XFS filesystem identities are shared or missing') + self.record('candidate-binary', self.node(names[0], 'identity')) + for name in names: + state = self.record('runtime', {'node': name, **self.node(name, 'state')}) + observed = state['observedEnvironment'] + require(observed.get('RUSTFS_UNSAFE_BYPASS_DISK_CHECK') == 'false' and observed.get('MINIO_CI') == 'false', + 'the actual service process enabled disk-check bypass') + self.phase = 'baseline' + self.s3(names[0]).create_bucket(Bucket=self.bucket) + self.write(names[0], 'before-outage', small=100) + replacement = names[self.target] + survivors = [name for name in names if name != replacement] + original_process = self.node(replacement, 'state') + self.stop([replacement]) + self.phase = 'outage' + before = [self.snapshot(name, 'before-outage', quiescent=name == replacement) for name in names] + layout = self.record('initial-layout', topology([disk for node in before for disk in node])) + for disk in before[self.target]: + system_metadata_manifest(disk) + self.wait_ready(survivors, writable=True) + self.write(survivors[0], 'during-outage') + self.read(survivors, 'degraded-read') + large = {key: value for key, value in self.objects.items() if value['size'] > 1024} + reference = [shard_manifest(files, self.bucket, large) for files in self.scenes(survivors[0])] + frozen = self.snapshot(replacement, 'retired-before-reset') + require([file_manifest(files) for files in frozen] == [file_manifest(files) for files in before[self.target]], + 'offline endpoint changed during degraded writes') + empty = self.record('replacement-empty', self.node(replacement, 'reset')) + require(empty['empty'] is True and empty['mounts'] == mounts[replacement], 'replacement changed XFS mounts or retained data') + require(all(not files for files in self.scenes(replacement)), 'replacement endpoints are not empty') + self.start([replacement], repair=True) + self.phase = 'rejoin' + new_process = self.record('replacement-process', self.node(replacement, 'state')) + require((new_process['pid'], new_process['startTicks']) != (original_process['pid'], original_process['startTicks']), + 'replacement did not start a new process') + # Health probes do not read the fixture. Physical repair is decided + # before any recovered-object GET could trigger read repair. + self.wait_ready(names) + self.health_snapshot(names, 'rejoined-health') + self.phase = 'repair' + try: + self.wait_repair(replacement, large, reference) + except ValueError: + # Preserve the failed physical verdict even if survivor-backed + # reads subsequently succeed, as observed in backlog #3283. + try: + self.read(names, 'recovered-read') + self.assertions['allNodeContentVerified'] = True + except Exception as exc: + self.record('post-failure-read-error', {'type': type(exc).__name__}) + raise + self.read(names, 'recovered-read') + self.assertions['allNodeContentVerified'] = True + self.stop(names) + repaired = [self.snapshot(name, 'repaired-physical') for name in names] + current = self.record('replacement-layout', topology([disk for node in repaired for disk in node], original=layout, + replaced_disks=range(self.target*self.drives, (self.target+1)*self.drives))) + for node, (old, new) in enumerate(zip(before, repaired, strict=True)): + if node == self.target: + continue + require(current['members'][node*self.drives:(node+1)*self.drives] + == layout['members'][node*self.drives:(node+1)*self.drives], 'a surviving disk identity changed') + self.record('replacement-pool-metadata', {'disks': [system_metadata_manifest(files) for files in repaired[self.target]]}) + before_large = {key: value for key, value in large.items() if value['phase'] == 'before-outage'} + for old, files in zip(before[self.target], repaired[self.target], strict=True): + require(all(files.get(f'{self.bucket}/{key}/xl.meta') for key in self.objects), 'replacement omitted inline object metadata') + original_parts = shard_manifest(old, self.bucket, before_large) + new_parts = shard_manifest(files, self.bucket, before_large) + require(all(original_parts[key]['sha256'] == row['sha256'] for key, row in new_parts.items()), + 'replacement changed pre-outage same-slot shard bytes') + retired = self.snapshot(replacement, 'retired-after-repair', view='retired') + require([file_manifest(files) for files in retired] == [file_manifest(files) for files in frozen], 'retired data changed during repair') + self.phase = 'forced-reconstruction' + self.forced_reconstruction(names, large, repaired) + return {'topology': f'4x{self.drives}', 'poolCount': 1, 'parity': self.parity, + 'endpointLayout': self.layout, 'unsafeBypassDiskCheck': False, 'minioCI': False, + 'storage': 'independent loop-backed XFS filesystems', 'objects': self.objects, + 'newProcessAndEmptyRoots': True, **self.assertions} + + +def cleanup(artifacts): + path = artifacts / 'resources.json' + if not path.exists(): + return [] + resources = json.loads(path.read_text()) + failures = [] + for kind, names in (('container', reversed(resources['containers'])), + ('volume', reversed(resources['volumes'])), ('network', [resources['network']])): + for name in names: + result = subprocess.run(['docker', kind, 'inspect', name], capture_output=True, text=True, timeout=30) + if result.returncode: + missing = {'container': f'no such container: {name}', 'volume': f'get {name}: no such volume', + 'network': f'network {name} not found'} + if missing[kind] in result.stderr.lower(): + continue + failures.append({'kind': kind, 'name': name, 'error': 'inspect failed'}) + continue + info = json.loads(result.stdout)[0] + labels = info.get('Labels') if kind != 'container' else info.get('Config', {}).get('Labels') + if (labels or {}).get(LABEL) != resources['runId']: + failures.append({'kind': kind, 'name': name, 'error': 'ownership label differs'}) + continue + if kind == 'container': + subprocess.run(['docker', 'stop', '--time', '45', name], capture_output=True, timeout=60) + removed = subprocess.run(['docker', kind, 'rm', *(['--force'] if kind == 'container' else []), name], + capture_output=True, text=True, timeout=90) + if removed.returncode: + failures.append({'kind': kind, 'name': name, 'error': 'removal failed'}) + (artifacts / '.runtime.env').unlink(missing_ok=True) + return failures + + +def finish(case, result): + for name in case.containers: + try: + captured = subprocess.run(['docker', 'logs', '--tail', '1000', name], capture_output=True, timeout=30) + text = (captured.stdout + captured.stderr).decode(errors='replace') + text = text.replace(case.access, '[REDACTED]').replace(case.secret, '[REDACTED]') + (case.artifacts / (name + '-bootstrap.log')).write_text(text) + except Exception as exc: + case.record('bootstrap-log-error', {'node': name, 'error': type(exc).__name__}) + try: + logs = case.docker('cp', name + ':/backing/stdout.log', '-', binary=True) + for path, data in archive_files(logs).items(): + text = data.decode(errors='replace').replace(case.access, '[REDACTED]').replace(case.secret, '[REDACTED]') + (case.artifacts / (name + '.log')).write_text(text) + log_directory = case.docker('cp', name + ':/backing/logs/.', '-', binary=True) + for index, data in enumerate(archive_files(log_directory).values()): + text = data.decode(errors='replace').replace(case.access, '[REDACTED]').replace(case.secret, '[REDACTED]') + (case.artifacts / (name + f'-file-{index}.log')).write_text(text) + except Exception as exc: + case.record('service-log-error', {'node': name, 'error': type(exc).__name__}) + try: + case.stop([name]) + scenes = case.snapshot(name, 'final-physical') + if case.containers.index(name) == case.target: + missing = [] + for disk, files in enumerate(scenes, 1): + try: + system_metadata_manifest(files) + except ValueError as exc: + missing.append({'disk': disk, 'error': str(exc)}) + case.assertions.setdefault('systemMetadataRecovered', not missing) + for key in SYSTEM_KEYS: + present = all(files.get(f'.rustfs.sys/{key}/xl.meta', b'').startswith(b'XL2 ') + and len(files.get(f'.rustfs.sys/{key}/xl.meta', b'')) > 32 for files in scenes) + field = {'pool.bin': 'poolBinRecovered', 'pool.bin.identity': 'poolIdentityRecovered'}[key] + case.assertions[field] = present + case.record('final-system-metadata', {'missing': missing}) + case.record('final-mounts', {'node': name, 'mounts': case.node(name, 'mounts')}) + except Exception as exc: + case.record('final-capture-error', {'node': name, 'error': str(exc)[:500]}) + try: + failures = cleanup(case.artifacts) + result.update(cleanup='FAIL' if failures else 'PASS', cleanupFailures=failures) + except Exception as exc: + result.update(cleanup='FAIL', cleanupError=type(exc).__name__) + case.env_file.unlink(missing_ok=True) + result['assertions'] = case.assertions + save(case.artifacts / 'result.json', result) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--image', help='image built with scripts/prepare_node_replacement.py') + parser.add_argument('--artifacts-dir', type=Path, required=True) + parser.add_argument('--timeout', type=int, default=300) + parser.add_argument('--drives-per-node', type=int, choices=(1, 4), default=4) + parser.add_argument('--target-node', type=int, choices=(1, 2, 3, 4), default=2) + parser.add_argument('--endpoint-layout', choices=('subdirectory', 'mountpoint'), default='subdirectory') + parser.add_argument('--cleanup', action='store_true') + args = parser.parse_args() + if args.cleanup: + failures = cleanup(args.artifacts_dir) + print(json.dumps({'cleanup': 'FAIL' if failures else 'PASS', 'failures': failures})) + return bool(failures) + require(args.image and 10 <= args.timeout <= 1800, 'image and a 10..1800 second timeout are required') + args.artifacts_dir.mkdir(parents=True, exist_ok=False) + case = Case(args.image, args.artifacts_dir.resolve(), args.timeout, args.drives_per_node, args.target_node, args.endpoint_layout) + result = {'caseId': 'node-replacement-xfs', 'verdict': 'RUNNING', 'cleanup': 'PENDING', + 'topology': f'4x{args.drives_per_node}', 'targetNode': args.target_node, + 'storage': 'independent loop-backed XFS filesystems', + 'endpointLayout': args.endpoint_layout, 'unsafeBypassDiskCheck': False, 'minioCI': False} + def interrupted(signum, frame): + raise KeyboardInterrupt(f'signal {signum}') + signal.signal(signal.SIGTERM, interrupted) + try: + result.update(case.run(), verdict='PASS') + except (Exception, KeyboardInterrupt) as exc: + message = str(exc).replace(case.access, '[REDACTED]').replace(case.secret, '[REDACTED]') + infrastructure = isinstance(exc, (RuntimeError, OSError, subprocess.SubprocessError)) + verdict = 'INFRA_ERROR' if case.phase == 'preparation' or infrastructure else 'FAIL' + result.update(verdict=verdict, phase=case.phase, error={'type': type(exc).__name__, 'message': message[:2000]}) + finally: + finish(case, result) + print(json.dumps({key: value for key, value in result.items() if key != 'objects'}, indent=2)) + return result['verdict'] != 'PASS' or result['cleanup'] != 'PASS' + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/node_replacement_node.py b/scripts/node_replacement_node.py new file mode 100644 index 0000000..855dd49 --- /dev/null +++ b/scripts/node_replacement_node.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Own disposable XFS image mounts and one RustFS process inside a test container.""" + +import hashlib +import json +import os +from pathlib import Path +import signal +import stat +import subprocess +import sys +import time + + +BACKING = Path('/backing') +STATE = BACKING / 'process.json' + + +def command(*args): + result = subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if result.returncode: + raise RuntimeError(f'{args[0]} failed: {result.stdout[:2000]}') + return result.stdout.strip() + + +def roots(): + return [Path(f'/data/rustfs{index}') for index in range(1, int(os.environ['NR_DRIVES']) + 1)] + + +def data_roots(view='data'): + return [root / ('mnmd' if view == 'data' else 'retired') for root in roots()] + + +def process(): + state = json.loads(STATE.read_text()) if STATE.exists() else {'pid': 0, 'startTicks': None} + path = Path(f"/proc/{state['pid']}/stat") + try: + fields = path.read_text().rsplit(')', 1)[1].split() + running = fields[0] != 'Z' and fields[19] == state['startTicks'] + except FileNotFoundError: + running = False + observed = {} + if running: + try: + raw = Path(f"/proc/{state['pid']}/environ").read_bytes().decode().split('\0') + environment = dict(item.split('=', 1) for item in raw if '=' in item) + observed = {key: environment.get(key) for key in ('RUSTFS_UNSAFE_BYPASS_DISK_CHECK', 'MINIO_CI', + 'RUSTFS_SCANNER_ENABLED', 'RUSTFS_HEAL_ENABLED', 'RUSTFS_HEAL_AUTO_HEAL_ENABLE', 'RUSTFS_HEAL_MRF_ENABLE')} + except (FileNotFoundError, ProcessLookupError): + running = False + return {**state, 'running': running, 'observedEnvironment': observed} + + +def stop(): + old = process() + if old['running']: + os.kill(old['pid'], signal.SIGTERM) + deadline = time.monotonic() + 30 + while process()['running'] and time.monotonic() < deadline: + time.sleep(.1) + if process()['running']: + os.kill(old['pid'], signal.SIGKILL) + deadline = time.monotonic() + 5 + while process()['running'] and time.monotonic() < deadline: + time.sleep(.1) + if process()['running']: + raise RuntimeError('owned RustFS process did not exit') + return process() + + +def start(repair=True): + if process()['running']: + raise RuntimeError('owned RustFS process is already running') + candidate = json.loads(Path('/opt/node_replacement_candidate.json').read_text()) + actual = hashlib.sha256(Path('/usr/bin/rustfs').read_bytes()).hexdigest() + if actual != candidate['binarySha256']: + raise RuntimeError('runtime binary differs from the verified official release archive') + environment = {**os.environ, 'RUSTFS_HEAL_ENABLED': str(repair).lower(), + 'RUSTFS_HEAL_AUTO_HEAL_ENABLE': str(repair).lower(), + 'RUSTFS_HEAL_MRF_ENABLE': str(repair).lower(), + 'RUSTFS_SCANNER_ENABLED': str(repair).lower(), + 'RUSTFS_UNSAFE_BYPASS_DISK_CHECK': 'false', 'MINIO_CI': 'false'} + volumes = [f'http://node{node}:9000/data/rustfs{drive}/mnmd' + for node in range(1, 5) for drive in range(1, len(roots()) + 1)] + with (BACKING / 'stdout.log').open('ab') as output: + child = subprocess.Popen(['/usr/bin/rustfs', *volumes], env=environment, + stdin=subprocess.DEVNULL, stdout=output, stderr=subprocess.STDOUT, start_new_session=True) + fields = Path(f'/proc/{child.pid}/stat').read_text().rsplit(')', 1)[1].split() + value = {'pid': child.pid, 'startTicks': fields[19], 'repairEnabled': repair, + 'unsafeBypassDiskCheck': False, 'minioCI': False} + temporary = STATE.with_suffix('.tmp') + temporary.write_text(json.dumps(value)) + temporary.replace(STATE) + return process() + + +def mounts(): + result = [] + root_device = os.stat('/').st_dev + for mount in roots(): + observed = json.loads(command('findmnt', '--json', '--mountpoint', str(mount), + '--output', 'TARGET,SOURCE,FSTYPE,UUID,MAJ:MIN,OPTIONS'))['filesystems'][0] + if observed['fstype'] != 'xfs' or not observed['uuid'] or os.stat(mount).st_dev == root_device: + raise RuntimeError('test endpoint lacks a distinct mounted XFS block device') + observed['endpoint'] = str(mount / 'mnmd') + probe = subprocess.run(['findmnt', '--mountpoint', str(mount / 'mnmd')], capture_output=True) + observed['endpointIsMountpoint'] = probe.returncode == 0 + result.append(observed) + if len({row['maj:min'] for row in result}) != len(result): + raise RuntimeError('data endpoints share a block device') + return result + + +def reset(): + if process()['running']: + raise RuntimeError('empty-root replacement requires the original process to exit') + for data, retired in zip(data_roots(), data_roots('retired'), strict=True): + if retired.exists(): + raise RuntimeError('retired data already exists') + if os.environ['NR_LAYOUT'] == 'mountpoint': + command('umount', str(data)) + data.rename(retired) + data.mkdir() + if os.environ['NR_LAYOUT'] == 'mountpoint': + command('mount', '--bind', str(data), str(data)) + return {'empty': all(not list(path.iterdir()) for path in data_roots()), 'mounts': mounts()} + + +def quarantine(keys): + if process()['running']: + raise RuntimeError('quarantine requires a stopped process') + for disk, data in enumerate(data_roots()): + archive = data / f'.nr-oracle-{time.time_ns()}' + archive.mkdir() + for index, relative in enumerate(keys[disk]): + path = data / relative + if not path.is_relative_to(data) or '..' in Path(relative).parts or path.is_symlink(): + raise RuntimeError('invalid owned shard path') + path.rename(archive / str(index)) + return {'quarantinedPerDisk': len(keys)} + + +def log_cursors(): + return {path.name: path.stat().st_size for path in (BACKING / 'logs').glob('*') if path.is_file()} + + +def metadata_failures(cursors): + matches = [] + for path in (BACKING / 'logs').glob('*'): + if not path.is_file(): + continue + size = path.stat().st_size + offset = min(cursors.get(path.name, 0), size) + with path.open('rb') as stream: + stream.seek(max(offset, size - 2 * 1024**2)) + text = stream.read().decode(errors='replace') + matches.extend(line for line in text.splitlines() if 'load_pool_meta' in line or 'classify_pool_meta_absence' in line + or 'pool.bin' in line and ('InsufficientReadQuorum' in line or 'quorum' in line.lower())) + text = '\n'.join(matches[-8:]) + for key in ('RUSTFS_ACCESS_KEY', 'RUSTFS_SECRET_KEY'): + text = text.replace(os.environ[key], '[REDACTED]') + return {'observed': bool(matches), 'signatures': text} + + +def serve(): + mounted = [] + (BACKING / 'logs').mkdir(exist_ok=True) + def interrupted(signum, frame): + raise KeyboardInterrupt() + signal.signal(signal.SIGTERM, interrupted) + try: + # Docker may expose only loop0..loop7 even when the kernel can allocate + # more loop devices. Allocation remains delegated to mount/loop-control. + for index in range(128): + device = Path(f'/dev/loop{index}') + if not device.exists(): + os.mknod(device, stat.S_IFBLK | 0o600, os.makedev(7, index)) + for index, mount in enumerate(roots(), 1): + image = BACKING / f"{os.environ['NR_RUN_ID']}-node{os.environ['NR_SLOT']}-disk{index}.xfs" + # Only a freshly and exclusively created regular file is formatted. + with image.open('xb') as output: + output.truncate(512 * 1024**2) + command('mkfs.xfs', '-f', str(image)) + mount.mkdir(parents=True) + command('mount', '-t', 'xfs', '-o', 'loop', str(image), str(mount)) + mounted.append(mount) + (mount / 'mnmd').mkdir() + if os.environ['NR_LAYOUT'] == 'mountpoint': + command('mount', '--bind', str(mount / 'mnmd'), str(mount / 'mnmd')) + print(json.dumps({'mounts': mounts()}), flush=True) + start() + while True: + try: + while os.waitpid(-1, os.WNOHANG)[0]: + pass + except ChildProcessError: + pass + time.sleep(.2) + except KeyboardInterrupt: + pass + finally: + stop() + for mount in reversed(mounted): + if os.environ['NR_LAYOUT'] == 'mountpoint': + command('umount', str(mount / 'mnmd')) + command('umount', str(mount)) + + +def main(): + action = sys.argv[1] + if action == 'serve': + serve() + return + request = json.load(sys.stdin) + if action == 'state': value = process() + elif action == 'stop': value = stop() + elif action == 'start': value = start(request.get('repair', True)) + elif action == 'mounts': value = mounts() + elif action == 'reset': value = reset() + elif action == 'quarantine': value = quarantine(request['paths']) + elif action == 'log-cursors': value = log_cursors() + elif action == 'metadata-failure': value = metadata_failures(request['cursors']) + elif action == 'identity': + binary = Path('/usr/bin/rustfs') + value = {'binarySha256': hashlib.sha256(binary.read_bytes()).hexdigest(), + 'version': command(str(binary), '--version'), 'kernel': command('uname', '-a'), + 'candidate': json.loads(Path('/opt/node_replacement_candidate.json').read_text())} + else: raise ValueError('unsupported node operation') + print(json.dumps(value)) + + +if __name__ == '__main__': + main() diff --git a/scripts/prepare_node_replacement.py b/scripts/prepare_node_replacement.py new file mode 100644 index 0000000..062e164 --- /dev/null +++ b/scripts/prepare_node_replacement.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Build the XFS test image with a digest-verified official GNU release binary.""" + +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import zipfile + + +def prepare(release, platform, output, tag): + architecture = {'linux/amd64': 'x86_64', 'linux/arm64': 'aarch64'}[platform] + metadata = json.loads(subprocess.check_output(['gh', 'release', 'view', release, + '--repo', 'rustfs/rustfs', '--json', 'tagName,assets'], text=True)) + name = f'rustfs-linux-{architecture}-gnu-v{release.removeprefix("v")}.zip' + asset = next(row for row in metadata['assets'] if row['name'] == name) + expected = asset.get('digest', '').removeprefix('sha256:') + if len(expected) != 64: + raise ValueError('official release asset lacks a SHA-256 digest') + output.mkdir(parents=True, exist_ok=True) + archive = output / 'rustfs.zip' + if not archive.exists() or hashlib.sha256(archive.read_bytes()).hexdigest() != expected: + subprocess.run(['curl', '--fail', '--location', '--retry', '2', '--continue-at', '-', + '--output', str(archive), asset['url']], check=True) + if hashlib.sha256(archive.read_bytes()).hexdigest() != expected: + raise ValueError('official release archive digest differs') + with zipfile.ZipFile(archive) as package: + binaries = [row for row in package.infolist() if Path(row.filename).name == 'rustfs' and not row.is_dir()] + if len(binaries) != 1: + raise ValueError('official archive must contain exactly one RustFS binary') + with package.open(binaries[0]) as source, (output / 'rustfs').open('wb') as target: + shutil.copyfileobj(source, target) + (output / 'rustfs').chmod(0o755) + source = Path(__file__).resolve().parent + for relative in ('node_replacement_node.py', 'node-replacement.Dockerfile'): + shutil.copyfile(source / relative, output / relative) + (output / '.dockerignore').write_text('*\n!rustfs\n!candidate.json\n!node_replacement_node.py\n!node-replacement.Dockerfile\n') + manifest = {'release': metadata['tagName'], 'platform': platform, 'assetUrl': asset['url'], + 'archiveSha256': expected, 'binarySha256': hashlib.sha256((output / 'rustfs').read_bytes()).hexdigest()} + (output / 'candidate.json').write_text(json.dumps(manifest, indent=2) + '\n') + subprocess.run(['docker', 'build', '--platform', platform, '-f', str(output / 'node-replacement.Dockerfile'), + '-t', tag, str(output)], check=True) + print(json.dumps(manifest, indent=2)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--release', default='1.0.5') + parser.add_argument('--platform', choices=('linux/amd64', 'linux/arm64'), default='linux/amd64') + parser.add_argument('--out', type=Path, required=True) + parser.add_argument('--tag', default='rustfs-node-replacement-xfs:local') + args = parser.parse_args() + prepare(args.release, args.platform, args.out, args.tag) + + +if __name__ == '__main__': + main() diff --git a/test_node_replacement.py b/test_node_replacement.py new file mode 100644 index 0000000..0b706f5 --- /dev/null +++ b/test_node_replacement.py @@ -0,0 +1,261 @@ +"""Regression tests for replacement evidence and resource ownership.""" + +import io +import hashlib +import json +from pathlib import Path +import subprocess +import tarfile +import tempfile +import types +import unittest +from unittest.mock import Mock, patch + +from scripts.node_replacement import Case, LABEL, archive_files, cleanup, reconstruction_sources, shard_manifest, system_metadata_manifest, topology +from scripts import node_replacement_node as node + + +def disks(members): + return [{".rustfs.sys/format.json": json.dumps({"format": "xl", "id": "pool", + "xl": {"this": identity, "sets": [members]}}).encode()} for identity in members] + + +class ReplacementEvidenceTests(unittest.TestCase): + def test_quarantine_stays_inside_the_endpoint_mount_and_preserves_metadata(self): + with tempfile.TemporaryDirectory() as directory: + data = Path(directory) / 'mnmd' + part = data / 'bucket/key/generation/part.1' + part.parent.mkdir(parents=True) + part.write_bytes(b'committed shard') + metadata = data / 'bucket/key/xl.meta' + metadata.write_bytes(b'committed metadata') + with patch.object(node, 'process', return_value={'running': False}), patch.object(node, 'data_roots', return_value=[data]): + node.quarantine([['bucket/key/generation']]) + self.assertFalse(part.exists()) + self.assertEqual(metadata.read_bytes(), b'committed metadata') + archived = list(data.glob('.nr-oracle-*/0/part.1')) + self.assertEqual(len(archived), 1) + self.assertEqual(archived[0].read_bytes(), b'committed shard') + + def test_cold_start_control_ignores_metadata_errors_from_earlier_processes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'logs').mkdir() + log = root / 'logs/rustfs.log' + log.write_text('old load_pool_meta failed\n') + with patch.object(node, 'BACKING', root), patch.dict('os.environ', {'RUSTFS_ACCESS_KEY': 'testaccess', 'RUSTFS_SECRET_KEY': 'testsecret'}): + cursor = node.log_cursors() + with log.open('a') as output: output.write('new harmless startup\n') + self.assertFalse(node.metadata_failures(cursor)['observed']) + with log.open('a') as output: output.write('InsufficientReadQuorum .rustfs.sys pool.bin\n') + self.assertTrue(node.metadata_failures(cursor)['observed']) + cursor = node.log_cursors() + with log.open('a') as output: output.write('init failed during classify_pool_meta_absence: pool.bin replica unavailable\n') + self.assertTrue(node.metadata_failures(cursor)['observed']) + + def test_live_archive_retries_only_expected_concurrent_file_changes(self): + with tempfile.TemporaryDirectory() as root: + case = self.case(root) + changing = subprocess.CompletedProcess([], 1, b'', b'tar: ./key: file changed as we read it\n') + success = subprocess.CompletedProcess([], 0, b'archive', b'') + with patch('scripts.node_replacement.subprocess.run', side_effect=[changing, success]) as run, \ + patch('scripts.node_replacement.time.sleep'): + self.assertEqual(case.docker('exec', binary=True, live_archive=True), b'archive') + self.assertEqual(run.call_count, 2) + for live, error in ((False, changing), (True, subprocess.CompletedProcess([], 2, b'', b'tar: Permission denied'))): + with patch('scripts.node_replacement.subprocess.run', return_value=error): + with self.assertRaises(RuntimeError): case.docker('exec', live_archive=live) + + def test_existing_backing_file_is_rejected_before_formatting(self): + with tempfile.TemporaryDirectory() as root: + backing = Path(root) + image = backing / 'test-node2-disk1.xfs' + image.write_bytes(b'preserved evidence') + with patch.object(node, 'BACKING', backing), patch.object(node, 'roots', return_value=[backing / 'mount']), \ + patch.object(node, 'stop'), patch.object(node.signal, 'signal'), patch.object(node.os, 'mknod'), \ + patch.object(node, 'command') as command, \ + patch.dict('os.environ', {'NR_RUN_ID': 'test', 'NR_SLOT': '2'}): + with self.assertRaises(FileExistsError): node.serve() + command.assert_not_called() + self.assertEqual(image.read_bytes(), b'preserved evidence') + + def test_mount_inventory_rejects_root_devices_wrong_filesystems_and_duplicate_devices(self): + for invalid in (None, 'root', 'fstype', 'duplicate'): + with self.subTest(invalid=invalid): + rows = [{'target': '/data/rustfs1', 'fstype': 'xfs', 'uuid': 'one', 'maj:min': '7:1'}, + {'target': '/data/rustfs2', 'fstype': 'xfs', 'uuid': 'two', 'maj:min': '7:2'}] + if invalid == 'fstype': rows[0]['fstype'] = 'ext4' + if invalid == 'duplicate': rows[1]['maj:min'] = '7:1' + outputs = [json.dumps({'filesystems': [row]}) for row in rows] + devices = [types.SimpleNamespace(st_dev=value) for value in (1, 1 if invalid == 'root' else 2, 3)] + with patch.object(node, 'roots', return_value=[Path(row['target']) for row in rows]), \ + patch.object(node, 'command', side_effect=outputs), patch.object(node.os, 'stat', side_effect=devices), \ + patch.object(node.subprocess, 'run', return_value=types.SimpleNamespace(returncode=1)): + if invalid: + with self.assertRaises(RuntimeError): node.mounts() + else: + observed = node.mounts() + self.assertFalse(observed[0]['endpointIsMountpoint']) + + def test_ready_and_complete_object_parts_cannot_hide_missing_pool_metadata(self): + with tempfile.TemporaryDirectory() as root: + case = Case('image', Path(root), 10) + files = {'bucket/key/xl.meta': b'meta', 'bucket/key/generation/part.1': b'shard'} + case.bucket = 'bucket' + case.scenes = lambda name: [files] * 4 + with patch('scripts.node_replacement.time.monotonic', side_effect=[0, 11]): + with self.assertRaisesRegex(ValueError, 'repair deadline'): + case.wait_repair('replacement', {'key': {}}) + self.assertFalse(case.assertions['systemMetadataRecovered']) + self.assertTrue(case.assertions['replacementShardsPhysicallyPresent']) + + def test_both_pool_metadata_objects_require_nonempty_xlmeta(self): + valid = {'.rustfs.sys/format.json': b'format', + '.rustfs.sys/pool.bin/xl.meta': b'XL2 ' + b'x' * 64, + '.rustfs.sys/pool.bin.identity/xl.meta': b'XL2 ' + b'y' * 64} + self.assertEqual(set(system_metadata_manifest(valid)), {'pool.bin', 'pool.bin.identity'}) + for path in valid: + for replacement in (None, b'', b'XL2 ', b'broken' * 16): + if path.endswith('format.json') and replacement: + continue + changed = dict(valid) + if replacement is None: + del changed[path] + else: + changed[path] = replacement + with self.subTest(path=path, replacement=replacement), self.assertRaises(ValueError): + system_metadata_manifest(changed) + + def test_forced_reconstruction_uses_actual_topology_quorum(self): + self.assertEqual(reconstruction_sources(1, 2), 1) + self.assertEqual(reconstruction_sources(4, 4), 2) + with self.assertRaises(ValueError): reconstruction_sources(4, 1) + + def case(self, root): + case = Case("image", Path(root), 30) + case.objects = {"key": {"size": 4, "sha256": hashlib.sha256(b"data").hexdigest()}} + return case + + def test_reconstructed_reads_must_match_the_acknowledged_bytes(self): + for value in (b"data", b"bad!", b"data-extra"): + with self.subTest(value=value), tempfile.TemporaryDirectory() as root: + case = self.case(root) + client = Mock() + client.get_object.return_value = {"Body": io.BytesIO(value)} + case.s3 = lambda name: client + if value == b"data": + case.read(["replacement"], "proof") + else: + with self.assertRaises(ValueError): case.read(["replacement"], "proof") + + def test_negative_control_requires_read_quorum_failure_and_rejects_success(self): + for status, code in ((503, "SlowDownRead"), (503, "InternalError"), (404, "NoSuchKey"), (403, "AccessDenied"), (200, None)): + with self.subTest(status=status, code=code), tempfile.TemporaryDirectory() as root: + case = self.case(root) + client = Mock() + if status == 200: + client.get_object.return_value = {"Body": io.BytesIO(b"data")} + else: + error = RuntimeError("request failed") + error.response = {"ResponseMetadata": {"HTTPStatusCode": status}, "Error": {"Code": code}} + client.get_object.side_effect = error + case.s3 = lambda name: client + if code == "SlowDownRead": + case.read_without_replacement("survivor") + else: + with self.assertRaises(ValueError): case.read_without_replacement("survivor") + + def test_new_disk_must_take_the_original_slot_with_inherited_or_fresh_uuid(self): + original = topology(disks(["a", "b", "c", "old"])) + changed = disks(["a", "b", "c", "new"]) + self.assertEqual(topology(changed, original=original)["members"][-1], "new") + self.assertEqual(topology(disks(["a", "b", "c", "old"]), original=original)["members"][-1], "old") + for invalid in (disks(["a", "b", "other", "new"]), + disks(["a", "b", "new", "c"])): + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + topology(invalid, original=original) + + def test_separate_pools_partial_membership_and_wrong_pool_fail(self): + original = topology(disks(["a", "b", "c", "old"])) + for field in ("pool", "sets", "this"): + changed = disks(["a", "b", "c", "new"]) + value = json.loads(changed[-1][".rustfs.sys/format.json"]) + if field == "pool": value["id"] = "another-pool" + elif field == "sets": value["xl"]["sets"] = [["a", "b"], ["c", "new"]] + else: value["xl"]["this"] = "c" + changed[-1][".rustfs.sys/format.json"] = json.dumps(value).encode() + with self.subTest(field=field), self.assertRaises(ValueError): + topology(changed, original=original) + + def test_metadata_only_or_another_objects_shard_cannot_count_as_repair(self): + files = {"bucket/key/xl.meta": b"meta", "bucket/key/generation/part.1": b"shard"} + self.assertEqual(shard_manifest(files, "bucket", ["key"])["key"]["size"], 5) + for changed in ({"bucket/key/xl.meta": b"meta"}, + {**files, "bucket/key/another-generation/part.1": b"stale"}, + {**files, "bucket/key/generation/part.1": b""}, + {**files, "bucket/key/xl.meta": b""}): + with self.subTest(changed=changed), self.assertRaises(ValueError): + shard_manifest(changed, "bucket", ["key"]) + with self.assertRaises(ValueError): + shard_manifest(files, "bucket", ["key", "offline-write"]) + + def test_archives_reject_links_traversal_and_duplicate_file_evidence(self): + for kind in ("good", "traversal", "absolute", "link", "duplicate"): + output = io.BytesIO() + with tarfile.open(fileobj=output, mode="w") as archive: + name = {"traversal": "../key", "absolute": "/key"}.get(kind, "./key") + item = tarfile.TarInfo(name) + item.size = 4 + if kind == "link": + item.type, item.linkname = tarfile.SYMTYPE, "other" + archive.addfile(item, io.BytesIO(b"data")) + if kind == "duplicate": archive.addfile(item, io.BytesIO(b"data")) + if kind == "good": + self.assertEqual(archive_files(output.getvalue()), {"key": b"data"}) + else: + with self.subTest(kind=kind), self.assertRaises(ValueError): + archive_files(output.getvalue()) + + def test_recovery_removes_only_owned_resources_and_keeps_cleanup_failures(self): + for failure in (None, "ownership", "daemon", "remove"): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + resources = {"runId": "run", "network": "net", "containers": ["old", "new"], "volumes": ["disk"]} + (root / "resources.json").write_text(json.dumps(resources)) + (root / ".runtime.env").write_text("private") + removed = [] + def invoke(argv, **kwargs): + kind, action, name = argv[1], argv[2], argv[-1] + if action == "inspect": + if name == "old": + return subprocess.CompletedProcess(argv, 1, "", "No such container: old") + if failure == "daemon": + return subprocess.CompletedProcess(argv, 1, "", "Cannot connect to daemon") + labels = {LABEL: "other" if failure == "ownership" else "run"} + value = {"Config": {"Labels": labels}} if kind == "container" else {"Labels": labels} + return subprocess.CompletedProcess(argv, 0, json.dumps([value]), "") + removed.append(name) + return subprocess.CompletedProcess(argv, int(failure == "remove"), "", "") + with patch("scripts.node_replacement.subprocess.run", side_effect=invoke): + failures = cleanup(root) + self.assertEqual(bool(failures), failure is not None) + self.assertFalse((root / ".runtime.env").exists()) + self.assertNotIn("old", removed) + if failure in ("ownership", "daemon"): + self.assertEqual(removed, []) + + def test_repeated_cleanup_accepts_dockers_distinct_missing_resource_errors(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "resources.json").write_text(json.dumps({"runId": "run", "network": "net", + "containers": ["old"], "volumes": ["disk"]})) + def invoke(argv, **kwargs): + errors = {"container": "No such container: old", "volume": "get disk: no such volume", + "network": "network net not found"} + return subprocess.CompletedProcess(argv, 1, "[]", "Error response from daemon: " + errors[argv[1]]) + with patch("scripts.node_replacement.subprocess.run", side_effect=invoke): + self.assertEqual(cleanup(root), []) + + +if __name__ == "__main__": + unittest.main()