diff --git a/README.md b/README.md index a2674bd7..72afb132 100644 --- a/README.md +++ b/README.md @@ -390,15 +390,16 @@ rc admin rebalance status local --json rc ping local rc ready local --timeout 2 -# Prefer the RustFS background-scanner snapshot -rc du local +# List S3 objects and sum their sizes +rc du local/photos +rc du local/photos/2026/ --versions +rc du local/photos --incomplete -# Explicitly permit a portable paginated S3 fallback -rc du local/photos/2026/ --fallback --versions -rc du local/photos --fallback --incomplete +# Read the RustFS background scanner's last cluster totals +rc du local --snapshot ``` -`rc du` never starts the potentially expensive client scan after an unsupported or unauthorized admin request unless `--fallback` is present. See the [operational utilities reference](docs/reference/rc/ops.md) for count, staleness, and partial-result semantics. +For large buckets, a live scan may take many S3 requests. Existing scripts using `--fallback` still try the snapshot first. See the [operational utilities reference](docs/reference/rc/ops.md) for permissions, version counts, and partial results. ## Command Overview @@ -437,7 +438,7 @@ For full command documentation, see the [`rc` command reference](docs/reference/ | `completions` | Generate shell completion scripts | | `ping` | Check service liveness and round-trip latency | | `ready` | Check service dependency readiness | -| `du` | Report server-snapshot or explicitly permitted client-scan usage | +| `du` | Sum object sizes through S3 or read a RustFS usage snapshot | ### Admin Subcommands diff --git a/crates/cli/src/commands/du.rs b/crates/cli/src/commands/du.rs index 9391f7b6..6dce8ea2 100644 --- a/crates/cli/src/commands/du.rs +++ b/crates/cli/src/commands/du.rs @@ -1,10 +1,10 @@ -//! Storage usage command with RustFS snapshot and explicit S3 fallback. +//! Storage usage from S3 listings or an explicit RustFS snapshot. use clap::Args; use jiff::Timestamp; use rc_core::admin::{CapabilityApi, CapabilityAvailability, CapabilityReport}; use rc_core::ops::{UsageBucket, UsageReport, UsageScanApi, UsageScanRequest, UsageSnapshotApi}; -use rc_core::{AliasManager, Error}; +use rc_core::{AliasManager, Error, ListOptions, ObjectStore, RemotePath}; use rc_s3::{AdminClient, S3Client}; use serde::Serialize; @@ -17,13 +17,17 @@ const DATA_USAGE_CAPABILITY: &str = "admin.data-usage"; const SNAPSHOT_STALE_AFTER_SECONDS: u64 = 3_600; const FALLBACK_SUGGESTION: &str = "Retry with --fallback to permit a slower client-side S3 scan."; -/// Summarize storage usage from a RustFS snapshot or explicit S3 scan. +/// Summarize storage usage by listing S3 objects. #[derive(Args, Debug)] pub struct DuArgs { - /// Alias, bucket, or prefix path (ALIAS[/BUCKET[/PREFIX]]) + /// Alias, bucket, or directory prefix (ALIAS[/BUCKET[/PREFIX]]) pub target: String, - /// Permit a slower client-side S3 scan when the server snapshot cannot be used + /// Read the RustFS usage snapshot for a cluster or bucket + #[arg(long)] + pub snapshot: bool, + + /// Try the RustFS snapshot first, then use a client-side S3 scan if needed #[arg(long)] pub fallback: bool, @@ -125,6 +129,34 @@ pub async fn execute(args: DuArgs, output_config: OutputConfig) -> ExitCode { ); } }; + let requires_scan = target.request.requires_client_scan(); + if requires_scan && args.snapshot && !args.fallback { + let error = Error::UnsupportedFeature( + "The RustFS data-usage snapshot does not provide exact prefix or incomplete-upload usage" + .to_string(), + ); + return ops_output::emit_error( + &formatter, + USAGE_FAMILY, + "Server usage snapshot cannot satisfy the requested dimensions", + &error, + Some(DATA_USAGE_CAPABILITY), + None, + Some(FALLBACK_SUGGESTION), + ); + } + + let scan = async { + let client = S3Client::new(alias.clone()).await?; + if !args.target.ends_with('/') { + check_directory_target(&client, &target).await?; + } + client.scan_usage(&target.request).await + }; + if requires_scan || !(args.snapshot || args.fallback) { + return emit_scan_result(scan.await, &target, None, &formatter); + } + let admin = match AdminClient::new(&alias) { Ok(client) => client, Err(error) => { @@ -139,74 +171,51 @@ pub async fn execute(args: DuArgs, output_config: OutputConfig) -> ExitCode { ); } }; - let scan_client = if args.fallback { - match S3Client::new(alias).await { - Ok(client) => Some(client), - Err(error) => { - return ops_output::emit_error( - &formatter, - USAGE_FAMILY, - "Failed to create usage scan client", - &error, - None, - None, - None, - ); - } - } - } else { - None - }; + execute_snapshot(&target, &admin, &admin, args.fallback, scan, &formatter).await +} - execute_with_apis( - &args, - &target, - &admin, - &admin, - scan_client - .as_ref() - .map(|client| client as &dyn UsageScanApi), - Timestamp::now(), - &formatter, - ) - .await +async fn check_directory_target( + store: &dyn ObjectStore, + target: &ParsedUsageTarget, +) -> Result<(), Error> { + let (Some(bucket), Some(prefix)) = (&target.request.bucket, &target.request.prefix) else { + return Ok(()); + }; + let path = RemotePath::new(&target.alias, bucket, prefix); + let listing = store + .list_objects( + &path, + ListOptions { + max_keys: Some(1), + recursive: true, + ..Default::default() + }, + ) + .await?; + if listing.items.is_empty() { + return Err(Error::NotFound(format!( + "Directory '{}' does not exist", + path.to_full_path() + ))); + } + Ok(()) } -async fn execute_with_apis( - args: &DuArgs, +async fn execute_snapshot( target: &ParsedUsageTarget, capabilities: &dyn CapabilityApi, snapshot_api: &dyn UsageSnapshotApi, - scan_api: Option<&dyn UsageScanApi>, - now: Timestamp, + fallback: bool, + scan: impl Future>, formatter: &Formatter, ) -> ExitCode { - if target.request.requires_client_scan() { - if !args.fallback { - let error = Error::UnsupportedFeature( - "The RustFS data-usage snapshot does not provide exact prefix or incomplete-upload usage" - .to_string(), - ); - return ops_output::emit_error( - formatter, - USAGE_FAMILY, - "Server usage snapshot cannot satisfy the requested dimensions", - &error, - Some(DATA_USAGE_CAPABILITY), - None, - Some(FALLBACK_SUGGESTION), - ); - } - return execute_scan(scan_api, target, None, now, formatter).await; - } - let capability_report = capabilities.discover_capabilities(false).await; let server_version = capability_report .as_ref() .ok() .and_then(|report| report.server_version.clone()); - let fast_path = match capability_report { + let snapshot = match capability_report { Ok(report) if data_usage_available(&report) => snapshot_api .usage_snapshot() .await @@ -215,11 +224,9 @@ async fn execute_with_apis( Err(error) => Err(error), }; - match fast_path { - Ok(report) => emit_report(report, server_version, now, formatter), - Err(_error) if args.fallback => { - execute_scan(scan_api, target, server_version, now, formatter).await - } + match snapshot { + Ok(report) => emit_report(report, server_version, formatter), + Err(_) if fallback => emit_scan_result(scan.await, target, server_version, formatter), Err(error) => ops_output::emit_error( formatter, USAGE_FAMILY, @@ -232,26 +239,16 @@ async fn execute_with_apis( } } -async fn execute_scan( - scan_api: Option<&dyn UsageScanApi>, +fn emit_scan_result( + result: Result, target: &ParsedUsageTarget, server_version: Option, - now: Timestamp, formatter: &Formatter, ) -> ExitCode { - let Some(scan_api) = scan_api else { - return ops_output::emit_message( - formatter, - USAGE_FAMILY, - ExitCode::GeneralError, - "Client-side usage fallback was permitted but no scan adapter is available", - None, - ); - }; - match scan_api.scan_usage(&target.request).await { + match result { Ok(mut report) => { report.path = target.request.path(&target.alias); - emit_report(report, server_version, now, formatter) + emit_report(report, server_version, formatter) } Err(scan_error) => ops_output::emit_error( formatter, @@ -268,9 +265,9 @@ async fn execute_scan( fn emit_report( report: UsageReport, server_version: Option, - now: Timestamp, formatter: &Formatter, ) -> ExitCode { + let now = Timestamp::now(); if formatter.is_json() { formatter.json(&success_output(&report, server_version, now)); } else { @@ -471,7 +468,13 @@ fn parse_target(value: &str) -> Result { .get(2) .copied() .filter(|component| !component.is_empty()) - .map(str::to_string); + .map(|prefix| { + if prefix.ends_with('/') { + prefix.to_owned() + } else { + format!("{prefix}/") + } + }); if parts.get(2).is_some() && bucket.is_none() { return Err(Error::InvalidPath( "A usage prefix requires a non-empty bucket name".to_string(), @@ -593,36 +596,19 @@ mod tests { } } - fn args(target: &str, fallback: bool) -> DuArgs { - DuArgs { - target: target.to_string(), - fallback, - versions: false, - incomplete: false, - } - } - - fn parsed(args: &DuArgs) -> ParsedUsageTarget { - let mut target = parse_target(&args.target).expect("test target should parse"); - apply_scan_flags(&mut target, args); - target - } - #[tokio::test] async fn du_uses_advertised_server_snapshot_without_scanning() { - let command = args("local", false); - let target = parsed(&command); + let target = parse_target("local").expect("valid target"); let scan = MockScan { calls: AtomicUsize::new(0), report: UsageReport::empty(UsageSource::ClientScan, UsageScope::Cluster, None), }; - let code = execute_with_apis( - &command, + let code = execute_snapshot( &target, &MockCapabilities(Ok(capability_report(true))), &MockSnapshot(Ok(snapshot(1_700_000_000))), - Some(&scan), - Timestamp::from_second(1_700_000_100).expect("timestamp"), + false, + scan.scan_usage(&target.request), &Formatter::default(), ) .await; @@ -633,19 +619,17 @@ mod tests { #[tokio::test] async fn auth_denial_never_scans_without_explicit_fallback() { - let command = args("local", false); - let target = parsed(&command); + let target = parse_target("local").expect("valid target"); let scan = MockScan { calls: AtomicUsize::new(0), report: UsageReport::empty(UsageSource::ClientScan, UsageScope::Cluster, None), }; - let code = execute_with_apis( - &command, + let code = execute_snapshot( &target, &MockCapabilities(Err(Error::Auth("denied".to_string()))), &MockSnapshot(Ok(snapshot(1_700_000_000))), - Some(&scan), - Timestamp::from_second(1_700_000_100).expect("timestamp"), + false, + scan.scan_usage(&target.request), &Formatter::default(), ) .await; @@ -656,19 +640,17 @@ mod tests { #[tokio::test] async fn unavailable_snapshot_without_fallback_returns_unsupported_without_scanning() { - let command = args("local", false); - let target = parsed(&command); + let target = parse_target("local").expect("valid target"); let scan = MockScan { calls: AtomicUsize::new(0), report: UsageReport::empty(UsageSource::ClientScan, UsageScope::Cluster, None), }; - let code = execute_with_apis( - &command, + let code = execute_snapshot( &target, &MockCapabilities(Ok(capability_report(false))), &MockSnapshot(Ok(snapshot(1_700_000_000))), - Some(&scan), - Timestamp::from_second(1_700_000_100).expect("timestamp"), + false, + scan.scan_usage(&target.request), &Formatter::default(), ) .await; @@ -679,19 +661,17 @@ mod tests { #[tokio::test] async fn explicit_fallback_scans_when_capability_is_unavailable() { - let command = args("local/photos", true); - let target = parsed(&command); + let target = parse_target("local/photos").expect("valid target"); let scan = MockScan { calls: AtomicUsize::new(0), report: UsageReport::empty(UsageSource::ClientScan, UsageScope::Bucket, None), }; - let code = execute_with_apis( - &command, + let code = execute_snapshot( &target, &MockCapabilities(Ok(capability_report(false))), &MockSnapshot(Ok(snapshot(1_700_000_000))), - Some(&scan), - Timestamp::from_second(1_700_000_100).expect("timestamp"), + true, + scan.scan_usage(&target.request), &Formatter::default(), ) .await; @@ -702,8 +682,7 @@ mod tests { #[tokio::test] async fn partial_fallback_returns_network_exit_with_partial_result() { - let command = args("local", true); - let target = parsed(&command); + let target = parse_target("local").expect("valid target"); let mut partial = UsageReport::empty(UsageSource::ClientScan, UsageScope::Cluster, None); partial.push_failure(UsageFailure { bucket: "private".to_string(), @@ -713,13 +692,12 @@ mod tests { calls: AtomicUsize::new(0), report: partial, }; - let code = execute_with_apis( - &command, + let code = execute_snapshot( &target, &MockCapabilities(Ok(capability_report(false))), &MockSnapshot(Ok(snapshot(1_700_000_000))), - Some(&scan), - Timestamp::from_second(1_700_000_100).expect("timestamp"), + true, + scan.scan_usage(&target.request), &Formatter::default(), ) .await; diff --git a/crates/cli/src/commands/mod.rs b/crates/cli/src/commands/mod.rs index a59d9e3c..7696b11d 100644 --- a/crates/cli/src/commands/mod.rs +++ b/crates/cli/src/commands/mod.rs @@ -354,7 +354,7 @@ pub enum Commands { /// Generate shell completion scripts Completions(completions::CompletionsArgs), - /// Summarize storage usage + /// Summarize storage usage by listing S3 objects Du(du::DuArgs), /// Check service liveness and round-trip latency diff --git a/crates/cli/tests/du.rs b/crates/cli/tests/du.rs new file mode 100644 index 00000000..a3e288a9 --- /dev/null +++ b/crates/cli/tests/du.rs @@ -0,0 +1,240 @@ +mod admin_support; + +use std::process::{Command, Output}; + +use admin_support::{rc_binary, rc_host_alias, start_admin_binary_sequence_test_server}; +use serde_json::Value; + +fn run_du(args: &[&str], endpoint: &str) -> Output { + let config = tempfile::tempdir().expect("temporary config"); + Command::new(rc_binary()) + .args(["--json", "du"]) + .args(args) + .env("RC_CONFIG_DIR", config.path()) + .env("RC_HOST_local", rc_host_alias(endpoint)) + .env("NO_PROXY", "127.0.0.1") + .env("no_proxy", "127.0.0.1") + .output() + .expect("run rc du") +} + +#[test] +fn missing_directory_needs_an_explicit_trailing_slash_to_report_zero() { + let empty = b"false"; + for (target, expected_exit) in [("local/photos/missing", 5), ("local/photos/missing/", 0)] { + let (endpoint, requests, server) = start_admin_binary_sequence_test_server(vec![( + "200 OK", + "application/xml", + empty.to_vec(), + )]); + let output = run_du(&[target], &endpoint); + assert_eq!( + output.status.code(), + Some(expected_exit), + "{target}: {}", + String::from_utf8_lossy(&output.stderr) + ); + server.join().expect("serve empty prefix"); + assert!( + requests + .into_iter() + .next() + .expect("lookup") + .target + .contains("prefix=missing%2F") + ); + if expected_exit == 0 { + let json: Value = serde_json::from_slice(&output.stdout).expect("usage JSON"); + assert_eq!(json["data"]["object_count"], 0); + } else { + let json: Value = serde_json::from_slice(&output.stderr).expect("error JSON"); + assert_eq!(json["error"]["type"], "not_found"); + } + } +} + +#[test] +fn scans_exclude_directory_markers_but_include_empty_files_and_nonempty_objects() { + let objects = b"empty/0zero0payload/9"; + let versions = b"empty/0truezero0truepayload/9truedeletedtrue"; + for (flags, response) in [ + (&[][..], &objects[..]), + (&["--versions"][..], &versions[..]), + ] { + let (endpoint, _requests, server) = start_admin_binary_sequence_test_server(vec![( + "200 OK", + "application/xml", + response.to_vec(), + )]); + let mut args = vec!["local/photos"]; + args.extend_from_slice(flags); + let output = run_du(&args, &endpoint); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + server.join().expect("serve listing"); + let json: Value = serde_json::from_slice(&output.stdout).expect("usage JSON"); + let data = &json["data"]; + assert_eq!(data["object_count"], 2); + assert_eq!(data["total_bytes"], 9); + if !flags.is_empty() { + assert_eq!(data["version_count"], 2); + assert_eq!(data["delete_marker_count"], 1); + } + } +} + +#[test] +fn default_du_lists_s3_objects_for_cluster_bucket_and_prefix() { + let buckets = "photos"; + let objects = "falselogs/a17"; + for (target, scope) in [ + ("local", "cluster"), + ("local/photos", "bucket"), + ("local/photos/logs", "prefix"), + ("local/photos/logs/", "prefix"), + ] { + let mut responses = Vec::new(); + if scope == "cluster" { + responses.push(("200 OK", "application/xml", buckets.as_bytes().to_vec())); + } + if scope == "prefix" && !target.ends_with('/') { + responses.push(("200 OK", "application/xml", objects.as_bytes().to_vec())); + } + responses.push(("200 OK", "application/xml", objects.as_bytes().to_vec())); + let (endpoint, requests, server) = start_admin_binary_sequence_test_server(responses); + let output = run_du(&[target], &endpoint); + assert!( + output.status.success(), + "{target}: {}", + String::from_utf8_lossy(&output.stderr) + ); + server.join().expect("serve S3 listings"); + let targets: Vec<_> = requests.into_iter().map(|request| request.target).collect(); + assert!(targets.iter().all(|path| !path.contains("/admin/"))); + let listing = targets.last().expect("object listing"); + assert!( + listing.starts_with("/photos") && listing.contains("list-type=2"), + "{listing}" + ); + if scope == "prefix" { + assert!( + listing + .split(['?', '&']) + .any(|param| param == "prefix=logs%2F"), + "{listing}" + ); + } + let json: Value = serde_json::from_slice(&output.stdout).expect("usage JSON"); + let data = &json["data"]; + assert_eq!(data["scope"], scope); + assert_eq!(data["source"], "client_scan"); + assert_eq!(data["total_bytes"], 17); + assert_eq!(data["object_count"], 1); + assert!(data["snapshot_at"].is_null()); + } +} + +#[test] +fn du_versions_and_incomplete_uploads_use_the_same_directory_prefix() { + let versions = b"false"; + let uploads = b"false"; + let directory = + b"logs/0"; + let (endpoint, requests, server) = start_admin_binary_sequence_test_server(vec![ + ("200 OK", "application/xml", directory.to_vec()), + ("200 OK", "application/xml", versions.to_vec()), + ("200 OK", "application/xml", uploads.to_vec()), + ]); + let output = run_du( + &[ + "--fallback", + "--versions", + "--incomplete", + "local/photos/logs", + ], + &endpoint, + ); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + server.join().expect("serve version and upload listings"); + let mut requests = requests.into_iter(); + for operation in ["max-keys=1", "versions", "uploads"] { + let target = requests.next().expect("listing request").target; + assert!(target.contains(operation), "{target}"); + assert!( + target + .split(['?', '&']) + .any(|param| param == "prefix=logs%2F"), + "{target}" + ); + } +} + +#[test] +fn snapshot_only_rejects_dimensions_that_require_listing() { + for args in [ + vec!["--snapshot", "local/photos/2026/"], + vec!["--snapshot", "--incomplete", "local/photos"], + ] { + let output = run_du(&args, "http://127.0.0.1:1"); + assert_eq!(output.status.code(), Some(7)); + let json: Value = serde_json::from_slice(&output.stderr).expect("usage error JSON"); + assert_eq!(json["error"]["capability"], "admin.data-usage"); + } +} + +#[test] +fn admin_denial_only_triggers_listing_with_explicit_fallback() { + for options in [ + vec!["--snapshot"], + vec!["--fallback"], + vec!["--snapshot", "--fallback"], + ] { + let fallback = options.contains(&"--fallback"); + let mut responses = vec![( + "403 Forbidden", + "application/json", + br#"{"Code":"AccessDenied","Message":"Access Denied"}"#.to_vec(), + )]; + if fallback { + responses.push(( + "200 OK", + "application/xml", + b"false".to_vec(), + )); + } + let (endpoint, requests, server) = start_admin_binary_sequence_test_server(responses); + let mut args = options; + args.push("local/photos"); + let output = run_du(&args, &endpoint); + assert_eq!( + output.status.code(), + Some(if fallback { 0 } else { 4 }), + "{args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + server.join().expect("serve responses"); + let mut requests = requests.into_iter(); + assert_eq!( + requests.next().expect("capability discovery").target, + "/rustfs/admin/v3/info" + ); + if fallback { + assert!( + requests + .next() + .expect("object listing") + .target + .contains("list-type=2") + ); + let json: Value = serde_json::from_slice(&output.stdout).expect("usage JSON"); + assert_eq!(json["data"]["source"], "client_scan"); + } + } +} diff --git a/crates/cli/tests/help_contract.rs b/crates/cli/tests/help_contract.rs index 3582481b..000d19c6 100644 --- a/crates/cli/tests/help_contract.rs +++ b/crates/cli/tests/help_contract.rs @@ -587,6 +587,7 @@ fn top_level_command_help_contract() { args: &["du"], usage: "Usage: rc du [OPTIONS] ", expected_tokens: &[ + "--snapshot", "--fallback", "--versions", "--incomplete", diff --git a/crates/core/src/ops.rs b/crates/core/src/ops.rs index 1612511f..9586a6c7 100644 --- a/crates/core/src/ops.rs +++ b/crates/core/src/ops.rs @@ -241,7 +241,7 @@ fn sum_optional(values: impl Iterator>) -> Option { values.try_fold(first, |total, value| Some(total.saturating_add(value?))) } -/// Options for an explicitly authorized client-side usage scan. +/// Scope and optional counts for a client-side S3 usage scan. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UsageScanRequest { /// Optional bucket; absent means all visible buckets. diff --git a/crates/s3/src/ops.rs b/crates/s3/src/ops.rs index 658d54bc..5825b0d5 100644 --- a/crates/s3/src/ops.rs +++ b/crates/s3/src/ops.rs @@ -281,7 +281,10 @@ async fn scan_current_objects( }, ) .await?; - for object in page.items.into_iter().filter(|object| !object.is_dir) { + for object in page.items { + if object.is_dir || is_directory_marker(&object.key, object.size_bytes) { + continue; + } objects = objects.saturating_add(1); bytes = bytes.saturating_add(non_negative_bytes(object.size_bytes)); } @@ -335,7 +338,7 @@ async fn scan_versions( for version in page.items { if version.is_delete_marker { delete_markers = delete_markers.saturating_add(1); - } else { + } else if !is_directory_marker(&version.key, version.size_bytes) { versions = versions.saturating_add(1); bytes = bytes.saturating_add(non_negative_bytes(version.size_bytes)); if version.is_latest { @@ -451,6 +454,10 @@ async fn scan_uploaded_parts( Ok(bytes) } +fn is_directory_marker(key: &str, size: Option) -> bool { + key.ends_with('/') && size == Some(0) +} + fn non_negative_bytes(value: Option) -> u64 { value .and_then(|value| u64::try_from(value).ok()) diff --git a/docs/reference/rc/mc-compatibility.md b/docs/reference/rc/mc-compatibility.md index f32bbdc6..c84c62eb 100644 --- a/docs/reference/rc/mc-compatibility.md +++ b/docs/reference/rc/mc-compatibility.md @@ -33,7 +33,7 @@ treated as proof that the operation works. | Legal hold | `rc object legal-hold` | `rc legalhold` | Alias | | Tags | `rc bucket tag` and `rc object tag` | `rc tag` | Alias; runtime gated | | Administrative workflows | `rc admin ...` | selected `mc admin`-compatible forms | Canonical, partial by documented subcommand | -| Health and usage | `rc ping`, `ready`, `du` | equivalent top-level names | Canonical; `du` fast path is runtime gated | +| Health and usage | `rc ping`, `ready`, `du` | equivalent top-level names | Canonical; `du` lists objects by default | | Events | `rc watch` | `mc watch`-style filters | Canonical; runtime gated | | Other data workflows | `rc diff`, `sql`, `undo` | equivalent top-level names | Canonical | @@ -50,7 +50,7 @@ on every S3-compatible target: | --- | --- | | Bucket notifications, CORS, versioning, tags, lifecycle, replication, and watch | RustFS runtime capability or authoritative S3 route response | | Admin diagnostics, scanner, storage observations, KMS, OIDC, replication, and configuration | RustFS admin capability and bounded typed response | -| `du` server snapshot | `admin.data-usage`; client scan requires explicit `--fallback` | +| `du --snapshot` | Requires `admin.data-usage`. Plain `du` uses S3 listing. | | Advanced transfer fidelity | Exact S3 headers and behavior supported by the target release | An unavailable gate returns exit code `7` (`unsupported_feature`) rather than diff --git a/docs/reference/rc/ops.md b/docs/reference/rc/ops.md index 2ec5a254..c674a8e8 100644 --- a/docs/reference/rc/ops.md +++ b/docs/reference/rc/ops.md @@ -2,7 +2,7 @@ ## Purpose -`rc ping`, `rc ready`, and `rc du` provide lightweight service checks and storage-usage reporting. The health commands use public RustFS endpoints. `rc du` prefers the RustFS background-scanner snapshot and can run a portable but potentially expensive S3 scan only when the user explicitly permits it. +`rc ping`, `rc ready`, and `rc du` provide lightweight service checks and storage-usage reporting. The health commands use public RustFS endpoints. `rc du` lists S3 objects by default and supports explicit RustFS usage snapshots. ## Syntax @@ -30,51 +30,51 @@ rc ready local --json ## Usage command -RustFS 1.0.0-beta.10 exposes a signed data-usage snapshot at `GET /rustfs/admin/v3/datausageinfo`. Capability discovery advertises this fast path as `admin.data-usage`. The snapshot is generated by the server's background scanner, so it can be older than the command invocation. +`rc du` recursively lists current objects through S3 and sums their sizes. An alias scans its visible buckets. A bucket or prefix scans only that target. The default requires S3 list permissions and sends no RustFS admin requests. Large scans can take many requests, and concurrent writes or deletes can affect what later pages see. -`rc du` includes `source` in JSON and prints it in human output: +Bucket-relative targets select a directory. Both `rc du local/photos/logs` and `rc du local/photos/logs/` scan keys under `logs/`, so a sibling such as `logs-old/a` is excluded. Without a trailing slash, `rc` first lists at most one current key under the directory. A single object or a missing directory returns the `not_found` exit code (5). An explicit trailing slash also allows empty prefixes and directories containing only historical versions or incomplete uploads. This boundary applies to all scan modes. -- `server_snapshot` means the RustFS data-usage snapshot was used; -- `client_scan` means the user permitted and the client performed paginated S3 requests. - -When a server snapshot includes `last_update`, output includes `snapshot_at` and `snapshot_age_seconds`. A snapshot older than one hour is marked `stale`; the command still reports the actual timestamp and age rather than presenting the statistics as current. +Use `--snapshot` for the last cluster or bucket totals from the RustFS background scanner. It uses `GET /rustfs/admin/v3/datausageinfo` when the server advertises `admin.data-usage`. Human and JSON output identify the source as `client_scan` or `server_snapshot`. For timestamped snapshots, JSON includes `snapshot_at` and `snapshot_age_seconds`. A snapshot older than one hour is marked `stale` and returned. ### Options | Option | Description | | --- | --- | -| `--fallback` | Permit a slower client-side S3 scan when capability discovery, the admin route, or authorization prevents use of the snapshot. | -| `--versions` | During a client scan, use paginated object-version listing and include version and delete-marker counts. A server snapshot reports the counts already supplied by RustFS. | -| `--incomplete` | During a client scan, paginate incomplete multipart uploads and each upload's parts; include upload counts and uploaded part bytes. | +| `--snapshot` | Read the RustFS cluster or bucket snapshot. Prefixes and incomplete uploads require a scan and return unsupported-feature exit code 7 in snapshot-only mode. | +| `--fallback` | Keep the previous snapshot-first behavior. Use an S3 scan if the snapshot is unavailable or lacks the requested bucket or dimensions. May also be combined with `--snapshot`. | +| `--versions` | On a scan, list every version and count delete markers separately. A snapshot already includes the version counts reported by RustFS. | +| `--incomplete` | Include incomplete multipart uploads and the bytes in their uploaded parts. | -Prefix totals and incomplete multipart totals are not available in the RustFS server snapshot. Those requests require `--fallback`; the command never starts the scan implicitly. An admin permission denial also preserves exit code 4 unless `--fallback` was supplied. +### Migrating from snapshot-first usage -Examples: +Scripts that relied on a fast snapshot or have only RustFS admin permissions should add `--snapshot`. For a versioned bucket, the default scan sums current object sizes. Use `--versions` to sum bytes across versions. Existing invocations with `--fallback` retain their snapshot-first selection. Scripts that used a partial name such as `logs` to match several directories must now specify each directory separately. Both sources use the existing v3 usage output. ```bash -# Fast cluster snapshot on supported RustFS versions -rc du local - -# Bucket snapshot, with explicit portable fallback if the route cannot be used -rc du local/photos --fallback +# Scan current objects in one bucket +rc du local/photos -# Prefix scan including every version and delete marker -rc du local/photos/2026/ --fallback --versions +# Scan a prefix including every version and delete marker +rc du local/photos/2026/ --versions # Include uploaded bytes from incomplete multipart uploads -rc du local/photos --fallback --incomplete +rc du local/photos --incomplete + +# Read the cluster snapshot without initiating a scan +rc du local --snapshot ``` ## Client-scan semantics -The fallback uses S3 pagination until completion: +The scan uses these S3 operations: - `ListBuckets` determines cluster scope; - `ListObjectsV2` counts current objects and bytes by default; - `ListObjectVersions` replaces current-object byte enumeration when `--versions` is selected; - `ListMultipartUploads` and `ListParts` supply incomplete-upload counts and bytes when `--incomplete` is selected. -With `--versions`, `object_count` counts current non-delete objects, `version_count` counts non-delete versions, and `delete_marker_count` counts delete markers. Incomplete uploaded-part bytes are included in `total_bytes` when `--incomplete` is selected and are also exposed separately as `incomplete_upload_bytes`. +With `--versions`, `object_count` counts current non-delete objects, `version_count` counts non-delete versions, and `delete_marker_count` counts delete markers. Scans exclude zero-byte directory markers (keys ending in `/`) from object and version counts. Ordinary zero-byte files and objects with a nonzero payload count normally. Incomplete uploaded-part bytes are included in `total_bytes` when `--incomplete` is selected and are also exposed separately as `incomplete_upload_bytes`. + +Directory-scoped `--incomplete` relies on the backend honoring a directory prefix in `ListMultipartUploads`. In a test with MinIO RELEASE.2025-09-06T17-38-46Z, listing the bucket returned an open upload for `logs/open`, but requesting the prefix `logs/` returned an empty list. Directory totals on that release can therefore omit uploaded parts. For an alias-wide scan, a bucket-specific failure does not discard successfully collected buckets. Output is marked `partial`, includes sorted failure records, and the process exits with retryable network code 3. A bucket- or prefix-scoped failure returns its mapped authentication, not-found, or network exit directly. @@ -84,10 +84,10 @@ All three commands use [output schema v3](output.md). Health records use the `he ## Compatibility -| Backend | `ping` / `ready` | `du` fast path | `du --fallback` | +| Backend | `ping` / `ready` | Default `du` | `du --snapshot` | | --- | --- | --- | --- | -| RustFS 1.0.0-beta.10 | `/health`, `/health/ready` | Supported | Supported through S3 APIs | -| Other RustFS versions | Supported when documented health routes exist | Used only when capability discovery advertises it | Supported through S3 APIs | -| AWS S3, MinIO, other S3-compatible services | RustFS health routes are not assumed | Not used | Best effort, subject to the service's list/version/multipart support and permissions | +| RustFS 1.0.0-beta.10 | `/health`, `/health/ready` | S3 listing | Supported | +| Other RustFS versions | Supported when documented health routes exist | S3 listing | Requires the advertised data-usage capability | +| AWS S3, MinIO, other S3-compatible services | RustFS health routes are not assumed | Subject to the service's list/version/multipart support and permissions | Requires RustFS admin APIs | Prometheus scraping and control of the RustFS background scanner are intentionally outside these commands.