diff --git a/Cargo.lock b/Cargo.lock index a360400f..41d29649 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,16 +170,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "assert-json-diff" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" -dependencies = [ - "serde", - "serde_json", -] - [[package]] name = "async-trait" version = "0.1.92" @@ -225,7 +215,7 @@ dependencies = [ "bytes", "fastrand", "hex", - "http 1.5.0", + "http", "sha1 0.10.7", "time", "tokio", @@ -287,7 +277,7 @@ dependencies = [ "bytes", "bytes-utils", "fastrand", - "http 1.5.0", + "http", "http-body", "percent-encoding", "pin-project-lite", @@ -321,7 +311,7 @@ dependencies = [ "fastrand", "hex", "hmac", - "http 1.5.0", + "http", "http-body", "lru", "percent-encoding", @@ -351,7 +341,7 @@ dependencies = [ "aws-types", "bytes", "fastrand", - "http 1.5.0", + "http", "regex-lite", "tracing", ] @@ -376,7 +366,7 @@ dependencies = [ "aws-types", "bytes", "fastrand", - "http 1.5.0", + "http", "regex-lite", "tracing", ] @@ -402,7 +392,7 @@ dependencies = [ "aws-smithy-xml", "aws-types", "fastrand", - "http 1.5.0", + "http", "regex-lite", "tracing", ] @@ -422,7 +412,7 @@ dependencies = [ "form_urlencoded", "hex", "hmac", - "http 1.5.0", + "http", "percent-encoding", "sha2 0.11.1", "time", @@ -451,7 +441,7 @@ dependencies = [ "bytes", "crc-fast", "hex", - "http 1.5.0", + "http", "http-body", "http-body-util", "md-5 0.11.0", @@ -485,7 +475,7 @@ dependencies = [ "bytes-utils", "futures-core", "futures-util", - "http 1.5.0", + "http", "http-body", "http-body-util", "percent-encoding", @@ -500,23 +490,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "859a81364f8185349c584fb41cd8765a400cf982f447adada590cf008a9f3b85" dependencies = [ "aws-smithy-async", - "aws-smithy-protocol-test", "aws-smithy-runtime-api", "aws-smithy-types", - "bytes", "h2", - "http 1.5.0", - "http-body", + "http", "hyper", "hyper-rustls", "hyper-util", - "indexmap", "pin-project-lite", "rustls", "rustls-native-certs", "rustls-pki-types", - "serde", - "serde_json", "tokio", "tokio-rustls", "tower", @@ -545,25 +529,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws-smithy-protocol-test" -version = "0.64.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f76511a0e223ce78deb6a78b8afebda99cb737cfbc8a58d96dcb190f012dd40a" -dependencies = [ - "assert-json-diff", - "aws-smithy-runtime-api", - "base64-simd", - "cbor-diag", - "ciborium", - "http 0.2.12", - "pretty_assertions", - "regex-lite", - "roxmltree", - "serde_json", - "thiserror", -] - [[package]] name = "aws-smithy-query" version = "0.64.0" @@ -592,7 +557,7 @@ dependencies = [ "aws-smithy-types", "bytes", "fastrand", - "http 1.5.0", + "http", "http-body", "http-body-util", "pin-project-lite", @@ -610,7 +575,7 @@ dependencies = [ "aws-smithy-runtime-api-macros", "aws-smithy-types", "bytes", - "http 1.5.0", + "http", "pin-project-lite", "tokio", "tracing", @@ -636,7 +601,7 @@ checksum = "f7ef30d39e131651359b60a1c560de9a9502a4c3754172ab2d6e8734fe21bcba" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", - "http 1.5.0", + "http", ] [[package]] @@ -649,7 +614,7 @@ dependencies = [ "bytes", "bytes-utils", "futures-core", - "http 1.5.0", + "http", "http-body", "http-body-util", "itoa", @@ -761,15 +726,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" -[[package]] -name = "bs58" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" -dependencies = [ - "tinyvec", -] - [[package]] name = "bumpalo" version = "3.20.3" @@ -798,25 +754,6 @@ dependencies = [ "either", ] -[[package]] -name = "cbor-diag" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc245b6ecd09b23901a4fbad1ad975701fd5061ceaef6afa93a2d70605a64429" -dependencies = [ - "bs58", - "chrono", - "data-encoding", - "half", - "nom", - "num-bigint", - "num-rational", - "num-traits", - "separator", - "url", - "uuid", -] - [[package]] name = "cc" version = "1.11.1" @@ -852,42 +789,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "chrono" -version = "0.4.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" -dependencies = [ - "num-traits", -] - -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - [[package]] name = "cipher" version = "0.4.4" @@ -1088,12 +989,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.7" @@ -1169,12 +1064,6 @@ version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - [[package]] name = "digest" version = "0.10.7" @@ -1387,6 +1276,21 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fundu" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47ed3102262b937cdbc46cc412c58f2243b8d13989657d0c9f027eb03168aca" +dependencies = [ + "fundu-core", +] + +[[package]] +name = "fundu-core" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "358c273f0d54362c071e0d60af2ed872c264697ceccaa8e43b73abc508142830" + [[package]] name = "futures" version = "0.3.34" @@ -1553,7 +1457,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.5.0", + "http", "indexmap", "slab", "tokio", @@ -1561,17 +1465,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -1604,17 +1497,6 @@ dependencies = [ "digest 0.11.3", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.5.0" @@ -1632,7 +1514,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", - "http 1.5.0", + "http", ] [[package]] @@ -1643,7 +1525,7 @@ checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", - "http 1.5.0", + "http", "http-body", "pin-project-lite", ] @@ -1663,12 +1545,6 @@ dependencies = [ "libm", ] -[[package]] -name = "humantime" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" - [[package]] name = "hybrid-array" version = "0.4.15" @@ -1689,7 +1565,7 @@ dependencies = [ "futures-channel", "futures-core", "h2", - "http 1.5.0", + "http", "http-body", "httparse", "itoa", @@ -1705,7 +1581,7 @@ version = "0.27.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" dependencies = [ - "http 1.5.0", + "http", "hyper", "hyper-util", "rustls", @@ -1726,7 +1602,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.5.0", + "http", "http-body", "httparse", "hyper", @@ -1852,8 +1728,6 @@ checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown", - "serde", - "serde_core", ] [[package]] @@ -2537,16 +2411,6 @@ dependencies = [ "termtree", ] -[[package]] -name = "pretty_assertions" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" -dependencies = [ - "diff", - "yansi", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -2712,7 +2576,7 @@ dependencies = [ "getrandom 0.4.3", "glob", "hex", - "http 1.5.0", + "http", "humansize", "jiff", "mockall", @@ -2745,17 +2609,15 @@ dependencies = [ "aws-sdk-s3", "aws-sigv4", "aws-smithy-checksums", - "aws-smithy-http-client", "aws-smithy-runtime-api", "aws-smithy-types", "base64 0.22.1", "bytes", "futures", "hex", - "http 1.5.0", + "http", "http-body", "http-body-util", - "humantime", "jiff", "md-5 0.10.6", "mockall", @@ -2875,7 +2737,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 1.5.0", + "http", "http-body", "http-body-util", "hyper", @@ -2921,15 +2783,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "roxmltree" -version = "0.14.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921904a62e410e37e215c40381b7117f830d9d89ba60ab5236170541dd25646b" -dependencies = [ - "xmlparser", -] - [[package]] name = "rsa" version = "0.9.10" @@ -2976,10 +2829,10 @@ dependencies = [ "clap_complete", "comfy-table", "console", + "fundu", "futures", "glob", "humansize", - "humantime", "indicatif", "insta", "jiff", @@ -3129,12 +2982,6 @@ version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -[[package]] -name = "separator" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f97841a747eef040fcd2e7b3b9a220a7205926e60488e673d9e4926d27772ce5" - [[package]] name = "serde" version = "1.0.229" @@ -3171,8 +3018,6 @@ version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ - "foldhash", - "indexmap", "itoa", "memchr", "serde", @@ -3654,7 +3499,7 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.5.0", + "http", "http-body", "pin-project-lite", "tower", @@ -4171,12 +4016,6 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "yoke" version = "0.8.4" diff --git a/Cargo.toml b/Cargo.toml index f0e16b5a..9c8940b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ console = "0.16" dirs = "6.0" jiff = { version = "0.2", features = ["serde"] } humansize = "2.1" -humantime = "2.3" +fundu = { version = "2.1.1", default-features = false, features = ["custom"] } bytes = "1.11" url = "2.5" futures = "0.3" diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 890789c2..f44fea50 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -50,7 +50,7 @@ console.workspace = true # Utilities jiff.workspace = true humansize.workspace = true -humantime.workspace = true +fundu.workspace = true mime_guess.workspace = true tempfile.workspace = true glob.workspace = true diff --git a/crates/cli/src/commands/admin/replicate.rs b/crates/cli/src/commands/admin/replicate.rs index 4cc3be31..72be9176 100644 --- a/crates/cli/src/commands/admin/replicate.rs +++ b/crates/cli/src/commands/admin/replicate.rs @@ -17,11 +17,12 @@ use super::{get_admin_client, normalize_admin_alias}; use crate::exit_code::ExitCode; use crate::output::Formatter; use rc_core::admin::{ - AdminApi, PeerSiteSpec, ReplicateEditStatus, SiteRemoveSpec, SiteReplicationInfo, - SiteReplicationPeer, SiteReplicationRepairApi, SiteReplicationRepairOperationStatus, - SiteReplicationRepairPreflight, SiteReplicationResyncOperation, SiteReplicationResyncStatus, - SiteStatusOptions, validate_site_replication_ca_bundle, - validate_site_replication_repair_operation_id, validate_site_replication_repair_token, + AdminApi, PeerSiteSpec, ReplicateEditStatus, SiteRemoveSpec, SiteRemoveV2Spec, + SiteReplicationInfo, SiteReplicationPeer, SiteReplicationRepairApi, + SiteReplicationRepairOperationStatus, SiteReplicationRepairPreflight, + SiteReplicationResyncOperation, SiteReplicationResyncStatus, SiteStatusOptions, + validate_site_replication_ca_bundle, validate_site_replication_repair_operation_id, + validate_site_replication_repair_token, }; use rc_core::{AliasManager, Error}; use rc_s3::AdminClient; @@ -247,6 +248,22 @@ pub struct RemoveArgs { /// Remove all sites (dissolve the cluster) #[arg(long)] pub all: bool, + + /// Permanently retire the selected remote sites through the durable v2 protocol + #[arg(long)] + pub force: bool, + + /// Remote sites allowed to miss acknowledgements when dissolving all sites + #[arg(long = "decommissioned-site", requires_all = ["all", "force"])] + pub decommissioned_sites: Vec, + + /// Resume an exact durable removal operation + #[arg(long, requires = "force")] + pub operation_id: Option, + + /// Confirm permanent decommissioning + #[arg(long, requires = "force")] + pub yes: bool, } /// Execute a site replication subcommand @@ -1581,11 +1598,86 @@ async fn execute_remove(args: RemoveArgs, formatter: &Formatter) -> ExitCode { return ExitCode::UsageError; } + if args.force && (!args.yes || (args.all && args.decommissioned_sites.is_empty())) { + formatter.error( + "Permanent decommissioning requires --yes; --all also requires --decommissioned-site", + ); + return ExitCode::UsageError; + } + let client = match get_admin_client(&args.alias, formatter) { Ok(c) => c, Err(code) => return code, }; + if args.force { + let mut request = SiteRemoveV2Spec { + mode: "dry-run".to_string(), + sites: args.sites, + all: args.all, + decommissioned_sites: args.decommissioned_sites, + operation_id: args.operation_id, + preflight_token: None, + }; + let preview = match client.site_replication_remove_v2(&request).await { + Ok(preview) => preview, + Err(err) => { + return emit_admin_error(formatter, &err, "site-replication-decommission", false); + } + }; + if !formatter.is_json() { + formatter.println(&format!( + "Removal deployments: {}", + preview.removed_deployment_ids.join(", ") + )); + formatter.println(&format!( + "Decommissioned deployments: {}", + preview.decommissioned_deployment_ids.join(", ") + )); + formatter.println(&format!("Operation ID: {}", preview.operation_id)); + } + request.mode = "execute".to_string(); + request.operation_id = Some(preview.operation_id.clone()); + request.preflight_token = Some(preview.preflight_token); + return match client.site_replication_remove_v2(&request).await { + Ok(result) => { + let complete = result.status == "success"; + if formatter.is_json() { + formatter.json(&result); + } else { + formatter.println(&format!("Site decommissioning: {}", result.status)); + formatter.println(&format!("Operation ID: {}", result.operation_id)); + formatter + .println("Configuration cleanup is independent of object-version cleanup."); + for error in &result.errors { + formatter.error(error); + } + if !complete { + formatter.println("Resume with the same selectors and --operation-id."); + } + } + if complete { + ExitCode::Success + } else { + ExitCode::Conflict + } + } + Err(err) => { + let code = ExitCode::from_i32(err.exit_code()).unwrap_or(ExitCode::GeneralError); + let id = request.operation_id.as_deref().unwrap_or("unknown"); + emit_admin_message_error( + formatter, + code, + "site-replication-decommission", + true, + format!( + "{err}; operation ID: {id}; inspect or resume with the same selectors and --operation-id {id}" + ), + ) + } + }; + } + let spec = SiteRemoveSpec { site_names: args.sites.clone(), remove_all: args.all, diff --git a/crates/cli/src/commands/replicate.rs b/crates/cli/src/commands/replicate.rs index c9031a0c..0dc606f8 100644 --- a/crates/cli/src/commands/replicate.rs +++ b/crates/cli/src/commands/replicate.rs @@ -2579,9 +2579,38 @@ fn validate_reset_id(reset_id: &str) -> Result<(), Error> { } fn parse_resync_duration(value: &str) -> Result { - let duration = humantime::parse_duration(value).map_err(|error| { + use fundu::{CustomDurationParser, CustomTimeUnit, Multiplier, TimeUnit::*}; + let mut parser = CustomDurationParser::with_time_units(&[ + CustomTimeUnit::with_default(NanoSecond, &["nanos", "nsec", "ns"]), + CustomTimeUnit::with_default(MicroSecond, &["usec", "us", "µs"]), + CustomTimeUnit::with_default(MilliSecond, &["millis", "msec", "ms"]), + CustomTimeUnit::with_default(Second, &["seconds", "second", "secs", "sec", "s"]), + CustomTimeUnit::with_default(Minute, &["minutes", "minute", "min", "mins", "m"]), + CustomTimeUnit::with_default(Hour, &["hours", "hour", "hr", "hrs", "h"]), + CustomTimeUnit::with_default(Day, &["days", "day", "d"]), + CustomTimeUnit::with_default(Week, &["weeks", "week", "wk", "wks", "w"]), + CustomTimeUnit::new( + Second, + &["months", "month", "M"], + Some(Multiplier(2_630_016, 0)), + ), + CustomTimeUnit::with_default(Year, &["years", "year", "yr", "yrs", "y"]), + ]); + parser + .allow_time_unit_delimiter(true) + .disable_exponent(true) + .parse_multiple(true, None); + if !resync_duration_components_have_units(value) { + return Err(Error::InvalidPath( + "--older-than requires an unsigned duration with a time unit".to_string(), + )); + } + let parsed = parser.parse(value.trim()).map_err(|error| { Error::InvalidPath(format!("Invalid --older-than duration '{value}': {error}")) })?; + let duration: std::time::Duration = parsed + .try_into() + .map_err(|_| Error::InvalidPath("--older-than exceeds the duration range".to_string()))?; if duration.is_zero() { return Err(Error::InvalidPath( "--older-than must be greater than zero".to_string(), @@ -2595,6 +2624,58 @@ fn parse_resync_duration(value: &str) -> Result { Ok(duration) } +fn resync_duration_components_have_units(value: &str) -> bool { + let mut characters = value.trim().chars().peekable(); + if characters.peek().is_none() { + return false; + } + while characters.peek().is_some() { + let mut digits = 0; + while characters.peek().is_some_and(char::is_ascii_digit) { + characters.next(); + digits += 1; + } + if digits == 0 { + return false; + } + if characters.peek() == Some(&'.') { + characters.next(); + let mut fraction = 0; + while characters.peek().is_some_and(char::is_ascii_digit) { + characters.next(); + fraction += 1; + } + if fraction == 0 { + return false; + } + } + while characters + .peek() + .is_some_and(|character| character.is_whitespace()) + { + characters.next(); + } + let mut unit = 0; + while characters + .peek() + .is_some_and(|character| character.is_alphabetic() || *character == 'µ') + { + characters.next(); + unit += 1; + } + if unit == 0 { + return false; + } + while characters + .peek() + .is_some_and(|character| character.is_whitespace()) + { + characters.next(); + } + } + true +} + // ==================== Helpers ==================== fn parse_bucket_path(path: &str) -> Result<(String, String), String> { @@ -3528,6 +3609,35 @@ mod tests { assert!(parse_resync_duration("9223372036854775808s").is_err()); } + #[test] + fn resync_duration_preserves_compound_units_aliases_and_month_length() { + for (input, expected) in [ + ( + "7d10h31s", + std::time::Duration::from_secs(7 * 86400 + 10 * 3600 + 31), + ), + ("2 hours 37 mins", std::time::Duration::from_secs(9420)), + ("1month", std::time::Duration::from_secs(2_630_016)), + ("4.2s", std::time::Duration::new(4, 200_000_000)), + ("1usec 2ns", std::time::Duration::from_nanos(1002)), + ] { + assert_eq!(parse_resync_duration(input).expect(input), expected); + } + for invalid in [ + "123", + "1h 2", + "1 2h", + ".3s", + "1.s", + "1e3s", + "+1h", + "1fortnight", + "1h,2m", + ] { + assert!(parse_resync_duration(invalid).is_err(), "{invalid}"); + } + } + #[test] fn replication_v3_status_output_preserves_server_state() { let target = status_target_output(ReplicationResyncTargetStatus { diff --git a/crates/cli/tests/admin_replicate.rs b/crates/cli/tests/admin_replicate.rs index 75c622aa..a6005201 100644 --- a/crates/cli/tests/admin_replicate.rs +++ b/crates/cli/tests/admin_replicate.rs @@ -489,6 +489,165 @@ fn replicate_remove_requires_site_or_all() { assert_eq!(output.status.code(), Some(2)); } +#[test] +fn replicate_remove_force_requires_explicit_confirmation_and_all_exclusions() { + for args in [ + vec!["--site", "secondary", "--force"], + vec!["--all", "--force", "--yes"], + ] { + let output = Command::new(rc_binary()) + .args(["admin", "replicate", "remove", "myalias"]) + .args(args) + .output() + .expect("run validation"); + assert_eq!(output.status.code(), Some(2)); + } +} + +const REMOVE_V2_PREVIEW: &str = r#"{"contractVersion":2,"operationId":"550e8400-e29b-41d4-a716-446655440000","status":"planned","preflightToken":"preview-token","removedDeploymentIds":["secondary-dep"],"decommissionedDeploymentIds":["secondary-dep"],"pendingPeers":[],"localCleanupComplete":false,"errors":[]}"#; +const REMOVE_V2_SUCCESS: &str = r#"{"contractVersion":2,"operationId":"550e8400-e29b-41d4-a716-446655440000","status":"success","preflightToken":"preview-token","removedDeploymentIds":["secondary-dep"],"decommissionedDeploymentIds":["secondary-dep"],"pendingPeers":[],"localCleanupComplete":true,"errors":[]}"#; +const REMOVE_V2_PARTIAL: &str = r#"{"contractVersion":2,"operationId":"550e8400-e29b-41d4-a716-446655440000","status":"Partial","preflightToken":"preview-token","removedDeploymentIds":["secondary-dep"],"decommissionedDeploymentIds":["secondary-dep"],"pendingPeers":["third-dep"],"localCleanupComplete":true,"errors":["peer unavailable"]}"#; + +#[test] +fn replicate_remove_force_uses_preview_and_v2_without_legacy_fallback() { + for (response, exit) in [(REMOVE_V2_SUCCESS, 0), (REMOVE_V2_PARTIAL, 6)] { + let (endpoint, requests, server) = start_admin_sequence_test_server(vec![ + ("200 OK", REMOVE_V2_PREVIEW), + ("200 OK", response), + ]); + let config = tempfile::tempdir().expect("config"); + let output = Command::new(rc_binary()) + .args([ + "--json", + "admin", + "replicate", + "remove", + "myalias", + "--site", + "secondary", + "--force", + "--yes", + ]) + .env("RC_CONFIG_DIR", config.path()) + .env("RC_HOST_myalias", rc_host_alias(&endpoint)) + .output() + .expect("run decommission"); + assert_eq!( + output.status.code(), + Some(exit), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + for mode in ["dry-run", "execute"] { + let request = requests + .recv_timeout(Duration::from_secs(5)) + .expect("request"); + assert_eq!( + request.target, + "/rustfs/admin/v3/site-replication/remove-v2" + ); + let body: serde_json::Value = serde_json::from_slice(&request.body).expect("JSON"); + assert_eq!(body["mode"], mode); + assert_eq!(body["sites"], serde_json::json!(["secondary"])); + if mode == "execute" { + assert_eq!(body["preflightToken"], "preview-token"); + } + } + let result: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("JSON output"); + assert_eq!( + result["status"], + if exit == 0 { "success" } else { "Partial" } + ); + server.join().expect("server finished"); + } +} + +#[test] +fn replicate_remove_force_rejects_old_server_without_sending_legacy_remove() { + let (endpoint, requests, server) = + start_admin_sequence_test_server(vec![("404 Not Found", "{}")]); + let config = tempfile::tempdir().expect("config"); + let output = Command::new(rc_binary()) + .args([ + "admin", + "replicate", + "remove", + "myalias", + "--site", + "secondary", + "--force", + "--yes", + ]) + .env("RC_CONFIG_DIR", config.path()) + .env("RC_HOST_myalias", rc_host_alias(&endpoint)) + .output() + .expect("run"); + assert_eq!(output.status.code(), Some(7)); + assert_eq!( + requests + .recv_timeout(Duration::from_secs(5)) + .expect("request") + .target, + "/rustfs/admin/v3/site-replication/remove-v2" + ); + server.join().expect("server finished"); +} + +#[test] +fn replicate_remove_force_retains_operation_id_when_execute_response_is_invalid() { + for json in [false, true] { + let (endpoint, requests, server) = start_admin_sequence_test_server(vec![ + ("200 OK", REMOVE_V2_PREVIEW), + ("200 OK", "invalid JSON"), + ]); + let config = tempfile::tempdir().expect("config"); + let mut command = Command::new(rc_binary()); + if json { + command.arg("--json"); + } + let output = command + .args([ + "admin", + "replicate", + "remove", + "myalias", + "--site", + "secondary", + "--force", + "--yes", + ]) + .env("RC_CONFIG_DIR", config.path()) + .env("RC_HOST_myalias", rc_host_alias(&endpoint)) + .output() + .expect("run"); + assert!(!output.status.success()); + let text = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + text.contains("550e8400-e29b-41d4-a716-446655440000"), + "{text}" + ); + assert!(text.contains("outcome is unknown"), "{text}"); + assert!(text.contains("--operation-id"), "{text}"); + if !json { + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("Removal deployments: secondary-dep")); + assert!(stdout.contains("Decommissioned deployments: secondary-dep")); + } + requests + .recv_timeout(Duration::from_secs(5)) + .expect("preview"); + requests + .recv_timeout(Duration::from_secs(5)) + .expect("execute"); + server.join().expect("server finished"); + } +} + #[test] fn replicate_info_json_uses_safe_projection() { let config_dir = tempfile::tempdir().expect("create config dir"); diff --git a/crates/core/src/admin/mod.rs b/crates/core/src/admin/mod.rs index 698684e9..7af2284a 100644 --- a/crates/core/src/admin/mod.rs +++ b/crates/core/src/admin/mod.rs @@ -138,14 +138,14 @@ pub use site::{ MAX_SITE_REPLICATION_CA_CERT_BYTES, MAX_SITE_REPLICATION_ERROR_RESPONSE_BYTES, MAX_SITE_REPLICATION_REPAIR_RESPONSE_BYTES, MAX_SITE_REPLICATION_REQUEST_BYTES, MAX_SITE_REPLICATION_SUCCESS_RESPONSE_BYTES, PeerSiteSpec, ReplicateEditStatus, - SITE_REPLICATION_REPAIR_CAPABILITY, ServiceActionResult, SiteRemoveSpec, SiteReplicationInfo, - SiteReplicationPeer, SiteReplicationRepairApi, SiteReplicationRepairCapabilityContract, - SiteReplicationRepairFamilyStatus, SiteReplicationRepairOperationStatus, - SiteReplicationRepairPreflight, SiteReplicationRepairRequest, SiteReplicationRepairSiteStatus, - SiteReplicationRepairTaskStatus, SiteReplicationResyncBucketStatus, - SiteReplicationResyncOperation, SiteReplicationResyncStatus, SiteStatusOptions, - validate_site_replication_ca_bundle, validate_site_replication_repair_operation_id, - validate_site_replication_repair_token, + SITE_REPLICATION_REPAIR_CAPABILITY, ServiceActionResult, SiteRemoveSpec, SiteRemoveV2Spec, + SiteRemoveV2Status, SiteReplicationInfo, SiteReplicationPeer, SiteReplicationRepairApi, + SiteReplicationRepairCapabilityContract, SiteReplicationRepairFamilyStatus, + SiteReplicationRepairOperationStatus, SiteReplicationRepairPreflight, + SiteReplicationRepairRequest, SiteReplicationRepairSiteStatus, SiteReplicationRepairTaskStatus, + SiteReplicationResyncBucketStatus, SiteReplicationResyncOperation, SiteReplicationResyncStatus, + SiteStatusOptions, validate_site_replication_ca_bundle, + validate_site_replication_repair_operation_id, validate_site_replication_repair_token, }; pub use tier::{ ManualTransitionJobResponse, ManualTransitionQueueSnapshot, ManualTransitionRunReport, @@ -420,6 +420,16 @@ pub trait AdminApi: Send + Sync { /// Remove sites from the site replication cluster async fn site_replication_remove(&self, spec: &SiteRemoveSpec) -> Result; + + /// Preview or execute durable decommissioning. Never falls back to legacy removal. + async fn site_replication_remove_v2( + &self, + _spec: &SiteRemoveV2Spec, + ) -> Result { + Err(crate::Error::UnsupportedFeature( + "Durable site decommissioning is unavailable".to_string(), + )) + } } /// Read-only RustFS runtime capability discovery. diff --git a/crates/core/src/admin/site.rs b/crates/core/src/admin/site.rs index c3430a46..6a9ff1e4 100644 --- a/crates/core/src/admin/site.rs +++ b/crates/core/src/admin/site.rs @@ -861,6 +861,35 @@ pub struct SiteRemoveSpec { pub remove_all: bool, } +/// Explicit durable decommissioning request for the native v2 route. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SiteRemoveV2Spec { + pub mode: String, + pub sites: Vec, + pub all: bool, + pub decommissioned_sites: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub operation_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub preflight_token: Option, +} + +/// Configuration cleanup progress; success does not mean the bucket is empty. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct SiteRemoveV2Status { + pub contract_version: u32, + pub operation_id: String, + pub status: String, + pub preflight_token: String, + pub removed_deployment_ids: Vec, + pub decommissioned_deployment_ids: Vec, + pub pending_peers: Vec, + pub local_cleanup_complete: bool, + pub errors: Vec, +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/s3/Cargo.toml b/crates/s3/Cargo.toml index f5d6acee..37fd6282 100644 --- a/crates/s3/Cargo.toml +++ b/crates/s3/Cargo.toml @@ -37,7 +37,6 @@ tracing.workspace = true # Utilities jiff.workspace = true -humantime.workspace = true bytes.workspace = true base64.workspace = true url.workspace = true @@ -61,5 +60,4 @@ serde_json.workspace = true quick-xml.workspace = true [dev-dependencies] -aws-smithy-http-client = { version = "1.1.5", features = ["test-util"] } mockall.workspace = true diff --git a/crates/s3/src/admin.rs b/crates/s3/src/admin.rs index da6263c1..20930001 100644 --- a/crates/s3/src/admin.rs +++ b/crates/s3/src/admin.rs @@ -5642,6 +5642,44 @@ impl AdminApi for AdminClient { self.request(Method::PUT, "/site-replication/remove", None, Some(&body)) .await } + + async fn site_replication_remove_v2( + &self, + spec: &rc_core::admin::SiteRemoveV2Spec, + ) -> Result { + let body = serde_json::to_vec(spec).map_err(Error::Json)?; + let result: rc_core::admin::SiteRemoveV2Status = self + .request_site_replication( + Method::PUT, + "/site-replication/remove-v2", + Some(&body), + "Site decommissioning", + (spec.mode == "execute").then_some("Site decommissioning"), + (spec.mode == "execute").then_some("Site decommissioning"), + ) + .await?; + if result.contract_version != 2 + || !matches!(result.status.as_str(), "planned" | "Partial" | "success") + || result.operation_id.is_empty() + || result.preflight_token.is_empty() + || spec + .operation_id + .as_ref() + .is_some_and(|id| *id != result.operation_id) + || result.decommissioned_deployment_ids.is_empty() + || result + .decommissioned_deployment_ids + .iter() + .any(|id| !result.removed_deployment_ids.contains(id)) + || (result.status == "success" + && (!result.local_cleanup_complete || !result.pending_peers.is_empty())) + { + return Err(Error::General( + "Invalid durable decommissioning response".to_string(), + )); + } + Ok(result) + } } #[async_trait] diff --git a/crates/s3/src/client.rs b/crates/s3/src/client.rs index 92604eec..53bfbeda 100644 --- a/crates/s3/src/client.rs +++ b/crates/s3/src/client.rs @@ -81,6 +81,38 @@ use crate::lifecycle_xml::{ const SINGLE_PUT_OBJECT_MAX_SIZE: u64 = crate::multipart::DEFAULT_PART_SIZE; const S3_SERVICE_NAME: &str = "s3"; const S3_REPLICATION_XML_NAMESPACE: &str = "http://s3.amazonaws.com/doc/2006-03-01/"; + +fn format_resync_duration(duration: Duration) -> String { + let mut remaining = duration.as_secs(); + let mut parts = Vec::new(); + for (seconds, suffix) in [ + (31_557_600, "years"), + (2_630_016, "months"), + (86_400, "days"), + (3600, "h"), + (60, "m"), + (1, "s"), + ] { + let count = remaining / seconds; + if count > 0 { + parts.push(format!("{count}{suffix}")); + remaining %= seconds; + } + } + let mut nanos = duration.subsec_nanos(); + for (unit, suffix) in [(1_000_000, "ms"), (1000, "us"), (1, "ns")] { + let count = nanos / unit; + if count > 0 { + parts.push(format!("{count}{suffix}")); + nanos %= unit; + } + } + if parts.is_empty() { + "0s".to_string() + } else { + parts.join(" ") + } +} const RUSTFS_FORCE_DELETE_HEADER: &str = "x-rustfs-force-delete"; const REPLICATION_EXTENSION_BODY_LIMIT: u64 = 1024 * 1024; const REPLICATION_CHECK_PROBE_NAMESPACE: &str = ".rustfs.sys/replication-check/"; @@ -7564,10 +7596,7 @@ impl ObjectStore for S3Client { query.push(("arn", target_arn)); } if let Some(older_than) = options.older_than { - query.push(( - "older-than", - humantime::format_duration(older_than).to_string(), - )); + query.push(("older-than", format_resync_duration(older_than))); } if let Some(reset_id) = options.reset_id { query.push(("reset-id", reset_id)); @@ -7800,7 +7829,7 @@ impl ControlExecutor for S3Client { #[cfg(test)] mod tests { use super::*; - use aws_smithy_http_client::test_util::{ + use crate::test_http::{ CaptureRequestReceiver, ReplayEvent, StaticReplayClient, capture_request, }; use rc_core::{ @@ -7819,6 +7848,20 @@ mod tests { use std::time::{Duration, Instant}; use tokio::sync::Notify; + #[test] + fn resync_duration_formatter_preserves_every_nanosecond() { + assert_eq!(format_resync_duration(Duration::from_secs(3600)), "1h"); + assert_eq!( + format_resync_duration(Duration::new(3661, 123_456_789)), + "1h 1m 1s 123ms 456us 789ns" + ); + assert_eq!(format_resync_duration(Duration::ZERO), "0s"); + assert_eq!( + format_resync_duration(Duration::from_secs(31_557_600 + 2_630_016)), + "1years 1months" + ); + } + #[derive(Debug)] struct CapturedXmlRequest { method: String, diff --git a/crates/s3/src/lib.rs b/crates/s3/src/lib.rs index 835ed003..1da85761 100644 --- a/crates/s3/src/lib.rs +++ b/crates/s3/src/lib.rs @@ -13,6 +13,8 @@ pub mod multipart; mod ops; mod payload_checksum; mod select; +#[cfg(test)] +mod test_http; mod watch; pub use admin::AdminClient; diff --git a/crates/s3/src/test_http.rs b/crates/s3/src/test_http.rs new file mode 100644 index 00000000..c5e239f2 --- /dev/null +++ b/crates/s3/src/test_http.rs @@ -0,0 +1,106 @@ +//! SDK test connectors owned by rc; request capture needs no protocol-format dependency. + +use aws_smithy_runtime_api::client::http::{ + HttpClient, HttpConnector, HttpConnectorFuture, HttpConnectorSettings, SharedHttpConnector, +}; +use aws_smithy_runtime_api::client::orchestrator::{HttpRequest, HttpResponse}; +use aws_smithy_runtime_api::client::result::ConnectorError; +use aws_smithy_runtime_api::client::runtime_components::RuntimeComponents; +use aws_smithy_types::body::SdkBody; +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; + +#[derive(Debug, Clone)] +pub(crate) struct StaticReplayClient { + responses: Arc>>, + requests: Arc>>>, +} + +pub(crate) struct ReplayEvent { + response: HttpResponse, +} + +impl ReplayEvent { + pub(crate) fn new(_request: http::Request, response: http::Response) -> Self { + Self { + response: response.try_into().expect("valid SDK response"), + } + } +} + +impl StaticReplayClient { + pub(crate) fn new(events: Vec) -> Self { + Self { + responses: Arc::new(Mutex::new( + events.into_iter().map(|event| event.response).collect(), + )), + requests: Arc::new(Mutex::new(Vec::new())), + } + } + + pub(crate) fn actual_requests(&self) -> impl Iterator> + use<> { + self.requests + .lock() + .expect("request capture lock") + .clone() + .into_iter() + } +} + +impl HttpConnector for StaticReplayClient { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + self.requests + .lock() + .expect("request capture lock") + .push(Arc::new(request)); + let response = self + .responses + .lock() + .expect("response queue lock") + .pop_front(); + HttpConnectorFuture::ready( + response.ok_or_else(|| ConnectorError::other("no test response remains".into(), None)), + ) + } +} + +impl HttpClient for StaticReplayClient { + fn http_connector( + &self, + _settings: &HttpConnectorSettings, + _components: &RuntimeComponents, + ) -> SharedHttpConnector { + SharedHttpConnector::new(self.clone()) + } +} + +pub(crate) struct CaptureRequestReceiver { + client: StaticReplayClient, +} + +impl CaptureRequestReceiver { + pub(crate) fn expect_request(self) -> Arc { + let requests = self.client.requests.lock().expect("request capture lock"); + assert_eq!(requests.len(), 1, "expected exactly one SDK request"); + requests[0].clone() + } + + pub(crate) fn expect_no_request(self) { + assert_eq!( + self.client.actual_requests().count(), + 0, + "expected no SDK request" + ); + } +} + +pub(crate) fn capture_request( + response: Option>, +) -> (StaticReplayClient, CaptureRequestReceiver) { + let response = response.unwrap_or_else(|| http::Response::new(SdkBody::empty())); + let client = StaticReplayClient::new(vec![ReplayEvent::new( + http::Request::new(SdkBody::empty()), + response, + )]); + (client.clone(), CaptureRequestReceiver { client }) +} diff --git a/docs/site-replication-resync.md b/docs/site-replication-resync.md index daf61f26..d3e66005 100644 --- a/docs/site-replication-resync.md +++ b/docs/site-replication-resync.md @@ -1,5 +1,13 @@ # Site Replication Resync Snapshots +## Permanent site decommissioning + +Use `rc admin replicate remove primary --site "Site 2" --force --yes` for a permanently retired remote site. This uses the native durable removal v2 route: the CLI previews the exact deployment identities and deletion scope, then executes with the returned operation ID and preflight token. Unsupported servers are rejected without falling back to ordinary removal. + +To dissolve all site replication while skipping only explicitly retired peers, use `rc admin replicate remove primary --all --force --decommissioned-site "Site 2" --yes`. All other participating sites must acknowledge durable local configuration cleanup. A `Partial` result exits with conflict status and reports its operation ID; retry with the same selectors and `--operation-id `. + +An existing ordinary pending removal can be resumed through force only with its original deletion scope. Success means site-derived replication rules, targets, and resync intents have been cleaned; ordinary objects, historical versions, and Object Lock protections remain. Background replication purge recovery can subsequently release versions already awaiting deletion once their retired target is no longer configured. Do not downgrade surviving nodes while a durable removal is pending. + `rc` exposes the RustFS site-replication resync handler through three bounded commands: ```text