diff --git a/CHANGELOG.md b/CHANGELOG.md index d95c35db..7eb7d4fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ All notable changes to RustFS Operator are documented in this file. The format i ### Added +- Tenant `spec.readinessProbe` with optional initial delay, timeout, period, failure + threshold, and success threshold settings, preserved during reconciliation. - Full mutual TLS (client-to-Pod and Pod-to-Pod) support when `caTrust.clientCaSecretRef` is configured with cert-manager, including client certificate/key volume projections, automatic `client auth` usage on managed default certificates, mTLS-aware container probes, and operator @@ -58,6 +60,16 @@ All notable changes to RustFS Operator are documented in this file. The format i opt-out on Tenants whose budgets you manage yourself, or remove your own budgets that select pool Pods, because Kubernetes rejects evictions of Pods selected by more than one budget. +### Changed + +- The default Tenant readiness probe timeout is five seconds, increased from the + previous effective one-second default. Upgrades update existing StatefulSet Pod + templates. Tenants without `spec.serviceAccountName` use enforced `RollingUpdate` + with partition zero for default ServiceAccount token protection; Tenants with + a custom ServiceAccount retain their existing update strategy and partition. + Apply the updated Tenant CRD explicitly before upgrading the Operator or + configuring readiness fields; Helm does not upgrade installed CRDs. + ### Fixed - Check an existing Tenant bucket with HeadBucket before CreateBucket, so a bucket under diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2cec430c..01b8eccf 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,6 +86,35 @@ Optional quick compile (not a separate `make` target): cargo check --all-targets ``` +The readiness field-management and rollout tests use a real Kubernetes API and +are ignored by default. Run them against a disposable cluster with the current +Tenant CRD: + +```bash +kind create cluster --name rustfs-readiness --kubeconfig /tmp/rustfs-readiness.kubeconfig +export KUBECONFIG=/tmp/rustfs-readiness.kubeconfig +export RUSTFS_READINESS_TEST_CONTEXT=kind-rustfs-readiness +kubectl --context "$RUSTFS_READINESS_TEST_CONTEXT" apply --server-side -f deploy/rustfs-operator/crds/tenant-crd.yaml +cargo test --lib readiness_ssa_live_preserves_values_ownership_and_external_fields -- --ignored +docker pull python:3.12-slim +kind load docker-image python:3.12-slim --name rustfs-readiness +cargo test --lib readiness_rollout_live_updates_complete_templates -- --ignored +kind delete cluster --name rustfs-readiness +``` + +Both tests create and delete isolated namespaces. The field-management test uses +zero-replica StatefulSets and checks +readiness timing ownership, manual configuration migration, and preservation of +external rollout policies and unrelated field ownership. + +The rollout test uses two Python HTTP server Pods with `imagePullPolicy: Never`, +so `python:3.12-slim` must be available on every node. It submits readiness timing +and server configuration changes through the production update helper, observes +all stored StatefulSet templates and Pod events, and waits for both updated Pods +to become Ready without manual Pod deletion. It exercises `Parallel` and +`OrderedReady` policies with an external manager sharing the readiness timeout. +Neither test requires Operator or RustFS container images. + ### 🔒 Git hooks (optional) The repository does **not** ship a `make setup-hooks` target. To run checks automatically on `git commit`, add your own `.git/hooks/pre-commit` that invokes `make pre-commit` (or the individual commands above). diff --git a/deploy/rustfs-operator/crds/tenant-crd.yaml b/deploy/rustfs-operator/crds/tenant-crd.yaml index a2e10796..b043e1c4 100644 --- a/deploy/rustfs-operator/crds/tenant-crd.yaml +++ b/deploy/rustfs-operator/crds/tenant-crd.yaml @@ -3158,6 +3158,41 @@ spec: priorityClassName: nullable: true type: string + readinessProbe: + description: Readiness timing for every RustFS Pod. Omitted fields use operator defaults. + nullable: true + properties: + failureThreshold: + description: Consecutive failures required to report NotReady. Defaults to 3. + format: int32 + minimum: 1.0 + nullable: true + type: integer + initialDelaySeconds: + description: Seconds before readiness checks begin. Defaults to 0. + format: int32 + minimum: 0.0 + nullable: true + type: integer + periodSeconds: + description: Interval between readiness checks, in seconds. Defaults to 10. + format: int32 + minimum: 1.0 + nullable: true + type: integer + successThreshold: + description: Consecutive successes required to report Ready. Defaults to 1. + format: int32 + minimum: 1.0 + nullable: true + type: integer + timeoutSeconds: + description: Timeout for each readiness check, in seconds. Defaults to 5. + format: int32 + minimum: 1.0 + nullable: true + type: integer + type: object rpcSecret: description: |- Optional Secret key selector for RustFS internode RPC authentication. diff --git a/deploy/rustfs-tenant/README.md b/deploy/rustfs-tenant/README.md index 5dfc8c0b..eb786b02 100644 --- a/deploy/rustfs-tenant/README.md +++ b/deploy/rustfs-tenant/README.md @@ -79,6 +79,7 @@ has no `appVersion`: it does not select a RustFS image through chart metadata. | `tenant.metadata.name` | Tenant name; defaults to release name; DNS-1035, max 55 characters | | `tenant.metadata.labels`, `annotations` | Native metadata maps for the Tenant | | `tenant.spec` | Native Tenant spec, rendered without `tpl` evaluation or environment injection | +| `tenant.spec.readinessProbe` | Optional readiness timing fields; the Operator supplies defaults and the health endpoint | | `ingress.api`, `ingress.console` | Independent Ingress settings; disabled by default | | `ingress.*.hosts` | Explicit DNS hostnames; no catch-all rule | | `ingress.*.ingressClassName`, `tls` | Native Ingress fields; TLS uses existing Secrets | @@ -101,6 +102,41 @@ Keep the complete desired lists in your values file. Clearing a default map uses source. The installed Tenant CRD remains the authority for validating native spec fields; chart schema validation covers the chart-owned settings. +### Readiness timing + +Configure readiness through the native Tenant spec in your values file: + +```yaml +tenant: + spec: + readinessProbe: + initialDelaySeconds: 0 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + successThreshold: 1 +``` + +These are the Operator defaults. Each field is optional, so a values file can +override only `timeoutSeconds`, for example. The Operator selects the readiness +handler from Tenant TLS: HTTP/HTTPS uses `/health/ready` on port `9000`, while +mTLS uses the authenticated exec probe or configured TCP probe. The five-second +timeout is the time Kubernetes allows for a probe response; the mTLS exec +command retains its separate two-second curl timeout, and server-side readiness +work has its own time budgets. + +Install the updated Tenant CRD explicitly before applying these values; Helm +does not upgrade CRDs from the Operator chart's `crds/` directory. Upgrading the +Operator also changes existing Tenants' default readiness timeout from one to +five seconds, which updates their StatefulSet Pod templates. When +`spec.serviceAccountName` is omitted, the Operator enforces `RollingUpdate` with +`rollingUpdate.partition: 0` for default ServiceAccount token protection, and +eligible Pods are replaced automatically. With an explicit custom ServiceAccount, +the Operator preserves the existing `RollingUpdate`, `rollingUpdate.partition`, +or `OnDelete` settings. +See [readiness timing and upgrade guidance](../../docs/operator-user-guide.md#readiness-probe-timing) +for validation rules, rollout behavior, and availability checks. + ## Upgrade and remove Keep the release name, namespace and Tenant name stable. Renaming a Tenant creates diff --git a/deploy/rustfs-tenant/values.yaml b/deploy/rustfs-tenant/values.yaml index 7033874b..7435df5c 100644 --- a/deploy/rustfs-tenant/values.yaml +++ b/deploy/rustfs-tenant/values.yaml @@ -25,6 +25,14 @@ tenant: spec: credsSecret: name: rustfs-credentials + # Optional timing overrides; omitted fields use these Operator defaults. + # The Operator selects the readiness handler from the Tenant TLS configuration. + # readinessProbe: + # initialDelaySeconds: 0 + # timeoutSeconds: 5 + # periodSeconds: 10 + # failureThreshold: 3 + # successThreshold: 1 pools: - name: pool-0 servers: 1 diff --git a/docs/operator-user-guide.md b/docs/operator-user-guide.md index ae9fea28..9ab2fab0 100644 --- a/docs/operator-user-guide.md +++ b/docs/operator-user-guide.md @@ -641,6 +641,7 @@ Useful Tenant-level fields: | `createServiceAccountRbac` | Deprecated compatibility field; ignored. Manage any custom ServiceAccount RBAC explicitly. | | `priorityClassName` | Tenant-level priority class. | | `lifecycle` | Kubernetes container lifecycle hooks. | +| `readinessProbe` | Optional readiness timing overrides for every RustFS Pool. | | `podManagementPolicy` | StatefulSet pod management policy. | | `podDeletionPolicyWhenNodeIsDown` | Node-down pod deletion behavior. | | `disablePodDisruptionBudget` | Set to `true` to stop creating per-pool PodDisruptionBudgets and delete the ones the operator created. | @@ -768,6 +769,66 @@ Multi-pool Tenants cannot contain a single-node single-disk pool. The Operator r Force deletion can have data consistency implications. Use it only when the storage backend and operational procedure are designed for that failure mode. Force deletion requires the Node object to be deleted or marked with an effective `node.kubernetes.io/out-of-service` taint that the target Pod does not tolerate, so volume detach fencing is explicit. Before using force policies, confirm the node is powered off or otherwise isolated; deleting the Node object is treated as that operational assertion. +#### Readiness probe timing + +Set `spec.readinessProbe` to configure the readiness probe on every RustFS +container in the Tenant. Each field is optional; omitted fields use the defaults +below. + +| Field | Default | Minimum | Meaning | +| --- | --- | --- | --- | +| `initialDelaySeconds` | `0` | `0` | Delay before readiness checks begin. | +| `timeoutSeconds` | `5` | `1` | Time Kubernetes allows for each probe response. | +| `periodSeconds` | `10` | `1` | Interval between readiness checks. | +| `failureThreshold` | `3` | `1` | Consecutive failures required to mark the container unready. | +| `successThreshold` | `1` | `1` | Consecutive successes required to become ready after a failure. | + +```yaml +spec: + readinessProbe: + initialDelaySeconds: 0 + timeoutSeconds: 5 + periodSeconds: 10 + failureThreshold: 3 + successThreshold: 1 +``` + +The Operator selects the probe handler from the Tenant TLS configuration. +HTTP/HTTPS readiness probes use `/health/ready` on port `9000`. mTLS readiness +uses the existing authenticated exec probe or configured TCP probe. The five +timing fields apply to each readiness handler. The mTLS exec command retains +its separate two-second curl timeout. Startup and liveness retain their existing +HTTP or TCP handlers. Kubernetes stops routing Service traffic to a Pod when it +becomes unready; a readiness failure alone does not restart its container. + +The five-second default gives distributed health checks more time to respond +during a peer failure. It is the Kubernetes probe timeout, not an upper bound on +the RustFS server's entire readiness handler. Choose timing values using measured +probe durations during normal operation and node failures, alongside the required +failure detection time. A successful readiness check reports the server's readiness +decision; verify S3 reads and writes separately when testing availability. + +Apply the updated Tenant CRD before using these fields or upgrading the Operator, +following [the upgrade instructions](#4-install-the-operator). Helm does not +upgrade an installed CRD from a chart's `crds/` directory. + +For existing Tenants that omit `readinessProbe`, upgrading changes the generated +readiness timeout from Kubernetes' previous effective default of `1` second to +`5` seconds. This changes the StatefulSet Pod template. When +`spec.serviceAccountName` is omitted, the Operator enforces `RollingUpdate` with +`rollingUpdate.partition: 0` as part of its default ServiceAccount token +protection, so eligible Pods are replaced automatically. With an explicit custom +ServiceAccount, the Operator preserves the existing StatefulSet update strategy: +`RollingUpdate` updates eligible Pods, `rollingUpdate.partition` limits which +ordinals are updated, and `OnDelete` applies the new template when Pods are +manually replaced. Plan the update according to the cluster's available quorum. +Set `spec.readinessProbe.timeoutSeconds: 1` explicitly if the deployment requires +the previous timeout. + +Keep the desired settings in the Tenant resource. Reconciliation restores them +after a manual StatefulSet probe change. Removing an override restores that +field's default and can also change the Pod template. + #### Pod disruption budgets The Operator creates one PodDisruptionBudget per pool, named after the pool StatefulSet and selecting the pool Pods. `minAvailable` is the pool's server count minus the number of servers that can be unavailable at once while every erasure set keeps write quorum for the `STANDARD` storage class. It is computed from the drive placement RustFS reports through the admin API: diff --git a/src/context.rs b/src/context.rs index 5f3bb6a1..dc46536e 100755 --- a/src/context.rs +++ b/src/context.rs @@ -17,7 +17,9 @@ use crate::types; use crate::types::v1alpha1::encryption::LocalKmsMasterKeySecretRef; use crate::types::v1alpha1::tenant::{OidcExtraCaCertSecretRef, RpcSecretRef, Tenant}; use k8s_openapi::NamespaceResourceScope; +use k8s_openapi::api::apps::v1::StatefulSet; use k8s_openapi::api::core::v1::Secret; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Status, StatusCause}; use kube::api::{DeleteParams, ListParams, ObjectList, Patch, PatchParams, PostParams}; use kube::runtime::events::{Event, EventType, Recorder, Reporter}; use kube::{Resource, ResourceExt, api::Api}; @@ -30,6 +32,22 @@ use std::io::Cursor; use std::path::{Component, Path, PathBuf}; use tracing::info; +#[derive(Debug, Snafu)] +#[snafu(display("Kubernetes StatefulSet apply error: {}", source))] +pub(crate) struct StatefulSetApplyError { + pub(crate) source: Box, + pub(crate) causes: Vec, +} + +impl From for StatefulSetApplyError { + fn from(source: kube::Error) -> Self { + Self { + source: Box::new(source), + causes: Vec::new(), + } + } +} + #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Kubernetes API error: {}", source))] @@ -696,6 +714,54 @@ impl Context { .await } + /// Preserves structured SSA conflict causes omitted by kube's ErrorResponse. + pub(crate) async fn apply_statefulset( + &self, + resource: &StatefulSet, + namespace: &str, + params: &PatchParams, + ) -> Result { + let api: Api = Api::namespaced(self.client.clone(), namespace); + let request = kube::core::Request::new(api.resource_url()) + .patch(&resource.name_any(), params, &Patch::Apply(resource)) + .map_err(kube::Error::BuildRequest)?; + let response = self + .client + .send(request.map(kube::client::Body::from)) + .await?; + let response_status = response.status(); + let body = response.into_body().collect_bytes().await?; + if response_status.is_success() { + return serde_json::from_slice(&body) + .map_err(kube::Error::SerdeError) + .map_err(Into::into); + } + + let error_response = serde_json::from_slice::(&body) + .unwrap_or_else(|_| kube::core::ErrorResponse { + status: response_status.to_string(), + message: String::from_utf8_lossy(&body).into_owned(), + reason: "Failed to parse error data".to_string(), + code: response_status.as_u16(), + }); + let causes = if response_status == http::StatusCode::CONFLICT + && error_response.code == 409 + && error_response.reason == "Conflict" + { + serde_json::from_slice::(&body) + .ok() + .and_then(|status| status.details) + .and_then(|details| details.causes) + .unwrap_or_default() + } else { + Vec::new() + }; + Err(StatefulSetApplyError { + source: Box::new(kube::Error::Api(error_response)), + causes, + }) + } + /// Force-applies a narrow security patch without taking ownership of the full resource. pub(crate) async fn force_apply_security_fields( &self, diff --git a/src/reconcile.rs b/src/reconcile.rs index bc504cf1..a843386a 100755 --- a/src/reconcile.rs +++ b/src/reconcile.rs @@ -34,6 +34,7 @@ mod phases; mod pod_disruption_budget; mod pool_lifecycle; mod provisioning; +mod readiness; mod tls; const OUT_OF_SERVICE_TAINT_KEY: &str = "node.kubernetes.io/out-of-service"; @@ -971,6 +972,7 @@ pub fn error_policy(object: Arc, error: &Error, _ctx: Arc) -> A types::error::Error::ImmutableFieldModified { .. } | types::error::Error::InvalidTenantName { .. } | types::error::Error::InvalidAdditionalVolumeSpec { .. } + | types::error::Error::InvalidReadinessProbe { .. } | types::error::Error::KmsMigrationBlocked { .. } | types::error::Error::InvalidWorkloadSecurityProfile { .. } | types::error::Error::WorkloadSecurityIncompatible { .. } @@ -1028,6 +1030,7 @@ fn reconcile_error_reason(error: &Error) -> &'static str { types::error::Error::InvalidAdditionalVolumeSpec { .. } => { "InvalidAdditionalVolumeSpec" } + types::error::Error::InvalidReadinessProbe { .. } => "InvalidReadinessProbe", types::error::Error::ImmutableFieldModified { .. } => "ImmutableFieldModified", types::error::Error::PoolDeleteBlocked { .. } => "PoolDeleteBlocked", types::error::Error::KmsMigrationBlocked { .. } => "KmsMigrationBlocked", diff --git a/src/reconcile/phases.rs b/src/reconcile/phases.rs index bc4086d8..ab0ee79e 100644 --- a/src/reconcile/phases.rs +++ b/src/reconcile/phases.rs @@ -124,6 +124,12 @@ pub(super) async fn validate_tenant_prerequisites( return Err(e.into()); } + if let Err(e) = tenant.validate_readiness_probe() { + let status_error = StatusError::from_types_error(&e); + patch_status_error(ctx, tenant, &status_error).await; + return Err(e.into()); + } + // Block known incompatible RustFS images before creating or rolling any StatefulSet. if let Err(e) = tenant.validate_workload_security_compatibility() { let status_error = StatusError::from_types_error(&e); @@ -1160,7 +1166,9 @@ async fn reconcile_existing_pool_statefulset( tenant, ) .await?; - if let Err(e) = ctx.apply(&desired, namespace).await { + if let Err(e) = + super::readiness::apply_statefulset(ctx, tenant, namespace, &existing_ss, desired).await + { let status_error = StatusError::statefulset_apply_failed(&ss_name); patch_status_error(ctx, tenant, &status_error).await; return Err(e.into()); @@ -1602,6 +1610,75 @@ mod tests { assert_eq!(status_patches.load(Ordering::SeqCst), 1); } + #[tokio::test] + async fn invalid_readiness_prerequisites_block_before_resource_requests() { + use http_body_util::BodyExt; + + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.spec.readiness_probe = Some(crate::types::v1alpha1::tenant::ReadinessProbeConfig { + timeout_seconds: Some(0), + ..Default::default() + }); + let status_patches = Arc::new(AtomicUsize::new(0)); + let service = service_fn({ + let tenant = tenant.clone(); + let status_patches = Arc::clone(&status_patches); + move |request: Request| { + let tenant = tenant.clone(); + let status_patches = Arc::clone(&status_patches); + async move { + let path = request.uri().path().to_string(); + let is_status = path.ends_with("/tenants/test-tenant/status"); + assert!( + is_status || path.contains("/events"), + "invalid readiness configuration must stop resource requests: {path}" + ); + if is_status { + assert_eq!(request.method(), Method::PATCH); + } + let body = request.into_body().collect().await.unwrap().to_bytes(); + let payload: Value = serde_json::from_slice(&body).unwrap(); + if is_status { + assert_eq!(payload["status"]["currentState"], "Blocked"); + assert!( + payload["status"]["conditions"] + .as_array() + .unwrap() + .iter() + .any(|condition| { + condition["type"] == "SpecValid" + && condition["status"] == "False" + && condition["reason"] == "InvalidReadinessProbe" + && condition["message"] + .as_str() + .unwrap() + .contains("spec.readinessProbe.timeoutSeconds") + }) + ); + status_patches.fetch_add(1, Ordering::SeqCst); + Ok::<_, Infallible>(kube_response( + StatusCode::OK, + serde_json::to_value(tenant).unwrap(), + )) + } else { + Ok(kube_response(StatusCode::OK, payload)) + } + } + } + }); + let ctx = Context::new(Client::new(service, "default")); + let error = validate_tenant_prerequisites(&ctx, &tenant) + .await + .expect_err("invalid readiness timing must stop reconciliation"); + assert!(matches!( + error, + Error::Types { + source: crate::types::error::Error::InvalidReadinessProbe { .. } + } + )); + assert_eq!(status_patches.load(Ordering::SeqCst), 1); + } + fn operator_managed_fields() -> Vec { vec![ManagedFieldsEntry { manager: Some("rustfs-operator".to_string()), diff --git a/src/reconcile/readiness.rs b/src/reconcile/readiness.rs new file mode 100644 index 00000000..4313b51d --- /dev/null +++ b/src/reconcile/readiness.rs @@ -0,0 +1,102 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::statefulset_owned_by_tenant; +use crate::context::{Context, Error}; +use crate::types; +use crate::types::v1alpha1::tenant::Tenant; +use k8s_openapi::api::apps::v1::StatefulSet; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::StatusCause; +use kube::ResourceExt; +use kube::api::PatchParams; + +/// Applies the complete Pod template atomically, reclaiming timing fields only +/// when Kubernetes reports that every conflict belongs to those fields. +pub(super) async fn apply_statefulset( + ctx: &Context, + tenant: &Tenant, + namespace: &str, + existing: &StatefulSet, + mut desired: StatefulSet, +) -> Result { + if !statefulset_owned_by_tenant(existing, tenant) { + return Err(types::error::Error::InternalError { + msg: format!( + "StatefulSet '{}' is not owned by Tenant '{}'", + existing.name_any(), + tenant.name() + ), + } + .into()); + } + let uid = existing + .metadata + .uid + .as_ref() + .ok_or_else(|| types::error::Error::InternalError { + msg: format!("StatefulSet '{}' has no UID", existing.name_any()), + })?; + let resource_version = existing.metadata.resource_version.as_ref().ok_or_else(|| { + types::error::Error::InternalError { + msg: format!( + "StatefulSet '{}' has no resourceVersion", + existing.name_any() + ), + } + })?; + desired.metadata.uid = Some(uid.clone()); + desired.metadata.resource_version = Some(resource_version.clone()); + + let params = PatchParams::apply("rustfs-operator"); + match ctx.apply_statefulset(&desired, namespace, ¶ms).await { + Ok(updated) => Ok(updated), + Err(error) if only_readiness_timing_conflicts(&error.causes) => { + // The rejected apply persisted nothing. Reuse its complete template + // and preconditions so concurrent changes cannot broaden ownership. + ctx.apply_statefulset(&desired, namespace, ¶ms.force()) + .await + .map_err(|error| Error::Kube { + source: *error.source, + }) + } + Err(error) => Err(Error::Kube { + source: *error.source, + }), + } +} + +fn only_readiness_timing_conflicts(causes: &[StatusCause]) -> bool { + !causes.is_empty() + && causes.iter().all(|cause| { + cause.reason.as_deref() == Some("FieldManagerConflict") + && matches!( + cause.field.as_deref(), + Some( + ".spec.template.spec.containers[name=\"rustfs\"].readinessProbe.initialDelaySeconds" + | ".spec.template.spec.containers[name=\"rustfs\"].readinessProbe.timeoutSeconds" + | ".spec.template.spec.containers[name=\"rustfs\"].readinessProbe.periodSeconds" + | ".spec.template.spec.containers[name=\"rustfs\"].readinessProbe.failureThreshold" + | ".spec.template.spec.containers[name=\"rustfs\"].readinessProbe.successThreshold" + ) + ) + }) +} + +#[cfg(test)] +#[path = "readiness_tests.rs"] +mod tests; + +#[cfg(test)] +#[path = "readiness_rollout_tests.rs"] +mod rollout_tests; diff --git a/src/reconcile/readiness_rollout_tests.rs b/src/reconcile/readiness_rollout_tests.rs new file mode 100644 index 00000000..cd507da4 --- /dev/null +++ b/src/reconcile/readiness_rollout_tests.rs @@ -0,0 +1,350 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::apply_statefulset; +use crate::context::{Context, Error}; +use crate::types::v1alpha1::tenant::Tenant; +use futures::{FutureExt, StreamExt, TryStreamExt}; +use k8s_openapi::api::apps::v1::StatefulSet; +use k8s_openapi::api::core::v1::{Container, Namespace, Pod, PodSpec}; +use kube::api::{ + DeleteParams, ListParams, Patch, PatchParams, PostParams, WatchEvent, WatchParams, +}; +use kube::config::KubeConfigOptions; +use kube::{Api, Client, Config, ResourceExt}; +use serde_json::json; +use std::collections::HashSet; +use std::panic::AssertUnwindSafe; +use std::time::Duration; + +const SERVER: &str = r#" +import os +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + time.sleep(float(os.environ["PROBE_DELAY"])) + self.send_response(200) + self.end_headers() + + def log_message(self, *args): + pass + +ThreadingHTTPServer(("0.0.0.0", 9000), Handler).serve_forever() +"#; + +fn rollout_template(tenant: &Tenant, policy: &str, delay: &str, timeout: i32) -> StatefulSet { + let mut desired = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("StatefulSet should render"); + let spec = desired.spec.as_mut().unwrap(); + spec.replicas = Some(2); + spec.pod_management_policy = Some(policy.to_string()); + spec.volume_claim_templates = None; + spec.template.spec = Some( + serde_json::from_value::(json!({ + "terminationGracePeriodSeconds": 1, + "containers": [{ + "name": "rustfs", + "image": "python:3.12-slim", + "imagePullPolicy": "Never", + "command": ["python", "-u", "-c", SERVER], + "env": [{"name": "PROBE_DELAY", "value": delay}], + "readinessProbe": { + "httpGet": {"path": "/health/ready", "port": 9000, "scheme": "HTTP"}, + "initialDelaySeconds": 0, + "timeoutSeconds": timeout, + "periodSeconds": 1, + "failureThreshold": 1, + "successThreshold": 1 + } + }] + })) + .expect("test Pod spec should deserialize"), + ); + desired +} + +fn probe_pair(container: &Container) -> (&str, i32) { + let delay = container + .env + .as_ref() + .unwrap() + .iter() + .find(|env| env.name == "PROBE_DELAY") + .and_then(|env| env.value.as_deref()) + .expect("test server should have a probe delay"); + let timeout = container + .readiness_probe + .as_ref() + .unwrap() + .timeout_seconds + .unwrap(); + (delay, timeout) +} + +fn assert_complete_template(container: &Container) { + let pair = probe_pair(container); + assert!( + matches!(pair, ("2", 5) | ("0", 1)), + "readiness timing and server configuration must change together, observed {pair:?}" + ); +} + +fn pod_ready(pod: &Pod) -> bool { + pod.metadata.deletion_timestamp.is_none() + && pod + .status + .as_ref() + .and_then(|status| status.conditions.as_ref()) + .is_some_and(|conditions| { + conditions + .iter() + .any(|c| c.type_ == "Ready" && c.status == "True") + }) +} + +async fn wait_for_rollout(statefulsets: &Api, pods: &Api, name: &str) { + tokio::time::timeout(Duration::from_secs(120), async { + let mut interval = tokio::time::interval(Duration::from_millis(200)); + loop { + interval.tick().await; + let statefulset = statefulsets + .get(name) + .await + .expect("StatefulSet should read"); + let Some(status) = statefulset.status.as_ref() else { + continue; + }; + if status.observed_generation != statefulset.metadata.generation + || status.updated_replicas != Some(2) + || status.ready_replicas != Some(2) + || status.current_revision.is_none() + || status.current_revision != status.update_revision + { + continue; + } + let observed = pods + .list(&ListParams::default()) + .await + .expect("Pods should list"); + let matching: Vec<_> = observed + .items + .iter() + .filter(|pod| pod.name_any().starts_with(&format!("{name}-"))) + .collect(); + if matching.len() == 2 + && matching.iter().all(|pod| { + pod_ready(pod) + && pod.labels().get("controller-revision-hash") + == status.update_revision.as_ref() + }) + { + return; + } + } + }) + .await + .expect("both Pods should become Ready at the current StatefulSet revision"); +} + +async fn run_rollout(client: Client, namespace: &str, policy: &str) { + let tenants: Api = Api::namespaced(client.clone(), namespace); + let statefulsets: Api = Api::namespaced(client.clone(), namespace); + let pods: Api = Api::namespaced(client.clone(), namespace); + let ctx = Context::new(client); + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.metadata.name = Some(format!("readiness-{}", policy.to_lowercase())); + tenant.metadata.namespace = Some(namespace.to_string()); + tenant.metadata.uid = None; + let tenant = tenants + .create(&PostParams::default(), &tenant) + .await + .expect("test Tenant should create"); + let initial = rollout_template(&tenant, policy, "2", 5); + let name = initial.name_any(); + ctx.apply(&initial, namespace) + .await + .expect("initial StatefulSet should create"); + wait_for_rollout(&statefulsets, &pods, &name).await; + + // Shared ownership forces the production helper to exercise conflict handling. + let existing = statefulsets + .patch( + &name, + &PatchParams::apply("external-readiness"), + &Patch::Apply(json!({ + "apiVersion": "apps/v1", "kind": "StatefulSet", + "metadata": {"name": name, "namespace": namespace}, + "spec": {"template": {"spec": {"containers": [{ + "name": "rustfs", "readinessProbe": {"timeoutSeconds": 5} + }]}}} + })), + ) + .await + .expect("external manager should share the current readiness timeout"); + let mut templates = statefulsets + .watch( + &WatchParams::default() + .fields(&format!("metadata.name={name}")) + .timeout(150), + existing.metadata.resource_version.as_deref().unwrap(), + ) + .await + .expect("StatefulSet watch should start") + .boxed(); + let pod_list = pods + .list(&ListParams::default()) + .await + .expect("initial Pods should list"); + let mut pod_events = pods + .watch( + &WatchParams::default().timeout(150), + pod_list.metadata.resource_version.as_deref().unwrap(), + ) + .await + .expect("Pod watch should start") + .boxed(); + let desired = rollout_template(&tenant, policy, "0", 1); + let updated = tokio::time::timeout(Duration::from_secs(15), async { + let mut current = existing; + loop { + match apply_statefulset(&ctx, &tenant, namespace, ¤t, desired.clone()).await { + Ok(updated) => return updated, + Err(Error::Kube { + source: kube::Error::Api(response), + }) if response.code == 409 + && response.message.contains("the object has been modified") => + { + current = statefulsets + .get(&name) + .await + .expect("StatefulSet should refresh"); + } + Err(error) => panic!("complete template update should succeed: {error}"), + } + } + }) + .await + .expect("template update should finish within its deadline"); + + let check_templates = async { + while let Some(event) = templates + .try_next() + .await + .expect("StatefulSet watch should read") + { + match event { + WatchEvent::Added(observed) | WatchEvent::Modified(observed) => { + let container = &observed + .spec + .as_ref() + .unwrap() + .template + .spec + .as_ref() + .unwrap() + .containers[0]; + assert_complete_template(container); + if observed.metadata.resource_version == updated.metadata.resource_version { + assert_eq!(probe_pair(container), ("0", 1)); + return; + } + } + WatchEvent::Error(error) => panic!("StatefulSet watch failed: {error:?}"), + _ => {} + } + } + panic!("StatefulSet watch ended before the submitted template"); + }; + let check_pods = async { + let mut ready = HashSet::new(); + while let Some(event) = pod_events.try_next().await.expect("Pod watch should read") { + match event { + WatchEvent::Added(pod) | WatchEvent::Modified(pod) + if pod.name_any().starts_with(&format!("{name}-")) => + { + let container = &pod.spec.as_ref().unwrap().containers[0]; + assert_complete_template(container); + if probe_pair(container) == ("0", 1) && pod_ready(&pod) { + ready.insert(pod.name_any()); + } else { + ready.remove(&pod.name_any()); + } + if ready.len() == 2 { + return; + } + } + WatchEvent::Deleted(pod) => { + ready.remove(&pod.name_any()); + } + WatchEvent::Error(error) => panic!("Pod watch failed: {error:?}"), + _ => {} + } + } + panic!("Pod watch ended before both updated Pods became Ready"); + }; + tokio::time::timeout(Duration::from_secs(120), async { + tokio::join!(check_templates, check_pods); + }) + .await + .expect("complete template rollout should finish without deleting Pods manually"); + wait_for_rollout(&statefulsets, &pods, &name).await; +} + +#[tokio::test] +#[ignore = "requires RUSTFS_READINESS_TEST_CONTEXT, the Tenant CRD and python:3.12-slim on every node"] +async fn readiness_rollout_live_updates_complete_templates() { + crate::install_rustls_crypto_provider(); + let context = std::env::var("RUSTFS_READINESS_TEST_CONTEXT") + .expect("set RUSTFS_READINESS_TEST_CONTEXT to a disposable Kubernetes context"); + let mut config = Config::from_kubeconfig(&KubeConfigOptions { + context: Some(context), + ..Default::default() + }) + .await + .expect("dedicated readiness test context should load"); + config.proxy_url = None; + let client = Client::try_from(config).expect("Kubernetes client should initialize"); + let namespaces: Api = Api::all(client.clone()); + let namespace = namespaces + .create( + &PostParams::default(), + &Namespace { + metadata: kube::core::ObjectMeta { + generate_name: Some("readiness-rollout-".to_string()), + ..Default::default() + }, + ..Default::default() + }, + ) + .await + .expect("isolated test namespace should create") + .name_any(); + let result = AssertUnwindSafe(async { + for policy in ["Parallel", "OrderedReady"] { + run_rollout(client.clone(), &namespace, policy).await; + } + }) + .catch_unwind() + .await; + let cleanup = namespaces + .delete(&namespace, &DeleteParams::default()) + .await; + if let Err(payload) = result { + std::panic::resume_unwind(payload); + } + cleanup.expect("isolated test namespace should delete"); +} diff --git a/src/reconcile/readiness_tests.rs b/src/reconcile/readiness_tests.rs new file mode 100644 index 00000000..fb4544b6 --- /dev/null +++ b/src/reconcile/readiness_tests.rs @@ -0,0 +1,988 @@ +// Copyright 2026 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use crate::types::v1alpha1::tenant::ReadinessProbeConfig; +use futures::{FutureExt, StreamExt, TryStreamExt}; +use http::{Method, Request, Response, StatusCode}; +use k8s_openapi::api::core::v1::{Container, Namespace, Probe}; +use kube::api::{DeleteParams, Patch, PatchParams, PostParams, WatchEvent, WatchParams}; +use kube::config::KubeConfigOptions; +use kube::{Api, Client, Config, client::Body}; +use serde_json::{Value, json}; +use std::convert::Infallible; +use std::panic::AssertUnwindSafe; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; +use tower::service_fn; + +#[derive(Debug)] +struct RecordedRequest { + method: Method, + path: String, + query: String, + body: Value, +} + +fn response(status: StatusCode, body: &Value) -> Response { + Response::builder() + .status(status) + .body(Body::from( + serde_json::to_vec(body).expect("response should serialize"), + )) + .expect("response should build") +} + +fn mock_context( + responses: Vec<(StatusCode, Value)>, +) -> (Context, Arc>>) { + let requests = Arc::new(Mutex::new(Vec::new())); + let service = service_fn({ + let requests = Arc::clone(&requests); + let responses = Arc::new(responses); + move |request: Request| { + let requests = Arc::clone(&requests); + let responses = Arc::clone(&responses); + async move { + let method = request.method().clone(); + let path = request.uri().path().to_string(); + let query = request.uri().query().unwrap_or_default().to_string(); + let body = serde_json::from_slice( + &request + .into_body() + .collect_bytes() + .await + .expect("request body should read"), + ) + .expect("request body should be JSON"); + let mut requests = requests.lock().expect("request lock should be available"); + let index = requests.len(); + requests.push(RecordedRequest { + method, + path, + query, + body, + }); + let (status, body) = responses.get(index).expect("unexpected API request"); + Ok::<_, Infallible>(response(*status, body)) + } + } + }); + (Context::new(Client::new(service, "default")), requests) +} + +fn fixture() -> (Tenant, StatefulSet, StatefulSet) { + let mut tenant = crate::tests::create_test_tenant(None, None); + let mut existing = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("StatefulSet should render"); + existing.metadata.uid = Some("statefulset-uid".to_string()); + existing.metadata.resource_version = Some("41".to_string()); + tenant.spec.readiness_probe = Some(ReadinessProbeConfig { + initial_delay_seconds: Some(2), + timeout_seconds: Some(8), + period_seconds: Some(7), + failure_threshold: Some(4), + success_threshold: Some(2), + }); + let desired = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("StatefulSet should render"); + (tenant, existing, desired) +} + +fn rustfs_container(statefulset: &StatefulSet) -> &Container { + statefulset + .spec + .as_ref() + .expect("StatefulSet should have spec") + .template + .spec + .as_ref() + .expect("Pod template should have spec") + .containers + .iter() + .find(|container| container.name == "rustfs") + .expect("RustFS container should exist") +} + +fn rustfs_container_mut(statefulset: &mut StatefulSet) -> &mut Container { + statefulset + .spec + .as_mut() + .expect("StatefulSet should have spec") + .template + .spec + .as_mut() + .expect("Pod template should have spec") + .containers + .iter_mut() + .find(|container| container.name == "rustfs") + .expect("RustFS container should exist") +} + +fn readiness_probe(statefulset: &StatefulSet) -> &Probe { + rustfs_container(statefulset) + .readiness_probe + .as_ref() + .expect("readiness probe should exist") +} + +fn successful_response( + mut statefulset: StatefulSet, + resource_version: &str, +) -> (StatusCode, Value) { + statefulset.metadata.resource_version = Some(resource_version.to_string()); + ( + StatusCode::OK, + serde_json::to_value(statefulset).expect("StatefulSet should serialize"), + ) +} + +fn api_failure(code: u16) -> (StatusCode, Value) { + ( + StatusCode::from_u16(code).expect("status code should be valid"), + json!({ + "apiVersion": "v1", + "kind": "Status", + "status": "Failure", + "reason": if code == 409 { "Conflict" } else { "InternalError" }, + "message": "injected apply failure", + "code": code, + }), + ) +} + +fn field_manager_conflict(field: &str, manager: &str) -> Value { + json!({ + "reason": "FieldManagerConflict", + "message": format!("conflict with \"{manager}\""), + "field": field, + }) +} + +fn conflict_response(causes: Vec) -> (StatusCode, Value) { + let (status, mut body) = api_failure(409); + body["details"] = json!({"causes": causes}); + (status, body) +} + +fn timing_conflict(field: &str) -> Value { + field_manager_conflict( + &format!(".spec.template.spec.containers[name=\"rustfs\"].readinessProbe.{field}"), + "external-readiness-manager", + ) +} + +fn expected_apply(existing: &StatefulSet, desired: &StatefulSet) -> Value { + let mut expected = desired.clone(); + expected.metadata.uid = existing.metadata.uid.clone(); + expected.metadata.resource_version = existing.metadata.resource_version.clone(); + serde_json::to_value(expected).expect("desired StatefulSet should serialize") +} + +#[tokio::test] +async fn ordinary_apply_submits_the_complete_template_once_without_force() { + let (tenant, existing, mut desired) = fixture(); + let container = rustfs_container_mut(&mut desired); + container.image = Some("example.invalid/rustfs:updated".to_string()); + container + .env + .get_or_insert_with(Vec::new) + .push(k8s_openapi::api::core::v1::EnvVar { + name: "READINESS_TEST_ENV".to_string(), + value: Some("updated".to_string()), + ..Default::default() + }); + let expected = expected_apply(&existing, &desired); + let (ctx, requests) = mock_context(vec![successful_response(desired.clone(), "42")]); + + let result = apply_statefulset(&ctx, &tenant, "default", &existing, desired) + .await + .expect("ordinary apply should succeed"); + assert_eq!(result.metadata.resource_version.as_deref(), Some("42")); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].method, Method::PATCH); + assert_eq!( + requests[0].path, + "/apis/apps/v1/namespaces/default/statefulsets/test-tenant-pool-0" + ); + assert!( + requests[0] + .query + .split('&') + .any(|parameter| parameter == "fieldManager=rustfs-operator") + ); + assert!(!requests[0].query.contains("force=true")); + assert_eq!(requests[0].body, expected); +} + +#[tokio::test] +async fn timing_conflicts_retry_the_identical_complete_template_with_force() { + let fields = [ + "initialDelaySeconds", + "timeoutSeconds", + "periodSeconds", + "failureThreshold", + "successThreshold", + ]; + let mut conflicts: Vec> = fields + .iter() + .map(|field| vec![timing_conflict(field)]) + .collect(); + conflicts.push( + fields + .iter() + .enumerate() + .map(|(index, field)| { + field_manager_conflict( + &format!( + ".spec.template.spec.containers[name=\"rustfs\"].readinessProbe.{field}" + ), + &format!("external-manager-{index}"), + ) + }) + .collect(), + ); + for causes in conflicts { + let (tenant, existing, mut desired) = fixture(); + rustfs_container_mut(&mut desired).image = + Some("example.invalid/rustfs:updated".to_string()); + let expected = expected_apply(&existing, &desired); + let (ctx, requests) = mock_context(vec![ + conflict_response(causes), + successful_response(desired.clone(), "42"), + ]); + apply_statefulset(&ctx, &tenant, "default", &existing, desired) + .await + .expect("timing-only conflicts should permit one force retry"); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + for request in requests.iter() { + assert_eq!(request.method, Method::PATCH); + assert!( + request + .query + .split('&') + .any(|parameter| parameter == "fieldManager=rustfs-operator") + ); + assert_eq!(request.body, expected); + assert_eq!(request.body["metadata"]["uid"], "statefulset-uid"); + assert_eq!(request.body["metadata"]["resourceVersion"], "41"); + } + assert_eq!(requests[0].path, requests[1].path); + assert!(!requests[0].query.contains("force=true")); + assert!(requests[1].query.contains("force=true")); + } +} + +#[tokio::test] +async fn conflicts_outside_readiness_timing_never_force_or_partially_update() { + let unrelated_fields = [ + ".spec.template.spec.containers[name=\"rustfs\"].image", + ".spec.template.spec.containers[name=\"rustfs\"].env[name=\"TEST\"].value", + ".spec.template.spec.containers[name=\"rustfs\"].readinessProbe.httpGet.path", + ".spec.template.spec.containers[name=\"sidecar\"].readinessProbe.timeoutSeconds", + ".spec.template.spec.containers[name=\"rustfs\"].livenessProbe.timeoutSeconds", + ".spec.template.spec.containers[name=\"rustfs\"].readinessProbe.timeoutSeconds.extra", + "spec.template.spec.containers[name=\"rustfs\"].readinessProbe.timeoutSeconds", + ]; + for field in unrelated_fields { + let (tenant, existing, desired) = fixture(); + let (ctx, requests) = mock_context(vec![conflict_response(vec![ + timing_conflict("timeoutSeconds"), + field_manager_conflict(field, "external-owner"), + ])]); + apply_statefulset(&ctx, &tenant, "default", &existing, desired) + .await + .expect_err("a mixed conflict must preserve all existing fields"); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 1, "unexpected retry for {field}"); + assert!(!requests[0].query.contains("force=true")); + } +} + +#[tokio::test] +async fn missing_or_invalid_conflict_details_never_enable_force() { + let valid_cause = timing_conflict("timeoutSeconds"); + let mut responses = vec![api_failure(409), api_failure(500)]; + for details in [ + Value::Null, + json!({}), + json!({"causes": []}), + json!({"causes": null}), + json!({"causes": "invalid"}), + json!({"causes": [null]}), + json!({"causes": [{"reason": "FieldManagerConflict"}]}), + json!({"causes": [{"field": valid_cause["field"]}]}), + json!({"causes": [{"reason": "FieldValueInvalid", "field": valid_cause["field"]}]}), + json!({"causes": [{"reason": "FieldManagerConflict", "field": 42}]}), + ] { + let (status, mut body) = api_failure(409); + body["details"] = details; + responses.push((status, body)); + } + let (status, mut wrong_code) = conflict_response(vec![valid_cause.clone()]); + wrong_code["code"] = json!(500); + responses.push((status, wrong_code)); + let (status, mut wrong_reason) = conflict_response(vec![valid_cause.clone()]); + wrong_reason["reason"] = json!("Invalid"); + responses.push((status, wrong_reason)); + let (_, valid_body) = conflict_response(vec![valid_cause]); + responses.push((StatusCode::INTERNAL_SERVER_ERROR, valid_body)); + + for failure in responses { + let (tenant, existing, desired) = fixture(); + let (ctx, requests) = mock_context(vec![failure]); + apply_statefulset(&ctx, &tenant, "default", &existing, desired) + .await + .expect_err("only complete structured timing conflicts may enable force"); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!(!requests[0].query.contains("force=true")); + } +} + +#[tokio::test] +async fn invalid_ownership_or_identity_prevents_all_apply_requests() { + for invalid in ["owner", "tenant_uid", "uid", "resource_version"] { + let (mut tenant, mut existing, desired) = fixture(); + match invalid { + "owner" => { + existing.metadata.owner_references.as_mut().unwrap()[0].uid = + "another-tenant".to_string() + } + "tenant_uid" => tenant.metadata.uid = None, + "uid" => existing.metadata.uid = None, + "resource_version" => existing.metadata.resource_version = None, + _ => unreachable!(), + } + let (ctx, requests) = mock_context(Vec::new()); + let error = apply_statefulset(&ctx, &tenant, "default", &existing, desired) + .await + .expect_err("invalid resource identity must stop the update"); + assert!( + matches!(error, Error::Types { .. }), + "unexpected error for {invalid}: {error}" + ); + assert!( + requests.lock().unwrap().is_empty(), + "unexpected API write for {invalid}" + ); + } +} + +#[tokio::test] +async fn force_retry_errors_are_propagated_without_further_requests() { + for code in [409, 500] { + let (tenant, existing, desired) = fixture(); + let (ctx, requests) = mock_context(vec![ + conflict_response(vec![timing_conflict("timeoutSeconds")]), + api_failure(code), + ]); + let error = apply_statefulset(&ctx, &tenant, "default", &existing, desired) + .await + .expect_err("retry errors must propagate to reconciliation"); + assert!( + matches!(error, Error::Kube { source: kube::Error::Api(ref response) } if response.code == code) + ); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].body, requests[1].body); + assert_eq!(requests[1].body["metadata"]["resourceVersion"], "41"); + } +} + +#[tokio::test] +async fn missing_existing_probe_is_restored_with_handler_and_timing_in_one_apply() { + let (tenant, mut existing, desired) = fixture(); + rustfs_container_mut(&mut existing).readiness_probe = None; + let (ctx, requests) = mock_context(vec![successful_response(desired.clone(), "42")]); + apply_statefulset(&ctx, &tenant, "default", &existing, desired) + .await + .expect("ordinary apply should restore the complete probe"); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!( + requests[0] + .query + .split('&') + .any(|parameter| parameter == "fieldManager=rustfs-operator") + ); + assert!(!requests[0].query.contains("force=true")); + assert_eq!(requests[0].body["metadata"]["uid"], "statefulset-uid"); + assert_eq!(requests[0].body["metadata"]["resourceVersion"], "41"); + let probe = &requests[0].body["spec"]["template"]["spec"]["containers"][0]["readinessProbe"]; + assert_eq!(probe["httpGet"]["path"], "/health/ready"); + assert_eq!(probe["timeoutSeconds"], 8); +} + +fn live_desired(tenant: &Tenant) -> StatefulSet { + let mut desired = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("StatefulSet should render"); + desired.spec.as_mut().unwrap().replicas = Some(0); + desired +} + +fn managed_container_fields<'a>(statefulset: &'a StatefulSet, manager: &str) -> Option<&'a Value> { + statefulset + .metadata + .managed_fields + .as_ref()? + .iter() + .find(|entry| entry.manager.as_deref() == Some(manager))? + .fields_v1 + .as_ref()? + .0 + .get("f:spec")? + .get("f:template")? + .get("f:spec")? + .get("f:containers")? + .get("k:{\"name\":\"rustfs\"}") +} + +async fn live_apply_with_status_retries( + ctx: &Context, + tenant: &Tenant, + namespace: &str, + api: &Api, + existing: &StatefulSet, +) -> Result { + let mut current = existing.clone(); + let mut retries = 0; + loop { + match apply_statefulset(ctx, tenant, namespace, ¤t, live_desired(tenant)).await { + Ok(updated) => return Ok(updated), + Err(Error::Kube { + source: kube::Error::Api(response), + }) if response.code == 409 + && response.message.contains("the object has been modified") + && retries < 5 => + { + // The StatefulSet controller can update status between applies. + // Reconciliation retries from a fresh observation in this case. + retries += 1; + current = api + .get(&existing.name_any()) + .await + .expect("StatefulSet should refresh after a concurrent status update"); + } + Err(error) => return Err(error), + } + } +} + +async fn live_apply_and_check_events( + ctx: &Context, + tenant: &Tenant, + namespace: &str, + api: &Api, + existing: &StatefulSet, + expected_timeout: i32, +) -> StatefulSet { + let resource_version = existing.metadata.resource_version.as_deref().unwrap(); + let mut events = api + .watch( + &WatchParams::default() + .fields(&format!("metadata.name={}", existing.name_any())) + .timeout(10), + resource_version, + ) + .await + .expect("StatefulSet watch should start") + .boxed(); + let updated = live_apply_with_status_retries(ctx, tenant, namespace, api, existing) + .await + .expect("readiness timing update should succeed"); + let expected_version = updated.metadata.resource_version.as_deref().unwrap(); + let before_strategy = + serde_json::to_value(&existing.spec.as_ref().unwrap().update_strategy).unwrap(); + let before_template = serde_json::to_value(&existing.spec.as_ref().unwrap().template).unwrap(); + let expected_template = serde_json::to_value(&updated.spec.as_ref().unwrap().template).unwrap(); + if expected_version != resource_version { + tokio::time::timeout(Duration::from_secs(15), async { + let mut target_observed = false; + while let Some(event) = events.try_next().await.expect("watch event should read") { + if let WatchEvent::Added(observed) | WatchEvent::Modified(observed) = event { + let template = + serde_json::to_value(&observed.spec.as_ref().unwrap().template).unwrap(); + target_observed |= template == expected_template; + assert!( + template == expected_template + || (!target_observed && template == before_template), + "stored revisions must contain the complete old or requested template" + ); + assert_eq!( + serde_json::to_value(&observed.spec.as_ref().unwrap().update_strategy) + .unwrap(), + before_strategy + ); + if observed.metadata.resource_version.as_deref() == Some(expected_version) { + return; + } + } + } + panic!("watch ended before the final apply revision"); + }) + .await + .expect("the complete applied template should be observed"); + } + assert_eq!( + readiness_probe(&updated).timeout_seconds, + Some(expected_timeout) + ); + let timing_fields = managed_container_fields(&updated, "rustfs-operator") + .and_then(|fields| fields.get("f:readinessProbe")) + .expect("the Operator should own readiness timing"); + for field in [ + "f:initialDelaySeconds", + "f:timeoutSeconds", + "f:periodSeconds", + "f:failureThreshold", + "f:successThreshold", + ] { + assert!( + timing_fields.get(field).is_some(), + "the Operator must own {field}" + ); + } + assert!(timing_fields.get("f:httpGet").is_some()); + let http = readiness_probe(&updated).http_get.as_ref().unwrap(); + assert_eq!(http.host.as_deref(), Some("127.0.0.1")); + assert_eq!( + http.http_headers.as_ref().unwrap()[0].name, + "X-Probe-Source" + ); + assert_eq!( + http.http_headers.as_ref().unwrap()[0].value, + "external-manager" + ); + let mut observed = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("StatefulSet should render for comparison"); + // Keep unrelated API defaults out of the readiness convergence check. + rustfs_container_mut(&mut observed).readiness_probe = + rustfs_container(&updated).readiness_probe.clone(); + assert!( + !tenant + .statefulset_needs_update(&observed, &tenant.spec.pools[0]) + .unwrap(), + "external probe headers and host must not cause repeated reconciliation writes" + ); + updated +} + +async fn run_live_scenario(client: Client, namespace: &str) { + let tenants: Api = Api::namespaced(client.clone(), namespace); + let statefulsets: Api = Api::namespaced(client.clone(), namespace); + let ctx = Context::new(client.clone()); + let mut tenant = + crate::tests::create_test_tenant(Some("external-workload-identity".to_string()), None); + tenant.metadata.namespace = Some(namespace.to_string()); + tenant.metadata.uid = None; + tenant = tenants + .create(&PostParams::default(), &tenant) + .await + .expect("test Tenant should create"); + + let mut legacy = live_desired(&tenant); + let legacy_probe = rustfs_container_mut(&mut legacy) + .readiness_probe + .as_mut() + .unwrap(); + legacy_probe.initial_delay_seconds = None; + legacy_probe.timeout_seconds = None; + legacy_probe.period_seconds = None; + legacy_probe.failure_threshold = None; + legacy_probe.success_threshold = None; + let name = legacy.name_any(); + let mut existing = statefulsets + .patch( + &name, + &PatchParams::apply("rustfs-operator"), + &Patch::Apply(&legacy), + ) + .await + .expect("legacy StatefulSet should create through SSA"); + assert_eq!(readiness_probe(&existing).timeout_seconds, Some(1)); + let manual = json!({"spec": {"template": {"spec": {"containers": [{ + "name": "rustfs", "readinessProbe": { + "timeoutSeconds": 5, + "httpGet": { + "host": "127.0.0.1", + "httpHeaders": [{"name": "X-Probe-Source", "value": "external-manager"}], + }, + }, + }]}}}}); + existing = statefulsets + .patch( + &name, + &PatchParams { + field_manager: Some("kubectl-patch".to_string()), + ..Default::default() + }, + &Patch::Strategic(&manual), + ) + .await + .expect("manual timeout workaround should apply"); + assert_eq!(readiness_probe(&existing).timeout_seconds, Some(5)); + + let on_delete = json!({"apiVersion": "apps/v1", "kind": "StatefulSet", "metadata": {"name": name, "namespace": namespace}, + "spec": {"updateStrategy": {"type": "OnDelete", "rollingUpdate": null}}}); + existing = statefulsets + .patch( + &name, + &PatchParams { + field_manager: Some("external-rollout-policy".to_string()), + ..Default::default() + }, + &Patch::Merge(&on_delete), + ) + .await + .expect("external OnDelete strategy should apply"); + + for timeout in [Some(5), Some(8), None] { + let config = timeout.map(|value| json!({"timeoutSeconds": value})); + tenant = tenants + .patch( + &tenant.name(), + &PatchParams::default(), + &Patch::Merge(json!({"spec": {"readinessProbe": config}})), + ) + .await + .expect("Tenant readiness configuration should persist"); + assert_eq!( + tenant + .spec + .readiness_probe + .as_ref() + .and_then(|probe| probe.timeout_seconds), + timeout + ); + existing = live_apply_and_check_events( + &ctx, + &tenant, + namespace, + &statefulsets, + &existing, + timeout.unwrap_or(5), + ) + .await; + if timeout == Some(5) { + let shared = json!({"apiVersion": "apps/v1", "kind": "StatefulSet", "metadata": {"name": name, "namespace": namespace}, + "spec": {"template": {"spec": {"containers": [{"name": "rustfs", "readinessProbe": {"timeoutSeconds": 5}}]}}}}); + existing = statefulsets + .patch( + &name, + &PatchParams::apply("external-shared-readiness"), + &Patch::Apply(&shared), + ) + .await + .expect("equal readiness values should permit shared ownership"); + assert!(managed_container_fields(&existing, "external-shared-readiness").unwrap()["f:readinessProbe"].get("f:timeoutSeconds").is_some()); + } + } + + let partitioned = json!({"apiVersion": "apps/v1", "kind": "StatefulSet", "metadata": {"name": name, "namespace": namespace}, + "spec": {"updateStrategy": {"type": "RollingUpdate", "rollingUpdate": {"partition": 3}}}}); + existing = statefulsets + .patch( + &name, + &PatchParams::apply("external-rollout-policy").force(), + &Patch::Apply(&partitioned), + ) + .await + .expect("external rolling partition should apply"); + existing = + live_apply_and_check_events(&ctx, &tenant, namespace, &statefulsets, &existing, 5).await; + + tenant.spec.env.push(k8s_openapi::api::core::v1::EnvVar { + name: "READINESS_TEST_ENV".to_string(), + value: Some("updated".to_string()), + ..Default::default() + }); + tenant.spec.readiness_probe = Some(ReadinessProbeConfig { + timeout_seconds: Some(8), + ..Default::default() + }); + let updated = + live_apply_and_check_events(&ctx, &tenant, namespace, &statefulsets, &existing, 8).await; + assert!( + rustfs_container(&updated) + .env + .as_ref() + .unwrap() + .iter() + .any(|variable| variable.name == "READINESS_TEST_ENV" + && variable.value.as_deref() == Some("updated")) + ); + + let external_image = "example.invalid/external-image:1"; + let image_patch = json!({"apiVersion": "apps/v1", "kind": "StatefulSet", "metadata": {"name": name, "namespace": namespace}, + "spec": {"template": {"spec": {"containers": [{"name": "rustfs", "image": external_image}]}}}}); + existing = statefulsets + .patch( + &name, + &PatchParams::apply("external-image-owner").force(), + &Patch::Apply(&image_patch), + ) + .await + .expect("external manager should own the image"); + tenant.spec.readiness_probe = Some(ReadinessProbeConfig { + timeout_seconds: Some(9), + ..Default::default() + }); + let error = live_apply_with_status_retries(&ctx, &tenant, namespace, &statefulsets, &existing) + .await + .expect_err("ordinary apply must preserve conflicts on unrelated fields"); + assert!( + matches!(error, Error::Kube { source: kube::Error::Api(ref response) } + if response.code == 409 && response.message.contains("external-image-owner") + && response.message.contains(".image")), + "expected an image ownership conflict, got {error}" + ); + let preserved = statefulsets + .get(&name) + .await + .expect("StatefulSet should remain readable"); + assert_eq!( + rustfs_container(&preserved).image.as_deref(), + Some(external_image) + ); + assert!( + managed_container_fields(&preserved, "external-image-owner") + .unwrap() + .get("f:image") + .is_some() + ); + assert_eq!( + serde_json::to_value(&preserved.spec).unwrap(), + serde_json::to_value(&existing.spec).unwrap(), + "an unrelated ownership conflict must leave the entire StatefulSet spec unchanged" + ); + assert_eq!(readiness_probe(&preserved).timeout_seconds, Some(8)); + assert_eq!(preserved.spec.as_ref().unwrap().replicas, Some(0)); + + for replace_existing in [false, true] { + live_force_retry_preserves_concurrent_change( + client.clone(), + &tenant, + namespace, + replace_existing, + ) + .await; + } +} + +async fn live_force_retry_preserves_concurrent_change( + client: Client, + tenant: &Tenant, + namespace: &str, + replace_existing: bool, +) { + let api: Api = Api::namespaced(client.clone(), namespace); + let mut desired = live_desired(tenant); + let suffix = if replace_existing { + "replacement" + } else { + "update" + }; + let name = format!("{}-{suffix}", desired.name_any()); + desired.metadata.name = Some(name.clone()); + api.patch( + &name, + &PatchParams::apply("rustfs-operator"), + &Patch::Apply(&desired), + ) + .await + .expect("concurrency test StatefulSet should create"); + let timing = json!({ + "apiVersion": "apps/v1", "kind": "StatefulSet", + "metadata": {"name": name, "namespace": namespace}, + "spec": {"template": {"spec": {"containers": [{ + "name": "rustfs", "readinessProbe": {"timeoutSeconds": 1} + }]}}} + }); + api.patch( + &name, + &PatchParams::apply("external-readiness").force(), + &Patch::Apply(&timing), + ) + .await + .expect("external timing ownership should create a real SSA conflict"); + let existing = tokio::time::timeout(Duration::from_secs(15), async { + loop { + let current = api + .get(&name) + .await + .expect("StatefulSet should remain readable"); + if current + .status + .as_ref() + .and_then(|status| status.observed_generation) + == current.metadata.generation + { + break current; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + .await + .expect("zero-replica StatefulSet status should converge before the race"); + + let request_count = Arc::new(AtomicUsize::new(0)); + let changed = Arc::new(Mutex::new(None::)); + let mut replacement = desired.clone(); + rustfs_container_mut(&mut replacement).image = + Some("example.invalid/concurrent-image:1".to_string()); + rustfs_container_mut(&mut replacement) + .readiness_probe + .as_mut() + .unwrap() + .timeout_seconds = Some(1); + let service = service_fn({ + let request_count = Arc::clone(&request_count); + let changed = Arc::clone(&changed); + let name = name.clone(); + let api = api.clone(); + move |request: Request| { + let request_count = Arc::clone(&request_count); + let changed = Arc::clone(&changed); + let name = name.clone(); + let api = api.clone(); + let client = client.clone(); + let replacement = replacement.clone(); + async move { + let index = request_count.fetch_add(1, Ordering::SeqCst); + assert!( + index < 2, + "only the ordinary apply and one force retry are allowed" + ); + let force = request + .uri() + .query() + .unwrap_or_default() + .split('&') + .any(|parameter| parameter == "force=true"); + assert_eq!(force, index == 1); + let response = client.send(request).await?; + if index == 0 { + assert_eq!(response.status(), StatusCode::CONFLICT); + let updated = if replace_existing { + api.delete(&name, &DeleteParams::default()).await?; + tokio::time::timeout(Duration::from_secs(15), async { + loop { + match api.get(&name).await { + Err(kube::Error::Api(error)) if error.code == 404 => break, + Ok(_) => tokio::time::sleep(Duration::from_millis(100)).await, + Err(error) => panic!("replacement wait failed: {error}"), + } + } + }) + .await + .expect("zero-replica StatefulSet should delete"); + api.create(&PostParams::default(), &replacement).await? + } else { + let patch = json!({ + "metadata": {"annotations": {"concurrent-update": "retained"}}, + "spec": {"template": {"spec": {"containers": [{ + "name": "rustfs", "image": "example.invalid/concurrent-image:1" + }]}}} + }); + api.patch(&name, &PatchParams::default(), &Patch::Strategic(&patch)) + .await? + }; + *changed + .lock() + .expect("concurrent result should be available") = Some(updated); + } + Ok::<_, kube::Error>(response) + } + } + }); + let ctx = Context::new(Client::new(service, namespace)); + let error = apply_statefulset(&ctx, tenant, namespace, &existing, desired) + .await + .expect_err("force retry must reject a concurrent update or replacement"); + assert!( + matches!(error, Error::Kube { source: kube::Error::Api(ref response) } + if matches!(response.code, 409 | 422)), + "expected a resource identity or version conflict, got {error}" + ); + assert_eq!(request_count.load(Ordering::SeqCst), 2); + let changed = changed + .lock() + .unwrap() + .take() + .expect("the race should have run"); + let preserved = api + .get(&name) + .await + .expect("concurrent resource should remain readable"); + assert_eq!( + serde_json::to_value(&preserved.spec).unwrap(), + serde_json::to_value(&changed.spec).unwrap(), + "the rejected retry must preserve the entire concurrent spec" + ); + assert_eq!(preserved.metadata.annotations, changed.metadata.annotations); + assert_eq!(preserved.metadata.uid, changed.metadata.uid); + assert_eq!( + preserved.metadata.uid != existing.metadata.uid, + replace_existing + ); +} + +#[tokio::test] +#[ignore = "requires RUSTFS_READINESS_TEST_CONTEXT and the Tenant CRD; run explicitly"] +async fn readiness_ssa_live_preserves_values_ownership_and_external_fields() { + crate::install_rustls_crypto_provider(); + let context = std::env::var("RUSTFS_READINESS_TEST_CONTEXT") + .expect("set RUSTFS_READINESS_TEST_CONTEXT to a disposable Kubernetes context"); + let mut config = Config::from_kubeconfig(&KubeConfigOptions { + context: Some(context), + ..Default::default() + }) + .await + .expect("dedicated readiness test context should load"); + config.proxy_url = None; + let client = Client::try_from(config).expect("Kubernetes client should initialize"); + let namespaces: Api = Api::all(client.clone()); + let namespace = namespaces + .create( + &PostParams::default(), + &Namespace { + metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { + generate_name: Some("readiness-ssa-".to_string()), + ..Default::default() + }, + ..Default::default() + }, + ) + .await + .expect("isolated test namespace should create") + .name_any(); + let result = AssertUnwindSafe(run_live_scenario(client, &namespace)) + .catch_unwind() + .await; + let cleanup = namespaces + .delete(&namespace, &DeleteParams::default()) + .await; + if let Err(payload) = result { + std::panic::resume_unwind(payload); + } + cleanup.expect("isolated test namespace should delete"); +} diff --git a/src/status.rs b/src/status.rs index 28835e8b..cae00cbb 100644 --- a/src/status.rs +++ b/src/status.rs @@ -201,6 +201,11 @@ impl StatusError { ConditionType::SpecValid, sanitize_message(message), ), + types::error::Error::InvalidReadinessProbe { message, .. } => Self::blocked( + Reason::InvalidReadinessProbe, + ConditionType::SpecValid, + sanitize_message(message), + ), types::error::Error::ImmutableFieldModified { field, .. } => Self::blocked( Reason::ImmutableFieldModified, ConditionType::SpecValid, @@ -887,6 +892,29 @@ mod tests { ); } + #[test] + fn status_builder_marks_invalid_readiness_probe_as_invalid_spec() { + let tenant = crate::tests::create_test_tenant(None, None); + let err = types::error::Error::InvalidReadinessProbe { + name: tenant.name(), + message: "spec.readinessProbe.timeoutSeconds must be at least 1".to_string(), + }; + + let status_error = StatusError::from_types_error(&err); + let mut builder = StatusBuilder::from_tenant(&tenant); + builder.mark_error(&status_error); + let status = builder.build(); + + let condition = status.condition(ConditionType::SpecValid).unwrap(); + assert_eq!(condition.status, "False"); + assert_eq!(condition.reason, "InvalidReadinessProbe"); + assert_eq!(status.current_state, "Blocked"); + assert_eq!( + crate::types::v1alpha1::status::next_actions_for_reason(&condition.reason), + vec!["fixReadinessProbe"] + ); + } + #[test] fn successful_status_reports_rpc_auth_ready_only_for_managed_secret() { let mut unmanaged = crate::tests::create_test_tenant(None, None); diff --git a/src/types/error.rs b/src/types/error.rs index 2fde6e29..89fd4682 100755 --- a/src/types/error.rs +++ b/src/types/error.rs @@ -46,6 +46,9 @@ pub enum Error { ))] InvalidAdditionalVolumeSpec { name: String, message: String }, + #[snafu(display("invalid readiness probe for tenant '{}': {}", name, message))] + InvalidReadinessProbe { name: String, message: String }, + #[snafu(display("KMS migration is blocked for tenant '{}': {}", name, message))] KmsMigrationBlocked { name: String, message: String }, diff --git a/src/types/v1alpha1/status.rs b/src/types/v1alpha1/status.rs index 8401376b..a0ad89dc 100755 --- a/src/types/v1alpha1/status.rs +++ b/src/types/v1alpha1/status.rs @@ -120,6 +120,7 @@ pub enum Reason { InvalidTenantName, InvalidPoolSpec, InvalidAdditionalVolumeSpec, + InvalidReadinessProbe, ImmutableFieldModified, CredentialSecretNotFound, CredentialSecretMissingKey, @@ -209,6 +210,7 @@ impl Reason { Self::InvalidTenantName => "InvalidTenantName", Self::InvalidPoolSpec => "InvalidPoolSpec", Self::InvalidAdditionalVolumeSpec => "InvalidAdditionalVolumeSpec", + Self::InvalidReadinessProbe => "InvalidReadinessProbe", Self::ImmutableFieldModified => "ImmutableFieldModified", Self::CredentialSecretNotFound => "CredentialSecretNotFound", Self::CredentialSecretMissingKey => "CredentialSecretMissingKey", @@ -515,6 +517,7 @@ pub fn is_blocked_reason(reason: &str) -> bool { "InvalidTenantName" | "InvalidPoolSpec" | "InvalidAdditionalVolumeSpec" + | "InvalidReadinessProbe" | "ImmutableFieldModified" | "CredentialSecretNotFound" | "CredentialSecretMissingKey" @@ -601,6 +604,7 @@ pub fn next_actions_for_reason(reason: &str) -> Vec<&'static str> { match reason { "InvalidPoolSpec" => vec!["fixPoolSpec"], "InvalidAdditionalVolumeSpec" => vec!["fixAdditionalVolumeSpec"], + "InvalidReadinessProbe" => vec!["fixReadinessProbe"], "CredentialSecretNotFound" => vec!["createCredentialSecret"], "CredentialSecretMissingKey" => vec!["addRequiredSecretKey"], "CredentialSecretInvalidEncoding" => vec!["replaceSecretValueWithUtf8"], diff --git a/src/types/v1alpha1/tenant.rs b/src/types/v1alpha1/tenant.rs index 8e2a961e..9958b96a 100755 --- a/src/types/v1alpha1/tenant.rs +++ b/src/types/v1alpha1/tenant.rs @@ -83,6 +83,65 @@ pub struct OidcConfig { pub extra_ca_cert_secret_ref: Option, } +/// Timing overrides for the RustFS readiness probe. +/// +/// The operator selects the probe handler according to the Tenant TLS configuration. +#[derive(Deserialize, Serialize, Clone, Debug, KubeSchema, Default, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct ReadinessProbeConfig { + /// Seconds before readiness checks begin. Defaults to 0. + #[schemars(range(min = 0))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub initial_delay_seconds: Option, + + /// Timeout for each readiness check, in seconds. Defaults to 5. + #[schemars(range(min = 1))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_seconds: Option, + + /// Interval between readiness checks, in seconds. Defaults to 10. + #[schemars(range(min = 1))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub period_seconds: Option, + + /// Consecutive failures required to report NotReady. Defaults to 3. + #[schemars(range(min = 1))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_threshold: Option, + + /// Consecutive successes required to report Ready. Defaults to 1. + #[schemars(range(min = 1))] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub success_threshold: Option, +} + +impl ReadinessProbeConfig { + fn validate(&self) -> Result<(), String> { + for (field, value, minimum) in [ + ("initialDelaySeconds", self.initial_delay_seconds, 0), + ("timeoutSeconds", self.timeout_seconds, 1), + ("periodSeconds", self.period_seconds, 1), + ("failureThreshold", self.failure_threshold, 1), + ("successThreshold", self.success_threshold, 1), + ] { + if value.is_some_and(|value| value < minimum) { + return Err(format!( + "spec.readinessProbe.{field} must be at least {minimum}" + )); + } + } + Ok(()) + } + + fn apply_to(&self, probe: &mut corev1::Probe) { + probe.initial_delay_seconds = Some(self.initial_delay_seconds.unwrap_or(0)); + probe.timeout_seconds = Some(self.timeout_seconds.unwrap_or(5)); + probe.period_seconds = Some(self.period_seconds.unwrap_or(10)); + probe.failure_threshold = Some(self.failure_threshold.unwrap_or(3)); + probe.success_threshold = Some(self.success_threshold.unwrap_or(1)); + } +} + #[derive(CustomResource, Deserialize, Serialize, Clone, Debug, KubeSchema, Default)] #[kube( group = "rustfs.com", @@ -194,6 +253,10 @@ pub struct TenantSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub oidc: Option, + /// Readiness timing for every RustFS Pod. Omitted fields use operator defaults. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub readiness_probe: Option, + // #[serde(default, skip_serializing_if = "Option::is_none")] // pub request_auto_cert: Option, // @@ -204,9 +267,6 @@ pub struct TenantSpec { // pub liveness: Option, // // #[serde(default, skip_serializing_if = "Option::is_none")] - // pub readiness: Option, - // - // #[serde(default, skip_serializing_if = "Option::is_none")] // pub startup: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub lifecycle: Option, @@ -404,6 +464,18 @@ impl Tenant { }) } + pub(crate) fn validate_readiness_probe(&self) -> Result<(), types::error::Error> { + if let Some(config) = &self.spec.readiness_probe { + config + .validate() + .map_err(|message| types::error::Error::InvalidReadinessProbe { + name: self.name(), + message, + })?; + } + Ok(()) + } + /// a new owner reference for tenant pub fn new_owner_ref(&self) -> metav1::OwnerReference { metav1::OwnerReference { @@ -616,12 +688,85 @@ pub fn validate_dns1035_label(name: &str) -> Result<(), types::error::Error> { #[cfg(test)] mod tests { - use super::{OidcConfig, OidcExtraCaCertSecretRef, TenantSpec}; + use super::{OidcConfig, OidcExtraCaCertSecretRef, ReadinessProbeConfig, Tenant, TenantSpec}; use crate::types::v1alpha1::status::pool::PoolState; use k8s_openapi::api::apps::v1::{StatefulSet, StatefulSetSpec, StatefulSetStatus}; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + use kube::CustomResourceExt; use std::collections::BTreeSet; + #[test] + fn readiness_probe_preserves_omitted_fields_and_camel_case_names() { + let legacy: TenantSpec = serde_json::from_value(serde_json::json!({ "pools": [] })) + .expect("existing Tenant specs should deserialize"); + assert!(legacy.readiness_probe.is_none()); + + let config = serde_json::json!({ "timeoutSeconds": 8 }); + let spec: TenantSpec = serde_json::from_value(serde_json::json!({ + "pools": [], + "readinessProbe": config, + })) + .expect("readiness timing should deserialize"); + assert_eq!( + spec.readiness_probe, + Some(ReadinessProbeConfig { + timeout_seconds: Some(8), + ..Default::default() + }) + ); + let value = serde_json::to_value(spec).expect("Tenant spec should serialize"); + assert_eq!(value["readinessProbe"], config); + } + + #[test] + fn readiness_probe_validation_accepts_boundaries_and_rejects_invalid_values() { + for (field, minimum) in [ + ("initialDelaySeconds", 0), + ("timeoutSeconds", 1), + ("periodSeconds", 1), + ("failureThreshold", 1), + ("successThreshold", 1), + ] { + for value in [minimum, i32::MAX] { + let config: ReadinessProbeConfig = + serde_json::from_value(serde_json::json!({ field: value })) + .expect("readiness timing should deserialize"); + assert!(config.validate().is_ok(), "{field} should accept {value}"); + } + let config: ReadinessProbeConfig = + serde_json::from_value(serde_json::json!({ field: minimum - 1 })) + .expect("invalid timing should reach runtime validation"); + let error = config.validate().expect_err("invalid timing must fail"); + assert!(error.contains(&format!("spec.readinessProbe.{field}"))); + } + assert!(ReadinessProbeConfig::default().validate().is_ok()); + } + + #[test] + fn readiness_probe_crd_exposes_only_optional_timing_fields_with_bounds() { + let crd = serde_json::to_value(Tenant::crd()).expect("Tenant CRD should serialize"); + let schema = &crd["spec"]["versions"][0]["schema"]["openAPIV3Schema"]["properties"]["spec"] + ["properties"]["readinessProbe"]; + assert_eq!( + schema["properties"].as_object().map(|fields| fields.len()), + Some(5) + ); + assert!(schema.get("required").is_none()); + for (field, minimum) in [ + ("initialDelaySeconds", 0), + ("timeoutSeconds", 1), + ("periodSeconds", 1), + ("failureThreshold", 1), + ("successThreshold", 1), + ] { + assert_eq!( + schema["properties"][field]["minimum"].as_f64(), + Some(f64::from(minimum)) + ); + assert_eq!(schema["properties"][field]["format"], "int32"); + } + } + #[test] fn oidc_extra_ca_secret_ref_defaults_to_ca_crt() { let config: OidcConfig = serde_json::from_value(serde_json::json!({ diff --git a/src/types/v1alpha1/tenant/workloads.rs b/src/types/v1alpha1/tenant/workloads.rs index cff64cd0..fc4c83fe 100755 --- a/src/types/v1alpha1/tenant/workloads.rs +++ b/src/types/v1alpha1/tenant/workloads.rs @@ -70,6 +70,44 @@ pub(crate) fn uses_unpartitioned_rolling_update( strategy_type == "RollingUpdate" && partition == 0 } +fn readiness_probes_equal( + existing: Option<&corev1::Probe>, + desired: Option<&corev1::Probe>, +) -> bool { + let (existing, desired) = match (existing, desired) { + (Some(existing), Some(desired)) => (existing, desired), + (None, None) => return true, + _ => return false, + }; + let http_equal = match (existing.http_get.as_ref(), desired.http_get.as_ref()) { + (Some(existing), Some(desired)) => { + existing.path.as_deref().unwrap_or("/") == desired.path.as_deref().unwrap_or("/") + && existing.port == desired.port + && existing.scheme.as_deref().unwrap_or("HTTP") + == desired.scheme.as_deref().unwrap_or("HTTP") + } + (None, None) => true, + _ => false, + }; + let timing_values = |probe: &corev1::Probe| { + [ + probe.initial_delay_seconds.unwrap_or(0), + probe.timeout_seconds.unwrap_or(1), + probe.period_seconds.unwrap_or(10), + probe.failure_threshold.unwrap_or(3), + probe.success_threshold.unwrap_or(1), + ] + }; + + // External managers can add HTTP headers or a host without changing the + // endpoint and timing fields that the Operator owns. + timing_values(existing) == timing_values(desired) + && http_equal + && existing.exec == desired.exec + && existing.grpc == desired.grpc + && existing.tcp_socket == desired.tcp_socket +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ImageUnverifiableReason { CustomRepository, @@ -1486,6 +1524,7 @@ impl Tenant { ) -> Result { self.validate_declared_workload_security_contexts()?; self.validate_additional_volumes()?; + self.validate_readiness_probe()?; let labels = self.pool_labels(pool); let selector_labels = self.pool_selector_labels(pool); @@ -1667,6 +1706,21 @@ impl Tenant { .. } = security; + let mut readiness_probe = if tls_plan.mtls_enabled { + if super::helper::use_tcp_mtls_readiness_probe(&self.spec.env) { + tcp_probe() + } else { + mtls_readiness_probe(&tls_plan.mount_path) + } + } else { + http_probe("/health/ready", tls_plan.probe_scheme) + }; + self.spec + .readiness_probe + .as_ref() + .unwrap_or(&super::ReadinessProbeConfig::default()) + .apply_to(&mut readiness_probe); + let container = corev1::Container { name: "rustfs".to_owned(), image: Some(super::helper::get_rustfs_image_or_default( @@ -1705,15 +1759,7 @@ impl Tenant { } else { http_probe("/health", tls_plan.probe_scheme) }), - readiness_probe: Some(if tls_plan.mtls_enabled { - if super::helper::use_tcp_mtls_readiness_probe(&self.spec.env) { - tcp_probe() - } else { - mtls_readiness_probe(&tls_plan.mount_path) - } - } else { - http_probe("/health/ready", tls_plan.probe_scheme) - }), + readiness_probe: Some(readiness_probe), startup_probe: Some(if tls_plan.mtls_enabled { tcp_probe() } else { @@ -2003,6 +2049,13 @@ impl Tenant { let existing_container = &existing_pod_spec.containers[0]; let desired_container = &desired_pod_spec.containers[0]; + if !readiness_probes_equal( + existing_container.readiness_probe.as_ref(), + desired_container.readiness_probe.as_ref(), + ) { + return Ok(true); + } + // Check image if existing_container.image != desired_container.image { return Ok(true); @@ -2051,8 +2104,6 @@ impl Tenant { // Check container probes (mTLS switches HTTP probes to TCP/exec probes) if normalize_probe_for_comparison(existing_container.liveness_probe.as_ref()) != normalize_probe_for_comparison(desired_container.liveness_probe.as_ref()) - || normalize_probe_for_comparison(existing_container.readiness_probe.as_ref()) - != normalize_probe_for_comparison(desired_container.readiness_probe.as_ref()) || normalize_probe_for_comparison(existing_container.startup_probe.as_ref()) != normalize_probe_for_comparison(desired_container.startup_probe.as_ref()) { @@ -2345,7 +2396,7 @@ mod tests { use super::{ DEFAULT_FS_GROUP, DEFAULT_RUN_AS_GROUP, DEFAULT_RUN_AS_USER, MAX_APP_ARMOR_LOCALHOST_PROFILE_LENGTH, RUNTIME_DEFAULT_IMAGE_ACK_ANNOTATION, - quantities_semantically_equal, uses_unpartitioned_rolling_update, + quantities_semantically_equal, readiness_probes_equal, uses_unpartitioned_rolling_update, validate_declared_app_armor_profile, validate_declared_seccomp_profile, }; use crate::types::v1alpha1::encryption::{ @@ -2354,9 +2405,9 @@ mod tests { use crate::types::v1alpha1::logging::{LoggingConfig, LoggingMode}; use crate::types::v1alpha1::security_context::{MAX_KUBERNETES_ID, PodSecurityContextOverride}; use crate::types::v1alpha1::tenant::{ - OidcConfig, OidcExtraCaCertSecretRef, RpcSecretRef, Tenant, + OidcConfig, OidcExtraCaCertSecretRef, ReadinessProbeConfig, RpcSecretRef, Tenant, }; - use crate::types::v1alpha1::tls::{SecretKeyReference, TlsPlan}; + use crate::types::v1alpha1::tls::{SecretKeyReference, TlsPlan, http_probe}; use k8s_openapi::api::apps::v1; use k8s_openapi::api::core::v1 as corev1; use k8s_openapi::apimachinery::pkg::api::resource::Quantity; @@ -4061,12 +4112,24 @@ mod tests { .into_iter() .flatten() { - probe.timeout_seconds = Some(1); - probe.period_seconds = Some(10); - probe.success_threshold = Some(1); - probe.failure_threshold = Some(3); - probe.initial_delay_seconds = Some(0); + probe.timeout_seconds.get_or_insert(1); + probe.period_seconds.get_or_insert(10); + probe.success_threshold.get_or_insert(1); + probe.failure_threshold.get_or_insert(3); + probe.initial_delay_seconds.get_or_insert(0); } + assert_eq!( + container.readiness_probe.as_ref().unwrap().timeout_seconds, + Some(5) + ); + assert_eq!( + container.liveness_probe.as_ref().unwrap().timeout_seconds, + Some(1) + ); + assert_eq!( + container.startup_probe.as_ref().unwrap().timeout_seconds, + Some(1) + ); let needs_update = tenant .statefulset_needs_update(&statefulset, pool) @@ -5637,6 +5700,385 @@ mod tests { ); } + #[test] + fn readiness_probe_configuration_preserves_tls_and_other_probes() { + for tls in [TlsPlan::disabled(), tls_plan("sha256:readiness")] { + let mut tenant = crate::tests::create_test_tenant(None, None); + let baseline = tenant + .new_statefulset_with_tls_plan(&tenant.spec.pools[0], &tls) + .expect("default StatefulSet should render"); + let baseline_container = &baseline.spec.unwrap().template.spec.unwrap().containers[0]; + + for (config, expected) in [ + (None, [0, 5, 10, 3, 1]), + (Some(ReadinessProbeConfig::default()), [0, 5, 10, 3, 1]), + ( + Some(ReadinessProbeConfig { + timeout_seconds: Some(8), + ..Default::default() + }), + [0, 8, 10, 3, 1], + ), + ( + Some(ReadinessProbeConfig { + initial_delay_seconds: Some(2), + timeout_seconds: Some(8), + period_seconds: Some(15), + failure_threshold: Some(4), + success_threshold: Some(2), + }), + [2, 8, 15, 4, 2], + ), + ] { + tenant.spec.readiness_probe = config; + let statefulset = tenant + .new_statefulset_with_tls_plan(&tenant.spec.pools[0], &tls) + .expect("configured StatefulSet should render"); + let container = &statefulset.spec.unwrap().template.spec.unwrap().containers[0]; + let probe = container.readiness_probe.as_ref().unwrap(); + let http_get = probe.http_get.as_ref().unwrap(); + assert_eq!(http_get.path.as_deref(), Some("/health/ready")); + assert_eq!( + http_get.port, + k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int(9000) + ); + assert_eq!(http_get.scheme.as_deref(), Some(tls.probe_scheme)); + assert_eq!( + [ + probe.initial_delay_seconds, + probe.timeout_seconds, + probe.period_seconds, + probe.failure_threshold, + probe.success_threshold, + ], + expected.map(Some) + ); + assert_eq!( + probe.http_get, + baseline_container + .readiness_probe + .as_ref() + .unwrap() + .http_get + ); + assert_eq!(container.liveness_probe, baseline_container.liveness_probe); + assert_eq!(container.startup_probe, baseline_container.startup_probe); + } + } + } + + #[test] + fn readiness_timing_preserves_mtls_handlers_and_converges() { + for mode in ["exec", "tcp"] { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.spec.env.push(corev1::EnvVar { + name: "TENANT_MTLS_READINESS_PROBE".to_string(), + value: Some(mode.to_string()), + ..Default::default() + }); + let mut tls = tls_plan("sha256:readiness-mtls"); + tls.mtls_enabled = true; + let pool = tenant.spec.pools[0].clone(); + let baseline = tenant.new_statefulset_with_tls_plan(&pool, &tls).unwrap(); + let baseline_container = &baseline + .spec + .as_ref() + .unwrap() + .template + .spec + .as_ref() + .unwrap() + .containers[0]; + let baseline_probe = baseline_container.readiness_probe.as_ref().unwrap(); + assert_eq!(baseline_probe.exec.is_some(), mode == "exec"); + assert_eq!(baseline_probe.tcp_socket.is_some(), mode == "tcp"); + assert!( + !tenant + .statefulset_needs_update_with_tls_plan(&baseline, &pool, &tls) + .unwrap() + ); + + tenant.spec.readiness_probe = Some(ReadinessProbeConfig { + initial_delay_seconds: Some(2), + timeout_seconds: Some(8), + period_seconds: Some(15), + failure_threshold: Some(4), + success_threshold: Some(2), + }); + assert!( + tenant + .statefulset_needs_update_with_tls_plan(&baseline, &pool, &tls) + .unwrap() + ); + let configured = tenant.new_statefulset_with_tls_plan(&pool, &tls).unwrap(); + let container = &configured + .spec + .as_ref() + .unwrap() + .template + .spec + .as_ref() + .unwrap() + .containers[0]; + let probe = container.readiness_probe.as_ref().unwrap(); + assert_eq!(probe.exec, baseline_probe.exec); + assert_eq!(probe.tcp_socket, baseline_probe.tcp_socket); + assert_eq!(container.liveness_probe, baseline_container.liveness_probe); + assert_eq!(container.startup_probe, baseline_container.startup_probe); + assert_eq!( + [ + probe.initial_delay_seconds, + probe.timeout_seconds, + probe.period_seconds, + probe.failure_threshold, + probe.success_threshold, + ], + [2, 8, 15, 4, 2].map(Some) + ); + assert!( + !tenant + .statefulset_needs_update_with_tls_plan(&configured, &pool, &tls) + .unwrap() + ); + + let mut changed_handler = probe.clone(); + if let Some(exec) = changed_handler.exec.as_mut() { + exec.command = Some(vec!["true".to_string()]); + } + if let Some(tcp) = changed_handler.tcp_socket.as_mut() { + tcp.port = k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int(9001); + } + assert!(!readiness_probes_equal(Some(&changed_handler), Some(probe))); + tenant.spec.readiness_probe = None; + assert!( + tenant + .statefulset_needs_update_with_tls_plan(&configured, &pool, &tls) + .unwrap() + ); + } + } + + #[test] + fn readiness_probe_invalid_timing_prevents_statefulset_rendering() { + let mut tenant = crate::tests::create_test_tenant(None, None); + tenant.spec.readiness_probe = Some(ReadinessProbeConfig { + timeout_seconds: Some(0), + ..Default::default() + }); + let error = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect_err("invalid readiness timing must not reach the Pod template"); + assert!(matches!( + error, + crate::types::error::Error::InvalidReadinessProbe { name, message } + if name == "test-tenant" && message.contains("timeoutSeconds") + )); + } + + #[test] + fn readiness_probe_timing_changes_and_removal_require_statefulset_updates() { + for (field, value) in [ + ("initialDelaySeconds", 2), + ("timeoutSeconds", 8), + ("periodSeconds", 15), + ("failureThreshold", 4), + ("successThreshold", 2), + ] { + let mut tenant = crate::tests::create_test_tenant(None, None); + let existing = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("default StatefulSet should render"); + tenant.spec.readiness_probe = Some( + serde_json::from_value(serde_json::json!({ field: value })) + .expect("readiness configuration should deserialize"), + ); + assert!( + tenant + .statefulset_needs_update(&existing, &tenant.spec.pools[0]) + .unwrap(), + "changing {field} must update the StatefulSet" + ); + let configured = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("configured StatefulSet should render"); + assert!( + !tenant + .statefulset_needs_update(&configured, &tenant.spec.pools[0]) + .unwrap() + ); + tenant.spec.readiness_probe = None; + assert!( + tenant + .statefulset_needs_update(&configured, &tenant.spec.pools[0]) + .unwrap(), + "removing {field} must restore the operator default" + ); + } + } + + #[test] + fn readiness_probe_comparison_uses_kubernetes_defaults_for_existing_objects() { + let mut tenant = crate::tests::create_test_tenant(None, None); + let mut existing = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("StatefulSet should render"); + let container = &mut existing + .spec + .as_mut() + .unwrap() + .template + .spec + .as_mut() + .unwrap() + .containers[0]; + let mut probe = http_probe("/health/ready", "HTTP"); + probe.timeout_seconds = Some(5); + container.readiness_probe = Some(probe); + assert!( + !tenant + .statefulset_needs_update(&existing, &tenant.spec.pools[0]) + .unwrap(), + "omitted Kubernetes defaults must not cause repeated updates" + ); + + existing + .spec + .as_mut() + .unwrap() + .template + .spec + .as_mut() + .unwrap() + .containers[0] + .readiness_probe + .as_mut() + .unwrap() + .timeout_seconds = None; + assert!( + tenant + .statefulset_needs_update(&existing, &tenant.spec.pools[0]) + .unwrap(), + "the historical one-second timeout must receive the new default" + ); + tenant.spec.readiness_probe = Some(ReadinessProbeConfig { + timeout_seconds: Some(1), + ..Default::default() + }); + assert!( + !tenant + .statefulset_needs_update(&existing, &tenant.spec.pools[0]) + .unwrap(), + "an explicit one-second timeout must match the Kubernetes default" + ); + existing + .spec + .as_mut() + .unwrap() + .template + .spec + .as_mut() + .unwrap() + .containers[0] + .readiness_probe = None; + assert!( + tenant + .statefulset_needs_update(&existing, &tenant.spec.pools[0]) + .unwrap(), + "a missing readiness probe must be restored" + ); + } + + #[test] + fn readiness_probe_comparison_preserves_unmanaged_http_fields() { + let mut tenant = crate::tests::create_test_tenant(None, None); + let mut existing = tenant + .new_statefulset(&tenant.spec.pools[0]) + .expect("StatefulSet should render"); + let probe = existing + .spec + .as_mut() + .unwrap() + .template + .spec + .as_mut() + .unwrap() + .containers[0] + .readiness_probe + .as_mut() + .unwrap(); + let http_get = probe.http_get.as_mut().unwrap(); + http_get.host = Some("127.0.0.1".to_string()); + http_get.http_headers = Some(vec![corev1::HTTPHeader { + name: "X-Probe-Source".to_string(), + value: "external-manager".to_string(), + }]); + probe.timeout_seconds = Some(8); + assert!( + tenant + .statefulset_needs_update(&existing, &tenant.spec.pools[0]) + .unwrap(), + "external fields must not hide changes to managed timing" + ); + + tenant.spec.readiness_probe = Some(ReadinessProbeConfig { + timeout_seconds: Some(8), + ..Default::default() + }); + assert!( + !tenant + .statefulset_needs_update(&existing, &tenant.spec.pools[0]) + .unwrap(), + "matching managed fields must converge with external HTTP fields present" + ); + } + + #[test] + fn readiness_probe_comparison_tracks_managed_http_fields() { + let desired = http_probe("/health/ready", "HTTP"); + let mut changed_port = desired.clone(); + changed_port.http_get.as_mut().unwrap().port = + k8s_openapi::apimachinery::pkg::util::intstr::IntOrString::Int(9001); + for existing in [ + http_probe("/health", "HTTP"), + http_probe("/health/ready", "HTTPS"), + changed_port, + ] { + assert!( + !readiness_probes_equal(Some(&existing), Some(&desired)), + "managed endpoint changes must require an update" + ); + } + + let mut default_scheme = desired.clone(); + default_scheme.http_get.as_mut().unwrap().scheme = None; + assert!( + readiness_probes_equal(Some(&default_scheme), Some(&desired)), + "an omitted scheme uses Kubernetes' HTTP default" + ); + } + + #[test] + fn readiness_probe_comparison_detects_missing_or_replaced_http_handler() { + let desired = http_probe("/health/ready", "HTTP"); + for existing in [ + corev1::Probe::default(), + corev1::Probe { + exec: Some(corev1::ExecAction { + command: Some(vec!["true".to_string()]), + }), + ..Default::default() + }, + ] { + assert!( + !readiness_probes_equal(Some(&existing), Some(&desired)), + "a missing or replaced HTTP handler must require an update" + ); + } + assert!(!readiness_probes_equal(None, Some(&desired))); + assert!(!readiness_probes_equal(Some(&desired), None)); + assert!(readiness_probes_equal(None, None)); + } + // Test: StatefulSet diff detection - no changes needed #[test] fn test_statefulset_no_update_needed() { diff --git a/tests/tenant_chart.rs b/tests/tenant_chart.rs index ab1cd160..37eb45bf 100644 --- a/tests/tenant_chart.rs +++ b/tests/tenant_chart.rs @@ -142,6 +142,52 @@ tenant: assert_resource_boundary(&docs); } +#[test] +fn readiness_timing_values_reach_generated_statefulsets() { + for (values, expected) in [ + ( + r#" +tenant: + spec: + readinessProbe: + initialDelaySeconds: 7 + timeoutSeconds: 9 + periodSeconds: 17 + failureThreshold: 4 + successThreshold: 2 +"#, + (7, 9, 17, 4, 2), + ), + ( + "tenant: {spec: {readinessProbe: {timeoutSeconds: 8}}}", + (0, 8, 10, 3, 1), + ), + ] { + let docs = documents(render(values, &[])); + let tenant = tenant(&docs); + let workload = tenant.new_statefulset(&tenant.spec.pools[0]).unwrap(); + let pod = workload.spec.unwrap().template.spec.unwrap(); + let container = pod + .containers + .iter() + .find(|container| container.name == "rustfs") + .unwrap(); + let probe = container.readiness_probe.as_ref().unwrap(); + assert_eq!( + ( + probe.initial_delay_seconds.unwrap_or_default(), + probe.timeout_seconds.unwrap(), + probe.period_seconds.unwrap(), + probe.failure_threshold.unwrap(), + probe.success_threshold.unwrap(), + ), + expected, + "chart values must reach the generated Pod readiness probe: {values}" + ); + assert_resource_boundary(&docs); + } +} + #[test] fn ingress_backends_match_operator_services_and_preserve_tls() { let docs = documents(render(