From 881765ce39e65129b70eb3ff9088c1848c4c894c Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 23 Oct 2026 16:02:41 +0200 Subject: [PATCH 1/5] Remove support for node certificates without domain components --- CHANGELOG.md | 8 ++++++++ .../src/controller/build/node_config.rs | 10 ---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c16750..86b18e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,13 @@ All notable changes to this project will be documented in this file. which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#172]). +### Removed + +- Remove support for node certificates whose distinguished names lack domain components, i.e. the + allowed node DN `CN=generated certificate for pod`. This entry was only required to make the + transition from SDP 26.3 to 26.7 possible. Since SDP 26.7, the generated certificates contain the + domain components, so the fallback is no longer needed in SDP 26.11 ([#202]). + [#166]: https://github.com/stackabletech/opensearch-operator/pull/166 [#171]: https://github.com/stackabletech/opensearch-operator/pull/171 [#172]: https://github.com/stackabletech/opensearch-operator/pull/172 @@ -49,6 +56,7 @@ All notable changes to this project will be documented in this file. [#198]: https://github.com/stackabletech/opensearch-operator/pull/198 [#199]: https://github.com/stackabletech/opensearch-operator/pull/199 [#201]: https://github.com/stackabletech/opensearch-operator/pull/201 +[#202]: https://github.com/stackabletech/opensearch-operator/pull/202 ## [26.7.0] - 2026-07-21 diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 88f0516..8f5db41 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -253,10 +253,6 @@ impl NodeConfig { /// The list looks similar to: /// - DC=local,DC=cluster,DC=svc,DC=my-namespace,DC=opensearch-nodes-cluster-manager-headless,DC=opensearch-nodes-cluster-manager-* /// - DC=local,DC=cluster,DC=svc,DC=my-namespace,DC=opensearch-nodes-data-headless,DC=opensearch-nodes-data-* - /// - CN=generated certificate for pod - /// - /// The entry "CN=generated certificate for pod" is still added to make the transition from - /// SDP 26.3 to 26.7 possible. fn nodes_dn(&self) -> Vec { self.cluster .role_group_configs @@ -284,11 +280,6 @@ impl NodeConfig { .collect::>() .join(",") }) - // TODO Remove "CN=generated certificate for pod" after the release of SDP 26.7 and - // adapt the comment of the function and the tests. - // - // tracked in https://github.com/stackabletech/opensearch-operator/issues/145 - .chain(iter::once("CN=generated certificate for pod".to_owned())) .collect() } @@ -715,7 +706,6 @@ mod tests { "plugins.security.authcz.admin_dn: CN=update-security-config.0b1e30e6-326e-4c1a-868d-ad6598b49e8b\n", "plugins.security.nodes_dn:\n", "- DC=local,DC=cluster,DC=svc,DC=default,DC=my-opensearch-cluster-nodes-default-headless,DC=my-opensearch-cluster-nodes-default-*\n", - "- CN=generated certificate for pod\n", "plugins.security.ssl.http.enabled: true\n", "plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/server/tls.crt\n", "plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/server/tls.key\n", From 3944ba2bdd79a98101aaed728ec6e6a5e226a672 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 23 Oct 2026 16:20:29 +0200 Subject: [PATCH 2/5] chore: Fix clippy warnings --- .../src/controller/build/node_config.rs | 2 +- .../controller/build/role_group_builder.rs | 31 ++++++++++--------- .../src/controller/dereference.rs | 4 ++- .../src/controller/update_status.rs | 4 ++- .../src/webhooks/conversion.rs | 6 +++- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/rust/operator-binary/src/controller/build/node_config.rs b/rust/operator-binary/src/controller/build/node_config.rs index 8f5db41..6f202a4 100644 --- a/rust/operator-binary/src/controller/build/node_config.rs +++ b/rust/operator-binary/src/controller/build/node_config.rs @@ -1,6 +1,6 @@ //! Configuration of an OpenSearch node -use std::{iter, str::FromStr}; +use std::str::FromStr; use serde_json::json; use stackable_operator::{ diff --git a/rust/operator-binary/src/controller/build/role_group_builder.rs b/rust/operator-binary/src/controller/build/role_group_builder.rs index ccf03bd..76190d5 100644 --- a/rust/operator-binary/src/controller/build/role_group_builder.rs +++ b/rust/operator-binary/src/controller/build/role_group_builder.rs @@ -821,22 +821,23 @@ impl<'a> RoleGroupBuilder<'a> { let opensearch_path_conf = self.node_config.opensearch_path_conf(); for file_type in settings { - let mount_path; - let sub_path; - - if use_sub_path { - mount_path = format!( - "{opensearch_path_conf}/opensearch-security/{filename}", - filename = file_type.filename.to_owned() - ); - sub_path = Some(file_type.filename.to_owned()); + let (mount_path, sub_path) = if use_sub_path { + ( + format!( + "{opensearch_path_conf}/opensearch-security/{filename}", + filename = file_type.filename.to_owned() + ), + Some(file_type.filename.to_owned()), + ) } else { - mount_path = format!( - "{opensearch_path_conf}/opensearch-security/{file_type}", - file_type = file_type.id - ); - sub_path = None; - } + ( + format!( + "{opensearch_path_conf}/opensearch-security/{file_type}", + file_type = file_type.id + ), + None, + ) + }; volume_mounts.push(VolumeMount { mount_path, diff --git a/rust/operator-binary/src/controller/dereference.rs b/rust/operator-binary/src/controller/dereference.rs index f0a8aff..0f7d142 100644 --- a/rust/operator-binary/src/controller/dereference.rs +++ b/rust/operator-binary/src/controller/dereference.rs @@ -31,7 +31,9 @@ pub enum Error { #[snafu(display("failed to fetch the discovery service listener"))] FetchDiscoveryServiceListener { - source: stackable_operator::client::Error, + // boxed because otherwise Clippy warns about a large enum variant + #[snafu(source(from(stackable_operator::client::Error, Box::new)))] + source: Box, }, } diff --git a/rust/operator-binary/src/controller/update_status.rs b/rust/operator-binary/src/controller/update_status.rs index b018533..46c3361 100644 --- a/rust/operator-binary/src/controller/update_status.rs +++ b/rust/operator-binary/src/controller/update_status.rs @@ -28,7 +28,9 @@ pub enum Error { #[snafu(display("failed to update status"))] UpdateStatus { - source: stackable_operator::client::Error, + // boxed because otherwise Clippy warns about a large enum variant + #[snafu(source(from(stackable_operator::client::Error, Box::new)))] + source: Box, }, } diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index 1655cf6..c339962 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -22,7 +22,11 @@ pub enum Error { MergeCrd { source: MergeError }, #[snafu(display("failed to create conversion webhook server"))] - CreateWebhook { source: WebhookServerError }, + CreateWebhook { + // boxed because otherwise Clippy warns about a large enum variant + #[snafu(source(from(WebhookServerError, Box::new)))] + source: Box, + }, } /// Creates and returns a [`WebhookServer`]. From d99439a76922ee363d3e2d329cb9b12d617dfaa8 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 23 Oct 2026 18:22:21 +0200 Subject: [PATCH 3/5] docs: Add upgrade notes --- CHANGELOG.md | 12 ++++++------ .../opensearch/pages/usage-guide/upgrade.adoc | 10 ++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86b18e0..21cf5ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,12 +31,6 @@ All notable changes to this project will be documented in this file. - Replace the unmaintained `serde_yaml` crate with `yaml_serde` ([#198]). - The operator Deployment runs with a restricted security context and a read-only root filesystem, so it can be deployed into namespaces that enforce the `restricted` Pod Security Standard ([#201]). -### Fixed - -- Fix a longstanding problem of including empty `categories`, `shortNames` and `additionalPrinterColumns` in the CRDs, - which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources. - See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#172]). - ### Removed - Remove support for node certificates whose distinguished names lack domain components, i.e. the @@ -44,6 +38,12 @@ All notable changes to this project will be documented in this file. transition from SDP 26.3 to 26.7 possible. Since SDP 26.7, the generated certificates contain the domain components, so the fallback is no longer needed in SDP 26.11 ([#202]). +### Fixed + +- Fix a longstanding problem of including empty `categories`, `shortNames` and `additionalPrinterColumns` in the CRDs, + which could cause problems with GitOps tools (e.g. ArgoCD) reporting a diff in the custom resources. + See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#172]). + [#166]: https://github.com/stackabletech/opensearch-operator/pull/166 [#171]: https://github.com/stackabletech/opensearch-operator/pull/171 [#172]: https://github.com/stackabletech/opensearch-operator/pull/172 diff --git a/docs/modules/opensearch/pages/usage-guide/upgrade.adoc b/docs/modules/opensearch/pages/usage-guide/upgrade.adoc index bb3adb3..ed11acd 100644 --- a/docs/modules/opensearch/pages/usage-guide/upgrade.adoc +++ b/docs/modules/opensearch/pages/usage-guide/upgrade.adoc @@ -1,6 +1,16 @@ = SDP upgrade notes :description: Instructions for upgrading the SDP versions. +== Upgrade from SDP 26.7 to 26.11 + +=== Node certificates + +The operator no longer accepts node certificates whose distinguished names lack domain components. +Such certificates were issued up to SDP 26.3. + +If you are upgrading from SDP 26.3, then you must upgrade to SDP 26.7 first and restart all OpenSearch Pods, so that they get certificates with domain components. +Otherwise, the nodes which are already restarted with SDP 26.11 reject the transport connections of the nodes which are not restarted yet, and the cluster can lose its quorum during the rolling restart. + == Upgrade from SDP 26.3 to 26.7 === Dedicated ConfigMap for security settings From 3e7f396626ecc7b89ea23deb1b5b825e0ca5b68a Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 26 Oct 2026 11:57:15 +0100 Subject: [PATCH 4/5] docs: Extend the upgrade notes --- .../opensearch/pages/usage-guide/upgrade.adoc | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/docs/modules/opensearch/pages/usage-guide/upgrade.adoc b/docs/modules/opensearch/pages/usage-guide/upgrade.adoc index ed11acd..8d9e317 100644 --- a/docs/modules/opensearch/pages/usage-guide/upgrade.adoc +++ b/docs/modules/opensearch/pages/usage-guide/upgrade.adoc @@ -3,6 +3,9 @@ == Upgrade from SDP 26.7 to 26.11 +A direct upgrade from SDP 26.3 to 26.11 is not supported. +Upgrade to SDP 26.7 first and restart all OpenSearch Pods, see the notes for the upgrade from SDP 26.3 to 26.7 below. + === Node certificates The operator no longer accepts node certificates whose distinguished names lack domain components. @@ -11,8 +14,64 @@ Such certificates were issued up to SDP 26.3. If you are upgrading from SDP 26.3, then you must upgrade to SDP 26.7 first and restart all OpenSearch Pods, so that they get certificates with domain components. Otherwise, the nodes which are already restarted with SDP 26.11 reject the transport connections of the nodes which are not restarted yet, and the cluster can lose its quorum during the rolling restart. +=== StatefulSet updates + +The `app.kubernetes.io/version` label was removed from the listener PVC templates of the StatefulSets. +Kubernetes restricts changes to `spec.volumeClaimTemplates` and other StatefulSet fields, so the operator cannot apply this change in place and the following error occurs: + +[source] +---- +ERROR stackable_operator::logging::controller: Failed to reconcile object ... +unable to patch resource "opensearch-nodes-default", +ApiError: StatefulSet.apps "opensearch-nodes-default" is invalid: +spec: Forbidden: updates to statefulset spec for fields other than 'replicas', ... +---- + +The OpenSearch cluster keeps running at first, so this error is easy to overlook. +The operator cannot apply any further change to the StatefulSets though, for instance to the number of replicas or to the product version, until the error is resolved. +The Pods also keep the outdated Pod template. +If such a Pod is restarted, then it can fail to start, because the operator already changed the resources which the outdated Pod template refers to. +For instance, a Pod which is restarted after an upgrade from SDP 26.3 stays in the state `ContainerCreating`, because the security settings moved into a dedicated ConfigMap: + +[source] +---- +MountVolume.SetUp failed for volume "security-config-file-config": +configmap references non-existent config key: config.yml +---- + +To resolve this error, you need to delete the StatefulSets right after the operator upgrade. +You can do this using the following command: + +[source,shell] +---- +kubectl delete statefulsets.apps --cascade=orphan opensearch-nodes-default +---- + +The operator will automatically recreate the StatefulSet. +`--cascade=orphan` retains the Pods, so that they are not all deleted at once. +The recreated StatefulSet adopts them and restarts them one after another to apply the new product image. + +Up to SDP 26.7, the label was set to the product version, so this error occurred on every product version upgrade. +Since the label is removed in SDP 26.11, the error does not occur in subsequent upgrades. + == Upgrade from SDP 26.3 to 26.7 +=== Node certificates + +Since SDP 26.7, the certificates issued by the Secret Operator contain domain components in their distinguished names, and the operator adds the corresponding entry to the list of allowed node distinguished names. +This list is read once at node startup, so the nodes which are still running with SDP 26.3 reject the transport connections of the nodes which are already restarted with SDP 26.7: + +[source] +---- +Node presenting certificate with SSL Principal {DC=local,DC=cluster,DC=svc, +DC=default,DC=opensearch-nodes-default-headless,DC=opensearch-nodes-default-0, +CN=generated certificate for pod} could not securely connect to the cluster. +Please ensure the principal is correct and present in the nodes_dn list. +---- + +The cluster can lose its quorum during the rolling restart and be unavailable for a short time. +Plan for a short downtime; the cluster recovers on its own once all Pods are restarted. + === Dedicated ConfigMap for security settings The security settings defined in the cluster specification are now stored in a separate ConfigMap named `-security-config`. From 7bddfaca3a79f39c542e2d98a0cd563a5248a0d1 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 26 Oct 2026 12:02:29 +0100 Subject: [PATCH 5/5] docs: Fix the upgrade notes --- .../modules/opensearch/pages/usage-guide/upgrade.adoc | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docs/modules/opensearch/pages/usage-guide/upgrade.adoc b/docs/modules/opensearch/pages/usage-guide/upgrade.adoc index 8d9e317..63af8ad 100644 --- a/docs/modules/opensearch/pages/usage-guide/upgrade.adoc +++ b/docs/modules/opensearch/pages/usage-guide/upgrade.adoc @@ -27,17 +27,8 @@ ApiError: StatefulSet.apps "opensearch-nodes-default" is invalid: spec: Forbidden: updates to statefulset spec for fields other than 'replicas', ... ---- -The OpenSearch cluster keeps running at first, so this error is easy to overlook. +The OpenSearch cluster keeps running and stays healthy, so this error is easy to overlook. The operator cannot apply any further change to the StatefulSets though, for instance to the number of replicas or to the product version, until the error is resolved. -The Pods also keep the outdated Pod template. -If such a Pod is restarted, then it can fail to start, because the operator already changed the resources which the outdated Pod template refers to. -For instance, a Pod which is restarted after an upgrade from SDP 26.3 stays in the state `ContainerCreating`, because the security settings moved into a dedicated ConfigMap: - -[source] ----- -MountVolume.SetUp failed for volume "security-config-file-config": -configmap references non-existent config key: config.yml ----- To resolve this error, you need to delete the StatefulSets right after the operator upgrade. You can do this using the following command: