From 381a2ee856c36501b5ea81c476aa4569d64ceb73 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Thu, 29 Oct 2026 23:01:33 +0800 Subject: [PATCH 01/19] fix(airgap): repoint default-catalog subscriptions to mirrored catalogs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operators created at runtime reference the default catalog sources (redhat-operators etc.), disabled in airgap. Observed: the MCH operator's multicluster-engine subscription pointed at redhat-operators and never resolved — MCH stuck Installing for hours on a missing MultiClusterEngine CRD. Step 3d repoints any such subscription to the cs-* mirrored catalogs. (cherry picked from commit 37090e3cfc7464fcce0a1a4fe3dfe5cc95190b85) --- scripts/airgap-post-install.sh | 38 ++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/scripts/airgap-post-install.sh b/scripts/airgap-post-install.sh index 6db90b48..065b5ed7 100755 --- a/scripts/airgap-post-install.sh +++ b/scripts/airgap-post-install.sh @@ -346,6 +346,43 @@ PYEOF # The old inline ITMS used registry_base with /mirror/ prefix which was # incorrect for single-registry (mirror-registry) architecture. +# ─── Step 3d: Repoint default-catalog subscriptions to the mirror catalog ─ +# Operators created at runtime (e.g. the MCH operator's multicluster-engine +# subscription) reference the DEFAULT catalog sources (redhat-operators etc.), +# which are disabled in airgap. Without repointing, the operator never +# resolves and upstream reconciles stall (observed: MCH stuck Installing on a +# missing MultiClusterEngine CRD for hours, with only a reconciler error). +repoint_default_catalog_subscriptions() { + step "3d" "Repoint default-catalog subscriptions to mirrored catalogs" + + local map=" +redhat-operators:cs-redhat-operator-index-v4-22 +certified-operators:cs-certified-operator-index-v4-22 +community-operators:cs-community-operator-index-v4-22 +" + while IFS= read -r line; do + [ -z "$line" ] && continue + local def="${line%%:*}" mirror="${line##*:}" + local subs + subs=$(oc get subscriptions.operators.coreos.com -A -o json 2>/dev/null | + python3 -c " +import json, sys +d = json.load(sys.stdin) +for s in d.get('items', []): + if s['spec'].get('source') == '$def': + print(s['metadata']['namespace'] + ' ' + s['metadata']['name']) +" 2>/dev/null) || true + local sub ns name + for sub in $subs; do + ns="${sub%% *}"; name="${sub##* }" + info " repointing $ns/$name: $def -> $mirror" + oc patch subscriptions.operators.coreos.com "$name" -n "$ns" --type merge \ + -p "{\"spec\":{\"source\":\"$mirror\",\"sourceNamespace\":\"openshift-marketplace\"}}" || true + done + done <<< "$map" + info "Subscription catalog repointing complete" +} + # ─── Step 5: Mirror OCI Helm charts and utility images ─────────── mirror_oci_charts() { step 5 "Mirror OCI Helm charts and utility images" @@ -859,6 +896,7 @@ disable_default_catalogs create_catalog_sources apply_ocmirror_resources normalise_mirror_source_policy +repoint_default_catalog_subscriptions mirror_oci_charts fix_manifest_lists add_argocd_ca From 602abe0d6037cdc17f128e13a7e405ac9ed2b891 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 05:31:53 +0000 Subject: [PATCH 02/19] fix(airgap): cover reg.kyverno.io in the manual ITMS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vendored kyverno chart pulls its images from reg.kyverno.io (not ghcr.io, which the imageset mirrors) — without a tag-mirror entry every kyverno pod ImagePullBackOffs in airgap. Add the registry to the manual ITMS (placeholder form, substituted at apply time). Observed live: kyverno pods crash-looped on image pull until the ITMS covered reg.kyverno.io. --- airgap/itms-manual-mirrors.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/airgap/itms-manual-mirrors.yaml b/airgap/itms-manual-mirrors.yaml index b3097324..3dee93c5 100644 --- a/airgap/itms-manual-mirrors.yaml +++ b/airgap/itms-manual-mirrors.yaml @@ -14,3 +14,10 @@ spec: - MIRROR_REGISTRY_HOST:8443/hashicorp # NOTE: quay.io/community-operator-pipeline-prod removed — oc-mirror 4.22 # generates ITMS for it in itms-oc-mirror.yaml automatically. + # Kyverno chart images: the vendored chart defaults to reg.kyverno.io + # (Kyverno's own registry), which is not mirrored by oc-mirror (the + # imageset mirrors the content-identical ghcr.io/kyverno/* images). + # Redirect tag pulls to the mirror's kyverno/* paths. + - source: reg.kyverno.io + mirrors: + - MIRROR_REGISTRY_HOST:8443 From a93cbec4f2cc29e1c2bf2f086c449d2679427ede Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 05:32:00 +0000 Subject: [PATCH 03/19] fix(airgap): pin vault 2.1.2-ubi in the imageset hashicorp-vault chart 0.1.11 deploys vault:2.1.2-ubi by default; the imageset still pinned 1.21.4-ubi, leaving the resolved operand image unmirrored (range-pin aging). Align the pinned operand with what the chart actually deploys. --- airgap/imageset-config-4.22.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airgap/imageset-config-4.22.yaml b/airgap/imageset-config-4.22.yaml index 9bd9794a..84000056 100644 --- a/airgap/imageset-config-4.22.yaml +++ b/airgap/imageset-config-4.22.yaml @@ -113,7 +113,8 @@ mirror: # NFD operand — worker/master/gc DaemonSets use this tag, NOT the operator digest. - name: registry.redhat.io/openshift4/ose-node-feature-discovery-rhel9:v4.22 # Vault — Red Hat certified UBI image from registry.connect.redhat.com - - name: registry.connect.redhat.com/hashicorp/vault:1.21.4-ubi + # (hashicorp-vault chart 0.1.11 deploys vault:2.1.2-ubi by default) + - name: registry.connect.redhat.com/hashicorp/vault:2.1.2-ubi # Kyverno (Helm chart, not OLM — images from ghcr.io) - name: ghcr.io/kyverno/kyverno:v1.17.2 - name: ghcr.io/kyverno/kyvernopre:v1.17.2 From 6fbf1e58280d0d21a0d8c4d1a40f35a55b050d6f Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 05:32:08 +0000 Subject: [PATCH 04/19] docs(airgap): org-prefixed mirror targets need the org in MIRROR_REGISTRY For mirror targets serving content under an org (e.g. quay...:443/mirror), the manual-mirror substitution must use the org-prefixed root or every tag-based operand pull (kyverno, certified intel/hashicorp operands) fails with unauthorized/404. Observed live: the kyverno pods ImagePullBackOff until the ITMS mirrors carried the org prefix. --- airgap/itms-manual-mirrors.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/airgap/itms-manual-mirrors.yaml b/airgap/itms-manual-mirrors.yaml index 3dee93c5..f4792edb 100644 --- a/airgap/itms-manual-mirrors.yaml +++ b/airgap/itms-manual-mirrors.yaml @@ -1,3 +1,9 @@ +# NOTE (org-prefixed registries): when the mirror target serves content +# under an org/namespace prefix (e.g. a Quay 'mirror' org), MIRROR_REGISTRY +# must be set to the ORG-PREFIXED root for this substitution — e.g. +# quay.example:443/mirror — so the entries below resolve to +# quay.example:443/mirror/. Using the bare host:port produces +# unauthorized/404 pulls for every tag-based operand image. apiVersion: config.openshift.io/v1 kind: ImageTagMirrorSet metadata: From 20febe56c44cbe08da9d60fe896e5e9b1146a812 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 05:32:31 +0000 Subject: [PATCH 05/19] fix(airgap): repoint intel operator subscriptions via the airgap overlay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The values shipped source: certified-operators (the default catalog, disabled in airgap). Each root-app sync attempt applied the default source, the subscription failed to resolve (CatalogSourcesUnhealthy), and the sync retried — re-running the vault hook and burning the retry budget. Repoint both intel-device-plugins and intel-tdx-dcap at the mirrored catalog from the airgap overlay, so the connected default (certified-operators) is preserved and the flip is automatic in disconnected mode. --- overrides/values-baremetal-airgap.yaml | 11 +++++++++++ values-baremetal.yaml | 10 ++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/overrides/values-baremetal-airgap.yaml b/overrides/values-baremetal-airgap.yaml index 32c5a34e..722e55b1 100644 --- a/overrides/values-baremetal-airgap.yaml +++ b/overrides/values-baremetal-airgap.yaml @@ -17,6 +17,17 @@ global: # Mirrored catalog for the Argo-rendered OLM subscriptions (oc-mirror output) catalogSource: cs-redhat-operator-index-v4-22 catalogSourceNamespace: openshift-marketplace +# The Intel operator subscriptions stay on the certified catalog in the +# connected default; repoint them at the mirrored certified catalog here. +# If left on certified-operators (disabled in airgap), each root-app sync +# attempt fails to resolve (CatalogSourcesUnhealthy) and retries — burning +# the sync retry budget and stalling the operators. +clusterGroup: + subscriptions: + intel-device-plugins: + source: cs-certified-operator-index-v4-22 + intel-tdx-dcap: + source: cs-certified-operator-index-v4-22 # Trustee: flip platform collateral to the offline file backend. # Collect with `make collect-dcap-collateral` before deploy. kbs: diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 271361b6..565cbb49 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -82,8 +82,9 @@ clusterGroup: namespace: openshift-operators # These Intel operators live in the certified-operators catalog, not redhat-operators source: certified-operators - # Required for airgap -- point at your mirrored catalog source instead - # source: cs-certified-operator-index-v4-22 + # Airgap: the default catalogs are disabled — the airgap overlay + # (overrides/values-baremetal-airgap.yaml) repoints these at the + # mirrored catalog source (cs-certified-operator-index-v4-22). channel: stable annotations: argocd.argoproj.io/sync-wave: "10" @@ -92,8 +93,9 @@ clusterGroup: namespace: intel-dcap-operator-system # These Intel operators live in the certified-operators catalog, not redhat-operators source: certified-operators - # Required for airgap -- point at your mirrored catalog source instead - # source: cs-certified-operator-index-v4-22 + # Airgap: repointed by overrides/values-baremetal-airgap.yaml — + # each root-app sync against the disabled default catalog burns the + # retry budget (CatalogSourcesUnhealthy) and stalls the operator. channel: alpha annotations: argocd.argoproj.io/sync-wave: "10" From 59ed821f3f7e895c2fa4f108cc759b1a7d7cf7fb Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 05:32:45 +0000 Subject: [PATCH 06/19] fix(airgap): trustee AS collateral backend follows the airgap flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The connected default (remote Intel PCS URL) is unreachable in airgap — every kata-cc pod's TEE-evidence verification failed with 401 (RcarAttestFailed). The previous mechanism was broken: the values-baremetal-airgap.yaml overlay cannot flip inline application parameters (its kbs.tdx block was inert dead config — removed here). The clustergroup chart tpl-renders override values, so select the file:// backend with an inline {{ if $.Values.global.airgap }} conditional on the collateralService parameter: connected renders the Intel PCS URL (unchanged behavior), airgap renders the offline file backend collected via make collect-dcap-collateral. --- overrides/values-baremetal-airgap.yaml | 5 ----- values-baremetal.yaml | 9 ++++++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/overrides/values-baremetal-airgap.yaml b/overrides/values-baremetal-airgap.yaml index 722e55b1..471509b5 100644 --- a/overrides/values-baremetal-airgap.yaml +++ b/overrides/values-baremetal-airgap.yaml @@ -28,8 +28,3 @@ clusterGroup: source: cs-certified-operator-index-v4-22 intel-tdx-dcap: source: cs-certified-operator-index-v4-22 -# Trustee: flip platform collateral to the offline file backend. -# Collect with `make collect-dcap-collateral` before deploy. -kbs: - tdx: - collateralService: "file:///opt/confidential-containers/attestation-service/tdx/platform_collaterals.json" diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 565cbb49..45a50dbb 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -224,10 +224,13 @@ clusterGroup: # TDX DCAP collateral — connected (default): the trustee AS fetches # platform collateral (TCB info, QE identity, CRLs) directly from # Intel PCS; these endpoints are public and need no API key. - # Airgap: overrides/values-baremetal-airgap.yaml flips this to the - # file:// backend (collateral collected via make collect-dcap-collateral). + # Airgap: the offline file backend (collateral collected via + # make collect-dcap-collateral, mounted from the tdx-collateral + # Secret) — the remote URL is unreachable and every TEE-evidence + # verification fails with 401. The clustergroup chart tpl-renders + # override values, so the global.airgap flag selects the backend. - name: kbs.tdx.collateralService - value: "https://api.trustedservices.intel.com/sgx/certification/v4/" + value: "{{ if $.Values.global.airgap }}file:///opt/confidential-containers/attestation-service/tdx/platform_collaterals.json{{ else }}https://api.trustedservices.intel.com/sgx/certification/v4/{{ end }}" - name: kbs.tdx.collateralMountPath value: "/opt/confidential-containers/attestation-service/tdx" - name: kbs.tdx.collateralFileName From 4c4a229c0629b11c639fe420491729b8666160ac Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 05:33:09 +0000 Subject: [PATCH 07/19] fix(airgap): activate the CoCo workload image overrides via the overlay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CoCo guest VMs pull images directly from the pod-spec reference — no IDMS/ITMS redirect inside the CVM — so in airgap the workloads must reference the mirror registry directly (reachable, trusted via the initdata CA, authenticated via the KBS regcred / cluster pull secret). Previously the overrides were commented examples requiring three manual uncomment-and-edit operations mid-deploy. Move them into the airgap overlay: they are absent in connected mode (chart defaults, behavior unchanged) and activate automatically in disconnected mode, with the mirror root taken from global.mirrorRegistry (placeholder convention; set at deploy time, org-prefixed for Quay-style targets). The clustergroup chart tpl-renders the override values. --- overrides/values-baremetal-airgap.yaml | 25 +++++++++++++++++++++++++ values-baremetal.yaml | 21 ++++++--------------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/overrides/values-baremetal-airgap.yaml b/overrides/values-baremetal-airgap.yaml index 471509b5..d55b4ada 100644 --- a/overrides/values-baremetal-airgap.yaml +++ b/overrides/values-baremetal-airgap.yaml @@ -17,6 +17,11 @@ global: # Mirrored catalog for the Argo-rendered OLM subscriptions (oc-mirror output) catalogSource: cs-redhat-operator-index-v4-22 catalogSourceNamespace: openshift-marketplace + # Mirror root for the CoCo workload image overrides (deploy-time: set to + # your mirror registry, org-prefixed where applicable — e.g. + # quay.example:443/mirror or mirror.example:8443). The workload overrides + # below are tpl-rendered against this value at chart render time. + mirrorRegistry: "MIRROR_REGISTRY_HOST:8443" # The Intel operator subscriptions stay on the certified catalog in the # connected default; repoint them at the mirrored certified catalog here. # If left on certified-operators (disabled in airgap), each root-app sync @@ -28,3 +33,23 @@ clusterGroup: source: cs-certified-operator-index-v4-22 intel-tdx-dcap: source: cs-certified-operator-index-v4-22 + # CoCo guest VMs pull images directly from the pod-spec reference — no + # IDMS/ITMS inside the CVM — so the workloads must reference the mirror + # registry directly (reachable, trusted via the initdata CA, authenticated + # via the KBS regcred / cluster pull secret). These overrides are absent + # in connected mode (chart defaults) and activate only in airgap. + applications: + hello-openshift: + overrides: + - name: image + value: "{{ $.Values.global.mirrorRegistry }}/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2" + kbs-access-curl: + overrides: + - name: image + value: "{{ $.Values.global.mirrorRegistry }}/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2" + - name: initImage + value: "{{ $.Values.global.mirrorRegistry }}/ubi9/ubi:latest" + kbs-access-sealed: + overrides: + - name: image + value: "{{ $.Values.global.mirrorRegistry }}/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2" diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 45a50dbb..2c379a22 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -348,11 +348,10 @@ clusterGroup: path: charts/coco-supported/hello-openshift annotations: argocd.argoproj.io/sync-wave: "30" - # Required for airgap — CoCo guest VMs pull images directly, no IDMS/ITMS in the CVM. - # Uncomment and set MIRROR_REGISTRY_HOST to your mirror registry. - # overrides: - # - name: image - # value: "MIRROR_REGISTRY_HOST:8443/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2" + # Airgap — CoCo guest VMs pull images directly from the pod-spec + # reference (no IDMS/ITMS inside the CVM): the airgap overlay + # (overrides/values-baremetal-airgap.yaml) activates the image + # overrides against global.mirrorRegistry. syncPolicy: automated: prune: true @@ -364,12 +363,7 @@ clusterGroup: path: charts/coco-supported/kbs-access-curl annotations: argocd.argoproj.io/sync-wave: "30" - # Required for airgap — CoCo guest VMs pull images directly, no IDMS/ITMS in the CVM. - # overrides: - # - name: image - # value: "MIRROR_REGISTRY_HOST:8443/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2" - # - name: initImage - # value: "MIRROR_REGISTRY_HOST:8443/ubi9/ubi:latest" + # Airgap — image overrides activated by the airgap overlay. syncPolicy: automated: prune: true @@ -381,10 +375,7 @@ clusterGroup: path: charts/coco-supported/kbs-access-sealed annotations: argocd.argoproj.io/sync-wave: "30" - # Required for airgap — CoCo guest VMs pull images directly, no IDMS/ITMS in the CVM. - # overrides: - # - name: image - # value: "MIRROR_REGISTRY_HOST:8443/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2" + # Airgap — image overrides activated by the airgap overlay. syncPolicy: automated: prune: true From bec203a0741decdbe06749b78fa3828ead33adaa Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 05:33:25 +0000 Subject: [PATCH 08/19] docs(airgap): helmRepoUrl is required for multiSourceConfig in airgap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With multiSourceConfig enabled, ArgoCD pulls the VP Helm charts from helmRepoUrl directly and the repo-server performs no IDMS redirect — in airgap this must point at the mirror or every chart-type application fails at the chart-fetch stage. Also clarify the global.airgap comment: the flag is set in values-global.yaml on the disconnected branch and loads the airgap overlay (the overlay does not set the flag itself). --- values-global.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/values-global.yaml b/values-global.yaml index 31e69af1..d4b5bd49 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -9,8 +9,11 @@ global: # Airgap / disconnected master switch. false (default, connected): # Intel DCAP zero-touch platform registration (vault/ESO PCS API key), # trustee fetches platform collateral directly from Intel PCS. - # true (disconnected): set by overrides/values-baremetal-airgap.yaml — - # manual PCK provisioning (scripts/dcap-pck.py) and file-backed collateral. + # true (disconnected): set here on the disconnected branch — loads + # overrides/values-baremetal-airgap.yaml (via the sharedValueFiles + # selector in values-baremetal.yaml), which repoints subscriptions and + # workload images at the mirror; requires manual PCK provisioning + # (scripts/dcap-pck.py) and file-backed collateral. airgap: false secretStore: # Warning: This must be present even if it is set to none. @@ -70,6 +73,9 @@ main: clusterGroupChartVersion: 0.9.58 # VP OCI Helm charts are mirrored to mirror-registry by oc-mirror additionalImages. # Use mirror-registry directly — no Quay dependency for airgap deploys. + # REQUIRED in airgap: with multiSourceConfig, ArgoCD pulls the VP Helm + # charts from this URL directly and the repo-server performs no IDMS + # redirect — point it at the mirror (e.g. :8443/validatedpatterns). helmRepoUrl: quay.io/validatedpatterns # Common secret store configuration used across multiple charts. From 5cb65e32d5498b7368ee19cfb470b7eb6e492f1b Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 10:49:42 +0000 Subject: [PATCH 09/19] fix(airgap): set the MCE subscription source via the ACM chart override MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MCH operator creates the multicluster-engine subscription at runtime from the mce-subscription-spec annotation on the MultiClusterHub, which the ACM chart templates from acm.mce_operator.source (default redhat-operators — disabled in airgap). Without the override the subscription is born on the dead default catalog and MCH stalls Installing until a manual repoint — observed in three consecutive disconnected runs (DEP3, DEP4, the PR #165 validation). Set the source from the airgap overlay so the subscription is born on the mirrored catalog; connected mode keeps the chart default. Chart- level mechanism (the annotation contract), not a post-MCH patch step. --- overrides/values-baremetal-airgap.yaml | 10 ++++++++++ values-baremetal.yaml | 8 ++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/overrides/values-baremetal-airgap.yaml b/overrides/values-baremetal-airgap.yaml index d55b4ada..c5e1a0bb 100644 --- a/overrides/values-baremetal-airgap.yaml +++ b/overrides/values-baremetal-airgap.yaml @@ -39,6 +39,16 @@ clusterGroup: # via the KBS regcred / cluster pull secret). These overrides are absent # in connected mode (chart defaults) and activate only in airgap. applications: + acm: + # The MCH operator creates the multicluster-engine subscription at + # runtime from the mce-subscription-spec annotation on the + # MultiClusterHub — the ACM chart templates that annotation from + # acm.mce_operator.source. Without this override the subscription is + # born on redhat-operators (disabled in airgap) and MCH stalls + # Installing until a manual repoint. + overrides: + - name: acm.mce_operator.source + value: cs-redhat-operator-index-v4-22 hello-openshift: overrides: - name: image diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 2c379a22..04a82ae8 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -170,10 +170,10 @@ clusterGroup: project: hub chart: acm chartVersion: 0.1.* - # Required for airgap -- set to your mirrored catalog source name - # overrides: - # - name: acm.mce_operator.source - # value: "cs-redhat-operator-index-v4-22" + # Airgap: the MCE subscription source is flipped by the airgap overlay + # (overrides/values-baremetal-airgap.yaml) via acm.mce_operator.source — + # the MCH operator honors the mce-subscription-spec annotation when it + # creates the multicluster-engine subscription at runtime. vault: name: vault From f4aacd5ef2c5266ff8480fc3318ae9f8e276d093 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 10:50:05 +0000 Subject: [PATCH 10/19] fix(airgap): enable application health check for sync-wave gating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The patterns-operator restores ArgoCD's Application health check (dropped in ArgoCD 1.8, argoproj/argo-cd#3781) when gitops.applicationHealthCheckEnabled is set on patterns-operator-config — it defaults to false. Without it, child Applications have unknown health and the root app applies all sync waves ungated: every child app is created within seconds and later-wave apps (workloads, wave 30) race earlier-wave dependencies (trustee/KBS, wave 20) instead of waiting for them to be healthy. Observed live: all 15 child apps created in ~4s during the PR #165 validation run, workloads wedged on the KBS race. Set the flag in step 10, before the Pattern CR deploys, so vp-gitops is created with the health check and the wave layout executes as designed. --- scripts/airgap-post-install.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/airgap-post-install.sh b/scripts/airgap-post-install.sh index 065b5ed7..da96fda9 100755 --- a/scripts/airgap-post-install.sh +++ b/scripts/airgap-post-install.sh @@ -769,8 +769,15 @@ data: gitops.channel: "${gitops_channel}" gitops.catalogSource: "${gitops_source}" gitops.sourceNamespace: "openshift-marketplace" + # App-of-apps wave gating: ArgoCD dropped the Application health check in + # 1.8 (argoproj/argo-cd#3781); the patterns-operator can restore it via + # this flag. Without it, child Applications have unknown health and the + # root app applies ALL sync waves ungated — every child app is created at + # once and later-wave apps (workloads, wave 30) race earlier-wave + # dependencies (trustee/KBS, wave 20) instead of waiting for them. + gitops.applicationHealthCheckEnabled: "true" EOF - info "patterns-operator-config ConfigMap created" + info "patterns-operator-config ConfigMap created (application health check enabled — sync-wave gating active)" } # ─── Step 11: Deploy Pattern CR directly ───────────────────────── From 33650b826b31e6da3d711ceb657d824a09c9a25d Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 12:08:56 +0000 Subject: [PATCH 11/19] =?UTF-8?q?fix(waves):=20acm=20app=20moves=20to=20wa?= =?UTF-8?q?ve=2015=20=E2=80=94=20breaks=20the=20CRD=20discovery=20deadlock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The acm application had no sync-wave annotation (wave 0), but the ACM chart renders the MultiClusterHub whose CRD is registered by the ACM operator — whose Subscription is rendered by the root app at wave 10. Under sync-wave gating this deadlocks: wave 0 holds the acm app, whose sync fails ArgoCD discovery on the not-yet-served operator.open-cluster-management.io/v1 group; wave 10, which would install the operator and register the CRD, is gated behind acm going healthy. Ungated deployments never noticed — the everything-at-once apply was load-bearing for ACM. Wave 15 places acm after the wave-10 operator subscriptions (whose health means the CSV is installed and the CRD is served) and before the wave-20 trustee/workloads. Observed live in the no-bounce experiment: acm stuck at discovery-retry 7/20 with all later waves held. --- values-baremetal.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 04a82ae8..e28e262a 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -170,6 +170,15 @@ clusterGroup: project: hub chart: acm chartVersion: 0.1.* + # Wave 15 — AFTER the wave-10 OLM subscriptions: the ACM chart renders + # the MultiClusterHub, whose CRD is registered by the ACM operator + # (Subscription rendered by the root app at wave 10). At wave 0 the + # app deadlocks under sync-wave gating: ArgoCD discovery fails on the + # not-yet-served operator.open-cluster-management.io/v1 group, the app + # never goes healthy, and wave 10 — which would install the operator — + # is gated behind it (observed live, 2026-11-02 no-bounce experiment). + annotations: + argocd.argoproj.io/sync-wave: "15" # Airgap: the MCE subscription source is flipped by the airgap overlay # (overrides/values-baremetal-airgap.yaml) via acm.mce_operator.source — # the MCH operator honors the mce-subscription-spec annotation when it From c82fc7f5df5bad50d663e13de0a5c0c4f3929b48 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 2 Nov 2026 15:14:34 +0000 Subject: [PATCH 12/19] style: trim inline comments to brief pointers The full rationale lives in the commit messages and the PR body; the values and script comments shrink to one-to-three-line pointers in the repo's existing style. --- airgap/itms-manual-mirrors.yaml | 13 ++------- overrides/values-baremetal-airgap.yaml | 24 +++------------- scripts/airgap-post-install.sh | 17 ++++-------- values-baremetal.yaml | 38 ++++++-------------------- values-global.yaml | 10 ++----- 5 files changed, 24 insertions(+), 78 deletions(-) diff --git a/airgap/itms-manual-mirrors.yaml b/airgap/itms-manual-mirrors.yaml index f4792edb..074cd6d2 100644 --- a/airgap/itms-manual-mirrors.yaml +++ b/airgap/itms-manual-mirrors.yaml @@ -1,9 +1,5 @@ -# NOTE (org-prefixed registries): when the mirror target serves content -# under an org/namespace prefix (e.g. a Quay 'mirror' org), MIRROR_REGISTRY -# must be set to the ORG-PREFIXED root for this substitution — e.g. -# quay.example:443/mirror — so the entries below resolve to -# quay.example:443/mirror/. Using the bare host:port produces -# unauthorized/404 pulls for every tag-based operand image. +# Org-prefixed mirror targets: MIRROR_REGISTRY must include the org path +# (e.g. quay.example:443/mirror) or every tag-based operand pull 404s. apiVersion: config.openshift.io/v1 kind: ImageTagMirrorSet metadata: @@ -20,10 +16,7 @@ spec: - MIRROR_REGISTRY_HOST:8443/hashicorp # NOTE: quay.io/community-operator-pipeline-prod removed — oc-mirror 4.22 # generates ITMS for it in itms-oc-mirror.yaml automatically. - # Kyverno chart images: the vendored chart defaults to reg.kyverno.io - # (Kyverno's own registry), which is not mirrored by oc-mirror (the - # imageset mirrors the content-identical ghcr.io/kyverno/* images). - # Redirect tag pulls to the mirror's kyverno/* paths. + # Kyverno chart images: the chart pulls from reg.kyverno.io, not ghcr.io. - source: reg.kyverno.io mirrors: - MIRROR_REGISTRY_HOST:8443 diff --git a/overrides/values-baremetal-airgap.yaml b/overrides/values-baremetal-airgap.yaml index c5e1a0bb..b0f3d9f1 100644 --- a/overrides/values-baremetal-airgap.yaml +++ b/overrides/values-baremetal-airgap.yaml @@ -17,35 +17,19 @@ global: # Mirrored catalog for the Argo-rendered OLM subscriptions (oc-mirror output) catalogSource: cs-redhat-operator-index-v4-22 catalogSourceNamespace: openshift-marketplace - # Mirror root for the CoCo workload image overrides (deploy-time: set to - # your mirror registry, org-prefixed where applicable — e.g. - # quay.example:443/mirror or mirror.example:8443). The workload overrides - # below are tpl-rendered against this value at chart render time. + # Mirror root for the workload image overrides (org-prefixed for Quay). mirrorRegistry: "MIRROR_REGISTRY_HOST:8443" -# The Intel operator subscriptions stay on the certified catalog in the -# connected default; repoint them at the mirrored certified catalog here. -# If left on certified-operators (disabled in airgap), each root-app sync -# attempt fails to resolve (CatalogSourcesUnhealthy) and retries — burning -# the sync retry budget and stalling the operators. +# Connected default: the certified-operators catalog, disabled in airgap. clusterGroup: subscriptions: intel-device-plugins: source: cs-certified-operator-index-v4-22 intel-tdx-dcap: source: cs-certified-operator-index-v4-22 - # CoCo guest VMs pull images directly from the pod-spec reference — no - # IDMS/ITMS inside the CVM — so the workloads must reference the mirror - # registry directly (reachable, trusted via the initdata CA, authenticated - # via the KBS regcred / cluster pull secret). These overrides are absent - # in connected mode (chart defaults) and activate only in airgap. + # CoCo guest VMs pull images directly — no IDMS/ITMS inside the CVM. applications: acm: - # The MCH operator creates the multicluster-engine subscription at - # runtime from the mce-subscription-spec annotation on the - # MultiClusterHub — the ACM chart templates that annotation from - # acm.mce_operator.source. Without this override the subscription is - # born on redhat-operators (disabled in airgap) and MCH stalls - # Installing until a manual repoint. + # Born on redhat-operators by default — disabled in airgap. overrides: - name: acm.mce_operator.source value: cs-redhat-operator-index-v4-22 diff --git a/scripts/airgap-post-install.sh b/scripts/airgap-post-install.sh index da96fda9..5f8ea22c 100755 --- a/scripts/airgap-post-install.sh +++ b/scripts/airgap-post-install.sh @@ -347,11 +347,8 @@ PYEOF # incorrect for single-registry (mirror-registry) architecture. # ─── Step 3d: Repoint default-catalog subscriptions to the mirror catalog ─ -# Operators created at runtime (e.g. the MCH operator's multicluster-engine -# subscription) reference the DEFAULT catalog sources (redhat-operators etc.), -# which are disabled in airgap. Without repointing, the operator never -# resolves and upstream reconciles stall (observed: MCH stuck Installing on a -# missing MultiClusterEngine CRD for hours, with only a reconciler error). +# Operators created at runtime reference the DEFAULT catalog sources, which +# are disabled in airgap — without repointing they never resolve. repoint_default_catalog_subscriptions() { step "3d" "Repoint default-catalog subscriptions to mirrored catalogs" @@ -769,15 +766,11 @@ data: gitops.channel: "${gitops_channel}" gitops.catalogSource: "${gitops_source}" gitops.sourceNamespace: "openshift-marketplace" - # App-of-apps wave gating: ArgoCD dropped the Application health check in - # 1.8 (argoproj/argo-cd#3781); the patterns-operator can restore it via - # this flag. Without it, child Applications have unknown health and the - # root app applies ALL sync waves ungated — every child app is created at - # once and later-wave apps (workloads, wave 30) race earlier-wave - # dependencies (trustee/KBS, wave 20) instead of waiting for them. + # App-of-apps sync-wave gating (Application health check) — see + # argoproj/argo-cd#3781; without it the root app applies all waves ungated. gitops.applicationHealthCheckEnabled: "true" EOF - info "patterns-operator-config ConfigMap created (application health check enabled — sync-wave gating active)" + info "patterns-operator-config ConfigMap created" } # ─── Step 11: Deploy Pattern CR directly ───────────────────────── diff --git a/values-baremetal.yaml b/values-baremetal.yaml index e28e262a..fe869f7e 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -82,9 +82,7 @@ clusterGroup: namespace: openshift-operators # These Intel operators live in the certified-operators catalog, not redhat-operators source: certified-operators - # Airgap: the default catalogs are disabled — the airgap overlay - # (overrides/values-baremetal-airgap.yaml) repoints these at the - # mirrored catalog source (cs-certified-operator-index-v4-22). + # Airgap: repointed by overrides/values-baremetal-airgap.yaml channel: stable annotations: argocd.argoproj.io/sync-wave: "10" @@ -93,9 +91,7 @@ clusterGroup: namespace: intel-dcap-operator-system # These Intel operators live in the certified-operators catalog, not redhat-operators source: certified-operators - # Airgap: repointed by overrides/values-baremetal-airgap.yaml — - # each root-app sync against the disabled default catalog burns the - # retry budget (CatalogSourcesUnhealthy) and stalls the operator. + # Airgap: repointed by overrides/values-baremetal-airgap.yaml channel: alpha annotations: argocd.argoproj.io/sync-wave: "10" @@ -170,19 +166,11 @@ clusterGroup: project: hub chart: acm chartVersion: 0.1.* - # Wave 15 — AFTER the wave-10 OLM subscriptions: the ACM chart renders - # the MultiClusterHub, whose CRD is registered by the ACM operator - # (Subscription rendered by the root app at wave 10). At wave 0 the - # app deadlocks under sync-wave gating: ArgoCD discovery fails on the - # not-yet-served operator.open-cluster-management.io/v1 group, the app - # never goes healthy, and wave 10 — which would install the operator — - # is gated behind it (observed live, 2026-11-02 no-bounce experiment). + # Wave 15: after the wave-10 ACM operator subscription (MCH CRD), + # before trustee at 20. annotations: argocd.argoproj.io/sync-wave: "15" - # Airgap: the MCE subscription source is flipped by the airgap overlay - # (overrides/values-baremetal-airgap.yaml) via acm.mce_operator.source — - # the MCH operator honors the mce-subscription-spec annotation when it - # creates the multicluster-engine subscription at runtime. + # Airgap: MCE source set by the airgap overlay (acm.mce_operator.source). vault: name: vault @@ -230,14 +218,9 @@ clusterGroup: # Inject mirror-registry CA into KBS trust store (required for airgap image pulls) - name: kbs.registryCa.enabled value: "true" - # TDX DCAP collateral — connected (default): the trustee AS fetches - # platform collateral (TCB info, QE identity, CRLs) directly from - # Intel PCS; these endpoints are public and need no API key. - # Airgap: the offline file backend (collateral collected via - # make collect-dcap-collateral, mounted from the tdx-collateral - # Secret) — the remote URL is unreachable and every TEE-evidence - # verification fails with 401. The clustergroup chart tpl-renders - # override values, so the global.airgap flag selects the backend. + # TDX DCAP collateral — connected: Intel PCS (public, no API key). + # Airgap: the offline file backend (make collect-dcap-collateral). + # The chart tpl-renders override values — global.airgap selects. - name: kbs.tdx.collateralService value: "{{ if $.Values.global.airgap }}file:///opt/confidential-containers/attestation-service/tdx/platform_collaterals.json{{ else }}https://api.trustedservices.intel.com/sgx/certification/v4/{{ end }}" - name: kbs.tdx.collateralMountPath @@ -357,10 +340,7 @@ clusterGroup: path: charts/coco-supported/hello-openshift annotations: argocd.argoproj.io/sync-wave: "30" - # Airgap — CoCo guest VMs pull images directly from the pod-spec - # reference (no IDMS/ITMS inside the CVM): the airgap overlay - # (overrides/values-baremetal-airgap.yaml) activates the image - # overrides against global.mirrorRegistry. + # Airgap: image overrides activated by the airgap overlay. syncPolicy: automated: prune: true diff --git a/values-global.yaml b/values-global.yaml index d4b5bd49..a3412471 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -9,10 +9,8 @@ global: # Airgap / disconnected master switch. false (default, connected): # Intel DCAP zero-touch platform registration (vault/ESO PCS API key), # trustee fetches platform collateral directly from Intel PCS. - # true (disconnected): set here on the disconnected branch — loads - # overrides/values-baremetal-airgap.yaml (via the sharedValueFiles - # selector in values-baremetal.yaml), which repoints subscriptions and - # workload images at the mirror; requires manual PCK provisioning + # true (disconnected): loads the airgap overlay (repoints subscriptions + # and workload images at the mirror); manual PCK provisioning # (scripts/dcap-pck.py) and file-backed collateral. airgap: false secretStore: @@ -73,9 +71,7 @@ main: clusterGroupChartVersion: 0.9.58 # VP OCI Helm charts are mirrored to mirror-registry by oc-mirror additionalImages. # Use mirror-registry directly — no Quay dependency for airgap deploys. - # REQUIRED in airgap: with multiSourceConfig, ArgoCD pulls the VP Helm - # charts from this URL directly and the repo-server performs no IDMS - # redirect — point it at the mirror (e.g. :8443/validatedpatterns). + # REQUIRED in airgap: no IDMS redirect in the ArgoCD repo-server. helmRepoUrl: quay.io/validatedpatterns # Common secret store configuration used across multiple charts. From 48831defd40b0ca1032c43f6e4d4e74d70af9a29 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 3 Nov 2026 01:51:13 +0000 Subject: [PATCH 13/19] fix(gating): set the ArgoCD application health check on every install path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ArgoCD 1.8 dropped the Application health check (argoproj/argo-cd#3781); the patterns-operator can restore it but defaults it off, so the root app applied all sync waves at once and wave ordering (workloads after trustee) was never enforced. Set main.gitops.applicationHealthCheckEnabled declaratively in values-global.yaml — rendered into the patterns-operator-config ConfigMap by pattern-install >= 0.0.19 (validatedpatterns/pattern-install-chart#52, fixed by #53). The disconnected bootstrap (airgap-post-install.sh step 10) keeps writing the key directly because the operator subscription predates any chart render there, and now asserts the key landed instead of hoping. --- scripts/airgap-post-install.sh | 16 ++++++++++++++++ values-global.yaml | 9 +++++++++ 2 files changed, 25 insertions(+) diff --git a/scripts/airgap-post-install.sh b/scripts/airgap-post-install.sh index 5f8ea22c..f6d2b931 100755 --- a/scripts/airgap-post-install.sh +++ b/scripts/airgap-post-install.sh @@ -768,9 +768,25 @@ data: gitops.sourceNamespace: "openshift-marketplace" # App-of-apps sync-wave gating (Application health check) — see # argoproj/argo-cd#3781; without it the root app applies all waves ungated. + # The connected path sets this declaratively via values-global.yaml + # (main.gitops.applicationHealthCheckEnabled) once pattern-install renders + # it (validatedpatterns/pattern-install-chart#52); this bootstrap writes it + # directly because the operator subscription predates any chart render. gitops.applicationHealthCheckEnabled: "true" EOF info "patterns-operator-config ConfigMap created" + + # Post-apply assertion: the operator only restores the Application health + # check when the key is present at its first reconcile — fail loudly here + # rather than let the root app apply all waves ungated. + local key + key=$(oc get configmap patterns-operator-config -n openshift-operators \ + -o jsonpath='{.data.gitops\.applicationHealthCheckEnabled}' 2>/dev/null || true) + if [[ "$key" != "true" ]]; then + error "gitops.applicationHealthCheckEnabled is not \"true\" on patterns-operator-config" + exit 1 + fi + info "Application health check key verified on patterns-operator-config" } # ─── Step 11: Deploy Pattern CR directly ───────────────────────── diff --git a/values-global.yaml b/values-global.yaml index a3412471..d5e8c0af 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -66,6 +66,15 @@ main: # operatorSource: cs-redhat-operator-index-v4-22 # Keep this channel aligned with airgap/imageset-config-4.22.yaml. channel: gitops-1.22 + # App-of-apps sync-wave gating. ArgoCD 1.8 dropped the Application + # health check (argoproj/argo-cd#3781); the patterns-operator can restore + # it but defaults it OFF. Without it the root Application treats child + # Applications as healthy and applies ALL waves at once, so wave ordering + # (e.g. workloads after trustee) is not enforced. Rendered into the + # patterns-operator-config ConfigMap by pattern-install >= 0.0.19 + # (validatedpatterns/pattern-install-chart#53); the disconnected bootstrap + # writes it directly (scripts/airgap-post-install.sh step 10). + applicationHealthCheckEnabled: "true" multiSourceConfig: enabled: true clusterGroupChartVersion: 0.9.58 From 4f2305eced04d6f261f30246919d3f814be24e21 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 3 Nov 2026 01:51:21 +0000 Subject: [PATCH 14/19] fix(airgap): correct, derive and make re-runnable the subscription repoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repoint loop split each 'namespace name' line on whitespace, so every oc patch targeted the wrong object and || true hid the failures while the step still logged success. Read line by line instead, drop the error masking, and fail loudly if anything is still on a disabled default catalog at the end. Scope it to what it can actually fix: subscriptions created at runtime by operators (e.g. MCH's multicluster-engine). ArgoCD-owned subscriptions are skipped and reported — they are declarative via the airgap overlay, and ArgoCD self-heal would revert an out-of-band patch on them anyway. Derive the mirrored catalog suffix from global.clusterVersion (4.22 → v4-22) instead of hardcoding -v4-22, verify the target CatalogSource exists before patching, and do the same derivation for the step-10 gitops source fallback (previously a stale -v4-21 default). New step 3e fails fast before the pattern deploys when the airgap overlay references a catalog that is not READY on the cluster, or when global.mirrorRegistry is missing/placeholder or does not match MIRROR_REGISTRY. The step is exposed standalone as 'make airgap-repoint-subs' (safe to re-run, e.g. after MCH deploys). --- Makefile | 4 + scripts/airgap-post-install.sh | 216 ++++++++++++++++++++++++++++++--- 2 files changed, 201 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index cba4a090..e31c81f7 100644 --- a/Makefile +++ b/Makefile @@ -205,6 +205,10 @@ airgap-mirror: ## Mirror content to disconnected registry (requires MIRROR_REGIS airgap-post-install: ## Post-install bootstrap (after airgap-mirror + labctl apply-mirror-resources) @scripts/airgap-post-install.sh +.PHONY: airgap-repoint-subs +airgap-repoint-subs: ## Re-run the default-catalog subscription repoint (safety net; safe any time, e.g. after MCH deploys) + @scripts/airgap-post-install.sh --repoint-subs + diff --git a/scripts/airgap-post-install.sh b/scripts/airgap-post-install.sh index f6d2b931..3b7152e8 100755 --- a/scripts/airgap-post-install.sh +++ b/scripts/airgap-post-install.sh @@ -6,7 +6,11 @@ # # Required env: # KUBECONFIG — path to cluster kubeconfig -# MIRROR_REGISTRY — registry host:port (e.g. MIRROR_REGISTRY_HOST:8443) +# MIRROR_REGISTRY — mirror root INCLUDING the org path for Quay targets +# (e.g. quay.example:443/mirror). Direct references +# (workload image rewrites, ITMS mirrors, chart pulls) +# all carry the org path; a bare host:port gets 401/404 +# on Quay. # # Optional env: # EXTRA_CA_CERT — path to CA cert file for the mirror registry @@ -46,6 +50,7 @@ case "${1:-}" in --sync-repos-only) MODE="sync"; shift ;; --deploy-pattern) MODE="deploy"; shift ;; --fix-manifest-lists) MODE="fix-manifests"; shift ;; + --repoint-subs) MODE="repoint"; shift ;; esac # ─── Step 1: Validate prerequisites ───────────────────────────── @@ -71,7 +76,8 @@ validate_prereqs() { fi if [[ -z "$MIRROR_REGISTRY" ]]; then - error "MIRROR_REGISTRY is not set (e.g. MIRROR_REGISTRY_HOST:8443)" + error "MIRROR_REGISTRY is not set (mirror root INCLUDING the org path," + error "e.g. quay.example:443/mirror — a bare host:port gets 401/404 on Quay)" exit 1 fi @@ -347,19 +353,70 @@ PYEOF # incorrect for single-registry (mirror-registry) architecture. # ─── Step 3d: Repoint default-catalog subscriptions to the mirror catalog ─ -# Operators created at runtime reference the DEFAULT catalog sources, which -# are disabled in airgap — without repointing they never resolve. +# Safety net for subscriptions created at runtime by operators (e.g. MCH +# creating the multicluster-engine subscription): they reference the DEFAULT +# catalog sources, which are disabled in airgap, and never resolve. +# +# The pattern's own subscriptions are declarative — sourceless ones follow +# global.catalogSource, the intel ones and the MCE source are set by +# overrides/values-baremetal-airgap.yaml and the acm override in +# values-baremetal.yaml — so ArgoCD-owned subscriptions are skipped here: +# ArgoCD self-heal would revert an out-of-band patch on the next sync. +# +# Idempotent and safe to re-run at any time (e.g. after MCH deploys): +# make airgap-repoint-subs repoint_default_catalog_subscriptions() { step "3d" "Repoint default-catalog subscriptions to mirrored catalogs" + # Derive the oc-mirror catalog suffix from global.clusterVersion + # (4.22 → v4-22), matching the operator index tags in the imageset. + local cluster_version catalog_suffix + cluster_version=$(python3 -c " +import yaml +with open('${PATTERN_DIR}/values-global.yaml') as f: + d = yaml.safe_load(f) +print(d.get('global', {}).get('clusterVersion', '')) +" 2>/dev/null || true) + if [[ -n "${cluster_version}" ]]; then + catalog_suffix="v${cluster_version//./-}" + info "Catalog suffix -${catalog_suffix} (from global.clusterVersion ${cluster_version})" + else + # Fallback: derive from the CatalogSources oc-mirror actually created. + catalog_suffix=$(oc get catalogsources -n openshift-marketplace \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null \ + | grep -oE 'index-v[0-9]+-[0-9]+' | head -1 | sed 's/.*index-//' || true) + if [[ -z "${catalog_suffix}" ]]; then + error "Cannot derive the catalog suffix: global.clusterVersion is empty" + error "and no cs-*-index-v* CatalogSources exist in openshift-marketplace." + exit 1 + fi + warn "Catalog suffix -${catalog_suffix} derived from live CatalogSources (clusterVersion empty)" + fi + local map=" -redhat-operators:cs-redhat-operator-index-v4-22 -certified-operators:cs-certified-operator-index-v4-22 -community-operators:cs-community-operator-index-v4-22 +redhat-operators:cs-redhat-operator-index-${catalog_suffix} +certified-operators:cs-certified-operator-index-${catalog_suffix} +community-operators:cs-community-operator-index-${catalog_suffix} " + local repointed=0 skipped=0 missing_catalog=0 while IFS= read -r line; do - [ -z "$line" ] && continue - local def="${line%%:*}" mirror="${line##*:}" + [[ -z "$line" ]] && continue + local def="${def:-}" mirror="${mirror:-}" + def="${line%%:*}" + mirror="${line##*:}" + + # Only repoint towards a CatalogSource that actually exists — + # patching to a missing source leaves the subscription unresolvable + # and is worse than leaving it on the disabled default. + if ! oc get catalogsource "$mirror" -n openshift-marketplace >/dev/null 2>&1; then + error " target CatalogSource $mirror not found — $def subscriptions left untouched" + missing_catalog=$((missing_catalog + 1)) + continue + fi + + # One "namespace name" pair per line — read line by line, NOT + # for-loop word splitting (which would feed each token to both + # ns and name and patch the wrong objects). local subs subs=$(oc get subscriptions.operators.coreos.com -A -o json 2>/dev/null | python3 -c " @@ -368,16 +425,118 @@ d = json.load(sys.stdin) for s in d.get('items', []): if s['spec'].get('source') == '$def': print(s['metadata']['namespace'] + ' ' + s['metadata']['name']) -" 2>/dev/null) || true - local sub ns name - for sub in $subs; do - ns="${sub%% *}"; name="${sub##* }" +" 2>/dev/null) || subs="" + + while IFS=' ' read -r ns name; do + [[ -z "${ns:-}" ]] && continue + # Skip ArgoCD-owned subscriptions: declarative via the overlay, + # and self-heal reverts out-of-band patches on them. + if oc get subscription.operators.coreos.com "$name" -n "$ns" \ + -o jsonpath='{.metadata.labels.argocd\.argoproj\.io/instance}' 2>/dev/null | grep -q .; then + info " skipping $ns/$name (ArgoCD-owned — declarative via the airgap overlay)" + skipped=$((skipped + 1)) + continue + fi info " repointing $ns/$name: $def -> $mirror" - oc patch subscriptions.operators.coreos.com "$name" -n "$ns" --type merge \ - -p "{\"spec\":{\"source\":\"$mirror\",\"sourceNamespace\":\"openshift-marketplace\"}}" || true - done + if oc patch subscriptions.operators.coreos.com "$name" -n "$ns" --type merge \ + -p "{\"spec\":{\"source\":\"$mirror\",\"sourceNamespace\":\"openshift-marketplace\"}}"; then + repointed=$((repointed + 1)) + else + error " failed to repoint $ns/$name" + fi + done <<< "$subs" done <<< "$map" - info "Subscription catalog repointing complete" + + info "Repointed: $repointed, skipped (ArgoCD-owned): $skipped, missing catalogs: $missing_catalog" + + # Final check — anything still on a disabled default catalog will never + # resolve. Fail loudly instead of leaving it for a stuck operator. + local stragglers + stragglers=$(oc get subscriptions.operators.coreos.com -A -o json 2>/dev/null | + python3 -c " +import json, sys +d = json.load(sys.stdin) +for s in d.get('items', []): + if s['spec'].get('source') in ('redhat-operators', 'certified-operators', 'community-operators'): + print(s['metadata']['namespace'] + '/' + s['metadata']['name'] + ' -> ' + s['spec']['source']) +" 2>/dev/null) || stragglers="" + if [[ -n "$stragglers" ]]; then + error "Subscriptions still on disabled default catalogs:" + while IFS= read -r s; do + [[ -z "$s" ]] && continue + error " $s" + done <<< "$stragglers" + error "ArgoCD-owned entries are declarative (airgap overlay); others need the repoint fixed." + exit 1 + fi + info "No subscriptions remain on disabled default catalogs" +} + +# ─── Step 3e: Validate the airgap overlay's catalog references ─── +# The overlay names mirrored CatalogSources literally (they track the +# imageset's operator index tags). Fail fast — before the pattern deploys — +# if any of them is missing, and check the mirror registry consistency while +# we are at it: a placeholder or mismatched global.mirrorRegistry would only +# surface as ImagePullBackOff inside the CVMs much later. +validate_airgap_overlay() { + step "3e" "Validate airgap overlay catalog references" + + local overlay="${PATTERN_DIR}/overrides/values-baremetal-airgap.yaml" + if [[ ! -f "$overlay" ]]; then + warn "No airgap overlay at $overlay — skipping validation" + return + fi + + # Every cs-* catalog named in the overlay must exist and be READY. + local catalogs + catalogs=$(python3 -c " +import re, sys +text = open('${overlay}').read() +for m in sorted(set(re.findall(r'cs-[a-z-]+-index-v[0-9-]+', text))): + print(m) +" 2>/dev/null) || catalogs="" + local missing=0 + while IFS= read -r cs; do + [[ -z "$cs" ]] && continue + local state + state=$(oc get catalogsource "$cs" -n openshift-marketplace \ + -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null) || state="" + if [[ "$state" == "READY" ]]; then + info " $cs: READY" + else + error " $cs: missing or not READY (state: '${state:-absent}')" + missing=$((missing + 1)) + fi + done <<< "$catalogs" + if [[ $missing -gt 0 ]]; then + error "The airgap overlay references catalogs that are not READY on this cluster." + error "Check that global.clusterVersion matches the mirrored operator index tags." + exit 1 + fi + + # Mirror registry consistency: global.mirrorRegistry must be a real, + # non-placeholder value and match $MIRROR_REGISTRY (org path included). + local mirror yaml_mirror + mirror="${MIRROR_REGISTRY%%/}" + yaml_mirror=$(python3 -c " +import yaml +with open('${PATTERN_DIR}/values-global.yaml') as f: + d = yaml.safe_load(f) +print(d.get('global', {}).get('mirrorRegistry', '') or '') +" 2>/dev/null) || yaml_mirror="" + if [[ -z "$yaml_mirror" || "$yaml_mirror" == *"MIRROR_REGISTRY_HOST"* ]]; then + error "global.mirrorRegistry is empty or a placeholder in values-global.yaml." + error "Set it to the mirror root INCLUDING the org path (e.g. quay.example:443/mirror)" + error "— the workload charts rewrite their images against it in airgap mode." + exit 1 + fi + if [[ "$yaml_mirror" != "$mirror" ]]; then + error "global.mirrorRegistry ('$yaml_mirror') != MIRROR_REGISTRY ('$mirror')." + error "They must be the same value (org path included) or workloads will pull" + error "from a different registry than the one the cluster mirrors." + exit 1 + fi + info "global.mirrorRegistry matches MIRROR_REGISTRY ($mirror)" } # ─── Step 5: Mirror OCI Helm charts and utility images ─────────── @@ -746,12 +905,23 @@ with open('${PATTERN_DIR}/values-global.yaml') as f: " 2>/dev/null || echo "gitops-1.22") local gitops_source + # operatorSource may be empty in connected mode — fall back to a + # clusterVersion-derived mirrored catalog name (same convention as the + # overlay and step 3d), never a hardcoded minor. + local cluster_version default_source + cluster_version=$(python3 -c " +import yaml +with open('${PATTERN_DIR}/values-global.yaml') as f: + d = yaml.safe_load(f) + print(d.get('global', {}).get('clusterVersion', '')) +" 2>/dev/null || true) + default_source="cs-redhat-operator-index-v${cluster_version//./-}" gitops_source=$(python3 -c " import yaml with open('${PATTERN_DIR}/values-global.yaml') as f: d = yaml.safe_load(f) - print(d.get('main',{}).get('gitops',{}).get('operatorSource','cs-redhat-operator-index-v4-21')) -" 2>/dev/null || echo "cs-redhat-operator-index-v4-21") + print(d.get('main',{}).get('gitops',{}).get('operatorSource','') or '${default_source}') +" 2>/dev/null || echo "$default_source") info "GitOps channel: $gitops_channel (from values-global.yaml)" info "GitOps source: $gitops_source" @@ -900,6 +1070,13 @@ case "$MODE" in fix_manifest_lists exit 0 ;; + repoint) + # Standalone safety net — safe to run any time after the bootstrap, + # e.g. once MCH has created its runtime subscriptions. + validate_prereqs + repoint_default_catalog_subscriptions + exit 0 + ;; esac echo "╔══════════════════════════════════════════════════════════════╗" @@ -913,6 +1090,7 @@ create_catalog_sources apply_ocmirror_resources normalise_mirror_source_policy repoint_default_catalog_subscriptions +validate_airgap_overlay mirror_oci_charts fix_manifest_lists add_argocd_ca From 70e96c608f66ac59e7bfc0919c24fa6c89ffcff4 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 3 Nov 2026 01:51:33 +0000 Subject: [PATCH 15/19] feat(workloads): charts rewrite their images against the mirror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CoCo guest VMs pull images directly — there is no IDMS/ITMS inside the CVM — so the workload charts now rewrite their image references against global.mirrorRegistry when global.airgap is true (oc-mirror source-path convention: registry hostname stripped, org-prefixed mirror root prefixed). Connected mode is unchanged. Rendering fails fast when the mirror is unset or a placeholder in airgap mode, instead of surfacing as ImagePullBackOff inside the CVMs later. global.mirrorRegistry moves to values-global.yaml as a deploy-time value on the disconnected branch (same convention as global.airgap) — the committed placeholder in the airgap overlay was never substituted and would have produced MIRROR_REGISTRY_HOST image references. This also removes the overlay's applications overrides entirely: Helm replaces lists when merging, so those lists would silently drop any entry later added to the same app in values-baremetal.yaml. The acm.mce_operator.source override moves into the base file as a single tpl-conditional entry (mirrored catalog in airgap, the chart default redhat-operators otherwise), leaving the overlay with scalars only. --- .../hello-openshift/templates/_helpers.tpl | 31 +++++++++++++ .../templates/insecure-policy-deployment.yaml | 2 +- .../templates/secure-deployment.yaml | 2 +- .../templates/standard-deployment.yaml | 2 +- .../hello-openshift/values.yaml | 6 +++ .../kbs-access-curl/templates/_helpers.tpl | 30 +++++++++++++ .../kbs-access-curl/templates/deployment.yaml | 4 +- .../kbs-access-curl/values.yaml | 6 +++ .../kbs-access-sealed/templates/_helpers.tpl | 30 +++++++++++++ .../templates/deployment.yaml | 2 +- .../kbs-access-sealed/values.yaml | 9 ++++ overrides/values-baremetal-airgap.yaml | 44 +++++++------------ values-baremetal.yaml | 29 +++++++++--- values-global.yaml | 7 +++ 14 files changed, 165 insertions(+), 39 deletions(-) create mode 100644 charts/coco-supported/kbs-access-curl/templates/_helpers.tpl create mode 100644 charts/coco-supported/kbs-access-sealed/templates/_helpers.tpl diff --git a/charts/coco-supported/hello-openshift/templates/_helpers.tpl b/charts/coco-supported/hello-openshift/templates/_helpers.tpl index 162bbf9d..741215cc 100644 --- a/charts/coco-supported/hello-openshift/templates/_helpers.tpl +++ b/charts/coco-supported/hello-openshift/templates/_helpers.tpl @@ -56,3 +56,34 @@ Determine runtime class name from global config. {{- define "hello-openshift.runtimeClassName" -}} {{- .Values.global.coco.runtimeClassName -}} {{- end }} + +{{/* +Render a workload container image for the CoCo test workloads. + +CoCo guest VMs pull their images directly — there is no IDMS/ITMS inside the +CVM, so a registry.redhat.io/ubi9/... reference cannot be redirected at pull +time. In airgap mode (global.airgap: true) the image is rewritten against +global.mirrorRegistry using the oc-mirror source-path convention: the registry +hostname is stripped and the mirror root prefixed, e.g. + + registry.redhat.io/ubi9/httpd-24@sha256:... + → quay.example:443/mirror/ubi9/httpd-24@sha256:... + +The mirror root must include the org path for Quay targets (it is the same +value oc-mirror was pointed at). Rendering fails fast when airgap mode is on +and the mirror is unset or still a placeholder — a missing value here would +otherwise surface as ImagePullBackOff inside the CVM minutes later. +*/}} +{{- define "hello-openshift.image" -}} +{{- $image := .image -}} +{{- if .root.Values.global.airgap -}} +{{- $mirror := .root.Values.global.mirrorRegistry | default "" -}} +{{- if or (not $mirror) (contains "MIRROR_REGISTRY_HOST" $mirror) -}} +{{- fail "global.mirrorRegistry must be set to the mirror root (e.g. quay.example:443/mirror) when global.airgap is true" -}} +{{- end -}} +{{- $path := regexReplaceAll "^[a-zA-Z0-9.-]+(:[0-9]+)?/(.*)$" $image "${2}" -}} +{{- printf "%s/%s" $mirror $path -}} +{{- else -}} +{{- $image -}} +{{- end -}} +{{- end -}} diff --git a/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml b/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml index 6bee4414..6d36083c 100644 --- a/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml +++ b/charts/coco-supported/hello-openshift/templates/insecure-policy-deployment.yaml @@ -23,7 +23,7 @@ spec: runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }} containers: - name: hello-openshift - image: {{ .Values.image }} + image: {{ include "hello-openshift.image" (dict "root" $ "image" .Values.image) }} ports: - containerPort: 8080 volumeMounts: diff --git a/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml b/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml index b36026b6..8a263ee0 100644 --- a/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml +++ b/charts/coco-supported/hello-openshift/templates/secure-deployment.yaml @@ -25,7 +25,7 @@ spec: runtimeClassName: {{ include "hello-openshift.runtimeClassName" . }} containers: - name: hello-openshift - image: {{ .Values.image }} + image: {{ include "hello-openshift.image" (dict "root" $ "image" .Values.image) }} ports: - containerPort: 8080 volumeMounts: diff --git a/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml b/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml index 561e1c28..0d0db831 100644 --- a/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml +++ b/charts/coco-supported/hello-openshift/templates/standard-deployment.yaml @@ -16,7 +16,7 @@ spec: spec: containers: - name: hello-openshift - image: {{ .Values.image }} + image: {{ include "hello-openshift.image" (dict "root" $ "image" .Values.image) }} ports: - containerPort: 8080 volumeMounts: diff --git a/charts/coco-supported/hello-openshift/values.yaml b/charts/coco-supported/hello-openshift/values.yaml index 9205e61a..13f673c2 100644 --- a/charts/coco-supported/hello-openshift/values.yaml +++ b/charts/coco-supported/hello-openshift/values.yaml @@ -11,3 +11,9 @@ runtimeClassName: "" # Global values used by this chart (overridden by values-global.yaml) global: clusterPlatform: "" # Cluster platform: "Azure" or "AWS" - determines runtime class + # Airgap mode comes from values-global.yaml (global.airgap); the default here + # is connected so standalone chart rendering uses upstream registries. + airgap: false + # Mirror root (org-prefixed for Quay targets) — required when airgap is true. + # The workload charts rewrite their images against it; see _helpers.tpl. + mirrorRegistry: "" diff --git a/charts/coco-supported/kbs-access-curl/templates/_helpers.tpl b/charts/coco-supported/kbs-access-curl/templates/_helpers.tpl new file mode 100644 index 00000000..23185a7c --- /dev/null +++ b/charts/coco-supported/kbs-access-curl/templates/_helpers.tpl @@ -0,0 +1,30 @@ +{{/* +Render a workload container image for the CoCo test workloads. + +CoCo guest VMs pull their images directly — there is no IDMS/ITMS inside the +CVM, so a registry.redhat.io/ubi9/... reference cannot be redirected at pull +time. In airgap mode (global.airgap: true) the image is rewritten against +global.mirrorRegistry using the oc-mirror source-path convention: the registry +hostname is stripped and the mirror root prefixed, e.g. + + registry.redhat.io/ubi9/httpd-24@sha256:... + → quay.example:443/mirror/ubi9/httpd-24@sha256:... + +The mirror root must include the org path for Quay targets (it is the same +value oc-mirror was pointed at). Rendering fails fast when airgap mode is on +and the mirror is unset or still a placeholder — a missing value here would +otherwise surface as ImagePullBackOff inside the CVM minutes later. +*/}} +{{- define "kbs-access-curl.image" -}} +{{- $image := .image -}} +{{- if .root.Values.global.airgap -}} +{{- $mirror := .root.Values.global.mirrorRegistry | default "" -}} +{{- if or (not $mirror) (contains "MIRROR_REGISTRY_HOST" $mirror) -}} +{{- fail "global.mirrorRegistry must be set to the mirror root (e.g. quay.example:443/mirror) when global.airgap is true" -}} +{{- end -}} +{{- $path := regexReplaceAll "^[a-zA-Z0-9.-]+(:[0-9]+)?/(.*)$" $image "${2}" -}} +{{- printf "%s/%s" $mirror $path -}} +{{- else -}} +{{- $image -}} +{{- end -}} +{{- end -}} diff --git a/charts/coco-supported/kbs-access-curl/templates/deployment.yaml b/charts/coco-supported/kbs-access-curl/templates/deployment.yaml index 48beeb75..a6f99026 100644 --- a/charts/coco-supported/kbs-access-curl/templates/deployment.yaml +++ b/charts/coco-supported/kbs-access-curl/templates/deployment.yaml @@ -37,7 +37,7 @@ spec: runAsUser: 0 containers: - name: httpd - image: {{ .Values.image }} + image: {{ include "kbs-access-curl.image" (dict "root" $ "image" .Values.image) }} ports: - containerPort: 8080 securityContext: @@ -50,7 +50,7 @@ spec: readOnly: true initContainers: - name: curl - image: {{ .Values.initImage }} + image: {{ include "kbs-access-curl.image" (dict "root" $ "image" .Values.initImage) }} command: ['sh', '-c', 'curl -s http://127.0.0.1:8006/cdh/resource/default/kbsres1/key3 > /var/www/html/secret.txt'] securityContext: privileged: true diff --git a/charts/coco-supported/kbs-access-curl/values.yaml b/charts/coco-supported/kbs-access-curl/values.yaml index 82fda729..88ca2919 100644 --- a/charts/coco-supported/kbs-access-curl/values.yaml +++ b/charts/coco-supported/kbs-access-curl/values.yaml @@ -9,3 +9,9 @@ defaultMemory: "" runtimeClassName: "" global: clusterPlatform: "" # Cluster platform: "Azure", "AWS", etc. - determines runtime class + # Airgap mode comes from values-global.yaml (global.airgap); the default here + # is connected so standalone chart rendering uses upstream registries. + airgap: false + # Mirror root (org-prefixed for Quay targets) — required when airgap is true. + # The workload charts rewrite their images against it; see _helpers.tpl. + mirrorRegistry: "" diff --git a/charts/coco-supported/kbs-access-sealed/templates/_helpers.tpl b/charts/coco-supported/kbs-access-sealed/templates/_helpers.tpl new file mode 100644 index 00000000..61e2ab98 --- /dev/null +++ b/charts/coco-supported/kbs-access-sealed/templates/_helpers.tpl @@ -0,0 +1,30 @@ +{{/* +Render a workload container image for the CoCo test workloads. + +CoCo guest VMs pull their images directly — there is no IDMS/ITMS inside the +CVM, so a registry.redhat.io/ubi9/... reference cannot be redirected at pull +time. In airgap mode (global.airgap: true) the image is rewritten against +global.mirrorRegistry using the oc-mirror source-path convention: the registry +hostname is stripped and the mirror root prefixed, e.g. + + registry.redhat.io/ubi9/httpd-24@sha256:... + → quay.example:443/mirror/ubi9/httpd-24@sha256:... + +The mirror root must include the org path for Quay targets (it is the same +value oc-mirror was pointed at). Rendering fails fast when airgap mode is on +and the mirror is unset or still a placeholder — a missing value here would +otherwise surface as ImagePullBackOff inside the CVM minutes later. +*/}} +{{- define "kbs-access-sealed.image" -}} +{{- $image := .image -}} +{{- if .root.Values.global.airgap -}} +{{- $mirror := .root.Values.global.mirrorRegistry | default "" -}} +{{- if or (not $mirror) (contains "MIRROR_REGISTRY_HOST" $mirror) -}} +{{- fail "global.mirrorRegistry must be set to the mirror root (e.g. quay.example:443/mirror) when global.airgap is true" -}} +{{- end -}} +{{- $path := regexReplaceAll "^[a-zA-Z0-9.-]+(:[0-9]+)?/(.*)$" $image "${2}" -}} +{{- printf "%s/%s" $mirror $path -}} +{{- else -}} +{{- $image -}} +{{- end -}} +{{- end -}} diff --git a/charts/coco-supported/kbs-access-sealed/templates/deployment.yaml b/charts/coco-supported/kbs-access-sealed/templates/deployment.yaml index ad2503fc..b80989c4 100644 --- a/charts/coco-supported/kbs-access-sealed/templates/deployment.yaml +++ b/charts/coco-supported/kbs-access-sealed/templates/deployment.yaml @@ -24,7 +24,7 @@ spec: runtimeClassName: {{ .Values.global.coco.runtimeClassName }} containers: - name: httpd - image: {{ .Values.image }} + image: {{ include "kbs-access-sealed.image" (dict "root" $ "image" .Values.image) }} ports: - containerPort: 8080 protocol: TCP diff --git a/charts/coco-supported/kbs-access-sealed/values.yaml b/charts/coco-supported/kbs-access-sealed/values.yaml index 7e06d347..2586ccfc 100644 --- a/charts/coco-supported/kbs-access-sealed/values.yaml +++ b/charts/coco-supported/kbs-access-sealed/values.yaml @@ -2,3 +2,12 @@ # Override via values-global.yaml or values-baremetal.yaml image: registry.redhat.io/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2 + +global: + clusterPlatform: "" # Cluster platform: "Azure", "AWS", etc. - determines runtime class + # Airgap mode comes from values-global.yaml (global.airgap); the default here + # is connected so standalone chart rendering uses upstream registries. + airgap: false + # Mirror root (org-prefixed for Quay targets) — required when airgap is true. + # The workload charts rewrite their images against it; see _helpers.tpl. + mirrorRegistry: "" diff --git a/overrides/values-baremetal-airgap.yaml b/overrides/values-baremetal-airgap.yaml index b0f3d9f1..2425d484 100644 --- a/overrides/values-baremetal-airgap.yaml +++ b/overrides/values-baremetal-airgap.yaml @@ -3,9 +3,12 @@ # The connected default (global.airgap: false) loads the inert # values-baremetal-connected.yaml instead. # -# SCOPE: this overlay applies when ArgoCD renders the clustergroup chart — -# i.e. to the clusterGroup OLM subscriptions (ESO, CNV, LVMS, trustee, -# cert-manager, intel-*, ACM, NFD) and the chart values below. +# SCOPE — this overlay reaches the ROOT clustergroup render: the +# patterns-operator resolves clusterGroup.sharedValueFiles itself when it +# builds the root Application (internal/controller/argo.go, +# commonApplicationSourceHelm → getSharedValueFiles), in addition to the +# chart forwarding them to child Applications. So the clusterGroup +# subscriptions and the app overrides below are all in scope. # # It does NOT reach the bootstrap operators: the openshift-gitops-operator # and patterns-operator subscriptions are created by the patterns-operator @@ -13,12 +16,20 @@ # be set directly in values-global.yaml: # main.gitops.operatorSource: cs-redhat-operator-index-v4-22 # main.patternsOperator.source: cs-community-operator-index-v4-22 +# +# LIST WARNING: Helm replaces lists when merging — an `overrides:` list here +# would drop any entry added to the same app in values-baremetal.yaml. This +# overlay therefore sets ONLY scalars (map values). Mode-dependent list +# entries (e.g. acm.mce_operator.source) are tpl-conditionals in the base +# file instead. global: - # Mirrored catalog for the Argo-rendered OLM subscriptions (oc-mirror output) + # Mirrored catalog for the Argo-rendered OLM subscriptions (oc-mirror output). + # The names track the operator index tags in airgap/imageset-config-.yaml + # (e.g. redhat/redhat-operator-index:v4.22 → cs-redhat-operator-index-v4-22 for + # global.clusterVersion 4.22); scripts/airgap-post-install.sh step 3e fails fast + # if any catalog referenced here is missing from the cluster. catalogSource: cs-redhat-operator-index-v4-22 catalogSourceNamespace: openshift-marketplace - # Mirror root for the workload image overrides (org-prefixed for Quay). - mirrorRegistry: "MIRROR_REGISTRY_HOST:8443" # Connected default: the certified-operators catalog, disabled in airgap. clusterGroup: subscriptions: @@ -26,24 +37,3 @@ clusterGroup: source: cs-certified-operator-index-v4-22 intel-tdx-dcap: source: cs-certified-operator-index-v4-22 - # CoCo guest VMs pull images directly — no IDMS/ITMS inside the CVM. - applications: - acm: - # Born on redhat-operators by default — disabled in airgap. - overrides: - - name: acm.mce_operator.source - value: cs-redhat-operator-index-v4-22 - hello-openshift: - overrides: - - name: image - value: "{{ $.Values.global.mirrorRegistry }}/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2" - kbs-access-curl: - overrides: - - name: image - value: "{{ $.Values.global.mirrorRegistry }}/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2" - - name: initImage - value: "{{ $.Values.global.mirrorRegistry }}/ubi9/ubi:latest" - kbs-access-sealed: - overrides: - - name: image - value: "{{ $.Values.global.mirrorRegistry }}/ubi9/httpd-24@sha256:68a91ff691092f455fea682330c499588747231c16516cd4f35aff821e6847f2" diff --git a/values-baremetal.yaml b/values-baremetal.yaml index fe869f7e..2bfee01f 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -170,15 +170,26 @@ clusterGroup: # before trustee at 20. annotations: argocd.argoproj.io/sync-wave: "15" - # Airgap: MCE source set by the airgap overlay (acm.mce_operator.source). + # The MCH operator creates the multicluster-engine subscription at + # runtime from the mce-subscription-spec annotation (rendered from this + # value), so it must be born on the right catalog. Connected: the + # default redhat-operators. Airgap: the mirrored catalog (the default + # catalogs are disabled). Kept as a single tpl-conditional list entry — + # an overlay list would replace this one wholesale. + overrides: + - name: acm.mce_operator.source + value: '{{ if $.Values.global.airgap }}cs-redhat-operator-index-v4-22{{ else }}redhat-operators{{ end }}' vault: name: vault namespace: vault project: vault chart: hashicorp-vault - # Require 0.1.10 or newer within the 0.1.x release series. - chartVersion: "~0.1.10" + # Exact pin: chart 0.1.11 deploys vault:2.1.2-ubi — the version the + # imageset mirrors. A range (~0.1.10) would resolve to whatever is + # newest at deploy time and desync from the mirrored vault image + # (0.1.10 deploys 1.21.4-ubi). + chartVersion: 0.1.11 annotations: argocd.argoproj.io/sync-wave: "0" @@ -340,7 +351,9 @@ clusterGroup: path: charts/coco-supported/hello-openshift annotations: argocd.argoproj.io/sync-wave: "30" - # Airgap: image overrides activated by the airgap overlay. + # Airgap: the chart rewrites its image against global.mirrorRegistry + # (see charts/coco-supported//templates/_helpers.tpl) — no + # overrides needed here in either mode. syncPolicy: automated: prune: true @@ -352,7 +365,9 @@ clusterGroup: path: charts/coco-supported/kbs-access-curl annotations: argocd.argoproj.io/sync-wave: "30" - # Airgap — image overrides activated by the airgap overlay. + # Airgap: the chart rewrites its image against global.mirrorRegistry + # (see charts/coco-supported//templates/_helpers.tpl) — no + # overrides needed here in either mode. syncPolicy: automated: prune: true @@ -364,7 +379,9 @@ clusterGroup: path: charts/coco-supported/kbs-access-sealed annotations: argocd.argoproj.io/sync-wave: "30" - # Airgap — image overrides activated by the airgap overlay. + # Airgap: the chart rewrites its image against global.mirrorRegistry + # (see charts/coco-supported//templates/_helpers.tpl) — no + # overrides needed here in either mode. syncPolicy: automated: prune: true diff --git a/values-global.yaml b/values-global.yaml index d5e8c0af..b3e76623 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -13,6 +13,13 @@ global: # and workload images at the mirror); manual PCK provisioning # (scripts/dcap-pck.py) and file-backed collateral. airgap: false + # Mirror root for the workload image rewrite — set on the disconnected + # deploy branch (config-only delta), the same value oc-mirror was pointed + # at. MUST include the org path for Quay targets (e.g. + # quay.example:443/mirror): the CoCo guest VMs pull images directly and + # every reference carries it. Empty is fine while connected; the workload + # charts fail rendering if this is empty (or a placeholder) in airgap mode. + mirrorRegistry: "" secretStore: # Warning: This must be present even if it is set to none. backend: vault # none, vault, kubernetes From 6e3d8516dd161efef8f1b57be547e47835d6caff Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 3 Nov 2026 01:51:40 +0000 Subject: [PATCH 16/19] fix(airgap): pin the vault chart to 0.1.11 The values range (~0.1.10) resolved whatever was newest at deploy time while the imageset pinned a single chart tag: chart 0.1.11 deploys vault:2.1.2-ubi but 0.1.10 deploys 1.21.4-ubi, so a range could desync the operand image from the mirror. Pin the exact version the imageset mirrors. --- airgap/imageset-config-4.22.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/airgap/imageset-config-4.22.yaml b/airgap/imageset-config-4.22.yaml index 84000056..07620e89 100644 --- a/airgap/imageset-config-4.22.yaml +++ b/airgap/imageset-config-4.22.yaml @@ -1,6 +1,7 @@ --- # ImageSetConfiguration for CoCo Validated Pattern airgap deployment -# Target: MIRROR_REGISTRY_HOST:8443 (mirror-registry — sole airgap target, Phase 25.2) +# Target: a single mirror registry (org-prefixed target for Quay, e.g. +# quay.example:443/mirror — MIRROR_REGISTRY includes the org path) # OCP: 4.22.17 # Topology: baremetal (full operator set from values-baremetal.yaml) # @@ -10,7 +11,7 @@ # Usage: # oc-mirror -c airgap/imageset-config-4.22.yaml \ # --workspace file://$HOME/oc-mirror-workspace \ -# docker://MIRROR_REGISTRY_HOST:8443 --v2 +# docker:// --v2 apiVersion: mirror.openshift.io/v2alpha1 kind: ImageSetConfiguration @@ -126,7 +127,7 @@ mirror: # --- VP Helm OCI charts --- - name: quay.io/validatedpatterns/clustergroup:0.9.58 - name: quay.io/validatedpatterns/acm:0.1.27 - # values pin ~0.1.10 resolves the newest 0.1.x — mirror that, not the floor + # values pin 0.1.11 exactly (chart 0.1.11 deploys vault:2.1.2-ubi) - name: quay.io/validatedpatterns/hashicorp-vault:0.1.11 - name: quay.io/validatedpatterns/openshift-external-secrets:0.0.4 - name: quay.io/validatedpatterns/trustee:0.10.2 From be2d0330293088d161814ac5b834bdd2cd795062 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 3 Nov 2026 01:51:40 +0000 Subject: [PATCH 17/19] =?UTF-8?q?docs(airgap):=20MIRROR=5FREGISTRY=20inclu?= =?UTF-8?q?des=20the=20org=20path=20=E2=80=94=20everywhere?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ITMS header said the org path is required while the bootstrap script's help and error text said bare host:port; following the script's own guidance produced the 404s the header warned about. State the org-prefixed mirror root consistently in the script help/validation, the ITMS comment and the imageset header. --- airgap/itms-manual-mirrors.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airgap/itms-manual-mirrors.yaml b/airgap/itms-manual-mirrors.yaml index 074cd6d2..18b2ef85 100644 --- a/airgap/itms-manual-mirrors.yaml +++ b/airgap/itms-manual-mirrors.yaml @@ -1,5 +1,6 @@ # Org-prefixed mirror targets: MIRROR_REGISTRY must include the org path -# (e.g. quay.example:443/mirror) or every tag-based operand pull 404s. +# (e.g. quay.example:443/mirror) — the same value scripts/airgap-post-install.sh +# requires — or every tag-based operand pull 404s. apiVersion: config.openshift.io/v1 kind: ImageTagMirrorSet metadata: From 67e7516c3014c6bf8b5e5cc0f36fb069231f618a Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 3 Nov 2026 01:51:47 +0000 Subject: [PATCH 18/19] test(airgap): render gate for both connectivity modes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renders the full root-app values chain (the same files the patterns-operator assembles: values-global + values- + the resolved sharedValueFiles) with the clustergroup chart, and asserts the airgap overlay's effect on subscriptions and app overrides, connected mode rendering the upstream defaults, and the workload charts' image rewriting — including the fail-fast when the mirror is unset in airgap mode. Lets reviewers confirm the overlay reaches the root render without a cluster. --- scripts/verify/render-gate.sh | 185 ++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100755 scripts/verify/render-gate.sh diff --git a/scripts/verify/render-gate.sh b/scripts/verify/render-gate.sh new file mode 100755 index 00000000..d6bedd35 --- /dev/null +++ b/scripts/verify/render-gate.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# Render gate for the CoCo pattern values chain — both connectivity modes. +# +# Reproduces, without a cluster, the exact values chain the patterns-operator +# assembles for the ROOT clustergroup Application (values-global.yaml + +# values-.yaml + the sharedValueFiles it resolves for the platform/ +# storage/version/hardware/airgap axes), then asserts: +# +# airgap mode +# - intel subscriptions render on the mirrored certified catalog +# - sourceless subscriptions render on the mirrored redhat catalog +# - the acm mce_operator override renders the mirrored catalog +# - the trustee collateral override renders the file:// backend +# connected mode +# - intel subscriptions stay on certified-operators +# - sourceless subscriptions stay on redhat-operators (chart default) +# - the acm mce_operator override renders redhat-operators (chart default) +# - the trustee collateral override renders the Intel PCS URL +# workload charts (each, standalone) +# - connected: upstream registry.redhat.io images unchanged +# - airgap: images rewritten against global.mirrorRegistry +# - airgap without a mirror: rendering FAILS (fail-fast, not a silent +# ImagePullBackOff in the CVM) +# +# Usage: scripts/verify/render-gate.sh [mirror-root] +# mirror-root defaults to quay.example:443/mirror (placeholder is fine — +# nothing is pulled; only the rendered strings are asserted). +# +# Requires: helm >= 3.14 and network access to quay.io for the chart pull. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PATTERN_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +CLUSTERGROUP_VERSION="0.9.58" +GROUP="baremetal" +PLATFORM="None" # SNO: no values- file is present +MIRROR_ROOT="${1:-quay.example:443/mirror}" +FAILS=0 + +pass() { echo -e "PASS: $*"; } +fail() { echo -e "FAIL: $*"; FAILS=$((FAILS + 1)); } + +# ── Root render: the operator's value chain ────────────────────────── +helm pull oci://quay.io/validatedpatterns/clustergroup \ + --version "$CLUSTERGROUP_VERSION" --untar --untardir "$WORK" >/dev/null +CHART="${WORK}/clustergroup" + +render_root() { # $1 = airgap|connected + local mode="$1" + helm template cg "${CHART}" \ + -f "${PATTERN_DIR}/values-global.yaml" \ + -f "${PATTERN_DIR}/values-${GROUP}.yaml" \ + -f "${PATTERN_DIR}/overrides/values-storage-lvm.yaml" \ + -f "${PATTERN_DIR}/overrides/values-4.22.yaml" \ + -f "${PATTERN_DIR}/overrides/values-hw-intel-tdx.yaml" \ + -f "${PATTERN_DIR}/overrides/values-${GROUP}-${mode}.yaml" \ + --set global.repoURL=https://example.com/coco-pattern.git \ + --set global.targetRevision=render-gate \ + --set global.airgap="$([ "$mode" = airgap ] && echo true || echo false)" \ + --set-string "global.mirrorRegistry=$([ "$mode" = airgap ] && echo "$MIRROR_ROOT" || echo '')" \ + 2>/dev/null +} + +sub_source() { # $1 = rendered manifest, $2 = subscription name + echo "$1" | awk -v wanted="$2" ' + /^ name: / { cur = $2 } + /^ source: / { if (cur == wanted) { print $2; exit } }' +} + +acm_override() { # $1 = rendered manifest + echo "$1" | grep -A1 'name: acm.mce_operator.source' | grep 'value:' | head -1 | sed "s/.*value: //; s/'//g; s/\"//g" +} + +collateral_override() { # $1 = rendered manifest + echo "$1" | grep -A1 'name: kbs.tdx.collateralService' | grep 'value:' | head -1 | sed "s/.*value: //; s/'//g; s/\"//g" +} + +# ── Airgap assertions ──────────────────────────────────────────────── +echo "── airgap mode (global.airgap=true, mirror=${MIRROR_ROOT})" +AIRGAP="$(render_root airgap)" || { fail "airgap root render errored"; } + +for sub in intel-device-plugins-operator intel-tdx-dcap-operator; do + src="$(sub_source "$AIRGAP" "$sub")" + [[ "$src" == "cs-certified-operator-index-v4-22" ]] \ + && pass "subscription $sub → $src" \ + || fail "subscription $sub → '${src}' (expected cs-certified-operator-index-v4-22)" +done + +for sub in openshift-external-secrets-operator trustee-operator sandboxed-containers-operator; do + src="$(sub_source "$AIRGAP" "$sub")" + [[ "$src" == "cs-redhat-operator-index-v4-22" ]] \ + && pass "subscription $sub → $src" \ + || fail "subscription $sub → '${src}' (expected cs-redhat-operator-index-v4-22)" +done + +val="$(acm_override "$AIRGAP")" +[[ "$val" == "cs-redhat-operator-index-v4-22" ]] \ + && pass "acm.mce_operator.source → $val" \ + || fail "acm.mce_operator.source → '${val}' (expected cs-redhat-operator-index-v4-22)" + +val="$(collateral_override "$AIRGAP")" +[[ "$val" == file:///opt/confidential-containers/attestation-service/tdx/platform_collaterals.json ]] \ + && pass "kbs.tdx.collateralService → file:// backend" \ + || fail "kbs.tdx.collateralService → '${val}' (expected the file:// backend)" + +# ── Connected assertions ───────────────────────────────────────────── +echo "── connected mode (global.airgap=false)" +CONNECTED="$(render_root connected)" || { fail "connected root render errored"; } + +for sub in intel-device-plugins-operator intel-tdx-dcap-operator; do + src="$(sub_source "$CONNECTED" "$sub")" + [[ "$src" == "certified-operators" ]] \ + && pass "subscription $sub → $src" \ + || fail "subscription $sub → '${src}' (expected certified-operators)" +done + +for sub in openshift-external-secrets-operator trustee-operator sandboxed-containers-operator; do + src="$(sub_source "$CONNECTED" "$sub")" + [[ "$src" == "redhat-operators" ]] \ + && pass "subscription $sub → $src" \ + || fail "subscription $sub → '${src}' (expected redhat-operators)" +done + +val="$(acm_override "$CONNECTED")" +[[ "$val" == "redhat-operators" ]] \ + && pass "acm.mce_operator.source → $val (chart default)" \ + || fail "acm.mce_operator.source → '${val}' (expected redhat-operators)" + +val="$(collateral_override "$CONNECTED")" +[[ "$val" == "https://api.trustedservices.intel.com/sgx/certification/v4/" ]] \ + && pass "kbs.tdx.collateralService → Intel PCS" \ + || fail "kbs.tdx.collateralService → '${val}' (expected Intel PCS URL)" + +# ── Workload charts ────────────────────────────────────────────────── +echo "── workload charts" +for chart in hello-openshift kbs-access-curl kbs-access-sealed; do + out="$(helm template wg "${PATTERN_DIR}/charts/coco-supported/${chart}" \ + --set global.coco.runtimeClassName=kata-cc 2>/dev/null)" \ + || { fail "$chart connected render errored"; continue; } + if echo "$out" | grep -q "registry.redhat.io/ubi9/httpd-24@sha256:68a91ff"; then + pass "$chart connected: upstream image unchanged" + else + fail "$chart connected: image not the upstream registry.redhat.io reference" + fi + + out="$(helm template wg "${PATTERN_DIR}/charts/coco-supported/${chart}" \ + --set global.coco.runtimeClassName=kata-cc --set global.airgap=true \ + --set-string "global.mirrorRegistry=${MIRROR_ROOT}" 2>/dev/null)" \ + || { fail "$chart airgap render errored"; continue; } + if echo "$out" | grep -q "${MIRROR_ROOT}/ubi9/httpd-24@sha256:68a91ff" \ + && ! echo "$out" | grep -q "registry.redhat.io/ubi9"; then + pass "$chart airgap: image rewritten to ${MIRROR_ROOT}/ubi9/..." + else + fail "$chart airgap: image not rewritten to the mirror root" + fi +done + +# initImage (registry.access.redhat.com source path) +out="$(helm template x "${PATTERN_DIR}/charts/coco-supported/kbs-access-curl" \ + --set global.coco.runtimeClassName=kata-cc --set global.airgap=true \ + --set-string "global.mirrorRegistry=${MIRROR_ROOT}" 2>/dev/null)" \ + && echo "$out" | grep -q "${MIRROR_ROOT}/ubi9/ubi:latest" \ + && pass "kbs-access-curl airgap: initImage rewritten (registry.access path)" \ + || fail "kbs-access-curl airgap: initImage not rewritten" + +# Fail-fast: airgap without a mirror must not render +if helm template x "${PATTERN_DIR}/charts/coco-supported/hello-openshift" \ + --set global.coco.runtimeClassName=kata-cc --set global.airgap=true \ + >/dev/null 2>&1; then + fail "airgap without global.mirrorRegistry rendered (should fail fast)" +else + pass "airgap without global.mirrorRegistry fails fast" +fi + +# ── Verdict ────────────────────────────────────────────────────────── +echo "" +if [[ $FAILS -eq 0 ]]; then + echo "RENDER GATE: PASS (0 failures)" +else + echo "RENDER GATE: FAIL (${FAILS} failures)" + exit 1 +fi From 344076bb966ab3c083a615ac1dfc46ea85611a68 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 3 Nov 2026 08:36:34 +0000 Subject: [PATCH 19/19] fix(rbac): bootstrap the ArgoCD controller cluster-admin at wave -1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ArgoCD applies wave-0 resources by kind, so the imperative ServiceAccounts are attempted before the cluster-admin ClusterRoleBinding that authorises the controller to create them — the applies fail forbidden in the first seconds of the first sync (observed live: SAs forbidden at sync start, the binding applied 8s in by the same sync, permitted because the operator's instance role already grants rbac.authorization.k8s.io: */*). Ungated installs self-heal on the next automated sync, which is why this was never seen before; with sync-wave gating the first operation parks on wave-0 health (ESO Degraded ← vault unconfigured ← imperative cronjobs cannot start ← SA missing) and the retry never fires — a hard deadlock on clean installs. Render a distinct-named cluster-admin binding at sync-wave -1 via clusterGroup.extraObjects. The controller can create it from the start (full rbac.* from the operator role) and ClusterRoleBindings are instantly healthy, so wave 0 then applies the ServiceAccounts with cluster-admin in place. Bridge, not a fix: patterns-operator#887 tracks the root cause with two upstream options (operator-created binding, or moving the clustergroup chart's own binding to wave -1). TODO marker included; delete this once either lands. --- scripts/verify/render-gate.sh | 24 ++++++++++++++++++++++++ values-baremetal.yaml | 27 +++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/scripts/verify/render-gate.sh b/scripts/verify/render-gate.sh index d6bedd35..cc207e01 100755 --- a/scripts/verify/render-gate.sh +++ b/scripts/verify/render-gate.sh @@ -134,6 +134,30 @@ val="$(collateral_override "$CONNECTED")" && pass "kbs.tdx.collateralService → Intel PCS" \ || fail "kbs.tdx.collateralService → '${val}' (expected Intel PCS URL)" +# ── RBAC bootstrap bridge (patterns-operator#887) ─────────────────── +echo "── RBAC bridge" +for mode in airgap connected; do + out="$(render_root "$mode")" || { fail "$mode root render errored"; continue; } + # All documents mentioning the bridge name (the CRB itself, plus the + # helm-values ConfigMap which embeds the values indented). The real CRB is + # distinguished by an UNINDENTED "kind: ClusterRoleBinding" line. + block="$(echo "$out" | awk -v RS='---' '/coco-pattern-argocd-cluster-admin-bootstrap/')" + if [[ -z "$block" ]]; then + fail "$mode: RBAC bridge CRB missing from root render" + continue + fi + ok=1 + echo "$block" | grep -q "^kind: ClusterRoleBinding" || ok=0 + echo "$block" | grep -q "cluster-admin$" || ok=0 + echo "$block" | grep -q "vp-gitops-argocd-application-controller" || ok=0 + echo "$block" | grep -q 'sync-wave: "-1"' || ok=0 + if [[ $ok -eq 1 ]]; then + pass "$mode: RBAC bridge CRB at wave -1 for the controller SA" + else + fail "$mode: RBAC bridge CRB malformed (kind/role/subject/wave)" + fi +done + # ── Workload charts ────────────────────────────────────────────────── echo "── workload charts" for chart in hello-openshift kbs-access-curl kbs-access-sealed; do diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 2bfee01f..8a996299 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -125,6 +125,33 @@ clusterGroup: - '/overrides/values-baremetal-{{ if $.Values.global.airgap }}airgap{{ else }}connected{{ end }}.yaml' extraObjects: + # Bridge for the first-sync ordering race (patterns-operator#887): ArgoCD + # applies wave-0 resources by kind, so the imperative ServiceAccounts are + # attempted before the cluster-admin ClusterRoleBinding that authorises the + # controller to create them. Ungated installs self-heal on the next + # auto-sync; with sync-wave gating the first operation parks on wave-0 + # health and deadlocks. This binding applies at wave -1, before any wave-0 + # kind, which the controller can create from the start (the operator's + # instance role already grants rbac.authorization.k8s.io: */*). + # Distinct name on purpose — matching the clustergroup chart's own + # vp-gitops-cluster-admin-rolebinding would collide in the manifest. + # TODO: remove once patterns-operator creates the binding (or the + # clustergroup chart moves its own to wave -1) — see patterns-operator#887. + argocd-cluster-admin-bootstrap: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: coco-pattern-argocd-cluster-admin-bootstrap + annotations: + argocd.argoproj.io/sync-wave: "-1" + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: vp-gitops-argocd-application-controller + namespace: vp-gitops wait-for-vault-unsealed: apiVersion: batch/v1 kind: Job