diff --git a/.github/workflows/golang-checks.yaml b/.github/workflows/golang-checks.yaml index d16e50a038..1dadfaa53f 100644 --- a/.github/workflows/golang-checks.yaml +++ b/.github/workflows/golang-checks.yaml @@ -79,6 +79,10 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: go-version: ${{ env.GOLANG_VERSION }} + - name: Install Helm + uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v4.3.2 - name: Setup Go Proxy id: setup-go-proxy uses: nv-gha-runners/setup-artifactory-go-proxy@a15ec16ff434e8ca3b0f76919f902080e66ebc14 # main diff --git a/README.md b/README.md index 10d887f398..f8763e9278 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,12 @@ After installation, the GPU Operator and its operands should be up and running. Note: To deploy the GPU Operator on OpenShift, follow the instructions in the [official documentation](https://docs.nvidia.com/datacenter/cloud-native/openshift/latest/steps-overview.html). +## Operator high availability + +Helm installations run one operator replica by default. Increase `operator.replicaCount` to run multiple replicas with leader election. Unless explicitly disabled, the chart also adds soft pod anti-affinity and a PodDisruptionBudget (PDB) requiring at least one available replica. + +OLM installations continue to deploy one operator replica and do not include the Helm chart's PDB or pod anti-affinity. Helm values do not configure OLM installations. Administrators can manually increase replicas and add anti-affinity through the installed ClusterServiceVersion (CSV), then create a PDB separately, following `deployments/gpu-operator/templates/poddisruptionbudget.yaml`. + ## Product Documentation For information on platform support and getting started, visit the official documentation [repository](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/overview.html). diff --git a/cmd/gpu-operator/main.go b/cmd/gpu-operator/main.go index 4578266f79..823c175cab 100644 --- a/cmd/gpu-operator/main.go +++ b/cmd/gpu-operator/main.go @@ -79,6 +79,7 @@ func main() { var probeAddr string var pprofAddr string var renewDeadline time.Duration + var leaderElectionWarmup bool flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&pprofAddr, "pprof-bind-address", "", @@ -95,6 +96,9 @@ func main() { "Only enabled when the --leader-elect flag is set. "+ "If undefined, the renew deadline defaults to the controller-runtime manager's default RenewDeadline. "+ "By setting this option, the LeaseDuration is also set as RenewDealine + 5s.") + flag.BoolVar(&leaderElectionWarmup, "leader-election-warmup", false, + "Enable informer cache warmup for the operator's standby replicas. "+ + "Only enabled when the --leader-elect flag is set.") opts := zap.Options{ StacktraceLevel: zapcore.PanicLevel, @@ -126,16 +130,17 @@ func main() { } options := ctrl.Options{ - Scheme: scheme, - Metrics: metricsOptions, - HealthProbeBindAddress: probeAddr, - PprofBindAddress: pprofAddr, - LeaderElection: enableLeaderElection, - LeaderElectionNamespace: leaderElectionNamespace, - LeaderElectionID: "53822513.nvidia.com", - WebhookServer: webhookServer, - Cache: operatorcache.Options(operatorNamespace), - NewCache: operatorcache.New, + Scheme: scheme, + Metrics: metricsOptions, + HealthProbeBindAddress: probeAddr, + PprofBindAddress: pprofAddr, + LeaderElection: enableLeaderElection, + LeaderElectionNamespace: leaderElectionNamespace, + LeaderElectionID: "53822513.nvidia.com", + LeaderElectionReleaseOnCancel: true, + WebhookServer: webhookServer, + Cache: operatorcache.Options(operatorNamespace), + NewCache: operatorcache.New, } if enableLeaderElection && int(renewDeadline) != 0 { @@ -145,6 +150,10 @@ func main() { options.LeaseDuration = &leaseDuration } + if enableLeaderElection && leaderElectionWarmup { + options.Controller.EnableWarmup = &leaderElectionWarmup + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options) if err != nil { setupLog.Error(err, "unable to start manager") @@ -247,6 +256,10 @@ func main() { setupLog.Error(err, "unable to set up ready check") os.Exit(1) } + if err := mgr.Add(operatorMetrics); err != nil { + setupLog.Error(err, "unable to add operator metrics registration") + os.Exit(1) + } setupLog.Info("starting manager") if err := mgr.Start(ctx); err != nil { diff --git a/controllers/operator_metrics.go b/controllers/operator_metrics.go index d80e31bb59..b1306d280b 100644 --- a/controllers/operator_metrics.go +++ b/controllers/operator_metrics.go @@ -17,6 +17,7 @@ package controllers import ( + "context" "fmt" promcli "github.com/prometheus/client_golang/prometheus" @@ -66,8 +67,8 @@ const ( operatorMetricsNamespace = "gpu_operator" ) -// InitOperatorMetrics registers all GPU operator Prometheus metrics with the -// controller-runtime registry and returns the initialised OperatorMetrics. +// InitOperatorMetrics creates GPU operator Prometheus metrics. The metrics are +// registered after the manager becomes leader. func InitOperatorMetrics() *OperatorMetrics { m := &OperatorMetrics{ gpuNodesTotal: promcli.NewGauge( @@ -195,8 +196,23 @@ func InitOperatorMetrics() *OperatorMetrics { }, ), } + return m +} + +// Start implements manager.Runnable. It registers metrics with the +// controller-runtime registry. +func (m *OperatorMetrics) Start(context.Context) error { + m.register(metrics.Registry) + return nil +} - metrics.Registry.MustRegister( +// NeedLeaderElection implements manager.LeaderElectionRunnable. Returning true +// makes the manager call Start only after this replica acquires the lease, so that a +// standby replica does not also expose reconciliation metrics. +func (m *OperatorMetrics) NeedLeaderElection() bool { return true } + +func (m *OperatorMetrics) register(registry promcli.Registerer) { + registry.MustRegister( m.gpuNodesTotal, m.reconciliationLastSuccess, @@ -218,6 +234,4 @@ func InitOperatorMetrics() *OperatorMetrics { m.upgradesFailed, m.upgradesPending, ) - - return m } diff --git a/controllers/operator_metrics_test.go b/controllers/operator_metrics_test.go new file mode 100644 index 0000000000..c8a6ba557e --- /dev/null +++ b/controllers/operator_metrics_test.go @@ -0,0 +1,55 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package controllers + +import ( + "testing" + + promcli "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +func TestOperatorMetricsExposedOnlyAfterRegistration(t *testing.T) { + globalBefore, err := metrics.Registry.Gather() + require.NoError(t, err) + m := InitOperatorMetrics() + globalAfter, err := metrics.Registry.Gather() + require.NoError(t, err) + require.Len(t, globalAfter, len(globalBefore), "standby construction must not register metrics") + + registry := promcli.NewRegistry() + const name = "gpu_operator_reconciliation_status" + + metricFamilies, err := registry.Gather() + require.NoError(t, err) + for _, family := range metricFamilies { + require.NotEqual(t, name, family.GetName(), "standby must not expose reconciliation status") + } + + m.register(registry) + metricFamilies, err = registry.Gather() + require.NoError(t, err) + for _, family := range metricFamilies { + if family.GetName() == name { + require.Len(t, family.GetMetric(), 1) + require.Zero(t, family.GetMetric()[0].GetGauge().GetValue()) + return + } + } + t.Fatal("leader did not expose reconciliation status after registration") +} diff --git a/controllers/upgrade_controller_test.go b/controllers/upgrade_controller_test.go index 615424cdba..5deb48430d 100644 --- a/controllers/upgrade_controller_test.go +++ b/controllers/upgrade_controller_test.go @@ -168,8 +168,7 @@ func (f *fakeUpgradeStateManager) ApplyState(_ context.Context, _ *upgrade.Clust return nil } -// newTestOperatorMetrics builds the gauges the upgrade reconciler touches without -// registering them; InitOperatorMetrics panics when registered twice per binary. +// newTestOperatorMetrics builds only the gauges the upgrade reconciler touches. func newTestOperatorMetrics() *OperatorMetrics { newGauge := func(name string) promcli.Gauge { return promcli.NewGauge(promcli.GaugeOpts{Name: name}) diff --git a/deployments/gpu-operator/.helmignore b/deployments/gpu-operator/.helmignore index 50af031725..9ba67b3f8c 100644 --- a/deployments/gpu-operator/.helmignore +++ b/deployments/gpu-operator/.helmignore @@ -20,3 +20,5 @@ .idea/ *.tmproj .vscode/ +# Go unit tests for the chart +*_test.go diff --git a/deployments/gpu-operator/chart_test.go b/deployments/gpu-operator/chart_test.go new file mode 100644 index 0000000000..80dfcd76fd --- /dev/null +++ b/deployments/gpu-operator/chart_test.go @@ -0,0 +1,184 @@ +/** +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package gpuoperator + +import ( + "fmt" + "os/exec" + "strings" + "testing" + + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + policyv1 "k8s.io/api/policy/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/yaml" +) + +func TestOperatorRejectsInvalidReplicaCount(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm binary not found in PATH") + } + + for _, replicas := range []int{0, -1} { + t.Run(fmt.Sprint(replicas), func(t *testing.T) { + cmd := exec.Command("helm", "template", "gpu-operator", ".", "--values", "-") + cmd.Stdin = strings.NewReader(fmt.Sprintf("operator:\n replicaCount: %d\n", replicas)) + out, err := cmd.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(out), "operator.replicaCount must be at least 1") + }) + } +} + +// renderChart returns the operator Deployment and, if rendered, its PodDisruptionBudget. +func renderChart(t *testing.T, values string) (*appsv1.Deployment, *policyv1.PodDisruptionBudget) { + t.Helper() + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm binary not found in PATH") + } + + cmd := exec.Command("helm", "template", "gpu-operator", ".", "--namespace", "gpu-operator", "--values", "-") + cmd.Stdin = strings.NewReader(values) + var stderr strings.Builder + cmd.Stderr = &stderr + out, err := cmd.Output() + require.NoError(t, err, stderr.String()) + + var deployment *appsv1.Deployment + var pdb *policyv1.PodDisruptionBudget + for doc := range strings.SplitSeq(string(out), "\n---\n") { + var meta metav1.PartialObjectMetadata + require.NoError(t, yaml.Unmarshal([]byte(doc), &meta)) + if meta.Name != "gpu-operator" { + continue + } + switch meta.Kind { + case "Deployment": + deployment = &appsv1.Deployment{} + require.NoError(t, yaml.Unmarshal([]byte(doc), deployment)) + case "PodDisruptionBudget": + pdb = &policyv1.PodDisruptionBudget{} + require.NoError(t, yaml.Unmarshal([]byte(doc), pdb)) + } + } + require.NotNil(t, deployment) + return deployment, pdb +} + +func requireSoftPodAntiAffinity(t *testing.T, deployment *appsv1.Deployment) { + t.Helper() + podAntiAffinity := deployment.Spec.Template.Spec.Affinity.PodAntiAffinity + require.NotNil(t, podAntiAffinity) + require.Empty(t, podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + require.Len(t, podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, 1) + term := podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm + require.Equal(t, "kubernetes.io/hostname", term.TopologyKey) + require.Equal(t, deployment.Spec.Selector, term.LabelSelector) +} + +func TestOperatorHighAvailability(t *testing.T) { + testCases := map[string]struct { + values string + check func(t *testing.T, deployment *appsv1.Deployment, pdb *policyv1.PodDisruptionBudget) + }{ + "defaults render one replica, no anti-affinity and no PDB": { + check: func(t *testing.T, deployment *appsv1.Deployment, pdb *policyv1.PodDisruptionBudget) { + require.Equal(t, int32(1), *deployment.Spec.Replicas) + require.Nil(t, deployment.Spec.Template.Spec.Affinity.PodAntiAffinity) + require.Nil(t, pdb) + }, + }, + "two replicas render soft anti-affinity and a PDB": { + values: "operator: {replicaCount: 2}", + check: func(t *testing.T, deployment *appsv1.Deployment, pdb *policyv1.PodDisruptionBudget) { + require.NotNil(t, deployment.Spec.Replicas) + require.Equal(t, int32(2), *deployment.Spec.Replicas) + requireSoftPodAntiAffinity(t, deployment) + require.NotNil(t, pdb) + require.Equal(t, intstr.FromInt32(1), *pdb.Spec.MinAvailable) + require.NotNil(t, pdb.Spec.UnhealthyPodEvictionPolicy) + require.Equal(t, policyv1.AlwaysAllow, *pdb.Spec.UnhealthyPodEvictionPolicy) + require.Equal(t, deployment.Spec.Selector, pdb.Spec.Selector) + }, + }, + "two replicas with user-provided affinity render both the user's node affinity and HA anti-affinity": { + // This is the user's intended node affinity + values: ` +operator: + replicaCount: 2 + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - {key: example.com/pool, operator: Exists} +`, + check: func(t *testing.T, deployment *appsv1.Deployment, _ *policyv1.PodDisruptionBudget) { + require.NotNil(t, deployment.Spec.Replicas) + require.Equal(t, int32(2), *deployment.Spec.Replicas) + // Check if the rendered helm template includes user's intended node affinity + terms := deployment.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution + require.Len(t, terms, 1) + require.Equal(t, "example.com/pool", terms[0].Preference.MatchExpressions[0].Key) + // Check if the rendered helm template includes the HA node affinity + requireSoftPodAntiAffinity(t, deployment) + }, + }, + "two replicas with HA keys missing (--reuse-values from an older chart) render soft anti-affinity and a PDB": { + // A null value deletes the key from the merged values. This copies the + // values that an older release has: the release has no podAntiAffinity + // or podDisruptionBudget keys. + values: ` +operator: + replicaCount: 2 + podAntiAffinity: null + podDisruptionBudget: null +`, + check: func(t *testing.T, deployment *appsv1.Deployment, pdb *policyv1.PodDisruptionBudget) { + require.NotNil(t, deployment.Spec.Replicas) + require.Equal(t, int32(2), *deployment.Spec.Replicas) + requireSoftPodAntiAffinity(t, deployment) + require.NotNil(t, pdb) + require.Equal(t, deployment.Spec.Selector, pdb.Spec.Selector) + }, + }, + "two replicas with HA features disabled render no anti-affinity and no PDB": { + values: ` +operator: + replicaCount: 2 + podAntiAffinity: {enabled: false} + podDisruptionBudget: {enabled: false} +`, + check: func(t *testing.T, deployment *appsv1.Deployment, pdb *policyv1.PodDisruptionBudget) { + require.NotNil(t, deployment.Spec.Replicas) + require.Equal(t, int32(2), *deployment.Spec.Replicas) + require.Nil(t, deployment.Spec.Template.Spec.Affinity.PodAntiAffinity) + require.Nil(t, pdb) + }, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + deployment, pdb := renderChart(t, tc.values) + tc.check(t, deployment, pdb) + }) + } +} diff --git a/deployments/gpu-operator/templates/_helpers.tpl b/deployments/gpu-operator/templates/_helpers.tpl index 83ecd97cf0..65b101c959 100644 --- a/deployments/gpu-operator/templates/_helpers.tpl +++ b/deployments/gpu-operator/templates/_helpers.tpl @@ -56,6 +56,16 @@ app.kubernetes.io/managed-by: {{ include "gpu-operator.name" . }} {{- end }} {{- end -}} +{{/* +Selector labels for the operator pods. The Deployment selector, the pod +anti-affinity and the PodDisruptionBudget use these labels. Do not add +labels that change between releases: the Deployment selector is immutable. +*/}} +{{- define "gpu-operator.selectorLabels" -}} +app.kubernetes.io/component: "gpu-operator" +app: "gpu-operator" +{{- end -}} + {{- define "gpu-operator.matchLabels" -}} app.kubernetes.io/name: {{ include "gpu-operator.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/deployments/gpu-operator/templates/operator.yaml b/deployments/gpu-operator/templates/operator.yaml index 530f49e3bc..9bb4ca94ed 100644 --- a/deployments/gpu-operator/templates/operator.yaml +++ b/deployments/gpu-operator/templates/operator.yaml @@ -8,17 +8,15 @@ metadata: app.kubernetes.io/component: "gpu-operator" nvidia.com/gpu-driver-upgrade-drain.skip: "true" spec: - replicas: 1 + replicas: {{ .Values.operator.replicaCount | default 1 }} selector: matchLabels: - app.kubernetes.io/component: "gpu-operator" - app: "gpu-operator" + {{- include "gpu-operator.selectorLabels" . | nindent 6 }} template: metadata: labels: {{- include "gpu-operator.labels" . | nindent 8 }} - app.kubernetes.io/component: "gpu-operator" - app: "gpu-operator" + {{- include "gpu-operator.selectorLabels" . | nindent 8 }} nvidia.com/gpu-driver-upgrade-drain.skip: "true" annotations: {{- toYaml .Values.operator.annotations | nindent 8 }} @@ -55,6 +53,9 @@ spec: {{- if .renewDeadline }} - --leader-lease-renew-deadline={{ .renewDeadline }} {{- end }} + {{- if .warmup }} + - --leader-election-warmup + {{- end }} {{- end }} {{- if .Values.operator.logging.develMode }} - --zap-devel @@ -112,9 +113,24 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.operator.affinity }} + {{- $affinity := .Values.operator.affinity | default dict }} + {{- $podAntiAffinity := .Values.operator.podAntiAffinity | default dict }} + {{- $addPodAntiAffinity := and (dig "enabled" true $podAntiAffinity) (gt (int .Values.operator.replicaCount) 1) (not (hasKey $affinity "podAntiAffinity")) }} + {{- if or $affinity $addPodAntiAffinity }} affinity: + {{- with $affinity }} {{- toYaml . | nindent 8 }} + {{- end }} + {{- if $addPodAntiAffinity }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + {{- include "gpu-operator.selectorLabels" . | nindent 18 }} + {{- end }} {{- end }} {{- with .Values.operator.tolerations }} tolerations: diff --git a/deployments/gpu-operator/templates/poddisruptionbudget.yaml b/deployments/gpu-operator/templates/poddisruptionbudget.yaml new file mode 100644 index 0000000000..1b19eaa508 --- /dev/null +++ b/deployments/gpu-operator/templates/poddisruptionbudget.yaml @@ -0,0 +1,17 @@ +{{- $pdb := .Values.operator.podDisruptionBudget | default dict }} +{{- if and (dig "enabled" true $pdb) (gt (int .Values.operator.replicaCount) 1) }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: gpu-operator + namespace: {{ .Release.Namespace }} + labels: + {{- include "gpu-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: "gpu-operator" +spec: + minAvailable: 1 + unhealthyPodEvictionPolicy: AlwaysAllow + selector: + matchLabels: + {{- include "gpu-operator.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/deployments/gpu-operator/templates/validations.yaml b/deployments/gpu-operator/templates/validations.yaml index bb4677ebbe..b278bd12bf 100644 --- a/deployments/gpu-operator/templates/validations.yaml +++ b/deployments/gpu-operator/templates/validations.yaml @@ -28,3 +28,7 @@ {{ fail "gpuCluster.deployCR=true requires a Kubernetes cluster that supports Dynamic Resource Allocation (no resource.k8s.io DeviceClass API is served). When rendering offline with 'helm template', pass --api-versions resource.k8s.io/v1/DeviceClass" }} {{- end }} {{- end }} + +{{- if and (hasKey .Values.operator "replicaCount") (lt (int .Values.operator.replicaCount) 1) }} +{{- fail "operator.replicaCount must be at least 1" }} +{{- end }} diff --git a/deployments/gpu-operator/values.yaml b/deployments/gpu-operator/values.yaml index 80d74ab7aa..bb8f41d82e 100644 --- a/deployments/gpu-operator/values.yaml +++ b/deployments/gpu-operator/values.yaml @@ -84,8 +84,10 @@ operator: #version: "" imagePullPolicy: IfNotPresent imagePullSecrets: [] - # leaderElection: - # renewDeadline: 60s + replicaCount: 1 + leaderElection: + # renewDeadline: 60s + warmup: false env: [] priorityClassName: system-node-critical runtimeClass: nvidia @@ -113,6 +115,13 @@ operator: - key: "node-role.kubernetes.io/control-plane" operator: In values: [""] + podAntiAffinity: + enabled: true + podDisruptionBudget: + # Created when operator.replicaCount > 1. + # If only one node can run operator pods, this budget can block draining that node. + # Set operator.replicaCount to 1 or disable this podDisruptionBudget to allow draining. + enabled: true logging: # Zap time encoding (one of 'epoch', 'millis', 'nano', 'iso8601', 'rfc3339' or 'rfc3339nano') timeEncoding: epoch diff --git a/hack/must-gather.sh b/hack/must-gather.sh index 99597cecb0..db6127d1fc 100755 --- a/hack/must-gather.sh +++ b/hack/must-gather.sh @@ -59,9 +59,9 @@ else fi echo "Get the operator namespaces" -OPERATOR_POD_NAME=$($K get pods -lapp=gpu-operator -oname -A) +OPERATOR_PODS=$($K get pods -lapp=gpu-operator -oname -A) -if [ -z "$OPERATOR_POD_NAME" ]; then +if [ -z "$OPERATOR_PODS" ]; then echo "FATAL: could not find the GPU Operator Pod ..." exit 1 fi @@ -217,29 +217,38 @@ echo "# Operator Pod" echo "#" echo -echo "Get the GPU Operator Pod (status)" -$K get "${OPERATOR_POD_NAME}" \ +echo "Get the GPU Operator Pods (status)" +$K get pods -lapp=gpu-operator \ -owide \ -n "${OPERATOR_NAMESPACE}" \ > "${ARTIFACT_DIR}/gpu_operator_pod.status" -echo "Get the GPU Operator Pod (yaml)" -$K get "${OPERATOR_POD_NAME}" \ +echo "Get the GPU Operator Pods (yaml)" +$K get pods -lapp=gpu-operator \ -oyaml \ -n "${OPERATOR_NAMESPACE}" \ > "${ARTIFACT_DIR}/gpu_operator_pod.yaml" -echo "Get the GPU Operator Pod logs" -$K logs "${OPERATOR_POD_NAME}" \ +echo "Get the Leases in ${OPERATOR_NAMESPACE}" +$K get leases -oyaml \ -n "${OPERATOR_NAMESPACE}" \ - --timestamps \ - > "${ARTIFACT_DIR}/gpu_operator_pod.log" + > "${ARTIFACT_DIR}/gpu_operator_namespace_leases.yaml" -$K logs "${OPERATOR_POD_NAME}" \ - -n "${OPERATOR_NAMESPACE}" \ - --timestamps \ - --previous \ - > "${ARTIFACT_DIR}/gpu_operator_pod.previous.log" +for operator_pod in ${OPERATOR_PODS}; do + operator_pod_name=$(echo "$operator_pod" | cut -d/ -f2) + + echo "Get the GPU Operator Pod logs ($operator_pod_name)" + $K logs "${operator_pod}" \ + -n "${OPERATOR_NAMESPACE}" \ + --timestamps \ + > "${ARTIFACT_DIR}/gpu_operator_pod_${operator_pod_name}.log" + + $K logs "${operator_pod}" \ + -n "${OPERATOR_NAMESPACE}" \ + --timestamps \ + --previous \ + > "${ARTIFACT_DIR}/gpu_operator_pod_${operator_pod_name}.previous.log" +done echo "" echo "#" @@ -272,7 +281,7 @@ do fi pod_name=$(echo "$pod" | cut -d/ -f2) - if [ "${pod}" == "${OPERATOR_POD_NAME}" ]; then + if echo "${OPERATOR_PODS}" | grep -Fqx "${pod}"; then echo "Skipping operator pod $pod_name ..." continue fi