Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/golang-checks.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ jobs:
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
with:
go-version: ${{ env.GOLANG_VERSION }}
- name: Install Helm
uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
Comment thread
aryangorwade marked this conversation as resolved.
Resolved
with:
version: v4.3.2
- name: Setup Go Proxy
id: setup-go-proxy
uses: nv-gha-runners/setup-artifactory-go-proxy@a15ec16ff434e8ca3b0f76919f902080e66ebc14 # main
Expand Down
6 changes: 6 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ After installation, the GPU Operator and its operands should be up and running.
Note:
To deploy the GPU Operator on OpenShift, follow the instructions in the [official documentation](https://docs.nvidia.com/datacenter/cloud-native/openshift/latest/steps-overview.html).

## Operator high availability

Helm installations run one operator replica by default. Increase `operator.replicaCount` to run multiple replicas with leader election. Unless explicitly disabled, the chart also adds soft pod anti-affinity and a PodDisruptionBudget (PDB) requiring at least one available replica.

OLM installations continue to deploy one operator replica and do not include the Helm chart's PDB or pod anti-affinity. Helm values do not configure OLM installations. Administrators can manually increase replicas and add anti-affinity through the installed ClusterServiceVersion (CSV), then create a PDB separately, following `deployments/gpu-operator/templates/poddisruptionbudget.yaml`.


## Product Documentation
For information on platform support and getting started, visit the official documentation [repository](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/overview.html).
Expand Down
33 changes: 23 additions & 10 deletions cmd/gpu-operator/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ func main() {
var probeAddr string
var pprofAddr string
var renewDeadline time.Duration
var leaderElectionWarmup bool

flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
flag.StringVar(&pprofAddr, "pprof-bind-address", "",
Expand All @@ -95,6 +96,9 @@ func main() {
"Only enabled when the --leader-elect flag is set. "+
"If undefined, the renew deadline defaults to the controller-runtime manager's default RenewDeadline. "+
"By setting this option, the LeaseDuration is also set as RenewDealine + 5s.")
flag.BoolVar(&leaderElectionWarmup, "leader-election-warmup", false,
"Enable informer cache warmup for the operator's standby replicas. "+
"Only enabled when the --leader-elect flag is set.")

opts := zap.Options{
StacktraceLevel: zapcore.PanicLevel,
Expand Down Expand Up @@ -126,16 +130,17 @@ func main() {
}

options := ctrl.Options{
Scheme: scheme,
Metrics: metricsOptions,
HealthProbeBindAddress: probeAddr,
PprofBindAddress: pprofAddr,
LeaderElection: enableLeaderElection,
LeaderElectionNamespace: leaderElectionNamespace,
LeaderElectionID: "53822513.nvidia.com",
WebhookServer: webhookServer,
Cache: operatorcache.Options(operatorNamespace),
NewCache: operatorcache.New,
Scheme: scheme,
Metrics: metricsOptions,
HealthProbeBindAddress: probeAddr,
PprofBindAddress: pprofAddr,
LeaderElection: enableLeaderElection,
LeaderElectionNamespace: leaderElectionNamespace,
LeaderElectionID: "53822513.nvidia.com",
LeaderElectionReleaseOnCancel: true,
WebhookServer: webhookServer,
Cache: operatorcache.Options(operatorNamespace),
NewCache: operatorcache.New,
}

if enableLeaderElection && int(renewDeadline) != 0 {
Expand All @@ -145,6 +150,10 @@ func main() {
options.LeaseDuration = &leaseDuration
}

if enableLeaderElection && leaderElectionWarmup {
Comment thread
aryangorwade marked this conversation as resolved.
Resolved
options.Controller.EnableWarmup = &leaderElectionWarmup
}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), options)
if err != nil {
setupLog.Error(err, "unable to start manager")
Expand Down Expand Up @@ -247,6 +256,10 @@ func main() {
setupLog.Error(err, "unable to set up ready check")
os.Exit(1)
}
if err := mgr.Add(operatorMetrics); err != nil {
setupLog.Error(err, "unable to add operator metrics registration")
os.Exit(1)
}

setupLog.Info("starting manager")
if err := mgr.Start(ctx); err != nil {
Expand Down
24 changes: 19 additions & 5 deletions controllers/operator_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package controllers

import (
"context"
"fmt"

promcli "github.com/prometheus/client_golang/prometheus"
Expand Down Expand Up @@ -66,8 +67,8 @@ const (
operatorMetricsNamespace = "gpu_operator"
)

// InitOperatorMetrics registers all GPU operator Prometheus metrics with the
// controller-runtime registry and returns the initialised OperatorMetrics.
// InitOperatorMetrics creates GPU operator Prometheus metrics. The metrics are
// registered after the manager becomes leader.
func InitOperatorMetrics() *OperatorMetrics {
m := &OperatorMetrics{
gpuNodesTotal: promcli.NewGauge(
Expand Down Expand Up @@ -195,8 +196,23 @@ func InitOperatorMetrics() *OperatorMetrics {
},
),
}
return m
}

// Start implements manager.Runnable. It registers metrics with the
// controller-runtime registry.
func (m *OperatorMetrics) Start(context.Context) error {
m.register(metrics.Registry)
return nil
}

metrics.Registry.MustRegister(
// NeedLeaderElection implements manager.LeaderElectionRunnable. Returning true
// makes the manager call Start only after this replica acquires the lease, so that a
// standby replica does not also expose reconciliation metrics.
func (m *OperatorMetrics) NeedLeaderElection() bool { return true }
Comment thread
aryangorwade marked this conversation as resolved.
Resolved

func (m *OperatorMetrics) register(registry promcli.Registerer) {
registry.MustRegister(
m.gpuNodesTotal,

m.reconciliationLastSuccess,
Expand All @@ -218,6 +234,4 @@ func InitOperatorMetrics() *OperatorMetrics {
m.upgradesFailed,
m.upgradesPending,
)

return m
}
55 changes: 55 additions & 0 deletions controllers/operator_metrics_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/

package controllers

import (
"testing"

promcli "github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)

func TestOperatorMetricsExposedOnlyAfterRegistration(t *testing.T) {
globalBefore, err := metrics.Registry.Gather()
require.NoError(t, err)
m := InitOperatorMetrics()
globalAfter, err := metrics.Registry.Gather()
require.NoError(t, err)
require.Len(t, globalAfter, len(globalBefore), "standby construction must not register metrics")

registry := promcli.NewRegistry()
const name = "gpu_operator_reconciliation_status"

metricFamilies, err := registry.Gather()
require.NoError(t, err)
for _, family := range metricFamilies {
require.NotEqual(t, name, family.GetName(), "standby must not expose reconciliation status")
}

m.register(registry)
metricFamilies, err = registry.Gather()
require.NoError(t, err)
for _, family := range metricFamilies {
if family.GetName() == name {
require.Len(t, family.GetMetric(), 1)
require.Zero(t, family.GetMetric()[0].GetGauge().GetValue())
return
}
}
t.Fatal("leader did not expose reconciliation status after registration")
}
3 changes: 1 addition & 2 deletions controllers/upgrade_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,7 @@ func (f *fakeUpgradeStateManager) ApplyState(_ context.Context, _ *upgrade.Clust
return nil
}

// newTestOperatorMetrics builds the gauges the upgrade reconciler touches without
// registering them; InitOperatorMetrics panics when registered twice per binary.
// newTestOperatorMetrics builds only the gauges the upgrade reconciler touches.
func newTestOperatorMetrics() *OperatorMetrics {
newGauge := func(name string) promcli.Gauge {
return promcli.NewGauge(promcli.GaugeOpts{Name: name})
Expand Down
2 changes: 2 additions & 0 deletions deployments/gpu-operator/.helmignore
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,5 @@
.idea/
*.tmproj
.vscode/
# Go unit tests for the chart
*_test.go
184 changes: 184 additions & 0 deletions deployments/gpu-operator/chart_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
/**
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
**/

package gpuoperator

import (
"fmt"
"os/exec"
"strings"
"testing"

"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
policyv1 "k8s.io/api/policy/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/yaml"
)

func TestOperatorRejectsInvalidReplicaCount(t *testing.T) {
if _, err := exec.LookPath("helm"); err != nil {
t.Skip("helm binary not found in PATH")
}

for _, replicas := range []int{0, -1} {
t.Run(fmt.Sprint(replicas), func(t *testing.T) {
cmd := exec.Command("helm", "template", "gpu-operator", ".", "--values", "-")
cmd.Stdin = strings.NewReader(fmt.Sprintf("operator:\n replicaCount: %d\n", replicas))
out, err := cmd.CombinedOutput()
require.Error(t, err)
require.Contains(t, string(out), "operator.replicaCount must be at least 1")
})
}
}

// renderChart returns the operator Deployment and, if rendered, its PodDisruptionBudget.
func renderChart(t *testing.T, values string) (*appsv1.Deployment, *policyv1.PodDisruptionBudget) {
t.Helper()
if _, err := exec.LookPath("helm"); err != nil {
t.Skip("helm binary not found in PATH")
}

cmd := exec.Command("helm", "template", "gpu-operator", ".", "--namespace", "gpu-operator", "--values", "-")
cmd.Stdin = strings.NewReader(values)
var stderr strings.Builder
cmd.Stderr = &stderr
out, err := cmd.Output()
require.NoError(t, err, stderr.String())

var deployment *appsv1.Deployment
var pdb *policyv1.PodDisruptionBudget
for doc := range strings.SplitSeq(string(out), "\n---\n") {
var meta metav1.PartialObjectMetadata
require.NoError(t, yaml.Unmarshal([]byte(doc), &meta))
if meta.Name != "gpu-operator" {
continue
}
switch meta.Kind {
case "Deployment":
deployment = &appsv1.Deployment{}
require.NoError(t, yaml.Unmarshal([]byte(doc), deployment))
case "PodDisruptionBudget":
pdb = &policyv1.PodDisruptionBudget{}
require.NoError(t, yaml.Unmarshal([]byte(doc), pdb))
}
}
require.NotNil(t, deployment)
return deployment, pdb
}

func requireSoftPodAntiAffinity(t *testing.T, deployment *appsv1.Deployment) {
t.Helper()
podAntiAffinity := deployment.Spec.Template.Spec.Affinity.PodAntiAffinity
require.NotNil(t, podAntiAffinity)
require.Empty(t, podAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution)
require.Len(t, podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, 1)
term := podAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm
require.Equal(t, "kubernetes.io/hostname", term.TopologyKey)
require.Equal(t, deployment.Spec.Selector, term.LabelSelector)
}

func TestOperatorHighAvailability(t *testing.T) {
testCases := map[string]struct {
values string
check func(t *testing.T, deployment *appsv1.Deployment, pdb *policyv1.PodDisruptionBudget)
}{
"defaults render one replica, no anti-affinity and no PDB": {
check: func(t *testing.T, deployment *appsv1.Deployment, pdb *policyv1.PodDisruptionBudget) {
require.Equal(t, int32(1), *deployment.Spec.Replicas)
require.Nil(t, deployment.Spec.Template.Spec.Affinity.PodAntiAffinity)
require.Nil(t, pdb)
},
},
"two replicas render soft anti-affinity and a PDB": {
values: "operator: {replicaCount: 2}",
check: func(t *testing.T, deployment *appsv1.Deployment, pdb *policyv1.PodDisruptionBudget) {
require.NotNil(t, deployment.Spec.Replicas)
require.Equal(t, int32(2), *deployment.Spec.Replicas)
requireSoftPodAntiAffinity(t, deployment)
require.NotNil(t, pdb)
require.Equal(t, intstr.FromInt32(1), *pdb.Spec.MinAvailable)
require.NotNil(t, pdb.Spec.UnhealthyPodEvictionPolicy)
require.Equal(t, policyv1.AlwaysAllow, *pdb.Spec.UnhealthyPodEvictionPolicy)
require.Equal(t, deployment.Spec.Selector, pdb.Spec.Selector)
},
},
"two replicas with user-provided affinity render both the user's node affinity and HA anti-affinity": {
// This is the user's intended node affinity
values: `
operator:
replicaCount: 2
affinity:
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
preference:
matchExpressions:
- {key: example.com/pool, operator: Exists}
`,
check: func(t *testing.T, deployment *appsv1.Deployment, _ *policyv1.PodDisruptionBudget) {
require.NotNil(t, deployment.Spec.Replicas)
require.Equal(t, int32(2), *deployment.Spec.Replicas)
// Check if the rendered helm template includes user's intended node affinity
terms := deployment.Spec.Template.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution
require.Len(t, terms, 1)
require.Equal(t, "example.com/pool", terms[0].Preference.MatchExpressions[0].Key)
// Check if the rendered helm template includes the HA node affinity
requireSoftPodAntiAffinity(t, deployment)
},
},
"two replicas with HA keys missing (--reuse-values from an older chart) render soft anti-affinity and a PDB": {
// A null value deletes the key from the merged values. This copies the
// values that an older release has: the release has no podAntiAffinity
// or podDisruptionBudget keys.
values: `
operator:
replicaCount: 2
podAntiAffinity: null
podDisruptionBudget: null
`,
check: func(t *testing.T, deployment *appsv1.Deployment, pdb *policyv1.PodDisruptionBudget) {
require.NotNil(t, deployment.Spec.Replicas)
require.Equal(t, int32(2), *deployment.Spec.Replicas)
requireSoftPodAntiAffinity(t, deployment)
require.NotNil(t, pdb)
require.Equal(t, deployment.Spec.Selector, pdb.Spec.Selector)
},
},
"two replicas with HA features disabled render no anti-affinity and no PDB": {
values: `
operator:
replicaCount: 2
podAntiAffinity: {enabled: false}
podDisruptionBudget: {enabled: false}
`,
check: func(t *testing.T, deployment *appsv1.Deployment, pdb *policyv1.PodDisruptionBudget) {
require.NotNil(t, deployment.Spec.Replicas)
require.Equal(t, int32(2), *deployment.Spec.Replicas)
require.Nil(t, deployment.Spec.Template.Spec.Affinity.PodAntiAffinity)
require.Nil(t, pdb)
},
},
}

for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
deployment, pdb := renderChart(t, tc.values)
tc.check(t, deployment, pdb)
})
}
}
Loading
Loading