Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 4 additions & 8 deletions charts/openstack-hypervisor-operator/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,6 @@ controllerManager:
- --metrics-bind-address=:8443
- --leader-elect
- --health-probe-bind-address=:8081
- --certificate-namespace=$(CERTIFICATE_NAMESPACE)
- --certificate-issuer-name=$(CERTIFICATE_ISSUER_NAME)
- --label-selector=$(LABEL_SELECTOR)
- --agent-namespaces=$(AGENT_NAMESPACES)
- --eviction-concurrency=$(EVICTION_CONCURRENCY)
- --eviction-trait-concurrency=$(EVICTION_TRAIT_CONCURRENCY)
containerSecurityContext:
allowPrivilegeEscalation: false
capabilities:
Expand All @@ -23,8 +17,10 @@ controllerManager:
# Default maximum number of VM migrations to run concurrently while draining a hypervisor.
# Keep this quoted because it is rendered as an environment variable.
evictionConcurrency: "1"
# Optional comma-separated TRAIT=N overrides for per-host migration concurrency.
# The lowest matching trait limit wins, e.g. "CUSTOM_HANA_EXCLUSIVE_HOST=1,CUSTOM_FOO=2".
# Optional JSON object with integer per-host migration limits >= 1.
# Example: '{"CUSTOM_HANA_EXCLUSIVE_HOST":1,"CUSTOM_FOO":2}'. Empty text clears the map.
# Trait names must be nonempty without surrounding whitespace. The lowest matching limit wins.
# CLI flags repeat --eviction-trait-concurrency=TRAIT:N to merge keys. Empty CLI input is invalid.
evictionTraitConcurrency: ""
osAuthUrl: ""
osProjectDomainName: ""
Expand Down
89 changes: 89 additions & 0 deletions cmd/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
SPDX-FileCopyrightText: Copyright 2025 SAP SE or an SAP affiliate company and cobaltcore-dev contributors
SPDX-License-Identifier: Apache-2.0

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"encoding/json"
"reflect"
"strings"

"github.com/go-viper/mapstructure/v2"
"github.com/knadh/koanf/providers/env"
"github.com/knadh/koanf/v2"

"github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/config"
"github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/controller"
"github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/controller/eviction"
)

type options struct {
config.Shared `koanf:",squash"`
Certificate controller.CertificateOptions `koanf:",squash"`
Eviction eviction.Options `koanf:",squash"`
}

// loadOptions reads the operator settings from the environment, starting from
// the built-in defaults for settings whose env var is not set. Each module
// owns its options struct and defaults. A new tagged field needs no loader
// changes or flag registration.
func loadOptions() (options, error) {
cfg := options{
Shared: config.DefaultOptions(),
Certificate: controller.DefaultCertificateOptions(),
Eviction: eviction.DefaultOptions(),
}
k := koanf.New(".")

// Environment variables use the same names as before, lowercased as keys.
if err := k.Load(env.ProviderWithValue("", ".", func(name, value string) (string, any) {
return strings.ToLower(name), value
}), nil); err != nil {
return cfg, err
}

// Decode lists and JSON maps by target type. Keep text unmarshaling for
// other types. Plain strings, including label selectors, stay unchanged.
decoderConfig := &mapstructure.DecoderConfig{
DecodeHook: mapstructure.ComposeDecodeHookFunc(
mapstructure.StringToSliceHookFunc(","),
stringToMapJSONHook,
mapstructure.TextUnmarshallerHookFunc(),
),
WeaklyTypedInput: true,
}

err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{
DecoderConfig: decoderConfig,
})
return cfg, err
}

// stringToMapJSONHook decodes JSON into a fresh map of the target type.
// An empty environment value clears the map. Errors do not change the target.
func stringToMapJSONHook(from, to reflect.Type, data any) (any, error) {
if from.Kind() != reflect.String || to.Kind() != reflect.Map {
return data, nil
}
result := reflect.New(to)
if text := data.(string); text != "" {
if err := json.Unmarshal([]byte(text), result.Interface()); err != nil {
return nil, err
}
}
return result.Elem().Interface(), nil
}
104 changes: 29 additions & 75 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,14 @@ package main
import (
"crypto/sha256"
"crypto/tls"
"errors"
"flag"
"fmt"
"os"
gruntime "runtime"
"strconv"
"strings"

"github.com/octago/sflags/gen/gflag"
"github.com/sapcc/go-api-declarations/bininfo"

// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"
Expand All @@ -55,7 +54,6 @@ import (
"github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/controller"
"github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/controller/eviction"
"github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/controller/ready"
"github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/global"
"github.com/cobaltcore-dev/openstack-hypervisor-operator/internal/logger"

// +kubebuilder:scaffold:imports
Expand Down Expand Up @@ -85,8 +83,6 @@ func main() {
var secureMetrics bool
var enableHTTP2 bool
var tlsOpts []func(*tls.Config)
var certificateNamespace string
var certificateIssuerName string

flag.BoolVar(&version, "version", false,
"Print the version and exit.")
Expand All @@ -100,23 +96,20 @@ func main() {
"If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.")
flag.BoolVar(&enableHTTP2, "enable-http2", false,
"If set, HTTP/2 will be enabled for the metrics and webhook servers")
flag.StringVar(&global.LabelSelector, "label-selector", "", "Label selector to filter watched resources (namely nodes).")

var agentNamespacesFlag string
flag.StringVar(&agentNamespacesFlag, "agent-namespaces", "",
"Comma-separated list of namespaces to search for agent pods (nova-compute, neutron) during offboarding.")

flag.IntVar(&global.EvictionConcurrency, "eviction-concurrency", 1,
"Default maximum number of VM migrations to run concurrently while draining a hypervisor. Defaults to 1 (serial).")

var evictionTraitConcurrencyFlag string
flag.StringVar(&evictionTraitConcurrencyFlag, "eviction-trait-concurrency", "",
"Comma-separated TRAIT=N overrides for per-host migration concurrency, e.g. "+
"CUSTOM_HANA_EXCLUSIVE_HOST=1,CUSTOM_FOO=2. The lowest matching trait limit wins.")
cfg, err := loadOptions()
if err != nil {
fmt.Fprintf(os.Stderr, "invalid configuration: %v\n", err)
os.Exit(1)
}

flag.StringVar(&certificateNamespace, "certificate-namespace", "monsoon3", "The namespace for the certificates. ")
flag.StringVar(&certificateIssuerName, "certificate-issuer-name", "nova-hypervisor-agents-ca-issuer",
"Name of the certificate issuer.")
// Register CLI flags from the env-filled structs so that explicit
// command-line arguments override environment values; the current
// (env-derived) field values become the flag defaults.
if err := gflag.ParseTo(&cfg, flag.CommandLine); err != nil {
fmt.Fprintf(os.Stderr, "invalid configuration: %v\n", err)
os.Exit(1)
}

opts := ctrlzap.Options{
Development: true,
Expand All @@ -137,62 +130,23 @@ func main() {
os.Exit(0)
}

if agentNamespacesFlag != "" {
// Deduplicate. Without this, repeated entries (e.g. from templated
// values) would cause redundant pod-list calls and inflate the
// AgentPodsEvicted condition message with duplicate pod names.
seen := map[string]struct{}{}
for ns := range strings.SplitSeq(agentNamespacesFlag, ",") {
ns = strings.TrimSpace(ns)
if ns == "" {
continue
}
if _, ok := seen[ns]; ok {
continue
}
seen[ns] = struct{}{}
global.AgentNamespaces = append(global.AgentNamespaces, ns)
}
}

if len(global.AgentNamespaces) == 0 {
setupLog.Error(errors.New("--agent-namespaces is required"), "invalid configuration")
os.Exit(1)
}
ctrl.SetLogger(ctrlzap.New(ctrlzap.UseFlagOptions(&opts)))

if global.EvictionConcurrency < 1 {
setupLog.Error(errors.New("--eviction-concurrency must be >= 1"), "invalid configuration")
if err := cfg.Validate(); err != nil {
setupLog.Error(err, "invalid configuration")
os.Exit(1)
}

for entry := range strings.SplitSeq(evictionTraitConcurrencyFlag, ",") {
entry = strings.TrimSpace(entry)
if entry == "" {
continue
}
trait, value, ok := strings.Cut(entry, "=")
trait = strings.TrimSpace(trait)
n, err := strconv.Atoi(strings.TrimSpace(value))
if !ok || trait == "" || err != nil || n < 1 {
setupLog.Error(errors.New("invalid --eviction-trait-concurrency entry: "+entry),
"invalid configuration", "expected", "TRAIT=N with N >= 1")
os.Exit(1)
}
global.EvictionTraitConcurrency[trait] = n
}

if certificateIssuerName == "" {
setupLog.Error(errors.New("certificate-issuer-name cannot be empty"), "invalid certificate issuer name")
if err := cfg.Eviction.Validate(); err != nil {
setupLog.Error(err, "invalid configuration")
os.Exit(1)
}

if certificateNamespace == "" {
setupLog.Error(errors.New("certificate-namespace cannot be empty"), "invalid certificate namespace")
if err := cfg.Certificate.Validate(); err != nil {
setupLog.Error(err, "invalid configuration")
os.Exit(1)
}

ctrl.SetLogger(ctrlzap.New(ctrlzap.UseFlagOptions(&opts)))

// if the enable-http2 flag is false (the default), http/2 should be disabled
// due to its vulnerabilities. More specifically, disabling http/2 will
// prevent from being vulnerable to the HTTP/2 Stream Cancellation and
Expand Down Expand Up @@ -260,9 +214,9 @@ func main() {
&policyv1.PodDisruptionBudget{}: maintenanceCacheConfig,
},
}
if global.LabelSelector != "" {
setupLog.Info("setting up cache with label selector", "selector", global.LabelSelector)
selector, err := labels.Parse(global.LabelSelector)
if cfg.LabelSelector != "" {
setupLog.Info("setting up cache with label selector", "selector", cfg.LabelSelector)
selector, err := labels.Parse(cfg.LabelSelector)
if err != nil {
setupLog.Error(err, "unable to parse label selector")
os.Exit(1)
Expand All @@ -273,7 +227,7 @@ func main() {

h := sha256.New()
h.Write([]byte(leaderElectionID)) // Seed it with something "unique" to the project
h.Write([]byte(global.LabelSelector))
h.Write([]byte(cfg.LabelSelector))
leaderElectionID = fmt.Sprintf("%x.cloud.sap", h.Sum(nil))
}

Expand Down Expand Up @@ -315,7 +269,7 @@ func main() {
if err = (&controller.HypervisorController{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
}).SetupWithManager(mgr, cfg.Shared); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Hypervisor")
os.Exit(1)
}
Expand All @@ -331,7 +285,7 @@ func main() {
if err = (&eviction.EvictionReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
}).SetupWithManager(mgr, cfg.Eviction); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Eviction")
os.Exit(1)
}
Expand All @@ -355,15 +309,15 @@ func main() {
if err = (&controller.NodeCertificateController{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr, certificateNamespace, certificateIssuerName); err != nil {
}).SetupWithManager(mgr, cfg.Certificate); err != nil {
setupLog.Error(err, "unable to create controller", "certificate", "NodeCertificate")
os.Exit(1)
}

if err = (&controller.GardenerNodeLifecycleController{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr, certificateNamespace); err != nil {
}).SetupWithManager(mgr, cfg.Certificate.Namespace); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "Maintenance")
os.Exit(1)
}
Expand Down
7 changes: 7 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,12 @@ go 1.27
require (
github.com/cert-manager/cert-manager v1.21.2
github.com/go-logr/logr v1.4.4
github.com/go-viper/mapstructure/v2 v2.4.0
github.com/gophercloud/gophercloud/v2 v2.15.0
github.com/gophercloud/utils/v2 v2.0.0-20260922174841-b4759ea0529b
github.com/knadh/koanf/providers/env v1.1.0
github.com/knadh/koanf/v2 v2.4.1
github.com/octago/sflags v0.3.1
github.com/onsi/ginkgo/v2 v2.33.1
github.com/onsi/gomega v1.44.0
github.com/sapcc/go-api-declarations v1.25.1
Expand Down Expand Up @@ -64,6 +68,9 @@ require (
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/knadh/koanf/maps v0.1.2 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
Expand Down
14 changes: 14 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAg
github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw=
github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/gofrs/uuid/v5 v5.5.1 h1:z1Ce19/JwNidXpy3tOQc3241lnJLKdKyq/xlNvlD4Ng=
Expand Down Expand Up @@ -115,6 +117,12 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo=
github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc=
github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY=
github.com/knadh/koanf/v2 v2.4.1 h1:MK2JDkLy4UGzEUkW1CCCKQftd9+M+NX8hAZ2axbf/jo=
github.com/knadh/koanf/v2 v2.4.1/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
Expand All @@ -125,6 +133,10 @@ github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo=
github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE=
github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand All @@ -133,6 +145,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/octago/sflags v0.3.1 h1:LW65z20iAQKteEyjsnnc+/lyoCUnIoRuAocggr6RB6A=
github.com/octago/sflags v0.3.1/go.mod h1:hVUkbnYwMU9kZiZJyOAIVN56YiVMMPxgJ46kRZ19jh0=
github.com/onsi/ginkgo/v2 v2.33.1 h1:62XfWJbEIjPSCSDYrsCPP/SHcmH7O0M8ne8zfqQ2iz4=
github.com/onsi/ginkgo/v2 v2.33.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44=
github.com/onsi/gomega v1.44.0 h1:eAiGl3Pw5jz5GQdDff0BcxYpAX1JxW8xD7mFUuwNfZQ=
Expand Down
Loading
Loading