From 56af7bae42aeb8a56dc069a03aeec8ab2228b6ce Mon Sep 17 00:00:00 2001 From: kerthcet Date: Thu, 22 Oct 2026 17:50:44 +0100 Subject: [PATCH 01/21] support kubernetes as a new provider Signed-off-by: kerthcet --- cmd/main.go | 24 +- config/default/kustomization.yaml | 3 + config/kubernetes-provider/clusters.yaml | 12 + config/kubernetes-provider/kustomization.yaml | 16 + config/kubernetes-provider/remote-rbac.yaml | 36 + config/manager/manager.yaml | 19 +- config/rbac/role.yaml | 1 + config/samples/nodepool.yaml | 5 + docs/architecture.md | 25 +- go.mod | 27 +- go.sum | 54 +- internal/controller/nodeclaim_controller.go | 2 +- internal/controller/pod_placement_helpers.go | 2 +- internal/webhook/v1/pod_webhook.go | 21 +- internal/webhook/v1/pod_webhook_test.go | 43 ++ pkg/provider/aws/aws.go | 4 +- pkg/provider/aws/aws_test.go | 4 +- pkg/provider/kubernetes/config.go | 125 ++++ pkg/provider/kubernetes/eks.go | 151 +++++ pkg/provider/kubernetes/karpenter.go | 143 ++++ pkg/provider/kubernetes/kubernetes.go | 629 ++++++++++++++++++ pkg/provider/kubernetes/kubernetes_test.go | 407 ++++++++++++ pkg/provider/kubernetes/node_autoscaler.go | 63 ++ pkg/provider/kubernetes/pod.go | 239 +++++++ pkg/provider/kubernetes/provision_test.go | 348 ++++++++++ pkg/provider/kubernetes/wait.go | 74 +++ pkg/provider/modal/modal.go | 4 +- pkg/provider/modal/modal_test.go | 4 +- pkg/provider/provider.go | 56 +- pkg/provider/runpod/runpod.go | 2 +- pkg/provider/runpod/runpod_test.go | 2 +- pkg/util/pod.go | 10 +- pkg/util/pod_test.go | 6 +- pkg/vnode/env.go | 39 +- pkg/vnode/env_test.go | 70 +- pkg/vnode/handler.go | 60 +- pkg/vnode/handler_test.go | 35 + pkg/vnode/status.go | 15 + 38 files changed, 2650 insertions(+), 130 deletions(-) create mode 100644 config/kubernetes-provider/clusters.yaml create mode 100644 config/kubernetes-provider/kustomization.yaml create mode 100644 config/kubernetes-provider/remote-rbac.yaml create mode 100644 pkg/provider/kubernetes/config.go create mode 100644 pkg/provider/kubernetes/eks.go create mode 100644 pkg/provider/kubernetes/karpenter.go create mode 100644 pkg/provider/kubernetes/kubernetes.go create mode 100644 pkg/provider/kubernetes/kubernetes_test.go create mode 100644 pkg/provider/kubernetes/node_autoscaler.go create mode 100644 pkg/provider/kubernetes/pod.go create mode 100644 pkg/provider/kubernetes/provision_test.go create mode 100644 pkg/provider/kubernetes/wait.go diff --git a/cmd/main.go b/cmd/main.go index d567e7a..5d39f1c 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -57,6 +57,7 @@ import ( "github.com/InftyAI/Nebula/pkg/provider" awsprovider "github.com/InftyAI/Nebula/pkg/provider/aws" "github.com/InftyAI/Nebula/pkg/provider/fake" + kubeprovider "github.com/InftyAI/Nebula/pkg/provider/kubernetes" "github.com/InftyAI/Nebula/pkg/provider/modal" "github.com/InftyAI/Nebula/pkg/provider/runpod" "github.com/InftyAI/Nebula/pkg/vnode" @@ -109,6 +110,7 @@ func main() { var costLabels string var kubeletServingTLSBootstrap bool var providers string + var kubernetesClusters string var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ "Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.") @@ -151,6 +153,10 @@ func main() { "x509: certificate signed by unknown authority. Off by default because it needs the "+ "RBAC to impersonate one virtual node identity — the signer signs for nobody else "+ "(see addServingCertificateBootstrap).") + flag.StringVar(&kubernetesClusters, "kubernetes-clusters", "", + "Path to the kubernetes provider's cluster table in the container, normally "+ + "/etc/nebula/kubernetes/clusters.yaml mounted from the kubernetes-clusters ConfigMap. "+ + "Empty leaves the kubernetes provider unregistered.") flag.StringVar(&providers, "providers", strings.Join(knownProviders, ","), "Comma-separated production providers to enable; all by default. A listed provider is "+ "skipped if it fails to initialize. Drain a provider before dropping it: nothing "+ @@ -327,7 +333,7 @@ func main() { // startup already sees its provider. The manager's client backs the AWS region // source (regions are read from NodePools at call time, not env), so it is // threaded in; the client is only queried at runtime, after the cache has synced. - registerProviders(context.Background(), mgr.GetClient(), enabledProviders) + registerProviders(context.Background(), mgr.GetClient(), enabledProviders, kubernetesClusters, attribution) // One shared failover blocklist, written by the virtual kubelet handlers on a // Provision failure and read by the placement controller to skip a candidate @@ -528,7 +534,7 @@ func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string, servingTLSBoots // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,resourceNames=nebula-kubelet-serving,verbs=delete;get // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests/approval,resourceNames=nebula-kubelet-serving,verbs=update // +kubebuilder:rbac:groups=certificates.k8s.io,resources=signers,resourceNames=kubernetes.io/kubelet-serving,verbs=approve -// +kubebuilder:rbac:groups="",resources=users,resourceNames={"system:node:nebula-aws","system:node:nebula-modal","system:node:nebula-runpod","system:node:nebula-fake"},verbs=impersonate +// +kubebuilder:rbac:groups="",resources=users,resourceNames={"system:node:nebula-aws","system:node:nebula-modal","system:node:nebula-runpod","system:node:nebula-kubernetes","system:node:nebula-fake"},verbs=impersonate // +kubebuilder:rbac:groups="",resources=groups,resourceNames="system:nodes",verbs=impersonate // addServingCertificateBootstrap requests a trusted serving certificate for the kubelet @@ -608,7 +614,8 @@ func setupVirtualNodes(mgr ctrl.Manager, blocklist vnode.Blocklister, kubeletSrv // crash loop. // // Only providers in enabled are attempted (see parseProviders). -func registerProviders(ctx context.Context, c client.Client, enabled map[string]bool) { +func registerProviders(ctx context.Context, c client.Client, enabled map[string]bool, + kubernetesClusters string, costLabels []string) { register := func(name string, build func() (provider.Provider, error)) { if !enabled[name] { setupLog.Info("provider disabled by --providers", "provider", name) @@ -639,6 +646,13 @@ func registerProviders(ctx context.Context, c client.Client, enabled map[string] return runpod.NewSDKClient(ctx) }) + register(provider.ProviderKubernetes, func() (provider.Provider, error) { + if kubernetesClusters == "" { + return nil, errors.New("no --kubernetes-clusters file") + } + return kubeprovider.NewFromFile(ctx, kubernetesClusters, costLabels) + }) + // The fake provider is an in-memory backend used only by the e2e suite to // exercise the full control-plane loop without cloud credentials. It ships in // the binary but registers ONLY when explicitly enabled, so it can never place @@ -652,7 +666,9 @@ func registerProviders(ctx context.Context, c client.Client, enabled map[string] // knownProviders are the names --providers accepts, one per register call above. The fake // provider is not among them: it stays gated on its env var alone. -var knownProviders = []string{provider.ProviderModal, provider.ProviderAWS, provider.ProviderRunPod} +var knownProviders = []string{ + provider.ProviderModal, provider.ProviderAWS, provider.ProviderRunPod, provider.ProviderKubernetes, +} // parseProviders turns --providers into the enabled set. An unknown name is an error rather // than ignored, so a typo cannot silently leave a provider off. diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 26c5452..5ab1745 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -22,6 +22,9 @@ resources: # the manager uses the CSVs embedded in its binary. Uncommenting this also needs # the catalog env var and volume in config/manager/manager.yaml. # - ../catalog +# The kubernetes provider's cluster table (config/kubernetes-provider/clusters.yaml). Off: +# without it the kubernetes provider is not registered. Fill in clusters.yaml first. +# - ../kubernetes-provider # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml - ../webhook diff --git a/config/kubernetes-provider/clusters.yaml b/config/kubernetes-provider/clusters.yaml new file mode 100644 index 0000000..88ddfd5 --- /dev/null +++ b/config/kubernetes-provider/clusters.yaml @@ -0,0 +1,12 @@ +# The kubernetes provider's cluster table (pkg/provider/kubernetes.Config), rendered into the +# manager's ConfigMap by this directory's kustomization. Read once at startup: restart the +# manager after a change. No credentials go here: each EKS cluster is reached with Nebula's +# own AWS identity. +clusters: + # The host cluster itself, the first target to verify the provider against. + - name: my-cluster # the EKS cluster's own name + provider: eks + region: us-east-1 # the EKS cluster's AWS region + geography: us # set explicitly; a NodePool's regions match either this or region + # roleARN: arn:aws:iam::123456789012:role/nebula-remote # only for another account + # nodeAutoscaler: karpenter # what adds nodes; the default and only one supported today diff --git a/config/kubernetes-provider/kustomization.yaml b/config/kubernetes-provider/kustomization.yaml new file mode 100644 index 0000000..b6a43af --- /dev/null +++ b/config/kubernetes-provider/kustomization.yaml @@ -0,0 +1,16 @@ +# Renders clusters.yaml into the ConfigMap the manager mounts at /etc/nebula/kubernetes. +# Included by config/default, which applies the `nebula-` prefix and `nebula-system` +# namespace and rewrites the Deployment's `kubernetes-clusters` reference to match. +# +# remote-rbac.yaml is deliberately NOT a resource here: it belongs in each target cluster. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +configMapGenerator: +- name: kubernetes-clusters + files: + - clusters.yaml + +generatorOptions: + # A stable name, so the volume reference and `kubectl edit` keep working. + disableNameSuffixHash: true diff --git a/config/kubernetes-provider/remote-rbac.yaml b/config/kubernetes-provider/remote-rbac.yaml new file mode 100644 index 0000000..5055e90 --- /dev/null +++ b/config/kubernetes-provider/remote-rbac.yaml @@ -0,0 +1,36 @@ +# What Nebula may do in a target cluster, the host cluster included. Apply it in every +# target cluster, and map Nebula's IAM identity to the "nebula" group with an EKS access +# entry: +# +# aws eks create-access-entry --cluster-name \ +# --principal-arn --kubernetes-groups nebula +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: nebula-remote +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update", "delete"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "create"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: nebula-remote +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nebula-remote +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: nebula diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 0543b5a..830bc0e 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -68,6 +68,9 @@ spec: # only way to turn AWS off, which registers even without credentials. Drain a # provider first: once dropped, nothing terminates its running instances. # - --providers=runpod + # The kubernetes provider's cluster table, from the optional ConfigMap mounted below. + # Absent ConfigMap = no file = the provider is skipped at registration. + - --kubernetes-clusters=/etc/nebula/kubernetes/clusters.yaml image: controller:latest name: manager imagePullPolicy: IfNotPresent @@ -189,14 +192,22 @@ spec: requests: cpu: "1" memory: 1Gi - # Empty, not absent: config/default appends the cert mounts here with a JSON - # patch (`.../volumeMounts/-`), which needs the key to exist. - volumeMounts: [] + volumeMounts: + - name: kubernetes-clusters + mountPath: /etc/nebula/kubernetes + readOnly: true # # The catalog ConfigMap mount, off with NEBULA_CATALOG_DIR above. # - name: catalog # mountPath: /etc/nebula/catalog # readOnly: true - volumes: [] + volumes: + - name: kubernetes-clusters + configMap: + # Unprefixed on purpose: with `- ../kubernetes-provider` in config/default, + # namePrefix rewrites this to the generated "nebula-kubernetes-clusters". + # optional, so without it the provider is just skipped. + name: kubernetes-clusters + optional: true # - name: catalog # configMap: # # Unprefixed on purpose: config/default's namePrefix rewrites this diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 85e70a6..b87e5ba 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -75,6 +75,7 @@ rules: resourceNames: - system:node:nebula-aws - system:node:nebula-fake + - system:node:nebula-kubernetes - system:node:nebula-modal - system:node:nebula-runpod resources: diff --git a/config/samples/nodepool.yaml b/config/samples/nodepool.yaml index a24f100..aaf9794 100644 --- a/config/samples/nodepool.yaml +++ b/config/samples/nodepool.yaml @@ -7,6 +7,11 @@ metadata: spec: # strategy (the inner, provider-ranking axis). providers: + # EKS clusters from --kubernetes-clusters. A region matches a cluster's geography ("us") or + # its exact AWS region ("us-east-1"), both as set in that file. + - name: kubernetes + regions: + - us-east-1 - name: aws regions: - us diff --git a/docs/architecture.md b/docs/architecture.md index d9a5d64..60f24ab 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -59,17 +59,17 @@ mapping), see [docs/status.md](status.md). declaration cannot yet prefer the cheapest region. Modal is the sharper case — a pinned region there costs 1.5x (group) or 1.75x (narrow) over its unconstrained default, which the catalog does not model. -- Failover on a provider that *queues*. The blocklist has exactly one writer: the - virtual kubelet's `Provision` error path. A provider that reports a capacity - shortfall synchronously (AWS `CreateFleet`) therefore fails over across zone, - region and tier, but one that ACCEPTS the request and queues for capacity returns - an instance id and no error — so nothing is blocklisted and placement is never - re-driven. Modal is that case: a `[modal, aws]` pool never advances to AWS on - capacity, and the Pod waits in Modal's queue at `Initializing` (which, per - [status](status.md#modal), is indistinguishable from booting). Modal also collapses - its regions into a single candidate, so it has no intra-provider region failover - either. Making failover live for such a provider needs a reserved-by deadline that - synthesizes `ErrNoCapacity` — the classification side already handles it. +- Failover on Modal's queue. The blocklist has two writers: the virtual kubelet's + `Provision` error path, and the poll loop when `List` reports an instance as + `InstanceUnschedulable`. A provider that reports a capacity shortfall synchronously + (AWS `CreateFleet`) fails over through the first; the `kubernetes` provider, which + returns unreserved and reports no capacity once its pod stays unbound past the + bind timeout, through the second. Modal reports neither: it ACCEPTS the request and + queues, so a `[modal, aws]` pool never advances to AWS on capacity, and the Pod + waits in Modal's queue at `Initializing` (which, per [status](status.md#modal), is + indistinguishable from booting). Modal also collapses its regions into a single + candidate, so it has no intra-provider region failover either. Making failover live + for it means reporting `InstanceUnschedulable` past a reserved-by deadline. - Bin-packing multiple unrelated Pods onto one external instance. The current model is one workload Pod to one external instance. - In-place migration. Recovery from reclaim, failure, or spec changes is @@ -226,7 +226,8 @@ Follow one GPU Pod from creation to teardown: and region from the failed request. The shared blocklist records that scope for the Pod's blocklist TTL plus jitter. The Pod is marked Failed; the placement controller deletes terminal controller-owned Pods so their owner recreates a - fresh gated Pod that skips the blocked candidate. + fresh gated Pod that skips the blocked candidate. An instance accepted unreserved + and later listed as `InstanceUnschedulable` takes the same path from the poll loop. 10. **Teardown.** The happy path is edge-triggered by VK `DeletePod`, which calls `provider.Terminate(instanceID)`. The backstop is level-triggered by the diff --git a/go.mod b/go.mod index d86fa72..b4aa1d6 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,13 @@ module github.com/InftyAI/Nebula go 1.25.0 require ( - github.com/aws/aws-sdk-go-v2 v1.43.0 + github.com/aws/aws-sdk-go-v2 v1.47.3 github.com/aws/aws-sdk-go-v2/config v1.32.31 + github.com/aws/aws-sdk-go-v2/credentials v1.20.9 github.com/aws/aws-sdk-go-v2/service/ec2 v1.317.0 - github.com/aws/smithy-go v1.27.3 + github.com/aws/aws-sdk-go-v2/service/eks v1.104.2 + github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 + github.com/aws/smithy-go v1.28.5 github.com/modal-labs/modal-client/go v0.9.0 github.com/onsi/ginkgo/v2 v2.27.2 github.com/onsi/gomega v1.38.2 @@ -29,17 +32,15 @@ require ( github.com/Masterminds/semver/v3 v3.4.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.19.30 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect diff --git a/go.sum b/go.sum index 7ce5be5..8967b8e 100644 --- a/go.sum +++ b/go.sum @@ -10,36 +10,38 @@ github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced h1:HxlRMDx/Ve github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced/go.mod h1:p7lmI+ecoe1RTyD11SPXWsSQ3H+pJ4cp5y7vtKW4QdM= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aws/aws-sdk-go-v2 v1.43.0 h1:fharf/WhbRAVZ1du0QL7roNFxZ6T/sWr+4Ni617bwSI= -github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= +github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= github.com/aws/aws-sdk-go-v2/config v1.32.31 h1:n4nY9O3QKoHIkL85EX+V8RcMFtOhlpTFhGArg915PXk= github.com/aws/aws-sdk-go-v2/config v1.32.31/go.mod h1:PN0NYDCCoOpGGsZ2+elDUidmHfQBPyYzN2GCgl8HEBs= -github.com/aws/aws-sdk-go-v2/credentials v1.19.30 h1:TTCvvzFU6gXa4iJecNG/0F/B0oYTiazoRECr2XyLHrY= -github.com/aws/aws-sdk-go-v2/credentials v1.19.30/go.mod h1:jKxAp2AEncnliinzpgOSZDFv6+VjvWhjw/AtbfsWT9U= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 h1:kfVL5wAunCJycL6MOQ6aNh6PlAYEymflcjuKmrWUA0o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31/go.mod h1:nWfRNDAppujCQgOUd43lKT4yeLv9z3nJ3bw1G3BgQKo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 h1:Z8F3hfCY33IGpJjFAnv0wvtv1FIKj1GHmRDEYqy64tw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 h1:hyOxUyXdh3AyjE93gBgsfziJag9ACwcs+ZpDBLzi8mw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 h1:0MrUL35H/Y4kdFfItoR5jCgtDQ4Z/8LudAoIHRfA4hE= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9 h1:SNO55PnWhaDafsOcv16lCA0kdoFMl/YlegWoVsU02s0= +github.com/aws/aws-sdk-go-v2/credentials v1.20.9/go.mod h1:+2reOTRtdD3VSBqpRDNwrZIYy8yMCYa7oK3WKTJ/I1I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 h1:gr3b87OLB29jymWPzp/CAyXwQRJVE+sC4Wm0qBIKHsk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3/go.mod h1:BRjf4L/Yx2hY54o+ZibPbLJyMMeSwqYQXwcJfqW66a8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 h1:CqL8NN3p4I0JTwwV5MEoatIl4Y2amZ2X9dRD+0hbRL0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6/go.mod h1:exdJ32hMqXl0Wyo6NwqEC5yIhtua3+PDaKE8P29m4qE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 h1:FpBq24TG6WVSObTNhQOv7Q0088rounUUlwn3zlNItkM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6/go.mod h1:ONzdPVYC95oBOu0VWNEsKEntcoemP3xVKjT+qWml1s4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 h1:Qmn60RWuk9+bd2ddvJJqwf3Cpac3VvGMOdbGemtUhX4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6/go.mod h1:h/YJhe4AVyQxEUMluph10iqV7mjajH7A9L+BHF5ppis= github.com/aws/aws-sdk-go-v2/service/ec2 v1.317.0 h1:IkqA16g2hkQntk/K5+srT65TueoTDa7vGhZwqG9w6T4= github.com/aws/aws-sdk-go-v2/service/ec2 v1.317.0/go.mod h1:dmz3SHr11/hwUijR6xfE/xDRNHcjJwJWZ9ASZdkjGeg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 h1:w2SIhW92DZPFrSL4ksVCr8IYff5OZwIcxg8+95tzvAI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg= -github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 h1:OHH5iTQvVGmfHjX/5Q+vFuA/Rf2x6/95aJ/75QCQSm4= -github.com/aws/aws-sdk-go-v2/service/signin v1.5.0/go.mod h1:mCF3AK9PpL49oOrhniUXWAfhVBVQ/XbytoE5eccZUIs= -github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 h1:CaJyYhxBE0M/HJX/YvSaSmQlsI91VHB0lKU8LtLxL3A= -github.com/aws/aws-sdk-go-v2/service/sso v1.33.0/go.mod h1:+e6BMRMPjBQoCw/WovYR9GLy2IU0z4Q77smOB1DraSg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 h1:tC323YV77QdafeBr6LUhLDTsboyuyHLNRwAyCP44kGU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0/go.mod h1:SfLK1sgviHmbI+MozR9iDwDjL4cdCVZtahsjoR+z7wg= -github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 h1:Pd6PNlp4t8PTXxqzstICl52Wsy78vpjFZ7PRUj44mJc= -github.com/aws/aws-sdk-go-v2/service/sts v1.45.0/go.mod h1:rmQ0TnHzuLPmabgjPcsywhsSOmaBDgzR4zvDxSPsGdg= -github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= -github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2/service/eks v1.104.2 h1:WvY97C6Zyx8E2Q16jXaL5c8wjHTKMmrOEUDY6+DmtsM= +github.com/aws/aws-sdk-go-v2/service/eks v1.104.2/go.mod h1:7Z9oc7QnBWbdizy5Zj5Ba0XKLlfO8eyGRSNeqyH1eZY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 h1:e6YW5uME30TkrElR+4/xcY7NB6CCj2dcv7vtmGXhfhQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21/go.mod h1:obpHAMP693+f3/A8PkLhIqfK1C+NWWn8zXzqD3OH520= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 h1:LHHdobjo5le1sjJdolitV7uVvYg3wkaeM1Ju2fDgeZQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6/go.mod h1:CBGB0evAiaMD3Lk4wPO7LQCcCZLuhzL5Qf/wfEUHQ5w= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 h1:tvpDjuUvgKURf3Yk4hOaGbZ4QdY+070OWcA5d6LUv2A= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.4/go.mod h1:3ibVKg5Ng82Zby8++jELTD2nHxiPk2DJRLEeW/LKp8M= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 h1:ChCtagixSFOxltv0o6E3NT8F7ksvZtZGPwUBynO5Nmk= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.4/go.mod h1:+FEO9jrVMpc9oN3CdAxJbtlQEu3z8UddDPSedpeM3PY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 h1:j6c+HA4AoS0EBEjArPjavFEBrucJEryfrQU0TffmACk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4/go.mod h1:RhghaXUk9q66VtOQUX1FdPddQuHnqQZzbB0fEYXmPRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4 h1:OKEfbf2tcfsF2pBk/S9QqNpo6alaAtiBs4hysCsBjPw= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.4/go.mod h1:mCd6qGo9b8hTtfjgsmNkFlzwjyce4owd7HDVFnCLXhM= +github.com/aws/smithy-go v1.28.5 h1:OqbqKtzY9NUnr2yAe9+hiTdFb20tNaK/lI0PMOGXX9k= +github.com/aws/smithy-go v1.28.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= diff --git a/internal/controller/nodeclaim_controller.go b/internal/controller/nodeclaim_controller.go index 6f09310..98025de 100644 --- a/internal/controller/nodeclaim_controller.go +++ b/internal/controller/nodeclaim_controller.go @@ -287,7 +287,7 @@ func (r *NodeClaimReconciler) provider(name string) (provider.Provider, bool) { // still reclaims by asking the provider what exists. func (r *NodeClaimReconciler) desiredPhase(nc *nebulav1alpha1.NodeClaim, pod *corev1.Pod) nebulav1alpha1.NodeClaimPhase { switch { - case util.IsTerminalPodPhase(pod.Status.Phase): + case util.Finished(pod.Status.Phase): return nebulav1alpha1.NodeClaimTerminated case !pod.DeletionTimestamp.IsZero(): return nebulav1alpha1.NodeClaimTerminating diff --git a/internal/controller/pod_placement_helpers.go b/internal/controller/pod_placement_helpers.go index 1dc9662..76378be 100644 --- a/internal/controller/pod_placement_helpers.go +++ b/internal/controller/pod_placement_helpers.go @@ -411,7 +411,7 @@ func (r *PodPlacementReconciler) reapTerminalPod(ctx context.Context, pod *corev if !pod.DeletionTimestamp.IsZero() { return true, nil // already being deleted; nothing more to place } - if !util.IsTerminalPodPhase(pod.Status.Phase) { + if !util.Finished(pod.Status.Phase) { return false, nil } if !isControllerOwned(pod) { diff --git a/internal/webhook/v1/pod_webhook.go b/internal/webhook/v1/pod_webhook.go index c09c95b..0ed646e 100644 --- a/internal/webhook/v1/pod_webhook.go +++ b/internal/webhook/v1/pod_webhook.go @@ -19,6 +19,7 @@ package v1 import ( "context" "fmt" + "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -112,9 +113,9 @@ func (d *PodCustomDefaulter) Default(_ context.Context, obj runtime.Object) erro // +kubebuilder:webhook:path=/validate--v1-pod,mutating=false,failurePolicy=fail,sideEffects=None,groups="",resources=pods,verbs=create;update,versions=v1,name=vpod-v1.nebula.inftyai.com,admissionReviewVersions=v1 // PodCustomValidator rejects an opted-in Pod the providers cannot run as written: they -// launch one container and no init containers, and expose at most one port (Modal's -// connect URL routes to one; see modal.firstPort). Admitting more would drop the rest -// silently. +// launch one container and no init containers, expose at most one port (Modal's connect URL +// routes to one; see modal.firstPort), and mount nothing from the cluster, so no volumes and +// no service account. Admitting more would drop the rest silently. type PodCustomValidator struct{} var _ webhook.CustomValidator = &PodCustomValidator{} @@ -187,9 +188,23 @@ func validatePod(pod *corev1.Pod) error { if c := pod.Spec.Containers[0]; len(c.Ports) > 1 { return fmt.Errorf("nebula exposes at most one port per Pod, container %q declares %d", c.Name, len(c.Ports)) } + if sa := pod.Spec.ServiceAccountName; sa != "" && sa != "default" { + return fmt.Errorf("nebula does not mount service accounts, got %q", sa) + } + for _, v := range pod.Spec.Volumes { + // The API server adds this one to every Pod before validation; it is simply not mounted. + if v.Projected != nil && strings.HasPrefix(v.Name, serviceAccountTokenVolumePrefix) { + continue + } + return fmt.Errorf("nebula does not mount volumes, got %q", v.Name) + } return nil } +// serviceAccountTokenVolumePrefix names the projected token volume the ServiceAccount +// admission plugin adds ("kube-api-access-"). +const serviceAccountTokenVolumePrefix = "kube-api-access-" + // hasNebulaGate reports whether the Pod carries Nebula's provider-selection scheduling gate. func hasNebulaGate(pod *corev1.Pod) bool { for _, g := range pod.Spec.SchedulingGates { diff --git a/internal/webhook/v1/pod_webhook_test.go b/internal/webhook/v1/pod_webhook_test.go index 305552e..11bde90 100644 --- a/internal/webhook/v1/pod_webhook_test.go +++ b/internal/webhook/v1/pod_webhook_test.go @@ -216,6 +216,49 @@ func TestValidateCreate_OneContainerOnePort(t *testing.T) { } } +func TestValidateCreate_NoVolumesNoServiceAccount(t *testing.T) { + saToken := corev1.Volume{ + Name: "kube-api-access-x7k2p", + VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{}}, + } + cases := []struct { + name string + mutate func(*corev1.Pod) + wantErr string + }{ + // What the API server itself adds to every Pod must be admitted. + {"default service account and its token", func(p *corev1.Pod) { + p.Spec.ServiceAccountName = "default" + p.Spec.Volumes = []corev1.Volume{saToken} + }, ""}, + {"own service account", func(p *corev1.Pod) { p.Spec.ServiceAccountName = "worker" }, "service accounts"}, + {"emptyDir", func(p *corev1.Pod) { + p.Spec.Volumes = []corev1.Volume{saToken, {Name: "scratch", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} + }, "volumes"}, + // Only the projected token volume is exempt, not anything that borrows its name. + {"lookalike name", func(p *corev1.Pod) { + p.Spec.Volumes = []corev1.Volume{{Name: "kube-api-access-fake", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} + }, "volumes"}, + } + v := &PodCustomValidator{} + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + pod := podWith(map[string]string{nebulav1alpha1.EnabledLabel: "true"}, "") + pod.Spec.Containers = []corev1.Container{{Name: "main"}} + tc.mutate(pod) + _, err := v.ValidateCreate(context.Background(), pod) + switch { + case tc.wantErr == "" && err != nil: + t.Fatalf("ValidateCreate err = %v, want nil", err) + case tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)): + t.Fatalf("ValidateCreate err = %v, want it to contain %q", err, tc.wantErr) + } + }) + } +} + func TestValidateCreate_RejectsNonPod(t *testing.T) { if _, err := (&PodCustomValidator{}).ValidateCreate(context.Background(), &corev1.Service{}); err == nil { t.Fatal("expected an error for a non-Pod object") diff --git a/pkg/provider/aws/aws.go b/pkg/provider/aws/aws.go index 7eac846..01c0bd7 100644 --- a/pkg/provider/aws/aws.go +++ b/pkg/provider/aws/aws.go @@ -816,7 +816,7 @@ func (p *Provider) instanceSpecFromPod( Image: c.Image, Command: append([]string{}, c.Command...), Args: append([]string{}, c.Args...), - // The caller's resolved environment, whole (provider.ProvisionRequest.Env). The Pod's + // The caller's resolved environment, whole (provider.Env.All). The Pod's // own env is not read: it holds references this adapter cannot follow. // // CAVEAT, and the reason this is called out: buildUserData renders env into cloud-init @@ -828,7 +828,7 @@ func (p *Provider) instanceSpecFromPod( // TODO: deliver Secret-derived values out-of-band — SSM Parameter Store / Secrets // Manager under the claim, fetched at boot with the instance profile — and keep only // non-sensitive values in user-data. - Env: req.Env, + Env: req.Env.All(), Spot: req.CapacityType == nebulav1alpha1.CapacitySpot, Region: req.Region, Tags: map[string]string{ClaimTagKey: req.ClaimName}, diff --git a/pkg/provider/aws/aws_test.go b/pkg/provider/aws/aws_test.go index ceea8d2..67b4ab5 100644 --- a/pkg/provider/aws/aws_test.go +++ b/pkg/provider/aws/aws_test.go @@ -186,7 +186,9 @@ func TestProvision_UsesResolvedEnv(t *testing.T) { if _, err := p.Provision(context.Background(), pod, provider.ProvisionRequest{ ClaimName: "claim-env", Region: "us-west-2", // EC2 needs one; the test client is not configured with a default - Env: map[string]string{"FOO": "bar", "TOKEN": "t0ken"}, + Env: provider.Env{ + Config: map[string]string{"FOO": "bar"}, Secrets: map[string]string{"TOKEN": "t0ken"}, + }, }); err != nil { t.Fatalf("Provision: %v", err) } diff --git a/pkg/provider/kubernetes/config.go b/pkg/provider/kubernetes/config.go new file mode 100644 index 0000000..d1e09e3 --- /dev/null +++ b/pkg/provider/kubernetes/config.go @@ -0,0 +1,125 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "errors" + "fmt" + "os" + + "sigs.k8s.io/yaml" + + "github.com/InftyAI/Nebula/pkg/provider" +) + +// ClusterProviderEKS is the only cluster provider supported today. +const ClusterProviderEKS = "eks" + +// clusterCatalogs maps a cluster provider to the price catalog its GPU mapping comes from: +// an EKS cluster's nodes are EC2 instances, so the AWS catalog's instance types say which +// families carry which GPU (see familiesByAccelerator). Supporting another cloud starts with +// an entry here. +var clusterCatalogs = map[string]string{ + ClusterProviderEKS: provider.ProviderAWS, +} + +// Config is the cluster table, read from the file --kubernetes-clusters names. It holds no +// credentials: an EKS cluster is reached with Nebula's own AWS identity (see newEKSClients). +type Config struct { + Clusters []ClusterConfig `json:"clusters"` +} + +// ClusterConfig is one placement target. A NodePool selects clusters by Region or Geography, +// both stated here rather than derived from each other. +type ClusterConfig struct { + // Name is the cluster's own name: for EKS, what DescribeCluster and the token take. + Name string `json:"name"` + Provider string `json:"provider"` + // Region is the cluster's AWS region: where DescribeCluster is called, and the token + // placement, the NodeClaim and the blocklist carry for this cluster. + Region string `json:"region"` + // Geography is the Nebula geography the cluster counts as in (see provider.Geographies). + Geography string `json:"geography"` + // RoleARN, when set, is assumed before talking to this cluster: the cross-account path. + RoleARN string `json:"roleARN,omitempty"` + // NodeAutoscaler is what adds nodes to the cluster (see nodeAutoscalers). Empty is Karpenter. + NodeAutoscaler string `json:"nodeAutoscaler,omitempty"` +} + +// nodeAutoscalerName defaults an empty NodeAutoscaler to Karpenter. +func (c ClusterConfig) nodeAutoscalerName() string { + if c.NodeAutoscaler == "" { + return NodeAutoscalerKarpenter + } + return c.NodeAutoscaler +} + +// LoadConfig reads and validates the cluster table at path. +func LoadConfig(path string) (*Config, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("kubernetes: read cluster config: %w", err) + } + var cfg Config + if err := yaml.UnmarshalStrict(raw, &cfg); err != nil { + return nil, fmt.Errorf("kubernetes: parse cluster config %s: %w", path, err) + } + if err := cfg.Validate(); err != nil { + return nil, err + } + return &cfg, nil +} + +// Validate rejects a table the provider could not act on. A region holds one cluster because +// the region is the token claims and blocks carry: two clusters in one would make teardown +// search the wrong one. Names are unique so each one is a single EKS cluster. +func (c *Config) Validate() error { + if len(c.Clusters) == 0 { + return errors.New("kubernetes: cluster config lists no clusters") + } + seen := make(map[string]bool, len(c.Clusters)) + regions := make(map[string]string, len(c.Clusters)) + for i, cl := range c.Clusters { + if cl.Name == "" { + return fmt.Errorf("kubernetes: cluster %d has no name", i) + } + if seen[cl.Name] { + return fmt.Errorf("kubernetes: duplicate cluster name %q", cl.Name) + } + seen[cl.Name] = true + if _, ok := clusterCatalogs[cl.Provider]; !ok { + return fmt.Errorf("kubernetes: cluster %q: unsupported provider %q (want %q)", + cl.Name, cl.Provider, ClusterProviderEKS) + } + if _, ok := nodeAutoscalers[cl.nodeAutoscalerName()]; !ok { + return fmt.Errorf("kubernetes: cluster %q: unsupported node autoscaler %q", cl.Name, cl.NodeAutoscaler) + } + if cl.Region == "" { + return fmt.Errorf("kubernetes: cluster %q: region is required", cl.Name) + } + if other, dup := regions[cl.Region]; dup { + return fmt.Errorf("kubernetes: clusters %q and %q are both in %s; a region holds one cluster", + other, cl.Name, cl.Region) + } + regions[cl.Region] = cl.Name + if !provider.IsGeography(cl.Geography) { + return fmt.Errorf("kubernetes: cluster %q: geography %q is not one of %v", + cl.Name, cl.Geography, provider.Geographies) + } + } + return nil +} diff --git a/pkg/provider/kubernetes/eks.go b/pkg/provider/kubernetes/eks.go new file mode 100644 index 0000000..5da9404 --- /dev/null +++ b/pkg/provider/kubernetes/eks.go @@ -0,0 +1,151 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "sync" + "time" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/eks" + "github.com/aws/aws-sdk-go-v2/service/sts" + smithyhttp "github.com/aws/smithy-go/transport/http" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +const ( + // eksTokenPrefix and clusterIDHeader are the EKS token format `aws eks get-token` emits: + // a presigned STS GetCallerIdentity URL, bound to one cluster by the signed header. + eksTokenPrefix = "k8s-aws-v1." + clusterIDHeader = "x-k8s-aws-id" + // tokenTTL is how long a minted token is reused. EKS accepts one for 15 minutes; the + // margin covers clock skew and a request in flight when it is swapped. + tokenTTL = 10 * time.Minute + + // clusterQPS and clusterBurst are each cluster client's own budget, separate from the + // home cluster's (see restConfigQPS in cmd/main.go). + clusterQPS = 50 + clusterBurst = 100 +) + +// clusterClients is what the provider needs from one cluster: pods, Secrets, namespaces and +// events, plus discovery to check the node autoscaler is installed. +type clusterClients struct { + kube k8s.Interface +} + +// newEKSClients builds the clients for one EKS cluster from Nebula's ambient AWS identity +// (the SDK's default chain: an access key today, Pod Identity later). Nothing is stored: +// the endpoint and CA come from DescribeCluster, the bearer token from STS. +func newEKSClients(ctx context.Context, cl ClusterConfig) (clusterClients, error) { + awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(cl.Region)) + if err != nil { + return clusterClients{}, fmt.Errorf("kubernetes: cluster %q: load AWS config: %w", cl.Name, err) + } + if cl.RoleARN != "" { + awsCfg.Credentials = awssdk.NewCredentialsCache( + stscreds.NewAssumeRoleProvider(sts.NewFromConfig(awsCfg), cl.RoleARN)) + } + + out, err := eks.NewFromConfig(awsCfg).DescribeCluster(ctx, + &eks.DescribeClusterInput{Name: awssdk.String(cl.Name)}) + if err != nil { + return clusterClients{}, fmt.Errorf("kubernetes: describe EKS cluster %q in %s: %w", cl.Name, cl.Region, err) + } + c := out.Cluster + if c == nil || c.Endpoint == nil || c.CertificateAuthority == nil || c.CertificateAuthority.Data == nil { + return clusterClients{}, fmt.Errorf("kubernetes: EKS cluster %q has no endpoint or CA yet", cl.Name) + } + ca, err := base64.StdEncoding.DecodeString(*c.CertificateAuthority.Data) + if err != nil { + return clusterClients{}, fmt.Errorf("kubernetes: cluster %q: decode CA: %w", cl.Name, err) + } + + tokens := &eksTokenSource{presign: sts.NewPresignClient(sts.NewFromConfig(awsCfg)), cluster: cl.Name} + restCfg := &rest.Config{ + Host: *c.Endpoint, + TLSClientConfig: rest.TLSClientConfig{CAData: ca}, + QPS: clusterQPS, + Burst: clusterBurst, + WrapTransport: func(rt http.RoundTripper) http.RoundTripper { + return &bearerTransport{base: rt, tokens: tokens} + }, + } + kube, err := k8s.NewForConfig(restCfg) + if err != nil { + return clusterClients{}, fmt.Errorf("kubernetes: cluster %q: build client: %w", cl.Name, err) + } + return clusterClients{kube: kube}, nil +} + +// eksTokenSource mints and caches the bearer token for one cluster. +type eksTokenSource struct { + presign *sts.PresignClient + cluster string + + mu sync.Mutex + token string + expires time.Time +} + +func (s *eksTokenSource) Token(ctx context.Context) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.token != "" && time.Now().Before(s.expires) { + return s.token, nil + } + req, err := s.presign.PresignGetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}, + func(o *sts.PresignOptions) { + o.ClientOptions = append(o.ClientOptions, func(so *sts.Options) { + so.APIOptions = append(so.APIOptions, smithyhttp.AddHeaderValue(clusterIDHeader, s.cluster)) + }) + }) + if err != nil { + return "", fmt.Errorf("kubernetes: presign EKS token for %q: %w", s.cluster, err) + } + s.token = eksTokenPrefix + base64.RawURLEncoding.EncodeToString([]byte(req.URL)) + s.expires = time.Now().Add(tokenTTL) + return s.token, nil +} + +// bearerTransport adds the current EKS token to every request. +type bearerTransport struct { + base http.RoundTripper + tokens interface { + Token(context.Context) (string, error) + } +} + +func (t *bearerTransport) RoundTrip(req *http.Request) (*http.Response, error) { + token, err := t.tokens.Token(req.Context()) + if err != nil { + return nil, err + } + req = req.Clone(req.Context()) // a RoundTripper must not mutate the caller's request + req.Header.Set("Authorization", "Bearer "+token) + return t.base.RoundTrip(req) +} + +// kubeClient is the typed client; an alias so tests can pass client-go's fake. +type kubeClient = k8s.Interface diff --git a/pkg/provider/kubernetes/karpenter.go b/pkg/provider/kubernetes/karpenter.go new file mode 100644 index 0000000..2b7a65d --- /dev/null +++ b/pkg/provider/kubernetes/karpenter.go @@ -0,0 +1,143 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + "fmt" + "slices" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime/schema" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +// Karpenter v1 (sigs.k8s.io/karpenter): its well-known labels, the annotation that keeps +// consolidation off a node, and the pod events it emits. FailedScheduling is also the +// kube-scheduler's reason, expected on every pod that needs a new node, so only Karpenter's +// counts. +const ( + instanceFamilyLabel = "karpenter.k8s.aws/instance-family" + capacityTypeLabel = "karpenter.sh/capacity-type" + doNotDisruptAnnotation = "karpenter.sh/do-not-disrupt" + capacityTypeSpot = "spot" + capacityTypeOnDemand = "on-demand" + + reasonInsufficientCapacity = "InsufficientCapacityError" + reasonFailedScheduling = "FailedScheduling" + karpenterComponent = "karpenter" + + // karpenterBindTimeout covers a healthy launch up to binding: Karpenter's decision, the + // EC2 launch and node boot until the GPUs are advertised. Measured p-family boots may need + // more; image pull comes after binding and is the readiness deadline's. + karpenterBindTimeout = 5 * time.Minute + // karpenterProvisionTimeout covers the slow end of Karpenter's decision: its batch window + // (up to 10s by default), the scheduling pass and the EC2 call. A refusal arriving after + // it is only caught by bindTimeout, so err long. + karpenterProvisionTimeout = 30 * time.Second +) + +// nodeClaimGV is Karpenter v1's API, only checked for presence (see verify). +var nodeClaimGV = schema.GroupVersion{Group: "karpenter.sh", Version: "v1"} + +type karpenter struct{} + +func (karpenter) bindTimeout() time.Duration { return karpenterBindTimeout } +func (karpenter) provisionTimeout() time.Duration { return karpenterProvisionTimeout } + +// selection pins the GPU model through the instance family (one model per NVIDIA family on +// AWS) and the tier through the capacity type, and keeps consolidation and drift from +// evicting a running sandbox to repack. +func (karpenter) selection(families []string, tier nebulav1alpha1.CapacityType) nodeSelection { + var reqs []corev1.NodeSelectorRequirement + if len(families) > 0 { + reqs = append(reqs, corev1.NodeSelectorRequirement{ + Key: instanceFamilyLabel, Operator: corev1.NodeSelectorOpIn, Values: families, + }) + } + if ct := karpenterCapacityType(tier); ct != "" { + reqs = append(reqs, corev1.NodeSelectorRequirement{ + Key: capacityTypeLabel, Operator: corev1.NodeSelectorOpIn, Values: []string{ct}, + }) + } + sel := nodeSelection{annotations: map[string]string{doNotDisruptAnnotation: labelTrue}} + if len(reqs) > 0 { + // Reschedule takes time, let's be conservative here. + sel.affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: reqs}}, + }, + }} + } + return sel +} + +func karpenterCapacityType(tier nebulav1alpha1.CapacityType) string { + switch tier { + case nebulav1alpha1.CapacitySpot: + return capacityTypeSpot + case nebulav1alpha1.CapacityOnDemand: + return capacityTypeOnDemand + default: + return "" + } +} + +// refused returns Karpenter's verdict that it will not get the pod a node, from the pod's +// own events, or nil. EC2 refusing a launch shows nowhere else: Karpenter deletes that +// NodeClaim without recording why. These events are rate-limited, so a dropped one only costs +// the bind timeout. +func (karpenter) refused(ctx context.Context, cc clusterClients, pod *corev1.Pod) error { + evs, err := cc.kube.CoreV1().Events(pod.Namespace).List(ctx, metav1.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("involvedObject.uid", string(pod.UID)).String(), + }) + if err != nil { + return nil + } + for _, ev := range evs.Items { + // Filtered again here: the selector is the server's job, and not every client honours it. + if ev.InvolvedObject.UID != pod.UID || !fromKarpenter(ev) { + continue + } + if ev.Reason == reasonInsufficientCapacity || ev.Reason == reasonFailedScheduling { + return fmt.Errorf("kubernetes: karpenter: %s: %w", ev.Message, provider.ErrNoCapacity) + } + } + return nil +} + +func (karpenter) verify(_ context.Context, cc clusterClients) error { + res, err := cc.kube.Discovery().ServerResourcesForGroupVersion(nodeClaimGV.String()) + if err == nil && slices.ContainsFunc(res.APIResources, func(r metav1.APIResource) bool { + return r.Name == "nodeclaims" + }) { + return nil + } + return fmt.Errorf("kubernetes: Karpenter %s NodeClaims not found (%v); is Karpenter v1 installed?", + nodeClaimGV, err) +} + +func fromKarpenter(ev corev1.Event) bool { + return strings.Contains(ev.Source.Component, karpenterComponent) || + strings.Contains(ev.ReportingController, karpenterComponent) +} diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go new file mode 100644 index 0000000..46f14a8 --- /dev/null +++ b/pkg/provider/kubernetes/kubernetes.go @@ -0,0 +1,629 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kubernetes implements provider.Provider on EKS clusters running Karpenter. Each +// cluster is one region: placement picks a cluster, this adapter creates the real pod there, +// and that cluster's Karpenter launches and packs the nodes. One NodeClaim maps to one real +// pod, not to a node; several pods can share a Karpenter node. +// +// Capacity is found by trying. Provision creates the pod and watches it briefly, failing at +// once on the node autoscaler's refusal (see waitForProvision); a pod still unbound after that +// is settled by the poll loop, which reports no capacity past the bind timeout (see +// toInstance). Either way the cluster is blocklisted like an AWS region. +// +// The real pod lives in the placeholder's namespace, named after the claim like every +// provider's instance, which is what lets the host cluster itself be a target. Nebula reports +// no cost for it: a shared node has no per-pod price here, and OpenCost covers the clusters. +package kubernetes + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/provider/catalog" + "github.com/InftyAI/Nebula/pkg/util" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +const ( + pollInterval = 5 * time.Second + spotNotice = 2 * time.Minute + // provisionSlack is the part of a Provision that is not the wait: the namespace, + // pod and Secret calls, and deleting a previous incarnation. It keeps the handler's + // deadline from cutting the wait short, which would read as no capacity. + provisionSlack = 10 * time.Second +) + +var _ provider.Provider = (*Provider)(nil) + +type cluster struct { + cfg ClusterConfig + clients clusterClients + nodeAutoscaler nodeAutoscaler + // families maps a canonical accelerator to the instance families carrying it in this + // cluster's cloud (see clusterCatalogs). + families map[string][]string +} + +// Provider is the kubernetes implementation of provider.Provider. +type Provider struct { + clusters map[string]cluster // by region, the token placement and blocks carry + order []string // config order, which is the order ResolveRegions walks + // offered is every cluster's families merged, for MapAccelerator: placement asks the + // provider, not a cluster, whether it serves an accelerator. + offered map[string][]string + costLabels []string + now func() time.Time // a seam for the bind timeout in toInstance +} + +// NewFromFile builds the provider from the cluster table at path, connecting to every +// cluster up front so a bad entry fails at startup rather than on the first Provision. +// costLabels are the --cost-labels keys, copied from each Pod onto its real pod. +func NewFromFile(ctx context.Context, path string, costLabels []string) (*Provider, error) { + cfg, err := LoadConfig(path) + if err != nil { + return nil, err + } + cat, err := catalog.Load() + if err != nil { + return nil, fmt.Errorf("kubernetes: load catalog: %w", err) + } + clients := make(map[string]clusterClients, len(cfg.Clusters)) + for _, cl := range cfg.Clusters { + cc, err := newEKSClients(ctx, cl) + if err != nil { + return nil, err + } + if err := nodeAutoscalers[cl.nodeAutoscalerName()].verify(ctx, cc); err != nil { + return nil, fmt.Errorf("kubernetes: cluster %q: %w", cl.Name, err) + } + clients[cl.Name] = cc + } + return New(cfg.Clusters, clients, cat, costLabels), nil +} + +// New returns a Provider over already-built clients, one per cluster in clusters, which +// must already be validated (see Config.Validate). +func New(clusters []ClusterConfig, clients map[string]clusterClients, cat catalog.Lookup, + costLabels []string) *Provider { + p := &Provider{ + clusters: make(map[string]cluster, len(clusters)), + offered: map[string][]string{}, + costLabels: costLabels, + now: time.Now, + } + for _, cl := range clusters { + families := familiesByAccelerator(cat.Offerings(clusterCatalogs[cl.Provider])) + p.clusters[cl.Region] = cluster{ + cfg: cl, clients: clients[cl.Name], nodeAutoscaler: nodeAutoscalers[cl.nodeAutoscalerName()], + families: families, + } + p.order = append(p.order, cl.Region) + for accel, fs := range families { + for _, f := range fs { + if !slices.Contains(p.offered[accel], f) { + p.offered[accel] = append(p.offered[accel], f) + } + } + } + } + for _, fs := range p.offered { + slices.Sort(fs) + } + return p +} + +// familiesByAccelerator turns a catalog's instance types into families per GPU model: +// H100 → [p5] on AWS, where each NVIDIA family carries one GPU model, so the family pins it. +func familiesByAccelerator(rows []provider.Offering) map[string][]string { + out := make(map[string][]string) + for _, o := range rows { + family, _, _ := strings.Cut(o.AcceleratorID, ".") + key := strings.ToUpper(o.AcceleratorType) + if family != "" && !slices.Contains(out[key], family) { + out[key] = append(out[key], family) + } + } + for _, fs := range out { + slices.Sort(fs) + } + return out +} + +// Name implements provider.Provider. +func (p *Provider) Name() string { return provider.ProviderKubernetes } + +// Capabilities implements provider.Provider. +func (p *Provider) Capabilities() provider.Capabilities { + return provider.Capabilities{ + SupportsStop: false, + SupportsSpot: true, // Karpenter's capacity type, see nodeAffinity + SupportsEgressPolicy: false, // no NetworkPolicy is written yet + SupportsCPUOnly: true, + NativeTags: true, + PreemptionNotice: spotNotice, + PollInterval: pollInterval, + ProvisionTimeout: p.longestProvision() + provisionSlack, + } +} + +// longestProvision is the slowest node autoscaler's provisionTimeout. Capabilities has one +// deadline for the provider, set to this plus provisionSlack, and each cluster's wait inside +// it is bounded by its own node autoscaler. +func (p *Provider) longestProvision() time.Duration { + var d time.Duration + for _, cl := range p.clusters { + d = max(d, cl.nodeAutoscaler.provisionTimeout()) + } + return d +} + +// Offerings implements provider.Provider. There is no price table: see the package doc. +func (p *Provider) Offerings(context.Context) ([]provider.Offering, error) { return nil, nil } + +// MapAccelerator implements provider.Provider: the families of every cluster that has the +// accelerator. A cluster that lacks it fails its own Provision with ErrUnsupportedAccelerator +// (see familiesFor), which blocks only that cluster. The ids are instance families, and count is +// ignored: Karpenter picks the size, which is what lets several pods pack onto one node. +func (p *Provider) MapAccelerator(canonical string, _ int32) ([]string, bool) { + fs, ok := p.offered[strings.ToUpper(canonical)] + return slices.Clone(fs), ok +} + +// Provision implements provider.Provider: create the real pod in the cluster req.Region +// names, then wait for capacity. A pod that gets none is deleted before returning, or +// Karpenter would launch a node for it later. +func (p *Provider) Provision( + ctx context.Context, pod *corev1.Pod, req provider.ProvisionRequest, +) (provider.ProvisionResult, error) { + if pod == nil || req.ClaimName == "" { + return provider.ProvisionResult{}, errors.New("kubernetes: nil pod or empty ClaimName") + } + if mode := req.Egress.ModeOrOpen(); mode != nebulav1alpha1.EgressOpen { + return provider.ProvisionResult{}, fmt.Errorf("kubernetes: cannot enforce egress mode %q yet", mode) + } + if req.RegistryAuth != nil { + // ECR pulls go through the node role; other registries need a pull Secret, not built yet. + return provider.ProvisionResult{}, req.RegistryAuth.Unsupported(provider.ProviderKubernetes) + } + cl, ok := p.clusters[req.Region] + if !ok { + return provider.ProvisionResult{}, fmt.Errorf("kubernetes: no cluster in region %q", req.Region) + } + families, err := familiesFor(cl, pod) + if err != nil { + return provider.ProvisionResult{}, err + } + real := buildPod(pod, podSpec{ + claim: req.ClaimName, + selection: cl.nodeAutoscaler.selection(families, req.CapacityType), + env: req.Env.Config, + hasSecrets: len(req.Env.Secrets) > 0, + costLabels: p.costLabels, + }) + kube := cl.clients.kube + + if err := ensureNamespace(ctx, kube, real.Namespace); err != nil { + return provider.ProvisionResult{}, err + } + created, err := createOrAdopt(ctx, cl.clients, real) + if err != nil { + return provider.ProvisionResult{}, err + } + if len(req.Env.Secrets) > 0 { + if err := ensureEnvSecret(ctx, kube, created, req.Env.Secrets); err != nil { + cleanup(ctx, cl.clients, real.Namespace, real.Name) + return provider.ProvisionResult{}, err + } + } + bound, err := waitForProvision(ctx, cl.clients, cl.nodeAutoscaler, real.Namespace, real.Name) + if err != nil { + cleanup(ctx, cl.clients, real.Namespace, real.Name) + return provider.ProvisionResult{}, withTier(req.CapacityType, err) + } + // Unbound is accepted, not reserved: the node may still be booting. List settles it, + // reporting no capacity once the pod is unbound past its node autoscaler's bindTimeout. + return provider.ProvisionResult{ + InstanceID: instanceID(req.Region, real.Namespace, real.Name), + Reserved: bound, + }, nil +} + +// cleanup deletes a pod that got no capacity, on a context that outlives the expired one. +// A failure is only logged: the Pod fails regardless, and the NodeClaim teardown finds the +// leftover through FindByClaim. +func cleanup(ctx context.Context, cc clusterClients, namespace, name string) { + if err := deleteInstance(context.WithoutCancel(ctx), cc, namespace, name); err != nil { + logf.FromContext(ctx).Error(err, "kubernetes: could not delete pod without capacity; teardown will retry", + "namespace", namespace, "pod", name) + } +} + +// familiesFor resolves the Pod's accelerator to cl's instance families; a CPU-only Pod gets none. +func familiesFor(cl cluster, pod *corev1.Pod) ([]string, error) { + canonical, count, err := util.AcceleratorRequest(pod) + if err != nil { + return nil, fmt.Errorf("kubernetes: %w", err) + } + if canonical == "" || count <= 0 { + return nil, nil + } + fs, ok := cl.families[strings.ToUpper(canonical)] + if !ok { + return nil, fmt.Errorf("kubernetes: unsupported accelerator %q: %w", canonical, provider.ErrUnsupportedAccelerator) + } + return fs, nil +} + +// ensureNamespace creates the Pod's namespace in a remote cluster on first use. In the host +// cluster it already exists. +func ensureNamespace(ctx context.Context, kube kubeClient, name string) error { + _, err := kube.CoreV1().Namespaces().Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return nil + } + if !apierrors.IsNotFound(err) { + return fmt.Errorf("kubernetes: get namespace %q: %w", name, classifyAPI(err)) + } + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{nebulav1alpha1.ManagedByLabel: nebulav1alpha1.ManagedByValue}, + }} + if _, err := kube.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}); err != nil && + !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("kubernetes: create namespace %q: %w", name, classifyAPI(err)) + } + return nil +} + +// Terminate implements provider.Provider. The region is redundant with the id. +func (p *Provider) Terminate(ctx context.Context, instanceID, _ string) error { + if instanceID == "" { + return nil + } + cl, ns, name, err := p.locate(instanceID) + if err != nil { + return err + } + return deleteInstance(ctx, cl.clients, ns, name) +} + +// Get implements provider.Provider. +func (p *Provider) Get(ctx context.Context, instanceID, _ string) (*provider.Instance, error) { + cl, ns, name, err := p.locate(instanceID) + if err != nil { + return nil, err + } + pod, err := cl.clients.kube.CoreV1().Pods(ns).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("kubernetes: get pod %s: %w", instanceID, classifyAPI(err)) + } + inst := p.toInstance(cl, pod) + return &inst, nil +} + +// List implements provider.Provider. One unreadable cluster fails the whole call: leaving +// its pods out would make the poll loop read every one of them as terminated. +func (p *Provider) List(ctx context.Context) ([]provider.Instance, error) { + var out []provider.Instance + for _, region := range p.order { + pods, err := p.listPods(ctx, p.clusters[region], "") + if err != nil { + return nil, err + } + for i := range pods { + out = append(out, p.toInstance(p.clusters[region], &pods[i])) + } + } + return out, nil +} + +// FindByClaim implements provider.Provider. region names the claim's cluster; an unknown one +// is searched for everywhere rather than reported absent, since absent means "nothing to tear down". +func (p *Provider) FindByClaim(ctx context.Context, claimName, region string) (*provider.Instance, error) { + regions := p.order + if _, ok := p.clusters[region]; ok { + regions = []string{region} + } + for _, r := range regions { + pods, err := p.listPods(ctx, p.clusters[r], fields.OneTermEqualSelector("metadata.name", claimName).String()) + if err != nil { + return nil, err + } + for i := range pods { + // Checked again here: the selector is the server's job, and not every client honours it. + if pods[i].Name == claimName { + inst := p.toInstance(p.clusters[r], &pods[i]) + return &inst, nil + } + } + } + return nil, nil +} + +// listPods lists this provider's pods in every namespace of cl, narrowed by fieldSelector. +func (p *Provider) listPods(ctx context.Context, cl cluster, fieldSelector string) ([]corev1.Pod, error) { + list, err := cl.clients.kube.CoreV1().Pods(metav1.NamespaceAll).List(ctx, metav1.ListOptions{ + LabelSelector: labels.Set(ownerLabels()).String(), + FieldSelector: fieldSelector, + // Served from the API server's watch cache rather than etcd. + ResourceVersion: "0", + }) + if err != nil { + return nil, fmt.Errorf("kubernetes: list pods in cluster %q: %w", cl.cfg.Name, classifyAPI(err)) + } + return list.Items, nil +} + +func (p *Provider) locate(id string) (cluster, string, string, error) { + region, ns, name, err := parseInstanceID(id) + if err != nil { + return cluster{}, "", "", err + } + cl, ok := p.clusters[region] + if !ok { + return cluster{}, "", "", fmt.Errorf("kubernetes: instance %q is in region %q, which has no cluster", id, region) + } + return cl, ns, name, nil +} + +// ClassifyProvisionError implements provider.Provider. Every block is confined to one region, +// that is one cluster, auth failures included. +func (p *Provider) ClassifyProvisionError(err error, accelerator, region string) provider.BlockScope { + if err == nil { + return provider.BlockScope{} + } + var te *tierError + var tier nebulav1alpha1.CapacityType + if errors.As(err, &te) { + tier = te.tier + } + scope := provider.ClassifyError(err, tier, accelerator) + if scope == (provider.BlockScope{}) { + return scope // block nothing; decorating it would create a block + } + r := region + if scope.DenyAll { + // DenyAll would block every region, but a 401/403 is one cluster's access entry or + // RBAC, not shared by the others. So block everything in this region instead. + anyAccelerator := "" + return provider.BlockScope{Accelerator: &anyAccelerator, Region: &r} + } + scope.Region = &r + return scope +} + +// ResolveRegions implements provider.Provider. Candidates are the clusters' AWS regions, one +// cluster each (see Config.Validate), so a shortage blocks that region, as EC2 capacity is the +// region's rather than a cluster's. A declared value matches a cluster's Geography or its exact +// Region, both from the cluster table; regions come back in declared order, then config order. +// +// nil/[] => every cluster +// ["us"] => clusters with geography us +// ["us-east-1"] => clusters in us-east-1 +// +// narrowTo, the Pod's geographies, keeps only clusters with one of them. +func (p *Provider) ResolveRegions(declared, narrowTo []string) []string { + var within []string + for _, g := range narrowTo { + if g = strings.ToLower(strings.TrimSpace(g)); provider.IsGeography(g) { + within = append(within, g) + } + } + tokens := declared + if len(tokens) == 0 { + tokens = []string{""} // matches every cluster + } + var out []string + for _, tok := range tokens { + tok = strings.TrimSpace(tok) + for _, region := range p.order { + cl := p.clusters[region].cfg + if matches(cl, tok) && (len(within) == 0 || slices.Contains(within, cl.Geography)) && + !slices.Contains(out, region) { + out = append(out, region) + } + } + } + return out +} + +func matches(cl ClusterConfig, token string) bool { + return token == "" || token == cl.Region || strings.EqualFold(token, cl.Geography) +} + +// instanceID is "//": the region names the cluster, and is part of the +// id because Get and Terminate are handed nothing else that names it. +func instanceID(region, namespace, name string) string { + return region + "/" + namespace + "/" + name +} + +func parseInstanceID(id string) (region, namespace, name string, err error) { + parts := strings.SplitN(id, "/", 3) + if len(parts) != 3 || parts[0] == "" || parts[1] == "" || parts[2] == "" { + return "", "", "", fmt.Errorf("kubernetes: malformed instance id %q", id) + } + return parts[0], parts[1], parts[2], nil +} + +// createOrAdopt creates the real pod, or returns the live one a crashed or retried Provision +// left behind. A finished one is the previous incarnation of a reused claim name and is +// replaced; anything else holding the name is refused rather than adopted. +func createOrAdopt(ctx context.Context, cc clusterClients, want *corev1.Pod) (*corev1.Pod, error) { + pods := cc.kube.CoreV1().Pods(want.Namespace) + got, err := pods.Get(ctx, want.Name, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return nil, fmt.Errorf("kubernetes: get pod %s/%s: %w", want.Namespace, want.Name, classifyAPI(err)) + case !ownedByUs(got): + return nil, fmt.Errorf("kubernetes: pod %s/%s exists and is not Nebula's", want.Namespace, want.Name) + case got.DeletionTimestamp == nil && !util.Finished(got.Status.Phase): + return got, nil + default: + if err := deleteAndWait(ctx, cc, want.Namespace, want.Name); err != nil { + return nil, err + } + } + created, err := pods.Create(ctx, want, metav1.CreateOptions{}) + if err != nil { + return nil, fmt.Errorf("kubernetes: create pod %s/%s: %w", want.Namespace, want.Name, classifyAPI(err)) + } + return created, nil +} + +// ensureEnvSecret writes the Secret-sourced env, owned by the pod so Kubernetes deletes it with +// the pod; nothing else deletes it. +func ensureEnvSecret(ctx context.Context, kube kubeClient, pod *corev1.Pod, env map[string]string) error { + want := envSecret(pod.Namespace, pod.Name, env) + want.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "v1", Kind: "Pod", Name: pod.Name, UID: pod.UID, + }} + secrets := kube.CoreV1().Secrets(pod.Namespace) + _, err := secrets.Create(ctx, want, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + err = replaceEnvSecret(ctx, kube, want) + } + if err != nil { + return fmt.Errorf("kubernetes: write env secret for %s/%s: %w", pod.Namespace, pod.Name, classifyAPI(err)) + } + return nil +} + +// replaceEnvSecret deletes the existing env Secret, a previous pod's of the same name, and +// creates it afresh. Only if Nebula made it: a user's own Secret with the name is refused, +// never deleted. +func replaceEnvSecret(ctx context.Context, kube kubeClient, want *corev1.Secret) error { + secrets := kube.CoreV1().Secrets(want.Namespace) + got, err := secrets.Get(ctx, want.Name, metav1.GetOptions{}) + if err == nil && !ownedByUs(got) { + return fmt.Errorf("secret %s/%s exists and is not Nebula's", want.Namespace, want.Name) + } + if err == nil { + err = secrets.Delete(ctx, want.Name, metav1.DeleteOptions{Preconditions: metav1.NewUIDPreconditions(string(got.UID))}) + } + if err != nil && !apierrors.IsNotFound(err) { + return err + } + _, err = secrets.Create(ctx, want, metav1.CreateOptions{}) + return err +} + +// deleteInstance removes the real pod; already gone is success. Its env Secret follows +// through the ownerReference (see ensureEnvSecret). +func deleteInstance(ctx context.Context, cc clusterClients, namespace, name string) error { + err := cc.kube.CoreV1().Pods(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("kubernetes: delete pod %s/%s: %w", namespace, name, classifyAPI(err)) + } + return nil +} + +// deleteAndWait deletes the pod and waits for the name to free up, so a replacement can take it. +func deleteAndWait(ctx context.Context, cc clusterClients, namespace, name string) error { + if err := deleteInstance(ctx, cc, namespace, name); err != nil { + return err + } + t := time.NewTicker(waitInterval) + defer t.Stop() + for { + _, err := cc.kube.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("kubernetes: previous pod %s/%s still terminating: %w", namespace, name, ctx.Err()) + case <-t.C: + } + } +} + +// toInstance normalizes a real pod. Region is the cluster's AWS region, the token the claim +// carries. +// +// A pod still unbound past its node autoscaler's bindTimeout reports InstanceUnschedulable: it is +// the bind wait an unreserved Provision handed over, timed from the pod's creation so a +// manager restart does not reset it. No events are read here; the fast refusal is caught in +// Provision (see waitForProvision). +func (p *Provider) toInstance(cl cluster, pod *corev1.Pod) provider.Instance { + inst := provider.Instance{ + ID: instanceID(cl.cfg.Region, pod.Namespace, pod.Name), + ClaimName: pod.Name, + State: toState(pod), + Region: cl.cfg.Region, + Endpoint: pod.Status.PodIP, + } + timeout := cl.nodeAutoscaler.bindTimeout() + if pod.Spec.NodeName == "" && inst.State == provider.InstancePending && + p.now().Sub(pod.CreationTimestamp.Time) > timeout { + inst.State = provider.InstanceUnschedulable + } + return inst +} + +// toState maps the real pod's phase. Unknown stays Pending, so a node that stopped reporting +// keeps the poll loop watching rather than going terminal on a pod that may still run. +func toState(pod *corev1.Pod) provider.InstanceState { + switch pod.Status.Phase { + case corev1.PodRunning: + return provider.InstanceRunning + case corev1.PodSucceeded: + return provider.InstanceTerminated + case corev1.PodFailed: + return provider.InstanceFailed + default: + return provider.InstancePending + } +} + +// classifyAPI marks a 401 or 403 as ErrAuth. ClassifyProvisionError narrows that to the one +// cluster, since another cluster's credentials and RBAC are separate. +func classifyAPI(err error) error { + if apierrors.IsUnauthorized(err) || apierrors.IsForbidden(err) { + return fmt.Errorf("%w: %w", provider.ErrAuth, err) + } + return err +} + +// tierError carries the requested tier to ClassifyProvisionError, which is not handed one, +// so a Spot shortage does not block OnDemand. +type tierError struct { + tier nebulav1alpha1.CapacityType + err error +} + +func (e *tierError) Error() string { return e.err.Error() } +func (e *tierError) Unwrap() error { return e.err } + +func withTier(tier nebulav1alpha1.CapacityType, err error) error { + return &tierError{tier: tier, err: err} +} diff --git a/pkg/provider/kubernetes/kubernetes_test.go b/pkg/provider/kubernetes/kubernetes_test.go new file mode 100644 index 0000000..20c4af4 --- /dev/null +++ b/pkg/provider/kubernetes/kubernetes_test.go @@ -0,0 +1,407 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + kubefake "k8s.io/client-go/kubernetes/fake" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/util" +) + +type fakeCatalog struct{ rows []provider.Offering } + +func (c fakeCatalog) Offerings(string) []provider.Offering { return c.rows } + +var testCatalog = fakeCatalog{rows: []provider.Offering{ + {AcceleratorType: "H100", AcceleratorID: "p5.4xlarge", GPUCount: 1}, + {AcceleratorType: "H100", AcceleratorID: "p5.48xlarge", GPUCount: 8}, + {AcceleratorType: "L4", AcceleratorID: "g6.xlarge", GPUCount: 1}, + {AcceleratorType: "L4", AcceleratorID: "gr6.4xlarge", GPUCount: 1}, +}} + +var testClusters = []ClusterConfig{ + {Name: "eks-use1", Provider: ClusterProviderEKS, Region: "us-east-1", Geography: "us"}, + {Name: "eks-eun1", Provider: ClusterProviderEKS, Region: "eu-north-1", Geography: "eu"}, + {Name: "eks-usw2", Provider: ClusterProviderEKS, Region: "us-west-2", Geography: "us"}, +} + +// newTestProvider fills any cluster clients leaves out with an empty fake one. +func newTestProvider(clients map[string]clusterClients) *Provider { + all := make(map[string]clusterClients, len(testClusters)) + for _, cl := range testClusters { + cc, ok := clients[cl.Name] + if !ok { + cc = clusterClients{kube: kubefake.NewClientset()} + } + all[cl.Name] = cc + } + return New(testClusters, all, testCatalog, []string{"org-id"}) +} + +// placeholder is a Pod as Nebula's webhook and placement leave it. +func placeholder() *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "work-1", Namespace: "org-1", + Labels: map[string]string{ + nebulav1alpha1.EnabledLabel: nebulav1alpha1.EnabledValue, + nebulav1alpha1.AcceleratorTypeLabel: "h100", + "org-id": "o1", + "batch.kubernetes.io/controller-uid": "job-uid", + }, + }, + Spec: corev1.PodSpec{ + NodeName: "nebula-kubernetes", + NodeSelector: map[string]string{nebulav1alpha1.ProviderLabel: provider.ProviderKubernetes}, + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{{ + Name: "main", Image: "img:1", Command: []string{"run"}, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{util.NvidiaGPUResource: resource.MustParse("1")}, + Limits: corev1.ResourceList{util.NvidiaGPUResource: resource.MustParse("1")}, + }, + }}, + Volumes: []corev1.Volume{{ + Name: "kube-api-access-abcde", + VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{}}, + }}, + }, + } +} + +func TestConfigValidate(t *testing.T) { + ok := ClusterConfig{Name: "a", Provider: "eks", Region: "us-east-1", Geography: "us"} + cases := map[string]struct { + clusters []ClusterConfig + wantErr string + }{ + "valid": {clusters: []ClusterConfig{ok}}, + "empty": {wantErr: "no clusters"}, + "duplicate": {clusters: []ClusterConfig{ok, ok}, wantErr: "duplicate"}, + "unsupported": { + clusters: []ClusterConfig{{Name: "a", Provider: "gke", Region: "r"}}, + wantErr: "unsupported provider", + }, + "missing": {clusters: []ClusterConfig{{Name: "a", Provider: "eks"}}, wantErr: "required"}, + "region twice": { + clusters: []ClusterConfig{ok, {Name: "b", Provider: "eks", Region: "us-east-1", Geography: "us"}}, + wantErr: "a region holds one cluster", + }, + "geography": { + clusters: []ClusterConfig{{Name: "a", Provider: "eks", Region: "us-east-1", Geography: "usa"}}, + wantErr: "geography", + }, + "node autoscaler": { + clusters: []ClusterConfig{{ + Name: "a", Provider: "eks", Region: "r", Geography: "us", NodeAutoscaler: "ca", + }}, + wantErr: "unsupported node autoscaler", + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + err := (&Config{Clusters: tc.clusters}).Validate() + if tc.wantErr == "" && err != nil || tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)) { + t.Fatalf("Validate() = %v, want error containing %q", err, tc.wantErr) + } + }) + } +} + +func TestEnvSecretName(t *testing.T) { + if got := envSecretName("org-1-work-1"); got != "org-1-work-1-env" { + t.Fatalf("envSecretName = %q", got) + } + long := strings.Repeat("a", maxNameLen) + a, b := envSecretName(long), envSecretName(long[:maxNameLen-1]+"b") + if len(a) > maxNameLen || !strings.HasSuffix(a, envSecretSuffix) { + t.Fatalf("envSecretName = %q (%d chars)", a, len(a)) + } + if a == b { + t.Fatal("two long names collided") + } +} + +func TestBuildPod(t *testing.T) { + got := buildPod(placeholder(), podSpec{ + claim: "org-1-work-1", + selection: karpenter{}.selection([]string{"p5"}, nebulav1alpha1.CapacitySpot), + costLabels: []string{"org-id"}, + env: map[string]string{"B": "2", "A": "1"}, + hasSecrets: true, + }) + if got.Name != "org-1-work-1" || got.Namespace != "org-1" { + t.Fatalf("got %s/%s", got.Namespace, got.Name) + } + if got.Labels["org-id"] != "o1" || !ownedByUs(got) { + t.Fatalf("labels = %v", got.Labels) + } + notCopied := []string{ + nebulav1alpha1.EnabledLabel, "batch.kubernetes.io/controller-uid", nebulav1alpha1.AcceleratorTypeLabel, + } + for _, k := range notCopied { + if _, ok := got.Labels[k]; ok { + t.Errorf("label %q was copied", k) + } + } + if got.Spec.NodeName != "" || len(got.Spec.NodeSelector) > 0 || len(got.Spec.Volumes) > 0 { + t.Errorf("Nebula scheduling or volumes leaked: %+v", got.Spec) + } + if got.Annotations[doNotDisruptAnnotation] != "true" { + t.Errorf("annotations = %v", got.Annotations) + } + c := got.Spec.Containers[0] + if len(c.EnvFrom) != 1 || c.EnvFrom[0].SecretRef.Name != "org-1-work-1-env" { + t.Errorf("envFrom = %+v, want the env Secret", c.EnvFrom) + } + if want := []corev1.EnvVar{{Name: "A", Value: "1"}, {Name: "B", Value: "2"}}; !slices.Equal(c.Env, want) { + t.Errorf("env = %+v, want plain values inline and sorted", c.Env) + } + noSecrets := buildPod(placeholder(), podSpec{claim: "org-1-work-1"}) + if len(noSecrets.Spec.Containers[0].EnvFrom) != 0 { + t.Error("with no Secrets there is no env Secret to load") + } + terms := got.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms + reqs := terms[0].MatchExpressions + want := map[string]string{instanceFamilyLabel: "p5", capacityTypeLabel: capacityTypeSpot} + for _, r := range reqs { + if want[r.Key] != strings.Join(r.Values, ",") { + t.Errorf("affinity %s = %v", r.Key, r.Values) + } + delete(want, r.Key) + } + if len(want) > 0 { + t.Errorf("missing affinity %v", want) + } +} + +func TestMapAccelerator(t *testing.T) { + p := newTestProvider(nil) + if got, ok := p.MapAccelerator("h100", 8); !ok || !slices.Equal(got, []string{"p5"}) { + t.Errorf("H100 = %v, %v", got, ok) + } + if got, ok := p.MapAccelerator("L4", 1); !ok || !slices.Equal(got, []string{"g6", "gr6"}) { + t.Errorf("L4 = %v, %v", got, ok) + } + if _, ok := p.MapAccelerator("TPU", 1); ok { + t.Error("TPU should be unsupported") + } +} + +func TestResolveRegions(t *testing.T) { + p := newTestProvider(nil) + cases := map[string]struct { + declared, narrowTo, want []string + }{ + "unconstrained": {want: []string{"us-east-1", "eu-north-1", "us-west-2"}}, + "geography": {declared: []string{"us"}, want: []string{"us-east-1", "us-west-2"}}, + "region": {declared: []string{"eu-north-1"}, want: []string{"eu-north-1"}}, + "cluster name": {declared: []string{"eks-usw2"}}, + "declared order": {declared: []string{"us-west-2", "us-east-1"}, want: []string{"us-west-2", "us-east-1"}}, + "narrowed": {narrowTo: []string{"eu"}, want: []string{"eu-north-1"}}, + "narrowed out": {declared: []string{"us-east-1"}, narrowTo: []string{"eu"}}, + "no match": {declared: []string{"ap-south-1"}}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + if got := p.ResolveRegions(tc.declared, tc.narrowTo); !slices.Equal(got, tc.want) { + t.Fatalf("ResolveRegions = %v, want %v", got, tc.want) + } + }) + } +} + +func TestClassifyProvisionError(t *testing.T) { + p := newTestProvider(nil) + region := "us-east-1" + + if s := p.ClassifyProvisionError(nil, "H100:1", region); s != (provider.BlockScope{}) { + t.Errorf("nil error blocked %+v", s) + } + + spot := p.ClassifyProvisionError(withTier(nebulav1alpha1.CapacitySpot, + fmt.Errorf("x: %w", provider.ErrNoCapacity)), "H100:1", region) + if spot.CapacityType != nebulav1alpha1.CapacitySpot || *spot.Region != region || *spot.Accelerator != "H100:1" { + t.Errorf("spot shortage scope = %+v", spot) + } + + auth := p.ClassifyProvisionError(classifyAPI(forbidden()), "H100:1", region) + if auth.DenyAll || *auth.Region != region || *auth.Accelerator != "" { + t.Errorf("auth failure must block only the cluster, got %+v", auth) + } +} + +func forbidden() error { + return apierrors.NewForbidden(schema.GroupResource{Resource: "pods"}, "x", fmt.Errorf("nope")) +} + +func TestAcceleratorTolerations(t *testing.T) { + c := corev1.Container{Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("4"), + corev1.ResourceMemory: resource.MustParse("8Gi"), + "amd.com/gpu": resource.MustParse("1"), + "hugepages-2Mi": resource.MustParse("1Gi"), + "aws.amazon.com/neuron": resource.MustParse("1"), + }, + Limits: corev1.ResourceList{"amd.com/gpu": resource.MustParse("1")}, + }} + tols := acceleratorTolerations(&c) + keys := make([]string, 0, len(tols)) + for _, tol := range tols { + if tol.Operator != corev1.TolerationOpExists || tol.Effect != corev1.TaintEffectNoSchedule { + t.Errorf("toleration %+v, want Exists/NoSchedule", tol) + } + keys = append(keys, tol.Key) + } + if want := []string{"amd.com/gpu", "aws.amazon.com/neuron"}; !slices.Equal(keys, want) { + t.Fatalf("tolerated keys = %v, want %v", keys, want) + } + if got := acceleratorTolerations(&corev1.Container{}); len(got) != 0 { + t.Fatalf("CPU-only container tolerates %v, want nothing", got) + } +} + +type catalogByProvider map[string][]provider.Offering + +func (c catalogByProvider) Offerings(name string) []provider.Offering { return c[name] } + +// Each cluster maps GPUs through its own cloud's catalog; the provider offers their union. +func TestFamiliesPerCluster(t *testing.T) { + clusterCatalogs["l4cloud"] = "l4-only" + t.Cleanup(func() { delete(clusterCatalogs, "l4cloud") }) + cat := catalogByProvider{ + provider.ProviderAWS: {{AcceleratorType: "H100", AcceleratorID: "p5.48xlarge", GPUCount: 8}}, + "l4-only": {{AcceleratorType: "L4", AcceleratorID: "g6.xlarge", GPUCount: 1}}, + } + clusters := []ClusterConfig{ + {Name: "eks-use1", Provider: ClusterProviderEKS, Region: "us-east-1", Geography: "us"}, + {Name: "l4", Provider: "l4cloud", Region: "us-east-2", Geography: "us"}, + } + tc := newTestCluster(t, true) + p := New(clusters, map[string]clusterClients{ + "eks-use1": {kube: kubefake.NewClientset()}, "l4": tc.clients(), + }, cat, nil) + + if fs, ok := p.MapAccelerator("H100", 1); !ok || !slices.Equal(fs, []string{"p5"}) { + t.Fatalf("MapAccelerator(H100) = %v, %v; the provider offers what any cluster has", fs, ok) + } + req := provider.ProvisionRequest{ClaimName: "org-1-work-1", Region: "us-east-2"} + _, err := p.Provision(context.Background(), placeholder(), req) + if !errors.Is(err, provider.ErrUnsupportedAccelerator) { + t.Fatalf("Provision of H100 on the L4-only cluster = %v, want ErrUnsupportedAccelerator", err) + } +} + +// The real pod keeps what the user wrote and loses what is Nebula's or the home cluster's. +func TestBuildPodCopiesThenOverrides(t *testing.T) { + local := placeholder() + probe := &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: []string{"ready"}}}} + local.Spec.Containers[0].ReadinessProbe = probe + local.Spec.Containers[0].Env = []corev1.EnvVar{{Name: "SECRET", ValueFrom: &corev1.EnvVarSource{}}} + local.Spec.SecurityContext = &corev1.PodSecurityContext{RunAsNonRoot: ptrTo(true)} + local.Spec.Tolerations = []corev1.Toleration{ + {Key: nebulav1alpha1.ProviderLabel, Operator: corev1.TolerationOpExists}, + {Key: "team", Operator: corev1.TolerationOpEqual, Value: "ml"}, + } + local.Spec.SchedulingGates = []corev1.PodSchedulingGate{{Name: nebulav1alpha1.ProviderSelectionGate}} + local.Spec.ServiceAccountName = "default" + local.Spec.PriorityClassName, local.Spec.Priority = "high", ptrTo(int32(1000)) + zone := func(z string) corev1.NodeSelectorTerm { + return corev1.NodeSelectorTerm{MatchExpressions: []corev1.NodeSelectorRequirement{ + {Key: "zone", Operator: corev1.NodeSelectorOpIn, Values: []string{z}}, + }} + } + local.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{zone("a"), zone("b")}, + }, + }} + + got := buildPod(local, podSpec{ + claim: "org-1-work-1", selection: karpenter{}.selection([]string{"p5"}, nebulav1alpha1.CapacityOnDemand), + }) + spec, c := got.Spec, got.Spec.Containers[0] + + if c.ReadinessProbe == nil || spec.SecurityContext == nil { + t.Error("the user's probe and security context must carry over") + } + // The placeholder's env holds a reference into the home cluster; only the resolved env may + // appear, and this test passes none. + if c.Env != nil || len(c.EnvFrom) != 0 || len(c.VolumeMounts) != 0 || len(spec.Volumes) != 0 { + t.Errorf("env/volumes leaked: env=%v envFrom=%v mounts=%v volumes=%v", c.Env, c.EnvFrom, c.VolumeMounts, spec.Volumes) + } + if spec.NodeName != "" || len(spec.SchedulingGates) != 0 || len(spec.NodeSelector) != 0 { + t.Errorf("Nebula's scheduling leaked: node=%q gates=%v selector=%v", + spec.NodeName, spec.SchedulingGates, spec.NodeSelector) + } + if spec.ServiceAccountName != "" { + t.Error("the service account is the home cluster's and must be cleared") + } + keys := make([]string, 0, len(spec.Tolerations)) + for _, tol := range spec.Tolerations { + keys = append(keys, tol.Key) + } + if slices.Contains(keys, nebulav1alpha1.ProviderLabel) || !slices.Contains(keys, "team") || + !slices.Contains(keys, string(util.NvidiaGPUResource)) { + t.Errorf("tolerations = %v; want the user's and the GPU's, not the virtual node's", keys) + } + // Both of the user's ORed terms must also carry the instance family, or one of them + // would let the pod land on any node. + for i, term := range spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + if !slices.ContainsFunc(term.MatchExpressions, func(r corev1.NodeSelectorRequirement) bool { + return r.Key == instanceFamilyLabel + }) { + t.Errorf("term %d lacks the instance family: %v", i, term.MatchExpressions) + } + } + if local.Spec.NodeName != "nebula-kubernetes" || len(local.Spec.Tolerations) != 2 { + t.Error("buildPod must not modify the placeholder") + } +} + +func TestIsExtendedResource(t *testing.T) { + for name, want := range map[corev1.ResourceName]bool{ + "nvidia.com/gpu": true, + "amd.com/gpu": true, + "aws.amazon.com/neuron": true, + corev1.ResourceCPU: false, + corev1.ResourceMemory: false, + "hugepages-2Mi": false, + "kubernetes.io/batch-cpu": false, // the reserved domain is native + "requests.nvidia.com/gpu": false, // a quota name, not a resource + "example.com/has space": false, // not a valid qualified name + } { + if got := isExtendedResource(name); got != want { + t.Errorf("isExtendedResource(%q) = %v, want %v", name, got, want) + } + } +} diff --git a/pkg/provider/kubernetes/node_autoscaler.go b/pkg/provider/kubernetes/node_autoscaler.go new file mode 100644 index 0000000..6c01021 --- /dev/null +++ b/pkg/provider/kubernetes/node_autoscaler.go @@ -0,0 +1,63 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + "time" + + corev1 "k8s.io/api/core/v1" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" +) + +// NodeAutoscalerKarpenter is the default, and today the only, node autoscaler. +const NodeAutoscalerKarpenter = "karpenter" + +// nodeAutoscaler is everything that depends on what adds nodes to a cluster: the labels a pod +// selects to get the right GPU and tier, how it says it gave up, and how long a healthy launch +// takes. Supporting another one is one more implementation in nodeAutoscalers. +type nodeAutoscaler interface { + // selection steers the pod onto nodes with the GPU families and tier. No families is a + // CPU-only pod; an empty tier leaves it to the cluster. + selection(families []string, tier nebulav1alpha1.CapacityType) nodeSelection + // refused looks once at a pending, unbound pod for the node autoscaler's verdict that it + // will not get it a node, returning an error wrapping provider.ErrNoCapacity, or nil. It + // only speeds up failure: success is the pod binding, and silence ends at bindTimeout. + refused(ctx context.Context, cc clusterClients, pod *corev1.Pod) error + // provisionTimeout is how long Provision watches a new pod for a binding or refused: long + // enough for the node autoscaler to decide, short enough not to hold a worker through + // node boot. Capabilities.ProvisionTimeout adds provisionSlack for the create calls. + provisionTimeout() time.Duration + // bindTimeout is how long a healthy launch may take to bind the pod, node boot included, + // counted from the pod's creation. Past it, List reports no capacity. + bindTimeout() time.Duration + // verify fails when the node autoscaler is not installed in the cluster, so a mismatch shows + // up at startup instead of as pods that never schedule. + verify(ctx context.Context, cc clusterClients) error +} + +// nodeSelection is what a node autoscaler adds to the real pod. +type nodeSelection struct { + affinity *corev1.Affinity + tolerations []corev1.Toleration + annotations map[string]string +} + +var nodeAutoscalers = map[string]nodeAutoscaler{ + NodeAutoscalerKarpenter: karpenter{}, +} diff --git a/pkg/provider/kubernetes/pod.go b/pkg/provider/kubernetes/pod.go new file mode 100644 index 0000000..65b3c65 --- /dev/null +++ b/pkg/provider/kubernetes/pod.go @@ -0,0 +1,239 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "crypto/sha256" + "encoding/hex" + "maps" + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +const ( + // maxNameLen is the object-name cap. The real pod is named after the claim, which + // util.ClaimName keeps within it; only the Secret's suffix can push past it. + maxNameLen = 253 + nameHashLen = 10 + envSecretSuffix = "-env" + + labelTrue = "true" +) + +// ownerLabels mark what this provider created. Placeholders carry neither (placement puts the +// provider in their nodeSelector, not their labels), so in the host cluster List never +// mistakes one for an instance; both are required in case a user sets one of them. +func ownerLabels() map[string]string { + return map[string]string{ + nebulav1alpha1.ManagedByLabel: nebulav1alpha1.ManagedByValue, + nebulav1alpha1.ProviderLabel: provider.ProviderKubernetes, + } +} + +func ownedByUs(obj metav1.Object) bool { + for k, v := range ownerLabels() { + if obj.GetLabels()[k] != v { + return false + } + } + return true +} + +// envSecretName is the env Secret's name for the real pod podName, with a hash of podName in +// place of its tail when the suffix would pass maxNameLen, so two long names never meet. +func envSecretName(podName string) string { + if len(podName)+len(envSecretSuffix) <= maxNameLen { + return podName + envSecretSuffix + } + sum := sha256.Sum256([]byte(podName)) + keep := maxNameLen - len(envSecretSuffix) - nameHashLen - 1 + return podName[:keep] + "-" + hex.EncodeToString(sum[:])[:nameHashLen] + envSecretSuffix +} + +// podSpec is everything buildPod needs that is not on the placeholder. +type podSpec struct { + claim string + selection nodeSelection + // env is ProvisionRequest.Env.Config, set inline; hasSecrets says Env.Secrets is + // non-empty, so the pod loads the env Secret holding it. + env map[string]string + hasSecrets bool + // costLabels are the placeholder's --cost-labels keys, copied so cost attribution follows + // the sandbox to the real pod. Nothing else is: a controller label such as + // batch.kubernetes.io/controller-uid would let the placeholder's Job adopt the real pod, + // since both share a namespace. + costLabels []string +} + +// buildPod renders the real pod from the placeholder: a copy of its spec, so the sandbox runs +// as the user wrote it (probes, hooks, security context, tolerations), minus what cannot +// carry over and with Nebula's own scheduling replaced. Metadata is not copied: labels are +// chosen below, since the opt-in label alone would send the pod back through Nebula's +// webhook in the host cluster, and a controller label would let the placeholder's Job adopt it. +func buildPod(local *corev1.Pod, s podSpec) *corev1.Pod { + // Named after the claim like every provider's instance (util.ClaimName: "-"), + // which also keeps it off the placeholder's own name in the host cluster. + name := s.claim + + labels := map[string]string{} + for _, k := range s.costLabels { + if v, ok := local.Labels[k]; ok { + labels[k] = v + } + } + maps.Copy(labels, ownerLabels()) // last, so a copied key cannot overwrite them + + spec := *local.Spec.DeepCopy() + + // Nebula's scheduling: the placeholder is bound to the virtual node through these. + spec.NodeName = "" + spec.SchedulingGates = nil + delete(spec.NodeSelector, nebulav1alpha1.ProviderLabel) + spec.Tolerations = slices.DeleteFunc(spec.Tolerations, func(t corev1.Toleration) bool { + return t.Key == nebulav1alpha1.ProviderLabel + }) + spec.Tolerations = append(spec.Tolerations, acceleratorTolerations(&spec.Containers[0])...) + spec.Tolerations = append(spec.Tolerations, s.selection.tolerations...) + spec.Affinity = withNodeRequirements(spec.Affinity, s.selection.affinity) + + // References to objects that live in the home cluster. The webhook admits no volume + // but the service-account token, which is simply not mounted here. + spec.ServiceAccountName, spec.DeprecatedServiceAccount = "", "" + spec.AutomountServiceAccountToken = ptrTo(false) + spec.Volumes = nil + spec.ImagePullSecrets = nil + spec.ResourceClaims = nil + spec.SchedulerName = "" + spec.Overhead = nil + + c := &spec.Containers[0] + c.VolumeMounts = nil + // The caller resolved every reference, so nothing here points back at the home cluster. + // Plain values go inline; Secret-sourced ones go in a Secret, since a pod spec is + // readable with `get pod`. The two never share a key, so precedence is moot. + c.Env = envVars(s.env) + c.EnvFrom = nil + if s.hasSecrets { + c.EnvFrom = []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: envSecretName(name)}}, + }} + } + + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: local.Namespace, + Labels: labels, + Annotations: s.selection.annotations, + }, + Spec: spec, + } +} + +// withNodeRequirements adds the node autoscaler's required node affinity to the user's own. +// Required terms are ORed, so the requirements go into every one of them; preferences and +// pod (anti-)affinity are left as the user wrote them. +func withNodeRequirements(user, ours *corev1.Affinity) *corev1.Affinity { + if ours == nil || ours.NodeAffinity == nil || ours.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + return user + } + ourRequired := ours.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + reqs := ourRequired.NodeSelectorTerms[0].MatchExpressions + if user == nil { + return ours + } + out := user.DeepCopy() + if out.NodeAffinity == nil { + out.NodeAffinity = &corev1.NodeAffinity{} + } + required := out.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution + if required == nil || len(required.NodeSelectorTerms) == 0 { + out.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = ourRequired.DeepCopy() + return out + } + for i := range required.NodeSelectorTerms { + required.NodeSelectorTerms[i].MatchExpressions = append(required.NodeSelectorTerms[i].MatchExpressions, reqs...) + } + return out +} + +// acceleratorTolerations tolerates, for each extended resource the container asks for +// (nvidia.com/gpu, amd.com/gpu, aws.amazon.com/neuron, ...), the NoSchedule taint of the same +// key, which is how accelerator nodes are conventionally kept for the pods that use them. +func acceleratorTolerations(c *corev1.Container) []corev1.Toleration { + var out []corev1.Toleration + seen := map[corev1.ResourceName]bool{} + for _, list := range []corev1.ResourceList{c.Resources.Requests, c.Resources.Limits} { + for name := range list { + if !isExtendedResource(name) || seen[name] { + continue + } + seen[name] = true + out = append(out, corev1.Toleration{ + Key: string(name), Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule, + }) + } + } + slices.SortFunc(out, func(a, b corev1.Toleration) int { return strings.Compare(a.Key, b.Key) }) + return out +} + +// isExtendedResource mirrors Kubernetes' IsExtendedResourceName (pkg/apis/core/v1/helper in +// k8s.io/kubernetes, not importable): a domain-qualified name outside the reserved +// kubernetes.io domain, not a quota name, and valid once given the quota prefix. +func isExtendedResource(name corev1.ResourceName) bool { + n := string(name) + if !strings.Contains(n, "/") || strings.Contains(n, corev1.ResourceDefaultNamespacePrefix) || + strings.HasPrefix(n, corev1.DefaultResourceRequestsPrefix) { + return false + } + return len(validation.IsQualifiedName(corev1.DefaultResourceRequestsPrefix+n)) == 0 +} + +// envVars renders env in sorted order, so the same request always yields the same spec. +func envVars(env map[string]string) []corev1.EnvVar { + if len(env) == 0 { + return nil + } + out := make([]corev1.EnvVar, 0, len(env)) + for _, k := range slices.Sorted(maps.Keys(env)) { + out = append(out, corev1.EnvVar{Name: k, Value: env[k]}) + } + return out +} + +// envSecret holds the Secret-sourced env (provider.Env.Secrets) for the real pod named podName. +func envSecret(namespace, podName string, env map[string]string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: envSecretName(podName), + Namespace: namespace, + Labels: ownerLabels(), + }, + Type: corev1.SecretTypeOpaque, + StringData: env, + } +} + +func ptrTo[T any](v T) *T { return &v } diff --git a/pkg/provider/kubernetes/provision_test.go b/pkg/provider/kubernetes/provision_test.go new file mode 100644 index 0000000..730f5d6 --- /dev/null +++ b/pkg/provider/kubernetes/provision_test.go @@ -0,0 +1,348 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + "errors" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" +) + +const testUID = types.UID("real-uid") + +// testCluster is one fake cluster. bind makes the fake "scheduler" bind every created pod. +type testCluster struct { + kube *kubefake.Clientset +} + +func newTestCluster(t *testing.T, bind bool) testCluster { + t.Helper() + kube := kubefake.NewClientset(&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "org-1"}}) + kube.PrependReactor("create", "pods", func(a k8stesting.Action) (bool, runtime.Object, error) { + pod := a.(k8stesting.CreateAction).GetObject().(*corev1.Pod) + pod.UID = testUID + if bind { + pod.Spec.NodeName = "node-1" + } + return false, nil, nil // let the tracker store the mutated pod + }) + return testCluster{kube: kube} +} + +func (c testCluster) clients() clusterClients { return clusterClients{kube: c.kube} } + +func provisionRequest() provider.ProvisionRequest { + return provider.ProvisionRequest{ + ClaimName: "org-1-work-1", Region: "us-east-1", + CapacityType: nebulav1alpha1.CapacityOnDemand, + Env: provider.Env{Config: map[string]string{"PLAIN": "p"}, Secrets: map[string]string{"K": "v"}}, + } +} + +func karpenterEvent(reason, msg string) *corev1.Event { + return &corev1.Event{ + ObjectMeta: metav1.ObjectMeta{Name: "ev-" + reason, Namespace: "org-1"}, + InvolvedObject: corev1.ObjectReference{Kind: "Pod", Name: "org-1-work-1", Namespace: "org-1", UID: testUID}, + Reason: reason, Message: msg, + Source: corev1.EventSource{Component: "karpenter"}, + } +} + +func TestProvisionBoundToWarmNode(t *testing.T) { + tc := newTestCluster(t, true) + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + + res, err := p.Provision(context.Background(), placeholder(), provisionRequest()) + if err != nil { + t.Fatal(err) + } + if !res.Reserved || res.InstanceID != "us-east-1/org-1/org-1-work-1" { + t.Fatalf("result = %+v", res) + } + sec, err := tc.kube.CoreV1().Secrets("org-1").Get(context.Background(), "org-1-work-1-env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if sec.StringData["K"] != "v" || len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].UID != testUID { + t.Errorf("secret = %+v", sec) + } + + inst, err := p.FindByClaim(context.Background(), "org-1-work-1", "eks-use1") + if err != nil || inst == nil || inst.ID != res.InstanceID { + t.Fatalf("FindByClaim = %+v, %v", inst, err) + } + list, err := p.List(context.Background()) + if err != nil || len(list) != 1 || list[0].ClaimName != "org-1-work-1" { + t.Fatalf("List = %+v, %v", list, err) + } + if err := p.Terminate(context.Background(), res.InstanceID, ""); err != nil { + t.Fatal(err) + } + if got, err := p.Get(context.Background(), res.InstanceID, ""); err != nil || got != nil { + t.Fatalf("after Terminate, Get = %+v, %v", got, err) + } + if err := p.Terminate(context.Background(), res.InstanceID, ""); err != nil { + t.Fatalf("Terminate is not idempotent: %v", err) + } +} + +func TestProvisionNoCapacityDeletesPod(t *testing.T) { + for _, reason := range []string{reasonInsufficientCapacity, reasonFailedScheduling} { + t.Run(reason, func(t *testing.T) { + tc := newTestCluster(t, false) + if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), + karpenterEvent(reason, "no p5 capacity"), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + + _, err := p.Provision(context.Background(), placeholder(), provisionRequest()) + if !errors.Is(err, provider.ErrNoCapacity) { + t.Fatalf("err = %v, want ErrNoCapacity", err) + } + assertGone(t, tc) + }) + } +} + +func TestProvisionIgnoresSchedulerFailedScheduling(t *testing.T) { + tc := newTestCluster(t, false) + ev := karpenterEvent(reasonFailedScheduling, "0/3 nodes are available") + ev.Source.Component = "default-scheduler" + if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), ev, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + + // The scheduler's event is no verdict, so the window closes without one: the pod is + // accepted unreserved and kept for the poll loop to settle. + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + res, err := p.Provision(ctx, placeholder(), provisionRequest()) + if err != nil || res.Reserved || res.InstanceID == "" { + t.Fatalf("Provision = %+v, %v; want accepted, unreserved", res, err) + } + _, err = tc.kube.CoreV1().Pods("org-1").Get(context.Background(), "org-1-work-1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("pod was deleted: %v", err) + } +} + +func TestListReportsUnschedulablePastBindTimeout(t *testing.T) { + tc := newTestCluster(t, false) + ctx := context.Background() + unbound := placeholder() + unbound.Name = "org-1-work-1" + unbound.Labels = ownerLabels() + unbound.Spec.NodeName = "" + unbound.CreationTimestamp = metav1.NewTime(time.Now()) + bound := unbound.DeepCopy() + bound.Name = "org-1-work-2" + bound.Spec.NodeName = "node-1" + for _, pod := range []*corev1.Pod{unbound, bound} { + if _, err := tc.kube.CoreV1().Pods("org-1").Create(ctx, pod, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + + states := func() map[string]provider.Instance { + list, err := p.List(ctx) + if err != nil { + t.Fatal(err) + } + out := map[string]provider.Instance{} + for _, inst := range list { + out[inst.ClaimName] = inst + } + return out + } + if got := states()["org-1-work-1"].State; got != provider.InstancePending { + t.Fatalf("young unbound pod = %s, want Pending", got) + } + + p.now = func() time.Time { return time.Now().Add(karpenterBindTimeout + time.Second) } + got := states() + if got["org-1-work-1"].State != provider.InstanceUnschedulable { + t.Errorf("old unbound pod = %+v, want Unschedulable", got["org-1-work-1"]) + } + if got["org-1-work-2"].State == provider.InstanceUnschedulable { + t.Error("a bound pod must never report Unschedulable, however old") + } +} + +func TestProvisionRejectsUnknownClusterAndPullSecret(t *testing.T) { + p := newTestProvider(map[string]clusterClients{}) + req := provisionRequest() + req.Region = "nowhere" + if _, err := p.Provision(context.Background(), placeholder(), req); err == nil { + t.Error("unknown cluster: want an error") + } + req = provisionRequest() + req.RegistryAuth = &provider.RegistryAuth{} + if _, err := p.Provision(context.Background(), placeholder(), req); err == nil { + t.Error("pull credential: want an error") + } +} + +func TestProvisionCreatesNamespaceInRemoteCluster(t *testing.T) { + tc := newTestCluster(t, true) + if err := tc.kube.CoreV1().Namespaces().Delete(context.Background(), "org-1", metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + if _, err := p.Provision(context.Background(), placeholder(), provisionRequest()); err != nil { + t.Fatal(err) + } + ns, err := tc.kube.CoreV1().Namespaces().Get(context.Background(), "org-1", metav1.GetOptions{}) + if err != nil || ns.Labels[nebulav1alpha1.ManagedByLabel] != nebulav1alpha1.ManagedByValue { + t.Fatalf("namespace = %+v, %v", ns, err) + } +} + +func assertGone(t *testing.T, tc testCluster) { + t.Helper() + ctx := context.Background() + _, err := tc.kube.CoreV1().Pods("org-1").Get(ctx, "org-1-work-1", metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + t.Errorf("real pod not deleted: %v", err) + } + // The fake clientset has no garbage collector, so what is checked is that a real one would + // take the env Secret with the pod: it is owned by it, or was never written. + sec, err := tc.kube.CoreV1().Secrets("org-1").Get(ctx, "org-1-work-1-env", metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + t.Errorf("get env secret: %v", err) + case len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].UID != testUID: + t.Errorf("env secret is not owned by the real pod: %+v", sec.OwnerReferences) + } +} + +func TestKarpenterVerify(t *testing.T) { + tc := newTestCluster(t, false) + if err := (karpenter{}).verify(context.Background(), tc.clients()); err == nil { + t.Fatal("verify passed on a cluster without Karpenter") + } + tc.kube.Resources = []*metav1.APIResourceList{{ + GroupVersion: nodeClaimGV.String(), + APIResources: []metav1.APIResource{{Name: "nodeclaims"}}, + }} + if err := (karpenter{}).verify(context.Background(), tc.clients()); err != nil { + t.Fatalf("verify = %v", err) + } +} + +// A new node: the pod binds a while after it was created. +func TestProvisionWaitsForBinding(t *testing.T) { + tc := newTestCluster(t, false) + go func() { + ctx := context.Background() + for { + pod, err := tc.kube.CoreV1().Pods("org-1").Get(ctx, "org-1-work-1", metav1.GetOptions{}) + if err == nil { + pod.Spec.NodeName = "node-1" + if _, err := tc.kube.CoreV1().Pods("org-1").Update(ctx, pod, metav1.UpdateOptions{}); err == nil { + return + } + } + time.Sleep(50 * time.Millisecond) + } + }() + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + + res, err := p.Provision(context.Background(), placeholder(), provisionRequest()) + if err != nil || !res.Reserved { + t.Fatalf("Provision = %+v, %v", res, err) + } +} + +// In the host cluster the placeholder sits beside the real pod; List must see only the latter. +func TestListIgnoresPodsWeDidNotCreate(t *testing.T) { + tc := newTestCluster(t, true) + ctx := context.Background() + half := placeholder() + half.Name = "users-own" + half.Labels = map[string]string{nebulav1alpha1.ProviderLabel: provider.ProviderKubernetes} + for _, pod := range []*corev1.Pod{placeholder(), half} { + if _, err := tc.kube.CoreV1().Pods("org-1").Create(ctx, pod, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + if _, err := p.Provision(ctx, placeholder(), provisionRequest()); err != nil { + t.Fatal(err) + } + + list, err := p.List(ctx) + if err != nil || len(list) != 1 || list[0].ClaimName != "org-1-work-1" { + t.Fatalf("List = %+v, %v; want only the real pod", list, err) + } +} + +// A user's own Secret that happens to carry the env Secret's name is refused, never overwritten. +func TestProvisionRefusesToOverwriteForeignSecret(t *testing.T) { + tc := newTestCluster(t, true) + theirs := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "org-1-work-1-env", Namespace: "org-1"}, + StringData: map[string]string{"THEIRS": "keep"}, + } + if _, err := tc.kube.CoreV1().Secrets("org-1").Create(context.Background(), theirs, + metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + + if _, err := p.Provision(context.Background(), placeholder(), provisionRequest()); err == nil { + t.Fatal("Provision overwrote a Secret Nebula did not create") + } + got, _ := tc.kube.CoreV1().Secrets("org-1").Get(context.Background(), "org-1-work-1-env", metav1.GetOptions{}) + if got.StringData["THEIRS"] != "keep" { + t.Fatalf("the user's Secret was changed: %+v", got.StringData) + } +} + +// A previous pod's env Secret, Nebula's own, is replaced with the new values and owner. +func TestProvisionReplacesOwnEnvSecret(t *testing.T) { + tc := newTestCluster(t, true) + old := envSecret("org-1", "org-1-work-1", map[string]string{"K": "stale"}) + old.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "Pod", Name: "org-1-work-1", UID: "old-uid"}} + if _, err := tc.kube.CoreV1().Secrets("org-1").Create(context.Background(), old, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + + if _, err := p.Provision(context.Background(), placeholder(), provisionRequest()); err != nil { + t.Fatal(err) + } + got, err := tc.kube.CoreV1().Secrets("org-1").Get(context.Background(), "org-1-work-1-env", metav1.GetOptions{}) + if err != nil || got.StringData["K"] != "v" || got.OwnerReferences[0].UID != testUID { + t.Fatalf("secret = %+v, %v; want the new values, owned by the new pod", got, err) + } +} diff --git a/pkg/provider/kubernetes/wait.go b/pkg/provider/kubernetes/wait.go new file mode 100644 index 0000000..87edae3 --- /dev/null +++ b/pkg/provider/kubernetes/wait.go @@ -0,0 +1,74 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/InftyAI/Nebula/pkg/util" +) + +// waitInterval is how often a pending Provision re-reads its pod. +const waitInterval = 2 * time.Second + +// waitForProvision watches a new pod for provisionTimeout: long enough to catch a warm node +// binding it or the node autoscaler refusing it, without holding a worker through node boot. +// Neither within the window is no answer yet (false, nil); the poll loop takes over from there +// (see toInstance). Image pull and startup after binding are the readiness deadline's. +func waitForProvision(ctx context.Context, cc clusterClients, as nodeAutoscaler, namespace, name string) (bool, error) { + ctx, cancel := context.WithTimeout(ctx, as.provisionTimeout()) + defer cancel() + t := time.NewTicker(waitInterval) + defer t.Stop() + for { + bound, err := checkBinding(ctx, cc, as, namespace, name) + if bound || err != nil { + return bound, err + } + select { + case <-ctx.Done(): + return false, nil + case <-t.C: + } + } +} + +// checkBinding is one pass of waitForProvision. A failed read is not a verdict, so it is +// swallowed and the next tick retries. +func checkBinding(ctx context.Context, cc clusterClients, as nodeAutoscaler, namespace, name string) (bool, error) { + pod, err := cc.kube.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return false, fmt.Errorf("kubernetes: pod %s/%s disappeared before it was bound", namespace, name) + } + if err != nil { + return false, nil + } + // With reserved capacity, the pod will have a NodeName assigned once it is scheduled. + if pod.Spec.NodeName != "" { + return true, nil + } + if util.Finished(pod.Status.Phase) { + return false, fmt.Errorf("kubernetes: pod %s/%s ended %s before it was bound: %s", + namespace, name, pod.Status.Phase, pod.Status.Message) + } + return false, as.refused(ctx, cc, pod) +} diff --git a/pkg/provider/modal/modal.go b/pkg/provider/modal/modal.go index 6363315..3f5bf15 100644 --- a/pkg/provider/modal/modal.go +++ b/pkg/provider/modal/modal.go @@ -644,8 +644,8 @@ func (p *Provider) sandboxSpecFromPod(pod *corev1.Pod, req provider.ProvisionReq // The caller's resolved map is the whole environment — the Pod's literals plus // everything envFrom/valueFrom referenced. pod.Spec.Containers[0].Env is NOT read // here: it holds references this adapter has no cluster access to follow. See - // provider.ProvisionRequest.Env. - Env: req.Env, + // provider.Env.All. + Env: req.Env.All(), CPU: physicalCores(requests.CPU), MemoryMiB: requests.MemoryMiB, CPULimit: physicalCores(limits.CPU), diff --git a/pkg/provider/modal/modal_test.go b/pkg/provider/modal/modal_test.go index b2437af..ae72842 100644 --- a/pkg/provider/modal/modal_test.go +++ b/pkg/provider/modal/modal_test.go @@ -1762,7 +1762,9 @@ func TestProvision_UsesResolvedEnv(t *testing.T) { if _, err := p.Provision(context.Background(), pod, provider.ProvisionRequest{ ClaimName: "claim-env", - Env: map[string]string{"FOO": "bar", "TOKEN": "t0ken"}, + Env: provider.Env{ + Config: map[string]string{"FOO": "bar"}, Secrets: map[string]string{"TOKEN": "t0ken"}, + }, }); err != nil { t.Fatalf("Provision: %v", err) } diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 529dccb..e3f80c9 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -22,6 +22,7 @@ import ( "context" "fmt" "io" + "maps" "sort" "strings" "time" @@ -234,19 +235,12 @@ type ProvisionRequest struct { // otherwise. Resolved from the NodePool at provision time, never from the Pod, which // the workload's own owner can patch (see vnode.Handler.egressFor). Egress *nebulav1alpha1.EgressPolicy - // Env is the container's environment, fully RESOLVED: literals plus everything - // envFrom/valueFrom referenced, merged in kubelet precedence (envFrom in listed order, - // then env overriding it). A provider forwards it and never re-reads the Pod's env, - // which holds references it cannot follow. + // Env is the container's environment, fully RESOLVED (see Env). A provider forwards it and + // never re-reads the Pod's env, which holds references it cannot follow. // // Resolved by the caller (pkg/vnode): reading a Secret needs cluster access an adapter // deliberately lacks, and kubelet precedence should be implemented once. - // - // SECRET-BEARING, and why this struct redacts itself — a secretKeyRef's value lands here - // in the clear. Never log the map, never put it on the Pod, never in an error string, - // the same rule as ProvisionResult.ConnectToken. Nil is normal: no env, or an - // unresolving caller. - Env map[string]string + Env Env // RegistryAuth authenticates the pull of the container image, from the Pod's // imagePullSecrets; nil is an anonymous pull. Resolved by the caller (pkg/vnode) for the // same reason as Env — it names a Secret an adapter cannot read. @@ -259,10 +253,44 @@ type ProvisionRequest struct { func (r ProvisionRequest) String() string { return fmt.Sprintf("ProvisionRequest{ClaimName:%s CapacityType:%s Region:%s Egress:%s Env:%s "+ "RegistryAuth:%s}", - r.ClaimName, r.CapacityType, r.Region, r.Egress.ModeOrOpen(), RedactedEnv(r.Env), - r.RegistryAuth) + r.ClaimName, r.CapacityType, r.Region, r.Egress.ModeOrOpen(), r.Env, r.RegistryAuth) +} + +// Env is a container's resolved environment: literals plus everything envFrom/valueFrom +// referenced, merged in kubelet precedence (envFrom in listed order, then env overriding it). +// It is split by how a value must be handled, and each variable is in exactly one part, +// decided by the source whose value won. +// +// SECRET-BEARING, and the reason it redacts itself: never log Secrets, never put them on a +// Pod or in an error string, the same rule as ProvisionResult.ConnectToken. Config is +// redacted too, since a literal can hold a secret as well. The zero value is no env. +type Env struct { + // Config is literals, ConfigMap values and fieldRefs. + Config map[string]string + // Secrets is the values that came from a Secret. + Secrets map[string]string } +// All is every variable, for a provider that ships the environment one way. The parts never +// share a key, so nothing is lost. SECRET-BEARING, like Secrets. +func (e Env) All() map[string]string { + if len(e.Config) == 0 && len(e.Secrets) == 0 { + return nil + } + out := make(map[string]string, len(e.Config)+len(e.Secrets)) + maps.Copy(out, e.Config) + maps.Copy(out, e.Secrets) + return out +} + +// String redacts both parts: key names print, values never do. +func (e Env) String() string { + return fmt.Sprintf("{Config:%s Secrets:%s}", RedactedEnv(e.Config), RedactedEnv(e.Secrets)) +} + +// GoString implements fmt.GoStringer so %#v is redacted too. +func (e Env) GoString() string { return e.String() } + // GoString implements fmt.GoStringer so %#v is redacted too. func (r ProvisionRequest) GoString() string { return r.String() } @@ -419,6 +447,10 @@ const ( InstanceTerminated InstanceState = "Terminated" // InstanceFailed: entered a terminal error state. InstanceFailed InstanceState = "Failed" + // InstanceUnschedulable: accepted by a Provision that returned unreserved, then never + // placed on capacity. The poll loop treats it like a Provision capacity error: it + // blocklists the candidate and fails the Pod. + InstanceUnschedulable InstanceState = "Unschedulable" ) // Offering is one row of the price/availability catalog. The price/availability diff --git a/pkg/provider/runpod/runpod.go b/pkg/provider/runpod/runpod.go index 5213ca4..455d403 100644 --- a/pkg/provider/runpod/runpod.go +++ b/pkg/provider/runpod/runpod.go @@ -530,7 +530,7 @@ func (p *Provider) podSpecFromPod(pod *corev1.Pod, req provider.ProvisionRequest // The caller's resolved map is the whole environment — the Pod's literals plus // everything envFrom/valueFrom referenced. pod.Spec.Containers[0].Env is NOT read // here: it holds references this adapter has no cluster access to follow. - Env: req.Env, + Env: req.Env.All(), ContainerDiskGiB: max(defaultContainerDiskGiB, util.PodEphemeralStorageGiB(pod)), Ports: containerPorts(&c), DataCenterIDs: dataCentersOf(req.Region), diff --git a/pkg/provider/runpod/runpod_test.go b/pkg/provider/runpod/runpod_test.go index f216582..805e061 100644 --- a/pkg/provider/runpod/runpod_test.go +++ b/pkg/provider/runpod/runpod_test.go @@ -176,7 +176,7 @@ func TestProvision_GPUPod(t *testing.T) { ClaimName: "claim-a", CapacityType: nebulav1alpha1.CapacityOnDemand, Region: "US-KS-2", - Env: map[string]string{"HF_TOKEN": "hf_secret"}, + Env: provider.Env{Secrets: map[string]string{"HF_TOKEN": "hf_secret"}}, }) if err != nil { t.Fatalf("Provision: %v", err) diff --git a/pkg/util/pod.go b/pkg/util/pod.go index c6fbbfc..75cb4ec 100644 --- a/pkg/util/pod.go +++ b/pkg/util/pod.go @@ -18,12 +18,8 @@ package util import corev1 "k8s.io/api/core/v1" -// IsTerminalPodPhase reports whether a Pod phase is an end state that will not progress. -// -// Shared because the two readers must agree: the control plane reaps and un-gates on a terminal -// Pod, while the virtual node refuses to move one (a provider still listing the instance would -// otherwise walk a Failed Pod back to Running). A predicate that drifted between them would -// resurrect exactly the Pods the other side had written off. -func IsTerminalPodPhase(phase corev1.PodPhase) bool { +// Finished reports whether a Pod phase is final (Succeeded or Failed). Shared so the control +// plane, which reaps finished Pods, and the virtual node, which never moves one back, agree. +func Finished(phase corev1.PodPhase) bool { return phase == corev1.PodFailed || phase == corev1.PodSucceeded } diff --git a/pkg/util/pod_test.go b/pkg/util/pod_test.go index c8f071b..2d5a00b 100644 --- a/pkg/util/pod_test.go +++ b/pkg/util/pod_test.go @@ -22,7 +22,7 @@ import ( corev1 "k8s.io/api/core/v1" ) -func TestIsTerminalPodPhase(t *testing.T) { +func TestFinished(t *testing.T) { // Pending is the one worth stating: a Pod that has not started yet is not terminal, and // treating it as one would reap a workload mid-provision. for phase, want := range map[corev1.PodPhase]bool{ @@ -33,8 +33,8 @@ func TestIsTerminalPodPhase(t *testing.T) { corev1.PodUnknown: false, "": false, } { - if got := IsTerminalPodPhase(phase); got != want { - t.Errorf("IsTerminalPodPhase(%q) = %v, want %v", phase, got, want) + if got := Finished(phase); got != want { + t.Errorf("Finished(%q) = %v, want %v", phase, got, want) } } } diff --git a/pkg/vnode/env.go b/pkg/vnode/env.go index 5f0b711..77f1121 100644 --- a/pkg/vnode/env.go +++ b/pkg/vnode/env.go @@ -29,9 +29,10 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" ) -// resolveEnv builds the environment a provider gets as ProvisionRequest.Env, following the -// references the Pod makes. The only place envFrom/valueFrom are read, since a provider has -// no cluster access to follow them with. +// resolveEnv builds the environment a provider gets as ProvisionRequest.Env (Config, Secrets), +// following the references the Pod makes. A variable lands in secrets when the value that +// won under kubelet precedence came from a Secret, and in env otherwise. The only place +// envFrom/valueFrom are read, since a provider has no cluster access to follow them with. // // Kubelet rules: envFrom first in listed order with each Prefix, then env overriding it. // optional means "the object or key may not exist" — NOT "we may fail to look", so a read @@ -46,25 +47,37 @@ import ( // Reads bypass node.go's Secret/ConfigMap informers on purpose: VK waits only for the POD // informer to sync before CreatePod, and a cold cache reports an existing Secret as absent — // the one signal this function acts on. A nil client is the usual test seam (literals only). -func resolveEnv(ctx context.Context, client kubernetes.Interface, pod *corev1.Pod) (map[string]string, error) { +func resolveEnv( + ctx context.Context, client kubernetes.Interface, pod *corev1.Pod, +) (env, secrets map[string]string, err error) { if len(pod.Spec.Containers) == 0 { - return nil, nil + return nil, nil, nil } // One container per Nebula Pod, as everywhere else that reads the workload. c := pod.Spec.Containers[0] if len(c.Env) == 0 && len(c.EnvFrom) == 0 { - return nil, nil + return nil, nil, nil } - r := &envResolver{client: client, pod: pod} + r := &envResolver{client: client, pod: pod, fromSecret: map[string]bool{}} out := make(map[string]string, len(c.Env)) if err := r.fromSources(ctx, out, c.EnvFrom); err != nil { - return nil, err + return nil, nil, err } if err := r.fromVars(ctx, out, c.Env); err != nil { - return nil, err + return nil, nil, err } - return out, nil + for k, v := range out { + dst := &env + if r.fromSecret[k] { + dst = &secrets + } + if *dst == nil { + *dst = map[string]string{} + } + (*dst)[k] = v + } + return env, secrets, nil } // envResolver resolves one container's references, memoizing what it reads so ten @@ -75,6 +88,9 @@ type envResolver struct { pod *corev1.Pod secrets map[string]*corev1.Secret configs map[string]*corev1.ConfigMap + // fromSecret records, per variable, whether its current value came from a Secret; the + // last write wins, as the value does. + fromSecret map[string]bool } // fromSources applies envFrom in listed order. Every key of each source becomes a variable, @@ -116,6 +132,7 @@ func (r *envResolver) fromVars(ctx context.Context, out map[string]string, vars for _, e := range vars { if e.ValueFrom == nil { out[e.Name] = e.Value + r.fromSecret[e.Name] = false continue } v, found, err := r.valueFrom(ctx, e.ValueFrom) @@ -126,6 +143,7 @@ func (r *envResolver) fromVars(ctx context.Context, out map[string]string, vars // variable unset, as the kubelet does. if found { out[e.Name] = v + r.fromSecret[e.Name] = e.ValueFrom.SecretKeyRef != nil } } return nil @@ -302,6 +320,7 @@ func (r *envResolver) put(ctx context.Context, out map[string]string, name, valu return } out[name] = value + r.fromSecret[name] = kind == "Secret" } // optional dereferences the *bool the API uses for optional, where nil means false. diff --git a/pkg/vnode/env_test.go b/pkg/vnode/env_test.go index c57f2f6..a143bc6 100644 --- a/pkg/vnode/env_test.go +++ b/pkg/vnode/env_test.go @@ -26,8 +26,11 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" + + "github.com/InftyAI/Nebula/pkg/provider" ) // envPod is a Pod whose single container carries the given env sources. @@ -74,7 +77,7 @@ func TestResolveEnv_Precedence(t *testing.T) { []corev1.EnvVar{{Name: "SHARED", Value: "explicit"}, {Name: "PLAIN", Value: "p"}}, ) - got, err := resolveEnv(context.Background(), client, pod) + got, err := resolveAll(context.Background(), client, pod) if err != nil { t.Fatalf("resolveEnv: %v", err) } @@ -98,7 +101,7 @@ func TestResolveEnv_Prefix(t *testing.T) { ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "cm"}}, }}, nil) - got, err := resolveEnv(context.Background(), client, pod) + got, err := resolveAll(context.Background(), client, pod) if err != nil { t.Fatalf("resolveEnv: %v", err) } @@ -135,7 +138,7 @@ func TestResolveEnv_SecretKeyRef(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got, err := resolveEnv(context.Background(), client, envPod(nil, tc.env)) + got, err := resolveAll(context.Background(), client, envPod(nil, tc.env)) switch { case tc.wantErr != "": if err == nil || !strings.Contains(err.Error(), tc.wantErr) { @@ -170,7 +173,7 @@ func TestResolveEnv_ReadFailureIsNeverOptional(t *testing.T) { }, }}, nil) - if _, err := resolveEnv(context.Background(), client, pod); err == nil { + if _, err := resolveAll(context.Background(), client, pod); err == nil { t.Fatal("expected a read failure to fail the resolve even though the ref is optional") } } @@ -198,7 +201,7 @@ func TestResolveEnv_MemoizesReads(t *testing.T) { []corev1.EnvVar{keyRef("X", "A"), keyRef("Y", "B")}, ) - got, err := resolveEnv(context.Background(), client, pod) + got, err := resolveAll(context.Background(), client, pod) if err != nil { t.Fatalf("resolveEnv: %v", err) } @@ -229,7 +232,7 @@ func TestResolveEnv_MemoizesMisses(t *testing.T) { }} } - got, err := resolveEnv(context.Background(), client, envPod(nil, []corev1.EnvVar{optRef("A"), optRef("B")})) + got, err := resolveAll(context.Background(), client, envPod(nil, []corev1.EnvVar{optRef("A"), optRef("B")})) if err != nil { t.Fatalf("resolveEnv: %v", err) } @@ -296,7 +299,7 @@ func TestResolveEnv_ResourceFieldRefRefused(t *testing.T) { ResourceFieldRef: &corev1.ResourceFieldSelector{Resource: "limits.memory"}, }}}) - _, err := resolveEnv(context.Background(), fake.NewSimpleClientset(), pod) + _, err := resolveAll(context.Background(), fake.NewSimpleClientset(), pod) if err == nil || !strings.Contains(err.Error(), "limits.memory") { t.Fatalf("expected a resourceFieldRef refusal naming the resource, got %v", err) } @@ -319,7 +322,7 @@ func TestResolveEnv_DropsIllegalNames(t *testing.T) { ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "cm"}}, }}, nil) - got, err := resolveEnv(context.Background(), client, pod) + got, err := resolveAll(context.Background(), client, pod) if err != nil { t.Fatalf("resolveEnv: %v", err) } @@ -344,7 +347,7 @@ func TestResolveEnv_NilClient(t *testing.T) { }, ) - got, err := resolveEnv(context.Background(), nil, pod) + got, err := resolveAll(context.Background(), nil, pod) if err != nil { t.Fatalf("resolveEnv: %v", err) } @@ -373,8 +376,10 @@ func TestCreatePod_PassesResolvedEnvToProvider(t *testing.T) { if err := h.CreatePod(context.Background(), pod); err != nil { t.Fatalf("CreatePod: %v", err) } - if !maps.Equal(fp.lastReq.Env, map[string]string{"PLAIN": "p", "TOKEN": "t0ken"}) { - t.Fatalf("provider got env %v", fp.lastReq.Env) + if !maps.Equal(fp.lastReq.Env.Config, map[string]string{"PLAIN": "p"}) || + !maps.Equal(fp.lastReq.Env.Secrets, map[string]string{"TOKEN": "t0ken"}) { + t.Fatalf("provider got config %v, secrets %v; the Secret's value belongs in Secrets", + fp.lastReq.Env.Config, fp.lastReq.Env.Secrets) } if vf := pod.Spec.Containers[0].Env[1].ValueFrom; vf == nil || vf.SecretKeyRef == nil { t.Fatal("the Pod's env must keep its reference; resolution belongs on the request") @@ -418,8 +423,49 @@ func TestCreatePod_UnresolvableEnvIsNonTerminal(t *testing.T) { // TestResolveEnv_NoEnvIsNil keeps the common case allocation-free and the request field nil, // so a provider can tell "nothing to set" from "an empty environment". func TestResolveEnv_NoEnvIsNil(t *testing.T) { - got, err := resolveEnv(context.Background(), fake.NewSimpleClientset(), testPod("default", "p1")) + got, err := resolveAll(context.Background(), fake.NewSimpleClientset(), testPod("default", "p1")) if err != nil || got != nil { t.Fatalf("expected (nil, nil) for a Pod with no env, got (%v, %v)", got, err) } } + +// resolveAll is resolveEnv with its two maps merged, for tests about which value wins rather +// than where it lands. +func resolveAll(ctx context.Context, client kubernetes.Interface, pod *corev1.Pod) (map[string]string, error) { + env, secrets, err := resolveEnv(ctx, client, pod) + if err != nil { + return nil, err + } + return provider.Env{Config: env, Secrets: secrets}.All(), nil +} + +// A variable is a secret when the value that WON came from a Secret, so precedence decides +// the map as well as the value. +func TestResolveEnv_SplitsSecrets(t *testing.T) { + client := fake.NewSimpleClientset( + configMapObj("cm", map[string]string{"FROM_CM": "c", "CM_THEN_SECRET": "c"}), + secretObj("sec", map[string]string{"FROM_SECRET": "s", "SECRET_THEN_LITERAL": "s"}), + ) + pod := envPod( + []corev1.EnvFromSource{ + {ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "cm"}}}, + {SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "sec"}}}, + }, + []corev1.EnvVar{ + {Name: "SECRET_THEN_LITERAL", Value: "plain"}, + {Name: "CM_THEN_SECRET", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "sec"}, Key: "FROM_SECRET", + }}}, + }, + ) + + env, secrets, err := resolveEnv(context.Background(), client, pod) + if err != nil { + t.Fatalf("resolveEnv: %v", err) + } + wantEnv := map[string]string{"FROM_CM": "c", "SECRET_THEN_LITERAL": "plain"} + wantSecrets := map[string]string{"FROM_SECRET": "s", "CM_THEN_SECRET": "s"} + if !maps.Equal(env, wantEnv) || !maps.Equal(secrets, wantSecrets) { + t.Fatalf("env = %v, secrets = %v; want %v and %v", env, secrets, wantEnv, wantSecrets) + } +} diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index d8d8b01..3f596ed 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -321,14 +321,14 @@ func (h *Handler) CreatePod(ctx context.Context, pod *corev1.Pod) error { // Deliberately NOT stored: tracking a pod makes GetPod return non-nil, which suppresses // the retry this branch wants. (The provision-failure branch below stores for exactly // that reason — it does not want one.) - env, err := resolveEnv(ctx, h.client, pod) + env, secrets, err := resolveEnv(ctx, h.client, pod) if err != nil { log.Error(err, "cannot resolve the Pod's environment; nothing provisioned, retrying") h.markStatus(pod, corev1.PodPending, reasonConfigError, err.Error()) h.emit(pod) return err } - req.Env = env + req.Env = provider.Env{Config: env, Secrets: secrets} // Same resolve-before-requesting rule, and the same non-terminal treatment, for the // image pull credential: an imagePullSecret names a Secret in the Pod's namespace that @@ -360,7 +360,7 @@ func (h *Handler) CreatePod(ctx context.Context, pod *corev1.Pod) error { // The env COUNT, never its content: a value here may have come from a Secret. log.Info("provisioning external instance", "capacityType", req.CapacityType, "region", req.Region, - "envVars", len(req.Env), "timeout", timeout.String()) + "envVars", len(req.Env.Config)+len(req.Env.Secrets), "timeout", timeout.String()) // Two clocks for two different waits: provisioningAt measures the end-to-end wait a // user feels (until the instance reports Running; handed to store below), callStart @@ -676,21 +676,36 @@ func (h *Handler) reconcileOnce(ctx context.Context) { byClaim[inst.ClaimName] = inst } + trackedCount := len(h.tracked) h.mu.Lock() - emit := make([]*corev1.Pod, 0, len(h.tracked)) - tracked := len(h.tracked) - matched, frozen := 0, 0 + emit := make([]*corev1.Pod, 0, trackedCount) + // failed are placements to block once the lock is released: the TTL comes from the + // pod's NodePool, and that read does not belong under h.mu. + var failed []failedPlacement + updated, finished, gone := 0, 0, 0 for _, tp := range h.tracked { inst, present := byClaim[tp.claimName] before := statusSignature(tp.pod) switch { // No reverse for a terminal or deleting pod. - case util.IsTerminalPodPhase(tp.pod.Status.Phase) || !tp.pod.DeletionTimestamp.IsZero(): - frozen++ + case util.Finished(tp.pod.Status.Phase) || !tp.pod.DeletionTimestamp.IsZero(): + finished++ case !present: + gone++ applyState(tp.pod, provider.InstanceTerminated, "", h.nowFn()) + case inst.State == provider.InstanceUnschedulable: + // An unreserved instance that never got capacity: the Provision error path, + // arriving late. Failing the Pod is once only, as Finished freezes it above. + updated++ + log.Info("no capacity for unreserved instance; failing the pod", + "pod", key(tp.pod.Namespace, tp.pod.Name), "instanceID", tp.instance) + applyNoCapacity(tp.pod, h.nowFn()) + failed = append(failed, failedPlacement{ + pod: tp.pod.DeepCopy(), region: inst.Region, + err: fmt.Errorf("external instance got no capacity: %w", provider.ErrNoCapacity), + }) default: - matched++ + updated++ if waited, over := h.readyExpired(tp, inst.State); over { // Failing the Pod is the whole action: teardown follows from the // terminal phase, via the reap (see readyExpired). @@ -717,12 +732,21 @@ func (h *Handler) reconcileOnce(ctx context.Context) { notify := h.notify h.mu.Unlock() - // V(1) so a healthy steady state stays quiet. tracked>0 with matched==0 means the - // claim names don't line up with what List returns — the classic "provisioned but - // never Running". frozen is what keeps that reading valid: a terminal or deleting pod is - // deliberately not matched, so without it those would look like the same failure. + for _, b := range failed { + pool, err := h.poolFor(ctx, b.pod) + if err != nil { + pool = nil // the default TTL; a block with the wrong length beats no block + } + h.recordBlock(ctx, b.pod, b.region, blocklistTTLOf(pool), b.err) + } + + // V(1) so a healthy steady state stays quiet. updated, finished and gone split the tracked + // pods between them. All of them gone means the claim names don't line up with what List + // returns — the classic "provisioned but never Running"; finished keeps that reading valid, + // since a finished or deleting pod is never matched and would otherwise look the same. log.V(1).Info("poll tick", - "listed", len(instances), "tracked", tracked, "matched", matched, "frozen", frozen) + "listedInstances", len(instances), "trackedCount", trackedCount, + "updated", updated, "finished", finished, "gone", gone) // Re-emit EVERY tracked pod each tick, which makes status propagation // level-triggered. VK dedups an emit against the last status IT received from us, @@ -1128,6 +1152,14 @@ func (h *Handler) recordBlock( h.blocklist.Record(h.prov.Name(), scope, ttl) } +// failedPlacement is a placement the poll loop found failed, whose block it records after +// unlocking. The provider's classification of err decides the scope. +type failedPlacement struct { + pod *corev1.Pod + region string + err error +} + func ptrNow(t metav1.Time) *metav1.Time { return &t } // Tracks reports whether this virtual node is the one running the Pod. The kubelet API diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go index 4a52497..c7dc963 100644 --- a/pkg/vnode/handler_test.go +++ b/pkg/vnode/handler_test.go @@ -1983,3 +1983,38 @@ func TestCreatePod_FailsClosedWithoutClusterState(t *testing.T) { }) } } + +// An unreserved instance the provider later reports as having no capacity is the Provision +// error path arriving late: the candidate is blocklisted with the pool's TTL, the Pod fails +// with the provision-failed reason, and a second tick neither re-fails nor re-blocks it. +func TestReconcileOnce_UnschedulableBlocksAndFailsPod(t *testing.T) { + scope := provider.BlockScope{Region: ptrTo("eks-use1")} + fp := &fakeProvider{provisionID: "inst-1", classifyScope: scope} + bl := &recordingBlocklist{} + h := NewHandler(fp, nil, bl, clusterWithTTL(30*time.Second)) + h.jitterFn = func() time.Duration { return 0 } + if err := h.CreatePod(context.Background(), testPod("default", "p1")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + + fp.list = []provider.Instance{{ + ID: "inst-1", ClaimName: "default-p1", Region: "eks-use1", + State: provider.InstanceUnschedulable, + }} + h.reconcileOnce(context.Background()) + + got, _ := h.GetPod(context.Background(), "default", "p1") + if got.Status.Phase != corev1.PodFailed || got.Status.Reason != reasonProvisionFailed { + t.Fatalf("status = %s/%s, want Failed/%s", got.Status.Phase, got.Status.Reason, reasonProvisionFailed) + } + if bl.calls != 1 || bl.ttl != 30*time.Second || fp.classifyRegion != "eks-use1" { + t.Fatalf("block = %d calls, ttl %v, region %q; want one, 30s, eks-use1", bl.calls, bl.ttl, fp.classifyRegion) + } + + h.reconcileOnce(context.Background()) + if bl.calls != 1 { + t.Fatalf("Record calls = %d after a second tick, want 1", bl.calls) + } +} + +func ptrTo[T any](v T) *T { return &v } diff --git a/pkg/vnode/status.go b/pkg/vnode/status.go index be3efe2..697612e 100644 --- a/pkg/vnode/status.go +++ b/pkg/vnode/status.go @@ -123,6 +123,21 @@ func applyReadinessTimeout(pod *corev1.Pod, waited time.Duration, now metav1.Tim }, false) } +// applyNoCapacity fails a pod whose unreserved instance was found to have no capacity, with +// the same reason as a Provision that failed outright. +func applyNoCapacity(pod *corev1.Pod, now metav1.Time) { + const msg = "external instance got no capacity" + setPhase(pod, corev1.PodFailed, reasonProvisionFailed, msg, now) + setReady(pod, corev1.ConditionFalse, now) + setContainerStatuses(pod, corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + Reason: reasonProvisionFailed, + Message: msg, + FinishedAt: now, + }, + }, false) +} + // setContainerStatuses mirrors the one instance's state onto every container in the spec. // The READY column, `kubectl wait`, and anything keying off container readiness read // Status.ContainerStatuses, so an empty array reads as 0/N even with Ready=True. There is From a39b2f183097557220e078887054619ec550f01c Mon Sep 17 00:00:00 2001 From: kerthcet Date: Thu, 22 Oct 2026 20:04:29 +0100 Subject: [PATCH 02/21] address comments Signed-off-by: kerthcet --- internal/webhook/v1/pod_webhook.go | 15 --------------- pkg/vnode/handler.go | 2 +- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/internal/webhook/v1/pod_webhook.go b/internal/webhook/v1/pod_webhook.go index 0ed646e..12fb3b4 100644 --- a/internal/webhook/v1/pod_webhook.go +++ b/internal/webhook/v1/pod_webhook.go @@ -19,7 +19,6 @@ package v1 import ( "context" "fmt" - "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -188,23 +187,9 @@ func validatePod(pod *corev1.Pod) error { if c := pod.Spec.Containers[0]; len(c.Ports) > 1 { return fmt.Errorf("nebula exposes at most one port per Pod, container %q declares %d", c.Name, len(c.Ports)) } - if sa := pod.Spec.ServiceAccountName; sa != "" && sa != "default" { - return fmt.Errorf("nebula does not mount service accounts, got %q", sa) - } - for _, v := range pod.Spec.Volumes { - // The API server adds this one to every Pod before validation; it is simply not mounted. - if v.Projected != nil && strings.HasPrefix(v.Name, serviceAccountTokenVolumePrefix) { - continue - } - return fmt.Errorf("nebula does not mount volumes, got %q", v.Name) - } return nil } -// serviceAccountTokenVolumePrefix names the projected token volume the ServiceAccount -// admission plugin adds ("kube-api-access-"). -const serviceAccountTokenVolumePrefix = "kube-api-access-" - // hasNebulaGate reports whether the Pod carries Nebula's provider-selection scheduling gate. func hasNebulaGate(pod *corev1.Pod) bool { for _, g := range pod.Spec.SchedulingGates { diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index 3f596ed..f18080d 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -676,8 +676,8 @@ func (h *Handler) reconcileOnce(ctx context.Context) { byClaim[inst.ClaimName] = inst } - trackedCount := len(h.tracked) h.mu.Lock() + trackedCount := len(h.tracked) emit := make([]*corev1.Pod, 0, trackedCount) // failed are placements to block once the lock is released: the TTL comes from the // pod's NodePool, and that read does not belong under h.mu. From de09e3d6852c5d94485fe953f07fea385054bc4b Mon Sep 17 00:00:00 2001 From: kerthcet Date: Thu, 22 Oct 2026 20:09:18 +0100 Subject: [PATCH 03/21] optimize the create pod process Signed-off-by: kerthcet --- pkg/provider/kubernetes/kubernetes.go | 91 +++++++++-------------- pkg/provider/kubernetes/provision_test.go | 66 +++++++++++++--- 2 files changed, 89 insertions(+), 68 deletions(-) diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index 46f14a8..5b6ce3b 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -231,12 +231,12 @@ func (p *Provider) Provision( if err := ensureNamespace(ctx, kube, real.Namespace); err != nil { return provider.ProvisionResult{}, err } - created, err := createOrAdopt(ctx, cl.clients, real) + created, adopted, err := createOrAdopt(ctx, cl.clients, real) if err != nil { return provider.ProvisionResult{}, err } if len(req.Env.Secrets) > 0 { - if err := ensureEnvSecret(ctx, kube, created, req.Env.Secrets); err != nil { + if err := ensureEnvSecret(ctx, kube, created, adopted, req.Env.Secrets); err != nil { cleanup(ctx, cl.clients, real.Namespace, real.Name) return provider.ProvisionResult{}, err } @@ -474,35 +474,46 @@ func parseInstanceID(id string) (region, namespace, name string, err error) { return parts[0], parts[1], parts[2], nil } -// createOrAdopt creates the real pod, or returns the live one a crashed or retried Provision -// left behind. A finished one is the previous incarnation of a reused claim name and is -// replaced; anything else holding the name is refused rather than adopted. -func createOrAdopt(ctx context.Context, cc clusterClients, want *corev1.Pod) (*corev1.Pod, error) { +// createOrAdopt creates the real pod, or adopts the live one an interrupted Provision left +// (adopted=true). Anything else holding the pod's or env Secret's name is refused, not +// replaced: a new pod could start on a warm node with whatever Secret of that name exists. +func createOrAdopt( + ctx context.Context, cc clusterClients, want *corev1.Pod, +) (pod *corev1.Pod, adopted bool, err error) { pods := cc.kube.CoreV1().Pods(want.Namespace) got, err := pods.Get(ctx, want.Name, metav1.GetOptions{}) switch { case apierrors.IsNotFound(err): case err != nil: - return nil, fmt.Errorf("kubernetes: get pod %s/%s: %w", want.Namespace, want.Name, classifyAPI(err)) + return nil, false, fmt.Errorf("kubernetes: get pod %s/%s: %w", want.Namespace, want.Name, classifyAPI(err)) case !ownedByUs(got): - return nil, fmt.Errorf("kubernetes: pod %s/%s exists and is not Nebula's", want.Namespace, want.Name) + return nil, false, fmt.Errorf("kubernetes: pod %s/%s exists and is not Nebula's", want.Namespace, want.Name) case got.DeletionTimestamp == nil && !util.Finished(got.Status.Phase): - return got, nil + return got, true, nil default: - if err := deleteAndWait(ctx, cc, want.Namespace, want.Name); err != nil { - return nil, err - } + return nil, false, fmt.Errorf("kubernetes: pod %s/%s from an earlier sandbox is still present", + want.Namespace, want.Name) + } + + secret := envSecretName(want.Name) + _, err = cc.kube.CoreV1().Secrets(want.Namespace).Get(ctx, secret, metav1.GetOptions{}) + switch { + case err == nil: + return nil, false, fmt.Errorf("kubernetes: secret %s/%s already exists", want.Namespace, secret) + case !apierrors.IsNotFound(err): + return nil, false, fmt.Errorf("kubernetes: get secret %s/%s: %w", want.Namespace, secret, classifyAPI(err)) } + created, err := pods.Create(ctx, want, metav1.CreateOptions{}) if err != nil { - return nil, fmt.Errorf("kubernetes: create pod %s/%s: %w", want.Namespace, want.Name, classifyAPI(err)) + return nil, false, fmt.Errorf("kubernetes: create pod %s/%s: %w", want.Namespace, want.Name, classifyAPI(err)) } - return created, nil + return created, false, nil } // ensureEnvSecret writes the Secret-sourced env, owned by the pod so Kubernetes deletes it with -// the pod; nothing else deletes it. -func ensureEnvSecret(ctx context.Context, kube kubeClient, pod *corev1.Pod, env map[string]string) error { +// it. An existing one is accepted only for an adopted pod, and only if it is Nebula's. +func ensureEnvSecret(ctx context.Context, kube kubeClient, pod *corev1.Pod, adopted bool, env map[string]string) error { want := envSecret(pod.Namespace, pod.Name, env) want.OwnerReferences = []metav1.OwnerReference{{ APIVersion: "v1", Kind: "Pod", Name: pod.Name, UID: pod.UID, @@ -510,7 +521,14 @@ func ensureEnvSecret(ctx context.Context, kube kubeClient, pod *corev1.Pod, env secrets := kube.CoreV1().Secrets(pod.Namespace) _, err := secrets.Create(ctx, want, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { - err = replaceEnvSecret(ctx, kube, want) + var got *corev1.Secret + switch got, err = secrets.Get(ctx, want.Name, metav1.GetOptions{}); { + case err != nil: + case !adopted: + err = fmt.Errorf("secret %s/%s appeared before this pod's", want.Namespace, want.Name) + case !ownedByUs(got): + err = fmt.Errorf("secret %s/%s exists and is not Nebula's", want.Namespace, want.Name) + } } if err != nil { return fmt.Errorf("kubernetes: write env secret for %s/%s: %w", pod.Namespace, pod.Name, classifyAPI(err)) @@ -518,25 +536,6 @@ func ensureEnvSecret(ctx context.Context, kube kubeClient, pod *corev1.Pod, env return nil } -// replaceEnvSecret deletes the existing env Secret, a previous pod's of the same name, and -// creates it afresh. Only if Nebula made it: a user's own Secret with the name is refused, -// never deleted. -func replaceEnvSecret(ctx context.Context, kube kubeClient, want *corev1.Secret) error { - secrets := kube.CoreV1().Secrets(want.Namespace) - got, err := secrets.Get(ctx, want.Name, metav1.GetOptions{}) - if err == nil && !ownedByUs(got) { - return fmt.Errorf("secret %s/%s exists and is not Nebula's", want.Namespace, want.Name) - } - if err == nil { - err = secrets.Delete(ctx, want.Name, metav1.DeleteOptions{Preconditions: metav1.NewUIDPreconditions(string(got.UID))}) - } - if err != nil && !apierrors.IsNotFound(err) { - return err - } - _, err = secrets.Create(ctx, want, metav1.CreateOptions{}) - return err -} - // deleteInstance removes the real pod; already gone is success. Its env Secret follows // through the ownerReference (see ensureEnvSecret). func deleteInstance(ctx context.Context, cc clusterClients, namespace, name string) error { @@ -547,26 +546,6 @@ func deleteInstance(ctx context.Context, cc clusterClients, namespace, name stri return nil } -// deleteAndWait deletes the pod and waits for the name to free up, so a replacement can take it. -func deleteAndWait(ctx context.Context, cc clusterClients, namespace, name string) error { - if err := deleteInstance(ctx, cc, namespace, name); err != nil { - return err - } - t := time.NewTicker(waitInterval) - defer t.Stop() - for { - _, err := cc.kube.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return nil - } - select { - case <-ctx.Done(): - return fmt.Errorf("kubernetes: previous pod %s/%s still terminating: %w", namespace, name, ctx.Err()) - case <-t.C: - } - } -} - // toInstance normalizes a real pod. Region is the cluster's AWS region, the token the claim // carries. // diff --git a/pkg/provider/kubernetes/provision_test.go b/pkg/provider/kubernetes/provision_test.go index 730f5d6..5db5e0b 100644 --- a/pkg/provider/kubernetes/provision_test.go +++ b/pkg/provider/kubernetes/provision_test.go @@ -328,21 +328,63 @@ func TestProvisionRefusesToOverwriteForeignSecret(t *testing.T) { } } -// A previous pod's env Secret, Nebula's own, is replaced with the new values and owner. -func TestProvisionReplacesOwnEnvSecret(t *testing.T) { +// Whatever already holds the names is refused before anything is created, so no container can +// start with a Secret that is not this request's; the error blocks nothing, the Pod's owner +// retries. +func TestProvisionRefusesWhatHoldsTheName(t *testing.T) { + finished := func() *corev1.Pod { + pod := buildPod(placeholder(), podSpec{claim: "org-1-work-1"}) + pod.Status.Phase = corev1.PodSucceeded + return pod + } + stale := func() *corev1.Secret { + sec := envSecret("org-1", "org-1-work-1", map[string]string{"K": "stale"}) + sec.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "Pod", Name: "org-1-work-1", UID: "old"}} + return sec + } + cases := map[string]runtime.Object{ + "earlier pod, finished": finished(), + "earlier pod's secret": stale(), + } + for name, obj := range cases { + t.Run(name, func(t *testing.T) { + tc := newTestCluster(t, true) + if err := tc.kube.Tracker().Add(obj); err != nil { + t.Fatal(err) + } + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + + _, err := p.Provision(context.Background(), placeholder(), provisionRequest()) + if err == nil { + t.Fatal("Provision took a name already held") + } + if scope := p.ClassifyProvisionError(err, "H100:1", "us-east-1"); scope != (provider.BlockScope{}) { + t.Errorf("a name collision blocked %+v; it says nothing about capacity", scope) + } + if sec, err := tc.kube.CoreV1().Secrets("org-1").Get(context.Background(), "org-1-work-1-env", + metav1.GetOptions{}); err == nil && sec.StringData["K"] == "v" { + t.Error("the request's Secret was written despite the refusal") + } + }) + } +} + +// A live pod of ours is the one an interrupted Provision left behind: adopted, with its Secret. +func TestProvisionAdoptsLivePod(t *testing.T) { tc := newTestCluster(t, true) - old := envSecret("org-1", "org-1-work-1", map[string]string{"K": "stale"}) - old.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "Pod", Name: "org-1-work-1", UID: "old-uid"}} - if _, err := tc.kube.CoreV1().Secrets("org-1").Create(context.Background(), old, metav1.CreateOptions{}); err != nil { - t.Fatal(err) + live := buildPod(placeholder(), podSpec{claim: "org-1-work-1"}) + live.UID, live.Spec.NodeName, live.Status.Phase = testUID, "node-1", corev1.PodRunning + sec := envSecret("org-1", "org-1-work-1", map[string]string{"K": "v"}) + sec.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "Pod", Name: "org-1-work-1", UID: testUID}} + for _, obj := range []runtime.Object{live, sec} { + if err := tc.kube.Tracker().Add(obj); err != nil { + t.Fatal(err) + } } p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) - if _, err := p.Provision(context.Background(), placeholder(), provisionRequest()); err != nil { - t.Fatal(err) - } - got, err := tc.kube.CoreV1().Secrets("org-1").Get(context.Background(), "org-1-work-1-env", metav1.GetOptions{}) - if err != nil || got.StringData["K"] != "v" || got.OwnerReferences[0].UID != testUID { - t.Fatalf("secret = %+v, %v; want the new values, owned by the new pod", got, err) + res, err := p.Provision(context.Background(), placeholder(), provisionRequest()) + if err != nil || !res.Reserved || res.InstanceID != "us-east-1/org-1/org-1-work-1" { + t.Fatalf("Provision = %+v, %v; want the live pod adopted", res, err) } } From 0c3f25356abb665b04c223ac3178026bc73502cc Mon Sep 17 00:00:00 2001 From: kerthcet Date: Thu, 22 Oct 2026 20:22:57 +0100 Subject: [PATCH 04/21] address comments Signed-off-by: kerthcet --- internal/controller/nodeclaim_controller.go | 24 +++++++++++++++- .../controller/nodeclaim_controller_test.go | 28 +++++++++++++++++-- pkg/provider/kubernetes/kubernetes.go | 4 ++- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/internal/controller/nodeclaim_controller.go b/internal/controller/nodeclaim_controller.go index 98025de..d94bfdf 100644 --- a/internal/controller/nodeclaim_controller.go +++ b/internal/controller/nodeclaim_controller.go @@ -203,9 +203,16 @@ func (r *NodeClaimReconciler) reconcileDelete(ctx context.Context, nc *nebulav1a // case this backstop exists for), or a failed Provision created an instance without // ever returning its id. The Pod-derived claim name is then the only way to reclaim // it. If DeletePod already ran, nothing matches and id stays "" (a no-op Terminate). + // + // A recorded id is only trusted once the provider says it is this claim's instance: it + // is copied from a Pod annotation the workload's owner can write (see recordInstanceID), + // so taken as-is it would let anyone have Nebula terminate an instance of their choosing. + claim := util.ClaimName(nc.Spec.PodRef.Namespace, nc.Spec.PodRef.Name) id := nc.Status.InstanceID + if id != "" && !r.instanceOfClaim(ctx, prov, id, claim, nc.Spec.Region) { + id = "" + } if id == "" { - claim := util.ClaimName(nc.Spec.PodRef.Namespace, nc.Spec.PodRef.Name) var err error if id, err = r.findInstanceID(ctx, prov, claim, nc.Spec.Region); err != nil { log.Error(err, "looking up the claim's instance for teardown; will retry") @@ -233,6 +240,21 @@ func (r *NodeClaimReconciler) reconcileDelete(ctx context.Context, nc *nebulav1a // findInstanceID returns the id of claim's live instance, or "" when none exists // (already gone or never provisioned), which Terminate treats as a no-op. +// instanceOfClaim reports whether the provider knows id as claim's instance. Anything short of +// that, gone, unreadable or another claim's, is false, and teardown falls back to the lookup +// by claim rather than retrying on an id that may be forged. +func (r *NodeClaimReconciler) instanceOfClaim( + ctx context.Context, prov provider.Provider, id, claim, region string, +) bool { + inst, err := prov.Get(ctx, id, region) + if err == nil && inst != nil && inst.ClaimName == claim { + return true + } + logf.FromContext(ctx).Info("recorded instance id is not this claim's; looking the instance up by claim", + "instanceID", id, "claim", claim, "error", err) + return false +} + func (r *NodeClaimReconciler) findInstanceID(ctx context.Context, prov provider.Provider, claim, region string) (string, error) { inst, err := prov.FindByClaim(ctx, claim, region) if err != nil || inst == nil { diff --git a/internal/controller/nodeclaim_controller_test.go b/internal/controller/nodeclaim_controller_test.go index abaa6f1..6b398f4 100644 --- a/internal/controller/nodeclaim_controller_test.go +++ b/internal/controller/nodeclaim_controller_test.go @@ -69,7 +69,12 @@ func (f *fakeProvider) Terminate(_ context.Context, id, region string) error { f.regions = append(f.regions, region) return f.terminateErr } -func (f *fakeProvider) Get(context.Context, string, string) (*provider.Instance, error) { +func (f *fakeProvider) Get(_ context.Context, id, _ string) (*provider.Instance, error) { + for _, inst := range f.list { + if inst.ID == id { + return &inst, nil + } + } return nil, nil } func (f *fakeProvider) List(context.Context) ([]provider.Instance, error) { @@ -509,7 +514,7 @@ func TestReconcileDelete_UsesRecordedInstanceID(t *testing.T) { claim := newClaim("c1", "p1", "default", "uid-1", "fake") claim.Status.InstanceID = "inst-recorded" deleteClaim(t, claim) - prov := &fakeProvider{name: "fake"} // List would return nothing + prov := &fakeProvider{name: "fake", list: []provider.Instance{{ID: "inst-recorded", ClaimName: "default-p1"}}} r, _ := newClaimReconciler(t, []client.Object{claim}, prov) reconcileClaim(t, r, "c1") @@ -522,6 +527,25 @@ func TestReconcileDelete_UsesRecordedInstanceID(t *testing.T) { } } +// The recorded id comes from a Pod annotation the workload's owner can write, so one naming +// another claim's instance must not be terminated: the backstop falls back to its own claim's. +func TestReconcileDelete_IgnoresForgedInstanceID(t *testing.T) { + claim := newClaim("c1", "p1", "default", "uid-1", "fake") + claim.Status.InstanceID = "someone-elses" + deleteClaim(t, claim) + prov := &fakeProvider{name: "fake", list: []provider.Instance{ + {ID: "someone-elses", ClaimName: "other-ns-victim"}, + {ID: "mine", ClaimName: "default-p1"}, + }} + r, _ := newClaimReconciler(t, []client.Object{claim}, prov) + + reconcileClaim(t, r, "c1") + + if len(prov.terminated) != 1 || prov.terminated[0] != "mine" { + t.Fatalf("terminated %v, want only this claim's instance", prov.terminated) + } +} + // The backstop runs when VK never did, so the region VK held in memory is gone too. It // must pass spec.Region, written before provisioning and never rewritten — otherwise a // region-partitioned provider has to search for the instance and can conclude "already diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index 5b6ce3b..58dd75d 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -258,7 +258,9 @@ func (p *Provider) Provision( // A failure is only logged: the Pod fails regardless, and the NodeClaim teardown finds the // leftover through FindByClaim. func cleanup(ctx context.Context, cc clusterClients, namespace, name string) { - if err := deleteInstance(context.WithoutCancel(ctx), cc, namespace, name); err != nil { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), provisionSlack) + defer cancel() + if err := deleteInstance(ctx, cc, namespace, name); err != nil { logf.FromContext(ctx).Error(err, "kubernetes: could not delete pod without capacity; teardown will retry", "namespace", namespace, "pod", name) } From f77668a7869e95f00fde9f9d2ca63a1ae2f7d50f Mon Sep 17 00:00:00 2001 From: kerthcet Date: Thu, 22 Oct 2026 21:06:18 +0100 Subject: [PATCH 05/21] add not ready state Signed-off-by: kerthcet --- internal/webhook/v1/pod_webhook_test.go | 43 ---------------------- pkg/provider/kubernetes/kubernetes.go | 15 +++++++- pkg/provider/kubernetes/kubernetes_test.go | 23 ++++++++++++ pkg/provider/provider.go | 4 ++ pkg/vnode/handler.go | 18 +++++---- pkg/vnode/handler_test.go | 33 +++++++++++++++++ pkg/vnode/status.go | 9 ++++- 7 files changed, 93 insertions(+), 52 deletions(-) diff --git a/internal/webhook/v1/pod_webhook_test.go b/internal/webhook/v1/pod_webhook_test.go index 11bde90..305552e 100644 --- a/internal/webhook/v1/pod_webhook_test.go +++ b/internal/webhook/v1/pod_webhook_test.go @@ -216,49 +216,6 @@ func TestValidateCreate_OneContainerOnePort(t *testing.T) { } } -func TestValidateCreate_NoVolumesNoServiceAccount(t *testing.T) { - saToken := corev1.Volume{ - Name: "kube-api-access-x7k2p", - VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{}}, - } - cases := []struct { - name string - mutate func(*corev1.Pod) - wantErr string - }{ - // What the API server itself adds to every Pod must be admitted. - {"default service account and its token", func(p *corev1.Pod) { - p.Spec.ServiceAccountName = "default" - p.Spec.Volumes = []corev1.Volume{saToken} - }, ""}, - {"own service account", func(p *corev1.Pod) { p.Spec.ServiceAccountName = "worker" }, "service accounts"}, - {"emptyDir", func(p *corev1.Pod) { - p.Spec.Volumes = []corev1.Volume{saToken, {Name: "scratch", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} - }, "volumes"}, - // Only the projected token volume is exempt, not anything that borrows its name. - {"lookalike name", func(p *corev1.Pod) { - p.Spec.Volumes = []corev1.Volume{{Name: "kube-api-access-fake", - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} - }, "volumes"}, - } - v := &PodCustomValidator{} - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - pod := podWith(map[string]string{nebulav1alpha1.EnabledLabel: "true"}, "") - pod.Spec.Containers = []corev1.Container{{Name: "main"}} - tc.mutate(pod) - _, err := v.ValidateCreate(context.Background(), pod) - switch { - case tc.wantErr == "" && err != nil: - t.Fatalf("ValidateCreate err = %v, want nil", err) - case tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)): - t.Fatalf("ValidateCreate err = %v, want it to contain %q", err, tc.wantErr) - } - }) - } -} - func TestValidateCreate_RejectsNonPod(t *testing.T) { if _, err := (&PodCustomValidator{}).ValidateCreate(context.Background(), &corev1.Service{}); err == nil { t.Fatal("expected an error for a non-Pod object") diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index 58dd75d..ef121af 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -576,7 +576,11 @@ func (p *Provider) toInstance(cl cluster, pod *corev1.Pod) provider.Instance { func toState(pod *corev1.Pod) provider.InstanceState { switch pod.Status.Phase { case corev1.PodRunning: - return provider.InstanceRunning + // Running only says the containers started; Ready is what makes the workload available. + if podReady(pod) { + return provider.InstanceRunning + } + return provider.InstanceNotReady case corev1.PodSucceeded: return provider.InstanceTerminated case corev1.PodFailed: @@ -586,6 +590,15 @@ func toState(pod *corev1.Pod) provider.InstanceState { } } +func podReady(pod *corev1.Pod) bool { + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + // classifyAPI marks a 401 or 403 as ErrAuth. ClassifyProvisionError narrows that to the one // cluster, since another cluster's credentials and RBAC are separate. func classifyAPI(err error) error { diff --git a/pkg/provider/kubernetes/kubernetes_test.go b/pkg/provider/kubernetes/kubernetes_test.go index 20c4af4..fbc6c98 100644 --- a/pkg/provider/kubernetes/kubernetes_test.go +++ b/pkg/provider/kubernetes/kubernetes_test.go @@ -405,3 +405,26 @@ func TestIsExtendedResource(t *testing.T) { } } } + +func TestToStateNeedsReady(t *testing.T) { + running := func(ready corev1.ConditionStatus) *corev1.Pod { + pod := &corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodRunning}} + if ready != "" { + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: ready}} + } + return pod + } + for name, tc := range map[string]struct { + pod *corev1.Pod + want provider.InstanceState + }{ + "ready": {running(corev1.ConditionTrue), provider.InstanceRunning}, + "probe failing": {running(corev1.ConditionFalse), provider.InstanceNotReady}, + "no condition yet": {running(""), provider.InstanceNotReady}, + "pending": {&corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending}}, provider.InstancePending}, + } { + if got := toState(tc.pod); got != tc.want { + t.Errorf("%s: toState = %s, want %s", name, got, tc.want) + } + } +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index e3f80c9..6b4a704 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -443,6 +443,10 @@ const ( InstancePending InstanceState = "Pending" // InstanceRunning: up and reachable. InstanceRunning InstanceState = "Running" + // InstanceNotReady: running, but not ready (a failing readiness probe, a crash loop). The + // Pod shows Running with Ready=False, and the readiness deadline fails it if it stays + // unusable that long. + InstanceNotReady InstanceState = "NotReady" // InstanceTerminated: gone (also the mapping for "absent from List"). InstanceTerminated InstanceState = "Terminated" // InstanceFailed: entered a terminal error state. diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index f18080d..6369ba4 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -706,7 +706,8 @@ func (h *Handler) reconcileOnce(ctx context.Context) { }) default: updated++ - if waited, over := h.readyExpired(tp, inst.State); over { + state := inst.State + if waited, over := h.readyExpired(tp, state); over { // Failing the Pod is the whole action: teardown follows from the // terminal phase, via the reap (see readyExpired). log.Info("readiness deadline exceeded; failing the pod", @@ -715,11 +716,11 @@ func (h *Handler) reconcileOnce(ctx context.Context) { applyReadinessTimeout(tp.pod, waited, h.nowFn()) break } - applyState(tp.pod, inst.State, inst.Endpoint, h.nowFn()) + applyState(tp.pod, state, inst.Endpoint, h.nowFn()) // The observed address, for a provider that cannot know it before boot. // Empty for one that published at create, which must not clear it. setEndpoint(tp.pod, inst.Endpoint) - h.observeReady(tp, inst.State) + h.observeReady(tp, state) } // Log before -> after every tick: a differing pair is the lifecycle moving // (Provisioning -> Initializing -> Running, or -> Terminated), an equal pair @@ -767,10 +768,13 @@ func (h *Handler) reconcileOnce(ctx context.Context) { } } -// readyExpired maintains the Initializing clock and reports whether it has run -// past the deadline, with how long the pod has been there. +// readyExpired maintains the not-usable clock and reports whether it has run past the +// deadline, with how long the pod has been there. It counts InstancePending and +// InstanceNotReady alike, so a workload that never comes up and one that stops being ready +// both fail once unusable for the whole deadline; Running resets it, so a brief unready spell +// costs nothing. // -// It anchors on the FIRST observation of InstancePending, not on provisioningAt: the +// It anchors on the FIRST observation of a not-usable state, not on provisioningAt: the // provision call is bounded separately (see defaultProvisionTimeout), so measuring from // there would spend the readiness budget on a box that had barely started booting. Any // other state RESETS the clock rather than merely pausing it — a demoted pod gets a fresh @@ -788,7 +792,7 @@ func (h *Handler) reconcileOnce(ctx context.Context) { // // Callers must hold h.mu. func (h *Handler) readyExpired(tp *trackedPod, state provider.InstanceState) (time.Duration, bool) { - if state != provider.InstancePending { + if state != provider.InstancePending && state != provider.InstanceNotReady { tp.initializingAt = time.Time{} return 0, false } diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go index c7dc963..6e10223 100644 --- a/pkg/vnode/handler_test.go +++ b/pkg/vnode/handler_test.go @@ -2018,3 +2018,36 @@ func TestReconcileOnce_UnschedulableBlocksAndFailsPod(t *testing.T) { } func ptrTo[T any](v T) *T { return &v } + +// NotReady shows as it is, Running with Ready=False, and counts toward the readiness deadline +// like Pending: unusable for the whole deadline fails the Pod, before or after it was ready. +// Running in between resets the clock. +func TestReconcileOnce_NotReadyIsShownAndCounted(t *testing.T) { + fp := &fakeProvider{provisionID: "inst-1"} + h := NewHandler(fp, nil, nil, openCluster()) + if err := h.CreatePod(context.Background(), testPod("default", "p1")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + tick := func(state provider.InstanceState) *corev1.Pod { + fp.list = []provider.Instance{{ID: "inst-1", ClaimName: "default-p1", State: state}} + h.reconcileOnce(context.Background()) + got, _ := h.GetPod(context.Background(), "default", "p1") + return got + } + + got := tick(provider.InstanceNotReady) + if got.Status.Phase != corev1.PodRunning || isPodReadyStatus(got) { + t.Fatalf("not-ready instance = %s, ready=%v; want Running, not ready", got.Status.Phase, isPodReadyStatus(got)) + } + if got := tick(provider.InstanceRunning); !isPodReadyStatus(got) { + t.Fatal("a ready instance must make the Pod ready") + } + + tick(provider.InstanceNotReady) // the clock starts again, from zero + rewindInitializing(t, h, "default", "p1", 11*time.Minute) + got = tick(provider.InstanceNotReady) + if got.Status.Phase != corev1.PodFailed || got.Status.Reason != reasonReadinessTimeout { + t.Fatalf("unusable past the deadline = %s/%s, want Failed/%s", + got.Status.Phase, got.Status.Reason, reasonReadinessTimeout) + } +} diff --git a/pkg/vnode/status.go b/pkg/vnode/status.go index 697612e..6f6b2e7 100644 --- a/pkg/vnode/status.go +++ b/pkg/vnode/status.go @@ -71,6 +71,13 @@ func applyState(pod *corev1.Pod, state provider.InstanceState, endpoint string, if endpoint != "" && net.ParseIP(endpoint) != nil { pod.Status.PodIP = endpoint } + case provider.InstanceNotReady: + setPhase(pod, corev1.PodRunning, reasonRunning, "external instance is running but not ready", now) + setReady(pod, corev1.ConditionFalse, now) + setContainerStatuses(pod, corev1.ContainerState{Running: &corev1.ContainerStateRunning{StartedAt: now}}, false) + if endpoint != "" && net.ParseIP(endpoint) != nil { + pod.Status.PodIP = endpoint + } case provider.InstancePending: setPhase(pod, corev1.PodPending, reasonInitializing, "external instance is initializing", now) setReady(pod, corev1.ConditionFalse, now) @@ -111,7 +118,7 @@ func applyState(pod *corev1.Pod, state provider.InstanceState, endpoint string, // verdict, so it must not read as a provider-reported failure — see // PodReasonReadinessTimeout. func applyReadinessTimeout(pod *corev1.Pod, waited time.Duration, now metav1.Time) { - msg := fmt.Sprintf("external instance did not become ready within %s", waited.Round(time.Second)) + msg := fmt.Sprintf("external instance was not ready for %s", waited.Round(time.Second)) setPhase(pod, corev1.PodFailed, reasonReadinessTimeout, msg, now) setReady(pod, corev1.ConditionFalse, now) setContainerStatuses(pod, corev1.ContainerState{ From 1fc979d4ee8b8dc66dcc400f5dd435c0031abe7e Mon Sep 17 00:00:00 2001 From: kerthcet Date: Thu, 22 Oct 2026 22:40:42 +0100 Subject: [PATCH 06/21] address comments Signed-off-by: kerthcet --- docs/metrics.md | 2 +- .../pod_placement_controller_test.go | 5 ++--- internal/webhook/v1/pod_webhook.go | 5 +++++ internal/webhook/v1/pod_webhook_test.go | 20 +++++++++++++++++++ pkg/provider/modal/modal_test.go | 1 + pkg/util/accelerator.go | 12 +++++------ pkg/util/accelerator_test.go | 2 +- pkg/vnode/handler_test.go | 3 +-- pkg/vnode/metrics_test.go | 12 +++++++++-- 9 files changed, 47 insertions(+), 15 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 4c75f50..da2c307 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -67,7 +67,7 @@ which is the whole reason for splitting them: | `reason` | Means | Clears when | | --- | --- | --- | | `no_pool` | The Pod names a NodePool that does not exist, or carries no pool label. | A human fixes the Pod (or the workload generating it). | -| `invalid_request` | The request is malformed — `nvidia.com/gpu` with no accelerator-type label (*not* treated as CPU-only), or a `nebula.inftyai.com/regions` annotation naming no known geography. | A human fixes the Pod spec. | +| `invalid_request` | The request is malformed — `nvidia.com/gpu` with no accelerator-type label (*not* treated as CPU-only), an accelerator-type label with no `nvidia.com/gpu` request, or a `nebula.inftyai.com/regions` annotation naming no known geography. | A human fixes the Pod spec. | | `all_blocked` | A servable candidate exists, but failover is holding every one of them off. | By itself — the Pod is already requeued for the block's expiry. | | `no_candidate` | No provider in the pool can serve this request at all. | An operator adds a provider, or a provider registers. | | `stale_claim` | A NodeClaim from a prior same-named Pod has not been reaped yet. | By itself, in seconds. A sustained rate means the NodeClaim backstop is stuck. | diff --git a/internal/controller/pod_placement_controller_test.go b/internal/controller/pod_placement_controller_test.go index 0da7557..db97a60 100644 --- a/internal/controller/pod_placement_controller_test.go +++ b/internal/controller/pod_placement_controller_test.go @@ -106,10 +106,9 @@ func gatedPod(name, ns, uid, pool, gpu string) *corev1.Pod { }, } if gpu != "" { - // The placement controller matches on the accelerator TYPE only, which - // rides on the accelerator-type label; the count (nvidia.com/gpu) is a - // provisioning detail the adapter reads, not needed here. + // The type and the nvidia.com/gpu count come together (see util.AcceleratorRequest). pod.Labels[nebulav1alpha1.AcceleratorTypeLabel] = gpu + pod.Spec.Containers[0].Resources.Limits = corev1.ResourceList{util.NvidiaGPUResource: resource.MustParse("1")} } return pod } diff --git a/internal/webhook/v1/pod_webhook.go b/internal/webhook/v1/pod_webhook.go index 12fb3b4..052094f 100644 --- a/internal/webhook/v1/pod_webhook.go +++ b/internal/webhook/v1/pod_webhook.go @@ -28,6 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/util" ) var podlog = logf.Log.WithName("pod-webhook") @@ -187,6 +188,10 @@ func validatePod(pod *corev1.Pod) error { if c := pod.Spec.Containers[0]; len(c.Ports) > 1 { return fmt.Errorf("nebula exposes at most one port per Pod, container %q declares %d", c.Name, len(c.Ports)) } + // The accelerator type and the GPU count must come together; see util.AcceleratorRequest. + if _, _, err := util.AcceleratorRequest(pod); err != nil { + return err + } return nil } diff --git a/internal/webhook/v1/pod_webhook_test.go b/internal/webhook/v1/pod_webhook_test.go index 305552e..df518db 100644 --- a/internal/webhook/v1/pod_webhook_test.go +++ b/internal/webhook/v1/pod_webhook_test.go @@ -22,6 +22,7 @@ import ( "testing" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" @@ -216,6 +217,25 @@ func TestValidateCreate_OneContainerOnePort(t *testing.T) { } } +// The accelerator type and the GPU count come together; a type alone is rejected up front, +// rather than placed as a GPU request the container itself does not make. +func TestValidateCreate_AcceleratorNeedsGPUCount(t *testing.T) { + pod := podWith(map[string]string{ + nebulav1alpha1.EnabledLabel: "true", + nebulav1alpha1.AcceleratorTypeLabel: "h100", + }, "") + pod.Spec.Containers = []corev1.Container{{Name: "main"}} + if _, err := (&PodCustomValidator{}).ValidateCreate(context.Background(), pod); err == nil || + !strings.Contains(err.Error(), "nvidia.com/gpu") { + t.Fatalf("ValidateCreate err = %v, want a missing nvidia.com/gpu rejection", err) + } + + pod.Spec.Containers[0].Resources.Limits = corev1.ResourceList{"nvidia.com/gpu": resource.MustParse("1")} + if _, err := (&PodCustomValidator{}).ValidateCreate(context.Background(), pod); err != nil { + t.Fatalf("ValidateCreate err = %v for a type with its count, want nil", err) + } +} + func TestValidateCreate_RejectsNonPod(t *testing.T) { if _, err := (&PodCustomValidator{}).ValidateCreate(context.Background(), &corev1.Service{}); err == nil { t.Fatal("expected an error for a non-Pod object") diff --git a/pkg/provider/modal/modal_test.go b/pkg/provider/modal/modal_test.go index ae72842..66936d3 100644 --- a/pkg/provider/modal/modal_test.go +++ b/pkg/provider/modal/modal_test.go @@ -294,6 +294,7 @@ func TestProvision_MapsResourcesPortsAndTimeout(t *testing.T) { corev1.ResourceCPU: resource.MustParse("2500m"), corev1.ResourceMemory: resource.MustParse("4Gi"), }, + Limits: corev1.ResourceList{util.NvidiaGPUResource: resource.MustParse("1")}, } c.Ports = []corev1.ContainerPort{{ContainerPort: 8000}, {ContainerPort: 9090}} deadline := int64(3600) diff --git a/pkg/util/accelerator.go b/pkg/util/accelerator.go index f1ebea9..5b09ada 100644 --- a/pkg/util/accelerator.go +++ b/pkg/util/accelerator.go @@ -43,13 +43,13 @@ const NvidiaGPUResource corev1.ResourceName = "nvidia.com/gpu" // // It returns: // - ("", 0, nil) no GPU type requested => a CPU-only workload. -// - (type, count, nil) a GPU workload; count defaults to 1 when the type is -// set but no nvidia.com/gpu resource is present, so authors can request a -// single GPU with just the label. +// - (type, count, nil) a GPU workload. // // The type is returned verbatim (case is normalized downstream by the provider -// catalog's MapAccelerator). An error is returned only for a genuinely -// contradictory request: an explicit nvidia.com/gpu count with no GPU type. +// catalog's MapAccelerator). Type and count must come together: a count with no +// type, or a type with no nvidia.com/gpu request, is an error. A type alone used to +// mean one GPU, but the container itself asks for none, so a provider running it +// would get no device. func AcceleratorRequest(pod *corev1.Pod) (accelerator string, count int32, err error) { typ := pod.Labels[nebulav1alpha1.AcceleratorTypeLabel] n := gpuCount(pod) @@ -61,7 +61,7 @@ func AcceleratorRequest(pod *corev1.Pod) (accelerator string, count int32, err e return "", 0, nil // CPU-only } if n == 0 { - n = 1 // a GPU type with no explicit count means one GPU + return "", 0, fmt.Errorf("%s=%s set without an nvidia.com/gpu request", nebulav1alpha1.AcceleratorTypeLabel, typ) } return typ, n, nil } diff --git a/pkg/util/accelerator_test.go b/pkg/util/accelerator_test.go index 574f3f9..aa18c35 100644 --- a/pkg/util/accelerator_test.go +++ b/pkg/util/accelerator_test.go @@ -55,7 +55,7 @@ func TestAcceleratorRequest(t *testing.T) { }{ {name: "no label is CPU-only", typ: "", gpuLimit: 0, wantAccel: "", wantCount: 0}, {name: "type and count", typ: "a100-40gb", gpuLimit: 8, wantAccel: "a100-40gb", wantCount: 8}, - {name: "type only defaults to 1", typ: "h100", gpuLimit: 0, wantAccel: "h100", wantCount: 1}, + {name: "type without count is rejected", typ: "h100", gpuLimit: 0, wantErr: true}, {name: "casing preserved", typ: "A100-80GB", gpuLimit: 2, wantAccel: "A100-80GB", wantCount: 2}, {name: "count without type is contradictory", typ: "", gpuLimit: 4, wantErr: true}, } diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go index 6e10223..8ef9ef6 100644 --- a/pkg/vnode/handler_test.go +++ b/pkg/vnode/handler_test.go @@ -516,8 +516,7 @@ func TestCreatePod_ProvisionFailureRecordsBlock(t *testing.T) { h := NewHandler(fp, nil, bl, clusterWithTTL(7*time.Minute)) h.jitterFn = func() time.Duration { return 0 } // pin jitter so the base TTL is asserted exactly - pod := testPod("default", "p1") - pod.Labels[nebulav1alpha1.AcceleratorTypeLabel] = "H100" + pod := withGPU(testPod("default", "p1"), "H100") if err := h.CreatePod(context.Background(), pod); err == nil { t.Fatal("expected CreatePod to return the provision error") diff --git a/pkg/vnode/metrics_test.go b/pkg/vnode/metrics_test.go index a6448fa..805e153 100644 --- a/pkg/vnode/metrics_test.go +++ b/pkg/vnode/metrics_test.go @@ -31,6 +31,8 @@ import ( nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" "github.com/InftyAI/Nebula/pkg/metrics" "github.com/InftyAI/Nebula/pkg/provider" + "github.com/InftyAI/Nebula/pkg/util" + "k8s.io/apimachinery/pkg/api/resource" ) // The collectors are package-level and registered once, so every test in this package @@ -60,8 +62,14 @@ func histCount(t *testing.T, h *prometheus.HistogramVec, l prometheus.Labels) ui // the pool identity is derived from. Pair it with metricCluster, which supplies the other // half — the tier and region, which are read from the NodeClaim, not the Pod. func metricPod(ns, name string) *corev1.Pod { - pod := testPod(ns, name) - pod.Labels[nebulav1alpha1.AcceleratorTypeLabel] = "H100" + return withGPU(testPod(ns, name), "H100") +} + +// withGPU makes pod a one-GPU request of type accel: the type label and the nvidia.com/gpu +// count, which util.AcceleratorRequest requires together. +func withGPU(pod *corev1.Pod, accel string) *corev1.Pod { + pod.Labels[nebulav1alpha1.AcceleratorTypeLabel] = accel + pod.Spec.Containers[0].Resources.Limits = corev1.ResourceList{util.NvidiaGPUResource: resource.MustParse("1")} return pod } From a0a8dac986ee8382d67b05de5b7004a3c2ce8db3 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Thu, 22 Oct 2026 23:00:10 +0100 Subject: [PATCH 07/21] fix error Signed-off-by: kerthcet --- pkg/provider/errors.go | 25 ++++++++++++++++++++++ pkg/provider/kubernetes/kubernetes.go | 23 ++------------------ pkg/provider/kubernetes/kubernetes_test.go | 12 ++++++++++- pkg/provider/kubernetes/pod.go | 7 ++++-- pkg/vnode/handler.go | 8 ++++++- pkg/vnode/handler_test.go | 12 +++++++++-- 6 files changed, 60 insertions(+), 27 deletions(-) diff --git a/pkg/provider/errors.go b/pkg/provider/errors.go index e3337b8..7193315 100644 --- a/pkg/provider/errors.go +++ b/pkg/provider/errors.go @@ -126,6 +126,31 @@ func ClassifyError(err error, capacityType nebulav1alpha1.CapacityType, accelera } } +// WithTier records the capacity tier a provision failure happened under. ClassifyProvisionError +// is not handed one, so an adapter whose errors do not say it themselves reads it back with +// TierOf, and a Spot shortage does not block OnDemand. The poll loop wraps the failures it +// finds late the same way, so they block what a synchronous Provision failure would. +func WithTier(tier nebulav1alpha1.CapacityType, err error) error { + return &tierError{tier: tier, err: err} +} + +// TierOf returns the tier WithTier recorded on err, or "" if none was. +func TierOf(err error) nebulav1alpha1.CapacityType { + var te *tierError + if errors.As(err, &te) { + return te.tier + } + return "" +} + +type tierError struct { + tier nebulav1alpha1.CapacityType + err error +} + +func (e *tierError) Error() string { return e.err.Error() } +func (e *tierError) Unwrap() error { return e.err } + // IsDeadline reports whether err is a provision deadline that fired. func IsDeadline(err error) bool { return err != nil && (errors.Is(err, context.DeadlineExceeded) || diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index ef121af..5927219 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -244,7 +244,7 @@ func (p *Provider) Provision( bound, err := waitForProvision(ctx, cl.clients, cl.nodeAutoscaler, real.Namespace, real.Name) if err != nil { cleanup(ctx, cl.clients, real.Namespace, real.Name) - return provider.ProvisionResult{}, withTier(req.CapacityType, err) + return provider.ProvisionResult{}, provider.WithTier(req.CapacityType, err) } // Unbound is accepted, not reserved: the node may still be booting. List settles it, // reporting no capacity once the pod is unbound past its node autoscaler's bindTimeout. @@ -403,12 +403,7 @@ func (p *Provider) ClassifyProvisionError(err error, accelerator, region string) if err == nil { return provider.BlockScope{} } - var te *tierError - var tier nebulav1alpha1.CapacityType - if errors.As(err, &te) { - tier = te.tier - } - scope := provider.ClassifyError(err, tier, accelerator) + scope := provider.ClassifyError(err, provider.TierOf(err), accelerator) if scope == (provider.BlockScope{}) { return scope // block nothing; decorating it would create a block } @@ -607,17 +602,3 @@ func classifyAPI(err error) error { } return err } - -// tierError carries the requested tier to ClassifyProvisionError, which is not handed one, -// so a Spot shortage does not block OnDemand. -type tierError struct { - tier nebulav1alpha1.CapacityType - err error -} - -func (e *tierError) Error() string { return e.err.Error() } -func (e *tierError) Unwrap() error { return e.err } - -func withTier(tier nebulav1alpha1.CapacityType, err error) error { - return &tierError{tier: tier, err: err} -} diff --git a/pkg/provider/kubernetes/kubernetes_test.go b/pkg/provider/kubernetes/kubernetes_test.go index fbc6c98..980c1a6 100644 --- a/pkg/provider/kubernetes/kubernetes_test.go +++ b/pkg/provider/kubernetes/kubernetes_test.go @@ -247,7 +247,7 @@ func TestClassifyProvisionError(t *testing.T) { t.Errorf("nil error blocked %+v", s) } - spot := p.ClassifyProvisionError(withTier(nebulav1alpha1.CapacitySpot, + spot := p.ClassifyProvisionError(provider.WithTier(nebulav1alpha1.CapacitySpot, fmt.Errorf("x: %w", provider.ErrNoCapacity)), "H100:1", region) if spot.CapacityType != nebulav1alpha1.CapacitySpot || *spot.Region != region || *spot.Accelerator != "H100:1" { t.Errorf("spot shortage scope = %+v", spot) @@ -428,3 +428,13 @@ func TestToStateNeedsReady(t *testing.T) { } } } + +// The kubelet expands $(VAR) and folds $$ in env values; resolved values must come out as +// they went in, so every $ is escaped. +func TestEnvVarsEscapeDollar(t *testing.T) { + got := envVars(map[string]string{"REF": "$(TOKEN)", "TWO": "a$$b", "PLAIN": "x"}) + want := []corev1.EnvVar{{Name: "PLAIN", Value: "x"}, {Name: "REF", Value: "$$(TOKEN)"}, {Name: "TWO", Value: "a$$$$b"}} + if !slices.Equal(got, want) { + t.Fatalf("envVars = %+v, want %+v", got, want) + } +} diff --git a/pkg/provider/kubernetes/pod.go b/pkg/provider/kubernetes/pod.go index 65b3c65..34a9888 100644 --- a/pkg/provider/kubernetes/pod.go +++ b/pkg/provider/kubernetes/pod.go @@ -211,14 +211,17 @@ func isExtendedResource(name corev1.ResourceName) bool { return len(validation.IsQualifiedName(corev1.DefaultResourceRequestsPrefix+n)) == 0 } -// envVars renders env in sorted order, so the same request always yields the same spec. +// envVars renders env in sorted order, so the same request always yields the same spec. Each +// "$" is doubled because the kubelet expands $(VAR) in env values (and folds "$$" to "$"): +// the values are already resolved and must reach the container verbatim, as resolveEnv +// promises, not pick up another variable such as one from the env Secret. func envVars(env map[string]string) []corev1.EnvVar { if len(env) == 0 { return nil } out := make([]corev1.EnvVar, 0, len(env)) for _, k := range slices.Sorted(maps.Keys(env)) { - out = append(out, corev1.EnvVar{Name: k, Value: env[k]}) + out = append(out, corev1.EnvVar{Name: k, Value: strings.ReplaceAll(env[k], "$", "$$")}) } return out } diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index 6369ba4..385f4f1 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -701,7 +701,7 @@ func (h *Handler) reconcileOnce(ctx context.Context) { "pod", key(tp.pod.Namespace, tp.pod.Name), "instanceID", tp.instance) applyNoCapacity(tp.pod, h.nowFn()) failed = append(failed, failedPlacement{ - pod: tp.pod.DeepCopy(), region: inst.Region, + pod: tp.pod.DeepCopy(), claim: tp.claimName, region: inst.Region, err: fmt.Errorf("external instance got no capacity: %w", provider.ErrNoCapacity), }) default: @@ -738,6 +738,11 @@ func (h *Handler) reconcileOnce(ctx context.Context) { if err != nil { pool = nil // the default TTL; a block with the wrong length beats no block } + // The tier the instance was placed under, from the claim that recorded it, so a + // late Spot shortage blocks Spot alone, as a failed Provision would. + if nc, err := h.claimFor(ctx, b.pod, b.claim); err == nil { + b.err = provider.WithTier(nc.Spec.CapacityType, b.err) + } h.recordBlock(ctx, b.pod, b.region, blocklistTTLOf(pool), b.err) } @@ -1160,6 +1165,7 @@ func (h *Handler) recordBlock( // unlocking. The provider's classification of err decides the scope. type failedPlacement struct { pod *corev1.Pod + claim string region string err error } diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go index 8ef9ef6..f0b42cb 100644 --- a/pkg/vnode/handler_test.go +++ b/pkg/vnode/handler_test.go @@ -79,6 +79,7 @@ type fakeProvider struct { // timeout does. It exists to prove the provision deadline does not leak into the // writes that follow. provisionBlocksUntilDeadline bool + classifyErr error // the error last passed to ClassifyProvisionError } func (f *fakeProvider) Name() string { return "fake" } @@ -139,7 +140,8 @@ func (f *fakeProvider) FindByClaim(_ context.Context, claim, _ string) (*provide func (f *fakeProvider) Offerings(context.Context) ([]provider.Offering, error) { return nil, nil } func (f *fakeProvider) MapAccelerator(c string, _ int32) ([]string, bool) { return []string{c}, true } func (f *fakeProvider) ResolveRegions(declared, _ []string) []string { return declared } -func (f *fakeProvider) ClassifyProvisionError(_ error, accel, region string) provider.BlockScope { +func (f *fakeProvider) ClassifyProvisionError(err error, accel, region string) provider.BlockScope { + f.classifyErr = err f.classifyAccel = accel f.classifyRegion = region return f.classifyScope @@ -1990,7 +1992,9 @@ func TestReconcileOnce_UnschedulableBlocksAndFailsPod(t *testing.T) { scope := provider.BlockScope{Region: ptrTo("eks-use1")} fp := &fakeProvider{provisionID: "inst-1", classifyScope: scope} bl := &recordingBlocklist{} - h := NewHandler(fp, nil, bl, clusterWithTTL(30*time.Second)) + cluster := clusterWithTTL(30 * time.Second) + cluster.claim.Spec.CapacityType = nebulav1alpha1.CapacitySpot + h := NewHandler(fp, nil, bl, cluster) h.jitterFn = func() time.Duration { return 0 } if err := h.CreatePod(context.Background(), testPod("default", "p1")); err != nil { t.Fatalf("CreatePod: %v", err) @@ -2009,6 +2013,10 @@ func TestReconcileOnce_UnschedulableBlocksAndFailsPod(t *testing.T) { if bl.calls != 1 || bl.ttl != 30*time.Second || fp.classifyRegion != "eks-use1" { t.Fatalf("block = %d calls, ttl %v, region %q; want one, 30s, eks-use1", bl.calls, bl.ttl, fp.classifyRegion) } + // The claim's tier rides on the error, so the provider blocks Spot alone, not every tier. + if tier := provider.TierOf(fp.classifyErr); tier != nebulav1alpha1.CapacitySpot { + t.Fatalf("classified a late failure with tier %q, want the claim's %q", tier, nebulav1alpha1.CapacitySpot) + } h.reconcileOnce(context.Background()) if bl.calls != 1 { From ad390f2fe0f5b0e4fe9807f24b8ebc1af046779e Mon Sep 17 00:00:00 2001 From: kerthcet Date: Fri, 23 Oct 2026 11:33:49 +0100 Subject: [PATCH 08/21] optimize the pod creation path Signed-off-by: kerthcet --- pkg/provider/kubernetes/eks.go | 7 ++ pkg/provider/kubernetes/karpenter.go | 40 ++++---- pkg/provider/kubernetes/kubernetes.go | 86 +++++++++-------- pkg/provider/kubernetes/node_autoscaler.go | 13 +-- pkg/provider/kubernetes/provision_test.go | 107 ++++++++++----------- pkg/provider/kubernetes/wait.go | 74 -------------- 6 files changed, 125 insertions(+), 202 deletions(-) delete mode 100644 pkg/provider/kubernetes/wait.go diff --git a/pkg/provider/kubernetes/eks.go b/pkg/provider/kubernetes/eks.go index 5da9404..5aa792d 100644 --- a/pkg/provider/kubernetes/eks.go +++ b/pkg/provider/kubernetes/eks.go @@ -30,6 +30,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/sts" smithyhttp "github.com/aws/smithy-go/transport/http" + "k8s.io/apimachinery/pkg/runtime" k8s "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) @@ -88,6 +89,12 @@ func newEKSClients(ctx context.Context, cl ClusterConfig) (clusterClients, error TLSClientConfig: rest.TLSClientConfig{CAData: ca}, QPS: clusterQPS, Burst: clusterBurst, + // Protobuf for the built-in types read every poll (pods, events): several times smaller + // and cheaper to decode than JSON. Custom resources only speak JSON, hence the fallback. + ContentConfig: rest.ContentConfig{ + ContentType: runtime.ContentTypeProtobuf, + AcceptContentTypes: runtime.ContentTypeProtobuf + "," + runtime.ContentTypeJSON, + }, WrapTransport: func(rt http.RoundTripper) http.RoundTripper { return &bearerTransport{base: rt, tokens: tokens} }, diff --git a/pkg/provider/kubernetes/karpenter.go b/pkg/provider/kubernetes/karpenter.go index 2b7a65d..b90d5db 100644 --- a/pkg/provider/kubernetes/karpenter.go +++ b/pkg/provider/kubernetes/karpenter.go @@ -27,9 +27,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" - "github.com/InftyAI/Nebula/pkg/provider" ) // Karpenter v1 (sigs.k8s.io/karpenter): its well-known labels, the annotation that keeps @@ -51,10 +51,6 @@ const ( // EC2 launch and node boot until the GPUs are advertised. Measured p-family boots may need // more; image pull comes after binding and is the readiness deadline's. karpenterBindTimeout = 5 * time.Minute - // karpenterProvisionTimeout covers the slow end of Karpenter's decision: its batch window - // (up to 10s by default), the scheduling pass and the EC2 call. A refusal arriving after - // it is only caught by bindTimeout, so err long. - karpenterProvisionTimeout = 30 * time.Second ) // nodeClaimGV is Karpenter v1's API, only checked for presence (see verify). @@ -62,8 +58,7 @@ var nodeClaimGV = schema.GroupVersion{Group: "karpenter.sh", Version: "v1"} type karpenter struct{} -func (karpenter) bindTimeout() time.Duration { return karpenterBindTimeout } -func (karpenter) provisionTimeout() time.Duration { return karpenterProvisionTimeout } +func (karpenter) bindTimeout() time.Duration { return karpenterBindTimeout } // selection pins the GPU model through the instance family (one model per NVIDIA family on // AWS) and the tier through the capacity type, and keeps consolidation and drift from @@ -103,27 +98,30 @@ func karpenterCapacityType(tier nebulav1alpha1.CapacityType) string { } } -// refused returns Karpenter's verdict that it will not get the pod a node, from the pod's -// own events, or nil. EC2 refusing a launch shows nowhere else: Karpenter deletes that -// NodeClaim without recording why. These events are rate-limited, so a dropped one only costs +// refusals reads Karpenter's refusals for every pod in the cluster at once: its warnings about +// pods, InsufficientCapacityError (EC2 had none) and FailedScheduling (no NodePool fits). EC2 +// refusing a launch shows nowhere else, since Karpenter deletes that NodeClaim without +// recording why. The selector leaves out the kube-scheduler's FailedScheduling, which every +// pod needing a new node gets. Karpenter rate-limits these events, so a dropped one only costs // the bind timeout. -func (karpenter) refused(ctx context.Context, cc clusterClients, pod *corev1.Pod) error { - evs, err := cc.kube.CoreV1().Events(pod.Namespace).List(ctx, metav1.ListOptions{ - FieldSelector: fields.OneTermEqualSelector("involvedObject.uid", string(pod.UID)).String(), +func (karpenter) refusals(ctx context.Context, cc clusterClients) (map[types.UID]bool, error) { + evs, err := cc.kube.CoreV1().Events(metav1.NamespaceAll).List(ctx, metav1.ListOptions{ + FieldSelector: fields.Set{ + "involvedObject.kind": "Pod", "source": karpenterComponent, "type": corev1.EventTypeWarning, + }.String(), + ResourceVersion: "0", // served from the API server's watch cache }) if err != nil { - return nil + return nil, err } + out := map[types.UID]bool{} for _, ev := range evs.Items { - // Filtered again here: the selector is the server's job, and not every client honours it. - if ev.InvolvedObject.UID != pod.UID || !fromKarpenter(ev) { - continue - } - if ev.Reason == reasonInsufficientCapacity || ev.Reason == reasonFailedScheduling { - return fmt.Errorf("kubernetes: karpenter: %s: %w", ev.Message, provider.ErrNoCapacity) + // Checked again here: the selector is the server's job, and not every client honours it. + if fromKarpenter(ev) && (ev.Reason == reasonInsufficientCapacity || ev.Reason == reasonFailedScheduling) { + out[ev.InvolvedObject.UID] = true } } - return nil + return out, nil } func (karpenter) verify(_ context.Context, cc clusterClients) error { diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index 5927219..a60092c 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -19,10 +19,10 @@ limitations under the License. // and that cluster's Karpenter launches and packs the nodes. One NodeClaim maps to one real // pod, not to a node; several pods can share a Karpenter node. // -// Capacity is found by trying. Provision creates the pod and watches it briefly, failing at -// once on the node autoscaler's refusal (see waitForProvision); a pod still unbound after that -// is settled by the poll loop, which reports no capacity past the bind timeout (see -// toInstance). Either way the cluster is blocklisted like an AWS region. +// Capacity is found by trying. Provision creates the pod and returns at once, accepted but +// unreserved; the poll loop settles it, reporting no capacity on the node autoscaler's refusal +// or once the pod stays unbound past the bind timeout (see toInstance), and the cluster is +// then blocklisted like an AWS region. // // The real pod lives in the placeholder's namespace, named after the claim like every // provider's instance, which is what lets the host cluster itself be a target. Nebula reports @@ -42,6 +42,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" "github.com/InftyAI/Nebula/pkg/provider" @@ -53,10 +54,8 @@ import ( const ( pollInterval = 5 * time.Second spotNotice = 2 * time.Minute - // provisionSlack is the part of a Provision that is not the wait: the namespace, - // pod and Secret calls, and deleting a previous incarnation. It keeps the handler's - // deadline from cutting the wait short, which would read as no capacity. - provisionSlack = 10 * time.Second + // cleanupTimeout bounds deleting a pod Provision could not finish setting up. + cleanupTimeout = 10 * time.Second ) var _ provider.Provider = (*Provider)(nil) @@ -168,21 +167,9 @@ func (p *Provider) Capabilities() provider.Capabilities { NativeTags: true, PreemptionNotice: spotNotice, PollInterval: pollInterval, - ProvisionTimeout: p.longestProvision() + provisionSlack, } } -// longestProvision is the slowest node autoscaler's provisionTimeout. Capabilities has one -// deadline for the provider, set to this plus provisionSlack, and each cluster's wait inside -// it is bounded by its own node autoscaler. -func (p *Provider) longestProvision() time.Duration { - var d time.Duration - for _, cl := range p.clusters { - d = max(d, cl.nodeAutoscaler.provisionTimeout()) - } - return d -} - // Offerings implements provider.Provider. There is no price table: see the package doc. func (p *Provider) Offerings(context.Context) ([]provider.Offering, error) { return nil, nil } @@ -241,27 +228,23 @@ func (p *Provider) Provision( return provider.ProvisionResult{}, err } } - bound, err := waitForProvision(ctx, cl.clients, cl.nodeAutoscaler, real.Namespace, real.Name) - if err != nil { - cleanup(ctx, cl.clients, real.Namespace, real.Name) - return provider.ProvisionResult{}, provider.WithTier(req.CapacityType, err) - } - // Unbound is accepted, not reserved: the node may still be booting. List settles it, - // reporting no capacity once the pod is unbound past its node autoscaler's bindTimeout. + // Unbound is accepted, not reserved: a node may still be coming. List settles it (see + // toInstance), so no worker waits on Karpenter here. return provider.ProvisionResult{ InstanceID: instanceID(req.Region, real.Namespace, real.Name), - Reserved: bound, + Reserved: created.Spec.NodeName != "", }, nil } -// cleanup deletes a pod that got no capacity, on a context that outlives the expired one. +// cleanup deletes a pod Provision could not finish setting up, on a context that outlives the +// caller's. // A failure is only logged: the Pod fails regardless, and the NodeClaim teardown finds the // leftover through FindByClaim. func cleanup(ctx context.Context, cc clusterClients, namespace, name string) { - ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), provisionSlack) + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) defer cancel() if err := deleteInstance(ctx, cc, namespace, name); err != nil { - logf.FromContext(ctx).Error(err, "kubernetes: could not delete pod without capacity; teardown will retry", + logf.FromContext(ctx).Error(err, "kubernetes: could not delete a half-set-up pod; teardown will retry", "namespace", namespace, "pod", name) } } @@ -328,26 +311,48 @@ func (p *Provider) Get(ctx context.Context, instanceID, _ string) (*provider.Ins if err != nil { return nil, fmt.Errorf("kubernetes: get pod %s: %w", instanceID, classifyAPI(err)) } - inst := p.toInstance(cl, pod) + inst := p.toInstance(cl, pod, false) return &inst, nil } // List implements provider.Provider. One unreadable cluster fails the whole call: leaving // its pods out would make the poll loop read every one of them as terminated. +// +// Note: list is expensive for kubernetes compared to other providers, especially with many pods. func (p *Provider) List(ctx context.Context) ([]provider.Instance, error) { var out []provider.Instance for _, region := range p.order { - pods, err := p.listPods(ctx, p.clusters[region], "") + cl := p.clusters[region] + // TODO: this may leads to performance issues if there are many pods + // in the cluster, informer-based listing should be considered for better performance. + pods, err := p.listPods(ctx, cl, "") if err != nil { return nil, err } + refused := p.refusals(ctx, cl, pods) for i := range pods { - out = append(out, p.toInstance(p.clusters[region], &pods[i])) + out = append(out, p.toInstance(cl, &pods[i], refused[pods[i].UID])) } } return out, nil } +// refusals reads cl's node autoscaler refusals, but only while some of pods is still waiting +// for a node: nothing else needs them. A failed read is no verdict; the bind timeout remains. +func (p *Provider) refusals(ctx context.Context, cl cluster, pods []corev1.Pod) map[types.UID]bool { + if !slices.ContainsFunc(pods, func(pod corev1.Pod) bool { return pod.Spec.NodeName == "" }) { + return nil + } + // TODO: we also have performance issue here if we have many pods. + // Consider using informer-based caching or batching refusals requests for better performance. + refused, err := cl.nodeAutoscaler.refusals(ctx, cl.clients) + if err != nil { + logf.FromContext(ctx).Info("kubernetes: could not read refusals; relying on the bind timeout", + "cluster", cl.cfg.Name, "error", err.Error()) + } + return refused +} + // FindByClaim implements provider.Provider. region names the claim's cluster; an unknown one // is searched for everywhere rather than reported absent, since absent means "nothing to tear down". func (p *Provider) FindByClaim(ctx context.Context, claimName, region string) (*provider.Instance, error) { @@ -363,7 +368,7 @@ func (p *Provider) FindByClaim(ctx context.Context, claimName, region string) (* for i := range pods { // Checked again here: the selector is the server's job, and not every client honours it. if pods[i].Name == claimName { - inst := p.toInstance(p.clusters[r], &pods[i]) + inst := p.toInstance(p.clusters[r], &pods[i], false) return &inst, nil } } @@ -546,11 +551,10 @@ func deleteInstance(ctx context.Context, cc clusterClients, namespace, name stri // toInstance normalizes a real pod. Region is the cluster's AWS region, the token the claim // carries. // -// A pod still unbound past its node autoscaler's bindTimeout reports InstanceUnschedulable: it is -// the bind wait an unreserved Provision handed over, timed from the pod's creation so a -// manager restart does not reset it. No events are read here; the fast refusal is caught in -// Provision (see waitForProvision). -func (p *Provider) toInstance(cl cluster, pod *corev1.Pod) provider.Instance { +// An unbound pod reports InstanceUnschedulable when its node autoscaler refused it, or once it +// stays unbound past bindTimeout, timed from the pod's creation so a manager restart does not +// reset it. That is the bind wait an unreserved Provision handed over. +func (p *Provider) toInstance(cl cluster, pod *corev1.Pod, refused bool) provider.Instance { inst := provider.Instance{ ID: instanceID(cl.cfg.Region, pod.Namespace, pod.Name), ClaimName: pod.Name, @@ -560,7 +564,7 @@ func (p *Provider) toInstance(cl cluster, pod *corev1.Pod) provider.Instance { } timeout := cl.nodeAutoscaler.bindTimeout() if pod.Spec.NodeName == "" && inst.State == provider.InstancePending && - p.now().Sub(pod.CreationTimestamp.Time) > timeout { + (refused || p.now().Sub(pod.CreationTimestamp.Time) > timeout) { inst.State = provider.InstanceUnschedulable } return inst diff --git a/pkg/provider/kubernetes/node_autoscaler.go b/pkg/provider/kubernetes/node_autoscaler.go index 6c01021..7b41ae7 100644 --- a/pkg/provider/kubernetes/node_autoscaler.go +++ b/pkg/provider/kubernetes/node_autoscaler.go @@ -21,6 +21,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" ) @@ -35,14 +36,10 @@ type nodeAutoscaler interface { // selection steers the pod onto nodes with the GPU families and tier. No families is a // CPU-only pod; an empty tier leaves it to the cluster. selection(families []string, tier nebulav1alpha1.CapacityType) nodeSelection - // refused looks once at a pending, unbound pod for the node autoscaler's verdict that it - // will not get it a node, returning an error wrapping provider.ErrNoCapacity, or nil. It - // only speeds up failure: success is the pod binding, and silence ends at bindTimeout. - refused(ctx context.Context, cc clusterClients, pod *corev1.Pod) error - // provisionTimeout is how long Provision watches a new pod for a binding or refused: long - // enough for the node autoscaler to decide, short enough not to hold a worker through - // node boot. Capabilities.ProvisionTimeout adds provisionSlack for the create calls. - provisionTimeout() time.Duration + // refusals returns the pods in the cluster the node autoscaler says it will not get a node + // for, by UID, in one read for all of them. It only speeds up failure: success is the pod + // binding, silence ends at bindTimeout, and an error means no answer this time. + refusals(ctx context.Context, cc clusterClients) (map[types.UID]bool, error) // bindTimeout is how long a healthy launch may take to bind the pod, node boot included, // counted from the pod's creation. Past it, List reports no capacity. bindTimeout() time.Duration diff --git a/pkg/provider/kubernetes/provision_test.go b/pkg/provider/kubernetes/provision_test.go index 5db5e0b..3329141 100644 --- a/pkg/provider/kubernetes/provision_test.go +++ b/pkg/provider/kubernetes/provision_test.go @@ -18,12 +18,10 @@ package kubernetes import ( "context" - "errors" "testing" "time" corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -47,6 +45,7 @@ func newTestCluster(t *testing.T, bind bool) testCluster { kube.PrependReactor("create", "pods", func(a k8stesting.Action) (bool, runtime.Object, error) { pod := a.(k8stesting.CreateAction).GetObject().(*corev1.Pod) pod.UID = testUID + pod.CreationTimestamp = metav1.Now() // as the API server does; the fake leaves it zero if bind { pod.Spec.NodeName = "node-1" } @@ -112,45 +111,70 @@ func TestProvisionBoundToWarmNode(t *testing.T) { } } -func TestProvisionNoCapacityDeletesPod(t *testing.T) { +// Provision never waits for Karpenter: it accepts the pod unreserved, and List reports a +// refusal on the next poll, whenever it arrives. +func TestListReportsKarpenterRefusal(t *testing.T) { for _, reason := range []string{reasonInsufficientCapacity, reasonFailedScheduling} { t.Run(reason, func(t *testing.T) { tc := newTestCluster(t, false) + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + + res, err := p.Provision(context.Background(), placeholder(), provisionRequest()) + if err != nil || res.Reserved { + t.Fatalf("Provision = %+v, %v; want accepted, unreserved", res, err) + } + if got := listState(t, p); got != provider.InstancePending { + t.Fatalf("before any refusal: %s, want Pending", got) + } if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), karpenterEvent(reason, "no p5 capacity"), metav1.CreateOptions{}); err != nil { t.Fatal(err) } - p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) - - _, err := p.Provision(context.Background(), placeholder(), provisionRequest()) - if !errors.Is(err, provider.ErrNoCapacity) { - t.Fatalf("err = %v, want ErrNoCapacity", err) + if got := listState(t, p); got != provider.InstanceUnschedulable { + t.Fatalf("after Karpenter's %s: %s, want Unschedulable", reason, got) } - assertGone(t, tc) }) } } -func TestProvisionIgnoresSchedulerFailedScheduling(t *testing.T) { +// listState is the one instance List reports. +func listState(t *testing.T, p *Provider) provider.InstanceState { + t.Helper() + list, err := p.List(context.Background()) + if err != nil || len(list) != 1 { + t.Fatalf("List = %+v, %v; want one instance", list, err) + } + return list[0].State +} + +// The kube-scheduler's FailedScheduling, which every pod waiting for a new node gets, is no +// refusal; nor is a Karpenter refusal about a pod that has since been bound. +func TestListIgnoresWhatIsNotARefusal(t *testing.T) { tc := newTestCluster(t, false) + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + if _, err := p.Provision(context.Background(), placeholder(), provisionRequest()); err != nil { + t.Fatal(err) + } ev := karpenterEvent(reasonFailedScheduling, "0/3 nodes are available") ev.Source.Component = "default-scheduler" if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), ev, metav1.CreateOptions{}); err != nil { t.Fatal(err) } - p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + if got := listState(t, p); got != provider.InstancePending { + t.Fatalf("with the scheduler's event: %s, want Pending", got) + } - // The scheduler's event is no verdict, so the window closes without one: the pod is - // accepted unreserved and kept for the poll loop to settle. - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - res, err := p.Provision(ctx, placeholder(), provisionRequest()) - if err != nil || res.Reserved || res.InstanceID == "" { - t.Fatalf("Provision = %+v, %v; want accepted, unreserved", res, err) + if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), + karpenterEvent(reasonInsufficientCapacity, "earlier attempt"), metav1.CreateOptions{}); err != nil { + t.Fatal(err) } - _, err = tc.kube.CoreV1().Pods("org-1").Get(context.Background(), "org-1-work-1", metav1.GetOptions{}) - if err != nil { - t.Fatalf("pod was deleted: %v", err) + pod, _ := tc.kube.CoreV1().Pods("org-1").Get(context.Background(), "org-1-work-1", metav1.GetOptions{}) + pod.Spec.NodeName = "node-1" + if _, err := tc.kube.CoreV1().Pods("org-1").Update(context.Background(), pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if got := listState(t, p); got == provider.InstanceUnschedulable { + t.Fatal("a bound pod must not be reported Unschedulable for an earlier refusal") } } @@ -226,25 +250,6 @@ func TestProvisionCreatesNamespaceInRemoteCluster(t *testing.T) { } } -func assertGone(t *testing.T, tc testCluster) { - t.Helper() - ctx := context.Background() - _, err := tc.kube.CoreV1().Pods("org-1").Get(ctx, "org-1-work-1", metav1.GetOptions{}) - if !apierrors.IsNotFound(err) { - t.Errorf("real pod not deleted: %v", err) - } - // The fake clientset has no garbage collector, so what is checked is that a real one would - // take the env Secret with the pod: it is owned by it, or was never written. - sec, err := tc.kube.CoreV1().Secrets("org-1").Get(ctx, "org-1-work-1-env", metav1.GetOptions{}) - switch { - case apierrors.IsNotFound(err): - case err != nil: - t.Errorf("get env secret: %v", err) - case len(sec.OwnerReferences) != 1 || sec.OwnerReferences[0].UID != testUID: - t.Errorf("env secret is not owned by the real pod: %+v", sec.OwnerReferences) - } -} - func TestKarpenterVerify(t *testing.T) { tc := newTestCluster(t, false) if err := (karpenter{}).verify(context.Background(), tc.clients()); err == nil { @@ -259,27 +264,13 @@ func TestKarpenterVerify(t *testing.T) { } } -// A new node: the pod binds a while after it was created. -func TestProvisionWaitsForBinding(t *testing.T) { +// Provision returns without waiting for a node: reserved only if the pod is already bound. +func TestProvisionDoesNotWaitForANode(t *testing.T) { tc := newTestCluster(t, false) - go func() { - ctx := context.Background() - for { - pod, err := tc.kube.CoreV1().Pods("org-1").Get(ctx, "org-1-work-1", metav1.GetOptions{}) - if err == nil { - pod.Spec.NodeName = "node-1" - if _, err := tc.kube.CoreV1().Pods("org-1").Update(ctx, pod, metav1.UpdateOptions{}); err == nil { - return - } - } - time.Sleep(50 * time.Millisecond) - } - }() p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) - res, err := p.Provision(context.Background(), placeholder(), provisionRequest()) - if err != nil || !res.Reserved { - t.Fatalf("Provision = %+v, %v", res, err) + if err != nil || res.Reserved || res.InstanceID != "us-east-1/org-1/org-1-work-1" { + t.Fatalf("Provision = %+v, %v; want accepted, unreserved, at once", res, err) } } diff --git a/pkg/provider/kubernetes/wait.go b/pkg/provider/kubernetes/wait.go deleted file mode 100644 index 87edae3..0000000 --- a/pkg/provider/kubernetes/wait.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2026 The InftyAI Team. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubernetes - -import ( - "context" - "fmt" - "time" - - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/InftyAI/Nebula/pkg/util" -) - -// waitInterval is how often a pending Provision re-reads its pod. -const waitInterval = 2 * time.Second - -// waitForProvision watches a new pod for provisionTimeout: long enough to catch a warm node -// binding it or the node autoscaler refusing it, without holding a worker through node boot. -// Neither within the window is no answer yet (false, nil); the poll loop takes over from there -// (see toInstance). Image pull and startup after binding are the readiness deadline's. -func waitForProvision(ctx context.Context, cc clusterClients, as nodeAutoscaler, namespace, name string) (bool, error) { - ctx, cancel := context.WithTimeout(ctx, as.provisionTimeout()) - defer cancel() - t := time.NewTicker(waitInterval) - defer t.Stop() - for { - bound, err := checkBinding(ctx, cc, as, namespace, name) - if bound || err != nil { - return bound, err - } - select { - case <-ctx.Done(): - return false, nil - case <-t.C: - } - } -} - -// checkBinding is one pass of waitForProvision. A failed read is not a verdict, so it is -// swallowed and the next tick retries. -func checkBinding(ctx context.Context, cc clusterClients, as nodeAutoscaler, namespace, name string) (bool, error) { - pod, err := cc.kube.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - return false, fmt.Errorf("kubernetes: pod %s/%s disappeared before it was bound", namespace, name) - } - if err != nil { - return false, nil - } - // With reserved capacity, the pod will have a NodeName assigned once it is scheduled. - if pod.Spec.NodeName != "" { - return true, nil - } - if util.Finished(pod.Status.Phase) { - return false, fmt.Errorf("kubernetes: pod %s/%s ended %s before it was bound: %s", - namespace, name, pod.Status.Phase, pod.Status.Message) - } - return false, as.refused(ctx, cc, pod) -} From 719e95ec79131530982589d6a91384711935c990 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Fri, 23 Oct 2026 18:05:18 +0100 Subject: [PATCH 09/21] fix aws permission issue Signed-off-by: kerthcet --- pkg/provider/kubernetes/eks.go | 10 +++- pkg/provider/kubernetes/eks_test.go | 74 +++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 pkg/provider/kubernetes/eks_test.go diff --git a/pkg/provider/kubernetes/eks.go b/pkg/provider/kubernetes/eks.go index 5aa792d..525b83e 100644 --- a/pkg/provider/kubernetes/eks.go +++ b/pkg/provider/kubernetes/eks.go @@ -40,6 +40,10 @@ const ( // a presigned STS GetCallerIdentity URL, bound to one cluster by the signed header. eksTokenPrefix = "k8s-aws-v1." clusterIDHeader = "x-k8s-aws-id" + // expiresHeader carries presignExpiry, the value `aws eks get-token` sends. EKS bounds a + // token's life itself (see tokenTTL), not by this. + expiresHeader = "X-Amz-Expires" + presignExpiry = "60" // tokenTTL is how long a minted token is reused. EKS accepts one for 15 minutes; the // margin covers clock skew and a request in flight when it is swapped. tokenTTL = 10 * time.Minute @@ -122,10 +126,14 @@ func (s *eksTokenSource) Token(ctx context.Context) (string, error) { if s.token != "" && time.Now().Before(s.expires) { return s.token, nil } + // The STS presign client sets no expiry, and EKS rejects a token without one as + // Unauthorized. As a header it is hoisted into the query like every X-Amz-* one. req, err := s.presign.PresignGetCallerIdentity(ctx, &sts.GetCallerIdentityInput{}, func(o *sts.PresignOptions) { o.ClientOptions = append(o.ClientOptions, func(so *sts.Options) { - so.APIOptions = append(so.APIOptions, smithyhttp.AddHeaderValue(clusterIDHeader, s.cluster)) + so.APIOptions = append(so.APIOptions, + smithyhttp.SetHeaderValue(clusterIDHeader, s.cluster), + smithyhttp.SetHeaderValue(expiresHeader, presignExpiry)) }) }) if err != nil { diff --git a/pkg/provider/kubernetes/eks_test.go b/pkg/provider/kubernetes/eks_test.go new file mode 100644 index 0000000..44b1233 --- /dev/null +++ b/pkg/provider/kubernetes/eks_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + "encoding/base64" + "net/url" + "slices" + "strings" + "testing" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/sts" +) + +// The token is the shape `aws eks get-token` emits. Each piece is one EKS checks: a missing +// X-Amz-Expires, or a cluster ID that is not signed, is rejected as Unauthorized with no +// further detail. +func TestEKSTokenSource_Format(t *testing.T) { + cfg := awssdk.Config{ + Region: "eu-north-1", + Credentials: credentials.NewStaticCredentialsProvider("AKIDEXAMPLE", "secret", "session"), + } + s := &eksTokenSource{presign: sts.NewPresignClient(sts.NewFromConfig(cfg)), cluster: "my-cluster"} + + tok, err := s.Token(context.Background()) + if err != nil { + t.Fatalf("Token: %v", err) + } + encoded, ok := strings.CutPrefix(tok, eksTokenPrefix) + if !ok { + t.Fatalf("token %q lacks the %q prefix", tok, eksTokenPrefix) + } + raw, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + t.Fatalf("token is not unpadded base64url: %v", err) + } + u, err := url.Parse(string(raw)) + if err != nil { + t.Fatalf("token is not a URL: %v", err) + } + q := u.Query() + + if u.Host != "sts.eu-north-1.amazonaws.com" || q.Get("Action") != "GetCallerIdentity" { + t.Errorf("presigned %s?Action=%s, want sts.eu-north-1.amazonaws.com?Action=GetCallerIdentity", + u.Host, q.Get("Action")) + } + if got := q.Get("X-Amz-Expires"); got != presignExpiry { + t.Errorf("X-Amz-Expires = %q, want %q", got, presignExpiry) + } + if signed := strings.Split(q.Get("X-Amz-SignedHeaders"), ";"); !slices.Contains(signed, clusterIDHeader) { + t.Errorf("signed headers %v lack %s", signed, clusterIDHeader) + } + + if again, _ := s.Token(context.Background()); again != tok { + t.Error("a second call within tokenTTL minted a new token") + } +} From 1b2b709730dd88841dc51fcc9abf2e46461ad242 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Fri, 23 Oct 2026 18:35:49 +0100 Subject: [PATCH 10/21] unschedulable event should be marked as capacity error Signed-off-by: kerthcet --- pkg/provider/kubernetes/karpenter.go | 22 +++++++-- pkg/provider/kubernetes/kubernetes.go | 24 ++++++---- pkg/provider/kubernetes/node_autoscaler.go | 6 ++- pkg/provider/kubernetes/provision_test.go | 55 +++++++++++++++++++--- pkg/vnode/handler_test.go | 26 ++++++++++ 5 files changed, 110 insertions(+), 23 deletions(-) diff --git a/pkg/provider/kubernetes/karpenter.go b/pkg/provider/kubernetes/karpenter.go index b90d5db..9bb45ef 100644 --- a/pkg/provider/kubernetes/karpenter.go +++ b/pkg/provider/kubernetes/karpenter.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/types" nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" ) // Karpenter v1 (sigs.k8s.io/karpenter): its well-known labels, the annotation that keeps @@ -104,7 +105,10 @@ func karpenterCapacityType(tier nebulav1alpha1.CapacityType) string { // recording why. The selector leaves out the kube-scheduler's FailedScheduling, which every // pod needing a new node gets. Karpenter rate-limits these events, so a dropped one only costs // the bind timeout. -func (karpenter) refusals(ctx context.Context, cc clusterClients) (map[types.UID]bool, error) { +// +// Only InsufficientCapacityError is capacity. FailedScheduling is usually the pod's own spec +// (a missing toleration, a selector no NodePool allows), so it fails the pod without blocking. +func (karpenter) refusals(ctx context.Context, cc clusterClients) (map[types.UID]provider.InstanceState, error) { evs, err := cc.kube.CoreV1().Events(metav1.NamespaceAll).List(ctx, metav1.ListOptions{ FieldSelector: fields.Set{ "involvedObject.kind": "Pod", "source": karpenterComponent, "type": corev1.EventTypeWarning, @@ -114,11 +118,21 @@ func (karpenter) refusals(ctx context.Context, cc clusterClients) (map[types.UID if err != nil { return nil, err } - out := map[types.UID]bool{} + out := map[types.UID]provider.InstanceState{} for _, ev := range evs.Items { // Checked again here: the selector is the server's job, and not every client honours it. - if fromKarpenter(ev) && (ev.Reason == reasonInsufficientCapacity || ev.Reason == reasonFailedScheduling) { - out[ev.InvolvedObject.UID] = true + if !fromKarpenter(ev) { + continue + } + uid := ev.InvolvedObject.UID + switch ev.Reason { + case reasonInsufficientCapacity: + out[uid] = provider.InstanceUnschedulable + case reasonFailedScheduling: + // A launch that reached EC2 had passed scheduling, so capacity wins. + if out[uid] != provider.InstanceUnschedulable { + out[uid] = provider.InstanceFailed + } } } return out, nil diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index a60092c..a8be6fd 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -311,7 +311,7 @@ func (p *Provider) Get(ctx context.Context, instanceID, _ string) (*provider.Ins if err != nil { return nil, fmt.Errorf("kubernetes: get pod %s: %w", instanceID, classifyAPI(err)) } - inst := p.toInstance(cl, pod, false) + inst := p.toInstance(cl, pod, "") return &inst, nil } @@ -339,7 +339,7 @@ func (p *Provider) List(ctx context.Context) ([]provider.Instance, error) { // refusals reads cl's node autoscaler refusals, but only while some of pods is still waiting // for a node: nothing else needs them. A failed read is no verdict; the bind timeout remains. -func (p *Provider) refusals(ctx context.Context, cl cluster, pods []corev1.Pod) map[types.UID]bool { +func (p *Provider) refusals(ctx context.Context, cl cluster, pods []corev1.Pod) map[types.UID]provider.InstanceState { if !slices.ContainsFunc(pods, func(pod corev1.Pod) bool { return pod.Spec.NodeName == "" }) { return nil } @@ -368,7 +368,7 @@ func (p *Provider) FindByClaim(ctx context.Context, claimName, region string) (* for i := range pods { // Checked again here: the selector is the server's job, and not every client honours it. if pods[i].Name == claimName { - inst := p.toInstance(p.clusters[r], &pods[i], false) + inst := p.toInstance(p.clusters[r], &pods[i], "") return &inst, nil } } @@ -551,10 +551,10 @@ func deleteInstance(ctx context.Context, cc clusterClients, namespace, name stri // toInstance normalizes a real pod. Region is the cluster's AWS region, the token the claim // carries. // -// An unbound pod reports InstanceUnschedulable when its node autoscaler refused it, or once it -// stays unbound past bindTimeout, timed from the pod's creation so a manager restart does not -// reset it. That is the bind wait an unreserved Provision handed over. -func (p *Provider) toInstance(cl cluster, pod *corev1.Pod, refused bool) provider.Instance { +// A pod still waiting for a node reports refusal ("" for none) if set, else +// InstanceUnschedulable once it outlasts bindTimeout, timed from creation so a restart does +// not reset it. The refusal wins so a non-capacity reason never becomes a block by waiting. +func (p *Provider) toInstance(cl cluster, pod *corev1.Pod, refusal provider.InstanceState) provider.Instance { inst := provider.Instance{ ID: instanceID(cl.cfg.Region, pod.Namespace, pod.Name), ClaimName: pod.Name, @@ -562,9 +562,13 @@ func (p *Provider) toInstance(cl cluster, pod *corev1.Pod, refused bool) provide Region: cl.cfg.Region, Endpoint: pod.Status.PodIP, } - timeout := cl.nodeAutoscaler.bindTimeout() - if pod.Spec.NodeName == "" && inst.State == provider.InstancePending && - (refused || p.now().Sub(pod.CreationTimestamp.Time) > timeout) { + if pod.Spec.NodeName != "" || inst.State != provider.InstancePending { + return inst + } + switch { + case refusal != "": + inst.State = refusal + case p.now().Sub(pod.CreationTimestamp.Time) > cl.nodeAutoscaler.bindTimeout(): inst.State = provider.InstanceUnschedulable } return inst diff --git a/pkg/provider/kubernetes/node_autoscaler.go b/pkg/provider/kubernetes/node_autoscaler.go index 7b41ae7..0d310f7 100644 --- a/pkg/provider/kubernetes/node_autoscaler.go +++ b/pkg/provider/kubernetes/node_autoscaler.go @@ -24,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/types" nebulav1alpha1 "github.com/InftyAI/Nebula/api/v1alpha1" + "github.com/InftyAI/Nebula/pkg/provider" ) // NodeAutoscalerKarpenter is the default, and today the only, node autoscaler. @@ -37,9 +38,10 @@ type nodeAutoscaler interface { // CPU-only pod; an empty tier leaves it to the cluster. selection(families []string, tier nebulav1alpha1.CapacityType) nodeSelection // refusals returns the pods in the cluster the node autoscaler says it will not get a node - // for, by UID, in one read for all of them. It only speeds up failure: success is the pod + // for, by UID, in one read for all of them, with the state each should report; only + // InstanceUnschedulable blocks the candidate. It only speeds up failure: success is the pod // binding, silence ends at bindTimeout, and an error means no answer this time. - refusals(ctx context.Context, cc clusterClients) (map[types.UID]bool, error) + refusals(ctx context.Context, cc clusterClients) (map[types.UID]provider.InstanceState, error) // bindTimeout is how long a healthy launch may take to bind the pod, node boot included, // counted from the pod's creation. Past it, List reports no capacity. bindTimeout() time.Duration diff --git a/pkg/provider/kubernetes/provision_test.go b/pkg/provider/kubernetes/provision_test.go index 3329141..f04176c 100644 --- a/pkg/provider/kubernetes/provision_test.go +++ b/pkg/provider/kubernetes/provision_test.go @@ -112,9 +112,13 @@ func TestProvisionBoundToWarmNode(t *testing.T) { } // Provision never waits for Karpenter: it accepts the pod unreserved, and List reports a -// refusal on the next poll, whenever it arrives. +// refusal on the next poll, whenever it arrives. Only EC2's InsufficientCapacityError is +// capacity; Karpenter's FailedScheduling is the pod's spec, so it fails without blocking. func TestListReportsKarpenterRefusal(t *testing.T) { - for _, reason := range []string{reasonInsufficientCapacity, reasonFailedScheduling} { + for reason, want := range map[string]provider.InstanceState{ + reasonInsufficientCapacity: provider.InstanceUnschedulable, + reasonFailedScheduling: provider.InstanceFailed, + } { t.Run(reason, func(t *testing.T) { tc := newTestCluster(t, false) p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) @@ -127,16 +131,53 @@ func TestListReportsKarpenterRefusal(t *testing.T) { t.Fatalf("before any refusal: %s, want Pending", got) } if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), - karpenterEvent(reason, "no p5 capacity"), metav1.CreateOptions{}); err != nil { + karpenterEvent(reason, "refused"), metav1.CreateOptions{}); err != nil { t.Fatal(err) } - if got := listState(t, p); got != provider.InstanceUnschedulable { - t.Fatalf("after Karpenter's %s: %s, want Unschedulable", reason, got) + if got := listState(t, p); got != want { + t.Fatalf("after Karpenter's %s: %s, want %s", reason, got, want) } }) } } +// A launch that reached EC2 had passed scheduling, so an ICE outranks a FailedScheduling on +// the same pod. +func TestListCapacityRefusalWins(t *testing.T) { + tc := newTestCluster(t, false) + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + if _, err := p.Provision(context.Background(), placeholder(), provisionRequest()); err != nil { + t.Fatal(err) + } + for _, reason := range []string{reasonInsufficientCapacity, reasonFailedScheduling} { + if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), + karpenterEvent(reason, "refused"), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + if got := listState(t, p); got != provider.InstanceUnschedulable { + t.Fatalf("with both refusals: %s, want Unschedulable", got) + } +} + +// A FailedScheduling outranks the bind timeout: a pod Karpenter has explained must not turn +// into a block just by outlasting the wait. +func TestListSpecRefusalOutranksBindTimeout(t *testing.T) { + tc := newTestCluster(t, false) + p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + if _, err := p.Provision(context.Background(), placeholder(), provisionRequest()); err != nil { + t.Fatal(err) + } + if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), + karpenterEvent(reasonFailedScheduling, "did not tolerate taint"), metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + p.now = func() time.Time { return time.Now().Add(karpenterBindTimeout + time.Second) } + if got := listState(t, p); got != provider.InstanceFailed { + t.Fatalf("past the bind timeout after a FailedScheduling: %s, want Failed", got) + } +} + // listState is the one instance List reports. func listState(t *testing.T, p *Provider) provider.InstanceState { t.Helper() @@ -173,8 +214,8 @@ func TestListIgnoresWhatIsNotARefusal(t *testing.T) { if _, err := tc.kube.CoreV1().Pods("org-1").Update(context.Background(), pod, metav1.UpdateOptions{}); err != nil { t.Fatal(err) } - if got := listState(t, p); got == provider.InstanceUnschedulable { - t.Fatal("a bound pod must not be reported Unschedulable for an earlier refusal") + if got := listState(t, p); got != provider.InstancePending { + t.Fatalf("a bound pod must not be reported refused for an earlier refusal, got %s", got) } } diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go index f0b42cb..0dd3563 100644 --- a/pkg/vnode/handler_test.go +++ b/pkg/vnode/handler_test.go @@ -2024,6 +2024,32 @@ func TestReconcileOnce_UnschedulableBlocksAndFailsPod(t *testing.T) { } } +// A Failed instance fails the Pod without blocking the candidate: that is how a provider +// reports a refusal that is not capacity (Karpenter's FailedScheduling, a missing toleration), +// which must not exclude the candidate for every other Pod. +func TestReconcileOnce_FailedFailsPodWithoutBlock(t *testing.T) { + fp := &fakeProvider{provisionID: "inst-1", classifyScope: provider.BlockScope{Region: ptrTo("eks-use1")}} + bl := &recordingBlocklist{} + h := NewHandler(fp, nil, bl, clusterWithTTL(30*time.Second)) + h.jitterFn = func() time.Duration { return 0 } + if err := h.CreatePod(context.Background(), testPod("default", "p1")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + + fp.list = []provider.Instance{{ + ID: "inst-1", ClaimName: "default-p1", Region: "eks-use1", State: provider.InstanceFailed, + }} + h.reconcileOnce(context.Background()) + + got, _ := h.GetPod(context.Background(), "default", "p1") + if got.Status.Phase != corev1.PodFailed || got.Status.Reason != reasonFailed { + t.Fatalf("status = %s/%s, want Failed/%s", got.Status.Phase, got.Status.Reason, reasonFailed) + } + if bl.calls != 0 { + t.Fatalf("Record calls = %d, want none", bl.calls) + } +} + func ptrTo[T any](v T) *T { return &v } // NotReady shows as it is, Running with Ready=False, and counts toward the readiness deadline From f88298f878cfca33a0be390043b865c50f15928d Mon Sep 17 00:00:00 2001 From: kerthcet Date: Fri, 23 Oct 2026 19:19:36 +0100 Subject: [PATCH 11/21] introduce provisioning Signed-off-by: kerthcet --- pkg/provider/kubernetes/kubernetes.go | 16 +++++++++------ pkg/provider/kubernetes/kubernetes_test.go | 5 ++++- pkg/provider/kubernetes/provision_test.go | 17 +++++++++------ pkg/provider/provider.go | 3 +++ pkg/vnode/handler.go | 3 ++- pkg/vnode/handler_test.go | 24 ++++++++++++++++++++++ pkg/vnode/status.go | 6 ++++++ 7 files changed, 60 insertions(+), 14 deletions(-) diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index a8be6fd..122ae2c 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -551,9 +551,9 @@ func deleteInstance(ctx context.Context, cc clusterClients, namespace, name stri // toInstance normalizes a real pod. Region is the cluster's AWS region, the token the claim // carries. // -// A pod still waiting for a node reports refusal ("" for none) if set, else -// InstanceUnschedulable once it outlasts bindTimeout, timed from creation so a restart does -// not reset it. The refusal wins so a non-capacity reason never becomes a block by waiting. +// A pod still provisioning reports refusal ("" for none) if set, else InstanceUnschedulable +// once it outlasts bindTimeout, timed from creation so a restart does not reset it. The +// refusal wins so a non-capacity reason never becomes a block by waiting. func (p *Provider) toInstance(cl cluster, pod *corev1.Pod, refusal provider.InstanceState) provider.Instance { inst := provider.Instance{ ID: instanceID(cl.cfg.Region, pod.Namespace, pod.Name), @@ -562,7 +562,7 @@ func (p *Provider) toInstance(cl cluster, pod *corev1.Pod, refusal provider.Inst Region: cl.cfg.Region, Endpoint: pod.Status.PodIP, } - if pod.Spec.NodeName != "" || inst.State != provider.InstancePending { + if inst.State != provider.InstanceProvisioning { return inst } switch { @@ -574,8 +574,9 @@ func (p *Provider) toInstance(cl cluster, pod *corev1.Pod, refusal provider.Inst return inst } -// toState maps the real pod's phase. Unknown stays Pending, so a node that stopped reporting -// keeps the poll loop watching rather than going terminal on a pod that may still run. +// toState maps the real pod's phase. Pending splits on binding: unbound has no node behind it +// yet. Unknown stays Pending, so a node that stopped reporting keeps the poll loop watching +// rather than going terminal on a pod that may still run. func toState(pod *corev1.Pod) provider.InstanceState { switch pod.Status.Phase { case corev1.PodRunning: @@ -589,6 +590,9 @@ func toState(pod *corev1.Pod) provider.InstanceState { case corev1.PodFailed: return provider.InstanceFailed default: + if pod.Spec.NodeName == "" { + return provider.InstanceProvisioning + } return provider.InstancePending } } diff --git a/pkg/provider/kubernetes/kubernetes_test.go b/pkg/provider/kubernetes/kubernetes_test.go index 980c1a6..410e62e 100644 --- a/pkg/provider/kubernetes/kubernetes_test.go +++ b/pkg/provider/kubernetes/kubernetes_test.go @@ -421,7 +421,10 @@ func TestToStateNeedsReady(t *testing.T) { "ready": {running(corev1.ConditionTrue), provider.InstanceRunning}, "probe failing": {running(corev1.ConditionFalse), provider.InstanceNotReady}, "no condition yet": {running(""), provider.InstanceNotReady}, - "pending": {&corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending}}, provider.InstancePending}, + "unbound": {&corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending}}, provider.InstanceProvisioning}, + "bound, starting": {&corev1.Pod{ + Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{Phase: corev1.PodPending}, + }, provider.InstancePending}, } { if got := toState(tc.pod); got != tc.want { t.Errorf("%s: toState = %s, want %s", name, got, tc.want) diff --git a/pkg/provider/kubernetes/provision_test.go b/pkg/provider/kubernetes/provision_test.go index f04176c..1de2cfc 100644 --- a/pkg/provider/kubernetes/provision_test.go +++ b/pkg/provider/kubernetes/provision_test.go @@ -127,8 +127,8 @@ func TestListReportsKarpenterRefusal(t *testing.T) { if err != nil || res.Reserved { t.Fatalf("Provision = %+v, %v; want accepted, unreserved", res, err) } - if got := listState(t, p); got != provider.InstancePending { - t.Fatalf("before any refusal: %s, want Pending", got) + if got := listState(t, p); got != provider.InstanceProvisioning { + t.Fatalf("before any refusal: %s, want Provisioning", got) } if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), karpenterEvent(reason, "refused"), metav1.CreateOptions{}); err != nil { @@ -201,8 +201,8 @@ func TestListIgnoresWhatIsNotARefusal(t *testing.T) { if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), ev, metav1.CreateOptions{}); err != nil { t.Fatal(err) } - if got := listState(t, p); got != provider.InstancePending { - t.Fatalf("with the scheduler's event: %s, want Pending", got) + if got := listState(t, p); got != provider.InstanceProvisioning { + t.Fatalf("with the scheduler's event: %s, want Provisioning", got) } if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), @@ -248,8 +248,13 @@ func TestListReportsUnschedulablePastBindTimeout(t *testing.T) { } return out } - if got := states()["org-1-work-1"].State; got != provider.InstancePending { - t.Fatalf("young unbound pod = %s, want Pending", got) + // Unbound has no node behind it yet; bound has one booting, which is Initializing's Pending. + young := states() + if got := young["org-1-work-1"].State; got != provider.InstanceProvisioning { + t.Fatalf("young unbound pod = %s, want Provisioning", got) + } + if got := young["org-1-work-2"].State; got != provider.InstancePending { + t.Fatalf("young bound pod = %s, want Pending", got) } p.now = func() time.Time { return time.Now().Add(karpenterBindTimeout + time.Second) } diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 6b4a704..6283a06 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -439,6 +439,9 @@ func RedactedEnv(m map[string]string) string { type InstanceState string const ( + // InstanceProvisioning: the instance exists (it has an id and must be terminated), but no + // capacity is allocated to it yet, e.g. a real pod Karpenter has not given a node. + InstanceProvisioning InstanceState = "Provisioning" // InstancePending: created but not yet reachable/ready. InstancePending InstanceState = "Pending" // InstanceRunning: up and reachable. diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index 385f4f1..0fd35ae 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -777,7 +777,8 @@ func (h *Handler) reconcileOnce(ctx context.Context) { // deadline, with how long the pod has been there. It counts InstancePending and // InstanceNotReady alike, so a workload that never comes up and one that stops being ready // both fail once unusable for the whole deadline; Running resets it, so a brief unready spell -// costs nothing. +// costs nothing. InstanceProvisioning is not counted: a wait for capacity is the provider's +// to bound (the Kubernetes provider's bindTimeout), and the budget is for booting. // // It anchors on the FIRST observation of a not-usable state, not on provisioningAt: the // provision call is bounded separately (see defaultProvisionTimeout), so measuring from diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go index 0dd3563..43422de 100644 --- a/pkg/vnode/handler_test.go +++ b/pkg/vnode/handler_test.go @@ -2050,6 +2050,30 @@ func TestReconcileOnce_FailedFailsPodWithoutBlock(t *testing.T) { } } +// Provisioning is an instance with no capacity yet: the Pod stays at Provisioning, so its +// claim is not Bound and nothing accrues, and the readiness clock does not start, since that +// wait is the provider's to bound. +func TestReconcileOnce_ProvisioningIsNotInitializing(t *testing.T) { + fp := &fakeProvider{provisionID: "inst-1"} + h := NewHandler(fp, nil, nil, openCluster()) + if err := h.CreatePod(context.Background(), testPod("default", "p1")); err != nil { + t.Fatalf("CreatePod: %v", err) + } + fp.list = []provider.Instance{{ID: "inst-1", ClaimName: "default-p1", State: provider.InstanceProvisioning}} + h.reconcileOnce(context.Background()) + + got, _ := h.GetPod(context.Background(), "default", "p1") + if got.Status.Phase != corev1.PodPending || got.Status.Reason != reasonProvisioning { + t.Fatalf("status = %s/%s, want Pending/%s", got.Status.Phase, got.Status.Reason, reasonProvisioning) + } + h.mu.Lock() + clock := h.tracked[key("default", "p1")].initializingAt + h.mu.Unlock() + if !clock.IsZero() { + t.Fatal("the readiness clock must not start while the instance has no capacity") + } +} + func ptrTo[T any](v T) *T { return &v } // NotReady shows as it is, Running with Ready=False, and counts toward the readiness deadline diff --git a/pkg/vnode/status.go b/pkg/vnode/status.go index 6f6b2e7..013ab7b 100644 --- a/pkg/vnode/status.go +++ b/pkg/vnode/status.go @@ -78,6 +78,12 @@ func applyState(pod *corev1.Pod, state provider.InstanceState, endpoint string, if endpoint != "" && net.ParseIP(endpoint) != nil { pod.Status.PodIP = endpoint } + case provider.InstanceProvisioning: + setPhase(pod, corev1.PodPending, reasonProvisioning, "allocating external instance", now) + setReady(pod, corev1.ConditionFalse, now) + setContainerStatuses(pod, corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{Reason: reasonProvisioning, Message: "allocating external instance"}, + }, false) case provider.InstancePending: setPhase(pod, corev1.PodPending, reasonInitializing, "external instance is initializing", now) setReady(pod, corev1.ConditionFalse, now) From 7e9f9df8960584301b92d4b93b6c85d77c32bd4b Mon Sep 17 00:00:00 2001 From: kerthcet Date: Fri, 23 Oct 2026 21:59:28 +0100 Subject: [PATCH 12/21] ignore nodeclaim event Signed-off-by: kerthcet --- pkg/provider/kubernetes/karpenter.go | 48 ++++++---- pkg/provider/kubernetes/kubernetes.go | 6 +- pkg/provider/kubernetes/provision_test.go | 111 +++++++++++++++++----- 3 files changed, 118 insertions(+), 47 deletions(-) diff --git a/pkg/provider/kubernetes/karpenter.go b/pkg/provider/kubernetes/karpenter.go index 9bb45ef..33429d7 100644 --- a/pkg/provider/kubernetes/karpenter.go +++ b/pkg/provider/kubernetes/karpenter.go @@ -44,9 +44,8 @@ const ( capacityTypeSpot = "spot" capacityTypeOnDemand = "on-demand" - reasonInsufficientCapacity = "InsufficientCapacityError" - reasonFailedScheduling = "FailedScheduling" - karpenterComponent = "karpenter" + reasonFailedScheduling = "FailedScheduling" + karpenterComponent = "karpenter" // karpenterBindTimeout covers a healthy launch up to binding: Karpenter's decision, the // EC2 launch and node boot until the GPUs are advertised. Measured p-family boots may need @@ -99,15 +98,13 @@ func karpenterCapacityType(tier nebulav1alpha1.CapacityType) string { } } -// refusals reads Karpenter's refusals for every pod in the cluster at once: its warnings about -// pods, InsufficientCapacityError (EC2 had none) and FailedScheduling (no NodePool fits). EC2 -// refusing a launch shows nowhere else, since Karpenter deletes that NodeClaim without -// recording why. The selector leaves out the kube-scheduler's FailedScheduling, which every -// pod needing a new node gets. Karpenter rate-limits these events, so a dropped one only costs -// the bind timeout. +// refusals reads Karpenter's FailedScheduling for every pod in the cluster at once: one per pod +// UID, so each attempt gets its own. The selector leaves out the kube-scheduler's, which every +// pod needing a new node gets. A lost event only costs the bind timeout. // -// Only InsufficientCapacityError is capacity. FailedScheduling is usually the pod's own spec -// (a missing toleration, a selector no NodePool allows), so it fails the pod without blocking. +// Every refusal fails the pod; only a capacity one (see capacityRefusal) blocks the candidate. +// EC2's InsufficientCapacityError is on the NodeClaim, which names no pod, so the pod's own +// message is the only per-pod signal of a shortage. func (karpenter) refusals(ctx context.Context, cc clusterClients) (map[types.UID]provider.InstanceState, error) { evs, err := cc.kube.CoreV1().Events(metav1.NamespaceAll).List(ctx, metav1.ListOptions{ FieldSelector: fields.Set{ @@ -124,20 +121,35 @@ func (karpenter) refusals(ctx context.Context, cc clusterClients) (map[types.UID if !fromKarpenter(ev) { continue } + if ev.Reason != reasonFailedScheduling { + continue + } uid := ev.InvolvedObject.UID - switch ev.Reason { - case reasonInsufficientCapacity: + switch { + case capacityRefusal(ev.Message): out[uid] = provider.InstanceUnschedulable - case reasonFailedScheduling: - // A launch that reached EC2 had passed scheduling, so capacity wins. - if out[uid] != provider.InstanceUnschedulable { - out[uid] = provider.InstanceFailed - } + case out[uid] != provider.InstanceUnschedulable: // a later spec refusal does not undo a shortage + out[uid] = provider.InstanceFailed } } return out, nil } +// capacityPhrases are the parts of Karpenter v1's FailedScheduling message that mean no capacity +// rather than the pod's own spec. A sold-out offering is filtered out as unavailable, so the +// instance-type filter reports a missing "offering"; the other two are NodePool limits. +var capacityPhrases = []string{"offering", "exceed limits", "limits have been exhausted"} + +// capacityRefusal reports whether a Karpenter refusal is about capacity. The wording is +// Karpenter's, not an API, so TestCapacityRefusal pins real messages. +// +// Known gap, kept on purpose: when every family in a NodePool is unavailable at once, +// Karpenter drops the pool before scheduling, so its clause, and the "offering", is missing. +// Such a refusal fails the pod without a block, until Karpenter's unavailability marks lapse. +func capacityRefusal(message string) bool { + return slices.ContainsFunc(capacityPhrases, func(p string) bool { return strings.Contains(message, p) }) +} + func (karpenter) verify(_ context.Context, cc clusterClients) error { res, err := cc.kube.Discovery().ServerResourcesForGroupVersion(nodeClaimGV.String()) if err == nil && slices.ContainsFunc(res.APIResources, func(r metav1.APIResource) bool { diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index 122ae2c..228f4d8 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -20,9 +20,9 @@ limitations under the License. // pod, not to a node; several pods can share a Karpenter node. // // Capacity is found by trying. Provision creates the pod and returns at once, accepted but -// unreserved; the poll loop settles it, reporting no capacity on the node autoscaler's refusal -// or once the pod stays unbound past the bind timeout (see toInstance), and the cluster is -// then blocklisted like an AWS region. +// unreserved; the poll loop settles it. The node autoscaler's refusal fails the pod, and a +// refusal for capacity, or a pod unbound past the bind timeout (see toInstance), is no +// capacity: the cluster is then blocklisted like an AWS region. // // The real pod lives in the placeholder's namespace, named after the claim like every // provider's instance, which is what lets the host cluster itself be a target. Nebula reports diff --git a/pkg/provider/kubernetes/provision_test.go b/pkg/provider/kubernetes/provision_test.go index 1de2cfc..c641cec 100644 --- a/pkg/provider/kubernetes/provision_test.go +++ b/pkg/provider/kubernetes/provision_test.go @@ -18,6 +18,7 @@ package kubernetes import ( "context" + "strings" "testing" "time" @@ -64,15 +65,52 @@ func provisionRequest() provider.ProvisionRequest { } } -func karpenterEvent(reason, msg string) *corev1.Event { +// karpenterEvent is Karpenter's FailedScheduling on the provisioned pod; name keeps several apart. +func karpenterEvent(name, msg string) *corev1.Event { return &corev1.Event{ - ObjectMeta: metav1.ObjectMeta{Name: "ev-" + reason, Namespace: "org-1"}, + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "org-1"}, InvolvedObject: corev1.ObjectReference{Kind: "Pod", Name: "org-1-work-1", Namespace: "org-1", UID: testUID}, - Reason: reason, Message: msg, + Type: corev1.EventTypeWarning, + Reason: reasonFailedScheduling, Message: msg, Source: corev1.EventSource{Component: "karpenter"}, } } +// Karpenter v1.14 FailedScheduling messages: "Failed to schedule pod, " and one reason per +// NodePool, joined by "; ". The first two were captured from a live cluster, the requirement +// dumps shortened; the capacity ones follow its InstanceTypeFilterError and limit wording. +var ( + refusalMissingToleration = refusal( + "no instance type met the scheduling requirements or had enough resources, "+ + "requirements=karpenter.sh/nodepool In [saas-on-demand], resources={\"nvidia.com/gpu\":\"1\"}", + "did not tolerate taint (taint=hiverge.ai/workload_class=customer:NoSchedule)", + "incompatible requirements, key karpenter.sh/capacity-type, "+ + "karpenter.sh/capacity-type In [on-demand] not in karpenter.sh/capacity-type In [spot]", + "did not tolerate taint (taint=aws.amazon.com/neuron:NoSchedule)", + ) + refusalTooManyGPUs = refusal( + "no instance type met the scheduling requirements or had enough resources, "+ + "requirements=karpenter.sh/nodepool In [saas-on-demand], resources={\"nvidia.com/gpu\":\"64\"}", + "did not tolerate taint (taint=aws.amazon.com/neuron:NoSchedule)", + "no instance type has enough resources, "+ + "requirements=karpenter.sh/nodepool In [gpu], resources={\"nvidia.com/gpu\":\"64\"}", + ) + refusalSoldOut = refusal( + "no instance type met the scheduling requirements or had enough resources, "+ + "requirements=karpenter.sh/nodepool In [saas-on-demand], resources={\"nvidia.com/gpu\":\"1\"}", + "no instance type which had enough resources and the required offering met the scheduling requirements, "+ + "requirements=karpenter.sh/nodepool In [gpu], resources={\"nvidia.com/gpu\":\"1\"}", + ) + refusalPoolLimit = refusal( + "all available instance types exceed limits for nodepool (NodePool=gpu)", + "did not tolerate taint (taint=aws.amazon.com/neuron:NoSchedule)", + ) +) + +func refusal(reasons ...string) string { + return "Failed to schedule pod, " + strings.Join(reasons, "; ") +} + func TestProvisionBoundToWarmNode(t *testing.T) { tc := newTestCluster(t, true) p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) @@ -112,16 +150,19 @@ func TestProvisionBoundToWarmNode(t *testing.T) { } // Provision never waits for Karpenter: it accepts the pod unreserved, and List reports a -// refusal on the next poll, whenever it arrives. Only EC2's InsufficientCapacityError is -// capacity; Karpenter's FailedScheduling is the pod's spec, so it fails without blocking. +// refusal on the next poll, whenever it arrives. Every refusal fails the pod; only one for +// capacity reports Unschedulable, which also blocks the candidate. func TestListReportsKarpenterRefusal(t *testing.T) { - for reason, want := range map[string]provider.InstanceState{ - reasonInsufficientCapacity: provider.InstanceUnschedulable, - reasonFailedScheduling: provider.InstanceFailed, + for name, tc := range map[string]struct { + msg string + want provider.InstanceState + }{ + "spec": {refusalMissingToleration, provider.InstanceFailed}, + "capacity": {refusalSoldOut, provider.InstanceUnschedulable}, } { - t.Run(reason, func(t *testing.T) { - tc := newTestCluster(t, false) - p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) + t.Run(name, func(t *testing.T) { + cl := newTestCluster(t, false) + p := newTestProvider(map[string]clusterClients{"eks-use1": cl.clients()}) res, err := p.Provision(context.Background(), placeholder(), provisionRequest()) if err != nil || res.Reserved { @@ -130,28 +171,27 @@ func TestListReportsKarpenterRefusal(t *testing.T) { if got := listState(t, p); got != provider.InstanceProvisioning { t.Fatalf("before any refusal: %s, want Provisioning", got) } - if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), - karpenterEvent(reason, "refused"), metav1.CreateOptions{}); err != nil { + if _, err := cl.kube.CoreV1().Events("org-1").Create(context.Background(), + karpenterEvent("refusal", tc.msg), metav1.CreateOptions{}); err != nil { t.Fatal(err) } - if got := listState(t, p); got != want { - t.Fatalf("after Karpenter's %s: %s, want %s", reason, got, want) + if got := listState(t, p); got != tc.want { + t.Fatalf("after Karpenter's %s refusal: %s, want %s", name, got, tc.want) } }) } } -// A launch that reached EC2 had passed scheduling, so an ICE outranks a FailedScheduling on -// the same pod. +// A pod refused for capacity stays so: a spec refusal recorded alongside does not undo the block. func TestListCapacityRefusalWins(t *testing.T) { tc := newTestCluster(t, false) p := newTestProvider(map[string]clusterClients{"eks-use1": tc.clients()}) if _, err := p.Provision(context.Background(), placeholder(), provisionRequest()); err != nil { t.Fatal(err) } - for _, reason := range []string{reasonInsufficientCapacity, reasonFailedScheduling} { + for name, msg := range map[string]string{"sold-out": refusalSoldOut, "spec": refusalMissingToleration} { if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), - karpenterEvent(reason, "refused"), metav1.CreateOptions{}); err != nil { + karpenterEvent(name, msg), metav1.CreateOptions{}); err != nil { t.Fatal(err) } } @@ -169,7 +209,7 @@ func TestListSpecRefusalOutranksBindTimeout(t *testing.T) { t.Fatal(err) } if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), - karpenterEvent(reasonFailedScheduling, "did not tolerate taint"), metav1.CreateOptions{}); err != nil { + karpenterEvent("refusal", refusalMissingToleration), metav1.CreateOptions{}); err != nil { t.Fatal(err) } p.now = func() time.Time { return time.Now().Add(karpenterBindTimeout + time.Second) } @@ -178,6 +218,21 @@ func TestListSpecRefusalOutranksBindTimeout(t *testing.T) { } } +func TestCapacityRefusal(t *testing.T) { + for msg, want := range map[string]bool{ + refusalMissingToleration: false, + refusalTooManyGPUs: false, + refusalSoldOut: true, + refusalPoolLimit: true, + refusal("node limits have been exhausted for nodepool (NodePool=gpu)"): true, + refusal("no instance type has the required offering, requirements=…"): true, + } { + if got := capacityRefusal(msg); got != want { + t.Errorf("capacityRefusal(%.80q…) = %v, want %v", msg, got, want) + } + } +} + // listState is the one instance List reports. func listState(t *testing.T, p *Provider) provider.InstanceState { t.Helper() @@ -196,17 +251,21 @@ func TestListIgnoresWhatIsNotARefusal(t *testing.T) { if _, err := p.Provision(context.Background(), placeholder(), provisionRequest()); err != nil { t.Fatal(err) } - ev := karpenterEvent(reasonFailedScheduling, "0/3 nodes are available") - ev.Source.Component = "default-scheduler" - if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), ev, metav1.CreateOptions{}); err != nil { - t.Fatal(err) + scheduler := karpenterEvent("scheduler", "0/3 nodes are available") + scheduler.Source.Component = "default-scheduler" + nominated := karpenterEvent("nominated", "Pod should schedule on: nodeclaim/gpu-abc") + nominated.Type, nominated.Reason = corev1.EventTypeNormal, "Nominated" + for _, ev := range []*corev1.Event{scheduler, nominated} { + if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), ev, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } } if got := listState(t, p); got != provider.InstanceProvisioning { - t.Fatalf("with the scheduler's event: %s, want Provisioning", got) + t.Fatalf("with the scheduler's FailedScheduling and Karpenter's Nominated: %s, want Provisioning", got) } if _, err := tc.kube.CoreV1().Events("org-1").Create(context.Background(), - karpenterEvent(reasonInsufficientCapacity, "earlier attempt"), metav1.CreateOptions{}); err != nil { + karpenterEvent("earlier", refusalSoldOut), metav1.CreateOptions{}); err != nil { t.Fatal(err) } pod, _ := tc.kube.CoreV1().Pods("org-1").Get(context.Background(), "org-1-work-1", metav1.GetOptions{}) From 3bea50e581f9efbe9b49bc8181c303ee45a93e35 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Sat, 24 Oct 2026 16:55:32 +0100 Subject: [PATCH 13/21] add deny policy Signed-off-by: kerthcet --- pkg/failover/blocklist.go | 90 +++++----------- pkg/failover/blocklist_test.go | 118 +++++++++------------ pkg/provider/aws/aws.go | 16 ++- pkg/provider/aws/aws_test.go | 13 ++- pkg/provider/errors.go | 21 ++-- pkg/provider/errors_test.go | 28 +++-- pkg/provider/kubernetes/kubernetes.go | 13 +-- pkg/provider/kubernetes/kubernetes_test.go | 9 +- pkg/provider/modal/modal.go | 8 +- pkg/provider/modal/modal_test.go | 26 ++--- pkg/provider/provider.go | 61 +++++------ pkg/provider/runpod/runpod.go | 11 +- pkg/provider/runpod/runpod_test.go | 28 ++--- pkg/vnode/handler_test.go | 25 +++-- 14 files changed, 201 insertions(+), 266 deletions(-) diff --git a/pkg/failover/blocklist.go b/pkg/failover/blocklist.go index 0ebd81a..99b08ce 100644 --- a/pkg/failover/blocklist.go +++ b/pkg/failover/blocklist.go @@ -20,10 +20,9 @@ limitations under the License. // // It follows SkyPilot's blocklist-granularity rule: record at the exact granularity // that failed (a provider.BlockScope), and block a candidate only if some live -// entry's scope MATCHES it. Empty scope fields are wildcards, so "no Spot H100 in -// us-east-1" never disqualifies an OnDemand request, another accelerator, or another -// region — while an auth failure (DenyAll) blocks the whole provider until its TTL -// lapses. +// entry's scope MATCHES it. So "no Spot H100 in us-east-1" (DenyCandidate) never +// disqualifies an OnDemand request, another accelerator, or another region — while an +// auth failure (DenyAll) blocks the whole provider until its TTL lapses. // // In-memory and per-process on purpose: this is a hint that bounds churn, not durable // state. After a restart it is empty and the next attempt re-probes the provider, @@ -84,10 +83,11 @@ func newWithClock(c clock.Clock) *Blocklist { return &Blocklist{clock: c} } -// Record adds a block for prov at scope, lapsing after ttl. A DenyAll scope blocks the -// whole provider; anything else is confined to its matching fields. A non-positive ttl -// or empty provider is a no-op — defensive, so a misfire cannot install a permanent or -// provider-less block. Recording also drops expired entries, bounding the slice. +// Record adds a block for prov at scope, lapsing after ttl; scope.Deny says how much of the +// provider it covers (see provider.DenyPolicy). A non-positive ttl, an empty provider or a +// scope with no known policy is a no-op — defensive, so a misfire cannot install a +// permanent, provider-less or unmatchable block. Recording also drops expired entries, +// bounding the slice. // // Records COALESCE: an existing live entry with the same provider and an identical // scope has its expiry extended (never shortened) instead of gaining a duplicate. So @@ -95,7 +95,7 @@ func newWithClock(c clock.Clock) *Blocklist { // distinct live scopes, not failure volume. Behaviour is unchanged, since Blocked/ // BlockedUntil already used the latest covering expiry. func (b *Blocklist) Record(prov string, scope provider.BlockScope, ttl time.Duration) { - if prov == "" || ttl <= 0 { + if prov == "" || ttl <= 0 || !knownPolicy(scope.Deny) { return } b.mu.Lock() @@ -108,7 +108,7 @@ func (b *Blocklist) Record(prov string, scope provider.BlockScope, ttl time.Dura // Coalesce into an existing live entry for the same scope. gcLocked has already // dropped expired entries, so every entry scanned here is still live. for i := range b.entries { - if b.entries[i].provider == prov && scopeEqual(b.entries[i].scope, scope) { + if b.entries[i].provider == prov && b.entries[i].scope == scope { if expiresAt.After(b.entries[i].expiresAt) { b.entries[i].expiresAt = expiresAt // extend; never shorten a live block } @@ -184,66 +184,28 @@ func (b *Blocklist) gcLocked(now time.Time) { b.entries = kept } -// scopeCovers reports whether scope (a recorded block) applies to candidate c. DenyAll -// covers the whole provider; otherwise every field must match, per BlockScope's -// three-state pointer semantics: -// -// - nil => axis not applicable: matches ONLY a candidate whose field is empty, so -// a CPU-only Pod or a region-simple provider blocks without widening. -// - &"" => wildcard: matches any value on that axis. -// - &"v" => exact: matches only a candidate whose field equals "v". -// -// This is the match half of the granularity rule: as broad as the failure, no broader. +// scopeCovers reports whether scope (a recorded block) applies to candidate c, per its +// policy: the whole provider, one region, or one candidate. This is the match half of the +// granularity rule: as broad as the failure, no broader. func scopeCovers(scope provider.BlockScope, c Candidate) bool { - if scope.DenyAll { + switch scope.Deny { + case provider.DenyAll: return true - } - if !ptrMatches(scope.Accelerator, c.Accelerator) { - return false - } - if scope.CapacityType != "" && scope.CapacityType != c.CapacityType { - return false - } - if !ptrMatches(scope.Region, c.Region) { + case provider.DenyRegion: + return scope.Region == c.Region + case provider.DenyCandidate: + return scope.Accelerator == c.Accelerator && + (scope.CapacityType == "" || scope.CapacityType == c.CapacityType) && + scope.Region == c.Region + default: return false } - return true } -// scopeEqual reports whether two BlockScopes are the SAME block — identical along -// every axis. It is used by Record to coalesce a repeated failure into an existing -// entry. The *string fields are compared by VALUE, not pointer identity: two -// independent failures for the same region allocate distinct *string pointers, so a -// pointer-wise "==" on BlockScope would never coalesce them (and pointer fields make -// "==" unsafe anyway). This is stricter than scopeCovers, which asks "does this block -// COVER that candidate"; here we need exact equality so distinct scopes stay distinct -// entries. -func scopeEqual(a, b provider.BlockScope) bool { - return a.DenyAll == b.DenyAll && - a.CapacityType == b.CapacityType && - ptrEqual(a.Accelerator, b.Accelerator) && - ptrEqual(a.Region, b.Region) -} - -// ptrEqual reports whether two *string are equal by value: both nil, or both non-nil -// pointing at equal strings. (nil and &"" are DIFFERENT — nil means "axis not -// applicable" while &"" means "wildcard", a distinction BlockScope depends on.) -func ptrEqual(x, y *string) bool { - if x == nil || y == nil { - return x == y // equal only if both nil - } - return *x == *y -} - -// ptrMatches applies BlockScope's three-state rule to one axis: nil matches only an -// empty candidate value (the axis is not applicable), &"" is a wildcard that matches -// anything, and &"v" matches only the exact value. -func ptrMatches(pattern *string, value string) bool { - if pattern == nil { - return value == "" - } - if *pattern == "" { +func knownPolicy(p provider.DenyPolicy) bool { + switch p { + case provider.DenyAll, provider.DenyRegion, provider.DenyCandidate: return true } - return *pattern == value + return false } diff --git a/pkg/failover/blocklist_test.go b/pkg/failover/blocklist_test.go index 53bc3fa..747e91d 100644 --- a/pkg/failover/blocklist_test.go +++ b/pkg/failover/blocklist_test.go @@ -30,14 +30,12 @@ import ( // deterministic (package scripts forbid wall-clock reads anyway). var baseTime = time.Date(2026, 7, 24, 0, 0, 0, 0, time.UTC) -// ptr is a compact helper for the three-state BlockScope pointer fields. -func ptr(s string) *string { return &s } - func spotH100EastScope() provider.BlockScope { return provider.BlockScope{ - Accelerator: ptr("H100"), + Deny: provider.DenyCandidate, + Accelerator: "H100", CapacityType: nebulav1alpha1.CapacitySpot, - Region: ptr("us-east-1"), + Region: "us-east-1", } } @@ -97,58 +95,56 @@ func TestBlocked_ScopeMatchIsPrecise(t *testing.T) { } } -func TestBlocked_WildcardFieldsMatchAnyValue(t *testing.T) { +func TestBlocked_DenyRegionBlocksOneRegion(t *testing.T) { fc := testclock.NewFakeClock(baseTime) bl := newWithClock(fc) - // Wildcard Accelerator + wildcard Region (&"") => blocks Spot on aws - // everywhere, any GPU. The pointers are deliberately non-nil-empty: nil would - // mean "no such axis" and match only an empty candidate field. - bl.Record("aws", provider.BlockScope{ - Accelerator: ptr(""), - CapacityType: spot, - Region: ptr(""), - }, 10*time.Minute) + bl.Record("kubernetes", provider.BlockScope{Deny: provider.DenyRegion, Region: "us-east-1"}, 10*time.Minute) - if !bl.Blocked(cand("aws", "H100", spot, "us-east-1")) { - t.Error("H100/us-east-1 Spot should be blocked by the wildcard Spot scope") + if !bl.Blocked(cand("kubernetes", "H100", spot, "us-east-1")) || + !bl.Blocked(cand("kubernetes", "A100", onDemand, "us-east-1")) { + t.Error("DenyRegion must block every accelerator and tier in its region") } - if !bl.Blocked(cand("aws", "A100", spot, "eu-west-1")) { - t.Error("A100/eu-west-1 Spot should also be blocked by the wildcard Spot scope") + if bl.Blocked(cand("kubernetes", "H100", spot, "us-west-1")) { + t.Error("DenyRegion must not block another region") } - if bl.Blocked(cand("aws", "H100", onDemand, "us-east-1")) { - t.Error("OnDemand must survive a Spot-only wildcard block") +} + +func TestBlocked_EmptyCapacityTypeMatchesEveryTier(t *testing.T) { + fc := testclock.NewFakeClock(baseTime) + bl := newWithClock(fc) + anyTier := provider.BlockScope{Deny: provider.DenyCandidate, Accelerator: "H100", Region: "us-east-1"} + bl.Record("aws", anyTier, 10*time.Minute) + + if !bl.Blocked(cand("aws", "H100", spot, "us-east-1")) || !bl.Blocked(cand("aws", "H100", onDemand, "us-east-1")) { + t.Error("an empty CapacityType must block both tiers") + } + if bl.Blocked(cand("aws", "A100", spot, "us-east-1")) { + t.Error("the accelerator still matches exactly") } } -// A nil axis is "not applicable": it matches only a candidate whose field is empty, -// never a populated one. This is how a region-simple provider (Modal: nil Region) -// or a CPU-only Pod (nil Accelerator) blocks without widening across an axis it -// never had. Contrast &"" (wildcard), which matches any value — see the test above. -func TestBlocked_NilFieldMatchesOnlyEmptyCandidate(t *testing.T) { +// Empty Accelerator and Region match only candidates without them: a CPU-only Pod or a +// region-less provider (Modal) blocks without widening to a GPU or a region-pinned one. +func TestBlocked_EmptyFieldsMatchOnlyEmptyCandidate(t *testing.T) { fc := testclock.NewFakeClock(baseTime) bl := newWithClock(fc) - // A region-simple provider's block: nil Region, nil Accelerator, Spot only. - bl.Record("modal", provider.BlockScope{CapacityType: spot}, 10*time.Minute) + bl.Record("modal", provider.BlockScope{Deny: provider.DenyCandidate, CapacityType: spot}, 10*time.Minute) - // A region-simple candidate carries an empty region and (CPU Pod) empty - // accelerator, so the nil axes match it. if !bl.Blocked(cand("modal", "", spot, "")) { - t.Error("nil-axis Spot block should match the empty-field candidate") + t.Error("the block should match the empty-field candidate") } - // A populated candidate field is NOT matched by a nil axis: it is out of scope, - // not wildcarded in. if bl.Blocked(cand("modal", "H100", spot, "")) { - t.Error("nil Accelerator must not match a populated accelerator") + t.Error("an empty Accelerator must not match a populated accelerator") } if bl.Blocked(cand("modal", "", spot, "us-east-1")) { - t.Error("nil Region must not match a populated region") + t.Error("an empty Region must not match a populated region") } } func TestBlocked_DenyAllBlocksWholeProvider(t *testing.T) { fc := testclock.NewFakeClock(baseTime) bl := newWithClock(fc) - bl.Record("aws", provider.BlockScope{DenyAll: true}, 10*time.Minute) + bl.Record("aws", provider.BlockScope{Deny: provider.DenyAll}, 10*time.Minute) // Every accelerator/tier/region on aws is blocked... if !bl.Blocked(cand("aws", "H100", onDemand, "us-west-2")) { @@ -222,9 +218,9 @@ func TestRecord_NoOpOnInvalidInput(t *testing.T) { bl := newWithClock(fc) // Empty provider and non-positive TTL must not install a block. - bl.Record("", provider.BlockScope{DenyAll: true}, 10*time.Minute) - bl.Record("aws", provider.BlockScope{DenyAll: true}, 0) - bl.Record("aws", provider.BlockScope{DenyAll: true}, -time.Minute) + bl.Record("", provider.BlockScope{Deny: provider.DenyAll}, 10*time.Minute) + bl.Record("aws", provider.BlockScope{Deny: provider.DenyAll}, 0) + bl.Record("aws", provider.BlockScope{Deny: provider.DenyAll}, -time.Minute) if bl.Blocked(Candidate{Provider: "aws", Accelerator: "H100"}) { t.Error("invalid Record inputs must not block anything") @@ -232,15 +228,21 @@ func TestRecord_NoOpOnInvalidInput(t *testing.T) { if bl.Blocked(Candidate{Provider: "", Accelerator: "H100"}) { t.Error("empty-provider block must not exist") } + + // A scope without a known policy covers nothing, so it is not recorded either. + bl.Record("aws", provider.BlockScope{}, 10*time.Minute) + bl.Record("aws", provider.BlockScope{Deny: "Everything", Region: "us-east-1"}, 10*time.Minute) + if n := len(bl.entries); n != 0 { + t.Errorf("len(entries) = %d, want 0 for scopes without a known policy", n) + } } func TestRecord_CoalescesIdenticalScope(t *testing.T) { fc := testclock.NewFakeClock(baseTime) bl := newWithClock(fc) - // Several Pods failing for the SAME (provider, accelerator, tier, region) reason, - // each allocating its own scope value (distinct *string pointers), must collapse - // to ONE entry rather than one per failure. + // Several Pods failing for the SAME (provider, accelerator, tier, region) reason must + // collapse to ONE entry rather than one per failure. for i := 0; i < 5; i++ { bl.Record("aws", spotH100EastScope(), 10*time.Minute) } @@ -281,32 +283,16 @@ func TestRecord_DistinctScopesStaySeparate(t *testing.T) { // Scopes differing on any single axis are different blocks and must NOT coalesce. bl.Record("aws", spotH100EastScope(), 10*time.Minute) // H100/Spot/us-east-1 - bl.Record("aws", provider.BlockScope{ // different region - Accelerator: ptr("H100"), CapacityType: spot, Region: ptr("us-west-2"), - }, 10*time.Minute) - bl.Record("aws", provider.BlockScope{ // different tier - Accelerator: ptr("H100"), CapacityType: onDemand, Region: ptr("us-east-1"), - }, 10*time.Minute) + westScope, onDemandScope := spotH100EastScope(), spotH100EastScope() + westScope.Region, onDemandScope.CapacityType = "us-west-2", onDemand + regionScope := provider.BlockScope{Deny: provider.DenyRegion, Region: "us-east-1"} + bl.Record("aws", westScope, 10*time.Minute) // different region + bl.Record("aws", onDemandScope, 10*time.Minute) // different tier + bl.Record("aws", regionScope, 10*time.Minute) // different policy bl.Record("modal", spotH100EastScope(), 10*time.Minute) // different provider - if n := len(bl.entries); n != 4 { - t.Fatalf("len(entries) = %d, want 4 (distinct scopes must not coalesce)", n) - } -} - -func TestRecord_NilAndWildcardScopesDoNotCoalesce(t *testing.T) { - fc := testclock.NewFakeClock(baseTime) - bl := newWithClock(fc) - - // nil (axis not applicable) and &"" (wildcard) are semantically different scopes, - // so scopeEqual must keep them as separate entries. - nilAxes := provider.BlockScope{CapacityType: spot} - wildcardAxes := provider.BlockScope{Accelerator: ptr(""), Region: ptr(""), CapacityType: spot} - bl.Record("aws", nilAxes, 10*time.Minute) - bl.Record("aws", wildcardAxes, 10*time.Minute) - - if n := len(bl.entries); n != 2 { - t.Fatalf("len(entries) = %d, want 2 (nil and wildcard scopes are distinct)", n) + if n := len(bl.entries); n != 5 { + t.Fatalf("len(entries) = %d, want 5 (distinct scopes must not coalesce)", n) } } @@ -318,7 +304,7 @@ func TestRecord_ExpiredEntriesAreGarbageCollected(t *testing.T) { // Let the first entry expire, then record another: the GC on Record should drop // the stale one so the slice does not grow unbounded. fc.Step(2 * time.Minute) - bl.Record("aws", provider.BlockScope{DenyAll: true}, time.Minute) + bl.Record("aws", provider.BlockScope{Deny: provider.DenyAll}, time.Minute) if n := len(bl.entries); n != 1 { t.Fatalf("len(entries) = %d, want 1 (stale entry should be GC'd on Record)", n) diff --git a/pkg/provider/aws/aws.go b/pkg/provider/aws/aws.go index 01c0bd7..6783eae 100644 --- a/pkg/provider/aws/aws.go +++ b/pkg/provider/aws/aws.go @@ -712,7 +712,7 @@ func (p *Provider) List(ctx context.Context) ([]provider.Instance, error) { // sentinel: a Spot shortage must block only Spot, leaving OnDemand serviceable. // - EC2 capacity is per-region, so a capacity or quota block is confined to the region // that actually failed — a shortage in us-east-1 must not disqualify us-west-2, and -// vCPU limits are regional too. Only DenyAll (auth) stays region-wide, since bad +// vCPU limits are regional too. Only DenyAll (auth) stays provider-wide, since bad // credentials fail everywhere. The region is a parameter because the error does not // carry it and there is no default region to assume. func (p *Provider) ClassifyProvisionError(err error, accelerator, region string) provider.BlockScope { @@ -731,15 +731,11 @@ func (p *Provider) ClassifyProvisionError(err error, accelerator, region string) if scope == (provider.BlockScope{}) { return scope } - // Confine an accelerator/capacity/quota block to the region that failed. A - // DenyAll (auth) fails in every region, so it stays region-wide (Region left nil). - // An empty region (should not happen — every request carries one) maps to &"" = - // the wildcard, which blocks the accelerator in ALL regions: the safe over-broad - // choice, since a stale-but-effective block beats a block pinned to a guessed - // region that never matches. - if !scope.DenyAll { - r := region - scope.Region = &r + // Confine a candidate block to the region that failed; DenyAll (auth) fails in every + // region, so it stays provider-wide. Every request carries a region; an empty one + // would only match region-less candidates, which AWS has none of. + if scope.Deny == provider.DenyCandidate { + scope.Region = region } return scope } diff --git a/pkg/provider/aws/aws_test.go b/pkg/provider/aws/aws_test.go index 67b4ab5..ce9b294 100644 --- a/pkg/provider/aws/aws_test.go +++ b/pkg/provider/aws/aws_test.go @@ -617,16 +617,15 @@ func TestSweepRegions_CoversEveryPoolsPlacement(t *testing.T) { func TestClassifyProvisionError(t *testing.T) { p := newTestProvider(&fakeClient{}) const accel = "H100" - denyAll := provider.BlockScope{DenyAll: true} - // EC2 capacity/accelerator/quota blocks are confined to the adapter's region (an - // exact-match Region pointer) AND to the requested accelerator (passed in by the - // caller — the provider owns the whole scope). Only auth (DenyAll) ignores both. - region, wantAccel := testRegion, accel + denyAll := provider.BlockScope{Deny: provider.DenyAll} + // EC2 capacity/accelerator/quota blocks are confined to the adapter's region AND to the + // requested accelerator (passed in by the caller — the provider owns the whole scope). + // Only auth (DenyAll) ignores both. onDemandRegional := provider.BlockScope{ - Accelerator: &wantAccel, CapacityType: nebulav1alpha1.CapacityOnDemand, Region: ®ion, + Deny: provider.DenyCandidate, Accelerator: accel, CapacityType: nebulav1alpha1.CapacityOnDemand, Region: testRegion, } spotRegional := provider.BlockScope{ - Accelerator: &wantAccel, CapacityType: nebulav1alpha1.CapacitySpot, Region: ®ion, + Deny: provider.DenyCandidate, Accelerator: accel, CapacityType: nebulav1alpha1.CapacitySpot, Region: testRegion, } // A Spot capacity failure the Client wraps with both sentinels: no-capacity diff --git a/pkg/provider/errors.go b/pkg/provider/errors.go index 7193315..3fc23b5 100644 --- a/pkg/provider/errors.go +++ b/pkg/provider/errors.go @@ -55,7 +55,7 @@ var ( // checking the shared sentinels first and falling back to string heuristics for raw API // messages. The rule it encodes: a narrow failure must not disqualify other accelerators, or // other regions. Only auth widens to the whole provider via DenyAll; a capacity refusal is -// scoped to the failing accelerator/tier/region so failover can route around it; and two cases +// DenyCandidate, the failing accelerator/tier/region, so failover can route around it; and two cases // block nothing at all — a failure that belongs to the REQUEST rather than the candidate, and // one that could not be attributed to either. // @@ -68,29 +68,22 @@ var ( // a scope elsewhere. An adapter that decorates the result must leave the ZERO scope alone — // it means "block nothing", and any field added to it becomes a block. // -// capacityType is stamped onto accelerator-scoped blocks so a Spot failure does not block -// OnDemand. accelerator is the request's POOL identity (type:count, "" for a CPU-only Pod), -// not the provider's SKU id, so the block stays truthful when a launch spans several -// interchangeable instance types: non-empty becomes an exact match, "" leaves it nil so the -// block does not widen across every accelerator. DenyAll ignores both. +// capacityType is stamped onto candidate blocks so a Spot failure does not block OnDemand. +// accelerator is the request's POOL identity (type:count, "" for a CPU-only Pod), not the +// provider's SKU id, so the block stays truthful when a launch spans several interchangeable +// instance types (see BlockScope.Accelerator). DenyAll ignores both. func ClassifyError(err error, capacityType nebulav1alpha1.CapacityType, accelerator string) BlockScope { if err == nil { return BlockScope{} } - // capacityScope builds an accelerator/tier-scoped block, promoting a non-empty - // accelerator pool to an exact-match pointer and leaving it nil otherwise. capacityScope := func() BlockScope { - s := BlockScope{CapacityType: capacityType} - if accelerator != "" { - s.Accelerator = &accelerator - } - return s + return BlockScope{Deny: DenyCandidate, Accelerator: accelerator, CapacityType: capacityType} } switch categorize(err) { case catAuth: - return BlockScope{DenyAll: true} + return BlockScope{Deny: DenyAll} case catCapacity: return capacityScope() case catRequest: diff --git a/pkg/provider/errors_test.go b/pkg/provider/errors_test.go index cb68d72..c84cce8 100644 --- a/pkg/provider/errors_test.go +++ b/pkg/provider/errors_test.go @@ -29,16 +29,14 @@ import ( func TestClassifyError(t *testing.T) { const tier = nebulav1alpha1.CapacityOnDemand const accel = "H100" - // capacityScope is the expected accelerator-scoped block: the accelerator id is - // promoted to an exact-match pointer. - capacityScope := BlockScope{CapacityType: tier, Accelerator: ptrStr(accel)} + capacityScope := BlockScope{Deny: DenyCandidate, CapacityType: tier, Accelerator: accel} tests := []struct { name string err error want BlockScope }{ {"nil", nil, BlockScope{}}, - {"auth sentinel", ErrAuth, BlockScope{DenyAll: true}}, + {"auth sentinel", ErrAuth, BlockScope{Deny: DenyAll}}, // Quota is scoped like capacity (per accelerator + tier), NOT DenyAll: cloud // quotas are per-resource and, for a multi-region adapter, per-region, so one // exhausted quota must not fence off other regions/accelerators. @@ -80,7 +78,7 @@ func TestClassifyError(t *testing.T) { "modal refusing us during a build fences the provider", fmt.Errorf("modal: image build: %w", errors.New( "rpc error: code = Unauthenticated desc = invalid token")), - BlockScope{DenyAll: true}, + BlockScope{Deny: DenyAll}, }, { "modal rate-limiting us during a build is capacity-scoped", @@ -89,7 +87,7 @@ func TestClassifyError(t *testing.T) { capacityScope, }, {"wrapped sentinel", fmt.Errorf("provision failed: %w", ErrNoCapacity), capacityScope}, - {"string unauthorized", fmt.Errorf("HTTP 401 unauthorized"), BlockScope{DenyAll: true}}, + {"string unauthorized", fmt.Errorf("HTTP 401 unauthorized"), BlockScope{Deny: DenyAll}}, {"string quota", fmt.Errorf("account limit exceeded"), capacityScope}, {"string capacity", fmt.Errorf("no capacity available"), capacityScope}, // An error we cannot attribute blocks NOTHING: it is no evidence against the @@ -124,7 +122,7 @@ func TestClassifyError(t *testing.T) { // Pod picked the same broken provider. {"mint failure on an auth refusal", fmt.Errorf("modal: mint connect credential for sandbox sb-1: %w", - errors.New("unauthorized")), BlockScope{DenyAll: true}}, + errors.New("unauthorized")), BlockScope{Deny: DenyAll}}, {"mint failure on a rate limit", fmt.Errorf("modal: mint connect credential for sandbox sb-1: %w", errors.New("429 rate limit exceeded")), capacityScope}, @@ -155,19 +153,19 @@ func TestClassifyError(t *testing.T) { // does not block OnDemand on the same provider. func TestClassifyError_TierStamped(t *testing.T) { got := ClassifyError(ErrNoCapacity, nebulav1alpha1.CapacitySpot, "H100") - if got.CapacityType != nebulav1alpha1.CapacitySpot || got.DenyAll { + if got.CapacityType != nebulav1alpha1.CapacitySpot || got.Deny != DenyCandidate { t.Fatalf("expected Spot accelerator-scoped block, got %+v", got) } } -// An empty accelerator (CPU-only Pod) must leave Accelerator nil ("not -// applicable"), never a wildcard that would widen the block across every GPU. -func TestClassifyError_EmptyAcceleratorStaysNil(t *testing.T) { +// A CPU-only Pod's block is a candidate block with an empty Accelerator, which matches only +// CPU-only candidates (see BlockScope.Accelerator), never a GPU one. +func TestClassifyError_EmptyAcceleratorStaysEmpty(t *testing.T) { got := ClassifyError(ErrNoCapacity, nebulav1alpha1.CapacityOnDemand, "") - if got.Accelerator != nil { - t.Fatalf("expected nil Accelerator for a CPU-only request, got %+v", got.Accelerator) + if got.Deny != DenyCandidate || got.Accelerator != "" { + t.Fatalf("expected a CPU-only candidate block, got %+v", got) } - if got.DenyAll || got.CapacityType != nebulav1alpha1.CapacityOnDemand { + if got.CapacityType != nebulav1alpha1.CapacityOnDemand { t.Fatalf("expected an OnDemand capacity scope, got %+v", got) } } @@ -184,5 +182,3 @@ func TestClassifyError_UnattributableBlocksNothing(t *testing.T) { t.Fatalf("an unattributable error must block nothing, got %+v", got) } } - -func ptrStr(s string) *string { return &s } diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index 228f4d8..6625daf 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -412,14 +412,11 @@ func (p *Provider) ClassifyProvisionError(err error, accelerator, region string) if scope == (provider.BlockScope{}) { return scope // block nothing; decorating it would create a block } - r := region - if scope.DenyAll { - // DenyAll would block every region, but a 401/403 is one cluster's access entry or - // RBAC, not shared by the others. So block everything in this region instead. - anyAccelerator := "" - return provider.BlockScope{Accelerator: &anyAccelerator, Region: &r} - } - scope.Region = &r + if scope.Deny == provider.DenyAll { + // A 401/403 is one cluster's access entry or RBAC, not shared by the others. + return provider.BlockScope{Deny: provider.DenyRegion, Region: region} + } + scope.Region = region return scope } diff --git a/pkg/provider/kubernetes/kubernetes_test.go b/pkg/provider/kubernetes/kubernetes_test.go index 410e62e..b2862d0 100644 --- a/pkg/provider/kubernetes/kubernetes_test.go +++ b/pkg/provider/kubernetes/kubernetes_test.go @@ -249,12 +249,15 @@ func TestClassifyProvisionError(t *testing.T) { spot := p.ClassifyProvisionError(provider.WithTier(nebulav1alpha1.CapacitySpot, fmt.Errorf("x: %w", provider.ErrNoCapacity)), "H100:1", region) - if spot.CapacityType != nebulav1alpha1.CapacitySpot || *spot.Region != region || *spot.Accelerator != "H100:1" { - t.Errorf("spot shortage scope = %+v", spot) + want := provider.BlockScope{Deny: provider.DenyCandidate, Accelerator: "H100:1", + CapacityType: nebulav1alpha1.CapacitySpot, Region: region} + if spot != want { + t.Errorf("spot shortage scope = %+v, want %+v", spot, want) } + // One cluster's access entry or RBAC, so the region (cluster) alone, never the provider. auth := p.ClassifyProvisionError(classifyAPI(forbidden()), "H100:1", region) - if auth.DenyAll || *auth.Region != region || *auth.Accelerator != "" { + if auth != (provider.BlockScope{Deny: provider.DenyRegion, Region: region}) { t.Errorf("auth failure must block only the cluster, got %+v", auth) } } diff --git a/pkg/provider/modal/modal.go b/pkg/provider/modal/modal.go index 3f5bf15..b46c7b3 100644 --- a/pkg/provider/modal/modal.go +++ b/pkg/provider/modal/modal.go @@ -574,8 +574,8 @@ func (p *Provider) List(ctx context.Context) ([]provider.Instance, error) { // ResolveRegions sends every declared region at once, so the token may name the whole set // and the block then covers all of it. That is honest — a multi-region create never says // which region was short — and a pool wanting per-region blocking declares per-region -// pools. An empty region leaves Region nil, which per BlockScope matches only candidates -// with no region, so the block never leaks onto region-pinned ones. +// pools. An empty region stays empty, which per BlockScope matches only candidates with no +// region, so the block never leaks onto region-pinned ones. func (p *Provider) ClassifyProvisionError(err error, accelerator, region string) provider.BlockScope { // No failure, no block. ClassifyError already returns the zero scope here, but // the region decoration below would repopulate it into a non-empty scope that @@ -593,8 +593,8 @@ func (p *Provider) ClassifyProvisionError(err error, accelerator, region string) } // DenyAll already covers every region (auth fails everywhere), so narrowing it // would contradict the category. - if region != "" && !scope.DenyAll { - scope.Region = ®ion + if scope.Deny == provider.DenyCandidate { + scope.Region = region } return scope } diff --git a/pkg/provider/modal/modal_test.go b/pkg/provider/modal/modal_test.go index 66936d3..b74a874 100644 --- a/pkg/provider/modal/modal_test.go +++ b/pkg/provider/modal/modal_test.go @@ -545,8 +545,8 @@ func TestProvision_DiskAboveDefaultRefused(t *testing.T) { func TestClassifyProvisionError(t *testing.T) { p := newTestProvider(&fakeClient{}) - denyAll := provider.BlockScope{DenyAll: true} - onDemand := provider.BlockScope{CapacityType: nebulav1alpha1.CapacityOnDemand} + denyAll := provider.BlockScope{Deny: provider.DenyAll} + onDemand := provider.BlockScope{Deny: provider.DenyCandidate, CapacityType: nebulav1alpha1.CapacityOnDemand} tests := []struct { name string err error @@ -1158,20 +1158,20 @@ func TestClassifyProvisionError_ConfinesToFailingRegion(t *testing.T) { // regions are independent pools, so without this the first regional failure would // block every region the pool lists. got := p.ClassifyProvisionError(provider.ErrNoCapacity, "H100:1", "us-east") - if got.Region == nil || *got.Region != "us-east" { - t.Fatalf("expected the block confined to us-east, got Region=%v", got.Region) + if got.Deny != provider.DenyCandidate || got.Region != "us-east" { + t.Fatalf("expected a candidate block confined to us-east, got %+v", got) } - // Unconstrained: no region axis was used, so Region stays nil — which per - // BlockScope's three-state rule matches only an empty-region candidate, and so - // cannot leak onto region-pinned ones. - if got := p.ClassifyProvisionError(provider.ErrNoCapacity, "H100:1", ""); got.Region != nil { - t.Fatalf("expected nil Region for an unconstrained request, got %q", *got.Region) + // Unconstrained: no region axis was used, so Region stays empty — which per BlockScope + // matches only an empty-region candidate, and so cannot leak onto region-pinned ones. + if got := p.ClassifyProvisionError(provider.ErrNoCapacity, "H100:1", ""); got.Region != "" { + t.Fatalf("expected an empty Region for an unconstrained request, got %q", got.Region) } // Auth fails in every region, so DenyAll must not be narrowed to one. - if got := p.ClassifyProvisionError(provider.ErrAuth, "H100:1", "us-east"); got.Region != nil { - t.Fatalf("DenyAll must not be confined to a region, got %q", *got.Region) + got = p.ClassifyProvisionError(provider.ErrAuth, "H100:1", "us-east") + if got != (provider.BlockScope{Deny: provider.DenyAll}) { + t.Fatalf("auth must deny the whole provider, got %+v", got) } // No error, no block. recordBlock installs any non-empty scope it is handed, so @@ -1229,7 +1229,7 @@ func TestClassifyProvisionError_ImageBuildNeverDeniesTheProvider(t *testing.T) { // an expired workspace token has to fence the provider, or every replacement Pod retries it. refused := fmt.Errorf("modal: image build: %w", errors.New("rpc error: code = Unauthenticated desc = invalid token")) - if got := p.ClassifyProvisionError(refused, "H100:1", "us-east"); !got.DenyAll { + if got := p.ClassifyProvisionError(refused, "H100:1", "us-east"); got.Deny != provider.DenyAll { t.Fatalf("Modal refusing our credentials must fence the provider, got %+v", got) } } @@ -1383,7 +1383,7 @@ func TestProvision_MintDeadlineBlocksTheCandidate(t *testing.T) { t.Fatal("Provision succeeded; the sandbox has no credential") } scope := provider.ClassifyError(err, nebulav1alpha1.CapacityOnDemand, "H100:1") - if scope == (provider.BlockScope{}) || scope.DenyAll { + if scope.Deny != provider.DenyCandidate { t.Fatalf("BlockScope = %+v, want an accelerator-scoped block for a spent budget", scope) } } diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 6283a06..0a25ade 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -494,38 +494,39 @@ type Offering struct { GPUCount int32 } +// DenyPolicy is how much of a provider a failed placement excludes, from widest to narrowest. +type DenyPolicy string + +const ( + // DenyAll excludes every candidate of the provider: a credential that serves all of + // them stopped working. Still scoped to this one provider; it never spans providers. + DenyAll DenyPolicy = "All" + // DenyRegion excludes every accelerator and capacity type in BlockScope.Region: a + // failure of that region alone, such as one kubernetes cluster's access entry. + DenyRegion DenyPolicy = "Region" + // DenyCandidate excludes the failing candidate: BlockScope.Accelerator at its + // CapacityType in its Region, so failover can route around just what failed. + DenyCandidate DenyPolicy = "Candidate" +) + // BlockScope is how narrowly a failed placement is excluded, matched to what actually -// failed. It is a MATCH PATTERN, not a value, so the pointer fields are three-state: +// failed. The zero scope blocks nothing: it is the ONLY such signal, so every block names +// its Deny policy, and the fields below are read only as that policy says. // -// - nil => axis not applicable; matches only a candidate whose field is also empty -// (a CPU-only Pod has no accelerator, a region-simple provider no region). -// - &"" => wildcard: matches any value on that axis. -// - &"v" => exact: only candidates equal to "v" (a failed H100 must not block A100). -// -// Pointers are needed because the scope is built in two places: the adapter classifies -// from the error alone (which knows the region but not the accelerator, a property of the -// request), and the vnode handler fills the accelerator in from the failing Pod. So -// "unset" must be distinguishable from a deliberate wildcard. DenyAll ignores the pattern -// fields entirely. -// -// Note this is the opposite sense to a value field like ProviderSpec.Regions, where empty -// means "the default region". A candidate is resolved to concrete values before being -// matched here, so the two never mix. +// The scope is built in two places: the adapter classifies from the error alone (which +// knows the region but not the accelerator, a property of the request), and the vnode +// handler supplies the accelerator from the failing Pod. type BlockScope struct { - // Accelerator: nil => not applicable (CPU-only Pod, or DenyAll); &"" => every - // accelerator; &"H100:8" => that one pool. The value is the request's POOL identity - // (type:count), not the provider's SKU id, so a block stays confined to what ran out: - // an L4:8 shortage must not exclude L4:1. It also stays stable when a launch spans - // several interchangeable instance types. Filled in by the vnode handler, since the - // adapter classifies from the error alone and cannot know it. - Accelerator *string - // CapacityType empty => blocks all capacity types. + Deny DenyPolicy + // Accelerator, for DenyCandidate, is the request's POOL identity (type:count), not the + // provider's SKU id, so a block stays confined to what ran out: an L4:8 shortage must + // not exclude L4:1. Always an exact match: "" is a CPU-only Pod's, never every + // accelerator. + Accelerator string + // CapacityType, for DenyCandidate: empty => every capacity type. CapacityType nebulav1alpha1.CapacityType - // Region: nil => the provider has no region axis (Modal, whose candidates carry an - // empty region too); &"us-east-1" => that region only, so a shortage there does not - // disqualify us-west-2. - Region *string - // DenyAll true => block everything on this provider (auth/quota errors), ignoring the - // fields above. Still scoped to this one provider; it never spans providers. - DenyAll bool + // Region, for DenyRegion and DenyCandidate: that region only, so a shortage there does + // not disqualify another. Exact like Accelerator: "" matches only candidates without a + // region, such as a Modal create spanning several, never the region-pinned ones. + Region string } diff --git a/pkg/provider/runpod/runpod.go b/pkg/provider/runpod/runpod.go index 455d403..e81342e 100644 --- a/pkg/provider/runpod/runpod.go +++ b/pkg/provider/runpod/runpod.go @@ -442,12 +442,11 @@ func (p *Provider) ClassifyProvisionError(err error, accelerator, region string) if scope == (provider.BlockScope{}) { return scope } - // DenyAll already covers every region (auth fails everywhere), so narrowing it would - // contradict the category. An empty region leaves Region nil, which per BlockScope - // matches only candidates that carry no region either — the unconstrained pool — so - // the block never leaks onto region-pinned candidates. - if region != "" && !scope.DenyAll { - scope.Region = ®ion + // Confine a candidate block to the DC that failed; DenyAll (auth) fails everywhere, so it + // stays provider-wide. An empty region matches only candidates without one, the + // unconstrained pool, so the block never leaks onto region-pinned candidates. + if scope.Deny == provider.DenyCandidate { + scope.Region = region } return scope } diff --git a/pkg/provider/runpod/runpod_test.go b/pkg/provider/runpod/runpod_test.go index 805e061..e0e9822 100644 --- a/pkg/provider/runpod/runpod_test.go +++ b/pkg/provider/runpod/runpod_test.go @@ -618,17 +618,17 @@ func TestClassifyProvisionError(t *testing.T) { t.Run("capacity is scoped to this accelerator, tier and region", func(t *testing.T) { err := fmt.Errorf("no instances available: %w", provider.ErrNoCapacity) got := p.ClassifyProvisionError(err, accel, region) - if got.DenyAll { - t.Error("DenyAll = true for a capacity shortage; only this candidate ran out") + if got.Deny != provider.DenyCandidate { + t.Errorf("Deny = %q for a capacity shortage, want %q; only this candidate ran out", got.Deny, provider.DenyCandidate) } - if got.Accelerator == nil || *got.Accelerator != accel { - t.Errorf("Accelerator = %v, want %q", got.Accelerator, accel) + if got.Accelerator != accel { + t.Errorf("Accelerator = %q, want %q", got.Accelerator, accel) } if got.CapacityType != nebulav1alpha1.CapacityOnDemand { t.Errorf("CapacityType = %q, want OnDemand", got.CapacityType) } - if got.Region == nil || *got.Region != region { - t.Errorf("Region = %v, want %q — a shortage in one DC must not disqualify another", + if got.Region != region { + t.Errorf("Region = %q, want %q — a shortage in one DC must not disqualify another", got.Region, region) } }) @@ -638,11 +638,11 @@ func TestClassifyProvisionError(t *testing.T) { // the category and keep trying the other candidates against the same dead key. err := fmt.Errorf("401: %w", provider.ErrAuth) got := p.ClassifyProvisionError(err, accel, region) - if !got.DenyAll { - t.Fatal("DenyAll = false for an auth failure") + if got.Deny != provider.DenyAll { + t.Fatalf("Deny = %q for an auth failure, want %q", got.Deny, provider.DenyAll) } - if got.Region != nil { - t.Errorf("Region = %v on a DenyAll scope, want nil", got.Region) + if got.Region != "" { + t.Errorf("Region = %q on a DenyAll scope, want none", got.Region) } }) @@ -657,12 +657,12 @@ func TestClassifyProvisionError(t *testing.T) { } }) - t.Run("an empty region leaves Region nil", func(t *testing.T) { - // nil matches only candidates that carry no region either — the unconstrained pool — + t.Run("an empty region stays empty", func(t *testing.T) { + // "" matches only candidates that carry no region either — the unconstrained pool — // so the block never leaks onto region-pinned candidates. err := fmt.Errorf("no capacity: %w", provider.ErrNoCapacity) - if got := p.ClassifyProvisionError(err, accel, ""); got.Region != nil { - t.Errorf("Region = %v, want nil", got.Region) + if got := p.ClassifyProvisionError(err, accel, ""); got.Region != "" { + t.Errorf("Region = %q, want none", got.Region) } }) } diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go index 43422de..87cb35b 100644 --- a/pkg/vnode/handler_test.go +++ b/pkg/vnode/handler_test.go @@ -140,6 +140,10 @@ func (f *fakeProvider) FindByClaim(_ context.Context, claim, _ string) (*provide func (f *fakeProvider) Offerings(context.Context) ([]provider.Offering, error) { return nil, nil } func (f *fakeProvider) MapAccelerator(c string, _ int32) ([]string, bool) { return []string{c}, true } func (f *fakeProvider) ResolveRegions(declared, _ []string) []string { return declared } + +// denyAll is the scope a provider-wide failure (auth) classifies to. +var denyAll = provider.BlockScope{Deny: provider.DenyAll} + func (f *fakeProvider) ClassifyProvisionError(err error, accel, region string) provider.BlockScope { f.classifyErr = err f.classifyAccel = accel @@ -505,11 +509,11 @@ func TestCreatePod_UnattributableErrorFailsPodWithoutBlocklisting(t *testing.T) } func TestCreatePod_ProvisionFailureRecordsBlock(t *testing.T) { - accel, region := "H100", "us-east-1" scope := provider.BlockScope{ - Accelerator: &accel, + Deny: provider.DenyCandidate, + Accelerator: "H100", CapacityType: nebulav1alpha1.CapacitySpot, - Region: ®ion, + Region: "us-east-1", } fp := &fakeProvider{provisionErr: errors.New("no capacity"), classifyScope: scope} bl := &recordingBlocklist{} @@ -576,7 +580,7 @@ func TestCreatePod_BlocklistTTLFallsBackToDefault(t *testing.T) { cluster: clusterWithTTL(0), }} { t.Run(tc.name, func(t *testing.T) { - fp := &fakeProvider{provisionErr: errors.New("no capacity"), classifyScope: provider.BlockScope{DenyAll: true}} + fp := &fakeProvider{provisionErr: errors.New("no capacity"), classifyScope: denyAll} bl := &recordingBlocklist{} h := NewHandler(fp, nil, bl, tc.cluster) h.jitterFn = func() time.Duration { return 0 } // pin jitter so the base default is exact @@ -599,7 +603,7 @@ func TestCreatePod_BlocklistTTLFallsBackToDefault(t *testing.T) { // is filed either. Blocking a candidate over OUR failure to read a pool would exclude // serviceable capacity for every tenant sharing the blocklist. func TestCreatePod_UnreadablePoolBlocksNothing(t *testing.T) { - fp := &fakeProvider{provisionErr: errors.New("no capacity"), classifyScope: provider.BlockScope{DenyAll: true}} + fp := &fakeProvider{provisionErr: errors.New("no capacity"), classifyScope: denyAll} bl := &recordingBlocklist{} h := NewHandler(fp, nil, bl, &fakeCluster{err: errors.New("cache not synced")}) @@ -617,7 +621,7 @@ func TestCreatePod_UnreadablePoolBlocksNothing(t *testing.T) { // The recorded TTL is the base (the pool's, or the default) PLUS the handler's jitter, so // Pods failing for one scope do not re-probe the freed candidate in lockstep. func TestCreatePod_BlocklistTTLAddsJitter(t *testing.T) { - fp := &fakeProvider{provisionErr: errors.New("no capacity"), classifyScope: provider.BlockScope{DenyAll: true}} + fp := &fakeProvider{provisionErr: errors.New("no capacity"), classifyScope: denyAll} bl := &recordingBlocklist{} h := NewHandler(fp, nil, bl, clusterWithTTL(30*time.Second)) h.jitterFn = func() time.Duration { return 20 * time.Second } // deterministic jitter @@ -1896,7 +1900,7 @@ func TestCreatePod_ReadsPlacementFromClaim(t *testing.T) { func TestCreatePod_BlocklistRegionComesFromTheClaim(t *testing.T) { fp := &fakeProvider{ provisionErr: errors.New("no capacity"), - classifyScope: provider.BlockScope{DenyAll: true}, + classifyScope: provider.BlockScope{Deny: provider.DenyAll}, } bl := &recordingBlocklist{} h := NewHandler(fp, nil, bl, clusterWithPlacement(nebulav1alpha1.CapacitySpot, "us-east-1")) @@ -1989,7 +1993,7 @@ func TestCreatePod_FailsClosedWithoutClusterState(t *testing.T) { // error path arriving late: the candidate is blocklisted with the pool's TTL, the Pod fails // with the provision-failed reason, and a second tick neither re-fails nor re-blocks it. func TestReconcileOnce_UnschedulableBlocksAndFailsPod(t *testing.T) { - scope := provider.BlockScope{Region: ptrTo("eks-use1")} + scope := provider.BlockScope{Deny: provider.DenyCandidate, Region: "eks-use1"} fp := &fakeProvider{provisionID: "inst-1", classifyScope: scope} bl := &recordingBlocklist{} cluster := clusterWithTTL(30 * time.Second) @@ -2028,7 +2032,8 @@ func TestReconcileOnce_UnschedulableBlocksAndFailsPod(t *testing.T) { // reports a refusal that is not capacity (Karpenter's FailedScheduling, a missing toleration), // which must not exclude the candidate for every other Pod. func TestReconcileOnce_FailedFailsPodWithoutBlock(t *testing.T) { - fp := &fakeProvider{provisionID: "inst-1", classifyScope: provider.BlockScope{Region: ptrTo("eks-use1")}} + scope := provider.BlockScope{Deny: provider.DenyCandidate, Region: "eks-use1"} + fp := &fakeProvider{provisionID: "inst-1", classifyScope: scope} bl := &recordingBlocklist{} h := NewHandler(fp, nil, bl, clusterWithTTL(30*time.Second)) h.jitterFn = func() time.Duration { return 0 } @@ -2074,8 +2079,6 @@ func TestReconcileOnce_ProvisioningIsNotInitializing(t *testing.T) { } } -func ptrTo[T any](v T) *T { return &v } - // NotReady shows as it is, Running with Ready=False, and counts toward the readiness deadline // like Pending: unusable for the whole deadline fails the Pod, before or after it was ready. // Running in between resets the clock. From 505e72ea4aab8919db2ad80d50a6219153edc7ed Mon Sep 17 00:00:00 2001 From: kerthcet Date: Sat, 24 Oct 2026 18:20:17 +0100 Subject: [PATCH 14/21] support describe instances Signed-off-by: kerthcet --- .../pod_placement_controller_test.go | 32 +++++ internal/controller/pod_placement_helpers.go | 13 +++ pkg/provider/kubernetes/eks.go | 38 +++++- pkg/provider/kubernetes/eks_offerings_test.go | 109 ++++++++++++++++++ pkg/provider/kubernetes/kubernetes.go | 52 ++++++++- pkg/provider/kubernetes/kubernetes_test.go | 4 +- pkg/provider/provider.go | 8 ++ 7 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 pkg/provider/kubernetes/eks_offerings_test.go diff --git a/internal/controller/pod_placement_controller_test.go b/internal/controller/pod_placement_controller_test.go index db97a60..2756aec 100644 --- a/internal/controller/pod_placement_controller_test.go +++ b/internal/controller/pod_placement_controller_test.go @@ -529,6 +529,38 @@ func TestPlacement_ExpandsRegionGroupIntoConcreteCandidates(t *testing.T) { } } +// regionalProvider offers its GPUs only in some regions, as a provider.RegionAccelerators. +type regionalProvider struct { + *fakeProvider + servedIn map[string]bool +} + +func (p regionalProvider) ServesAcceleratorIn(_, region string) bool { + return p.servedIn[region] +} + +// A region that does not offer the accelerator is skipped up front, without a placement there +// that fails and without a block. +func TestPlacement_SkipsRegionWithoutTheAccelerator(t *testing.T) { + pod := gatedPod("p1", "default", "uid-1", "pool-a", "H100") + pool := poolWithRegions("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, + provider.ProviderKubernetes, "us") + prov := regionalProvider{ + fakeProvider: &fakeProvider{ + name: provider.ProviderKubernetes, gpus: []string{"H100"}, + expandRegions: func([]string) []string { return []string{"us-east-1", "us-west-1"} }, + }, + servedIn: map[string]bool{"us-west-1": true}, + } + r, c := newPlacementReconciler(t, []client.Object{pod, pool}, prov) + + reconcilePod(t, r, "default", "p1") + + if region := getClaim(t, c, "default-p1").Spec.Region; region != "us-west-1" { + t.Fatalf("placed in %q, want us-west-1, the only region offering the accelerator", region) + } +} + func TestRequestedGeographies(t *testing.T) { cases := []struct { name string diff --git a/internal/controller/pod_placement_helpers.go b/internal/controller/pod_placement_helpers.go index 76378be..b6934bb 100644 --- a/internal/controller/pod_placement_helpers.go +++ b/internal/controller/pod_placement_helpers.go @@ -176,6 +176,12 @@ func (r *PodPlacementReconciler) selectPlacement(ctx context.Context, pod *corev continue } for _, region := range regions { // inner: region + if accel != "" && !servesAcceleratorIn(prov, accel, region) { + metrics.RecordCandidateSkip(ref.Name, tier, region, metrics.SkipAcceleratorUnsupported) + log.V(1).Info("skipping candidate: region does not offer the accelerator", + "provider", ref.Name, "accelerator", accelerator, "region", region) + continue + } if until, blocked := r.blockedUntil(ref.Name, accelerator, tier, region); blocked { // Servable but failed recently; try the next region, then the next // tier, and remember when this one frees so we can requeue for it. @@ -213,6 +219,13 @@ func (r *PodPlacementReconciler) selectPlacement(ctx context.Context, pod *corev return placement{}, false, soonest } +// servesAcceleratorIn reports whether prov offers the accelerator in region, which only a +// provider.RegionAccelerators narrows below what MapAccelerator already said. +func servesAcceleratorIn(prov provider.Provider, accel, region string) bool { + ra, ok := prov.(provider.RegionAccelerators) + return !ok || ra.ServesAcceleratorIn(accel, region) +} + // placementLabels renders the metric label set for a completed placement. It mirrors // the virtual kubelet's Handler.metricLabels field for field — that symmetry is the // point, not a coincidence: the handler reads region and capacity type back off the diff --git a/pkg/provider/kubernetes/eks.go b/pkg/provider/kubernetes/eks.go index 525b83e..af0f691 100644 --- a/pkg/provider/kubernetes/eks.go +++ b/pkg/provider/kubernetes/eks.go @@ -21,12 +21,15 @@ import ( "encoding/base64" "fmt" "net/http" + "strings" "sync" "time" awssdk "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/eks" "github.com/aws/aws-sdk-go-v2/service/sts" smithyhttp "github.com/aws/smithy-go/transport/http" @@ -58,6 +61,39 @@ const ( // events, plus discovery to check the node autoscaler is installed. type clusterClients struct { kube k8s.Interface + // ec2 answers which instance families the cluster's region offers (see offeredFamilies); + // nil keeps every family the catalog has. + ec2 describeOfferings +} + +// describeOfferings is the one EC2 call used, so a test can stand in for the client. +type describeOfferings interface { + DescribeInstanceTypeOfferings(context.Context, *ec2.DescribeInstanceTypeOfferingsInput, + ...func(*ec2.Options)) (*ec2.DescribeInstanceTypeOfferingsOutput, error) +} + +// offeredFamilies returns which of families the client's region offers any instance type of. +func offeredFamilies(ctx context.Context, api describeOfferings, families []string) (map[string]bool, error) { + patterns := make([]string, len(families)) + for i, f := range families { + patterns[i] = f + ".*" // the family's own types only, never another family's prefix + } + found := map[string]bool{} + pages := ec2.NewDescribeInstanceTypeOfferingsPaginator(api, &ec2.DescribeInstanceTypeOfferingsInput{ + LocationType: ec2types.LocationTypeRegion, + Filters: []ec2types.Filter{{Name: awssdk.String("instance-type"), Values: patterns}}, + }) + for pages.HasMorePages() { + page, err := pages.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("kubernetes: describe instance type offerings: %w", err) + } + for _, off := range page.InstanceTypeOfferings { + family, _, _ := strings.Cut(string(off.InstanceType), ".") + found[family] = true + } + } + return found, nil } // newEKSClients builds the clients for one EKS cluster from Nebula's ambient AWS identity @@ -107,7 +143,7 @@ func newEKSClients(ctx context.Context, cl ClusterConfig) (clusterClients, error if err != nil { return clusterClients{}, fmt.Errorf("kubernetes: cluster %q: build client: %w", cl.Name, err) } - return clusterClients{kube: kube}, nil + return clusterClients{kube: kube, ec2: ec2.NewFromConfig(awsCfg)}, nil } // eksTokenSource mints and caches the bearer token for one cluster. diff --git a/pkg/provider/kubernetes/eks_offerings_test.go b/pkg/provider/kubernetes/eks_offerings_test.go new file mode 100644 index 0000000..71abff8 --- /dev/null +++ b/pkg/provider/kubernetes/eks_offerings_test.go @@ -0,0 +1,109 @@ +/* +Copyright 2026 The InftyAI Team. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + "errors" + "slices" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + kubefake "k8s.io/client-go/kubernetes/fake" + + "github.com/InftyAI/Nebula/pkg/provider" +) + +// fakeEC2 answers DescribeInstanceTypeOfferings from types, recording the filter it was given. +type fakeEC2 struct { + types []string + err error + filter []string +} + +func (f *fakeEC2) DescribeInstanceTypeOfferings(_ context.Context, in *ec2.DescribeInstanceTypeOfferingsInput, + _ ...func(*ec2.Options)) (*ec2.DescribeInstanceTypeOfferingsOutput, error) { + f.filter = in.Filters[0].Values + if f.err != nil { + return nil, f.err + } + out := &ec2.DescribeInstanceTypeOfferingsOutput{} + for _, t := range f.types { + out.InstanceTypeOfferings = append(out.InstanceTypeOfferings, + ec2types.InstanceTypeOffering{InstanceType: ec2types.InstanceType(t)}) + } + return out, nil +} + +func TestOfferedFamilies(t *testing.T) { + // us-west-1 as it is: g4dn and g6, but no p4d. + api := &fakeEC2{types: []string{"g4dn.xlarge", "g4dn.metal", "g6.xlarge"}} + got, err := offeredFamilies(context.Background(), api, []string{"p4d", "g4dn"}) + if err != nil { + t.Fatal(err) + } + if got["p4d"] || !got["g4dn"] { + t.Fatalf("offered = %v, want g4dn and not p4d", got) + } + if !slices.Equal(api.filter, []string{"p4d.*", "g4dn.*"}) { + t.Fatalf("filter = %v, want one pattern per family", api.filter) + } +} + +// Each cluster keeps only what its region offers (H100 is p5, L4 g6 or gr6 here), which is +// what lets placement skip a region rather than fail there. A failed lookup keeps all. +func TestNewKeepsTheFamiliesEachRegionOffers(t *testing.T) { + clients := map[string]clusterClients{ + "eks-use1": {kube: kubefake.NewClientset(), ec2: &fakeEC2{types: []string{"p5.48xlarge", "g6.xlarge"}}}, + "eks-eun1": {kube: kubefake.NewClientset(), ec2: &fakeEC2{types: []string{"g6.xlarge"}}}, + "eks-usw2": {kube: kubefake.NewClientset(), ec2: &fakeEC2{err: errors.New("throttled")}}, + } + p := newTestProvider(clients) + + for _, tc := range []struct { + accel, region string + want bool + }{ + {"H100", "us-east-1", true}, + {"H100", "eu-north-1", false}, // no p5 there + {"H100", "us-west-2", true}, // the lookup failed, so nothing was dropped + {"L4", "eu-north-1", true}, // g6 alone is enough + {"L4", "nowhere-1", false}, // no cluster in that region + } { + if got := p.ServesAcceleratorIn(tc.accel, tc.region); got != tc.want { + t.Errorf("ServesAcceleratorIn(%s, %s) = %v, want %v", tc.accel, tc.region, got, tc.want) + } + } + if fs := p.clusters["eu-north-1"].families["L4"]; !slices.Equal(fs, []string{"g6"}) { + t.Errorf("eu-north-1 L4 families = %v, want [g6]: gr6 is not offered there", fs) + } + // Provider-wide, H100 is still offered, by the clusters that have it. + if _, ok := p.MapAccelerator("H100", 1); !ok { + t.Error("MapAccelerator(H100) = false, want true while some cluster offers it") + } + + none := newTestProvider(map[string]clusterClients{ + "eks-use1": {kube: kubefake.NewClientset(), ec2: &fakeEC2{}}, + "eks-eun1": {kube: kubefake.NewClientset(), ec2: &fakeEC2{}}, + "eks-usw2": {kube: kubefake.NewClientset(), ec2: &fakeEC2{}}, + }) + if _, ok := none.MapAccelerator("H100", 1); ok { + t.Error("MapAccelerator(H100) = true with no region offering it, want false") + } + var _ provider.RegionAccelerators = none +} diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index 6625daf..a7eb017 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -58,7 +58,10 @@ const ( cleanupTimeout = 10 * time.Second ) -var _ provider.Provider = (*Provider)(nil) +var ( + _ provider.Provider = (*Provider)(nil) + _ provider.RegionAccelerators = (*Provider)(nil) +) type cluster struct { cfg ClusterConfig @@ -103,12 +106,14 @@ func NewFromFile(ctx context.Context, path string, costLabels []string) (*Provid } clients[cl.Name] = cc } - return New(cfg.Clusters, clients, cat, costLabels), nil + return New(ctx, cfg.Clusters, clients, cat, costLabels), nil } // New returns a Provider over already-built clients, one per cluster in clusters, which -// must already be validated (see Config.Validate). -func New(clusters []ClusterConfig, clients map[string]clusterClients, cat catalog.Lookup, +// must already be validated (see Config.Validate). Each cluster keeps only the families its +// region offers, read once here: what a region offers changes in months, and a restart +// picks a change up. +func New(ctx context.Context, clusters []ClusterConfig, clients map[string]clusterClients, cat catalog.Lookup, costLabels []string) *Provider { p := &Provider{ clusters: make(map[string]cluster, len(clusters)), @@ -118,6 +123,9 @@ func New(clusters []ClusterConfig, clients map[string]clusterClients, cat catalo } for _, cl := range clusters { families := familiesByAccelerator(cat.Offerings(clusterCatalogs[cl.Provider])) + if api := clients[cl.Name].ec2; api != nil { + families = inRegion(ctx, cl, api, families) + } p.clusters[cl.Region] = cluster{ cfg: cl, clients: clients[cl.Name], nodeAutoscaler: nodeAutoscalers[cl.nodeAutoscalerName()], families: families, @@ -154,6 +162,31 @@ func familiesByAccelerator(rows []provider.Offering) map[string][]string { return out } +// inRegion keeps the families of each accelerator that cl's region offers, dropping an +// accelerator with none left. A failed lookup keeps them all: a cluster wrongly ruled out is +// worse than one refused once. +func inRegion( + ctx context.Context, cl ClusterConfig, api describeOfferings, byAccel map[string][]string, +) map[string][]string { + var all []string + for _, fs := range byAccel { + all = append(all, fs...) + } + offered, err := offeredFamilies(ctx, api, all) + if err != nil { + logf.FromContext(ctx).Info("kubernetes: could not read the region's instance families; keeping all", + "cluster", cl.Name, "region", cl.Region, "error", err.Error()) + return byAccel + } + out := make(map[string][]string, len(byAccel)) + for accel, fs := range byAccel { + if kept := slices.DeleteFunc(slices.Clone(fs), func(f string) bool { return !offered[f] }); len(kept) > 0 { + out[accel] = kept + } + } + return out +} + // Name implements provider.Provider. func (p *Provider) Name() string { return provider.ProviderKubernetes } @@ -182,6 +215,17 @@ func (p *Provider) MapAccelerator(canonical string, _ int32) ([]string, bool) { return slices.Clone(fs), ok } +// ServesAcceleratorIn implements provider.RegionAccelerators: whether the cluster in region +// has the accelerator's families, so placement skips one whose region does not offer them. +func (p *Provider) ServesAcceleratorIn(canonical, region string) bool { + cl, ok := p.clusters[region] + if !ok { + return false + } + _, ok = cl.families[strings.ToUpper(canonical)] + return ok +} + // Provision implements provider.Provider: create the real pod in the cluster req.Region // names, then wait for capacity. A pod that gets none is deleted before returning, or // Karpenter would launch a node for it later. diff --git a/pkg/provider/kubernetes/kubernetes_test.go b/pkg/provider/kubernetes/kubernetes_test.go index b2862d0..d61564d 100644 --- a/pkg/provider/kubernetes/kubernetes_test.go +++ b/pkg/provider/kubernetes/kubernetes_test.go @@ -63,7 +63,7 @@ func newTestProvider(clients map[string]clusterClients) *Provider { } all[cl.Name] = cc } - return New(testClusters, all, testCatalog, []string{"org-id"}) + return New(context.Background(), testClusters, all, testCatalog, []string{"org-id"}) } // placeholder is a Pod as Nebula's webhook and placement leave it. @@ -310,7 +310,7 @@ func TestFamiliesPerCluster(t *testing.T) { {Name: "l4", Provider: "l4cloud", Region: "us-east-2", Geography: "us"}, } tc := newTestCluster(t, true) - p := New(clusters, map[string]clusterClients{ + p := New(context.Background(), clusters, map[string]clusterClients{ "eks-use1": {kube: kubefake.NewClientset()}, "l4": tc.clients(), }, cat, nil) diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 0a25ade..372ede5 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -167,6 +167,14 @@ type LogStreamer interface { Logs(ctx context.Context, instanceID string) (io.ReadCloser, error) } +// RegionAccelerators is OPTIONAL, for a provider whose regions do not all offer the same +// accelerators. MapAccelerator answers for the provider as a whole; placement asks this per +// region and skips one that cannot serve the request, rather than learning so from a failed +// placement. A provider without it serves in every region what MapAccelerator says. +type RegionAccelerators interface { + ServesAcceleratorIn(canonical, region string) bool +} + // Executor is the other OPTIONAL half: run one command inside a live instance, which is // what makes `kubectl exec` work. A backend with no way in (no agent, no SSH key) does // not implement it, and the virtual node answers NotFound. From 051065febe598246b09a944e490b2c2a9aeb4ec4 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Sat, 24 Oct 2026 19:21:38 +0100 Subject: [PATCH 15/21] add half open mechanism Signed-off-by: kerthcet --- .../controller/pod_placement_controller.go | 15 +-- .../pod_placement_controller_test.go | 46 ++++++- internal/controller/pod_placement_helpers.go | 16 +-- pkg/failover/blocklist.go | 115 ++++++++++++------ pkg/failover/blocklist_test.go | 98 ++++++++++++++- 5 files changed, 232 insertions(+), 58 deletions(-) diff --git a/internal/controller/pod_placement_controller.go b/internal/controller/pod_placement_controller.go index 533c882..d652edd 100644 --- a/internal/controller/pod_placement_controller.go +++ b/internal/controller/pod_placement_controller.go @@ -77,14 +77,15 @@ type PodPlacementReconciler struct { Blocklist Blocklister } -// Blocklister is the read side of pkg/failover.Blocklist the placement controller -// depends on: it asks whether a candidate placement is currently excluded and, -// when it is, how long until it frees (so a Pod stuck on failover can requeue for -// the exact moment a block lapses rather than idling until the periodic resync). -// The concrete type is injected from main; this narrow interface keeps the -// controller decoupled and a nil value a no-op. +// Blocklister is the side of pkg/failover.Blocklist the placement controller depends on: +// whether a candidate placement may be tried now and, when not, how long until it could +// be (so a Pod stuck on failover can requeue for the exact moment a block lapses rather +// than idling until the periodic resync). Admitting a lapsed candidate claims its one +// probe (see failover.Blocklist.Admit), so only a placement about to be made may ask. +// The concrete type is injected from main; this narrow interface keeps the controller +// decoupled and a nil value a no-op. type Blocklister interface { - BlockedUntil(c failover.Candidate) (retryAfter time.Duration, blocked bool) + Admit(c failover.Candidate) (retryAfter time.Duration, admitted bool) } // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;update;patch;delete diff --git a/internal/controller/pod_placement_controller_test.go b/internal/controller/pod_placement_controller_test.go index 2756aec..41a93e6 100644 --- a/internal/controller/pod_placement_controller_test.go +++ b/internal/controller/pod_placement_controller_test.go @@ -18,6 +18,7 @@ package controller import ( "context" + "fmt" "slices" "strings" "testing" @@ -50,7 +51,12 @@ type fakeBlocklist struct { until time.Duration } -func (b *fakeBlocklist) BlockedUntil(c failover.Candidate) (time.Duration, bool) { +func (b *fakeBlocklist) Admit(c failover.Candidate) (time.Duration, bool) { + until, blocked := b.blockedUntil(c) + return until, !blocked +} + +func (b *fakeBlocklist) blockedUntil(c failover.Candidate) (time.Duration, bool) { for _, e := range b.blocked { if e.Provider != "" && e.Provider != c.Provider { continue @@ -561,6 +567,44 @@ func TestPlacement_SkipsRegionWithoutTheAccelerator(t *testing.T) { } } +// When a block lapses, the Pods waiting on it do not all go: the first is the probe and is +// placed, the others stay gated and requeue, so a candidate still out costs one attempt. +func TestPlacement_LapsedBlockAdmitsOneProbe(t *testing.T) { + pool := poolWithRegions("pool-a", []nebulav1alpha1.CapacityType{nebulav1alpha1.CapacityOnDemand}, + provider.ProviderModal, "us-east-1") + objs := make([]client.Object, 0, 6) + objs = append(objs, pool) + for i := range 5 { + objs = append(objs, gatedPod(fmt.Sprintf("p%d", i), "default", fmt.Sprintf("uid-%d", i), "pool-a", "H100")) + } + r, c := newPlacementReconciler(t, objs, &fakeProvider{name: provider.ProviderModal, gpus: []string{"H100"}}) + bl := failover.New() + bl.Record(provider.ProviderModal, provider.BlockScope{Deny: provider.DenyCandidate, Accelerator: "H100:1", + CapacityType: nebulav1alpha1.CapacityOnDemand, Region: "us-east-1"}, time.Millisecond) + time.Sleep(5 * time.Millisecond) // the block lapses: half-open + r.Blocklist = bl + + placed := 0 + for i := range 5 { + res, err := r.Reconcile(context.Background(), reconcile.Request{ + NamespacedName: types.NamespacedName{Namespace: "default", Name: fmt.Sprintf("p%d", i)}, + }) + if err != nil { + t.Fatalf("reconcile p%d: %v", i, err) + } + var nc nebulav1alpha1.NodeClaim + switch err := c.Get(context.Background(), types.NamespacedName{Name: fmt.Sprintf("default-p%d", i)}, &nc); { + case err == nil: + placed++ + case res.RequeueAfter <= 0: + t.Errorf("p%d was held back without a requeue", i) + } + } + if placed != 1 { + t.Fatalf("placed %d Pods against the lapsed block, want 1, the probe", placed) + } +} + func TestRequestedGeographies(t *testing.T) { cases := []struct { name string diff --git a/internal/controller/pod_placement_helpers.go b/internal/controller/pod_placement_helpers.go index b6934bb..fc928fe 100644 --- a/internal/controller/pod_placement_helpers.go +++ b/internal/controller/pod_placement_helpers.go @@ -276,22 +276,24 @@ func requestedGeographies(pod *corev1.Pod) (narrowTo []string, ok bool) { return out, len(out) > 0 } -// blockedUntil reports whether the (provider, accelerator, tier, region) -// candidate is currently excluded by the failover blocklist and, if so, how long -// until it frees (for the requeue hint). accelerator is the request's pool -// identity (type:count; see selectPlacement), so a capacity block matches only -// candidates on the same pool. It is nil-safe: with no blocklist wired (tests, or -// a blocklist-less build) nothing is ever blocked. +// blockedUntil reports whether the failover blocklist holds the (provider, accelerator, +// tier, region) candidate back and, if so, how long until it frees (for the requeue hint). +// A false result admits the candidate, as a lapsed block's probe if it is one, so call it +// only for the candidate about to be selected. accelerator is the request's pool identity +// (type:count; see selectPlacement), so a capacity block matches only candidates on the +// same pool. It is nil-safe: with no blocklist wired (tests, or a blocklist-less build) +// nothing is ever blocked. func (r *PodPlacementReconciler) blockedUntil(provName, accelerator string, tier nebulav1alpha1.CapacityType, region string) (time.Duration, bool) { if r.Blocklist == nil { return 0, false } - return r.Blocklist.BlockedUntil(failover.Candidate{ + until, admitted := r.Blocklist.Admit(failover.Candidate{ Provider: provName, Accelerator: accelerator, CapacityType: tier, Region: region, }) + return until, !admitted } // requeueJitter maps a Pod UID to a stable offset in [0, blockRequeueJitter) to diff --git a/pkg/failover/blocklist.go b/pkg/failover/blocklist.go index 99b08ce..70cf39a 100644 --- a/pkg/failover/blocklist.go +++ b/pkg/failover/blocklist.go @@ -24,6 +24,12 @@ limitations under the License. // disqualifies an OnDemand request, another accelerator, or another region — while an // auth failure (DenyAll) blocks the whole provider until its TTL lapses. // +// A lapsed block is HALF-OPEN rather than gone: the first placement to ask is admitted as +// a probe (see Admit), and the candidate stays blocked for everyone else for ProbeWindow. +// A refused probe is recorded like any failure and blocks again; a ProbeWindow with no +// failure means it was not refused, and the candidate opens. So a candidate that is still +// out costs one attempt per expiry, not one per waiting Pod. +// // In-memory and per-process on purpose: this is a hint that bounds churn, not durable // state. After a restart it is empty and the next attempt re-probes the provider, // costing at worst one rejected attempt that is then re-recorded. No blocklist CRD, no @@ -40,12 +46,32 @@ import ( "github.com/InftyAI/Nebula/pkg/provider" ) -// entry is one recorded block: the provider it applies to, the scope that failed, -// and when the block lapses. It is compared against a query candidate by Blocked. +// ProbeWindow is how long a half-open candidate's one probe has to be refused before the +// candidate opens for everyone. Long enough for most refusals: a synchronous provider's +// (AWS, Modal) fails Provision itself, and Karpenter's arrives within ~3-8s, poll included, +// while it still remembers the shortage. One that rechecks EC2 can take ~30s; such a late +// refusal only lets the waiting Pods in as before, never worse. +const ProbeWindow = 15 * time.Second + +// entry is one recorded block: the provider it applies to, the scope that failed, when the +// block lapses, and, once half-open, until when its admitted probe holds the others back. type entry struct { - provider string - scope provider.BlockScope - expiresAt time.Time + provider string + scope provider.BlockScope + expiresAt time.Time + probeUntil time.Time // zero until a probe is admitted +} + +// heldUntil is when e stops holding candidates back: its expiry while blocked, its probe +// window while probing, zero once half-open with no probe admitted. +func (e entry) heldUntil(now time.Time) time.Time { + switch { + case now.Before(e.expiresAt): + return e.expiresAt + case now.Before(e.probeUntil): + return e.probeUntil + } + return time.Time{} } // Candidate is a placement being considered — the (provider, accelerator, tier, @@ -89,11 +115,10 @@ func newWithClock(c clock.Clock) *Blocklist { // permanent, provider-less or unmatchable block. Recording also drops expired entries, // bounding the slice. // -// Records COALESCE: an existing live entry with the same provider and an identical -// scope has its expiry extended (never shortened) instead of gaining a duplicate. So -// many Pods failing for the same reason produce ONE entry — the slice is bounded by -// distinct live scopes, not failure volume. Behaviour is unchanged, since Blocked/ -// BlockedUntil already used the latest covering expiry. +// Records COALESCE: an existing entry with the same provider and an identical scope has its +// expiry extended (never shortened) instead of gaining a duplicate. So many Pods failing for +// the same reason produce ONE entry — the slice is bounded by distinct live scopes, not +// failure volume. A half-open entry, its probe refused, is blocked again for ttl. func (b *Blocklist) Record(prov string, scope provider.BlockScope, ttl time.Duration) { if prov == "" || ttl <= 0 || !knownPolicy(scope.Deny) { return @@ -105,13 +130,12 @@ func (b *Blocklist) Record(prov string, scope provider.BlockScope, ttl time.Dura b.gcLocked(now) expiresAt := now.Add(ttl) - // Coalesce into an existing live entry for the same scope. gcLocked has already - // dropped expired entries, so every entry scanned here is still live. for i := range b.entries { - if b.entries[i].provider == prov && b.entries[i].scope == scope { - if expiresAt.After(b.entries[i].expiresAt) { - b.entries[i].expiresAt = expiresAt // extend; never shorten a live block + if e := &b.entries[i]; e.provider == prov && e.scope == scope { + if expiresAt.After(e.expiresAt) { + e.expiresAt = expiresAt // extend; never shorten a live block } + e.probeUntil = time.Time{} return } } @@ -122,58 +146,69 @@ func (b *Blocklist) Record(prov string, scope provider.BlockScope, ttl time.Dura }) } -// Blocked reports whether c is currently excluded by any live entry. It is the -// query the placement loop makes before selecting a candidate: a true result means -// "skip this (provider, accelerator, tier, region) and try the next one". +// Blocked reports whether c is currently excluded, without admitting a probe: a half-open +// candidate reads as not blocked. Placement asks Admit instead. func (b *Blocklist) Blocked(c Candidate) bool { + _, blocked := b.BlockedUntil(c) + return blocked +} + +// BlockedUntil reports whether c is currently excluded and, if so, how long until it could +// become servable: until the LATEST covering entry stops holding it (its expiry, or its +// probe's window). It never admits a probe, so it is safe for anything but placement. +func (b *Blocklist) BlockedUntil(c Candidate) (retryAfter time.Duration, blocked bool) { b.mu.Lock() defer b.mu.Unlock() - now := b.clock.Now() b.gcLocked(now) - for i := range b.entries { - if b.entries[i].provider == c.Provider && scopeCovers(b.entries[i].scope, c) { - return true - } - } - return false + return b.heldLocked(c, now) } -// BlockedUntil reports whether c is currently excluded and, if so, how long until -// it could become servable — the duration until the LATEST covering entry lapses -// (c stays blocked while ANY covering entry is live, so it frees only when the -// last one expires). Placement uses this to requeue a gated Pod exactly when a -// candidate frees, instead of leaving it idle until the periodic resync. A false -// result (retryAfter 0) means c is not blocked now. -func (b *Blocklist) BlockedUntil(c Candidate) (retryAfter time.Duration, blocked bool) { +// Admit is the placement loop's query: whether c may be tried now, and if not, how long +// until it could be. A candidate held back by any covering entry is not admitted. One that +// is half-open is admitted as the probe: every covering half-open entry then holds the +// others back for ProbeWindow, until a refusal (Record) blocks it again or the window ends. +func (b *Blocklist) Admit(c Candidate) (retryAfter time.Duration, admitted bool) { b.mu.Lock() defer b.mu.Unlock() - now := b.clock.Now() b.gcLocked(now) - var latest time.Time + if until, held := b.heldLocked(c, now); held { + return until, false + } for i := range b.entries { - if b.entries[i].provider == c.Provider && scopeCovers(b.entries[i].scope, c) { - if b.entries[i].expiresAt.After(latest) { - latest = b.entries[i].expiresAt + if e := &b.entries[i]; e.provider == c.Provider && scopeCovers(e.scope, c) { + e.probeUntil = now.Add(ProbeWindow) + } + } + return 0, true +} + +func (b *Blocklist) heldLocked(c Candidate, now time.Time) (time.Duration, bool) { + var latest time.Time + for _, e := range b.entries { + if e.provider == c.Provider && scopeCovers(e.scope, c) { + if until := e.heldUntil(now); until.After(latest) { + latest = until } } } if latest.IsZero() { return 0, false } - // gcLocked has dropped every expired entry, so latest is strictly after now. return latest.Sub(now), true } -// gcLocked drops entries that have expired as of now. Caller holds b.mu. +// gcLocked drops the entries whose probe window passed without a refusal: the candidate is +// open. An unprobed half-open entry stays, so the next placement probes it; entries stay +// bounded by distinct scopes. Caller holds b.mu. func (b *Blocklist) gcLocked(now time.Time) { if len(b.entries) == 0 { return } kept := b.entries[:0] for _, e := range b.entries { - if e.expiresAt.After(now) { + if e.probeUntil.IsZero() || now.Before(e.probeUntil) { kept = append(kept, e) } } diff --git a/pkg/failover/blocklist_test.go b/pkg/failover/blocklist_test.go index 747e91d..14c6928 100644 --- a/pkg/failover/blocklist_test.go +++ b/pkg/failover/blocklist_test.go @@ -301,12 +301,104 @@ func TestRecord_ExpiredEntriesAreGarbageCollected(t *testing.T) { bl := newWithClock(fc) bl.Record("aws", spotH100EastScope(), time.Minute) - // Let the first entry expire, then record another: the GC on Record should drop - // the stale one so the slice does not grow unbounded. - fc.Step(2 * time.Minute) + // Let the first entry expire and its probe pass unrefused, then record another: the GC + // on Record should drop the stale one so the slice stays bounded. + fc.Step(time.Minute) + bl.Admit(cand("aws", "H100", spot, "us-east-1")) + fc.Step(ProbeWindow) bl.Record("aws", provider.BlockScope{Deny: provider.DenyAll}, time.Minute) if n := len(bl.entries); n != 1 { t.Fatalf("len(entries) = %d, want 1 (stale entry should be GC'd on Record)", n) } } + +// A lapsed block admits ONE probe; every other caller is held back for ProbeWindow, so a +// candidate still out costs one attempt per expiry instead of one per waiting Pod. +func TestAdmit_HalfOpenAdmitsOneProbe(t *testing.T) { + fc := testclock.NewFakeClock(baseTime) + bl := newWithClock(fc) + c := cand("aws", "H100", spot, "us-east-1") + bl.Record("aws", spotH100EastScope(), 30*time.Second) + + if until, ok := bl.Admit(c); ok || until != 30*time.Second { + t.Fatalf("while blocked: Admit = (%v, %v), want (30s, false)", until, ok) + } + fc.Step(30 * time.Second) + if _, ok := bl.Admit(c); !ok { + t.Fatal("the first Admit after the block lapsed must be the probe") + } + for i := 0; i < 3; i++ { + if until, ok := bl.Admit(c); ok || until != ProbeWindow { + t.Fatalf("Admit %d during the probe = (%v, %v), want (%v, false)", i, until, ok, ProbeWindow) + } + } + if !bl.Blocked(c) { + t.Error("a probing candidate must read as blocked to everyone else") + } +} + +// A refused probe is recorded like any failure and blocks the candidate again for its TTL. +func TestAdmit_RefusedProbeBlocksAgain(t *testing.T) { + fc := testclock.NewFakeClock(baseTime) + bl := newWithClock(fc) + c := cand("aws", "H100", spot, "us-east-1") + bl.Record("aws", spotH100EastScope(), 30*time.Second) + fc.Step(30 * time.Second) + if _, ok := bl.Admit(c); !ok { + t.Fatal("want the probe admitted") + } + + fc.Step(15 * time.Second) // the node autoscaler refuses the probe + bl.Record("aws", spotH100EastScope(), 30*time.Second) + if until, ok := bl.Admit(c); ok || until != 30*time.Second { + t.Fatalf("after the probe was refused: Admit = (%v, %v), want (30s, false)", until, ok) + } + // And the cycle repeats: one probe once that block lapses. + fc.Step(30 * time.Second) + if _, ok := bl.Admit(c); !ok { + t.Fatal("want the next probe admitted once the renewed block lapsed") + } +} + +// A probe not refused within ProbeWindow was let through, so the candidate opens for all. +func TestAdmit_UnrefusedProbeOpensTheCandidate(t *testing.T) { + fc := testclock.NewFakeClock(baseTime) + bl := newWithClock(fc) + c := cand("aws", "H100", spot, "us-east-1") + bl.Record("aws", spotH100EastScope(), 30*time.Second) + fc.Step(30 * time.Second) + bl.Admit(c) + + fc.Step(ProbeWindow) + for i := 0; i < 3; i++ { + if _, ok := bl.Admit(c); !ok { + t.Fatalf("Admit %d after an unrefused probe = false, want the candidate open", i) + } + } + if n := len(bl.entries); n != 0 { + t.Fatalf("len(entries) = %d, want 0 once the probe window passed", n) + } +} + +// Only Admit claims the probe: a read-only query of a half-open candidate sees it as not +// blocked and leaves the probe for placement. +func TestBlockedUntil_NeverAdmitsAProbe(t *testing.T) { + fc := testclock.NewFakeClock(baseTime) + bl := newWithClock(fc) + c := cand("aws", "H100", spot, "us-east-1") + bl.Record("aws", spotH100EastScope(), 30*time.Second) + fc.Step(30 * time.Second) + + for i := 0; i < 3; i++ { + if until, blocked := bl.BlockedUntil(c); blocked || until != 0 { + t.Fatalf("BlockedUntil %d of a half-open candidate = (%v, %v), want (0, false)", i, until, blocked) + } + } + if _, ok := bl.Admit(c); !ok { + t.Fatal("the probe must still be available to Admit") + } + if _, ok := bl.Admit(cand("aws", "A100", spot, "us-east-1")); !ok { + t.Fatal("a candidate no entry covers is always admitted") + } +} From 267f8dde0d207a35f17cef1e6dc88d84bbd34f9e Mon Sep 17 00:00:00 2001 From: kerthcet Date: Sat, 24 Oct 2026 21:19:41 +0100 Subject: [PATCH 16/21] fix pod not ready bug Signed-off-by: kerthcet --- pkg/provider/kubernetes/kubernetes.go | 4 +++- pkg/provider/kubernetes/kubernetes_test.go | 8 +++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index a7eb017..d8db6fc 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -622,7 +622,9 @@ func toState(pod *corev1.Pod) provider.InstanceState { switch pod.Status.Phase { case corev1.PodRunning: // Running only says the containers started; Ready is what makes the workload available. - if podReady(pod) { + // A terminating pod is not available whatever its Ready says: it can read True through + // the whole grace period while its own cluster already routes nothing to it. + if podReady(pod) && pod.DeletionTimestamp == nil { return provider.InstanceRunning } return provider.InstanceNotReady diff --git a/pkg/provider/kubernetes/kubernetes_test.go b/pkg/provider/kubernetes/kubernetes_test.go index d61564d..834ef07 100644 --- a/pkg/provider/kubernetes/kubernetes_test.go +++ b/pkg/provider/kubernetes/kubernetes_test.go @@ -424,7 +424,13 @@ func TestToStateNeedsReady(t *testing.T) { "ready": {running(corev1.ConditionTrue), provider.InstanceRunning}, "probe failing": {running(corev1.ConditionFalse), provider.InstanceNotReady}, "no condition yet": {running(""), provider.InstanceNotReady}, - "unbound": {&corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending}}, provider.InstanceProvisioning}, + "terminating, still ready": {func() *corev1.Pod { + pod := running(corev1.ConditionTrue) + now := metav1.Now() + pod.DeletionTimestamp = &now + return pod + }(), provider.InstanceNotReady}, + "unbound": {&corev1.Pod{Status: corev1.PodStatus{Phase: corev1.PodPending}}, provider.InstanceProvisioning}, "bound, starting": {&corev1.Pod{ Spec: corev1.PodSpec{NodeName: "node-1"}, Status: corev1.PodStatus{Phase: corev1.PodPending}, }, provider.InstancePending}, From 67a912f7b5b1f99b28b2c4c40e261d838545509a Mon Sep 17 00:00:00 2001 From: kerthcet Date: Sat, 24 Oct 2026 21:27:06 +0100 Subject: [PATCH 17/21] fix recreate pod error Signed-off-by: kerthcet --- pkg/provider/kubernetes/kubernetes.go | 2 -- pkg/vnode/handler.go | 8 +++++++ pkg/vnode/handler.go.rej | 24 +++++++++++++++++++++ pkg/vnode/handler_test.go | 30 ++++++++++++++++++++++++++- pkg/vnode/handler_test.go.rej | 13 ++++++++++++ 5 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 pkg/vnode/handler.go.rej create mode 100644 pkg/vnode/handler_test.go.rej diff --git a/pkg/provider/kubernetes/kubernetes.go b/pkg/provider/kubernetes/kubernetes.go index d8db6fc..0cd8ebc 100644 --- a/pkg/provider/kubernetes/kubernetes.go +++ b/pkg/provider/kubernetes/kubernetes.go @@ -425,8 +425,6 @@ func (p *Provider) listPods(ctx context.Context, cl cluster, fieldSelector strin list, err := cl.clients.kube.CoreV1().Pods(metav1.NamespaceAll).List(ctx, metav1.ListOptions{ LabelSelector: labels.Set(ownerLabels()).String(), FieldSelector: fieldSelector, - // Served from the API server's watch cache rather than etcd. - ResourceVersion: "0", }) if err != nil { return nil, fmt.Errorf("kubernetes: list pods in cluster %q: %w", cl.cfg.Name, classifyAPI(err)) diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index 0fd35ae..b614764 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -117,6 +117,11 @@ type Handler struct { // provision is wrong and unrecoverable. tracked map[string]*trackedPod + // storeSeq numbers stores into tracked, only so a poll tick can tell which pods were + // stored after its List began: such a pod can be missing from that List while alive, + // so the List is no verdict on it (see reconcileOnce). + storeSeq uint64 + notify func(*corev1.Pod) // nowFn, pollEvery and readyDeadline are seams for tests. readyDeadline is @@ -137,6 +142,7 @@ type trackedPod struct { pod *corev1.Pod claimName string instance string + storeSeq uint64 // see Handler.storeSeq // patchedMeta is what VK has already patched onto the Pod in the API server. notify // fires every poll tick for the pod's whole life, and the pod it hands us is our own // stamped copy — which always carries these values, so it cannot say whether they ever @@ -690,6 +696,8 @@ func (h *Handler) reconcileOnce(ctx context.Context) { // No reverse for a terminal or deleting pod. case util.Finished(tp.pod.Status.Phase) || !tp.pod.DeletionTimestamp.IsZero(): finished++ + case !present && tp.storeSeq > listSeq: + // Stored while List was in flight; the next tick decides. case !present: gone++ applyState(tp.pod, provider.InstanceTerminated, "", h.nowFn()) diff --git a/pkg/vnode/handler.go.rej b/pkg/vnode/handler.go.rej new file mode 100644 index 0000000..74ee71d --- /dev/null +++ b/pkg/vnode/handler.go.rej @@ -0,0 +1,24 @@ +diff a/pkg/vnode/handler.go b/pkg/vnode/handler.go (rejected hunks) +@@ -680,6 +686,9 @@ func (h *Handler) pollLoop(ctx context.Context) { + // List always fails would otherwise strand every pod in its last status with no signal. + func (h *Handler) reconcileOnce(ctx context.Context) { + log := logf.FromContext(ctx).WithName("vnode-poll").WithValues("provider", h.prov.Name()) ++ h.mu.Lock() ++ listSeq := h.storeSeq ++ h.mu.Unlock() + instances, err := h.prov.List(ctx) + // This tick doubles as the process's credential probe: it runs on a cadence and goes + // through the provider's auth path, so a token that dies mid-life shows up here every +@@ -990,10 +1001,12 @@ func (h *Handler) store( + ) { + h.mu.Lock() + defer h.mu.Unlock() ++ h.storeSeq++ + h.tracked[key(pod.Namespace, pod.Name)] = &trackedPod{ + pod: pod.DeepCopy(), + claimName: claim, + instance: instance, ++ storeSeq: h.storeSeq, + provisioningAt: provisioningAt, + placement: place, + } diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go index 87cb35b..b7acf3f 100644 --- a/pkg/vnode/handler_test.go +++ b/pkg/vnode/handler_test.go @@ -124,7 +124,11 @@ func (f *fakeProvider) Get(context.Context, string, string) (*provider.Instance, return nil, nil } func (f *fakeProvider) List(context.Context) ([]provider.Instance, error) { - return f.list, f.listErr + list, err := f.list, f.listErr + if f.listHook != nil { + f.listHook() + } + return list, err } func (f *fakeProvider) FindByClaim(_ context.Context, claim, _ string) (*provider.Instance, error) { if f.listErr != nil { @@ -1508,6 +1512,30 @@ func TestReconcileOnce_AbsentInstanceIsTerminated(t *testing.T) { } } +func TestReconcileOnce_PodStoredDuringListIsNotGone(t *testing.T) { + // The race seen live: a pod provisioned while List was in flight is missing from that + // snapshot, yet alive. Only a List begun after it was stored is a verdict on it. + fp := &fakeProvider{provisionID: "inst-1"} + h := NewHandler(fp, nil, nil, openCluster()) + fp.listHook = func() { + fp.listHook = nil + _ = h.CreatePod(context.Background(), testPod("default", "p1")) + } + h.reconcileOnce(context.Background()) + + got, _ := h.GetPod(context.Background(), "default", "p1") + if got.Status.Phase == corev1.PodFailed { + t.Fatalf("a pod stored during List was failed as gone: %q", got.Status.Reason) + } + + h.reconcileOnce(context.Background()) // begun after the store, and still missing it + got, _ = h.GetPod(context.Background(), "default", "p1") + if got.Status.Phase != corev1.PodFailed || got.Status.Reason != "Terminated" { + t.Fatalf("phase/reason = %q/%q, want Failed/Terminated once a later List misses it", + got.Status.Phase, got.Status.Reason) + } +} + func TestReconcileOnce_ReadinessDeadlineFailsPod(t *testing.T) { // An instance the provider keeps reporting Pending forever — Modal's probe never // reporting pass. Past the deadline the Pod goes terminal, which is the whole action: diff --git a/pkg/vnode/handler_test.go.rej b/pkg/vnode/handler_test.go.rej new file mode 100644 index 0000000..455c03e --- /dev/null +++ b/pkg/vnode/handler_test.go.rej @@ -0,0 +1,13 @@ +diff a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go (rejected hunks) +@@ -63,7 +63,10 @@ type fakeProvider struct { + terminateErr error + list []provider.Instance + listErr error +- capabilities provider.Capabilities ++ // listHook runs inside List, before it answers, so a test can act in the window in ++ // which the snapshot is already taken. ++ listHook func() ++ capabilities provider.Capabilities + // The outbound cut: how many landed, on which instance, plus a canned failure so the + // retry path can be driven. + egressCnt int From 8951582f16791935396ed3d168f0459edfede141 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Sun, 25 Oct 2026 00:00:14 +0100 Subject: [PATCH 18/21] fix recreate pod error Signed-off-by: kerthcet --- pkg/vnode/handler.go | 5 +++++ pkg/vnode/handler.go.rej | 24 ------------------------ pkg/vnode/handler_test.go | 5 ++++- pkg/vnode/handler_test.go.rej | 13 ------------- 4 files changed, 9 insertions(+), 38 deletions(-) delete mode 100644 pkg/vnode/handler.go.rej delete mode 100644 pkg/vnode/handler_test.go.rej diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index b614764..bb3b101 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -672,6 +672,9 @@ func (h *Handler) pollLoop(ctx context.Context) { // List always fails would otherwise strand every pod in its last status with no signal. func (h *Handler) reconcileOnce(ctx context.Context) { log := logf.FromContext(ctx).WithName("vnode-poll").WithValues("provider", h.prov.Name()) + h.mu.Lock() + listSeq := h.storeSeq + h.mu.Unlock() instances, err := h.prov.List(ctx) if err != nil { log.Error(err, "provider List failed; tracked pod statuses will not advance this tick") @@ -915,10 +918,12 @@ func statusSignature(pod *corev1.Pod) string { func (h *Handler) store(pod *corev1.Pod, claim, instance string, place *placement) { h.mu.Lock() defer h.mu.Unlock() + h.storeSeq++ h.tracked[key(pod.Namespace, pod.Name)] = &trackedPod{ pod: pod.DeepCopy(), claimName: claim, instance: instance, + storeSeq: h.storeSeq, placement: place, } } diff --git a/pkg/vnode/handler.go.rej b/pkg/vnode/handler.go.rej deleted file mode 100644 index 74ee71d..0000000 --- a/pkg/vnode/handler.go.rej +++ /dev/null @@ -1,24 +0,0 @@ -diff a/pkg/vnode/handler.go b/pkg/vnode/handler.go (rejected hunks) -@@ -680,6 +686,9 @@ func (h *Handler) pollLoop(ctx context.Context) { - // List always fails would otherwise strand every pod in its last status with no signal. - func (h *Handler) reconcileOnce(ctx context.Context) { - log := logf.FromContext(ctx).WithName("vnode-poll").WithValues("provider", h.prov.Name()) -+ h.mu.Lock() -+ listSeq := h.storeSeq -+ h.mu.Unlock() - instances, err := h.prov.List(ctx) - // This tick doubles as the process's credential probe: it runs on a cadence and goes - // through the provider's auth path, so a token that dies mid-life shows up here every -@@ -990,10 +1001,12 @@ func (h *Handler) store( - ) { - h.mu.Lock() - defer h.mu.Unlock() -+ h.storeSeq++ - h.tracked[key(pod.Namespace, pod.Name)] = &trackedPod{ - pod: pod.DeepCopy(), - claimName: claim, - instance: instance, -+ storeSeq: h.storeSeq, - provisioningAt: provisioningAt, - placement: place, - } diff --git a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go index b7acf3f..bfadf92 100644 --- a/pkg/vnode/handler_test.go +++ b/pkg/vnode/handler_test.go @@ -63,7 +63,10 @@ type fakeProvider struct { terminateErr error list []provider.Instance listErr error - capabilities provider.Capabilities + // listHook runs inside List, before it answers, so a test can act in the window in + // which the snapshot is already taken. + listHook func() + capabilities provider.Capabilities // classifyScope is what ClassifyProvisionError returns for a failure; the zero // value (empty scope) means "not blocklistable". classifyAccel/classifyRegion // record what the handler passed in, so a test can assert it resolved them off the diff --git a/pkg/vnode/handler_test.go.rej b/pkg/vnode/handler_test.go.rej deleted file mode 100644 index 455c03e..0000000 --- a/pkg/vnode/handler_test.go.rej +++ /dev/null @@ -1,13 +0,0 @@ -diff a/pkg/vnode/handler_test.go b/pkg/vnode/handler_test.go (rejected hunks) -@@ -63,7 +63,10 @@ type fakeProvider struct { - terminateErr error - list []provider.Instance - listErr error -- capabilities provider.Capabilities -+ // listHook runs inside List, before it answers, so a test can act in the window in -+ // which the snapshot is already taken. -+ listHook func() -+ capabilities provider.Capabilities - // The outbound cut: how many landed, on which instance, plus a canned failure so the - // retry path can be driven. - egressCnt int From 19e28dfe37699e6d73247bb5243a88f51ffe7434 Mon Sep 17 00:00:00 2001 From: kerthcet Date: Sun, 25 Oct 2026 00:02:30 +0100 Subject: [PATCH 19/21] fix modal timeout issue Signed-off-by: kerthcet --- pkg/provider/modal/client.go | 20 +++++++++++++++----- pkg/provider/modal/modal_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/pkg/provider/modal/client.go b/pkg/provider/modal/client.go index 137b0fd..bafd94d 100644 --- a/pkg/provider/modal/client.go +++ b/pkg/provider/modal/client.go @@ -327,16 +327,14 @@ func (c *sdkClient) registrySecret(ctx context.Context, kv map[string]string) (* // no read-back, so a dropped error loses the credential of a sandbox that exists and is // billing — silently, since a caller handed an empty pair has nothing to log. The Pod fails // terminally and its owner recreates it — the only recovery, since this sandbox can never be -// given a credential. The API error is passed through unlabelled, because it is what decides -// the blocklist scope: an "unauthorized" here is OUR Modal credential, so it must fence the -// provider off rather than be scoped to one request. +// given a credential. The API error is passed through, because it is what decides the +// blocklist scope (see mintError). func (c *sdkClient) mintCredential(ctx context.Context, sb *modal.Sandbox, port int) (Credential, error) { creds, err := sb.CreateConnectToken(ctx, &modal.SandboxCreateConnectTokenParams{ Port: port, }) if err != nil { - return Credential{}, fmt.Errorf("modal: mint connect credential for sandbox %s on port %d: %w", - sb.SandboxID, port, err) + return Credential{}, mintError(sb.SandboxID, port, err) } // A token-less success is the same outcome as an error — an address with nothing to // authenticate against it — so it is reported as one. Nothing to wrap: Modal reported @@ -348,6 +346,18 @@ func (c *sdkClient) mintCredential(ctx context.Context, sb *modal.Sandbox, port return Credential{URL: creds.URL, Token: creds.Token}, nil } +// mintError wraps a failed mint. A DeadlineExceeded is Modal's own cap on the wait for a worker +// running out (~3m43s), so no GPU came free: it is labelled a shortage, which blocks the +// candidate and lets replacement Pods fail over instead of queueing behind it again. Anything +// else stays unlabelled: an "unauthorized" here is OUR Modal credential, and must fence the +// provider off rather than be scoped to one request. +func mintError(sandboxID string, port int, err error) error { + if status.Code(err) == codes.DeadlineExceeded { + err = fmt.Errorf("%w: %w", provider.ErrNoCapacity, err) + } + return fmt.Errorf("modal: mint connect credential for sandbox %s on port %d: %w", sandboxID, port, err) +} + // modalProbe maps a Pod readinessProbe onto Modal's Probe. Modal supports only // TCP and Exec probes, so an HTTPGet probe degrades to a TCP probe on its port // (readiness ≈ the port accepting connections). Returns (nil, nil) when p is nil diff --git a/pkg/provider/modal/modal_test.go b/pkg/provider/modal/modal_test.go index b74a874..7c70dfd 100644 --- a/pkg/provider/modal/modal_test.go +++ b/pkg/provider/modal/modal_test.go @@ -28,6 +28,8 @@ import ( "testing" "time" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -1151,6 +1153,29 @@ func TestResolveRegions_RoundTripsThroughProvision(t *testing.T) { } } +func TestMintTimeoutBlocksTheCandidate(t *testing.T) { + // Modal gives up waiting for a worker with a server-side DeadlineExceeded, not our own + // context deadline. Unlabelled, it read as unattributable and blocked nothing, so every + // replacement Pod queued behind the same full Modal for another ~3m43s. + p := newTestProvider(&fakeClient{}) + timedOut := mintError("sb-1", 8080, + status.Error(codes.DeadlineExceeded, "Sandbox task ID not found within timeout")) + want := provider.BlockScope{ + Deny: provider.DenyCandidate, Accelerator: "A100-40GB:8", CapacityType: nebulav1alpha1.CapacityOnDemand, + } + if got := p.ClassifyProvisionError(timedOut, "A100-40GB:8", ""); got != want { + t.Fatalf("mint timeout: got %+v, want %+v", got, want) + } + + denied := mintError("sb-1", 8080, status.Error(codes.PermissionDenied, "token revoked")) + if errors.Is(denied, provider.ErrNoCapacity) { + t.Fatal("a mint failure other than the worker wait was labelled a shortage") + } + if got := p.ClassifyProvisionError(denied, "A100-40GB:8", ""); got.Deny != provider.DenyAll { + t.Fatalf("mint auth failure: got %+v, want DenyAll", got) + } +} + func TestClassifyProvisionError_ConfinesToFailingRegion(t *testing.T) { p := newTestProvider(&fakeClient{}) From 9d75d29c1df42878ce900efb5c0c59a7585e46ca Mon Sep 17 00:00:00 2001 From: kerthcet Date: Sun, 25 Oct 2026 00:03:56 +0100 Subject: [PATCH 20/21] change modal timeout to 1min Signed-off-by: kerthcet --- pkg/provider/modal/client.go | 8 ++++---- pkg/provider/modal/modal.go | 20 ++++++++------------ pkg/provider/modal/modal_test.go | 11 +++++------ 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/pkg/provider/modal/client.go b/pkg/provider/modal/client.go index bafd94d..5c0ce46 100644 --- a/pkg/provider/modal/client.go +++ b/pkg/provider/modal/client.go @@ -320,8 +320,8 @@ func (c *sdkClient) registrySecret(ctx context.Context, kv map[string]string) (* // // CreateConnectToken BLOCKS until Modal has assigned the sandbox a worker, and that wait is // load-bearing: it is what makes a returning CreateSandbox proof of a placed GPU rather than -// of an accepted request (see Provision). Modal caps it itself, at ~3m40s, so a queue longer -// than the cap fails a Pod whose sandbox would have been placed. +// of an accepted request (see Provision). It is bounded by provisionTimeout, and by Modal's +// own ~3m43s cap after the SDK's retries. // // A failure is REPORTED, never swallowed into a zero credential. Minting is one-shot with // no read-back, so a dropped error loses the credential of a sandbox that exists and is @@ -346,8 +346,8 @@ func (c *sdkClient) mintCredential(ctx context.Context, sb *modal.Sandbox, port return Credential{URL: creds.URL, Token: creds.Token}, nil } -// mintError wraps a failed mint. A DeadlineExceeded is Modal's own cap on the wait for a worker -// running out (~3m43s), so no GPU came free: it is labelled a shortage, which blocks the +// mintError wraps a failed mint. A DeadlineExceeded is the wait for a worker running out, +// whether Modal's own cap or ours, so no GPU came free: it is labelled a shortage, which blocks the // candidate and lets replacement Pods fail over instead of queueing behind it again. Anything // else stays unlabelled: an "unauthorized" here is OUR Modal credential, and must fence the // provider off rather than be scoped to one request. diff --git a/pkg/provider/modal/modal.go b/pkg/provider/modal/modal.go index b46c7b3..fcd1b67 100644 --- a/pkg/provider/modal/modal.go +++ b/pkg/provider/modal/modal.go @@ -70,16 +70,12 @@ import ( // different ceiling sets spec.activeDeadlineSeconds, which maps straight through. const defaultSandboxTimeout = 24 * time.Hour -// provisionTimeout raises the vnode handler's generic Provision deadline, because Provision here -// BLOCKS twice: a cold image is pulled into Modal's cache on this call (see sdkClient.buildImage), -// and the credential mint then waits for a worker (see sdkClient.mintCredential). -// -// The two legs SHARE this budget, and the mint is the greedy one: Modal caps it at ~3m43s of its -// own accord, so it always fails before this deadline does and the build is left roughly the -// remaining minute. That is the cost of a tight ceiling here — a cold pull of a large image can -// outrun it — and it is deliberate: a call that will die at 3m43s anyway should not hold a -// pod-controller worker for ten minutes first. -const provisionTimeout = 5 * time.Minute +// provisionTimeout bounds Provision, which blocks on the image build and then on the mint's wait +// for a worker (see sdkClient.mintCredential). A minute, well under Modal's own ~3m43s cap on +// the mint, on purpose: Modal is a fallback, so a queued GPU is a shortage to fail over from +// rather than wait in. Its cost is the queued sandboxes that would have been placed later +// (measured 89s to ~220s), and a cold build of a large image that outruns it. +const provisionTimeout = time.Minute // compile-time assertions that Provider satisfies the interfaces. LogStreamer and // Executor are the optional halves: they are what make `kubectl logs` and `kubectl exec` @@ -446,8 +442,8 @@ func (p *Provider) Capabilities() provider.Capabilities { // // A fresh sandbox comes back RESERVED, but not because Modal's create says so — it accepts a // sandbox whose GPU is still queued. The mint is what waits (see mintCredential), so id and -// capacity arrive together, at the price of a call that blocks for as long as the queue does — -// bounded by Modal's own ~3m43s cap on the mint, not by provisionTimeout. +// capacity arrive together, at the price of a call that blocks for as long as the queue does, +// up to provisionTimeout. // // The connect credential comes back on this call and only this call, since Modal mints it // once with no read-back, and only for a Pod that declares a containerPort. The caller must diff --git a/pkg/provider/modal/modal_test.go b/pkg/provider/modal/modal_test.go index 7c70dfd..ba8acdb 100644 --- a/pkg/provider/modal/modal_test.go +++ b/pkg/provider/modal/modal_test.go @@ -583,12 +583,11 @@ func TestCapabilities(t *testing.T) { if caps.SupportsStop || caps.SupportsSpot || caps.PreemptionNotice != 0 || !caps.NativeTags { t.Fatalf("unexpected caps: %+v", caps) } - // Must be STATED, and above the handler's generic default: Provision blocks on the image - // build here, so this adapter needs more than a provider whose create just returns an id. - // Zero is the regression to guard — it silently reverts to that default. + // Must be STATED: Provision blocks on the build and the mint here, so this adapter needs + // more than the handler's generic default. Zero is the regression to guard — it silently + // reverts to that default. if caps.ProvisionTimeout != provisionTimeout { - t.Fatalf("ProvisionTimeout = %v, want %v to cover a cold image build", - caps.ProvisionTimeout, provisionTimeout) + t.Fatalf("ProvisionTimeout = %v, want %v", caps.ProvisionTimeout, provisionTimeout) } if p.Name() != provider.ProviderModal { t.Fatalf("name = %q", p.Name()) @@ -1156,7 +1155,7 @@ func TestResolveRegions_RoundTripsThroughProvision(t *testing.T) { func TestMintTimeoutBlocksTheCandidate(t *testing.T) { // Modal gives up waiting for a worker with a server-side DeadlineExceeded, not our own // context deadline. Unlabelled, it read as unattributable and blocked nothing, so every - // replacement Pod queued behind the same full Modal for another ~3m43s. + // replacement Pod queued behind the same full Modal again. p := newTestProvider(&fakeClient{}) timedOut := mintError("sb-1", 8080, status.Error(codes.DeadlineExceeded, "Sandbox task ID not found within timeout")) From eca3363e29fef648b52da025828d48ae4bc50b9e Mon Sep 17 00:00:00 2001 From: ky-hiverge Date: Sun, 25 Oct 2026 10:32:14 +0000 Subject: [PATCH 21/21] remove role Cherry-picked from hiverge/nebula c0c90be, without chart/values-prod.yaml, which only exists in that fork. Signed-off-by: ky-hiverge --- config/kubernetes-provider/clusters.yaml | 1 - config/kubernetes-provider/remote-rbac.yaml | 9 +- pkg/provider/kubernetes/config.go | 2 - pkg/provider/kubernetes/eks.go | 12 +-- pkg/vnode/env.go | 94 ++++++++++----------- pkg/vnode/env_test.go | 16 ++-- pkg/vnode/handler.go | 4 +- 7 files changed, 62 insertions(+), 76 deletions(-) diff --git a/config/kubernetes-provider/clusters.yaml b/config/kubernetes-provider/clusters.yaml index 88ddfd5..bff1e92 100644 --- a/config/kubernetes-provider/clusters.yaml +++ b/config/kubernetes-provider/clusters.yaml @@ -8,5 +8,4 @@ clusters: provider: eks region: us-east-1 # the EKS cluster's AWS region geography: us # set explicitly; a NodePool's regions match either this or region - # roleARN: arn:aws:iam::123456789012:role/nebula-remote # only for another account # nodeAutoscaler: karpenter # what adds nodes; the default and only one supported today diff --git a/config/kubernetes-provider/remote-rbac.yaml b/config/kubernetes-provider/remote-rbac.yaml index 5055e90..48beb9b 100644 --- a/config/kubernetes-provider/remote-rbac.yaml +++ b/config/kubernetes-provider/remote-rbac.yaml @@ -1,9 +1,6 @@ -# What Nebula may do in a target cluster, the host cluster included. Apply it in every -# target cluster, and map Nebula's IAM identity to the "nebula" group with an EKS access -# entry: -# -# aws eks create-access-entry --cluster-name \ -# --principal-arn --kubernetes-groups nebula +# What Nebula may do in a target cluster, the host cluster included. Apply it in every target +# cluster, and map the identity Nebula authenticates to that cluster with into the "nebula" +# group, the way the cluster's provider maps identities to Kubernetes groups. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/pkg/provider/kubernetes/config.go b/pkg/provider/kubernetes/config.go index d1e09e3..f69cd50 100644 --- a/pkg/provider/kubernetes/config.go +++ b/pkg/provider/kubernetes/config.go @@ -54,8 +54,6 @@ type ClusterConfig struct { Region string `json:"region"` // Geography is the Nebula geography the cluster counts as in (see provider.Geographies). Geography string `json:"geography"` - // RoleARN, when set, is assumed before talking to this cluster: the cross-account path. - RoleARN string `json:"roleARN,omitempty"` // NodeAutoscaler is what adds nodes to the cluster (see nodeAutoscalers). Empty is Karpenter. NodeAutoscaler string `json:"nodeAutoscaler,omitempty"` } diff --git a/pkg/provider/kubernetes/eks.go b/pkg/provider/kubernetes/eks.go index af0f691..c17631d 100644 --- a/pkg/provider/kubernetes/eks.go +++ b/pkg/provider/kubernetes/eks.go @@ -27,7 +27,6 @@ import ( awssdk "github.com/aws/aws-sdk-go-v2/aws" awsconfig "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials/stscreds" "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/eks" @@ -96,18 +95,15 @@ func offeredFamilies(ctx context.Context, api describeOfferings, families []stri return found, nil } -// newEKSClients builds the clients for one EKS cluster from Nebula's ambient AWS identity -// (the SDK's default chain: an access key today, Pod Identity later). Nothing is stored: -// the endpoint and CA come from DescribeCluster, the bearer token from STS. +// newEKSClients builds the clients for one EKS cluster from Nebula's ambient AWS identity (the +// SDK's default chain, the controller's IRSA role in a deployment), which must be in the +// cluster's account: EKS has no resource policy to grant DescribeCluster across accounts. +// Nothing is stored: the endpoint and CA come from DescribeCluster, the bearer token from STS. func newEKSClients(ctx context.Context, cl ClusterConfig) (clusterClients, error) { awsCfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(cl.Region)) if err != nil { return clusterClients{}, fmt.Errorf("kubernetes: cluster %q: load AWS config: %w", cl.Name, err) } - if cl.RoleARN != "" { - awsCfg.Credentials = awssdk.NewCredentialsCache( - stscreds.NewAssumeRoleProvider(sts.NewFromConfig(awsCfg), cl.RoleARN)) - } out, err := eks.NewFromConfig(awsCfg).DescribeCluster(ctx, &eks.DescribeClusterInput{Name: awssdk.String(cl.Name)}) diff --git a/pkg/vnode/env.go b/pkg/vnode/env.go index 77f1121..f456a93 100644 --- a/pkg/vnode/env.go +++ b/pkg/vnode/env.go @@ -27,11 +27,13 @@ import ( "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/kubernetes" logf "sigs.k8s.io/controller-runtime/pkg/log" + + "github.com/InftyAI/Nebula/pkg/provider" ) // resolveEnv builds the environment a provider gets as ProvisionRequest.Env (Config, Secrets), -// following the references the Pod makes. A variable lands in secrets when the value that -// won under kubelet precedence came from a Secret, and in env otherwise. The only place +// following the references the Pod makes. A variable lands in Secrets when the value that +// won under kubelet precedence came from a Secret, and in Config otherwise. The only place // envFrom/valueFrom are read, since a provider has no cluster access to follow them with. // // Kubelet rules: envFrom first in listed order with each Prefix, then env overriding it. @@ -47,55 +49,54 @@ import ( // Reads bypass node.go's Secret/ConfigMap informers on purpose: VK waits only for the POD // informer to sync before CreatePod, and a cold cache reports an existing Secret as absent — // the one signal this function acts on. A nil client is the usual test seam (literals only). -func resolveEnv( - ctx context.Context, client kubernetes.Interface, pod *corev1.Pod, -) (env, secrets map[string]string, err error) { +func resolveEnv(ctx context.Context, client kubernetes.Interface, pod *corev1.Pod) (provider.Env, error) { if len(pod.Spec.Containers) == 0 { - return nil, nil, nil + return provider.Env{}, nil } // One container per Nebula Pod, as everywhere else that reads the workload. c := pod.Spec.Containers[0] if len(c.Env) == 0 && len(c.EnvFrom) == 0 { - return nil, nil, nil + return provider.Env{}, nil } - r := &envResolver{client: client, pod: pod, fromSecret: map[string]bool{}} - out := make(map[string]string, len(c.Env)) - if err := r.fromSources(ctx, out, c.EnvFrom); err != nil { - return nil, nil, err - } - if err := r.fromVars(ctx, out, c.Env); err != nil { - return nil, nil, err + r := &envResolver{client: client, pod: pod} + var out provider.Env + if err := r.fromSources(ctx, &out, c.EnvFrom); err != nil { + return provider.Env{}, err } - for k, v := range out { - dst := &env - if r.fromSecret[k] { - dst = &secrets - } - if *dst == nil { - *dst = map[string]string{} - } - (*dst)[k] = v + if err := r.fromVars(ctx, &out, c.Env); err != nil { + return provider.Env{}, err } - return env, secrets, nil + return out, nil } // envResolver resolves one container's references, memoizing what it reads so ten // secretKeyRefs into one Secret cost one GET. Misses are memoized too (a nil entry). It // lives for a single resolveEnv call, so nothing here can go stale. type envResolver struct { - client kubernetes.Interface - pod *corev1.Pod - secrets map[string]*corev1.Secret - configs map[string]*corev1.ConfigMap - // fromSecret records, per variable, whether its current value came from a Secret; the - // last write wins, as the value does. - fromSecret map[string]bool + client kubernetes.Interface + pod *corev1.Pod + secretObjs map[string]*corev1.Secret + configMapObjs map[string]*corev1.ConfigMap +} + +// set records name's winning value in the part its source belongs to. A later source +// overrides an earlier one, possibly of the other kind, so the variable leaves that part. +func set(out *provider.Env, name, value string, secret bool) { + dst, other := &out.Config, out.Secrets + if secret { + dst, other = &out.Secrets, out.Config + } + delete(other, name) + if *dst == nil { + *dst = map[string]string{} + } + (*dst)[name] = value } // fromSources applies envFrom in listed order. Every key of each source becomes a variable, // with the source's Prefix prepended. -func (r *envResolver) fromSources(ctx context.Context, out map[string]string, sources []corev1.EnvFromSource) error { +func (r *envResolver) fromSources(ctx context.Context, out *provider.Env, sources []corev1.EnvFromSource) error { for _, src := range sources { switch { case src.SecretRef != nil: @@ -128,11 +129,10 @@ func (r *envResolver) fromSources(ctx context.Context, out map[string]string, so } // fromVars applies the explicit env list, overriding anything envFrom contributed. -func (r *envResolver) fromVars(ctx context.Context, out map[string]string, vars []corev1.EnvVar) error { +func (r *envResolver) fromVars(ctx context.Context, out *provider.Env, vars []corev1.EnvVar) error { for _, e := range vars { if e.ValueFrom == nil { - out[e.Name] = e.Value - r.fromSecret[e.Name] = false + set(out, e.Name, e.Value, false) continue } v, found, err := r.valueFrom(ctx, e.ValueFrom) @@ -142,8 +142,7 @@ func (r *envResolver) fromVars(ctx context.Context, out map[string]string, vars // Not found means optional-and-absent (a required miss errored), so leave the // variable unset, as the kubelet does. if found { - out[e.Name] = v - r.fromSecret[e.Name] = e.ValueFrom.SecretKeyRef != nil + set(out, e.Name, v, e.ValueFrom.SecretKeyRef != nil) } } return nil @@ -248,7 +247,7 @@ func subscript(path, prefix string) (string, bool) { // absent; a NotFound on a required ref is an error, as is any other read failure whether or // not the ref is optional. func (r *envResolver) secret(ctx context.Context, name string, opt bool) (*corev1.Secret, error) { - s, memoized := r.secrets[name] + s, memoized := r.secretObjs[name] if !memoized { if r.client == nil { return nil, nil // test seam: no cluster to read from @@ -263,10 +262,10 @@ func (r *envResolver) secret(ctx context.Context, name string, opt bool) (*corev default: s = got } - if r.secrets == nil { - r.secrets = map[string]*corev1.Secret{} + if r.secretObjs == nil { + r.secretObjs = map[string]*corev1.Secret{} } - r.secrets[name] = s + r.secretObjs[name] = s } if s == nil && !opt { return nil, r.absentErr("Secret", name) @@ -276,7 +275,7 @@ func (r *envResolver) secret(ctx context.Context, name string, opt bool) (*corev // configMap is the ConfigMap half of secret, with the same contract. func (r *envResolver) configMap(ctx context.Context, name string, opt bool) (*corev1.ConfigMap, error) { - cm, memoized := r.configs[name] + cm, memoized := r.configMapObjs[name] if !memoized { if r.client == nil { return nil, nil // test seam: no cluster to read from @@ -290,10 +289,10 @@ func (r *envResolver) configMap(ctx context.Context, name string, opt bool) (*co default: cm = got } - if r.configs == nil { - r.configs = map[string]*corev1.ConfigMap{} + if r.configMapObjs == nil { + r.configMapObjs = map[string]*corev1.ConfigMap{} } - r.configs[name] = cm + r.configMapObjs[name] = cm } if cm == nil && !opt { return nil, r.absentErr("ConfigMap", name) @@ -312,15 +311,14 @@ func (r *envResolver) absentErr(kind, name string) error { // still starts): a ConfigMap consumed wholesale carries whatever keys it was written with. // Logged, because this is the one place a variable disappears without an error. The bar is // Kubernetes' own, laxer than C_IDENTIFIER — "app.conf" passes, a leading digit does not. -func (r *envResolver) put(ctx context.Context, out map[string]string, name, value, kind, source string) { +func (r *envResolver) put(ctx context.Context, out *provider.Env, name, value, kind, source string) { if errs := validation.IsEnvVarName(name); len(errs) > 0 { logf.FromContext(ctx).WithName("vnode-env").Info("dropping environment variable with an illegal name", "pod", key(r.pod.Namespace, r.pod.Name), "name", name, "source", kind+"/"+source, "reason", strings.Join(errs, "; ")) return } - out[name] = value - r.fromSecret[name] = kind == "Secret" + set(out, name, value, kind == "Secret") } // optional dereferences the *bool the API uses for optional, where nil means false. diff --git a/pkg/vnode/env_test.go b/pkg/vnode/env_test.go index a143bc6..000743e 100644 --- a/pkg/vnode/env_test.go +++ b/pkg/vnode/env_test.go @@ -29,8 +29,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" - - "github.com/InftyAI/Nebula/pkg/provider" ) // envPod is a Pod whose single container carries the given env sources. @@ -429,14 +427,14 @@ func TestResolveEnv_NoEnvIsNil(t *testing.T) { } } -// resolveAll is resolveEnv with its two maps merged, for tests about which value wins rather +// resolveAll is resolveEnv with its two parts merged, for tests about which value wins rather // than where it lands. func resolveAll(ctx context.Context, client kubernetes.Interface, pod *corev1.Pod) (map[string]string, error) { - env, secrets, err := resolveEnv(ctx, client, pod) + env, err := resolveEnv(ctx, client, pod) if err != nil { return nil, err } - return provider.Env{Config: env, Secrets: secrets}.All(), nil + return env.All(), nil } // A variable is a secret when the value that WON came from a Secret, so precedence decides @@ -459,13 +457,13 @@ func TestResolveEnv_SplitsSecrets(t *testing.T) { }, ) - env, secrets, err := resolveEnv(context.Background(), client, pod) + env, err := resolveEnv(context.Background(), client, pod) if err != nil { t.Fatalf("resolveEnv: %v", err) } - wantEnv := map[string]string{"FROM_CM": "c", "SECRET_THEN_LITERAL": "plain"} + wantConfig := map[string]string{"FROM_CM": "c", "SECRET_THEN_LITERAL": "plain"} wantSecrets := map[string]string{"FROM_SECRET": "s", "CM_THEN_SECRET": "s"} - if !maps.Equal(env, wantEnv) || !maps.Equal(secrets, wantSecrets) { - t.Fatalf("env = %v, secrets = %v; want %v and %v", env, secrets, wantEnv, wantSecrets) + if !maps.Equal(env.Config, wantConfig) || !maps.Equal(env.Secrets, wantSecrets) { + t.Fatalf("config = %v, secrets = %v; want %v and %v", env.Config, env.Secrets, wantConfig, wantSecrets) } } diff --git a/pkg/vnode/handler.go b/pkg/vnode/handler.go index bb3b101..b15686a 100644 --- a/pkg/vnode/handler.go +++ b/pkg/vnode/handler.go @@ -327,14 +327,14 @@ func (h *Handler) CreatePod(ctx context.Context, pod *corev1.Pod) error { // Deliberately NOT stored: tracking a pod makes GetPod return non-nil, which suppresses // the retry this branch wants. (The provision-failure branch below stores for exactly // that reason — it does not want one.) - env, secrets, err := resolveEnv(ctx, h.client, pod) + env, err := resolveEnv(ctx, h.client, pod) if err != nil { log.Error(err, "cannot resolve the Pod's environment; nothing provisioned, retrying") h.markStatus(pod, corev1.PodPending, reasonConfigError, err.Error()) h.emit(pod) return err } - req.Env = provider.Env{Config: env, Secrets: secrets} + req.Env = env // Same resolve-before-requesting rule, and the same non-terminal treatment, for the // image pull credential: an imagePullSecret names a Secret in the Pod's namespace that