feat: support kubernetes as a new provider

cmd/main.go

@@ -57,6 +57,7 @@ import (
 	"github.com/InftyAI/Nebula/pkg/provider"
 	awsprovider "github.com/InftyAI/Nebula/pkg/provider/aws"
 	"github.com/InftyAI/Nebula/pkg/provider/fake"
+	kubeprovider "github.com/InftyAI/Nebula/pkg/provider/kubernetes"
 	"github.com/InftyAI/Nebula/pkg/provider/modal"
 	"github.com/InftyAI/Nebula/pkg/provider/runpod"
 	"github.com/InftyAI/Nebula/pkg/vnode"
@@ -109,6 +110,7 @@ func main() {
 	var costLabels string
 	var kubeletServingTLSBootstrap bool
 	var providers string
+	var kubernetesClusters string
 	var tlsOpts []func(*tls.Config)
 	flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+
 		"Use :8443 for HTTPS or :8080 for HTTP, or leave as 0 to disable the metrics service.")
@@ -151,6 +153,10 @@ func main() {
 			"x509: certificate signed by unknown authority. Off by default because it needs the "+
 			"RBAC to impersonate one virtual node identity — the signer signs for nobody else "+
 			"(see addServingCertificateBootstrap).")
+	flag.StringVar(&kubernetesClusters, "kubernetes-clusters", "",
+		"Path to the kubernetes provider's cluster table in the container, normally "+
+			"/etc/nebula/kubernetes/clusters.yaml mounted from the kubernetes-clusters ConfigMap. "+
+			"Empty leaves the kubernetes provider unregistered.")
 	flag.StringVar(&providers, "providers", strings.Join(knownProviders, ","),
 		"Comma-separated production providers to enable; all by default. A listed provider is "+
 			"skipped if it fails to initialize. Drain a provider before dropping it: nothing "+
@@ -327,7 +333,7 @@ func main() {
 	// startup already sees its provider. The manager's client backs the AWS region
 	// source (regions are read from NodePools at call time, not env), so it is
 	// threaded in; the client is only queried at runtime, after the cache has synced.
-	registerProviders(context.Background(), mgr.GetClient(), enabledProviders)
+	registerProviders(context.Background(), mgr.GetClient(), enabledProviders, kubernetesClusters, attribution)
 
 	// One shared failover blocklist, written by the virtual kubelet handlers on a
 	// Provision failure and read by the placement controller to skip a candidate
@@ -528,7 +534,7 @@ func setupKubeletServer(mgr ctrl.Manager, addr, clientCA string, servingTLSBoots
 // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests,resourceNames=nebula-kubelet-serving,verbs=delete;get
 // +kubebuilder:rbac:groups=certificates.k8s.io,resources=certificatesigningrequests/approval,resourceNames=nebula-kubelet-serving,verbs=update
 // +kubebuilder:rbac:groups=certificates.k8s.io,resources=signers,resourceNames=kubernetes.io/kubelet-serving,verbs=approve
-// +kubebuilder:rbac:groups="",resources=users,resourceNames={"system:node:nebula-aws","system:node:nebula-modal","system:node:nebula-runpod","system:node:nebula-fake"},verbs=impersonate
+// +kubebuilder:rbac:groups="",resources=users,resourceNames={"system:node:nebula-aws","system:node:nebula-modal","system:node:nebula-runpod","system:node:nebula-kubernetes","system:node:nebula-fake"},verbs=impersonate
 // +kubebuilder:rbac:groups="",resources=groups,resourceNames="system:nodes",verbs=impersonate
 
 // addServingCertificateBootstrap requests a trusted serving certificate for the kubelet
@@ -608,7 +614,8 @@ func setupVirtualNodes(mgr ctrl.Manager, blocklist vnode.Blocklister, kubeletSrv
 // crash loop.
 //
 // Only providers in enabled are attempted (see parseProviders).
-func registerProviders(ctx context.Context, c client.Client, enabled map[string]bool) {
+func registerProviders(ctx context.Context, c client.Client, enabled map[string]bool,
+	kubernetesClusters string, costLabels []string) {
 	register := func(name string, build func() (provider.Provider, error)) {
 		if !enabled[name] {
 			setupLog.Info("provider disabled by --providers", "provider", name)
@@ -639,6 +646,13 @@ func registerProviders(ctx context.Context, c client.Client, enabled map[string]
 		return runpod.NewSDKClient(ctx)
 	})
 
+	register(provider.ProviderKubernetes, func() (provider.Provider, error) {
+		if kubernetesClusters == "" {
+			return nil, errors.New("no --kubernetes-clusters file")
+		}
+		return kubeprovider.NewFromFile(ctx, kubernetesClusters, costLabels)
+	})
+
 	// The fake provider is an in-memory backend used only by the e2e suite to
 	// exercise the full control-plane loop without cloud credentials. It ships in
 	// the binary but registers ONLY when explicitly enabled, so it can never place
@@ -652,7 +666,9 @@ func registerProviders(ctx context.Context, c client.Client, enabled map[string]
 
 // knownProviders are the names --providers accepts, one per register call above. The fake
 // provider is not among them: it stays gated on its env var alone.
-var knownProviders = []string{provider.ProviderModal, provider.ProviderAWS, provider.ProviderRunPod}
+var knownProviders = []string{
+	provider.ProviderModal, provider.ProviderAWS, provider.ProviderRunPod, provider.ProviderKubernetes,
+}
 
 // parseProviders turns --providers into the enabled set. An unknown name is an error rather
 // than ignored, so a typo cannot silently leave a provider off.

config/default/kustomization.yaml

@@ -22,6 +22,9 @@ resources:
 # the manager uses the CSVs embedded in its binary. Uncommenting this also needs
 # the catalog env var and volume in config/manager/manager.yaml.
 # - ../catalog
+# The kubernetes provider's cluster table (config/kubernetes-provider/clusters.yaml). Off:
+# without it the kubernetes provider is not registered. Fill in clusters.yaml first.
+# - ../kubernetes-provider
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 - ../webhook

config/kubernetes-provider/clusters.yaml

@@ -0,0 +1,11 @@
+# The kubernetes provider's cluster table (pkg/provider/kubernetes.Config), rendered into the
+# manager's ConfigMap by this directory's kustomization. Read once at startup: restart the
+# manager after a change. No credentials go here: each EKS cluster is reached with Nebula's
+# own AWS identity.
+clusters:
+  # The host cluster itself, the first target to verify the provider against.
+  - name: my-cluster          # the EKS cluster's own name
+    provider: eks
+    region: us-east-1          # the EKS cluster's AWS region
+    geography: us              # set explicitly; a NodePool's regions match either this or region
+    # nodeAutoscaler: karpenter   # what adds nodes; the default and only one supported today

config/kubernetes-provider/kustomization.yaml

@@ -0,0 +1,16 @@
+# Renders clusters.yaml into the ConfigMap the manager mounts at /etc/nebula/kubernetes.
+# Included by config/default, which applies the `nebula-` prefix and `nebula-system`
+# namespace and rewrites the Deployment's `kubernetes-clusters` reference to match.
+#
+# remote-rbac.yaml is deliberately NOT a resource here: it belongs in each target cluster.
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+configMapGenerator:
+- name: kubernetes-clusters
+  files:
+  - clusters.yaml
+
+generatorOptions:
+  # A stable name, so the volume reference and `kubectl edit` keep working.
+  disableNameSuffixHash: true

config/kubernetes-provider/remote-rbac.yaml

@@ -0,0 +1,33 @@
+# What Nebula may do in a target cluster, the host cluster included. Apply it in every target
+# cluster, and map the identity Nebula authenticates to that cluster with into the "nebula"
+# group, the way the cluster's provider maps identities to Kubernetes groups.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: nebula-remote
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "create", "update", "delete"]
+  - apiGroups: [""]
+    resources: ["namespaces"]
+    verbs: ["get", "create"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: nebula-remote
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nebula-remote
+subjects:
+  - apiGroup: rbac.authorization.k8s.io
+    kind: Group
+    name: nebula

config/manager/manager.yaml

@@ -68,6 +68,9 @@ spec:
           # only way to turn AWS off, which registers even without credentials. Drain a
           # provider first: once dropped, nothing terminates its running instances.
           # - --providers=runpod
+          # The kubernetes provider's cluster table, from the optional ConfigMap mounted below.
+          # Absent ConfigMap = no file = the provider is skipped at registration.
+          - --kubernetes-clusters=/etc/nebula/kubernetes/clusters.yaml
         image: controller:latest
         name: manager
         imagePullPolicy: IfNotPresent
@@ -189,14 +192,22 @@ spec:
           requests:
             cpu: "1"
             memory: 1Gi
-        # Empty, not absent: config/default appends the cert mounts here with a JSON
-        # patch (`.../volumeMounts/-`), which needs the key to exist.
-        volumeMounts: []
+        volumeMounts:
+          - name: kubernetes-clusters
+            mountPath: /etc/nebula/kubernetes
+            readOnly: true
         #   # The catalog ConfigMap mount, off with NEBULA_CATALOG_DIR above.
         #   - name: catalog
         #     mountPath: /etc/nebula/catalog
         #     readOnly: true
-      volumes: []
+      volumes:
+        - name: kubernetes-clusters
+          configMap:
+            # Unprefixed on purpose: with `- ../kubernetes-provider` in config/default,
+            # namePrefix rewrites this to the generated "nebula-kubernetes-clusters".
+            # optional, so without it the provider is just skipped.
+            name: kubernetes-clusters
+            optional: true
       #   - name: catalog
       #     configMap:
       #       # Unprefixed on purpose: config/default's namePrefix rewrites this

config/rbac/role.yaml

@@ -75,6 +75,7 @@ rules:
   resourceNames:
   - system:node:nebula-aws
   - system:node:nebula-fake
+  - system:node:nebula-kubernetes
   - system:node:nebula-modal
   - system:node:nebula-runpod
   resources:

config/samples/nodepool.yaml

@@ -7,6 +7,11 @@ metadata:
 spec:
   # strategy (the inner, provider-ranking axis).
   providers:
+  # EKS clusters from --kubernetes-clusters. A region matches a cluster's geography ("us") or
+  # its exact AWS region ("us-east-1"), both as set in that file.
+  - name: kubernetes
+    regions:
+    - us-east-1
   - name: aws
     regions:
     - us

docs/architecture.md

@@ -59,17 +59,17 @@ mapping), see [docs/status.md](status.md).
   declaration cannot yet prefer the cheapest region. Modal is the sharper case — a
   pinned region there costs 1.5x (group) or 1.75x (narrow) over its unconstrained
   default, which the catalog does not model.
-- Failover on a provider that *queues*. The blocklist has exactly one writer: the
-  virtual kubelet's `Provision` error path. A provider that reports a capacity
-  shortfall synchronously (AWS `CreateFleet`) therefore fails over across zone,
-  region and tier, but one that ACCEPTS the request and queues for capacity returns
-  an instance id and no error — so nothing is blocklisted and placement is never
-  re-driven. Modal is that case: a `[modal, aws]` pool never advances to AWS on
-  capacity, and the Pod waits in Modal's queue at `Initializing` (which, per
-  [status](status.md#modal), is indistinguishable from booting). Modal also collapses
-  its regions into a single candidate, so it has no intra-provider region failover
-  either. Making failover live for such a provider needs a reserved-by deadline that
-  synthesizes `ErrNoCapacity` — the classification side already handles it.
+- Failover on Modal's queue. The blocklist has two writers: the virtual kubelet's
+  `Provision` error path, and the poll loop when `List` reports an instance as
+  `InstanceUnschedulable`. A provider that reports a capacity shortfall synchronously
+  (AWS `CreateFleet`) fails over through the first; the `kubernetes` provider, which
+  returns unreserved and reports no capacity once its pod stays unbound past the
+  bind timeout, through the second. Modal reports neither: it ACCEPTS the request and
+  queues, so a `[modal, aws]` pool never advances to AWS on capacity, and the Pod
+  waits in Modal's queue at `Initializing` (which, per [status](status.md#modal), is
+  indistinguishable from booting). Modal also collapses its regions into a single
+  candidate, so it has no intra-provider region failover either. Making failover live
+  for it means reporting `InstanceUnschedulable` past a reserved-by deadline.
 - Bin-packing multiple unrelated Pods onto one external instance. The current
   model is one workload Pod to one external instance.
 - In-place migration. Recovery from reclaim, failure, or spec changes is
@@ -226,7 +226,8 @@ Follow one GPU Pod from creation to teardown:
    and region from the failed request. The shared blocklist records that scope for
    the Pod's blocklist TTL plus jitter. The Pod is marked Failed; the placement
    controller deletes terminal controller-owned Pods so their owner recreates a
-   fresh gated Pod that skips the blocked candidate.
+   fresh gated Pod that skips the blocked candidate. An instance accepted unreserved
+   and later listed as `InstanceUnschedulable` takes the same path from the poll loop.
 
 10. **Teardown.** The happy path is edge-triggered by VK `DeletePod`, which calls
     `provider.Terminate(instanceID)`. The backstop is level-triggered by the

docs/metrics.md

@@ -67,7 +67,7 @@ which is the whole reason for splitting them:
 | `reason` | Means | Clears when |
 | --- | --- | --- |
 | `no_pool` | The Pod names a NodePool that does not exist, or carries no pool label. | A human fixes the Pod (or the workload generating it). |
-| `invalid_request` | The request is malformed — `nvidia.com/gpu` with no accelerator-type label (*not* treated as CPU-only), or a `nebula.inftyai.com/regions` annotation naming no known geography. | A human fixes the Pod spec. |
+| `invalid_request` | The request is malformed — `nvidia.com/gpu` with no accelerator-type label (*not* treated as CPU-only), an accelerator-type label with no `nvidia.com/gpu` request, or a `nebula.inftyai.com/regions` annotation naming no known geography. | A human fixes the Pod spec. |
 | `all_blocked` | A servable candidate exists, but failover is holding every one of them off. | By itself — the Pod is already requeued for the block's expiry. |
 | `no_candidate` | No provider in the pool can serve this request at all. | An operator adds a provider, or a provider registers. |
 | `stale_claim` | A NodeClaim from a prior same-named Pod has not been reaped yet. | By itself, in seconds. A sustained rate means the NodeClaim backstop is stuck. |

go.mod

@@ -3,10 +3,13 @@ module github.com/InftyAI/Nebula
 go 1.25.0
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.43.0
+	github.com/aws/aws-sdk-go-v2 v1.47.3
 	github.com/aws/aws-sdk-go-v2/config v1.32.31
+	github.com/aws/aws-sdk-go-v2/credentials v1.20.9
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.317.0
-	github.com/aws/smithy-go v1.27.3
+	github.com/aws/aws-sdk-go-v2/service/eks v1.104.2
+	github.com/aws/aws-sdk-go-v2/service/sts v1.51.4
+	github.com/aws/smithy-go v1.28.5
 	github.com/modal-labs/modal-client/go v0.9.0
 	github.com/onsi/ginkgo/v2 v2.27.2
 	github.com/onsi/gomega v1.38.2
@@ -29,17 +32,15 @@ require (
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/aristanetworks/gomap v0.0.0-20230726210543-f4e41046dced // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.19.30 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 // indirect
-	github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/signin v1.10.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.38.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.4 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect