diff --git a/.env.example b/.env.example index 1fd43f30b..1a775c40d 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ NODE_ENV=development # EdgeOps Console static embed (npm run build:console → dev/console/build) EDGEOPS_CONSOLE_PATH=dev/console/build # must be absolute path -EDGEOPS_CONSOLE_VERSION=v1.0.13 +EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.6 # EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=datasance diff --git a/.github/actions/set-build-env/action.yml b/.github/actions/set-build-env/action.yml index b36ee5cb9..b022057aa 100644 --- a/.github/actions/set-build-env/action.yml +++ b/.github/actions/set-build-env/action.yml @@ -8,7 +8,7 @@ runs: shell: bash run: | VERSION="${{ env.EDGEOPS_CONSOLE_VERSION }}" - if [ -z "$VERSION" ]; then VERSION="1.0.13"; fi + if [ -z "$VERSION" ]; then VERSION="1.1.0-rc.6"; fi echo "EDGEOPS_CONSOLE_VERSION=$VERSION" >> "${GITHUB_ENV}" REPO="${{ env.EDGEOPS_CONSOLE_REPO }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index cea273343..107bf8b30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,129 @@ # Changelog +## [v3.9.0-rc.9] + +### Added + +- **Fog status (host capacity + OS)** — agent `PUT /api/v3/agent/status` accepts and persists `systemCpus`, `systemTotalMemory`, `systemAvailableMemory`, `systemTotalDisk`, `systemAvailableDisk`, `systemOs`, `systemOsVersion`, and `systemKernelVersion`. User `GET /api/v3/iofog` and list return the stored values. `systemTotalCpu` remains host CPU busy **0–100%**; `diskUsage` remains Edgelet data-directory usage in **GiB**, not host filesystem totals. Columns on 3.9.0 baseline and 3.8→3.9 upgrade (sqlite, mysql, postgres). Omitted keys leave prior values unchanged. + +### Changed +- Embedded **EdgeOps Console** default version **v1.1.0-rc.5** → **v1.1.0-rc.6** + +## [v3.9.0-rc.8] + +### Changed +- Embedded **EdgeOps Console** default version **v1.1.0-rc.4** → **v1.1.0-rc.5** + +## [v3.9.0-rc.7] + +### Changed + +- **Fog status** — user GET and list return `knowledgeStatus` as the stored JSON string, matching `modelStatus`. Callers parse it. +- Dockerfile runtime base image digest pin refreshed for **`ubi9/nodejs-24-minimal`**. + +## [v3.9.0-rc.6] + +Fleet Knowledge artifacts: curated documents, datasets, and vector indexes distributed like Models, with a separate name namespace and Hub dataset pulls. + +### Added + +- **Fleet Knowledge API** — `GET/POST/PATCH/DELETE /api/v3/knowledge`, YAML create/upsert (`kind: Knowledge`), and volumeMount-style `GET/POST/DELETE …/knowledge/:name/link`; server-generated uuid; agent `GET /api/v3/agent/knowledge` returns linked rows only (`{ "knowledge": [ … ] }`). +- **`PATCH /api/v3/microservices/:uuid/knowledge`** — user microservices only; `microserviceKnowledge` change flag and rebuild rules for bind path / permissions / empty↔non-empty catalog transitions. +- **Fog status** — `knowledgeStatus` (JSON string), `activeKnowledge` (managed count), `knowledgeLastUpdate` (Unix ms). +- **v3.9.0 schema** — `Knowledge`, `FogKnowledge`, catalog tables, change-tracking flags, fog status columns on baseline and 3.8→3.9 upgrade. + +### Changed + +- **`modelLastUpdate`** — documented as Unix milliseconds (same clock as `knowledgeLastUpdate`). +- **MS catalog validation** — `knowledge.bindPath` must not collide with volumes, tmpfs, or the models catalog path. A non-empty `bindPath` on a models or knowledge catalog requires at least one item. +- **RBAC + swagger** — resource `knowledge`; viewer get/list; SRE/developer/admin full + link. +- Dockerfile base image digest pins refreshed for **`node:24-bookworm`** (console-builder and builder stages) +- Embedded **EdgeOps Console** default version **v1.1.0-rc.3** → **v1.1.0-rc.4** + +### Notes + +- Knowledge names are a separate namespace from Models. HF Knowledge uses the Hub dataset API; there is no `repoType` field. +- `format` is a soft hint. Known values persist; anything else stores `unknown`. Unknown format is not a 400. +- `getChanges.prune` still does not delete volumes. Unused **local** Knowledge prune is agent-side on the same flag. `system prune` does not drop Knowledge trees. + +## [v3.9.0-rc.5] + +Additive microservice `VOLUME` mapping `scope` for Edgelet private vs node-global shared claims. Reclaim stays on-node (`edgelet volume`); no new Controller prune flag. + +### Added + +- **`volumeMappings[].scope`** — `private` (default) or `shared`. Meaningful only when `type` is `volume`. Omit, null, or empty → `private`. Unknown values on `type: volume` are **400**. Other mapping types always store `private`. +- **`VolumeMappings.scope`** column on 3.9.0 baseline and 3.8→3.9 upgrade (sqlite, mysql, postgres). + +### Changed + +- **Agent microservice list** includes `scope` on each volume mapping (lowercase). +- **Volume-mapping create/delete** sets microservice `rebuild` and `microserviceCommon` so the agent applies the change. +- **`type: volume` `hostDestination`** charset matches Edgelet: `[a-zA-Z0-9][a-zA-Z0-9_.-]*`. +- Embedded **EdgeOps Console** default version **v1.1.0-rc.2** → **v1.1.0-rc.3** + +### Notes + +- Shared names are node-global. Controller does not unique-check names or `runAsUser` across consumers (local workloads exist). Use distinct names (`nodered-config`). +- Controller and system microservices always persist `private`. Explicit `scope: shared` on those workloads is **400**. +- `deleteWithCleanup` remains unused; operators reclaim with `edgelet volume` / `deprovision --purge-volumes`. +- Switching private → shared does not copy existing UUID directories. Older agents ignore `scope` (every `VOLUME` stays per-UUID) with no merge after upgrade. + +## [v3.9.0-rc.3] + +Observed microservice status and fog liveness: live CPU/memory only while a microservice is RUNNING, and node quietness judged by when Controller last received a status PUT. + +### Fixed + +- **Stopped/inactive microservices** no longer keep the last CPU/memory sample after stop, deactivate, or a non-RUNNING agent report. +- **Fog liveness** uses Controller receipt time (`lastStatusTime`), not the agent's clock, so clock skew no longer flaps nodes UNKNOWN. +- **Status jobs** walk the fleet in small transactions and no longer delete microservices from the liveness cleaner. + +### Added + +- **Microservice last-crash columns** — `last_error`, `last_error_at`, `restart_count` on `MicroserviceStatuses` (baseline 3.9.0 and 3.8→3.9 upgrade). + +### Changed + +- **GET microservice meters** — `status.cpuUsage` and `status.memoryUsage` are 0 unless observed `status` is `RUNNING`. +- **Operator stop / application deactivate** — GET shows `STOPPING` and zero meters immediately, before the next agent PUT. +- **Microservice last crash** — GET `status` always includes `lastError`, `lastErrorAt`, and `restartCount`. `errorMessage` is the current failure (cleared by the agent after 30s RUNNING); last crash is kept after recovery. +- Embedded **EdgeOps Console** default version **v1.1.0-rc.1** → **v1.1.0-rc.2** + +## [v3.9.0-rc.2] + +### Fixed + +- **`PATCH /api/v3/iofog/{uuid}` after Controller upgrade** — stale `bluetoothEnabled` / `abstractedHardwareEnabled` keys in `"FogPlatformSpecs".spec_json` (written before HAL removal) caused validation failure with misleading `Invalid fog platform spec field 'instance'`. Platform spec merge and upsert now whitelist allowed fields only; validation errors name the rejected field. +- **`POST /api/v3/runtimeClasses/{name}/link`** — refused link when the RuntimeClass name is not in the target fog `availableRuntimes` (same rule as microservice deploy runtime validation). + + +## [v3.9.0-rc.1] + +Fleet models, RuntimeClass, registry extensions, microservice catalog PATCH, MicroserviceTemplate, and HAL/BLE removal for Edgelet v1.1+ handoff. Targets **v3.9.0** schema (user tags). + +### Added + +- **Fleet models API** — `GET/POST/PATCH/DELETE /api/v3/models`, YAML create/upsert, and volumeMount-style `GET/POST/DELETE …/models/:name/link`; server-generated uuid; agent `GET /api/v3/agent/models` returns linked rows only. +- **RuntimeClass API** — `GET/POST/PATCH/DELETE /api/v3/runtimeClasses`, YAML, link endpoints (edgelet fogs only), and agent `GET /api/v3/agent/runtimeClasses`. +- **Registry `type` / `ca` / `insecure`** — Hub system registry by Hugging Face url+`hf`; MS and catalog image pulls remain **oci-only**. +- **`PATCH /api/v3/microservices/:uuid/models`** — `microserviceModels` change flag and rebuild rules for bind path / permissions / empty↔non-empty catalog transitions. +- **Microservice container fields** — commands/cmd alias, CDI devices, capabilities, health check, and related agent wire fields. +- **MicroserviceTemplate** — CRUD + YAML; deploy overlay via `POST /microservices` + `template`. +- **Agent status ingest** — `modelStatus`, `activeModels`, `modelLastUpdate`, `runtimeClasses`, `availableCdiDevices`, `podId`; getChanges flags `models`, `runtimeClasses`, `microserviceModels`. +- **v3.9.0 schema** — `Models`, `RuntimeClasses`, link tables, registry columns, template tables. Fresh install uses baseline 3.9.0 migrate+seed; upgrade from 3.8.0 applies incremental 3.9.0 only. + +### Changed + +- **MS deploy** — auto-links referenced fleet models and RuntimeClass on edgelet agents; validates runtime against `availableRuntimes`. +- **RBAC + swagger** — models, runtimeClasses, microserviceTemplates, and agent model/runtime routes documented and catalogued. +- Embedded **EdgeOps Console** default version **v1.0.13** → **v1.1.0-rc.1** + +### Removed + +- **HAL/BLE provisioning** — user HAL/USB routes, RESTBlue/HAL system catalog items, fog `bluetooth`/`hal` flags, and `deviceScanFrequency` from agent config wire (Edge Guard retained). +- **Agent HAL routes** — hardware inventory PUT/GET paths under `/api/v3/agent/hal/*`. + ## [v3.8.3-rc.3] - September 2026 diff --git a/Dockerfile b/Dockerfile index 07d61d889..7c8c0a65c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,10 @@ # ioFog overrides: EDGEOPS_CONSOLE_REPO=https://github.com/eclipse-iofog/edgeops-console # EDGEOPS_CONSOLE_FLAVOR=iofog # node:24-bookworm — pin manifest list digest for reproducible multi-arch builds -FROM node:24-bookworm@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2 AS console-builder +FROM node:24-bookworm@sha256:64af3819f9275802414d7cdc38c27e9d82bd564dec4d4da87d008255d36c63b4 AS console-builder ARG EDGEOPS_CONSOLE_REPO=https://github.com/Datasance/edgeops-console -ARG EDGEOPS_CONSOLE_VERSION=v1.0.13 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.6 ARG EDGEOPS_CONSOLE_FLAVOR=datasance RUN apt-get update \ @@ -28,7 +28,7 @@ RUN test -f build/index.html \ && cp -a build /tmp/console/build -FROM node:24-bookworm@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2 AS builder +FROM node:24-bookworm@sha256:64af3819f9275802414d7cdc38c27e9d82bd564dec4d4da87d008255d36c63b4 AS builder ARG PKG_VERSION @@ -48,9 +48,9 @@ RUN npm pack # ubi9/nodejs-24-minimal:latest — pin manifest list digest for reproducible multi-arch builds -FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:c7b8cb436feace96ca0b43eab7326394893e0ac9c2c6b46895727e3b9202fb9d +FROM registry.access.redhat.com/ubi9/nodejs-24-minimal@sha256:e3392c129520d2ebce748c3d0048343d0ec60be560cf220b60a7ba5d2bd2484a -ARG EDGEOPS_CONSOLE_VERSION=v1.0.13 +ARG EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.6 ARG IMAGE_REGISTRY ARG OCI_SOURCE_REPO ARG CONTROLLER_DISTRIBUTION=iofog diff --git a/Makefile b/Makefile index b68149b83..91905bef5 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Local Docker build — mirrors CI/release build-args (see .github/actions/set-build-env). -# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.0.13 +# Override any variable: make build FLAVOR=iofog EDGEOPS_CONSOLE_VERSION=v1.1.0-rc.6 FLAVOR ?= datasance IMAGE_NAME ?= controller @@ -25,7 +25,7 @@ else $(error FLAVOR must be "datasance" or "iofog", got "$(FLAVOR)") endif -EDGEOPS_CONSOLE_VERSION ?= v1.0.13 +EDGEOPS_CONSOLE_VERSION ?= v1.1.0-rc.6 IMAGE_REF = $(IMAGE_REGISTRY)/$(IMAGE_NAME):$(DOCKER_TAG) diff --git a/README.md b/README.md index 513dde289..74280ccf5 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,9 @@ API: `http://localhost:51121` · Console (embedded or split): set `CONSOLE_URL` |-------|-----| | Architecture | [docs/architecture.md](docs/architecture.md) | | RBAC | [docs/rbac-reference.md](docs/rbac-reference.md) | -| External OIDC | [docs/external-oidc-client-setup.md](docs/external-oidc-client-setup.md) | +| OIDC (embedded and external) | [docs/oidc-configuration.md](docs/oidc-configuration.md) | +| External OIDC client | [docs/external-oidc-client-setup.md](docs/external-oidc-client-setup.md) | +| External OIDC providers | [docs/external-oidc-providers.md](docs/external-oidc-providers.md) | ## License diff --git a/docs/architecture.md b/docs/architecture.md index bf36dd52d..40c64c9e6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -109,7 +109,7 @@ Used by EdgeOps Console, `potctl`, `iofogctl`, and automation. | Certificates | `/api/v3/certificate` | PKI operations | | WebSocket | `ws` routes in `src/routes/` | Exec and logs (Bearer token) | -Browser login uses the OAuth BFF (`GET /api/v3/user/oauth/authorize`); CLI uses `POST /api/v3/user/login`. See [oidc-configuration.md](oidc-configuration.md), [external-oidc-client-setup.md](external-oidc-client-setup.md), and [rbac-reference.md](rbac-reference.md). +Browser login uses the OAuth BFF (`GET /api/v3/user/oauth/authorize`); CLI uses `POST /api/v3/user/login`. See [oidc-configuration.md](oidc-configuration.md), [external-oidc-client-setup.md](external-oidc-client-setup.md), [external-oidc-providers.md](external-oidc-providers.md), and [rbac-reference.md](rbac-reference.md). ### Agent API (fog token — no OIDC) @@ -131,7 +131,7 @@ Full request/response shapes: **`docs/swagger.yaml`** (agent paths). | Job | File | Role | |-----|------|------| -| Fog status | `fog-status-job.js` | Mark agents offline when status pings lapse | +| Fog status | `fog-status-job.js` | Liveness cleaner: chunked transactions mark quiet fogs UNKNOWN using Controller receipt time (`lastStatusTime`). Desired-active microservices become UNKNOWN with CPU/memory 0. Desired-inactive STOPPED workloads are left alone. Does not delete microservices. Overlapping runs on the same process are skipped. | | Controller heartbeat | `controller-heartbeat-job.js` | Control-plane liveness | | Fog token cleanup | `fog-token-cleanup-job.js` | Expire stale agent tokens | | Controller cleanup | `controller-cleanup-job.js` | Orphaned controller MS housekeeping | @@ -139,7 +139,7 @@ Full request/response shapes: **`docs/swagger.yaml`** (agent paths). | NATS reconcile | `nats-reconcile-worker-job.js` | NATS operator sync | | Platform reconcile | `platform-reconcile-worker-job.js` | Fog + service platform claim/reconcile (one job, two queues) | | Fog platform sweep | `fog-platform-sweep-job.js` | Drift detection; re-enqueue stale fog/service tasks | -| Stopped app status | `stopped-app-status-job.js` | Application state maintenance | +| Stopped app status | `stopped-app-status-job.js` | Desired-inactive safety net: leftover RUNNING, STOPPING, DELETED, or DELETING observed status is rewritten to STOPPED with CPU/memory 0. Overlapping runs on the same process are skipped. | ### Platform reconcile (fog + service + resolver) @@ -356,6 +356,12 @@ Controller v3.8 and Edgelet share a **frozen field-agent REST contract** on `/ap **Volume mounts** — `GET volumeMounts` returns bind/volume shapes plus system-injected immutable `serviceAccount` entries. +**Volume mapping scope** — microservice `volumeMappings[]` may include `scope` (`private` default / `shared`). Meaningful only when `type` is `volume`. Shared names are node-global on that fog. Reclaim is on-node (`edgelet volume` / `deprovision --purge-volumes`); Controller does not prune volumes. `deleteWithCleanup` remains unused. + +**Fleet models** — weight artifacts at `/api/v3/models`, linked per fog. Agent `GET /api/v3/agent/models` returns linked rows only. User microservices bind them with a `models` catalog (`bindPath`, `permissions`, `items[].name`). + +**Fleet Knowledge** — documents, datasets, and vector indexes, same distribution shape as models and a separate name namespace. Operator CRUD, YAML (`kind: Knowledge`), and per-fog link live at `/api/v3/knowledge`. User microservices bind a `knowledge` catalog and can change it with `PATCH /api/v3/microservices/:uuid/knowledge`. Agent `GET /api/v3/agent/knowledge` returns linked rows only. `format` is a soft hint. Hugging Face pulls use the dataset API. Fog `modelLastUpdate` and `knowledgeLastUpdate` are Unix milliseconds. + **Service account RBAC** — microservice roles use apiGroup **`edgelet.iofog.org/v1`**; SA/role changes trigger `microserviceList` change tracking. For the full bilateral contract (including ControlPlane env vars and verification references), see Edgelet documentation: @@ -466,6 +472,7 @@ Agent routes and WebSocket exec/logs for agents are **outside** OIDC — see [rb | [pki.md](pki.md) | Central CAs, cert renewal, NATS operator rotation | | [oidc-configuration.md](oidc-configuration.md) | Embedded/external auth modes and env vars | | [external-oidc-client-setup.md](external-oidc-client-setup.md) | External IdP client configuration | +| [external-oidc-providers.md](external-oidc-providers.md) | Default groups/roles and IdP recipes | | [operations/database-transactions.md](operations/database-transactions.md) | Transaction runner, OTEL metrics, SQLITE_BUSY runbook | | [operations/sizing.md](operations/sizing.md) | Hardware sizing by fog count (K8s and Remote CP) | | [CONTRIBUTING](../CONTRIBUTING) | Dual-mirror CI and development | diff --git a/docs/external-oidc-client-setup.md b/docs/external-oidc-client-setup.md index 4e75ea4d0..777e86d88 100644 --- a/docs/external-oidc-client-setup.md +++ b/docs/external-oidc-client-setup.md @@ -1,12 +1,14 @@ -# External OIDC provider — minimum client setup for Controller +# External OIDC provider — client setup for Controller **Audience:** Platform and IdP administrators **Controller mode:** `AUTH_MODE=external` -**Applies to:** Any OIDC-compliant provider (Keycloak, Auth0, Okta, Azure AD, etc.) +**Applies to:** Any OIDC-compliant provider (Keycloak, Microsoft Entra ID, Okta, Auth0, and similar) + +This page is the **OIDC client contract**: client type, grants, redirect URIs, scopes, and access-token rules. For default groups/roles and per-provider recipes, see [external-oidc-providers.md](external-oidc-providers.md). Operator env and login flows: [oidc-configuration.md](oidc-configuration.md). ## Overview -Controller uses one **confidential** OIDC client for browser and CLI authentication when `AUTH_MODE=external`. +Controller uses **one confidential** OIDC client for browser and CLI authentication when `AUTH_MODE=external`. The EdgeOps Console does not talk to the IdP as a public SPA. Controller is an OAuth BFF for the browser and a password-grant front door for the CLI. | Use case | Grant / flow | Controller endpoint | |----------|--------------|---------------------| @@ -14,37 +16,67 @@ Controller uses one **confidential** OIDC client for browser and CLI authenticat | CLI (potctl) | Resource owner password (direct access) | `POST /api/v3/user/login` | | Session refresh | Refresh token | `POST /api/v3/user/refresh` | | Profile | Bearer access token (+ UserInfo) | `GET /api/v3/user/profile` | +| User APIs and operator WebSockets | Bearer access JWT | `/api/v3/*` except agent routes | +| Agent routes | Fog token | `/api/v3/agent/*` — OIDC does **not** apply | + +In external mode: -In external mode, access and refresh tokens are **issued by the IdP**. Controller validates access JWTs via the issuer JWKS and maps claims to RBAC. +- Access and refresh tokens are **issued by the IdP**. Controller does not mint them. +- Controller validates **access** JWTs via the issuer JWKS (`aud` must be `OIDC_CLIENT_ID`). +- There is **no local `AuthUsers` row** for IdP users. RBAC comes from JWT claims (and optional RoleBindings). +- MFA, password policy, and forced password change are **owned by the IdP**. Controller does not run embedded interaction UI in this mode. ## Controller environment (minimum) | Variable | Required | Example | |----------|----------|---------| | `AUTH_MODE` | Yes | `external` | -| `OIDC_ISSUER_URL` | Yes | `https://auth.example.com/realms/myrealm` | +| `OIDC_ISSUER_URL` | Yes | Full issuer URL (see [providers](external-oidc-providers.md)) | | `OIDC_CLIENT_ID` | Yes | `pot-controller` | | `OIDC_CLIENT_SECRET` | Yes | Confidential client secret | | `CONTROLLER_PUBLIC_URL` | Yes | `https://controller.example.com` | | `CONSOLE_URL` | Yes (browser login) | `https://console.example.com` | +| `TRUST_PROXY` | When TLS terminates at ingress | `true` | +| `AUTH_SESSION_STORE_TYPE` | HA + browser login | `database` (mysql/postgres) | | `AUTH_INSECURE_ALLOW_HTTP` | Development only | `true` when using `http://localhost:*` | +`OIDC_ISSUER_URL` must be the **issuer string**, not a generic IdP homepage. Discovery is: + +```text +GET {OIDC_ISSUER_URL}/.well-known/openid-configuration +``` + +Do **not** set legacy `KC_*`, `auth.realm`, or `auth.realmKey`. + +Optional `OIDC_CONSOLE_CLIENT_ID` / `AUTH_CONSOLE_CLIENT_ENABLED` registers a **future SPA-direct** public client. Default is **off**. Primary Console login uses this confidential Controller client via the BFF. Do not point Console at a second public client unless you intend to enable that path. + ## IdP client — required settings ### Client type -- OpenID Connect -- **Confidential client** (client authentication enabled; uses `OIDC_CLIENT_SECRET`) -- Public clients are not supported for the Controller OAuth BFF +Create **one** client for Controller. Do not use a public SPA, native, or device client as this client. + +| Setting | Required value | Why | +|---------|----------------|-----| +| Protocol | OpenID Connect | Controller is OIDC-only | +| Client type | **Confidential** (web application) | BFF uses `OIDC_CLIENT_SECRET` at the token endpoint | +| Client authentication | `client_secret_basic` or `client_secret_post` | Public (`none`) is not supported | +| PKCE | **S256 required or allowed** | Browser authorize always sends `code_challenge_method=S256` | +| Implicit / hybrid | **Off** | Not used | +| Device code | Off | Not used | +| Client credentials grant | Unused | Not used for operator login | ### Enabled grant types / flows | Flow | Required for | Notes | |------|--------------|-------| | Standard flow (authorization code) | Browser Sign in | Required | +| Refresh token (`offline_access`) | Console/CLI session refresh | Required for `POST /api/v3/user/refresh` | | Direct access grants (ROPC) | CLI `POST /user/login` | Required if potctl uses password login against the IdP | | Implicit flow | — | Off (not used) | +If your organization has disabled ROPC (common on Entra ID), browser login still works; CLI password login will not. + ### PKCE Controller always sends PKCE S256 on browser authorize (`code_challenge`, `code_challenge_method=S256`). @@ -58,7 +90,7 @@ Disabling PKCE on the IdP is a development workaround only, not recommended for ### Redirect URIs -Register exactly: +Register exactly as a **Web** redirect URI (not SPA): ```text {CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback @@ -73,19 +105,19 @@ Avoid overly broad wildcards in production. ### Web origins (CORS) -If the Console calls the Controller API from the browser, allow: +If the Console calls the Controller API from a different origin, allow: ```text {CONSOLE_URL} ``` -Example: `http://localhost:3000` +Example: `http://localhost:3000` or `https://console.example.com` ## Scopes ### Requested by Controller (browser OAuth BFF) -Controller sends this scope string on authorize: +Controller sends this scope string on authorize (not configurable): ```text openid profile email groups offline_access @@ -96,7 +128,7 @@ openid profile email groups offline_access | `openid` | OIDC baseline; `sub`, `id_token` | | `profile` | `preferred_username`, display name | | `email` | Email claim; identity linking | -| `groups` | RBAC group membership (see below) | +| `groups` | RBAC group membership — name must be **`groups`**, not `group` | | `offline_access` | Refresh token on authorization code flow | ### IdP client scope assignment @@ -106,36 +138,24 @@ Every requested scope must be assigned to the client as a **default** and/or **o | Scope | Typical assignment | |-------|-------------------| | `openid`, `profile`, `email`, `roles` | Default | -| `groups` | Optional (scope name must be **`groups`**, not `group`) | +| `groups` | Optional (scope name must be **`groups`**) | | `offline_access` | Optional | -**Common error:** `invalid_scope` when a scope is requested but not assigned to the client, or when the scope name is wrong (`group` vs `groups`). - -## RBAC / groups mapping +**Common error:** `invalid_scope` when a scope is requested but not assigned to the client, when the scope name is wrong (`group` vs `groups`), or when the provider only accepts a full URI scope (`api://…/groups`) instead of the short name `groups`. -Controller resolves RBAC **Group** subjects from the access token (in order): +## Access token requirements -1. `resource_access[{OIDC_CLIENT_ID}].roles` — Keycloak client roles (recommended) -2. Top-level `roles` array (if present) -3. `groups` array — from the `groups` scope and a group mapper +Controller validates the **access token**, not the ID token, on API calls. Groups and roles on the ID token alone are ignored for authorization. -Group names are lowercased before RBAC lookup. IdP role and group names should align with Controller RBAC groups (for example `admin`, `viewer`). +The access token **must**: -### Keycloak options +1. Be a **JWT** (opaque tokens fail). +2. Have `iss` equal to the discovered issuer. +3. Have `aud` equal to **`OIDC_CLIENT_ID`** (string or array containing it). +4. Carry RBAC claims on the **access** token (see [external-oidc-providers.md](external-oidc-providers.md)). +5. Not be a refresh token (`token_use` must not be `refresh`). -**Option A — Client roles (simplest)** -Assign roles on the client matching `OIDC_CLIENT_ID`. The default `roles` scope adds `resource_access` to the token. - -**Option B — `groups` scope** -Create a realm client scope named exactly `groups`. Add a Group Membership or Realm Role mapper that emits a `groups` claim. - -### User identity (User subject) - -Controller reads the User subject from JWT claims (first match): - -`preferred_username` → `username` → `email` → `sub` - -Ensure at least one stable identifier is present for RBAC User bindings. +If `aud` is Microsoft Graph, Okta `api://default`, or any API identifier that is **not** the client ID, Bearer validation fails and APIs return 401. ## Issuer discovery (minimum metadata) @@ -149,43 +169,46 @@ The issuer at `OIDC_ISSUER_URL` must expose: | `userinfo_endpoint` | `GET /user/profile` in external mode | | `revocation_endpoint` | Optional; best-effort logout | -## Keycloak checklist +## Groups, roles, and user identity + +Create IdP groups or application roles named `admin`, `sre`, `developer`, and `viewer`. Those names map **directly** to Controller system roles (no RoleBinding required). Full mapping, claim order, and Keycloak / Entra ID / Okta / Auth0 recipes: [external-oidc-providers.md](external-oidc-providers.md). -Example client: `pot-controller` +**User** subject (first match): `preferred_username` → `username` → `email` → `sub`. -| Setting | Value | -|---------|-------| -| Client authentication | On | -| Standard flow | On | -| Direct access grants | On (if CLI login is required) | -| PKCE Method | S256 | -| Valid redirect URIs | `{CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback` | -| Web origins | `{CONSOLE_URL}` | -| Client scopes | `openid`, `profile`, `email`, `roles` (default); `groups`, `offline_access` (optional) | +**Group** subjects from the access token (all lowercased): -### MFA and forced password change +1. `resource_access[{OIDC_CLIENT_ID}].roles` +2. Top-level `roles` array +3. Top-level `groups` array + +## MFA and forced password change - **Browser:** enforced by the IdP during authorize (for example Keycloak required action `UPDATE_PASSWORD`) - **CLI:** IdP password-grant policy applies - Controller does not run embedded interaction UI in external mode +## High availability + +For multiple Controller replicas with browser login, set `AUTH_SESSION_STORE_TYPE=database` (mysql/postgres) so OAuth `state` and the PKCE verifier survive any replica. `memory` is only safe for a single replica. + ## Verification ### Browser -1. Viewer Sign in → IdP login page (no `invalid_scope` or PKCE errors) +1. Console Sign in → IdP login page (no `invalid_scope` or PKCE errors) 2. Callback → `{CONSOLE_URL}/login#accessToken=...&refreshToken=...` -3. `GET /api/v3/user/profile` with `Authorization: Bearer ` → 200 +3. Decode the **access** token: `aud` is `OIDC_CLIENT_ID`; `iss` matches the issuer; `roles` / `groups` / `resource_access` contains a system role name +4. `GET /api/v3/user/profile` with `Authorization: Bearer ` → 200 ### CLI ```bash curl -sS -X POST '{CONTROLLER_PUBLIC_URL}/api/v3/user/login' \ -H 'Content-Type: application/json' \ - -d '{"email":"","password":"","totp":""}' + -d '{"email":"","password":"","totp":""}' ``` -Expect `{ "accessToken", "refreshToken" }` (IdP tokens). +`email` is the IdP **login identifier** (username or UPN). Expect `{ "accessToken", "refreshToken" }` (IdP tokens). ### Refresh @@ -199,12 +222,20 @@ curl -sS -X POST '{CONTROLLER_PUBLIC_URL}/api/v3/user/refresh' \ | Error | Likely cause | |-------|----------------| -| `invalid_scope` | Missing `groups` or `offline_access` on the client; wrong scope name (`group` vs `groups`) | +| `invalid_scope` | Missing `groups` or `offline_access` on the client; wrong scope name (`group` vs `groups`); provider requires a full URI scope | | `Missing parameter: code_challenge_method` | PKCE required on IdP but not sent by Controller (upgrade Controller) | -| `redirect_uri` mismatch | Callback URL not registered exactly | +| `redirect_uri` mismatch | Callback URL not registered exactly; URI registered as SPA instead of Web | | Login works, no `refreshToken` in browser hash | `offline_access` not requested or not assigned on the IdP client | -| 403 on API routes | Token valid but RBAC groups/roles not mapped; check client roles or `groups` claim | - -## Related API documentation - -See `docs/swagger.yaml` for `/user/oauth/authorize`, `/user/oauth/callback`, `/user/login`, `/user/refresh`, and `/user/profile`. +| 401 on API routes after login | Opaque access token; `aud` is not `OIDC_CLIENT_ID`; wrong `iss` | +| 403 on API routes | Token valid but RBAC groups/roles not on the **access** token; see [external-oidc-providers.md](external-oidc-providers.md) | +| CLI login fails, browser works | ROPC / direct access grants disabled | +| OAuth `state` errors on HA | `AUTH_SESSION_STORE_TYPE` still `memory` | + +## Related documentation + +| Document | Topic | +|----------|-------| +| [external-oidc-providers.md](external-oidc-providers.md) | Default groups/roles; Keycloak, Entra ID, Okta, Auth0 recipes | +| [oidc-configuration.md](oidc-configuration.md) | Auth modes and environment variables | +| [rbac-reference.md](rbac-reference.md) | System roles, verbs, RoleBindings | +| [swagger.yaml](swagger.yaml) | `/user/oauth/authorize`, `/user/oauth/callback`, `/user/login`, `/user/refresh`, `/user/profile` | diff --git a/docs/external-oidc-providers.md b/docs/external-oidc-providers.md new file mode 100644 index 000000000..8926c6d8d --- /dev/null +++ b/docs/external-oidc-providers.md @@ -0,0 +1,269 @@ +# External OIDC providers — groups, roles, and IdP recipes + +**Audience:** IdP and platform administrators +**Controller mode:** `AUTH_MODE=external` + +This page maps IdP groups and roles onto Controller RBAC and gives setup recipes for common OpenID Connect providers. Register the OIDC **client** first using [external-oidc-client-setup.md](external-oidc-client-setup.md) (confidential client, PKCE S256, redirect URI, scopes, access-token `aud`). + +System role permissions: [rbac-reference.md](rbac-reference.md). + +## Default groups and roles + +Controller ships four **human** system roles. Names are matched **case-insensitively** and compared in **lowercase**. + +| IdP group or role name | Controller system role | Typical use | +|------------------------|------------------------|-------------| +| `admin` | **admin** | Full cluster administration (`resources: ['*']`, `verbs: ['*']`) | +| `sre` | **sre** | Day-2 operations; read-only on `roles`, `roleBindings`, and some NATS operator objects | +| `developer` | **developer** | CRUD on workloads; read-only on infra | +| `viewer` | **viewer** | `get` / `list` only; no exec, tunnels, or events | + +Create these four names as **groups**, **client roles**, or **application roles** on the IdP, then assign users there. That is the intended drop-in setup. + +Two more system roles exist but must **not** be assigned to human IdP users: + +| Name | Purpose | +|------|---------| +| `agent-admin` | Edgelet service accounts (`edgelet.iofog.org/v1`) | +| `microservice` | Limited self-service for running workloads | + +Embedded mode also seeds `AuthGroups` rows named `admin`, `sre`, `developer`, and `viewer`. Those table rows are for **embedded** user membership. In **external** mode they are not consulted for login; JWT claims are the source of Group subjects. + +### Direct mapping (no RoleBinding) + +If the **access** token contains a group or role whose name is exactly `admin`, `sre`, `developer`, or `viewer`, Controller grants that system role immediately. You do not need a RoleBinding. + +Custom IdP names (`Platform Admins`, Entra group object IDs, Keycloak paths like `/admin`) do **not** match. Either: + +- Change the IdP mapper so the claim value is `admin` / `sre` / `developer` / `viewer`, or +- Create a Controller RoleBinding whose Group subject is the **exact lowercase claim value**. + +### How Controller reads the access token + +Group subjects (`src/lib/rbac/middleware.js`), all lowercased: + +1. `resource_access[{OIDC_CLIENT_ID}].roles` — Keycloak-style **client roles** (recommended on Keycloak) +2. Top-level `roles` array — Entra app roles; many Okta/Auth0 mappings +3. Top-level `groups` array — `groups` scope / group mapper + +Namespaced claims such as `https://controller.example.com/groups` are **not** read. Map them to `groups` or `roles`. + +ID-token-only claims are ignored for API authorization. The same claims must appear on the **access** token. + +User subject (first match): `preferred_username` → `username` → `email` → `sub`. + +If you bind a **User** (not Group) RoleBinding, the `name` must match that first non-empty claim. Entra `preferred_username` is usually the UPN. Keycloak’s is often a short username, not email. + +### Optional RoleBindings for custom names + +When the token cannot emit the four system names: + +```yaml +apiVersion: datasance.com/v3 +kind: RoleBinding +metadata: + name: platform-admins-admin +subjects: + - kind: Group + name: platform-admins +roleRef: + kind: Role + name: admin +``` + +User binding (when groups cannot be put in the token): + +```yaml +apiVersion: datasance.com/v3 +kind: RoleBinding +metadata: + name: alice-developer +subjects: + - kind: User + name: alice@example.com +roleRef: + kind: Role + name: developer +``` + +The User `name` must match the JWT user claim described above. + +## Switching from embedded to external + +1. Export users/groups: `POST /api/v3/auth/migration/export` (no password hashes). +2. Create matching users and groups (`admin` / `sre` / `developer` / `viewer`) on the IdP. +3. Switch env to `AUTH_MODE=external` plus issuer and client credentials ([client setup](external-oidc-client-setup.md)). +4. **External → embedded is not supported.** + +Assign at least one operator to `admin` before cutting over. + +--- + +## Provider recipes + +Every recipe assumes the confidential client, redirect URI, and hardcoded scopes from [external-oidc-client-setup.md](external-oidc-client-setup.md): + +```text +openid profile email groups offline_access +``` + +Access token `aud` must equal `OIDC_CLIENT_ID`. + +### Keycloak + +**Issuer** + +```text +https://auth.example.com/realms/{realm} +``` + +**Client** (`pot-controller` or your `OIDC_CLIENT_ID`) + +| Setting | Value | +|---------|-------| +| Client authentication | On (confidential) | +| Standard flow | On | +| Direct access grants | On (if CLI login is required) | +| Implicit | Off | +| PKCE Method | S256 | +| Valid redirect URIs | `{CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback` | +| Web origins | `{CONSOLE_URL}` | +| Default scopes | `openid`, `profile`, `email`, `roles` | +| Optional scopes | `groups`, `offline_access` | + +**RBAC (pick one)** + +- **Option A (simplest):** Client roles on this client named `admin`, `sre`, `developer`, `viewer`. The default `roles` scope puts them in `resource_access[{clientId}].roles`. +- **Option B:** Realm client scope named exactly `groups`. Group Membership mapper: claim name `groups`, **full group path = OFF** so the claim is `admin`, not `/admin`. + +Realm roles in a top-level `roles` array also work if you add that mapper to the access token. + +### Microsoft Entra ID (Azure AD) + +**Issuer** (v2) + +```text +https://login.microsoftonline.com/{tenant-id}/v2.0 +``` + +**App registration** + +| Setting | Value | +|---------|-------| +| Application type | **Web** (confidential), not SPA | +| Redirect URI | `{CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback` (platform **Web**) | +| Certificates & secrets | Client secret → `OIDC_CLIENT_SECRET` | +| `OIDC_CLIENT_ID` | Application (client) ID | +| Allow public client flows | On **only** if you need ROPC for potctl | +| Implicit | Off | + +**Audience.** Entra issues a Microsoft Graph access token if the client only requests OIDC scopes. Controller then rejects `aud`. To make `aud` the application ID: + +1. **Expose an API** on the same app. +2. Prefer an Application ID URI that yields access tokens whose `aud` is the **application (client) ID**. +3. Add a scope whose **short name is `groups`** so the authorize request is accepted. +4. Authorize this same app to that scope. +5. Confirm a decoded access token has `"aud": ""` and is a JWT. + +If Entra will only accept full-URI scopes (`api:///groups`), authorize returns `invalid_scope` because Controller always requests the short name `groups`. + +**RBAC (recommended: App roles, not directory groups).** Directory `groups` claims are often **object GUIDs**, and large tenants hit the groups overage claim (no array). Controller only reads a `groups` **array** of names. + +Define **App roles** (allowed member type: Users/Groups) with **values**: + +```text +admin +sre +developer +viewer +``` + +Assign users or security groups to those app roles. Entra puts them on the access token as: + +```json +"roles": ["admin"] +``` + +Controller maps `roles[]` to Group subjects. No RoleBinding needed. + +**ROPC:** Microsoft is deprecating this grant. Treat CLI password login as optional; prefer Console OAuth. + +### Okta + +**Issuer** (custom authorization server recommended) + +```text +https://{okta-domain}/oauth2/{authorizationServerId} +``` + +**Application** + +| Setting | Value | +|---------|-------| +| Sign-in method | OIDC | +| Application type | **Web** | +| Grant types | Authorization Code, Refresh Token, and Resource Owner Password if CLI is required | +| Client authentication | Client secret | +| Sign-in redirect | `{CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback` | +| Trusted origins | `{CONSOLE_URL}` as needed | + +**Audience:** the authorization server **Audience** must be the **client ID**. The Okta default `api://default` fails JWT `aud` checks. + +**Scopes:** add a custom scope named `groups` on that authorization server, plus `offline_access`. + +**Claims:** on the **access token**, add claim `groups` (or `roles`) whose value is group names, filtered to `admin`, `sre`, `developer`, `viewer`. ID-token-only claims are not enough. + +Create Okta groups with those same names and assign users. + +### Auth0 (and similar “API + application” IdPs) + +1. Create a **Regular Web Application** (confidential). +2. Create an **API** whose **identifier is exactly `OIDC_CLIENT_ID`** so `aud` matches. +3. On that API, add a permission/scope named `groups`. +4. Enable Authorization Code, Refresh Token, and Password (if CLI). +5. Enable RBAC on the API; add roles named `admin`, `sre`, `developer`, `viewer`. +6. Add a Login / post-login Action that copies roles onto the **access token** as `roles` or `groups` (not a namespaced claim). +7. Without an API audience, Auth0 often returns an **opaque** access token — Controller cannot validate that. + +### Generic OpenID Connect + +Any provider works if it satisfies this contract: + +| Requirement | Value | +|-------------|--------| +| Discovery | `{issuer}/.well-known/openid-configuration` | +| Client | Confidential + secret | +| Code flow | Authorization code + PKCE S256 | +| Refresh | `offline_access` | +| Password grant | Only if CLI login is required | +| Redirect | `{CONTROLLER_PUBLIC_URL}/api/v3/user/oauth/callback` | +| Access token | JWT, `aud` = client ID | +| Groups | `resource_access[clientId].roles` **or** `roles[]` **or** `groups[]` with names `admin` / `sre` / `developer` / `viewer` | + +--- + +## Verification (RBAC) + +After [client-setup verification](external-oidc-client-setup.md#verification): + +1. Decode the access token and confirm one of `resource_access`, `roles`, or `groups` contains `admin`, `sre`, `developer`, or `viewer`. +2. Sign in as `viewer`: list applications succeeds; mutating routes return 403. +3. Sign in as `admin`: admin-only routes (for example JWKS rotate is embedded-only; role mutations) succeed per [rbac-reference.md](rbac-reference.md). + +## Troubleshooting (groups and roles) + +| Symptom | Likely cause | +|---------|----------------| +| 403 after a successful login | Claims on the ID token only; Keycloak **full group path** left on (`/admin` ≠ `admin`); Entra group **GUIDs**; Auth0 namespaced claims | +| Login works, no groups in token | `groups` / `roles` mapper not added to the **access** token; client roles not assigned to the user | +| `aud` is Graph or `api://default` | See Entra and Okta audience notes above | +| User RoleBinding never matches | Binding uses email but JWT User subject is `preferred_username` | + +## Related documentation + +| Document | Topic | +|----------|-------| +| [external-oidc-client-setup.md](external-oidc-client-setup.md) | Confidential client, grants, scopes, redirect URI, access-token `aud` | +| [oidc-configuration.md](oidc-configuration.md) | Auth modes and environment variables | +| [rbac-reference.md](rbac-reference.md) | System roles, verbs, custom RoleBindings | +| [swagger.yaml](swagger.yaml) | User and RBAC HTTP APIs | diff --git a/docs/oidc-configuration.md b/docs/oidc-configuration.md index fa91a692a..8cb8b3a39 100644 --- a/docs/oidc-configuration.md +++ b/docs/oidc-configuration.md @@ -142,7 +142,7 @@ After login, API calls use `Authorization: Bearer `. Controller re **User** subject: `preferred_username` → `username` → `email` → `sub`. -External IdP setup (scopes, PKCE, redirect URIs, Keycloak checklist): [external-oidc-client-setup.md](external-oidc-client-setup.md). +External IdP client (scopes, PKCE, redirect URIs, access-token `aud`): [external-oidc-client-setup.md](external-oidc-client-setup.md). Default groups/roles and Keycloak / Entra ID / Okta / Auth0 recipes: [external-oidc-providers.md](external-oidc-providers.md). --- @@ -203,7 +203,8 @@ See [external-oidc-client-setup.md](external-oidc-client-setup.md) § Verificati | Document | Topic | |----------|-------| -| [external-oidc-client-setup.md](external-oidc-client-setup.md) | External IdP client, scopes, Keycloak checklist | +| [external-oidc-client-setup.md](external-oidc-client-setup.md) | External IdP client, scopes, redirect URI, access-token `aud` | +| [external-oidc-providers.md](external-oidc-providers.md) | Default groups/roles; Keycloak, Entra ID, Okta, Auth0 recipes | | [rbac-reference.md](rbac-reference.md) | Roles, bindings, public routes | | [architecture.md](architecture.md) | Auth modes summary, API surfaces | | [swagger.yaml](swagger.yaml) | User and auth admin endpoints | diff --git a/docs/rbac-reference.md b/docs/rbac-reference.md index 951aafb23..7d54c2c86 100644 --- a/docs/rbac-reference.md +++ b/docs/rbac-reference.md @@ -1,7 +1,7 @@ # Controller RBAC — operator reference **Audience:** Platform operators, SREs, and integrators configuring access to Controller v3.8 -**Applies to:** User and admin HTTP APIs under `/api/v3/*` (not Edgelet agent wire protocol) +**Applies to:** User and admin HTTP APIs under `/api/v3/*` (not Edgelet agent wire protocol). Includes fleet models, fleet Knowledge, RuntimeClass, and microservice templates. ## Overview @@ -32,7 +32,7 @@ Client request | **User** | `preferred_username`, `username`, `email`, or `sub` | | **Group** | `resource_access[].roles`, `roles[]`, `groups[]` (lowercased) | -Group names should match RoleBinding subjects and IdP role names (for example `admin`, `developer`, `viewer`). See [external-oidc-client-setup.md](external-oidc-client-setup.md) for external IdP claim mapping. +Group names should match RoleBinding subjects and IdP role names (for example `admin`, `developer`, `viewer`). Direct mapping of those names (no RoleBinding) and per-provider recipes: [external-oidc-providers.md](external-oidc-providers.md). Client registration: [external-oidc-client-setup.md](external-oidc-client-setup.md). ### Password-change gate @@ -79,13 +79,14 @@ The table below lists **user API** resources in `rbac-resources.yaml` and which | applications | ✓ | ✓ | ✓ | Replaces legacy `flows` | | systemApplications | ✓ | read | ✓ | | | applicationTemplates | ✓ | ✓ | ✓ | | +| microserviceTemplates | ✓ | ✓ | ✓ | | | services | ✓ | ✓ | ✓ | | | router | ✓ | read | ✓ | | | cluster | ✓ | read | ✓ | v3.8 HA controllers | | natsOperator, natsBootstrap, natsHub | read / ✓ | read | read | SRE has full NATS CRUD except operator objects (read) | | natsAccounts, natsUsers, natsAccountRules, natsUserRules | ✓ | ✓ | ✓ | | | catalog, registries | ✓ | ✓ | ✓ | | -| secrets, configMaps, volumeMounts | ✓ | ✓ | ✓ | | +| secrets, configMaps, volumeMounts, models, knowledge, runtimeClasses | ✓ | ✓ | ✓ | Model, Knowledge, and RuntimeClass link uses `patch` (same as volumeMounts). Viewer is get/list only | | tunnels | ✓ | read | — | Viewer intentionally omits exec/tunnel paths | | certificates, capabilities | ✓ | ✓ | ✓ | | | execSessions, logs | ✓ | ✓ | — | | @@ -139,6 +140,10 @@ Some sub-resource actions map to `patch` (for example microservice start/stop). | `POST /api/v3/agent/controller/register` | Added under **`agent`** resource (fog token) | | OIDC auth routes (OAuth BFF, interactions) | Catalogued under **`users`** with auth-only verbs | +Fleet models, fleet Knowledge, RuntimeClass, and microservice templates are user RBAC resources (`models`, `knowledge`, `runtimeClasses`, `microserviceTemplates`). Viewer gets `get`/`list`. SRE, developer, and admin get full CRUD including link (`patch` on `…/:name/link`). Agent `GET /api/v3/agent/models`, `GET /api/v3/agent/knowledge`, and `GET /api/v3/agent/runtimeClasses` are fog-token routes catalogued under **`agent`** for drift auditing only. Catalog `PATCH /api/v3/microservices/:uuid/models` and `PATCH /api/v3/microservices/:uuid/knowledge` stay on the **`microservices`** resource. + +Removed user HAL/USB inventory routes (`GET /api/v3/iofog/:uuid/hal/hw`, `GET /api/v3/iofog/:uuid/hal/usb`) and agent HAL PUT routes have no yaml entries. + Orphan RBAC entries for removed APIs must not reappear. CI and local checks enforce this (see Maintenance). ## Custom roles @@ -157,7 +162,7 @@ Keep `rbac-resources.yaml`, live routes, and system roles aligned when adding or ```bash nvm use 24 -# Compare Express routes to rbac-resources.yaml (243 routes as) +# Compare Express routes to rbac-resources.yaml npm run rbac-audit # grep gates — banned legacy terms must be absent; @@ -186,5 +191,6 @@ Optional CI wiring is planned. | Authorizer | `src/lib/rbac/authorizer.js` | | Route inventory (generated) | `node scripts/route-inventory.js` | | Drift script | `scripts/rbac-audit.js`, `npm run rbac-audit` | -| External IdP groups | `docs/external-oidc-client-setup.md` | +| External IdP client | [external-oidc-client-setup.md](external-oidc-client-setup.md) | +| External IdP groups and providers | [external-oidc-providers.md](external-oidc-providers.md) | | HTTP API spec | `docs/swagger.yaml` | diff --git a/docs/swagger.yaml b/docs/swagger.yaml index cc215ad94..709e0f253 100755 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -1,6 +1,6 @@ openapi : "3.0.0" info: - version: 3.8.2 + version: 3.9.0 title: Cloud Native Controller for Edge Native Workloads paths: /live: @@ -400,52 +400,12 @@ paths: description: Not Authorized "500": description: Internal Server Error - "/iofog/{uuid}/hal/hw": - get: - tags: - - ioFog - summary: Retrieves HAL hardware info - operationId: getFogHalHardwareInfo - parameters: - - in: path - name: uuid - description: ioFog node id - required: true - schema: - type: string - security: - - authToken: [] - responses: - "200": - description: Success - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number - content: - application/json: - schema: - $ref: "#/components/schemas/HalInfo" - "401": - description: Not Authorized - "404": - description: Invalid Node Id - "500": - description: Internal Server Error - "/iofog/{uuid}/hal/usb": + /application: get: tags: - - ioFog - summary: Retrieves HAL USB info - operationId: getFogHalUsbInfo - parameters: - - in: path - name: uuid - description: ioFog node id - required: true - schema: - type: string + - Application + summary: Lists all applications + operationId: listApplication security: - authToken: [] responses: @@ -459,24 +419,29 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/HalInfo" - "401": - description: Not Authorized + $ref: "#/components/schemas/ApplicationListResponse" + "400": + description: Bad Request "404": - description: Invalid Node Id + description: Not Found "500": description: Internal Server Error - /application: - get: + post: tags: - Application - summary: Lists all applications - operationId: listApplication + summary: Creates an application + operationId: createApplication security: - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ApplicationCreateRequest" responses: - "200": - description: Success + "201": + description: Created headers: X-Timestamp: description: FogController server timestamp @@ -485,11 +450,11 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/ApplicationListResponse" + $ref: "#/components/schemas/ApplicationCreateResponse" "400": description: Bad Request - "404": - description: Not Found + "401": + description: Not Authorized "500": description: Internal Server Error /application/system: @@ -615,6 +580,40 @@ paths: "500": description: Internal Server Error "/application/{name}": + put: + tags: + - Application + summary: Updates an application state (microservices, etc.) + operationId: updateApplication + parameters: + - in: path + name: name + description: Application name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ApplicationUpdateRequest" + responses: + "204": + description: Success + headers: + X-Timestamp: + description: FogController server timestamp + schema: + type: number + "400": + description: Bad Request + "404": + description: Not Found + "500": + description: Internal Server Error patch: tags: - Application @@ -633,14 +632,7 @@ paths: content: application/json: schema: - type: object - properties: - name: - type: string - description: - type: string - isActivated: - type: boolean + $ref: "#/components/schemas/ApplicationPatchRequest" required: true responses: "204": @@ -931,7 +923,7 @@ paths: schema: type: object properties: - application: + template: type: string format: binary responses: @@ -1439,88 +1431,82 @@ paths: description: Not Authorized "500": description: Internal Server Error - /agent/tunnel: + /agent/models: get: tags: - Agent - summary: Get an ioFog node tunnel configuration - operationId: getIOFogNodeTunnelConfig + summary: Gets fleet models linked to this fog + description: >- + Fog-token only. Returns models linked to the authenticated fog. + Rows missing uuid or name are omitted. A missing route is treated as + an empty list by the agent. + operationId: getAgentLinkedModels security: - authToken: [] responses: "200": description: Success - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number content: application/json: schema: - $ref: "#/components/schemas/IOFogNodeTunnelConfigResponse" + $ref: "#/components/schemas/AgentModelsListResponse" "401": description: Not Authorized - "404": - description: Tunnel Not Found "500": description: Internal Server Error - /agent/version: + /agent/knowledge: get: tags: - Agent - summary: Get change version command - operationId: getChangeVersion + summary: Gets fleet Knowledge linked to this fog + description: >- + Fog-token only. Returns Knowledge linked to the authenticated fog. + Rows missing uuid or name are omitted. An unlinked fog returns + { "knowledge": [] }. A missing route is treated as an empty list + by the agent. + operationId: getAgentLinkedKnowledge security: - authToken: [] responses: "200": description: Success - headers: - X-Timestamp: - description: FogController server timestamp - schema: - type: number content: application/json: schema: - $ref: "#/components/schemas/VersionCommandResponse" + $ref: "#/components/schemas/AgentKnowledgeListResponse" "401": description: Not Authorized - "404": - description: Version Command Not Found "500": description: Internal Server Error - /agent/hal/hw: - put: + /agent/runtimeClasses: + get: tags: - Agent - summary: Updates HAL hardware info - operationId: putHalHardwareInfo + summary: Gets RuntimeClasses linked to this fog + description: >- + Fog-token only. Returns RuntimeClasses linked to the authenticated fog. + Rows missing name or handler are omitted. Docker and podman agents + store the snapshot and do not apply it. + operationId: getAgentLinkedRuntimeClasses security: - authToken: [] - requestBody: - $ref: "#/components/requestBodies/HalInfo" responses: - "204": + "200": description: Success - headers: - X-Timestamp: - description: FogController server timestamp + content: + application/json: schema: - type: number - "400": - description: Bad Request + $ref: "#/components/schemas/AgentRuntimeClassesListResponse" "401": description: Not Authorized "500": description: Internal Server Error - /agent/hal/usb: + /agent/tunnel: get: tags: - - ioFog - summary: Retrieves HAL USB info - operationId: getAgentHalUsbInfo + - Agent + summary: Get an ioFog node tunnel configuration + operationId: getIOFogNodeTunnelConfig security: - authToken: [] responses: @@ -1534,34 +1520,37 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/HalInfo" + $ref: "#/components/schemas/IOFogNodeTunnelConfigResponse" "401": description: Not Authorized "404": - description: Invalid Node Id + description: Tunnel Not Found "500": description: Internal Server Error - put: + /agent/version: + get: tags: - Agent - summary: Updates HAL USB info - operationId: putHalUsbInfo + summary: Get change version command + operationId: getChangeVersion security: - authToken: [] - requestBody: - $ref: "#/components/requestBodies/HalInfo" responses: - "204": + "200": description: Success headers: X-Timestamp: description: FogController server timestamp schema: type: number - "400": - description: Bad Request + content: + application/json: + schema: + $ref: "#/components/schemas/VersionCommandResponse" "401": description: Not Authorized + "404": + description: Version Command Not Found "500": description: Internal Server Error /agent/delete-node: @@ -1791,6 +1780,12 @@ paths: tags: - Microservices summary: Creates a new microservice + description: >- + Optional template overlay: send template.name plus optional template.variables. + Request fields name, application, and iofogUuid/agentName win over the template. + Catalog models.items[].name are auto-linked to the target fog. Non-empty runtime + must exist as a RuntimeClass row and be in the fog availableRuntimes list. + registryId must resolve to type oci. operationId: createMicroservice security: - authToken: [] @@ -1823,6 +1818,11 @@ paths: tags: - Microservices summary: Creates a new microservice in an Application + description: >- + YAML kind Microservice. Optional spec.template applies the same overlay as + JSON POST /microservices (template.name plus optional template.variables). + Request fields metadata.name, spec.application, and spec.agent.name win over + the template. operationId: createMicroserviceYAML security: - authToken: [] @@ -2710,6 +2710,85 @@ paths: description: Not Found "500": description: Internal Server Error + "/microservices/{uuid}/models": + patch: + tags: + - Microservices + summary: Updates the microservice model catalog + description: >- + Catalog-only update. Sets the microserviceModels change flag only + (does not set microserviceList). Rebuilds the workload when the catalog + goes empty to non-empty or non-empty to empty, or when bindPath or + permissions change. Adding or removing items with the same bindPath + and permissions does not rebuild when the catalog is already non-empty. + Catalog items reference managed model names only. + operationId: patchMicroserviceModels + parameters: + - in: path + name: uuid + description: Microservice Uuid + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceCatalog" + responses: + "204": + description: Catalog updated + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + "/microservices/{uuid}/knowledge": + patch: + tags: + - Microservices + summary: Updates the microservice Knowledge catalog + description: >- + User microservices only. Catalog-only update. Sets the + microserviceKnowledge change flag only (does not set microserviceList). + Rebuilds the workload when the catalog goes empty to non-empty or + non-empty to empty, or when bindPath or permissions change. Adding or + removing items with the same bindPath and permissions does not rebuild + when the catalog is already non-empty. Catalog items reference managed + Knowledge names only. System and controller microservices are rejected. + operationId: patchMicroserviceKnowledge + parameters: + - in: path + name: uuid + description: Microservice Uuid + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/KnowledgeCatalog" + responses: + "204": + description: Catalog updated + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error "/microservices/{uuid}/rebuild": patch: tags: @@ -5504,40 +5583,1234 @@ paths: description: Not Found "500": description: Internal Server Error - /events: + + /models: get: tags: - - Events - summary: List audit events - description: Retrieve audit events with optional filters and pagination. - operationId: listEvents + - Models + summary: Returns fleet models + operationId: listModels security: - authToken: [] - parameters: - - in: query - name: limit - description: Maximum number of events to return (default 200, max 1000) - schema: - type: integer - minimum: 1 - maximum: 1000 - required: false - - in: query - name: offset - description: Number of events to skip before collecting results - schema: - type: integer - minimum: 0 - required: false - - in: query - name: startTime - description: Start of time range (Unix timestamp in ms or ISO 8601) - schema: - type: string - required: false - - in: query - name: endTime - description: End of time range (Unix timestamp in ms or ISO 8601) + responses: + "200": + description: List of fleet models + content: + application/json: + schema: + $ref: "#/components/schemas/ModelsListResponse" + "401": + description: Not Authorized + "500": + description: Internal Server Error + post: + tags: + - Models + summary: Creates a fleet model + description: Server generates uuid. Name is unique and immutable after create. + operationId: createModel + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ModelCreate" + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + /models/yaml: + post: + tags: + - Models + summary: Creates a fleet model from YAML + operationId: createModelYaml + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + model: + type: string + format: binary + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + "/models/{name}": + get: + tags: + - Models + summary: Gets a fleet model + description: Does not include linked fog uuids. Use GET /models/{name}/link. + operationId: getModel + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + patch: + tags: + - Models + summary: Updates a fleet model + description: Name is immutable. + operationId: updateModel + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ModelUpdate" + responses: + "200": + description: Updated + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - Models + summary: Deletes a fleet model + description: Refused when any microservice binds the model name. + operationId: deleteModel + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "202": + description: Accepted + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Model is bound by a microservice + "500": + description: Internal Server Error + + "/models/yaml/{name}": + put: + tags: + - Models + summary: Creates or updates a fleet model from YAML + operationId: upsertModelYaml + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + model: + type: string + format: binary + responses: + "200": + description: Upserted + content: + application/json: + schema: + $ref: "#/components/schemas/Model" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Conflict + "500": + description: Internal Server Error + + "/models/{name}/link": + get: + tags: + - Models + summary: Gets fogs linked to a fleet model + operationId: getModelLink + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + post: + tags: + - Models + summary: Links a fleet model to fog nodes + operationId: linkModel + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - Models + summary: Unlinks a fleet model from fog nodes + description: Refused when a microservice on that fog still binds the model name. + operationId: unlinkModel + parameters: + - in: path + name: name + description: Model name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Model is bound by a microservice on the fog + "500": + description: Internal Server Error + + /knowledge: + get: + tags: + - Knowledge + summary: Returns fleet Knowledge + operationId: listKnowledge + security: + - authToken: [] + responses: + "200": + description: List of fleet Knowledge + content: + application/json: + schema: + $ref: "#/components/schemas/KnowledgeListResponse" + "401": + description: Not Authorized + "500": + description: Internal Server Error + post: + tags: + - Knowledge + summary: Creates fleet Knowledge + description: >- + Server generates uuid. Name is unique and immutable after create. + Names are a separate namespace from Models. Hugging Face pulls use + the dataset API. files is Hugging Face only; a non-empty files array on an + OCI registry is rejected. format is an optional soft hint (see + Knowledge.format). + operationId: createKnowledge + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/KnowledgeCreate" + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/Knowledge" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + /knowledge/yaml: + post: + tags: + - Knowledge + summary: Creates fleet Knowledge from YAML + description: >- + kind Knowledge. spec.registry is an alias of registryId. + apiVersion iofog.org/v3 or datasance.com/v3. + operationId: createKnowledgeYaml + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + knowledge: + type: string + format: binary + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/Knowledge" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + "/knowledge/{name}": + get: + tags: + - Knowledge + summary: Gets fleet Knowledge + description: Does not include linked fog uuids. Use GET /knowledge/{name}/link. + operationId: getKnowledge + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/Knowledge" + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + patch: + tags: + - Knowledge + summary: Updates fleet Knowledge + description: Name is immutable. format follows the same soft-hint rules as create. + operationId: updateKnowledge + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/KnowledgeUpdate" + responses: + "200": + description: Updated + content: + application/json: + schema: + $ref: "#/components/schemas/Knowledge" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - Knowledge + summary: Deletes fleet Knowledge + description: Refused when any microservice binds the Knowledge name. + operationId: deleteKnowledge + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "202": + description: Accepted + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Knowledge is bound by a microservice + "500": + description: Internal Server Error + + "/knowledge/yaml/{name}": + put: + tags: + - Knowledge + summary: Creates or updates fleet Knowledge from YAML + operationId: upsertKnowledgeYaml + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + knowledge: + type: string + format: binary + responses: + "200": + description: Upserted + content: + application/json: + schema: + $ref: "#/components/schemas/Knowledge" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Conflict + "500": + description: Internal Server Error + + "/knowledge/{name}/link": + get: + tags: + - Knowledge + summary: Gets fogs linked to fleet Knowledge + operationId: getKnowledgeLink + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + post: + tags: + - Knowledge + summary: Links fleet Knowledge to fog nodes + operationId: linkKnowledge + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - Knowledge + summary: Unlinks fleet Knowledge from fog nodes + description: Refused when a microservice on that fog still binds the Knowledge name. + operationId: unlinkKnowledge + parameters: + - in: path + name: name + description: Knowledge name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Knowledge is bound by a microservice on the fog + "500": + description: Internal Server Error + + /runtimeClasses: + get: + tags: + - RuntimeClasses + summary: Returns fleet RuntimeClasses + operationId: listRuntimeClasses + security: + - authToken: [] + responses: + "200": + description: List of RuntimeClasses + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClassesListResponse" + "401": + description: Not Authorized + "500": + description: Internal Server Error + post: + tags: + - RuntimeClasses + summary: Creates a RuntimeClass + description: Name plus handler. Built-in catalog rows are not seeded. + operationId: createRuntimeClass + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClassCreate" + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClass" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + /runtimeClasses/yaml: + post: + tags: + - RuntimeClasses + summary: Creates a RuntimeClass from YAML + operationId: createRuntimeClassYaml + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + runtimeClass: + type: string + format: binary + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClass" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + "/runtimeClasses/{name}": + get: + tags: + - RuntimeClasses + summary: Gets a RuntimeClass + description: Does not include linked fog uuids. Use GET /runtimeClasses/{name}/link. + operationId: getRuntimeClass + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClass" + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + patch: + tags: + - RuntimeClasses + summary: Updates a RuntimeClass + description: Name is immutable. Non-catalog handlers may bounce the data plane. + operationId: updateRuntimeClass + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClassUpdate" + responses: + "200": + description: Updated + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClass" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - RuntimeClasses + summary: Deletes a RuntimeClass + description: Refused when any microservice pins runtime to this name. + operationId: deleteRuntimeClass + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "202": + description: Accepted + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: RuntimeClass is pinned by a microservice + "500": + description: Internal Server Error + + "/runtimeClasses/yaml/{name}": + put: + tags: + - RuntimeClasses + summary: Creates or updates a RuntimeClass from YAML + operationId: upsertRuntimeClassYaml + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + runtimeClass: + type: string + format: binary + responses: + "200": + description: Upserted + content: + application/json: + schema: + $ref: "#/components/schemas/RuntimeClass" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Conflict + "500": + description: Internal Server Error + + "/runtimeClasses/{name}/link": + get: + tags: + - RuntimeClasses + summary: Gets fogs linked to a RuntimeClass + operationId: getRuntimeClassLink + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + post: + tags: + - RuntimeClasses + summary: Links a RuntimeClass to fog nodes + description: Attach is refused when fog containerEngine is not edgelet or when the RuntimeClass name is not in the fog availableRuntimes list. + operationId: linkRuntimeClass + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Fog is not edgelet, or request invalid + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - RuntimeClasses + summary: Unlinks a RuntimeClass from fog nodes + description: Refused when a microservice on that fog still pins runtime to this name. + operationId: unlinkRuntimeClass + parameters: + - in: path + name: name + description: RuntimeClass name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkRequest" + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/FogLinkSet" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: RuntimeClass is pinned by a microservice on the fog + "500": + description: Internal Server Error + + /microserviceTemplates: + get: + tags: + - Microservice Templates + summary: Returns microservice templates + operationId: listMicroserviceTemplates + security: + - authToken: [] + responses: + "200": + description: List of microservice templates + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplatesListResponse" + "401": + description: Not Authorized + "500": + description: Internal Server Error + post: + tags: + - Microservice Templates + summary: Creates a microservice template + operationId: createMicroserviceTemplate + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplateCreate" + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplate" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + /microserviceTemplates/yaml: + post: + tags: + - Microservice Templates + summary: Creates a microservice template from YAML + operationId: createMicroserviceTemplateYaml + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + template: + type: string + format: binary + responses: + "200": + description: Created + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplate" + "400": + description: Bad Request + "401": + description: Not Authorized + "409": + description: Name already exists + "500": + description: Internal Server Error + + "/microserviceTemplates/{name}": + get: + tags: + - Microservice Templates + summary: Gets a microservice template + operationId: getMicroserviceTemplate + parameters: + - in: path + name: name + description: Template name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplate" + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + patch: + tags: + - Microservice Templates + summary: Updates a microservice template + description: Name is immutable. + operationId: updateMicroserviceTemplate + parameters: + - in: path + name: name + description: Template name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplateUpdate" + responses: + "200": + description: Updated + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplate" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + delete: + tags: + - Microservice Templates + summary: Deletes a microservice template + operationId: deleteMicroserviceTemplate + parameters: + - in: path + name: name + description: Template name + required: true + schema: + type: string + security: + - authToken: [] + responses: + "202": + description: Accepted + "401": + description: Not Authorized + "404": + description: Not Found + "500": + description: Internal Server Error + + "/microserviceTemplates/yaml/{name}": + put: + tags: + - Microservice Templates + summary: Creates or updates a microservice template from YAML + operationId: upsertMicroserviceTemplateYaml + parameters: + - in: path + name: name + description: Template name + required: true + schema: + type: string + security: + - authToken: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + template: + type: string + format: binary + responses: + "200": + description: Upserted + content: + application/json: + schema: + $ref: "#/components/schemas/MicroserviceTemplate" + "400": + description: Bad Request + "401": + description: Not Authorized + "404": + description: Not Found + "409": + description: Conflict + "500": + description: Internal Server Error + /events: + get: + tags: + - Events + summary: List audit events + description: Retrieve audit events with optional filters and pagination. + operationId: listEvents + security: + - authToken: [] + parameters: + - in: query + name: limit + description: Maximum number of events to return (default 200, max 1000) + schema: + type: integer + minimum: 1 + maximum: 1000 + required: false + - in: query + name: offset + description: Number of events to skip before collecting results + schema: + type: integer + minimum: 0 + required: false + - in: query + name: startTime + description: Start of time range (Unix timestamp in ms or ISO 8601) + schema: + type: string + required: false + - in: query + name: endTime + description: End of time range (Unix timestamp in ms or ISO 8601) schema: type: string required: false @@ -7495,6 +8768,12 @@ tags: description: Manage your services - name: VolumeMounts description: Manage your volume mounts + - name: Models + description: Manage your AI Model Catalogs + - name: Knowledge + description: Manage fleet Knowledge artifacts (documents, datasets, and vector indexes) + - name: RuntimeClasses + description: Manage your runtime classes - name: ConfigMap description: Manage your config maps - name: Events @@ -7592,12 +8871,6 @@ components: $ref: "#/components/schemas/CreateUpdateCatalogItemRequestBody" description: Microservice Catalog Item Info required: true - HalInfo: - content: - application/json: - schema: - $ref: "#/components/schemas/HalInfo" - required: true ApplicationTemplateCreateRequest: content: application/json: @@ -7781,8 +9054,19 @@ components: type: string value: type: string + ApplicationSpecBody: + type: object + properties: + microservices: + type: array + items: + $ref: "#/components/schemas/NewMicroserviceRequest" + natsConfig: + $ref: "#/components/schemas/NatsConfigObject" ApplicationCreateRequest: type: object + required: + - name properties: name: type: string @@ -7790,80 +9074,47 @@ components: type: string isActivated: type: boolean + isSystem: + type: boolean microservices: type: array items: - type: object - properties: - name: - type: string - config: - type: string - annotations: - type: string - catalogItemId: - type: integer - registryId: - type: integer - images: - type: array - items: - properties: - containerImage: - type: string - archId: - type: string - iofogUuid: - type: string - hostNetworkMode: - type: boolean - isPrivileged: - type: boolean - logSize: - type: number - rebuild: - type: boolean - volumeMappings: - type: array - items: - $ref: "#/components/schemas/VolumeMapping" - envs: - type: array - items: - type: object - properties: - key: - type: string - value: - type: string - runAsUser: - type: string - platform: - type: string - runtime: - type: string - cdiDevices: - type: array - items: - type: string - capAdd: - type: array - items: - type: string - capDrop: - type: array - items: - type: string - cmd: - type: array - items: - type: string - ports: - type: array - items: - $ref: "#/components/schemas/PortMappingsRequest" - natsConfig: - $ref: "#/components/schemas/NatsConfigObject" + $ref: "#/components/schemas/NewMicroserviceRequest" + description: >- + Nested microservice objects use the same schema as POST /microservices. + The application field on each item is set from the parent application name. + natsConfig: + $ref: "#/components/schemas/NatsConfigObject" + ApplicationUpdateRequest: + type: object + properties: + name: + type: string + description: Application resource name is immutable; must match path name if provided. + description: + type: string + isActivated: + type: boolean + isSystem: + type: boolean + microservices: + type: array + items: + $ref: "#/components/schemas/NewMicroserviceRequest" + natsConfig: + $ref: "#/components/schemas/NatsConfigObject" + ApplicationPatchRequest: + type: object + properties: + name: + type: string + description: Application resource name is immutable; must match path name if provided. + description: + type: string + isActivated: + type: boolean + isSystem: + type: boolean natsConfig: $ref: "#/components/schemas/NatsConfigObject" ApplicationCreateResponse: @@ -7889,7 +9140,7 @@ components: microservices: type: array items: - $ref: "#/components/schemas/GetMicroservicesResponse" + $ref: "#/components/schemas/MicroserviceInfoResponse" ApplicationListResponse: type: object properties: @@ -7916,86 +9167,7 @@ components: defaultValue: type: string application: - type: object - properties: - microservices: - type: array - items: - type: object - properties: - name: - type: string - config: - type: string - annotations: - type: string - catalogItemId: - type: integer - registryId: - type: integer - images: - type: array - items: - properties: - containerImage: - type: string - archId: - type: string - iofogUuid: - type: string - agentName: - type: string - hostNetworkMode: - type: boolean - isPrivileged: - type: boolean - logSize: - type: number - rebuild: - type: boolean - volumeMappings: - type: array - items: - $ref: "#/components/schemas/VolumeMapping" - envs: - type: array - items: - type: object - properties: - key: - type: string - value: - type: string - runAsUser: - type: number - platform: - type: string - runtime: - type: string - cdiDevices: - type: array - items: - type: string - capAdd: - type: array - items: - type: string - capDrop: - type: array - items: - type: string - cmd: - type: array - items: - type: string - ports: - type: array - items: - $ref: "#/components/schemas/PortMappingsRequest" - natsConfig: - $ref: "#/components/schemas/NatsConfigObject" - natsConfig: - $ref: "#/components/schemas/NatsConfigObject" + $ref: "#/components/schemas/ApplicationSpecBody" ApplicationTemplateCreateResponse: type: object properties: @@ -8020,14 +9192,7 @@ components: description: type: string application: - type: object - properties: - microservices: - type: array - items: - $ref: "#/components/schemas/GetMicroservicesResponse" - natsConfig: - $ref: "#/components/schemas/NatsConfigObject" + $ref: "#/components/schemas/ApplicationSpecBody" ApplicationTemplateListResponse: type: object properties: @@ -8124,6 +9289,9 @@ components: type: number lastStatusTime: type: number + description: >- + Controller receipt time in milliseconds of the last successful + agent status PUT. Not the agent's clock. lastCommandTime: type: number logFileCount: @@ -8148,6 +9316,7 @@ components: type: number diskUsage: type: number + description: Edgelet data-directory usage in GiB, not host filesystem total. cpuUsage: type: number memoryViolation: @@ -8156,8 +9325,33 @@ components: type: boolean cpuViolation: type: boolean + systemAvailableDisk: + type: integer + description: Free space on the agent diskDirectory filesystem in bytes. + systemAvailableMemory: + type: integer + description: Host RAM available in bytes. systemTotalCpu: type: number + description: Host CPU busy percentage 0–100, not logical core count. + systemCpus: + type: integer + description: Host logical CPU count. + systemTotalMemory: + type: integer + description: Host RAM capacity in bytes. + systemTotalDisk: + type: integer + description: diskDirectory filesystem size in bytes. + systemOs: + type: string + description: Host OS family (linux, darwin, windows, …). + systemOsVersion: + type: string + description: Human-oriented distro or release; empty string when unknown. + systemKernelVersion: + type: string + description: Linux kernel release; empty string on non-Linux. catalogItemStatus: type: string repositoryStatus: @@ -8186,14 +9380,12 @@ components: type: number cpuLimit: type: number + minimum: 5 + maximum: 400 logLimit: type: number logDirectory: type: string - bluetoothEnabled: - type: boolean - abstractedHardwareEnabled: - type: boolean version: type: string isReadyToUpgrade: @@ -8204,8 +9396,6 @@ components: type: number changeFrequency: type: number - deviceScanFrequency: - type: number tunnel: type: string watchdogEnabled: @@ -8270,6 +9460,38 @@ components: type: array items: type: string + runtimeClasses: + type: string + description: >- + JSON string of applied RuntimeClasses + (array of { name, handler, source }). Empty array when none. + availableCdiDevices: + type: string + description: JSON string of unique sorted fully-qualified CDI names. + modelStatus: + type: string + description: >- + JSON string of local and managed model status items. Parse for + display. Managed rows include uuid; local rows omit uuid. + activeModels: + type: integer + description: Count of managed fleet models only. + modelLastUpdate: + type: integer + description: Unix milliseconds; 0 when the managed list is empty. + knowledgeStatus: + type: string + description: >- + JSON string of local and managed Knowledge status items. Parse for + display. Managed rows include uuid; local rows omit uuid and set + source to local. source is local or managed. activeKnowledge is + the managed count, not the length of this array. + activeKnowledge: + type: integer + description: Count of managed fleet Knowledge only, not the knowledgeStatus row count. + knowledgeLastUpdate: + type: integer + description: Unix milliseconds; 0 when the managed list is empty. platformStatus: description: Present on GET single fog only; omitted on list responses allOf: @@ -8349,6 +9571,8 @@ components: cpuLimit: type: number default: 80 + minimum: 5 + maximum: 400 format: percent logLimit: type: number @@ -8368,19 +9592,9 @@ components: type: number default: 20 format: seconds - deviceScanFrequency: - type: number - default: 20 - format: seconds - bluetoothEnabled: - type: boolean - default: false watchdogEnabled: type: boolean default: true - abstractedHardwareEnabled: - type: boolean - default: false archId: type: number minimum: 0 @@ -8520,9 +9734,9 @@ components: type: boolean deleteNode: type: boolean - microservicesList: + microserviceList: type: boolean - microservicesConfig: + microserviceConfig: type: boolean registries: type: boolean @@ -8532,6 +9746,27 @@ components: type: boolean routerChanged: type: boolean + volumeMounts: + type: boolean + execSessions: + type: boolean + microserviceLogs: + type: boolean + fogLogs: + type: boolean + models: + type: boolean + runtimeClasses: + type: boolean + microserviceModels: + type: boolean + description: Catalog-only refresh; agent uses GET microservices (no dedicated GET). + knowledge: + type: boolean + description: Replace managed Knowledge from GET /agent/knowledge. + microserviceKnowledge: + type: boolean + description: Catalog-only refresh; agent uses GET microservices (no dedicated GET). IOFogNodeConfig: type: object properties: @@ -8555,6 +9790,8 @@ components: type: number cpuLimit: type: number + minimum: 5 + maximum: 400 logLimit: type: number logDirectory: @@ -8565,8 +9802,6 @@ components: type: number changeFrequency: type: number - deviceScanFrequency: - type: number watchdogEnabled: type: boolean latitude: @@ -8606,6 +9841,8 @@ components: type: number cpuLimit: type: number + minimum: 5 + maximum: 400 logLimit: type: number logDirectory: @@ -8616,8 +9853,6 @@ components: type: number changeFrequency: type: number - deviceScanFrequency: - type: number watchdogEnabled: type: boolean latitude: @@ -8654,6 +9889,7 @@ components: type: number diskUsage: type: number + description: Edgelet data-directory usage in GiB, not host filesystem total. cpuUsage: type: number memoryViolation: @@ -8664,16 +9900,41 @@ components: type: boolean systemAvailableDisk: type: integer + description: Free space on the agent diskDirectory filesystem in bytes. systemAvailableMemory: type: integer + description: Host RAM available in bytes. systemTotalCpu: type: number + description: Host CPU busy percentage 0–100, not logical core count. + systemCpus: + type: integer + description: Host logical CPU count. + systemTotalMemory: + type: integer + description: Host RAM capacity in bytes. + systemTotalDisk: + type: integer + description: diskDirectory filesystem size in bytes. + systemOs: + type: string + description: Host OS family (linux, darwin, windows, …). + systemOsVersion: + type: string + description: Human-oriented distro or release; empty string when unknown. + systemKernelVersion: + type: string + description: Linux kernel release; empty string on non-Linux. securityStatus: type: string securityViolationInfo: type: string microserviceStatus: type: string + description: >- + JSON array of per-microservice observed status objects. Items may + include additive lastError, lastErrorAt, and restartCount. Unknown + keys must not fail this PUT. repositoryCount: type: number repositoryStatus: @@ -8682,6 +9943,9 @@ components: type: number lastStatusTime: type: number + description: >- + Agent-supplied timestamp in milliseconds. Controller liveness uses + the PUT receipt time instead of this value. ipAddress: type: string ipAddressExternal: @@ -8710,6 +9974,46 @@ components: type: number gpsStatus: type: string + runtimeClasses: + type: array + description: Applied RuntimeClasses only, sorted by name. Docker/podman send []. + items: + type: object + properties: + name: + type: string + handler: + type: string + source: + type: string + enum: + - local + - managed + availableCdiDevices: + type: array + items: + type: string + description: Unique sorted fully-qualified CDI names (nvidia.com/gpu=0). + modelStatus: + type: string + description: JSON string of local and managed model status items. + activeModels: + type: integer + description: Count of managed fleet models only. + modelLastUpdate: + type: integer + description: Unix milliseconds; 0 when the managed list is empty. + knowledgeStatus: + type: string + description: >- + JSON string of local and managed Knowledge status items. Managed + rows include uuid; local rows omit uuid. source is local or managed. + activeKnowledge: + type: integer + description: Count of managed fleet Knowledge only, not the knowledgeStatus row count. + knowledgeLastUpdate: + type: integer + description: Unix milliseconds; 0 when the managed list is empty. IOFogNodeTunnelConfigResponse: type: object properties: @@ -8793,6 +10097,48 @@ components: type: array items: type: string + description: Alias of commands. commands wins when both are present. + commands: + type: array + items: + type: string + entrypoint: + type: array + items: + type: string + runAsGroup: + type: string + readOnlyRootFilesystem: + type: boolean + workingDir: + type: string + cpus: + type: number + memoryReservation: + type: integer + memorySwap: + type: integer + shmSize: + type: integer + sysctls: + type: object + ulimits: + type: object + devices: + type: array + items: + $ref: "#/components/schemas/ContainerDevice" + tmpfs: + type: array + items: + $ref: "#/components/schemas/ContainerTmpfs" + models: + $ref: "#/components/schemas/MicroserviceCatalog" + knowledge: + $ref: "#/components/schemas/KnowledgeCatalog" + description: >- + Fleet Knowledge bound at {bindPath}/{name}/. Items are Knowledge + names only. idConsumer: type: boolean isRouter: @@ -8950,6 +10296,18 @@ components: accessMode: type: string example: rw + scope: + type: string + enum: + - private + - shared + default: private + description: >- + Meaningful only when type is volume. Omit, null, or empty stores + private. Unknown values on type volume return 400. Other mapping + types always store private. Runtime accepts any case and stores + lowercase. + example: private VolumeMappingResponse: type: object properties: @@ -8964,6 +10322,10 @@ components: type: number hostDestination: type: string + description: >- + When type is volume, a managed volume name + ([a-zA-Z0-9][a-zA-Z0-9_.-]*). When type is bind (default), an + absolute host path (Unix or Windows drive path). example: /var/dest containerDestination: type: string @@ -8974,6 +10336,18 @@ components: type: type: string example: bind + scope: + type: string + enum: + - private + - shared + default: private + description: >- + Meaningful only when type is volume. Omit, null, or empty stores + private. Unknown values on type volume return 400. Other mapping + types always store private. Runtime accepts any case and stores + lowercase. GET always returns private or shared in lowercase. + example: private PortMappingsResponse: type: object properties: @@ -9077,8 +10451,25 @@ components: type: string userEmail: type: string + type: + type: string + enum: + - oci + - hf + default: oci + ca: + type: string + description: Optional base64-encoded PEM CA bundle + insecure: + type: boolean + default: false RegistryBody: type: object + description: >- + type defaults to oci. Microservice and catalog-item registryId must be + type oci. Hugging Face registries are for fleet models. ca is additional + trust and does not replace system CAs. insecure true allows http and + skips TLS verify. Image-pull ca/insecure apply on edgelet only. properties: url: type: string @@ -9090,6 +10481,18 @@ components: type: string email: type: string + type: + type: string + enum: + - oci + - hf + default: oci + ca: + type: string + description: Optional base64-encoded PEM CA bundle + insecure: + type: boolean + default: false ActionBody: type: object properties: @@ -9202,16 +10605,33 @@ components: type: string name: type: string + application: + type: string + flowId: + type: integer + description: Deprecated alias of applicationId for API retrocompatibility. config: type: string annotations: type: string + catalogItemId: + type: integer + registryId: + type: integer hostNetworkMode: type: boolean isPrivileged: type: boolean - logLimit: + logSize: type: number + rebuild: + type: boolean + isActivated: + type: boolean + schedule: + type: integer + minimum: 0 + maximum: 100 images: type: array items: @@ -9219,24 +10639,43 @@ components: containerImage: type: string archId: - type: string + type: integer iofogUuid: type: string - envs: + env: type: array items: - type: object - properties: - key: - type: string - value: - type: string + $ref: "#/components/schemas/MicroserviceEnvVar" + extraHosts: + type: array + items: + $ref: "#/components/schemas/MicroserviceExtraHost" runAsUser: type: string + runAsGroup: + type: string + readOnlyRootFilesystem: + type: boolean platform: type: string runtime: - type: string + type: string + pidMode: + type: string + ipcMode: + type: string + cpuSetCpus: + type: string + memoryLimit: + type: integer + cpus: + type: number + memoryReservation: + type: integer + memorySwap: + type: integer + shmSize: + type: integer cdiDevices: type: array items: @@ -9253,6 +10692,16 @@ components: type: array items: type: string + commands: + type: array + items: + type: string + entrypoint: + type: array + items: + type: string + workingDir: + type: string volumeMappings: type: array items: @@ -9261,6 +10710,8 @@ components: type: array items: $ref: "#/components/schemas/PortMappingsResponse" + healthCheck: + $ref: "#/components/schemas/MicroserviceHealthCheck" natsAccess: type: boolean natsConfig: @@ -9270,18 +10721,135 @@ components: properties: status: type: string + description: >- + Observed runtime state such as RUNNING, STOPPING, STOPPED, + UNKNOWN, or FAILED. Operator stop or application deactivate + sets STOPPING immediately, before the agent confirms STOPPED. operatingDuration: type: number + description: Live while status is RUNNING; otherwise 0. startTime: type: number + description: Live while status is RUNNING; otherwise 0. cpuUsage: type: number + description: >- + CPU sample for this microservice. Live only when status is + RUNNING (0 is valid while running). Otherwise 0. memoryUsage: type: number + description: >- + Memory sample for this microservice. Live only when status is + RUNNING (0 is valid while running). Otherwise 0. containerId: type: string + podId: + type: string + description: >- + Pause/sandbox id on edgelet. Same as containerId on docker/podman + when set. Omitted when unknown. + errorMessage: + type: string + description: >- + Current failure text. Agent keeps this populated while the + workload is failing, restarting, or has been RUNNING for less + than 30 seconds after a crash. After 30 seconds of continuous + RUNNING the agent sends an empty string so this field clears. + lastError: + type: string + description: >- + Last crash text. Not cleared on recovery. Empty string when + Controller has never stored a crash for this microservice. + lastErrorAt: + type: integer + description: >- + Unix milliseconds for lastError. 0 when lastError is empty. + restartCount: + type: integer + description: >- + Restart events since the last operator rebuild. 0 after rebuild + leftover history or when never restarted. May drop to 1 after + an agent process restart if the container is still crashed + (new episode). + execStatus: + type: object + models: + $ref: "#/components/schemas/MicroserviceCatalog" + knowledge: + $ref: "#/components/schemas/KnowledgeCatalog" + description: >- + Fleet Knowledge bound at {bindPath}/{name}/. Items are Knowledge + names only. User microservices only. + sysctls: + type: object + ulimits: + type: object + devices: + type: array + items: + $ref: "#/components/schemas/ContainerDevice" + tmpfs: + type: array + items: + $ref: "#/components/schemas/ContainerTmpfs" + serviceAccount: + type: object + nullable: true + properties: + roleRef: + $ref: "#/components/schemas/RoleRef" + additionalProperties: false + MicroserviceEnvVar: + type: object + required: + - key + properties: + key: + type: string + value: + type: string + valueFromSecret: + type: string + description: Format secret-name/key + valueFromConfigMap: + type: string + description: Format configmap-name/key + MicroserviceExtraHost: + type: object + required: + - name + - address + properties: + name: + type: string + address: + type: string + value: + type: string + description: Present on GET responses when address is a template expression. + MicroserviceHealthCheck: + type: object + required: + - test + properties: + test: + type: array + items: + type: string + interval: + type: integer + timeout: + type: integer + startPeriod: + type: integer + startInterval: + type: integer + retries: + type: integer NewMicroserviceRequest: type: object + required: + - name properties: name: type: string @@ -9291,14 +10859,18 @@ components: type: string catalogItemId: type: integer + registryId: + type: integer images: type: array + maxItems: 4 + minItems: 1 items: properties: containerImage: type: string archId: - type: string + type: integer application: type: string iofogUuid: @@ -9310,7 +10882,18 @@ components: isPrivileged: type: boolean logSize: - type: number + type: integer + minimum: 0 + rebuild: + type: boolean + schedule: + type: integer + minimum: 0 + maximum: 100 + pidMode: + type: string + ipcMode: + type: string volumeMappings: type: array items: @@ -9319,12 +10902,38 @@ components: type: array items: $ref: "#/components/schemas/PortMappingsRequest" + env: + type: array + items: + $ref: "#/components/schemas/MicroserviceEnvVar" + extraHosts: + type: array + items: + $ref: "#/components/schemas/MicroserviceExtraHost" runAsUser: type: string + runAsGroup: + type: string + readOnlyRootFilesystem: + type: boolean platform: type: string runtime: - type: string + type: string + cpuSetCpus: + type: string + memoryLimit: + type: integer + cpus: + type: number + memoryReservation: + type: integer + minimum: 1 + memorySwap: + type: integer + shmSize: + type: integer + minimum: 1 cdiDevices: type: array items: @@ -9341,8 +10950,45 @@ components: type: array items: type: string + description: Alias of commands. commands wins when both are present. + commands: + type: array + items: + type: string + entrypoint: + type: array + items: + type: string + workingDir: + type: string + sysctls: + type: object + ulimits: + type: object + devices: + type: array + items: + $ref: "#/components/schemas/ContainerDevice" + tmpfs: + type: array + items: + $ref: "#/components/schemas/ContainerTmpfs" + healthCheck: + $ref: "#/components/schemas/MicroserviceHealthCheck" + models: + $ref: "#/components/schemas/MicroserviceCatalog" + knowledge: + $ref: "#/components/schemas/KnowledgeCatalog" + description: >- + Fleet Knowledge bound at {bindPath}/{name}/. Items are Knowledge + names only. User microservices only. + template: + $ref: "#/components/schemas/MicroserviceTemplateRef" natsConfig: $ref: "#/components/schemas/NatsConfigObject" + natsEnabled: + type: boolean + description: YAML alias for natsConfig.natsAccess serviceAccount: type: object properties: @@ -9368,26 +11014,62 @@ components: type: boolean isPrivileged: type: boolean - logLimit: - type: number + logSize: + type: integer + minimum: 0 + schedule: + type: integer + minimum: 0 + maximum: 100 + pidMode: + type: string + ipcMode: + type: string volumeMappings: type: array items: $ref: "#/components/schemas/VolumeMapping" images: type: array + maxItems: 4 items: properties: containerImage: type: string archId: - type: string + type: integer + env: + type: array + items: + $ref: "#/components/schemas/MicroserviceEnvVar" + extraHosts: + type: array + items: + $ref: "#/components/schemas/MicroserviceExtraHost" runAsUser: type: string + runAsGroup: + type: string + readOnlyRootFilesystem: + type: boolean platform: type: string runtime: - type: string + type: string + cpuSetCpus: + type: string + memoryLimit: + type: integer + cpus: + type: number + memoryReservation: + type: integer + minimum: 1 + memorySwap: + type: integer + shmSize: + type: integer + minimum: 1 cdiDevices: type: array items: @@ -9404,8 +11086,45 @@ components: type: array items: type: string + description: Alias of commands. commands wins when both are present. + commands: + type: array + items: + type: string + entrypoint: + type: array + items: + type: string + workingDir: + type: string + sysctls: + type: object + ulimits: + type: object + devices: + type: array + items: + $ref: "#/components/schemas/ContainerDevice" + tmpfs: + type: array + items: + $ref: "#/components/schemas/ContainerTmpfs" + healthCheck: + $ref: "#/components/schemas/MicroserviceHealthCheck" + models: + $ref: "#/components/schemas/MicroserviceCatalog" + knowledge: + $ref: "#/components/schemas/KnowledgeCatalog" + description: >- + Fleet Knowledge bound at {bindPath}/{name}/. Items are Knowledge + names only. User microservices only. + template: + $ref: "#/components/schemas/MicroserviceTemplateRef" natsConfig: $ref: "#/components/schemas/NatsConfigObject" + natsEnabled: + type: boolean + description: YAML alias for natsConfig.natsAccess serviceAccount: type: object properties: @@ -10165,13 +11884,6 @@ components: type: string description: Target semver for upgrade or rollback pattern: ^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:[+]([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ - HalInfo: - type: object - required: - - info - properties: - info: - type: string PostTrackingRequest: type: array items: @@ -10685,6 +12397,391 @@ components: minItems: 1 required: - fogUuids + FogLinkRequest: + type: object + properties: + fogUuids: + type: array + items: + type: string + minItems: 1 + required: + - fogUuids + FogLinkSet: + type: object + properties: + fogUuids: + type: array + items: + type: string + Model: + type: object + properties: + uuid: + type: string + name: + type: string + repo: + type: string + description: Upstream path without host + revision: + type: string + description: Pin. Empty means latest (OCI) or main (HF). + registryId: + type: integer + files: + type: array + items: + type: string + description: Hugging Face only. Ignored for OCI registries. + format: + type: string + enum: + - gguf + - safetensors + - onnx + - pytorch + - tensorrt + - unknown + ModelsListResponse: + type: object + properties: + models: + type: array + items: + $ref: "#/components/schemas/Model" + AgentModelsListResponse: + type: object + properties: + models: + type: array + items: + $ref: "#/components/schemas/Model" + ModelCreate: + type: object + required: + - name + - repo + - registryId + properties: + name: + type: string + repo: + type: string + revision: + type: string + registryId: + type: integer + files: + type: array + items: + type: string + format: + type: string + enum: + - gguf + - safetensors + - onnx + - pytorch + - tensorrt + - unknown + ModelUpdate: + type: object + properties: + repo: + type: string + revision: + type: string + registryId: + type: integer + files: + type: array + items: + type: string + format: + type: string + enum: + - gguf + - safetensors + - onnx + - pytorch + - tensorrt + - unknown + Knowledge: + type: object + properties: + uuid: + type: string + name: + type: string + description: DNS-1123. Unique on the controller and immutable. Separate namespace from Models. + repo: + type: string + description: Hub dataset id or OCI path without host + revision: + type: string + description: Optional pin. Empty means latest (OCI) or main (HF). + registryId: + type: integer + description: Registry row. Hugging Face uses the dataset API. OCI uses ORAS. + files: + type: array + items: + type: string + description: Hugging Face only. Empty or omitted means the full dataset. Non-empty files on an OCI registry is rejected. + format: + type: string + description: >- + Optional soft hint. Known values persist lowercase: markdown, pdf, + jsonl, parquet, arrow, sqlite, faiss, chroma, lance. Any other + non-empty string is stored as unknown. Omit, null, or empty stays + omitted. Unknown format is not rejected. + KnowledgeListResponse: + type: object + properties: + knowledge: + type: array + items: + $ref: "#/components/schemas/Knowledge" + AgentKnowledgeListResponse: + type: object + properties: + knowledge: + type: array + items: + $ref: "#/components/schemas/Knowledge" + KnowledgeCreate: + type: object + required: + - name + - repo + - registryId + properties: + name: + type: string + repo: + type: string + revision: + type: string + registryId: + type: integer + files: + type: array + items: + type: string + format: + type: string + description: >- + Optional soft hint. Known values persist lowercase: markdown, pdf, + jsonl, parquet, arrow, sqlite, faiss, chroma, lance. Any other + non-empty string is stored as unknown. Omit, null, or empty stays + omitted. Unknown format is not rejected. + KnowledgeUpdate: + type: object + properties: + repo: + type: string + revision: + type: string + registryId: + type: integer + files: + type: array + items: + type: string + format: + type: string + description: >- + Optional soft hint. Known values persist lowercase: markdown, pdf, + jsonl, parquet, arrow, sqlite, faiss, chroma, lance. Any other + non-empty string is stored as unknown. Omit, null, or empty stays + omitted. Unknown format is not rejected. + KnowledgeCatalogItem: + type: object + required: + - name + properties: + name: + type: string + description: Managed fleet Knowledge name. Never uuid or a host path. + KnowledgeCatalog: + type: object + properties: + bindPath: + type: string + description: Required when items is non-empty. A non-empty bindPath requires at least one item. Absolute container path. Projection is {bindPath}/{name}/. + permissions: + type: string + enum: + - ro + - rw + default: ro + items: + type: array + items: + $ref: "#/components/schemas/KnowledgeCatalogItem" + RuntimeClass: + type: object + properties: + name: + type: string + handler: + type: string + RuntimeClassesListResponse: + type: object + properties: + runtimeClasses: + type: array + items: + $ref: "#/components/schemas/RuntimeClass" + AgentRuntimeClassesListResponse: + type: object + properties: + runtimeClasses: + type: array + items: + $ref: "#/components/schemas/RuntimeClass" + RuntimeClassCreate: + type: object + required: + - name + - handler + properties: + name: + type: string + handler: + type: string + RuntimeClassUpdate: + type: object + properties: + handler: + type: string + MicroserviceCatalogItem: + type: object + required: + - name + properties: + name: + type: string + description: Managed fleet model name. Never uuid or a host path. + MicroserviceCatalog: + type: object + properties: + bindPath: + type: string + description: Required when items is non-empty. A non-empty bindPath requires at least one item. Absolute container path. + permissions: + type: string + enum: + - ro + - rw + default: ro + items: + type: array + items: + $ref: "#/components/schemas/MicroserviceCatalogItem" + ContainerDevice: + type: object + required: + - hostPath + - containerPath + properties: + hostPath: + type: string + containerPath: + type: string + permissions: + type: string + ContainerTmpfs: + type: object + required: + - containerPath + properties: + containerPath: + type: string + size: + type: integer + mode: + type: string + MicroserviceTemplateVariable: + type: object + required: + - key + properties: + key: + type: string + description: + type: string + defaultValue: {} + MicroserviceTemplate: + type: object + properties: + name: + type: string + description: + type: string + variables: + type: array + items: + $ref: "#/components/schemas/MicroserviceTemplateVariable" + microservice: + type: object + description: Microservice spec without instance identity fields. + MicroserviceTemplatesListResponse: + type: object + properties: + microserviceTemplates: + type: array + items: + $ref: "#/components/schemas/MicroserviceTemplate" + MicroserviceTemplateCreate: + type: object + required: + - name + - microservice + properties: + name: + type: string + description: + type: string + variables: + type: array + items: + $ref: "#/components/schemas/MicroserviceTemplateVariable" + microservice: + type: object + MicroserviceTemplateUpdate: + type: object + properties: + description: + type: string + variables: + type: array + items: + $ref: "#/components/schemas/MicroserviceTemplateVariable" + microservice: + type: object + MicroserviceTemplateRef: + type: object + required: + - name + properties: + name: + type: string + variables: + description: Map of key to value, or array of { key, value }. + oneOf: + - type: object + additionalProperties: true + - type: array + items: + type: object + properties: + key: + type: string + value: {} + required: + - key RbacRule: type: object properties: diff --git a/package-lock.json b/package-lock.json index a837a696c..5fde23393 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "controller", - "version": "3.8.3", + "version": "3.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "controller", - "version": "3.8.3", + "version": "3.9.0", "hasInstallScript": true, "license": "EPL-2.0", "dependencies": { @@ -44,10 +44,10 @@ "helmet": "7.1.0", "is-elevated": "3.0.0", "jose": "^4.15.9", - "js-yaml": "4.3.1", + "js-yaml": "4.3.2", "jsonschema": "1.5.0", "moment": "2.30.1", - "multer": "2.2.0", + "multer": "2.3.0", "mysql2": "3.23.1", "nconf": "0.12.1", "node-fetch-npm": "^2.0.4", @@ -78,7 +78,7 @@ "chai-as-promised": "7.1.2", "chai-http": "4.4.0", "eslint": "8.57.1", - "js-yaml": "4.3.1", + "js-yaml": "4.3.2", "mocha": "11.7.6", "mocha-junit-reporter": "2.2.1", "nyc": "18.0.0", @@ -8626,9 +8626,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.1.tgz", - "integrity": "sha512-CY6crGq313MX8GkwvB7tzgp99vjQxY1++5y10/BKN/GUfHqWaOGQMNZkBvqSzsZKWk/ijwHlWzzkLulsGHhjWQ==", + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", "funding": [ { "type": "github", @@ -9628,9 +9628,9 @@ "license": "MIT" }, "node_modules/multer": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/multer/-/multer-2.2.0.tgz", - "integrity": "sha512-6rdyFg2kLrMh9Jee7/BMPuV9lEAd7lLW2YUpF9/YxR7njyoUwwQ0ZPh3TaIY50Sw6vlyD2HW3wGOkTS4P79xrQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/multer/-/multer-2.3.0.tgz", + "integrity": "sha512-cjNbm3sttszgZeGfJR124D+jFEfkXCVAsoPBmFn9X7UxmDSFHWqE2CoEj0vrmSpuAFnqWR1Szcm9QTsiHr60Xw==", "license": "MIT", "dependencies": { "append-field": "^1.0.0", diff --git a/package.json b/package.json index 4aa290875..8409d2fa3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "controller", - "version": "3.8.3", + "version": "3.9.0", "description": "A cloud-native operations controller for managing edge computing workloads, edgelet nodes, and deployments across Eclipse IoFog and Datasance PoT.", "main": "./src/main.js", "author": "Eclipse ioFog Project", @@ -58,6 +58,7 @@ "test:all": "node scripts/run-test.js test-all", "test:k8s-client": "node scripts/run-test.js test test/integration/k8s-client-integration.test.js", "test:integration:first-fog": "RUN_INTEGRATION=1 node scripts/run-test.js test test/src/integration/first-fog-reconcile-sqlite.test.js", + "test:integration:fleet-models": "RUN_INTEGRATION=1 node scripts/run-test.js test test/src/integration/fleet-models-runtime-sqlite.test.js", "test:ws-load": "node test/load/ws-pairing-load.js", "test:load:tx": "node test/load/transaction-safety-load.js --fogs 200 --soak-minutes 5", "precli-tests": "npm run lint", @@ -109,10 +110,10 @@ "helmet": "7.1.0", "is-elevated": "3.0.0", "jose": "^4.15.9", - "js-yaml": "4.3.1", + "js-yaml": "4.3.2", "jsonschema": "1.5.0", "moment": "2.30.1", - "multer": "2.2.0", + "multer": "2.3.0", "mysql2": "3.23.1", "@nats-io/transport-node": "^3.4.0", "nconf": "0.12.1", @@ -141,7 +142,7 @@ "chai-as-promised": "7.1.2", "chai-http": "4.4.0", "eslint": "8.57.1", - "js-yaml": "4.3.1", + "js-yaml": "4.3.2", "mocha": "11.7.6", "mocha-junit-reporter": "2.2.1", "nyc": "18.0.0", @@ -163,7 +164,7 @@ "overrides": { "tar": "^7.5.19", "protobufjs": "^7.6.5", - "js-yaml": "4.3.1", + "js-yaml": "4.3.2", "brace-expansion": "5.0.9", "uuid": "11.1.1", "diff": "8.0.3", diff --git a/scripts/build-console-dev.js b/scripts/build-console-dev.js index e9ee93590..82c41bf61 100644 --- a/scripts/build-console-dev.js +++ b/scripts/build-console-dev.js @@ -9,7 +9,7 @@ const CONSOLE_DIR = path.join(DEV_DIR, 'console') const BUILD_OUT = path.join(CONSOLE_DIR, 'build') const REPO = process.env.EDGEOPS_CONSOLE_REPO || 'https://github.com/Datasance/edgeops-console' -const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.0.13' +const VERSION = process.env.EDGEOPS_CONSOLE_VERSION || 'v1.1.0-rc.6' const FLAVOR = process.env.EDGEOPS_CONSOLE_FLAVOR || 'datasance' function normalizeTag (version) { diff --git a/src/config/config.yaml b/src/config/config.yaml index 110fb8b30..f13eb8a64 100644 --- a/src/config/config.yaml +++ b/src/config/config.yaml @@ -75,6 +75,7 @@ settings: # defaultJobInterval: 120 # Default job interval in seconds fogStatusUpdateInterval: 30 # Fog status update interval in seconds fogStatusUpdateTolerance: 3 # Fog status update tolerance + fogStatusLivenessChunkSize: 50 # Fogs processed per quiet-node liveness transaction (default: 50) fogExpiredTokenCleanupInterval: 300 # Fog expired token cleanup interval in seconds eventRetentionDays: 7 # Days to retain events (default: 7) eventCleanupInterval: 86400 # Cleanup job interval in seconds (default: 24 hours) diff --git a/src/config/env-mapping.js b/src/config/env-mapping.js index 870f00cc6..4d7069c02 100644 --- a/src/config/env-mapping.js +++ b/src/config/env-mapping.js @@ -58,6 +58,7 @@ module.exports = { // Settings Configuration FOG_STATUS_UPDATE_INTERVAL: 'settings.fogStatusUpdateInterval', FOG_STATUS_UPDATE_TOLERANCE: 'settings.fogStatusUpdateTolerance', + FOG_STATUS_LIVENESS_CHUNK_SIZE: 'settings.fogStatusLivenessChunkSize', FOG_EXPIRED_TOKEN_CLEANUP_INTERVAL: 'settings.fogExpiredTokenCleanupInterval', EVENT_RETENTION_DAYS: 'settings.eventRetentionDays', EVENT_CLEANUP_INTERVAL: 'settings.eventCleanupInterval', diff --git a/src/config/rbac-resources.yaml b/src/config/rbac-resources.yaml index cd2db3be1..ef5e1663d 100644 --- a/src/config/rbac-resources.yaml +++ b/src/config/rbac-resources.yaml @@ -55,6 +55,14 @@ resources: methods: DELETE: [patch] resourceNameParam: uuid + - path: /api/v3/microservices/:uuid/models + methods: + PATCH: [patch] + resourceNameParam: uuid + - path: /api/v3/microservices/:uuid/knowledge + methods: + PATCH: [patch] + resourceNameParam: uuid - path: /api/v3/microservices/:uuid/start methods: PATCH: [patch] @@ -138,14 +146,6 @@ resources: methods: POST: [patch] resourceNameParam: uuid - - path: /api/v3/iofog/:uuid/hal/hw - methods: - GET: [get] - resourceNameParam: uuid - - path: /api/v3/iofog/:uuid/hal/usb - methods: - GET: [get] - resourceNameParam: uuid - path: /api/v3/iofog/:uuid/prune methods: POST: [patch] @@ -529,6 +529,112 @@ resources: DELETE: [patch] resourceNameParam: name + # Fleet models + models: + basePath: /api/v3/models + routes: + - path: /api/v3/models + methods: + GET: [list] + POST: [create] + - path: /api/v3/models/yaml + methods: + POST: [create] + - path: /api/v3/models/:name + methods: + GET: [get] + PATCH: [patch] + DELETE: [delete] + resourceNameParam: name + - path: /api/v3/models/yaml/:name + methods: + PUT: [update] + resourceNameParam: name + - path: /api/v3/models/:name/link + methods: + GET: [get] + POST: [patch] + DELETE: [patch] + resourceNameParam: name + + # Fleet Knowledge + knowledge: + basePath: /api/v3/knowledge + routes: + - path: /api/v3/knowledge + methods: + GET: [list] + POST: [create] + - path: /api/v3/knowledge/yaml + methods: + POST: [create] + - path: /api/v3/knowledge/:name + methods: + GET: [get] + PATCH: [patch] + DELETE: [delete] + resourceNameParam: name + - path: /api/v3/knowledge/yaml/:name + methods: + PUT: [update] + resourceNameParam: name + - path: /api/v3/knowledge/:name/link + methods: + GET: [get] + POST: [patch] + DELETE: [patch] + resourceNameParam: name + + # Runtime classes + runtimeClasses: + basePath: /api/v3/runtimeClasses + routes: + - path: /api/v3/runtimeClasses + methods: + GET: [list] + POST: [create] + - path: /api/v3/runtimeClasses/yaml + methods: + POST: [create] + - path: /api/v3/runtimeClasses/:name + methods: + GET: [get] + PATCH: [patch] + DELETE: [delete] + resourceNameParam: name + - path: /api/v3/runtimeClasses/yaml/:name + methods: + PUT: [update] + resourceNameParam: name + - path: /api/v3/runtimeClasses/:name/link + methods: + GET: [get] + POST: [patch] + DELETE: [patch] + resourceNameParam: name + + # Microservice templates + microserviceTemplates: + basePath: /api/v3/microserviceTemplates + routes: + - path: /api/v3/microserviceTemplates + methods: + GET: [list] + POST: [create] + - path: /api/v3/microserviceTemplates/yaml + methods: + POST: [create] + - path: /api/v3/microserviceTemplates/:name + methods: + GET: [get] + PATCH: [patch] + DELETE: [delete] + resourceNameParam: name + - path: /api/v3/microserviceTemplates/yaml/:name + methods: + PUT: [update] + resourceNameParam: name + # Tunnels tunnels: basePath: /api/v3/iofog @@ -644,6 +750,15 @@ resources: - path: /api/v3/agent/volumeMounts methods: GET: [list] + - path: /api/v3/agent/models + methods: + GET: [list] + - path: /api/v3/agent/knowledge + methods: + GET: [list] + - path: /api/v3/agent/runtimeClasses + methods: + GET: [list] - path: /api/v3/agent/microservices methods: GET: [list] @@ -660,12 +775,6 @@ resources: - path: /api/v3/agent/version methods: GET: [get] - - path: /api/v3/agent/hal/hw - methods: - PUT: [update] - - path: /api/v3/agent/hal/usb - methods: - PUT: [update] - path: /api/v3/agent/delete-node methods: DELETE: [delete] diff --git a/src/config/rbac-system-roles.js b/src/config/rbac-system-roles.js index 3dbe4ee1b..58e457c25 100644 --- a/src/config/rbac-system-roles.js +++ b/src/config/rbac-system-roles.js @@ -40,7 +40,7 @@ module.exports = { rules: [ { apiGroups: [''], - resources: ['microservices', 'systemMicroservices', 'fogs', 'applications', 'systemApplications', 'applicationTemplates', 'services', 'router', 'networkTopology', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'tunnels', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'events', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'execSessions', 'systemExecSessions', 'logs', 'systemLogs'], + resources: ['microservices', 'systemMicroservices', 'fogs', 'applications', 'systemApplications', 'applicationTemplates', 'microserviceTemplates', 'services', 'router', 'networkTopology', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'knowledge', 'runtimeClasses', 'tunnels', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'events', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'execSessions', 'systemExecSessions', 'logs', 'systemLogs'], verbs: ['*'] }, { @@ -62,7 +62,7 @@ module.exports = { rules: [ { apiGroups: [''], - resources: ['microservices', 'applications', 'applicationTemplates', 'services', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'certificates', 'capabilities', 'serviceAccounts', 'controller', 'execSessions', 'logs'], + resources: ['microservices', 'applications', 'applicationTemplates', 'microserviceTemplates', 'services', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'knowledge', 'runtimeClasses', 'certificates', 'capabilities', 'serviceAccounts', 'controller', 'execSessions', 'logs'], verbs: ['get', 'list', 'create', 'update', 'patch', 'delete'] }, { @@ -84,7 +84,7 @@ module.exports = { rules: [ { apiGroups: [''], - resources: ['microservices', 'fogs', 'applications', 'systemMicroservices', 'systemApplications', 'applicationTemplates', 'services', 'router', 'networkTopology', 'natsOperator', 'natsBootstrap', 'natsHub', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'roles', 'roleBindings'], + resources: ['microservices', 'fogs', 'applications', 'systemMicroservices', 'systemApplications', 'applicationTemplates', 'microserviceTemplates', 'services', 'router', 'networkTopology', 'natsOperator', 'natsBootstrap', 'natsHub', 'natsAccounts', 'natsUsers', 'natsAccountRules', 'natsUserRules', 'catalog', 'registries', 'secrets', 'configMaps', 'volumeMounts', 'models', 'knowledge', 'runtimeClasses', 'certificates', 'capabilities', 'cluster', 'serviceAccounts', 'users', 'authUsers', 'authGroups', 'config', 'controller', 'roles', 'roleBindings'], verbs: ['get', 'list'] } ] diff --git a/src/controllers/agent-controller.js b/src/controllers/agent-controller.js index 1c6419f99..f798cd18e 100644 --- a/src/controllers/agent-controller.js +++ b/src/controllers/agent-controller.js @@ -52,6 +52,18 @@ const getAgentLinkedVolumeMountsEndpoint = async function (req, fog) { return { volumeMounts: await AgentService.getAgentLinkedVolumeMounts(fog) } } +const getAgentLinkedModelsEndpoint = async function (req, fog) { + return { models: await AgentService.getAgentLinkedModels(fog) } +} + +const getAgentLinkedKnowledgeEndpoint = async function (req, fog) { + return { knowledge: await AgentService.getAgentLinkedKnowledge(fog) } +} + +const getAgentLinkedRuntimeClassesEndpoint = async function (req, fog) { + return { runtimeClasses: await AgentService.getAgentLinkedRuntimeClasses(fog) } +} + const getAgentLogSessionsEndPoint = async function (req, fog) { return AgentService.getAgentLogSessions(fog) } @@ -78,18 +90,6 @@ const getAgentChangeVersionCommandEndPoint = async function (req, fog) { return AgentService.getAgentChangeVersionCommand(fog) } -const updateHalHardwareInfoEndPoint = async function (req, fog) { - const hardwareData = req.body - - return AgentService.updateHalHardwareInfo(hardwareData, fog) -} - -const updateHalUsbInfoEndPoint = async function (req, fog) { - const usbData = req.body - - return AgentService.updateHalUsbInfo(usbData, fog) -} - const deleteNodeEndPoint = async function (req, fog) { return AgentService.deleteNode(fog) } @@ -115,11 +115,12 @@ module.exports = { getAgentRegistriesEndPoint: AuthDecorator.checkFogToken(getAgentRegistriesEndPoint), getAgentTunnelEndPoint: AuthDecorator.checkFogToken(getAgentTunnelEndPoint), getAgentChangeVersionCommandEndPoint: AuthDecorator.checkFogToken(getAgentChangeVersionCommandEndPoint), - updateHalHardwareInfoEndPoint: AuthDecorator.checkFogToken(updateHalHardwareInfoEndPoint), - updateHalUsbInfoEndPoint: AuthDecorator.checkFogToken(updateHalUsbInfoEndPoint), deleteNodeEndPoint: AuthDecorator.checkFogToken(deleteNodeEndPoint), resetAgentConfigChangesEndPoint: AuthDecorator.checkFogToken(resetAgentConfigChangesEndPoint), getAgentLinkedVolumeMountsEndpoint: AuthDecorator.checkFogToken(getAgentLinkedVolumeMountsEndpoint), + getAgentLinkedModelsEndpoint: AuthDecorator.checkFogToken(getAgentLinkedModelsEndpoint), + getAgentLinkedKnowledgeEndpoint: AuthDecorator.checkFogToken(getAgentLinkedKnowledgeEndpoint), + getAgentLinkedRuntimeClassesEndpoint: AuthDecorator.checkFogToken(getAgentLinkedRuntimeClassesEndpoint), getControllerCAEndPoint: AuthDecorator.checkFogToken(getControllerCAEndPoint), getAgentLogSessionsEndPoint: AuthDecorator.checkFogToken(getAgentLogSessionsEndPoint), getAgentExecSessionsEndPoint: AuthDecorator.checkFogToken(getAgentExecSessionsEndPoint), diff --git a/src/controllers/iofog-controller.js b/src/controllers/iofog-controller.js index 61d72345f..a49ea8be1 100644 --- a/src/controllers/iofog-controller.js +++ b/src/controllers/iofog-controller.js @@ -67,20 +67,6 @@ async function setFogRebootCommandEndPoint (req) { return FogService.setFogRebootCommandEndPoint(fog, false) } -async function getHalHardwareInfoEndPoint (req) { - const uuidObj = { - uuid: req.params.uuid - } - return FogService.getHalHardwareInfoEndPoint(uuidObj, false) -} - -async function getHalUsbInfoEndPoint (req) { - const uuidObj = { - uuid: req.params.uuid - } - return FogService.getHalUsbInfoEndPoint(uuidObj, false) -} - async function setFogPruneCommandEndPoint (req) { const fog = { uuid: req.params.uuid @@ -116,8 +102,6 @@ module.exports = { generateProvisioningKeyEndPoint: (generateProvisionKeyEndPoint), setFogVersionCommandEndPoint, setFogRebootCommandEndPoint, - getHalHardwareInfoEndPoint, - getHalUsbInfoEndPoint, setFogPruneCommandEndPoint, enableNodeExecEndPoint, disableNodeExecEndPoint diff --git a/src/controllers/knowledge-controller.js b/src/controllers/knowledge-controller.js new file mode 100644 index 000000000..a6743c61c --- /dev/null +++ b/src/controllers/knowledge-controller.js @@ -0,0 +1,63 @@ +const KnowledgeService = require('../services/knowledge-service') +const YAMLParserService = require('../services/yaml-parser-service') + +const listKnowledgeEndpoint = async (req) => { + return KnowledgeService.listKnowledgeEndpoint() +} + +const getKnowledgeEndpoint = async (req) => { + return KnowledgeService.getKnowledgeEndpoint(req.params.name) +} + +const createKnowledgeEndpoint = async (req) => { + return KnowledgeService.createKnowledgeEndpoint(req.body) +} + +const updateKnowledgeEndpoint = async (req) => { + return KnowledgeService.updateKnowledgeEndpoint(req.params.name, req.body) +} + +const deleteKnowledgeEndpoint = async (req) => { + return KnowledgeService.deleteKnowledgeEndpoint(req.params.name) +} + +const createKnowledgeYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const knowledgeData = await YAMLParserService.parseKnowledgeFile(fileContent) + return KnowledgeService.createKnowledgeEndpoint(knowledgeData) +} + +const upsertKnowledgeYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const name = req.params.name + const knowledgeData = await YAMLParserService.parseKnowledgeFile(fileContent, { + isUpdate: true, + knowledgeName: name + }) + return KnowledgeService.upsertKnowledgeEndpoint(name, knowledgeData) +} + +const getKnowledgeLinkEndpoint = async (req) => { + return KnowledgeService.getKnowledgeLinkEndpoint(req.params.name) +} + +const linkKnowledgeEndpoint = async (req) => { + return KnowledgeService.linkKnowledgeEndpoint(req.params.name, req.body.fogUuids) +} + +const unlinkKnowledgeEndpoint = async (req) => { + return KnowledgeService.unlinkKnowledgeEndpoint(req.params.name, req.body.fogUuids) +} + +module.exports = { + listKnowledgeEndpoint, + getKnowledgeEndpoint, + createKnowledgeEndpoint, + updateKnowledgeEndpoint, + deleteKnowledgeEndpoint, + createKnowledgeYamlEndpoint, + upsertKnowledgeYamlEndpoint, + getKnowledgeLinkEndpoint, + linkKnowledgeEndpoint, + unlinkKnowledgeEndpoint +} diff --git a/src/controllers/microservice-template-controller.js b/src/controllers/microservice-template-controller.js new file mode 100644 index 000000000..6ad39621e --- /dev/null +++ b/src/controllers/microservice-template-controller.js @@ -0,0 +1,67 @@ +const MicroserviceTemplateService = require('../services/microservice-template-service') +const YAMLParserService = require('../services/yaml-parser-service') +const errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const { rvaluesVarSubstition } = require('../helpers/template-helper') + +async function _applyVariableSelfSubstitution (templateData) { + if (!templateData.variables) { + return templateData + } + await rvaluesVarSubstition(templateData.variables, { self: templateData.variables }) + return templateData +} + +const listMicroserviceTemplatesEndpoint = async (req) => { + return MicroserviceTemplateService.listMicroserviceTemplatesEndpoint() +} + +const getMicroserviceTemplateEndpoint = async (req) => { + return MicroserviceTemplateService.getMicroserviceTemplateEndpoint(req.params.name) +} + +const createMicroserviceTemplateEndpoint = async (req) => { + return MicroserviceTemplateService.createMicroserviceTemplateEndpoint(req.body) +} + +const updateMicroserviceTemplateEndpoint = async (req) => { + return MicroserviceTemplateService.updateMicroserviceTemplateEndpoint(req.params.name, req.body) +} + +const deleteMicroserviceTemplateEndpoint = async (req) => { + return MicroserviceTemplateService.deleteMicroserviceTemplateEndpoint(req.params.name) +} + +const createMicroserviceTemplateYamlEndpoint = async (req) => { + if (!req.file) { + throw new errors.ValidationError(ErrorMessages.MICROSERVICE_TEMPLATE_FILE_NOT_FOUND) + } + const fileContent = req.file.buffer.toString() + const templateData = await YAMLParserService.parseMicroserviceTemplateFile(fileContent) + await _applyVariableSelfSubstitution(templateData) + return MicroserviceTemplateService.createMicroserviceTemplateEndpoint(templateData) +} + +const upsertMicroserviceTemplateYamlEndpoint = async (req) => { + if (!req.file) { + throw new errors.ValidationError(ErrorMessages.MICROSERVICE_TEMPLATE_FILE_NOT_FOUND) + } + const name = req.params.name + const fileContent = req.file.buffer.toString() + const templateData = await YAMLParserService.parseMicroserviceTemplateFile(fileContent, { + isUpdate: true, + templateName: name + }) + await _applyVariableSelfSubstitution(templateData) + return MicroserviceTemplateService.upsertMicroserviceTemplateEndpoint(name, templateData) +} + +module.exports = { + listMicroserviceTemplatesEndpoint, + getMicroserviceTemplateEndpoint, + createMicroserviceTemplateEndpoint, + updateMicroserviceTemplateEndpoint, + deleteMicroserviceTemplateEndpoint, + createMicroserviceTemplateYamlEndpoint, + upsertMicroserviceTemplateYamlEndpoint +} diff --git a/src/controllers/microservices-controller.js b/src/controllers/microservices-controller.js index ab737c291..efa5bcc65 100644 --- a/src/controllers/microservices-controller.js +++ b/src/controllers/microservices-controller.js @@ -62,6 +62,18 @@ const updateSystemMicroserviceYAMLEndPoint = async function (req) { return MicroservicesService.updateSystemMicroserviceEndPoint(microserviceUuid, microservice, false) } +const updateMicroserviceCatalogEndPoint = async function (req) { + const microserviceUuid = req.params.uuid + const catalog = req.body + return MicroservicesService.updateMicroserviceCatalogEndPoint(microserviceUuid, catalog, false) +} + +const updateMicroserviceKnowledgeEndPoint = async function (req) { + const microserviceUuid = req.params.uuid + const catalog = req.body + return MicroservicesService.updateMicroserviceKnowledgeEndPoint(microserviceUuid, catalog, false) +} + const updateMicroserviceConfigEndPoint = async function (req) { const microserviceUuid = req.params.uuid const config = req.body @@ -214,6 +226,8 @@ module.exports = { createMicroserviceYAMLEndPoint, updateMicroserviceYAMLEndPoint, updateSystemMicroserviceYAMLEndPoint, + updateMicroserviceCatalogEndPoint, + updateMicroserviceKnowledgeEndPoint, updateMicroserviceConfigEndPoint, getMicroserviceConfigEndPoint, updateSystemMicroserviceConfigEndPoint, diff --git a/src/controllers/model-controller.js b/src/controllers/model-controller.js new file mode 100644 index 000000000..c92061504 --- /dev/null +++ b/src/controllers/model-controller.js @@ -0,0 +1,63 @@ +const ModelService = require('../services/model-service') +const YAMLParserService = require('../services/yaml-parser-service') + +const listModelsEndpoint = async (req) => { + return ModelService.listModelsEndpoint() +} + +const getModelEndpoint = async (req) => { + return ModelService.getModelEndpoint(req.params.name) +} + +const createModelEndpoint = async (req) => { + return ModelService.createModelEndpoint(req.body) +} + +const updateModelEndpoint = async (req) => { + return ModelService.updateModelEndpoint(req.params.name, req.body) +} + +const deleteModelEndpoint = async (req) => { + return ModelService.deleteModelEndpoint(req.params.name) +} + +const createModelYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const modelData = await YAMLParserService.parseModelFile(fileContent) + return ModelService.createModelEndpoint(modelData) +} + +const upsertModelYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const name = req.params.name + const modelData = await YAMLParserService.parseModelFile(fileContent, { + isUpdate: true, + modelName: name + }) + return ModelService.upsertModelEndpoint(name, modelData) +} + +const getModelLinkEndpoint = async (req) => { + return ModelService.getModelLinkEndpoint(req.params.name) +} + +const linkModelEndpoint = async (req) => { + return ModelService.linkModelEndpoint(req.params.name, req.body.fogUuids) +} + +const unlinkModelEndpoint = async (req) => { + return ModelService.unlinkModelEndpoint(req.params.name, req.body.fogUuids) +} + +module.exports = { + listModelsEndpoint, + getModelEndpoint, + createModelEndpoint, + updateModelEndpoint, + deleteModelEndpoint, + createModelYamlEndpoint, + upsertModelYamlEndpoint, + getModelLinkEndpoint, + linkModelEndpoint, + unlinkModelEndpoint +} diff --git a/src/controllers/runtime-class-controller.js b/src/controllers/runtime-class-controller.js new file mode 100644 index 000000000..6ae145870 --- /dev/null +++ b/src/controllers/runtime-class-controller.js @@ -0,0 +1,63 @@ +const RuntimeClassService = require('../services/runtime-class-service') +const YAMLParserService = require('../services/yaml-parser-service') + +const listRuntimeClassesEndpoint = async (req) => { + return RuntimeClassService.listRuntimeClassesEndpoint() +} + +const getRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.getRuntimeClassEndpoint(req.params.name) +} + +const createRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.createRuntimeClassEndpoint(req.body) +} + +const updateRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.updateRuntimeClassEndpoint(req.params.name, req.body) +} + +const deleteRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.deleteRuntimeClassEndpoint(req.params.name) +} + +const createRuntimeClassYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const runtimeClassData = await YAMLParserService.parseRuntimeClassFile(fileContent) + return RuntimeClassService.createRuntimeClassEndpoint(runtimeClassData) +} + +const upsertRuntimeClassYamlEndpoint = async (req) => { + const fileContent = req.file.buffer.toString() + const name = req.params.name + const runtimeClassData = await YAMLParserService.parseRuntimeClassFile(fileContent, { + isUpdate: true, + runtimeClassName: name + }) + return RuntimeClassService.upsertRuntimeClassEndpoint(name, runtimeClassData) +} + +const getRuntimeClassLinkEndpoint = async (req) => { + return RuntimeClassService.getRuntimeClassLinkEndpoint(req.params.name) +} + +const linkRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.linkRuntimeClassEndpoint(req.params.name, req.body.fogUuids) +} + +const unlinkRuntimeClassEndpoint = async (req) => { + return RuntimeClassService.unlinkRuntimeClassEndpoint(req.params.name, req.body.fogUuids) +} + +module.exports = { + listRuntimeClassesEndpoint, + getRuntimeClassEndpoint, + createRuntimeClassEndpoint, + updateRuntimeClassEndpoint, + deleteRuntimeClassEndpoint, + createRuntimeClassYamlEndpoint, + upsertRuntimeClassYamlEndpoint, + getRuntimeClassLinkEndpoint, + linkRuntimeClassEndpoint, + unlinkRuntimeClassEndpoint +} diff --git a/src/data/managers/application-manager.js b/src/data/managers/application-manager.js index 38c9d7cc5..9720e5f09 100644 --- a/src/data/managers/application-manager.js +++ b/src/data/managers/application-manager.js @@ -2,6 +2,20 @@ const BaseManager = require('./base-manager') const models = require('../models') const Application = models.Application const Microservice = models.Microservice +const RbacServiceAccount = models.RbacServiceAccount + +const microserviceInclude = { + model: Microservice, + as: 'microservices', + required: false, + include: [ + { + model: RbacServiceAccount, + as: 'serviceAccount', + required: false + } + ] +} class ApplicationManager extends BaseManager { getEntity () { @@ -44,13 +58,7 @@ class ApplicationManager extends BaseManager { async findOnePopulated (where, attributes, transaction) { const application = await Application.findOne({ - include: [ - { - model: Microservice, - as: 'microservices', - required: false - } - ], + include: [microserviceInclude], where, attributes }, { transaction }) @@ -66,13 +74,7 @@ class ApplicationManager extends BaseManager { async findAllPopulated (where, attributes, transaction) { const applications = await Application.findAll({ - include: [ - { - model: Microservice, - as: 'microservices', - required: false - } - ], + include: [microserviceInclude], where, attributes }, { transaction }) diff --git a/src/data/managers/fleet-knowledge-manager.js b/src/data/managers/fleet-knowledge-manager.js new file mode 100644 index 000000000..ae6812e4d --- /dev/null +++ b/src/data/managers/fleet-knowledge-manager.js @@ -0,0 +1,33 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const FleetKnowledge = models.FleetKnowledge + +const knowledgeExcludedFields = [ + 'created_at', + 'updated_at' +] + +class FleetKnowledgeManager extends BaseManager { + getEntity () { + return FleetKnowledge + } + + findOne (where, transaction) { + return FleetKnowledge.findOne({ + where, + attributes: { exclude: knowledgeExcludedFields }, + transaction + }) + } + + findAll (where, transaction) { + return FleetKnowledge.findAll({ + where, + attributes: { exclude: knowledgeExcludedFields }, + transaction + }) + } +} + +const instance = new FleetKnowledgeManager() +module.exports = instance diff --git a/src/data/managers/fleet-model-manager.js b/src/data/managers/fleet-model-manager.js new file mode 100644 index 000000000..921c707f8 --- /dev/null +++ b/src/data/managers/fleet-model-manager.js @@ -0,0 +1,33 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const FleetModel = models.FleetModel + +const modelExcludedFields = [ + 'created_at', + 'updated_at' +] + +class FleetModelManager extends BaseManager { + getEntity () { + return FleetModel + } + + findOne (where, transaction) { + return FleetModel.findOne({ + where, + attributes: { exclude: modelExcludedFields }, + transaction + }) + } + + findAll (where, transaction) { + return FleetModel.findAll({ + where, + attributes: { exclude: modelExcludedFields }, + transaction + }) + } +} + +const instance = new FleetModelManager() +module.exports = instance diff --git a/src/data/managers/fog-knowledge-manager.js b/src/data/managers/fog-knowledge-manager.js new file mode 100644 index 000000000..fbe57ecbb --- /dev/null +++ b/src/data/managers/fog-knowledge-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const FogKnowledge = models.FogKnowledge + +class FogKnowledgeManager extends BaseManager { + getEntity () { + return FogKnowledge + } +} + +const instance = new FogKnowledgeManager() +module.exports = instance diff --git a/src/data/managers/fog-platform-spec-manager.js b/src/data/managers/fog-platform-spec-manager.js index 492fe1c40..28d231b6a 100644 --- a/src/data/managers/fog-platform-spec-manager.js +++ b/src/data/managers/fog-platform-spec-manager.js @@ -3,7 +3,8 @@ const models = require('../models') const { validateFogPlatformSpec, parseSpecJson, - serializeSpecJson + serializeSpecJson, + pickPlatformSpecFields } = require('../../schemas/fog-platform-spec') class FogPlatformSpecManager extends BaseManager { @@ -24,8 +25,9 @@ class FogPlatformSpecManager extends BaseManager { } async upsertSpec (fogUuid, specObject, transaction) { - await validateFogPlatformSpec(specObject) - const specJson = serializeSpecJson(specObject) + const sanitized = pickPlatformSpecFields(specObject) + await validateFogPlatformSpec(sanitized) + const specJson = serializeSpecJson(sanitized) const existing = await this.findOne({ fogUuid }, transaction) if (existing) { diff --git a/src/data/managers/hw-info-manager.js b/src/data/managers/hw-info-manager.js deleted file mode 100644 index c65e89a37..000000000 --- a/src/data/managers/hw-info-manager.js +++ /dev/null @@ -1,12 +0,0 @@ -const BaseManager = require('./base-manager') -const models = require('../models') -const HWInfo = models.HWInfo - -class HWInfoManager extends BaseManager { - getEntity () { - return HWInfo - } -} - -const instance = new HWInfoManager() -module.exports = instance diff --git a/src/data/managers/iofog-manager.js b/src/data/managers/iofog-manager.js index db5b7ed03..bb7a45f78 100644 --- a/src/data/managers/iofog-manager.js +++ b/src/data/managers/iofog-manager.js @@ -76,6 +76,25 @@ class FogManager extends BaseManager { return rows.map((row) => row.uuid) } + async findStatusLivenessCandidates ({ afterUuid, limit, heardBefore, daemonStatuses }, transaction) { + const where = { + daemonStatus: { [Op.in]: daemonStatuses } + } + if (heardBefore != null) { + where.lastStatusTime = { [Op.lt]: heardBefore } + } + if (afterUuid) { + where.uuid = { [Op.gt]: afterUuid } + } + return Fog.findAll({ + attributes: ['uuid', 'statusFrequency', 'lastStatusTime', 'daemonStatus'], + where, + order: [['uuid', 'ASC']], + limit, + transaction + }) + } + updateLastActive (uuid, timestamp, transaction) { return Fog.update({ lastActive: timestamp diff --git a/src/data/managers/microservice-device-manager.js b/src/data/managers/microservice-device-manager.js new file mode 100644 index 000000000..bd0c77051 --- /dev/null +++ b/src/data/managers/microservice-device-manager.js @@ -0,0 +1,22 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceDevice = models.MicroserviceDevice + +const excludedFields = [ + 'id', + 'microservice_uuid', + 'microserviceUuid' +] + +class MicroserviceDeviceManager extends BaseManager { + getEntity () { + return MicroserviceDevice + } + + findAllExcludeFields (where, transaction) { + return this.findAllWithAttributes(where, { exclude: excludedFields }, transaction) + } +} + +const instance = new MicroserviceDeviceManager() +module.exports = instance diff --git a/src/data/managers/microservice-entrypoint-manager.js b/src/data/managers/microservice-entrypoint-manager.js new file mode 100644 index 000000000..90f3fca3c --- /dev/null +++ b/src/data/managers/microservice-entrypoint-manager.js @@ -0,0 +1,22 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceEntrypoint = models.MicroserviceEntrypoint + +const excludedFields = [ + 'id', + 'microservice_uuid', + 'microserviceUuid' +] + +class MicroserviceEntrypointManager extends BaseManager { + getEntity () { + return MicroserviceEntrypoint + } + + findAllExcludeFields (where, transaction) { + return this.findAllWithAttributes(where, { exclude: excludedFields }, transaction) + } +} + +const instance = new MicroserviceEntrypointManager() +module.exports = instance diff --git a/src/data/managers/microservice-knowledge-item-manager.js b/src/data/managers/microservice-knowledge-item-manager.js new file mode 100644 index 000000000..7830672b5 --- /dev/null +++ b/src/data/managers/microservice-knowledge-item-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceKnowledgeItem = models.MicroserviceKnowledgeItem + +class MicroserviceKnowledgeItemManager extends BaseManager { + getEntity () { + return MicroserviceKnowledgeItem + } +} + +const instance = new MicroserviceKnowledgeItemManager() +module.exports = instance diff --git a/src/data/managers/microservice-knowledge-manager.js b/src/data/managers/microservice-knowledge-manager.js new file mode 100644 index 000000000..505c53482 --- /dev/null +++ b/src/data/managers/microservice-knowledge-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceKnowledge = models.MicroserviceKnowledge + +class MicroserviceKnowledgeManager extends BaseManager { + getEntity () { + return MicroserviceKnowledge + } +} + +const instance = new MicroserviceKnowledgeManager() +module.exports = instance diff --git a/src/data/managers/microservice-manager.js b/src/data/managers/microservice-manager.js index ea2f96180..f918c3db0 100644 --- a/src/data/managers/microservice-manager.js +++ b/src/data/managers/microservice-manager.js @@ -9,6 +9,14 @@ const MicroserviceCdiDev = models.MicroserviceCdiDev const MicroserviceCapAdd = models.MicroserviceCapAdd const MicroserviceCapDrop = models.MicroserviceCapDrop const VolumeMapping = models.VolumeMapping +const MicroserviceEntrypoint = models.MicroserviceEntrypoint +const MicroserviceDevice = models.MicroserviceDevice +const MicroserviceTmpfs = models.MicroserviceTmpfs +const MicroserviceUlimit = models.MicroserviceUlimit +const MicroserviceModel = models.MicroserviceModel +const MicroserviceModelItem = models.MicroserviceModelItem +const MicroserviceKnowledge = models.MicroserviceKnowledge +const MicroserviceKnowledgeItem = models.MicroserviceKnowledgeItem const CatalogItem = models.CatalogItem const CatalogItemImage = models.CatalogItemImage const Fog = models.Fog @@ -31,6 +39,59 @@ const microserviceExcludedFields = [ 'iofog_uuid' ] +function containerChildIncludes () { + return [ + { + model: MicroserviceEntrypoint, + as: 'entrypoint', + required: false, + attributes: ['id', 'entrypoint'] + }, + { + model: MicroserviceDevice, + as: 'devices', + required: false, + attributes: ['id', 'hostPath', 'containerPath', 'permissions'] + }, + { + model: MicroserviceTmpfs, + as: 'tmpfs', + required: false, + attributes: ['id', 'containerPath', 'size', 'mode'] + }, + { + model: MicroserviceUlimit, + as: 'ulimits', + required: false, + attributes: ['id', 'name', 'soft', 'hard'] + }, + { + model: MicroserviceModel, + as: 'microserviceModel', + required: false, + attributes: ['bindPath', 'permissions'] + }, + { + model: MicroserviceModelItem, + as: 'modelItems', + required: false, + attributes: ['id', 'name'] + }, + { + model: MicroserviceKnowledge, + as: 'microserviceKnowledge', + required: false, + attributes: ['bindPath', 'permissions'] + }, + { + model: MicroserviceKnowledgeItem, + as: 'knowledgeItems', + required: false, + attributes: ['id', 'name'] + } + ] +} + class MicroserviceManager extends BaseManager { getEntity () { return Microservice @@ -56,6 +117,7 @@ class MicroserviceManager extends BaseManager { required: false, attributes: ['cmd'] }, + ...containerChildIncludes(), { model: MicroserviceCdiDev, as: 'cdiDevices', @@ -84,7 +146,7 @@ class MicroserviceManager extends BaseManager { model: VolumeMapping, as: 'volumeMappings', required: false, - attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type'] + attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type', 'scope'] }, { model: CatalogItemImage, @@ -152,6 +214,7 @@ class MicroserviceManager extends BaseManager { required: false, attributes: ['cmd', 'id'] }, + ...containerChildIncludes(), { model: MicroserviceCdiDev, as: 'cdiDevices', @@ -180,7 +243,7 @@ class MicroserviceManager extends BaseManager { model: VolumeMapping, as: 'volumeMappings', required: false, - attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type'] + attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type', 'scope'] }, { model: CatalogItemImage, @@ -272,6 +335,7 @@ class MicroserviceManager extends BaseManager { required: false, attributes: ['cmd'] }, + ...containerChildIncludes(), { model: MicroserviceCdiDev, as: 'cdiDevices', @@ -300,7 +364,7 @@ class MicroserviceManager extends BaseManager { model: VolumeMapping, as: 'volumeMappings', required: false, - attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type'] + attributes: ['hostDestination', 'containerDestination', 'accessMode', 'type', 'scope'] }, { model: CatalogItemImage, diff --git a/src/data/managers/microservice-model-item-manager.js b/src/data/managers/microservice-model-item-manager.js new file mode 100644 index 000000000..9d05f67ec --- /dev/null +++ b/src/data/managers/microservice-model-item-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceModelItem = models.MicroserviceModelItem + +class MicroserviceModelItemManager extends BaseManager { + getEntity () { + return MicroserviceModelItem + } +} + +const instance = new MicroserviceModelItemManager() +module.exports = instance diff --git a/src/data/managers/microservice-model-manager.js b/src/data/managers/microservice-model-manager.js new file mode 100644 index 000000000..3c0f3c9af --- /dev/null +++ b/src/data/managers/microservice-model-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceModel = models.MicroserviceModel + +class MicroserviceModelManager extends BaseManager { + getEntity () { + return MicroserviceModel + } +} + +const instance = new MicroserviceModelManager() +module.exports = instance diff --git a/src/data/managers/microservice-template-manager.js b/src/data/managers/microservice-template-manager.js new file mode 100644 index 000000000..340442122 --- /dev/null +++ b/src/data/managers/microservice-template-manager.js @@ -0,0 +1,58 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceTemplate = models.MicroserviceTemplate + +const excludedFields = [ + 'created_at', + 'updated_at' +] + +const variablesInclude = { + model: models.MicroserviceTemplateVariable, + as: 'variables', + required: false, + attributes: { exclude: excludedFields } +} + +class MicroserviceTemplateManager extends BaseManager { + getEntity () { + return MicroserviceTemplate + } + + findOne (where, transaction) { + return MicroserviceTemplate.findOne({ + where, + attributes: { exclude: excludedFields }, + transaction + }) + } + + findAll (where, transaction) { + return MicroserviceTemplate.findAll({ + where, + attributes: { exclude: excludedFields }, + transaction + }) + } + + findOnePopulated (where, transaction) { + return MicroserviceTemplate.findOne({ + include: [variablesInclude], + where, + attributes: { exclude: excludedFields }, + transaction + }) + } + + findAllPopulated (where, transaction) { + return MicroserviceTemplate.findAll({ + include: [variablesInclude], + where, + attributes: { exclude: excludedFields }, + transaction + }) + } +} + +const instance = new MicroserviceTemplateManager() +module.exports = instance diff --git a/src/data/managers/microservice-template-variable-manager.js b/src/data/managers/microservice-template-variable-manager.js new file mode 100644 index 000000000..d775f5324 --- /dev/null +++ b/src/data/managers/microservice-template-variable-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceTemplateVariable = models.MicroserviceTemplateVariable + +class MicroserviceTemplateVariableManager extends BaseManager { + getEntity () { + return MicroserviceTemplateVariable + } +} + +const instance = new MicroserviceTemplateVariableManager() +module.exports = instance diff --git a/src/data/managers/microservice-tmpfs-manager.js b/src/data/managers/microservice-tmpfs-manager.js new file mode 100644 index 000000000..207c5569f --- /dev/null +++ b/src/data/managers/microservice-tmpfs-manager.js @@ -0,0 +1,22 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceTmpfs = models.MicroserviceTmpfs + +const excludedFields = [ + 'id', + 'microservice_uuid', + 'microserviceUuid' +] + +class MicroserviceTmpfsManager extends BaseManager { + getEntity () { + return MicroserviceTmpfs + } + + findAllExcludeFields (where, transaction) { + return this.findAllWithAttributes(where, { exclude: excludedFields }, transaction) + } +} + +const instance = new MicroserviceTmpfsManager() +module.exports = instance diff --git a/src/data/managers/microservice-ulimit-manager.js b/src/data/managers/microservice-ulimit-manager.js new file mode 100644 index 000000000..395c8fb10 --- /dev/null +++ b/src/data/managers/microservice-ulimit-manager.js @@ -0,0 +1,22 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const MicroserviceUlimit = models.MicroserviceUlimit + +const excludedFields = [ + 'id', + 'microservice_uuid', + 'microserviceUuid' +] + +class MicroserviceUlimitManager extends BaseManager { + getEntity () { + return MicroserviceUlimit + } + + findAllExcludeFields (where, transaction) { + return this.findAllWithAttributes(where, { exclude: excludedFields }, transaction) + } +} + +const instance = new MicroserviceUlimitManager() +module.exports = instance diff --git a/src/data/managers/runtime-class-manager.js b/src/data/managers/runtime-class-manager.js new file mode 100644 index 000000000..f12e266fb --- /dev/null +++ b/src/data/managers/runtime-class-manager.js @@ -0,0 +1,12 @@ +const BaseManager = require('./base-manager') +const models = require('../models') +const RuntimeClass = models.RuntimeClass + +class RuntimeClassManager extends BaseManager { + getEntity () { + return RuntimeClass + } +} + +const instance = new RuntimeClassManager() +module.exports = instance diff --git a/src/data/managers/usb-info-manager.js b/src/data/managers/usb-info-manager.js deleted file mode 100644 index a3e818717..000000000 --- a/src/data/managers/usb-info-manager.js +++ /dev/null @@ -1,12 +0,0 @@ -const BaseManager = require('./base-manager') -const models = require('../models') -const USBInfo = models.USBInfo - -class USBInfoManager extends BaseManager { - getEntity () { - return USBInfo - } -} - -const instance = new USBInfoManager() -module.exports = instance diff --git a/src/data/managers/volume-mapping-manager.js b/src/data/managers/volume-mapping-manager.js index a4c40f277..88fd68e38 100644 --- a/src/data/managers/volume-mapping-manager.js +++ b/src/data/managers/volume-mapping-manager.js @@ -10,7 +10,7 @@ class VolumeMappingManager extends BaseManager { findAll (where, transaction) { return VolumeMapping.findAll({ where, - attributes: ['hostDestination', 'containerDestination', 'accessMode', 'id', 'type'], + attributes: ['hostDestination', 'containerDestination', 'accessMode', 'id', 'type', 'scope'], transaction }) } diff --git a/src/data/migrations/README.md b/src/data/migrations/README.md index 8347fb784..193bd2a0d 100644 --- a/src/data/migrations/README.md +++ b/src/data/migrations/README.md @@ -1,12 +1,32 @@ # Database migrations -## Fresh install (Controller v3.8+) +## Fresh install (empty database) -New installations use **`db_migration_*_v3.8.0.sql`** and **`db_seeder_*_v3.8.0.sql`** -(sqlite, mysql, postgres). The migration runner in `src/data/providers/database-provider.js` -records schema version **`3.8.0`** (includes embedded auth tables). +New installations apply **baseline 3.9.0** schema and seed files only. The runner in +`src/data/providers/database-provider.js` uses this path when `schema_version` is unset +and `untilVersion` is not provided. `schema_version` and `seeder_version` are then recorded +as `3.9.0`. Baseline files do not create hardware-inventory tables, architecture HAL/BLE +foreign keys, or HAL/RESTBlue catalog rows. -Greenfield only: wipe the data directory or database before upgrading from pre-3.8 builds. +| Version | Migration | Seeder | +|---------|-----------|--------| +| 3.9.0 (fresh) | `db_migration_*_baseline_v3.9.0.sql` | `db_seeder_*_baseline_v3.9.0.sql` | + +## Upgrade from 3.8.0 + +Existing 3.8.0 databases apply **incremental** 3.9.0 migrate-then-seed only. Do **not** +edit `db_*_v3.8.0.sql` files. The incremental 3.9 migration drops architecture HAL/BLE +foreign keys before deleting HAL/RESTBlue catalog rows. + +`untilVersion: '3.8.0'` still builds a 3.8 fixture via the incremental chain (used by tests). +Never apply baseline 3.9 and the 3.8 chain on the same database. + +| Version | Migration | Seeder | +|---------|-----------|--------| +| 3.8.0 | `db_migration_*_v3.8.0.sql` | `db_seeder_*_v3.8.0.sql` | +| 3.9.0 (upgrade) | `db_migration_*_v3.9.0.sql` | `db_seeder_*_v3.9.0.sql` | + +Greenfield only vs pre-3.8: wipe the data directory or database before upgrading from pre-3.8 builds. There is no v3.7→v3.8 incremental migrator. ## Pre-3.8 historical files diff --git a/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql new file mode 100644 index 000000000..72ed58db6 --- /dev/null +++ b/src/data/migrations/mysql/db_migration_mysql_baseline_v3.9.0.sql @@ -0,0 +1,1421 @@ +START TRANSACTION; + +CREATE TABLE IF NOT EXISTS Applications ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) UNIQUE, + description VARCHAR(255) DEFAULT '', + is_activated BOOLEAN DEFAULT false, + is_system BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + nats_access BOOLEAN DEFAULT false, + nats_rule_id INTEGER +); + +CREATE TABLE IF NOT EXISTS Registries ( + id INT AUTO_INCREMENT PRIMARY KEY, + url VARCHAR(255), + is_public BOOLEAN, + user_name TEXT, + password TEXT, + user_email TEXT, + type VARCHAR(16) NOT NULL DEFAULT 'oci', + ca TEXT, + insecure BOOLEAN NOT NULL DEFAULT false +); + +CREATE TABLE IF NOT EXISTS CatalogItems ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) UNIQUE, + description VARCHAR(255), + category TEXT, + config_example VARCHAR(255) DEFAULT '{}', + publisher TEXT, + disk_required BIGINT DEFAULT 0, + ram_required BIGINT DEFAULT 0, + picture VARCHAR(255) DEFAULT 'images/shared/default.png', + is_public BOOLEAN DEFAULT false, + registry_id INT, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_catalog_item_registry_id ON CatalogItems (registry_id); + +CREATE TABLE IF NOT EXISTS Architectures ( + id INT PRIMARY KEY, + name TEXT, + image TEXT, + description TEXT, + network_catalog_item_id INT, + FOREIGN KEY (network_catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_architecture_network_catalog_item_id ON Architectures (network_catalog_item_id); + +CREATE TABLE IF NOT EXISTS Fogs ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) DEFAULT 'Unnamed ioFog 1', + location TEXT, + gps_mode TEXT, + latitude FLOAT, + longitude FLOAT, + description TEXT, + last_active BIGINT, + daemon_status VARCHAR(36) DEFAULT 'NOT_PROVISIONED', + daemon_operating_duration BIGINT DEFAULT 0, + daemon_last_start BIGINT, + memory_usage FLOAT DEFAULT 0.000, + disk_usage FLOAT DEFAULT 0.000, + cpu_usage FLOAT DEFAULT 0.00, + memory_violation TEXT, + disk_violation TEXT, + cpu_violation TEXT, + system_available_disk BIGINT, + system_available_memory BIGINT, + system_total_cpu FLOAT, + system_cpus BIGINT, + system_total_memory BIGINT, + system_total_disk BIGINT, + system_os TEXT, + system_os_version TEXT, + system_kernel_version TEXT, + security_status VARCHAR(36) DEFAULT 'OK', + security_violation_info VARCHAR(36) DEFAULT 'No violation', + catalog_item_status TEXT, + repository_count BIGINT DEFAULT 0, + repository_status TEXT, + system_time BIGINT, + last_status_time BIGINT, + ip_address VARCHAR(36) DEFAULT '0.0.0.0', + ip_address_external VARCHAR(36) DEFAULT '0.0.0.0', + host VARCHAR(36), + catalog_item_message_counts TEXT, + last_command_time BIGINT, + network_interface VARCHAR(36) DEFAULT 'dynamic', + docker_url VARCHAR(255) DEFAULT 'unix:///run/edgelet/contaienrd.sock', + disk_limit FLOAT DEFAULT 50, + disk_directory VARCHAR(255) DEFAULT '/var/lib/iofog-agent/', + memory_limit FLOAT DEFAULT 4096, + cpu_limit FLOAT DEFAULT 80, + log_limit FLOAT DEFAULT 10, + log_directory VARCHAR(255) DEFAULT '/var/log/iofog/', + log_file_count BIGINT DEFAULT 10, + `version` TEXT, + is_ready_to_upgrade BOOLEAN DEFAULT TRUE, + is_ready_to_rollback BOOLEAN DEFAULT FALSE, + status_frequency INT DEFAULT 10, + change_frequency INT DEFAULT 20, + tunnel VARCHAR(255) DEFAULT '', + isolated_docker_container BOOLEAN DEFAULT FALSE, + docker_pruning_freq INT DEFAULT 0, + available_disk_threshold FLOAT DEFAULT 20, + log_level VARCHAR(10) DEFAULT 'INFO', + is_system BOOLEAN DEFAULT FALSE, + router_id INT DEFAULT 0, + time_zone VARCHAR(36) DEFAULT 'Etc/UTC', + created_at DATETIME, + updated_at DATETIME, + arch_id INT DEFAULT 0, + container_engine VARCHAR(36), + deployment_type VARCHAR(36), + active_volume_mounts BIGINT DEFAULT 0, + volume_mount_last_update BIGINT DEFAULT 0, + warning_message TEXT, + gps_device VARCHAR(36), + gps_scan_frequency INT DEFAULT 60, + edge_guard_frequency INT DEFAULT 0, + gps_status VARCHAR(32), + nats_id INT, + available_runtimes TEXT, + model_status TEXT, + runtime_classes TEXT, + available_cdi_devices TEXT, + active_models BIGINT DEFAULT 0, + model_last_update BIGINT DEFAULT 0, + knowledge_status TEXT, + active_knowledge BIGINT DEFAULT 0, + knowledge_last_update BIGINT DEFAULT 0, + FOREIGN KEY (arch_id) REFERENCES Architectures (id) +); + +CREATE INDEX idx_fog_arch_id ON Fogs (arch_id); + +CREATE TABLE IF NOT EXISTS ChangeTrackings ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_config BOOLEAN DEFAULT false, + reboot BOOLEAN DEFAULT false, + deletenode BOOLEAN DEFAULT false, + version BOOLEAN DEFAULT false, + microservice_list BOOLEAN DEFAULT false, + config BOOLEAN DEFAULT false, + registries BOOLEAN DEFAULT false, + tunnel BOOLEAN DEFAULT false, + router_changed BOOLEAN DEFAULT false, + prune BOOLEAN DEFAULT false, + last_updated VARCHAR(255) DEFAULT false, + iofog_uuid VARCHAR(36), + volume_mounts BOOLEAN DEFAULT false, + exec_sessions BOOLEAN DEFAULT false, + microservice_logs BOOLEAN DEFAULT false, + fog_logs BOOLEAN DEFAULT false, + models BOOLEAN DEFAULT false, + runtime_classes BOOLEAN DEFAULT false, + microservice_models BOOLEAN DEFAULT false, + knowledge BOOLEAN DEFAULT false, + microservice_knowledge BOOLEAN DEFAULT false, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_change_tracking_iofog_uuid ON ChangeTrackings (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogProvisionKeys ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + provisioning_string VARCHAR(100), + expiration_time BIGINT, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_provision_keys_iofogUuid ON FogProvisionKeys (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogVersionCommands ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + version_command VARCHAR(100), + semver VARCHAR(100), + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_version_commands_iofogUuid ON FogVersionCommands (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Tunnels ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + username TEXT, + password TEXT, + host TEXT, + remote_port INT, + local_port INT DEFAULT 22, + rsa_key TEXT, + closed BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_tunnels_iofogUuid ON Tunnels (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Microservices ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + config TEXT, + name VARCHAR(255) DEFAULT 'New Microservice', + config_last_updated BIGINT, + rebuild BOOLEAN DEFAULT false, + log_size BIGINT DEFAULT 0, + `delete` BOOLEAN DEFAULT false, + delete_with_cleanup BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + catalog_item_id INT, + registry_id INT DEFAULT 1, + iofog_uuid VARCHAR(36), + application_id INT, + run_as_user TEXT, + platform TEXT, + runtime TEXT, + annotations TEXT, + pid_mode VARCHAR(36), + ipc_mode VARCHAR(36), + schedule INT DEFAULT 50, + cpu_set_cpus TEXT, + memory_limit FLOAT, + is_activated BOOLEAN DEFAULT true, + host_network_mode BOOLEAN DEFAULT false, + is_privileged BOOLEAN DEFAULT false, + nats_rule_id INT, + nats_access BOOLEAN DEFAULT false, + nats_account_id INT, + nats_user_id INT, + nats_creds_secret_name TEXT, + is_controller BOOLEAN DEFAULT false, + run_as_group TEXT, + read_only_root_filesystem BOOLEAN DEFAULT false, + working_dir TEXT, + cpus FLOAT, + memory_reservation FLOAT, + memory_swap FLOAT, + shm_size FLOAT, + sysctls TEXT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES Applications (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservices_catalogItemId ON Microservices (catalog_item_id); +CREATE INDEX idx_microservices_registryId ON Microservices (registry_id); +CREATE INDEX idx_microservices_iofogUuid ON Microservices (iofog_uuid); +CREATE INDEX idx_microservices_applicationId ON Microservices (application_id); + +CREATE TABLE IF NOT EXISTS MicroserviceArgs ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + cmd TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_args_microserviceUuid ON MicroserviceArgs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceEnvs ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + `key` TEXT, + `value` TEXT, + microservice_uuid VARCHAR(36), + value_from_secret TEXT, + value_from_config_map TEXT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_envs_microserviceUuid ON MicroserviceEnvs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceExtraHost ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + template_type TEXT, + name TEXT, + template TEXT, + `value` TEXT, + microservice_uuid VARCHAR(36), + target_microservice_uuid VARCHAR(36), + target_fog_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_extra_host_microserviceUuid ON MicroserviceExtraHost (microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetMicroserviceUuid ON MicroserviceExtraHost (target_microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetFogUuid ON MicroserviceExtraHost (target_fog_uuid); + +CREATE TABLE IF NOT EXISTS MicroservicePorts ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + port_internal INT, + port_external INT, + is_udp BOOLEAN, + created_at DATETIME, + updated_at DATETIME, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_port_microserviceUuid ON MicroservicePorts (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceStatuses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + status VARCHAR(255) DEFAULT 'QUEUED', + operating_duration BIGINT DEFAULT 0, + start_time BIGINT DEFAULT 0, + cpu_usage FLOAT DEFAULT 0.000, + memory_usage BIGINT DEFAULT 0, + container_id VARCHAR(255) DEFAULT '', + percentage FLOAT DEFAULT 0.00, + error_message TEXT, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + ip_address TEXT, + exec_session_ids TEXT, + health_status TEXT, + pod_id TEXT, + last_error TEXT, + last_error_at BIGINT DEFAULT 0, + restart_count INT DEFAULT 0, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_status_microserviceUuid ON MicroserviceStatuses (microservice_uuid); + +CREATE TABLE IF NOT EXISTS VolumeMappings ( + uuid INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + host_destination TEXT, + container_destination TEXT, + access_mode TEXT, + type TEXT, + scope VARCHAR(16) NOT NULL DEFAULT 'private', + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mappings_microserviceUuid ON VolumeMappings (microservice_uuid); + +CREATE TABLE IF NOT EXISTS CatalogItemImages ( + id INT AUTO_INCREMENT PRIMARY KEY, + container_image TEXT, + catalog_item_id INT, + microservice_uuid VARCHAR(36), + arch_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (arch_id) REFERENCES Architectures (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_image_catalog_item_id ON CatalogItemImages (catalog_item_id); +CREATE INDEX idx_catalog_item_image_microservice_uuid ON CatalogItemImages (microservice_uuid); +CREATE INDEX idx_catalog_item_image_arch_id ON CatalogItemImages (arch_id); + +CREATE TABLE IF NOT EXISTS CatalogItemInputTypes ( + id INT AUTO_INCREMENT PRIMARY KEY, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_input_type_catalog_item_id ON CatalogItemInputTypes (catalog_item_id); + +CREATE TABLE IF NOT EXISTS CatalogItemOutputTypes ( + id INT AUTO_INCREMENT PRIMARY KEY, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_output_type_catalog_item_id ON CatalogItemOutputTypes (catalog_item_id); + +CREATE TABLE IF NOT EXISTS Routers ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + is_edge BOOLEAN DEFAULT true, + messaging_port INT DEFAULT 5671, + edge_router_port INT, + inter_router_port INT, + host TEXT, + is_default BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_router_iofogUuid ON Routers (iofog_uuid); + +CREATE TABLE RouterConnections ( + id INT AUTO_INCREMENT PRIMARY KEY, + source_router INT, + dest_router INT, + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL, + FOREIGN KEY (source_router) REFERENCES Routers(id) ON DELETE CASCADE, + FOREIGN KEY (dest_router) REFERENCES Routers(id) ON DELETE CASCADE +); + +CREATE INDEX idx_routerconnections_sourceRouter ON RouterConnections (source_router); +CREATE INDEX idx_routerconnections_destRouter ON RouterConnections (dest_router); + +CREATE TABLE IF NOT EXISTS Config ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + `key` VARCHAR(255) NOT NULL UNIQUE, + value VARCHAR(255) NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_config_key ON Config (`key`); + +CREATE TABLE IF NOT EXISTS Tags ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + value VARCHAR(255) UNIQUE NOT NULL +); + +CREATE TABLE IF NOT EXISTS IofogTags ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_iofogtags_fog_uuid ON IofogTags (fog_uuid); +CREATE INDEX idx_iofogtags_tag_id ON IofogTags (tag_id); + +CREATE TABLE IF NOT EXISTS ApplicationTemplates ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL DEFAULT 'new-application', + description VARCHAR(255) DEFAULT '', + schema_version VARCHAR(255) DEFAULT '', + application_json LONGTEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS ApplicationTemplateVariables ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + application_template_id INT NOT NULL, + `key` TEXT, + description VARCHAR(255) DEFAULT '', + default_value VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (application_template_id) REFERENCES ApplicationTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX idx_applicationtemplatevariables_application_template_id ON ApplicationTemplateVariables (application_template_id); + +CREATE TABLE IF NOT EXISTS MicroserviceCdiDevices ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + cdi_devices TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_cdiDevices_microserviceUuid ON MicroserviceCdiDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroservicePubTags ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceSubTags ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservicepubtags_microservice_uuid ON MicroservicePubTags (microservice_uuid); +CREATE INDEX idx_microservicesubtags_microservice_uuid ON MicroserviceSubTags (microservice_uuid); +CREATE INDEX idx_microservicepubtags_tag_id ON MicroservicePubTags (tag_id); +CREATE INDEX idx_microservicesubtags_tag_id ON MicroserviceSubTags (tag_id); + +CREATE TABLE IF NOT EXISTS MicroserviceCapAdd ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + cap_add TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capAdd_microserviceUuid ON MicroserviceCapAdd (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceCapDrop ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + cap_drop TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capDrop_microserviceUuid ON MicroserviceCapDrop (microservice_uuid); + +CREATE TABLE IF NOT EXISTS FogPublicKeys ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + public_key TEXT, + iofog_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_public_keys_iofogUuid ON FogPublicKeys (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogUsedTokens ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + jti VARCHAR(255) NOT NULL, + iofog_uuid VARCHAR(36), + expiry_time BIGINT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_used_tokens_iofogUuid ON FogUsedTokens (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Secrets ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + type VARCHAR(50) NOT NULL CHECK (type IN ('Opaque', 'tls')), + data TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_secrets_name ON Secrets (name); + +CREATE TABLE IF NOT EXISTS Certificates ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + subject TEXT NOT NULL, + is_ca BOOLEAN DEFAULT false, + signed_by_id INT, + hosts TEXT, + valid_from DATETIME NOT NULL, + valid_to DATETIME NOT NULL, + serial_number TEXT NOT NULL, + secret_id INT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (signed_by_id) REFERENCES Certificates (id) ON DELETE SET NULL, + FOREIGN KEY (secret_id) REFERENCES Secrets (id) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX idx_certificates_name_unique ON Certificates (name(255)); +CREATE INDEX idx_certificates_valid_to ON Certificates (valid_to); +CREATE INDEX idx_certificates_is_ca ON Certificates (is_ca); +CREATE INDEX idx_certificates_signed_by_id ON Certificates (signed_by_id); +CREATE INDEX idx_certificates_secret_id ON Certificates (secret_id); + +CREATE TABLE IF NOT EXISTS Services ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + type VARCHAR(50) NOT NULL, + resource TEXT NOT NULL, + target_port INT NOT NULL, + service_port INT, + k8s_type TEXT, + bridge_port INT, + default_bridge TEXT, + service_endpoint TEXT, + created_at DATETIME, + updated_at DATETIME, + provisioning_status VARCHAR(36) DEFAULT 'pending', + provisioning_error TEXT +); + +CREATE INDEX idx_services_name ON Services (name); +CREATE INDEX idx_services_id ON Services (id); + +CREATE TABLE IF NOT EXISTS ServiceTags ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + service_id INT NOT NULL, + tag_id INT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (service_id) REFERENCES Services (id) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_service_tags_service_id ON ServiceTags (service_id); +CREATE INDEX idx_service_tags_tag_id ON ServiceTags (tag_id); + +CREATE TABLE IF NOT EXISTS ConfigMaps ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + immutable BOOLEAN DEFAULT false, + data TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + use_vault BOOLEAN DEFAULT true +); + +CREATE INDEX idx_config_maps_name ON ConfigMaps (name); + +CREATE TABLE IF NOT EXISTS VolumeMounts ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + config_map_name VARCHAR(255), + secret_name VARCHAR(255), + version INT DEFAULT 1, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (config_map_name) REFERENCES ConfigMaps (name) ON DELETE CASCADE, + FOREIGN KEY (secret_name) REFERENCES Secrets (name) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mounts_uuid ON VolumeMounts (uuid); +CREATE INDEX idx_volume_mounts_config_map_name ON VolumeMounts (config_map_name); +CREATE INDEX idx_volume_mounts_secret_name ON VolumeMounts (secret_name); + +CREATE TABLE IF NOT EXISTS FogVolumeMounts ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + volume_mount_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (volume_mount_uuid) REFERENCES VolumeMounts (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_volume_mounts_fog_uuid ON FogVolumeMounts (fog_uuid); +CREATE INDEX idx_fog_volume_mounts_volume_mount_uuid ON FogVolumeMounts (volume_mount_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceExecStatuses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + status VARCHAR(255) DEFAULT 'INACTIVE', + exec_session_id VARCHAR(255), + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_status_microservice_uuid ON MicroserviceExecStatuses (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceHealthChecks ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + test TEXT, + `interval` BIGINT, + timeout BIGINT, + start_period BIGINT, + start_interval BIGINT, + retries INT, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_health_check_microservice_uuid ON MicroserviceHealthChecks (microservice_uuid); + +CREATE TABLE IF NOT EXISTS Events ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + timestamp BIGINT NOT NULL, + event_type VARCHAR(20) NOT NULL, + endpoint_type VARCHAR(10) NOT NULL, + actor_id VARCHAR(255), + method VARCHAR(10), + resource_type VARCHAR(50), + resource_id VARCHAR(255), + endpoint_path TEXT NOT NULL, + ip_address VARCHAR(45), + status VARCHAR(20) NOT NULL, + status_code INT, + status_message TEXT, + request_id VARCHAR(255), + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_events_timestamp ON Events (timestamp); +CREATE INDEX idx_events_endpoint_type ON Events (endpoint_type); +CREATE INDEX idx_events_actor_id ON Events (actor_id); +CREATE INDEX idx_events_resource_type ON Events (resource_type); +CREATE INDEX idx_events_status ON Events (status); +CREATE INDEX idx_events_method ON Events (method); +CREATE INDEX idx_events_event_type ON Events (event_type); +CREATE INDEX idx_events_created_at ON Events (created_at); + +CREATE TABLE IF NOT EXISTS MicroserviceLogStatuses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + log_session_id TEXT, + session_id VARCHAR(255) UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_log_status_microservice_uuid ON MicroserviceLogStatuses (microservice_uuid); +CREATE INDEX idx_microservice_log_status_session_id ON MicroserviceLogStatuses (session_id); + +CREATE TABLE IF NOT EXISTS MicroserviceExecSessions ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + session_id VARCHAR(255) UNIQUE NOT NULL, + status TEXT, + user_connected BOOLEAN DEFAULT false, + agent_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_sessions_microservice_uuid ON MicroserviceExecSessions (microservice_uuid); +CREATE INDEX idx_microservice_exec_sessions_session_id ON MicroserviceExecSessions (session_id); + +CREATE TABLE IF NOT EXISTS FogLogStatuses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + iofog_uuid VARCHAR(36), + log_session_id TEXT, + session_id VARCHAR(255) UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_log_status_iofog_uuid ON FogLogStatuses (iofog_uuid); +CREATE INDEX idx_fog_log_status_session_id ON FogLogStatuses (session_id); + +CREATE TABLE IF NOT EXISTS RbacRoles ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + kind TEXT, + created_at DATETIME, + updated_at DATETIME, + UNIQUE KEY unique_name (name(255)) +); + +CREATE TABLE IF NOT EXISTS RbacRoleRules ( + id INT AUTO_INCREMENT PRIMARY KEY, + role_id INT NOT NULL, + api_groups TEXT NOT NULL, + resources TEXT NOT NULL, + verbs TEXT NOT NULL, + resource_names TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (role_id) REFERENCES RbacRoles (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS RbacRoleBindings ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + kind TEXT, + role_ref TEXT NOT NULL, + subjects TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + UNIQUE KEY unique_name (name(255)), + role_id INTEGER +); + +CREATE TABLE IF NOT EXISTS RbacServiceAccounts ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + role_ref TEXT, + role_id INT, + microservice_uuid VARCHAR(36), + application_id INT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (role_id) REFERENCES RbacRoles (id), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES Applications (id) ON DELETE SET NULL +); + +CREATE INDEX idx_rbac_role_rules_role_id ON RbacRoleRules (role_id); +CREATE INDEX idx_rbac_roles_name ON RbacRoles (name(255)); +CREATE INDEX idx_rbac_role_bindings_name ON RbacRoleBindings (name(255)); +CREATE INDEX idx_rbac_service_accounts_name ON RbacServiceAccounts (name(255)); + +CREATE TABLE IF NOT EXISTS RbacCacheVersion ( + id INT PRIMARY KEY DEFAULT 1, + version BIGINT NOT NULL DEFAULT 1, + created_at DATETIME, + updated_at DATETIME, + CONSTRAINT single_row CHECK (id = 1) +); + +CREATE INDEX idx_rbac_role_bindings_role_id ON RbacRoleBindings (role_id); + +CREATE INDEX idx_rbac_service_accounts_role_id ON RbacServiceAccounts (role_id); +CREATE UNIQUE INDEX idx_rbac_service_accounts_microservice_uuid_unique ON RbacServiceAccounts (microservice_uuid); +CREATE UNIQUE INDEX idx_rbac_service_accounts_application_id_name_unique ON RbacServiceAccounts (application_id, name(255)); + +CREATE TABLE IF NOT EXISTS ClusterControllers ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + host VARCHAR(255), + process_id INT, + last_heartbeat DATETIME, + is_active BOOLEAN DEFAULT true, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_cluster_controllers_uuid ON ClusterControllers (uuid); +CREATE INDEX idx_cluster_controllers_host ON ClusterControllers (host); +CREATE INDEX idx_cluster_controllers_active ON ClusterControllers (is_active, last_heartbeat); + +CREATE TABLE IF NOT EXISTS NatsOperators ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + UNIQUE KEY nats_operators_name (name(255)) +); + +CREATE TABLE IF NOT EXISTS NatsAccounts ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + is_system BOOLEAN DEFAULT false, + is_leaf_system BOOLEAN DEFAULT false, + operator_id INT NOT NULL, + application_id INT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (operator_id) REFERENCES NatsOperators (id) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES Applications (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsUsers ( + id INT AUTO_INCREMENT PRIMARY KEY, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + creds_secret_name TEXT NOT NULL, + is_bearer BOOLEAN DEFAULT false, + account_id INT NOT NULL, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + nats_user_rule_id INTEGER, + FOREIGN KEY (account_id) REFERENCES NatsAccounts (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS NatsInstances ( + id INT AUTO_INCREMENT PRIMARY KEY, + iofog_uuid VARCHAR(36), + is_leaf BOOLEAN DEFAULT true, + is_hub BOOLEAN DEFAULT false, + host TEXT, + server_port INT, + leaf_port INT, + cluster_port INT, + mqtt_port INT, + http_port INT, + configmap_name TEXT, + jwt_dir_mount_name TEXT, + cert_secret_name TEXT, + js_storage_size TEXT, + js_memory_store_size TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsConnections ( + id INT AUTO_INCREMENT PRIMARY KEY, + source_nats INT NOT NULL, + dest_nats INT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (source_nats) REFERENCES NatsInstances (id) ON DELETE CASCADE, + FOREIGN KEY (dest_nats) REFERENCES NatsInstances (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsReconcileTasks ( + id INT AUTO_INCREMENT PRIMARY KEY, + reason VARCHAR(64) NOT NULL, + application_id INT, + account_rule_id INT, + user_rule_id INT, + fog_uuids TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS NatsAccountRules ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT, + info_url TEXT, + max_connections INT, + max_leaf_node_connections INT, + max_data BIGINT, + max_exports INT, + max_imports INT, + max_msg_payload INT, + max_subscriptions INT, + exports_allow_wildcards BOOLEAN DEFAULT true, + disallow_bearer BOOLEAN, + response_permissions TEXT, + resp_max INT, + resp_ttl BIGINT, + imports TEXT, + exports TEXT, + mem_storage BIGINT, + disk_storage BIGINT, + streams INT, + consumer INT, + max_ack_pending INT, + mem_max_stream_bytes BIGINT, + disk_max_stream_bytes BIGINT, + max_bytes_required BOOLEAN, + tiered_limits TEXT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS NatsUserRules ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT, + max_subscriptions INT, + max_payload INT, + max_data BIGINT, + bearer_token BOOLEAN DEFAULT false, + proxy_required BOOLEAN, + allowed_connection_types TEXT, + src TEXT, + times TEXT, + times_location TEXT, + resp_max INT, + resp_ttl BIGINT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + tags TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE UNIQUE INDEX idx_nats_accounts_application_id_unique ON NatsAccounts (application_id); +CREATE INDEX idx_nats_accounts_application_id ON NatsAccounts (application_id); +CREATE UNIQUE INDEX idx_nats_users_account_id_name ON NatsUsers (account_id, name(255)); +CREATE INDEX idx_nats_users_account_id ON NatsUsers (account_id); +CREATE INDEX idx_nats_users_microservice_uuid ON NatsUsers (microservice_uuid); +CREATE INDEX idx_nats_users_nats_user_rule_id ON NatsUsers (nats_user_rule_id); +CREATE UNIQUE INDEX idx_nats_instances_iofog_uuid_unique ON NatsInstances (iofog_uuid); +CREATE INDEX idx_nats_instances_iofog_uuid ON NatsInstances (iofog_uuid); +CREATE UNIQUE INDEX idx_nats_connections_source_dest_unique ON NatsConnections (source_nats, dest_nats); +CREATE INDEX idx_nats_connections_source_nats ON NatsConnections (source_nats); +CREATE INDEX idx_nats_connections_dest_nats ON NatsConnections (dest_nats); +CREATE INDEX idx_nats_account_rules_name ON NatsAccountRules (name); +CREATE INDEX idx_nats_user_rules_name ON NatsUserRules (name); +CREATE INDEX idx_nats_reconcile_tasks_status_claimed ON NatsReconcileTasks (status, claimed_at); + +CREATE INDEX idx_applications_nats_rule_id ON Applications (nats_rule_id); +CREATE INDEX idx_microservices_nats_rule_id ON Microservices (nats_rule_id); +CREATE INDEX idx_microservices_nats_account_id ON Microservices (nats_account_id); +CREATE INDEX idx_microservices_nats_user_id ON Microservices (nats_user_id); + +CREATE TABLE IF NOT EXISTS AuthUsers ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + password_history_hashes TEXT, + must_change_password BOOLEAN DEFAULT false, + is_bootstrap BOOLEAN DEFAULT false, + failed_attempts INT DEFAULT 0, + locked_until DATETIME, + deleted_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_users_email ON AuthUsers (email); +CREATE INDEX idx_auth_users_deleted_at ON AuthUsers (deleted_at); + +CREATE TABLE IF NOT EXISTS AuthGroups ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL UNIQUE, + is_system BOOLEAN DEFAULT false, + mfa_required BOOLEAN NOT NULL DEFAULT false, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS AuthUserGroups ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id VARCHAR(36) NOT NULL, + group_id INT NOT NULL, + created_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE, + FOREIGN KEY (group_id) REFERENCES AuthGroups (id) ON DELETE CASCADE, + UNIQUE KEY uk_auth_user_groups_user_group (user_id, group_id) +); + +CREATE INDEX idx_auth_user_groups_user_id ON AuthUserGroups (user_id); +CREATE INDEX idx_auth_user_groups_group_id ON AuthUserGroups (group_id); + +CREATE TABLE IF NOT EXISTS AuthMfa ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id VARCHAR(36) NOT NULL UNIQUE, + totp_secret_encrypted TEXT, + enabled BOOLEAN DEFAULT false, + recovery_codes_hash TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_mfa_user_id ON AuthMfa (user_id); + +CREATE TABLE IF NOT EXISTS AuthPasswordResetSessions ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + user_id VARCHAR(36) NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_password_reset_sessions_user_id ON AuthPasswordResetSessions (user_id); +CREATE INDEX idx_auth_password_reset_sessions_expires_at ON AuthPasswordResetSessions (expires_at); + +CREATE TABLE IF NOT EXISTS AuthRefreshTokens ( + id INT AUTO_INCREMENT PRIMARY KEY, + token_hash VARCHAR(255) NOT NULL, + user_id VARCHAR(36) NOT NULL, + family_id VARCHAR(36) NOT NULL, + expires_at DATETIME NOT NULL, + revoked BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_refresh_tokens_token_hash ON AuthRefreshTokens (token_hash); +CREATE INDEX idx_auth_refresh_tokens_user_id ON AuthRefreshTokens (user_id); +CREATE INDEX idx_auth_refresh_tokens_family_id ON AuthRefreshTokens (family_id); +CREATE INDEX idx_auth_refresh_tokens_expires_at ON AuthRefreshTokens (expires_at); + +CREATE TABLE IF NOT EXISTS AuthOidcKeys ( + id INT AUTO_INCREMENT PRIMARY KEY, + kid VARCHAR(255) NOT NULL UNIQUE, + key_material_encrypted TEXT, + vault_ref TEXT, + active BOOLEAN DEFAULT true, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_oidc_keys_active ON AuthOidcKeys (active); + +CREATE TABLE IF NOT EXISTS AuthOidcClients ( + id INT AUTO_INCREMENT PRIMARY KEY, + client_id VARCHAR(255) NOT NULL UNIQUE, + secret_ref TEXT, + client_type VARCHAR(32) NOT NULL DEFAULT 'confidential', + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS AuthOidcProviderStates ( + id INT AUTO_INCREMENT PRIMARY KEY, + model VARCHAR(64) NOT NULL, + record_id VARCHAR(255) NOT NULL, + payload TEXT NOT NULL, + expires_at DATETIME, + grant_id VARCHAR(255), + uid VARCHAR(255), + user_code VARCHAR(255), + consumed BOOLEAN DEFAULT false, + consumed_at DATETIME, + created_at DATETIME, + updated_at DATETIME, + UNIQUE KEY uq_auth_oidc_provider_states_model_record (model, record_id) +); + +CREATE INDEX idx_auth_oidc_provider_states_grant_id ON AuthOidcProviderStates (grant_id); +CREATE INDEX idx_auth_oidc_provider_states_uid ON AuthOidcProviderStates (uid); +CREATE INDEX idx_auth_oidc_provider_states_user_code ON AuthOidcProviderStates (user_code); +CREATE INDEX idx_auth_oidc_provider_states_expires_at ON AuthOidcProviderStates (expires_at); + +CREATE TABLE IF NOT EXISTS AuthBffSessions ( + sid VARCHAR(255) PRIMARY KEY NOT NULL, + data TEXT NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_bff_sessions_expires_at ON AuthBffSessions (expires_at); + +CREATE TABLE IF NOT EXISTS AuthInteractionStates ( + uid VARCHAR(255) PRIMARY KEY NOT NULL, + payload TEXT NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_interaction_states_expires_at ON AuthInteractionStates (expires_at); + +CREATE TABLE IF NOT EXISTS AuthBootstrapMeta ( + id INT AUTO_INCREMENT PRIMARY KEY, + completed_at DATETIME, + bootstrap_admin_user_id VARCHAR(36), + session_secret_ref TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (bootstrap_admin_user_id) REFERENCES AuthUsers (id) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS AuthPolicy ( + id INT PRIMARY KEY, + min_password_length INT DEFAULT 12, + require_uppercase BOOLEAN DEFAULT true, + require_lowercase BOOLEAN DEFAULT true, + require_digit BOOLEAN DEFAULT true, + password_max_age_days INT DEFAULT 0, + password_history_count INT DEFAULT 5, + max_failed_attempts INT DEFAULT 5, + lockout_duration_minutes INT DEFAULT 15, + access_token_ttl_seconds INT DEFAULT 900, + refresh_token_ttl_seconds INT DEFAULT 3600, + refresh_rotation BOOLEAN DEFAULT true, + max_concurrent_sessions INT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS FogPlatformSpecs ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + spec_json TEXT NOT NULL, + generation INT NOT NULL DEFAULT 1, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS FogPlatformStatuses ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + observed_generation INT NOT NULL DEFAULT 0, + phase VARCHAR(32) NOT NULL DEFAULT 'Pending', + last_error TEXT, + last_transition_at DATETIME, + conditions_json TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS FogPlatformReconcileTasks ( + id INT AUTO_INCREMENT PRIMARY KEY, + fog_uuid VARCHAR(36) NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_generation INT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + next_attempt_at DATETIME, + attempts INT NOT NULL DEFAULT 0, + last_error TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_platform_reconcile_tasks_fog_status ON FogPlatformReconcileTasks (fog_uuid, status); +CREATE INDEX idx_fog_platform_reconcile_tasks_status_claimed ON FogPlatformReconcileTasks (status, claimed_at); +CREATE INDEX idx_fog_platform_reconcile_tasks_next_attempt ON FogPlatformReconcileTasks (next_attempt_at); + +CREATE TABLE IF NOT EXISTS ServicePlatformReconcileTasks ( + id INT AUTO_INCREMENT PRIMARY KEY, + service_name TEXT NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_snapshot TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + next_attempt_at DATETIME, + attempts INT NOT NULL DEFAULT 0, + last_error TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_service_platform_reconcile_tasks_name_status ON ServicePlatformReconcileTasks (service_name(255), status); +CREATE INDEX idx_service_platform_reconcile_tasks_status_claimed ON ServicePlatformReconcileTasks (status, claimed_at); +CREATE INDEX idx_service_platform_reconcile_tasks_next_attempt ON ServicePlatformReconcileTasks (next_attempt_at); + +CREATE TABLE IF NOT EXISTS ReconcileOutbox ( + id INT AUTO_INCREMENT PRIMARY KEY, + kind VARCHAR(32) NOT NULL, + payload TEXT NOT NULL, + idempotency_key VARCHAR(255) NOT NULL, + created_at DATETIME, + processed_at DATETIME, + last_error TEXT, + UNIQUE KEY uk_reconcile_outbox_idempotency_key (idempotency_key) +); + +CREATE INDEX idx_reconcile_outbox_unprocessed ON ReconcileOutbox (processed_at, id); + +CREATE TABLE IF NOT EXISTS HubRouterConfigLocks ( + id INT PRIMARY KEY, + leader_uuid VARCHAR(36), + claimed_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_registries_url_type ON Registries (url, type); + +CREATE TABLE IF NOT EXISTS Models ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_models_registry_id ON Models (registry_id); +CREATE INDEX idx_models_name ON Models (name); + +CREATE TABLE IF NOT EXISTS RuntimeClasses ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS FogModels ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES Models (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_models_fog_uuid ON FogModels (fog_uuid); +CREATE INDEX idx_fog_models_model_uuid ON FogModels (model_uuid); + +CREATE TABLE IF NOT EXISTS FogRuntimeClasses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES RuntimeClasses (name) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_runtime_classes_fog_uuid ON FogRuntimeClasses (fog_uuid); +CREATE INDEX idx_fog_runtime_classes_runtime_class_name ON FogRuntimeClasses (runtime_class_name); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplates ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json LONGTEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplateVariables ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_template_id INT NOT NULL, + `key` TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_template_id) REFERENCES MicroserviceTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservicetemplatevariables_microservice_template_id ON MicroserviceTemplateVariables (microservice_template_id); + +CREATE TABLE IF NOT EXISTS MicroserviceEntrypoints ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_entrypoints_microserviceUuid ON MicroserviceEntrypoints (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceDevices ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_devices_microserviceUuid ON MicroserviceDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceTmpfs ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + container_path TEXT, + size FLOAT, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_tmpfs_microserviceUuid ON MicroserviceTmpfs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceUlimits ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(64) NOT NULL, + soft FLOAT, + hard FLOAT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_ulimits_microserviceUuid ON MicroserviceUlimits (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceModels ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL, + UNIQUE (microservice_uuid), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceModelItems ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Models (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_model_items_microserviceUuid ON MicroserviceModelItems (microservice_uuid); +CREATE INDEX idx_microservice_model_items_name ON MicroserviceModelItems (name); + +CREATE TABLE IF NOT EXISTS Knowledge ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_knowledge_registry_id ON Knowledge (registry_id); +CREATE INDEX idx_knowledge_name ON Knowledge (name); + +CREATE TABLE IF NOT EXISTS FogKnowledge ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES Knowledge (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_knowledge_fog_uuid ON FogKnowledge (fog_uuid); +CREATE INDEX idx_fog_knowledge_knowledge_uuid ON FogKnowledge (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledge ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL, + UNIQUE (microservice_uuid), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledgeItems ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Knowledge (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_knowledge_items_microserviceUuid ON MicroserviceKnowledgeItems (microservice_uuid); +CREATE INDEX idx_microservice_knowledge_items_name ON MicroserviceKnowledgeItems (name); + +COMMIT; diff --git a/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql new file mode 100644 index 000000000..66a8536a5 --- /dev/null +++ b/src/data/migrations/mysql/db_migration_mysql_v3.9.0.sql @@ -0,0 +1,242 @@ +ALTER TABLE Architectures DROP FOREIGN KEY Architectures_ibfk_2; +ALTER TABLE Architectures DROP FOREIGN KEY Architectures_ibfk_3; +ALTER TABLE Architectures DROP INDEX idx_architecture_hal_catalog_item_id; +ALTER TABLE Architectures DROP INDEX idx_architecture_bluetooth_catalog_item_id; +ALTER TABLE Architectures DROP COLUMN hal_catalog_item_id; +ALTER TABLE Architectures DROP COLUMN bluetooth_catalog_item_id; + +DELETE FROM Microservices WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemImages WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemInputTypes WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemOutputTypes WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue'); + +ALTER TABLE Registries ADD COLUMN type VARCHAR(16) NOT NULL DEFAULT 'oci'; +ALTER TABLE Registries ADD COLUMN ca TEXT; +ALTER TABLE Registries ADD COLUMN insecure BOOLEAN NOT NULL DEFAULT false; + +CREATE INDEX idx_registries_url_type ON Registries (url, type); + +CREATE TABLE IF NOT EXISTS Models ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_models_registry_id ON Models (registry_id); +CREATE INDEX idx_models_name ON Models (name); + +CREATE TABLE IF NOT EXISTS RuntimeClasses ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS FogModels ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES Models (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_models_fog_uuid ON FogModels (fog_uuid); +CREATE INDEX idx_fog_models_model_uuid ON FogModels (model_uuid); + +CREATE TABLE IF NOT EXISTS FogRuntimeClasses ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES RuntimeClasses (name) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_runtime_classes_fog_uuid ON FogRuntimeClasses (fog_uuid); +CREATE INDEX idx_fog_runtime_classes_runtime_class_name ON FogRuntimeClasses (runtime_class_name); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplates ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json LONGTEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplateVariables ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + microservice_template_id INT NOT NULL, + `key` TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_template_id) REFERENCES MicroserviceTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservicetemplatevariables_microservice_template_id ON MicroserviceTemplateVariables (microservice_template_id); + +ALTER TABLE ChangeTrackings ADD COLUMN models BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN runtime_classes BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN microservice_models BOOLEAN DEFAULT false; + +ALTER TABLE Fogs DROP COLUMN device_scan_frequency; +ALTER TABLE Fogs DROP COLUMN bluetooth; +ALTER TABLE Fogs DROP COLUMN hal; + +ALTER TABLE Fogs ADD COLUMN model_status TEXT; +ALTER TABLE Fogs ADD COLUMN runtime_classes TEXT; +ALTER TABLE Fogs ADD COLUMN available_cdi_devices TEXT; +ALTER TABLE Fogs ADD COLUMN active_models BIGINT DEFAULT 0; +ALTER TABLE Fogs ADD COLUMN model_last_update BIGINT DEFAULT 0; + +ALTER TABLE Microservices ADD COLUMN run_as_group TEXT; +ALTER TABLE Microservices ADD COLUMN read_only_root_filesystem BOOLEAN DEFAULT false; +ALTER TABLE Microservices ADD COLUMN working_dir TEXT; +ALTER TABLE Microservices ADD COLUMN cpus FLOAT; +ALTER TABLE Microservices ADD COLUMN memory_reservation FLOAT; +ALTER TABLE Microservices ADD COLUMN memory_swap FLOAT; +ALTER TABLE Microservices ADD COLUMN shm_size FLOAT; +ALTER TABLE Microservices ADD COLUMN sysctls TEXT; + +CREATE TABLE IF NOT EXISTS MicroserviceEntrypoints ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_entrypoints_microserviceUuid ON MicroserviceEntrypoints (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceDevices ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_devices_microserviceUuid ON MicroserviceDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceTmpfs ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + container_path TEXT, + size FLOAT, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_tmpfs_microserviceUuid ON MicroserviceTmpfs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceUlimits ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(64) NOT NULL, + soft FLOAT, + hard FLOAT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_ulimits_microserviceUuid ON MicroserviceUlimits (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceModels ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL, + UNIQUE (microservice_uuid), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceModelItems ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Models (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_model_items_microserviceUuid ON MicroserviceModelItems (microservice_uuid); +CREATE INDEX idx_microservice_model_items_name ON MicroserviceModelItems (name); + +ALTER TABLE MicroserviceStatuses ADD COLUMN pod_id TEXT; +ALTER TABLE MicroserviceStatuses ADD COLUMN last_error TEXT; +ALTER TABLE MicroserviceStatuses ADD COLUMN last_error_at BIGINT DEFAULT 0; +ALTER TABLE MicroserviceStatuses ADD COLUMN restart_count INT DEFAULT 0; + +ALTER TABLE VolumeMappings ADD COLUMN scope VARCHAR(16) NOT NULL DEFAULT 'private'; + +DROP TABLE IF EXISTS HWInfos; +DROP TABLE IF EXISTS USBInfos; + +CREATE TABLE IF NOT EXISTS Knowledge ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_knowledge_registry_id ON Knowledge (registry_id); +CREATE INDEX idx_knowledge_name ON Knowledge (name); + +CREATE TABLE IF NOT EXISTS FogKnowledge ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES Knowledge (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_knowledge_fog_uuid ON FogKnowledge (fog_uuid); +CREATE INDEX idx_fog_knowledge_knowledge_uuid ON FogKnowledge (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledge ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL, + UNIQUE (microservice_uuid), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledgeItems ( + id INT AUTO_INCREMENT PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Knowledge (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX idx_microservice_knowledge_items_microserviceUuid ON MicroserviceKnowledgeItems (microservice_uuid); +CREATE INDEX idx_microservice_knowledge_items_name ON MicroserviceKnowledgeItems (name); + +ALTER TABLE ChangeTrackings ADD COLUMN knowledge BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN microservice_knowledge BOOLEAN DEFAULT false; + +ALTER TABLE Fogs ADD COLUMN knowledge_status TEXT; +ALTER TABLE Fogs ADD COLUMN active_knowledge BIGINT DEFAULT 0; +ALTER TABLE Fogs ADD COLUMN knowledge_last_update BIGINT DEFAULT 0; + +ALTER TABLE Fogs ADD COLUMN system_cpus BIGINT; +ALTER TABLE Fogs ADD COLUMN system_total_memory BIGINT; +ALTER TABLE Fogs ADD COLUMN system_total_disk BIGINT; +ALTER TABLE Fogs ADD COLUMN system_os TEXT; +ALTER TABLE Fogs ADD COLUMN system_os_version TEXT; +ALTER TABLE Fogs ADD COLUMN system_kernel_version TEXT; diff --git a/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql new file mode 100644 index 000000000..742c5d39d --- /dev/null +++ b/src/data/migrations/postgres/db_migration_pg_baseline_v3.9.0.sql @@ -0,0 +1,1407 @@ +CREATE TABLE IF NOT EXISTS "Applications" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(255) UNIQUE, + description VARCHAR(255) DEFAULT '', + is_activated BOOLEAN DEFAULT false, + is_system BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + nats_access BOOLEAN DEFAULT false, + nats_rule_id INTEGER +); + +CREATE TABLE IF NOT EXISTS "Registries" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + url VARCHAR(255), + is_public BOOLEAN, + user_name TEXT, + password TEXT, + user_email TEXT, + type VARCHAR(16) NOT NULL DEFAULT 'oci', + ca TEXT, + insecure BOOLEAN NOT NULL DEFAULT false +); + +CREATE TABLE IF NOT EXISTS "CatalogItems" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name VARCHAR(255) UNIQUE, + description VARCHAR(255), + category TEXT, + config_example VARCHAR(255) DEFAULT '{}', + publisher TEXT, + disk_required BIGINT DEFAULT 0, + ram_required BIGINT DEFAULT 0, + picture VARCHAR(255) DEFAULT 'images/shared/default.png', + is_public BOOLEAN DEFAULT false, + registry_id INT, + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL +); + +CREATE INDEX idx_catalog_item_registry_id ON "CatalogItems" (registry_id); + +CREATE TABLE IF NOT EXISTS "Architectures" ( + id INT PRIMARY KEY, + name TEXT, + image TEXT, + description TEXT, + network_catalog_item_id INT, + FOREIGN KEY (network_catalog_item_id) REFERENCES "CatalogItems" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_architecture_network_catalog_item_id ON "Architectures" (network_catalog_item_id); + +CREATE TABLE IF NOT EXISTS "Fogs" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) DEFAULT 'Unnamed ioFog 1', + location TEXT, + gps_mode TEXT, + latitude DOUBLE PRECISION, + longitude DOUBLE PRECISION, + description TEXT, + last_active BIGINT, + daemon_status VARCHAR(36) DEFAULT 'NOT_PROVISIONED', + daemon_operating_duration BIGINT DEFAULT 0, + daemon_last_start BIGINT, + memory_usage DOUBLE PRECISION DEFAULT 0.000, + disk_usage DOUBLE PRECISION DEFAULT 0.000, + cpu_usage DOUBLE PRECISION DEFAULT 0.00, + memory_violation TEXT, + disk_violation TEXT, + cpu_violation TEXT, + system_available_disk BIGINT, + system_available_memory BIGINT, + system_total_cpu DOUBLE PRECISION, + system_cpus BIGINT, + system_total_memory BIGINT, + system_total_disk BIGINT, + system_os TEXT, + system_os_version TEXT, + system_kernel_version TEXT, + security_status VARCHAR(36) DEFAULT 'OK', + security_violation_info VARCHAR(36) DEFAULT 'No violation', + catalog_item_status TEXT, + repository_count BIGINT DEFAULT 0, + repository_status TEXT, + system_time BIGINT, + last_status_time BIGINT, + ip_address VARCHAR(36) DEFAULT '0.0.0.0', + ip_address_external VARCHAR(36) DEFAULT '0.0.0.0', + host VARCHAR(36), + catalog_item_message_counts TEXT, + last_command_time BIGINT, + network_interface VARCHAR(36) DEFAULT 'dynamic', + docker_url VARCHAR(255) DEFAULT 'unix:///run/edgelet/contaienrd.sock', + disk_limit DOUBLE PRECISION DEFAULT 50, + disk_directory VARCHAR(255) DEFAULT '/var/lib/iofog-agent/', + memory_limit DOUBLE PRECISION DEFAULT 4096, + cpu_limit DOUBLE PRECISION DEFAULT 80, + log_limit DOUBLE PRECISION DEFAULT 10, + log_directory VARCHAR(255) DEFAULT '/var/log/iofog/', + log_file_count BIGINT DEFAULT 10, + version TEXT, + is_ready_to_upgrade BOOLEAN DEFAULT TRUE, + is_ready_to_rollback BOOLEAN DEFAULT FALSE, + status_frequency INT DEFAULT 10, + change_frequency INT DEFAULT 20, + tunnel VARCHAR(255) DEFAULT '', + isolated_docker_container BOOLEAN DEFAULT FALSE, + docker_pruning_freq INT DEFAULT 0, + available_disk_threshold DOUBLE PRECISION DEFAULT 20, + log_level VARCHAR(10) DEFAULT 'INFO', + is_system BOOLEAN DEFAULT FALSE, + router_id INT DEFAULT 0, + time_zone VARCHAR(36) DEFAULT 'Etc/UTC', + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + arch_id INT DEFAULT 0, + container_engine VARCHAR(36), + deployment_type VARCHAR(36), + active_volume_mounts BIGINT DEFAULT 0, + volume_mount_last_update BIGINT DEFAULT 0, + warning_message TEXT, + gps_device VARCHAR(36), + gps_scan_frequency INT DEFAULT 60, + edge_guard_frequency INT DEFAULT 0, + gps_status VARCHAR(32), + nats_id INT, + available_runtimes TEXT, + model_status TEXT, + runtime_classes TEXT, + available_cdi_devices TEXT, + active_models BIGINT DEFAULT 0, + model_last_update BIGINT DEFAULT 0, + knowledge_status TEXT, + active_knowledge BIGINT DEFAULT 0, + knowledge_last_update BIGINT DEFAULT 0, + FOREIGN KEY (arch_id) REFERENCES "Architectures" (id) +); + +CREATE INDEX idx_fog_arch_id ON "Fogs" (arch_id); + +CREATE TABLE IF NOT EXISTS "ChangeTrackings" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_config BOOLEAN DEFAULT false, + reboot BOOLEAN DEFAULT false, + deletenode BOOLEAN DEFAULT false, + version BOOLEAN DEFAULT false, + microservice_list BOOLEAN DEFAULT false, + config BOOLEAN DEFAULT false, + registries BOOLEAN DEFAULT false, + tunnel BOOLEAN DEFAULT false, + router_changed BOOLEAN DEFAULT false, + prune BOOLEAN DEFAULT false, + last_updated VARCHAR(255) DEFAULT false, + iofog_uuid VARCHAR(36), + volume_mounts BOOLEAN DEFAULT false, + exec_sessions BOOLEAN DEFAULT false, + microservice_logs BOOLEAN DEFAULT false, + fog_logs BOOLEAN DEFAULT false, + models BOOLEAN DEFAULT false, + runtime_classes BOOLEAN DEFAULT false, + microservice_models BOOLEAN DEFAULT false, + knowledge BOOLEAN DEFAULT false, + microservice_knowledge BOOLEAN DEFAULT false, + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_change_tracking_iofog_uuid ON "ChangeTrackings" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "FogProvisionKeys" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + provisioning_string VARCHAR(100), + expiration_time BIGINT, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_provision_keys_iofogUuid ON "FogProvisionKeys" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "FogVersionCommands" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + version_command VARCHAR(100), + semver VARCHAR(100), + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_version_commands_iofogUuid ON "FogVersionCommands" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "Tunnels" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + username TEXT, + password TEXT, + host TEXT, + remote_port INT, + local_port INT DEFAULT 22, + rsa_key TEXT, + closed BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_tunnels_iofogUuid ON "Tunnels" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "Microservices" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + config TEXT, + name VARCHAR(255) DEFAULT 'New Microservice', + config_last_updated BIGINT, + rebuild BOOLEAN DEFAULT false, + log_size BIGINT DEFAULT 0, + delete BOOLEAN DEFAULT false, + delete_with_cleanup BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + catalog_item_id INT, + registry_id INT DEFAULT 1, + iofog_uuid VARCHAR(36), + application_id INT, + run_as_user TEXT, + platform TEXT, + runtime TEXT, + annotations TEXT, + pid_mode VARCHAR(36), + ipc_mode VARCHAR(36), + schedule INT DEFAULT 50, + cpu_set_cpus TEXT, + memory_limit DOUBLE PRECISION, + is_activated BOOLEAN DEFAULT true, + host_network_mode BOOLEAN DEFAULT false, + is_privileged BOOLEAN DEFAULT false, + nats_rule_id INT, + nats_access BOOLEAN DEFAULT false, + nats_account_id INT, + nats_user_id INT, + nats_creds_secret_name TEXT, + is_controller BOOLEAN DEFAULT false, + run_as_group TEXT, + read_only_root_filesystem BOOLEAN DEFAULT false, + working_dir TEXT, + cpus DOUBLE PRECISION, + memory_reservation DOUBLE PRECISION, + memory_swap DOUBLE PRECISION, + shm_size DOUBLE PRECISION, + sysctls TEXT, + FOREIGN KEY (catalog_item_id) REFERENCES "CatalogItems" (id) ON DELETE CASCADE, + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL, + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES "Applications" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservices_catalogItemId ON "Microservices" (catalog_item_id); +CREATE INDEX idx_microservices_registryId ON "Microservices" (registry_id); +CREATE INDEX idx_microservices_iofogUuid ON "Microservices" (iofog_uuid); +CREATE INDEX idx_microservices_applicationId ON "Microservices" (application_id); + +CREATE TABLE IF NOT EXISTS "MicroserviceArgs" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + cmd TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_args_microserviceUuid ON "MicroserviceArgs" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceEnvs" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + key TEXT, + value TEXT, + microservice_uuid VARCHAR(36), + value_from_secret TEXT, + value_from_config_map TEXT, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_envs_microserviceUuid ON "MicroserviceEnvs" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceExtraHost" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + template_type TEXT, + name TEXT, + template TEXT, + value TEXT, + microservice_uuid VARCHAR(36), + target_microservice_uuid VARCHAR(36), + target_fog_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_extra_host_microserviceUuid ON "MicroserviceExtraHost" (microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetMicroserviceUuid ON "MicroserviceExtraHost" (target_microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetFogUuid ON "MicroserviceExtraHost" (target_fog_uuid); + +CREATE TABLE IF NOT EXISTS "MicroservicePorts" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + port_internal INT, + port_external INT, + is_udp BOOLEAN, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_port_microserviceUuid ON "MicroservicePorts" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceStatuses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + status VARCHAR(255) DEFAULT 'QUEUED', + operating_duration BIGINT DEFAULT 0, + start_time BIGINT DEFAULT 0, + cpu_usage DOUBLE PRECISION DEFAULT 0.000, + memory_usage BIGINT DEFAULT 0, + container_id VARCHAR(255) DEFAULT '', + percentage DOUBLE PRECISION DEFAULT 0.00, + error_message TEXT, + microservice_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + ip_address TEXT, + exec_session_ids TEXT, + health_status TEXT, + pod_id TEXT DEFAULT '', + last_error TEXT DEFAULT '', + last_error_at BIGINT DEFAULT 0, + restart_count INTEGER DEFAULT 0, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_status_microserviceUuid ON "MicroserviceStatuses" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "VolumeMappings" ( + uuid INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + host_destination TEXT, + container_destination TEXT, + access_mode TEXT, + type TEXT, + scope VARCHAR(16) NOT NULL DEFAULT 'private', + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mappings_microserviceUuid ON "VolumeMappings" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "CatalogItemImages" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + container_image TEXT, + catalog_item_id INT, + microservice_uuid VARCHAR(36), + arch_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES "CatalogItems" (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (arch_id) REFERENCES "Architectures" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_image_catalog_item_id ON "CatalogItemImages" (catalog_item_id); +CREATE INDEX idx_catalog_item_image_microservice_uuid ON "CatalogItemImages" (microservice_uuid); +CREATE INDEX idx_catalog_item_image_arch_id ON "CatalogItemImages" (arch_id); + +CREATE TABLE IF NOT EXISTS "CatalogItemInputTypes" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES "CatalogItems" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_input_type_catalog_item_id ON "CatalogItemInputTypes" (catalog_item_id); + +CREATE TABLE IF NOT EXISTS "CatalogItemOutputTypes" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES "CatalogItems" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_output_type_catalog_item_id ON "CatalogItemOutputTypes" (catalog_item_id); + +CREATE TABLE IF NOT EXISTS "Routers" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + is_edge BOOLEAN DEFAULT true, + messaging_port INT DEFAULT 5671, + edge_router_port INT, + inter_router_port INT, + host TEXT, + is_default BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_router_iofogUuid ON "Routers" (iofog_uuid); + +CREATE TABLE "RouterConnections" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + source_router INT, + dest_router INT, + created_at TIMESTAMPTZ(0) NOT NULL, + updated_at TIMESTAMPTZ(0) NOT NULL, + FOREIGN KEY (source_router) REFERENCES "Routers"(id) ON DELETE CASCADE, + FOREIGN KEY (dest_router) REFERENCES "Routers"(id) ON DELETE CASCADE +); + +CREATE INDEX idx_routerconnections_sourceRouter ON "RouterConnections" (source_router); +CREATE INDEX idx_routerconnections_destRouter ON "RouterConnections" (dest_router); + +CREATE TABLE IF NOT EXISTS "Config" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + key VARCHAR(255) NOT NULL UNIQUE, + value VARCHAR(255) NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_config_key ON "Config" (key); + +CREATE TABLE IF NOT EXISTS "Tags" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + value VARCHAR(255) UNIQUE NOT NULL +); + +CREATE TABLE IF NOT EXISTS "IofogTags" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES "Tags" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_iofogtags_fog_uuid ON "IofogTags" (fog_uuid); +CREATE INDEX idx_iofogtags_tag_id ON "IofogTags" (tag_id); + +CREATE TABLE IF NOT EXISTS "ApplicationTemplates" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL DEFAULT 'new-application', + description VARCHAR(255) DEFAULT '', + schema_version VARCHAR(255) DEFAULT '', + application_json TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "ApplicationTemplateVariables" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + application_template_id INT NOT NULL, + key TEXT, + description VARCHAR(255) DEFAULT '', + default_value VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (application_template_id) REFERENCES "ApplicationTemplates" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_applicationtemplatevariables_application_template_id ON "ApplicationTemplateVariables" (application_template_id); + +CREATE TABLE IF NOT EXISTS "MicroserviceCdiDevices" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + cdi_devices TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_cdiDevices_microserviceUuid ON "MicroserviceCdiDevices" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroservicePubTags" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES "Tags" (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "MicroserviceSubTags" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES "Tags" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservicepubtags_microservice_uuid ON "MicroservicePubTags" (microservice_uuid); +CREATE INDEX idx_microservicesubtags_microservice_uuid ON "MicroserviceSubTags" (microservice_uuid); +CREATE INDEX idx_microservicepubtags_tag_id ON "MicroservicePubTags" (tag_id); +CREATE INDEX idx_microservicesubtags_tag_id ON "MicroserviceSubTags" (tag_id); + +CREATE TABLE IF NOT EXISTS "MicroserviceCapAdd" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + cap_add TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capAdd_microserviceUuid ON "MicroserviceCapAdd" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceCapDrop" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + cap_drop TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capDrop_microserviceUuid ON "MicroserviceCapDrop" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "FogPublicKeys" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + public_key TEXT, + iofog_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_public_keys_iofogUuid ON "FogPublicKeys" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "FogUsedTokens" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + jti VARCHAR(255) NOT NULL, + iofog_uuid VARCHAR(36), + expiry_time BIGINT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_used_tokens_iofogUuid ON "FogUsedTokens" (iofog_uuid); + +CREATE TABLE IF NOT EXISTS "Secrets" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + type VARCHAR(50) NOT NULL CHECK (type IN ('Opaque', 'tls')), + data TEXT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_secrets_name ON "Secrets" (name); + +CREATE TABLE IF NOT EXISTS "Certificates" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL, + subject TEXT NOT NULL, + is_ca BOOLEAN DEFAULT false, + signed_by_id INT, + hosts TEXT, + valid_from TIMESTAMPTZ(0) NOT NULL, + valid_to TIMESTAMPTZ(0) NOT NULL, + serial_number TEXT NOT NULL, + secret_id INT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (signed_by_id) REFERENCES "Certificates" (id) ON DELETE SET NULL, + FOREIGN KEY (secret_id) REFERENCES "Secrets" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_certificates_name ON "Certificates" (name); +CREATE INDEX idx_certificates_valid_to ON "Certificates" (valid_to); +CREATE INDEX idx_certificates_is_ca ON "Certificates" (is_ca); +CREATE INDEX idx_certificates_signed_by_id ON "Certificates" (signed_by_id); +CREATE INDEX idx_certificates_secret_id ON "Certificates" (secret_id); + +CREATE TABLE IF NOT EXISTS "Services" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL, + type TEXT NOT NULL, + resource TEXT NOT NULL, + target_port INTEGER NOT NULL, + service_port INTEGER, + k8s_type TEXT, + bridge_port INTEGER, + default_bridge TEXT, + service_endpoint TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + provisioning_status VARCHAR(36) DEFAULT 'pending', + provisioning_error TEXT +); + +CREATE INDEX idx_services_name ON "Services" (name); +CREATE INDEX idx_services_id ON "Services" (id); + +CREATE TABLE IF NOT EXISTS "ServiceTags" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + service_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (service_id) REFERENCES "Services" (id) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES "Tags" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_service_tags_service_id ON "ServiceTags" (service_id); +CREATE INDEX idx_service_tags_tag_id ON "ServiceTags" (tag_id); + +CREATE TABLE IF NOT EXISTS "ConfigMaps" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + immutable BOOLEAN DEFAULT false, + data TEXT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + use_vault BOOLEAN DEFAULT true +); + +CREATE INDEX idx_config_maps_name ON "ConfigMaps" (name); + +CREATE TABLE IF NOT EXISTS "VolumeMounts" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + config_map_name VARCHAR(255), + secret_name VARCHAR(255), + version INT DEFAULT 1, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (config_map_name) REFERENCES "ConfigMaps" (name) ON DELETE CASCADE, + FOREIGN KEY (secret_name) REFERENCES "Secrets" (name) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mounts_uuid ON "VolumeMounts" (uuid); +CREATE INDEX idx_volume_mounts_config_map_name ON "VolumeMounts" (config_map_name); +CREATE INDEX idx_volume_mounts_secret_name ON "VolumeMounts" (secret_name); + +CREATE TABLE IF NOT EXISTS "FogVolumeMounts" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + volume_mount_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (volume_mount_uuid) REFERENCES "VolumeMounts" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_volume_mounts_fog_uuid ON "FogVolumeMounts" (fog_uuid); +CREATE INDEX idx_fog_volume_mounts_volume_mount_uuid ON "FogVolumeMounts" (volume_mount_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceExecStatuses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + status VARCHAR(255) DEFAULT 'INACTIVE', + exec_session_id VARCHAR(255), + microservice_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_status_microservice_uuid ON "MicroserviceExecStatuses" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceHealthChecks" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + test TEXT, + interval DOUBLE PRECISION, + timeout DOUBLE PRECISION, + start_period DOUBLE PRECISION, + start_interval DOUBLE PRECISION, + retries INT, + microservice_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_health_check_microservice_uuid ON "MicroserviceHealthChecks" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "Events" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + timestamp BIGINT NOT NULL, + event_type VARCHAR(20) NOT NULL, + endpoint_type VARCHAR(10) NOT NULL, + actor_id VARCHAR(255), + method VARCHAR(10), + resource_type VARCHAR(50), + resource_id VARCHAR(255), + endpoint_path TEXT NOT NULL, + ip_address VARCHAR(45), + status VARCHAR(20) NOT NULL, + status_code INT, + status_message TEXT, + request_id VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_events_timestamp ON "Events" (timestamp); +CREATE INDEX idx_events_endpoint_type ON "Events" (endpoint_type); +CREATE INDEX idx_events_actor_id ON "Events" (actor_id); +CREATE INDEX idx_events_resource_type ON "Events" (resource_type); +CREATE INDEX idx_events_status ON "Events" (status); +CREATE INDEX idx_events_method ON "Events" (method); +CREATE INDEX idx_events_event_type ON "Events" (event_type); +CREATE INDEX idx_events_created_at ON "Events" (created_at); + +CREATE TABLE IF NOT EXISTS "MicroserviceLogStatuses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36), + log_session_id TEXT, + session_id VARCHAR(255) UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_log_status_microservice_uuid ON "MicroserviceLogStatuses" (microservice_uuid); +CREATE INDEX idx_microservice_log_status_session_id ON "MicroserviceLogStatuses" (session_id); + +CREATE TABLE IF NOT EXISTS "MicroserviceExecSessions" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + user_connected BOOLEAN DEFAULT false, + agent_connected BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_sessions_microservice_uuid ON "MicroserviceExecSessions" (microservice_uuid); +CREATE INDEX idx_microservice_exec_sessions_session_id ON "MicroserviceExecSessions" (session_id); + +CREATE TABLE IF NOT EXISTS "FogLogStatuses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + iofog_uuid VARCHAR(36), + log_session_id TEXT, + session_id VARCHAR(255) UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_log_status_iofog_uuid ON "FogLogStatuses" (iofog_uuid); +CREATE INDEX idx_fog_log_status_session_id ON "FogLogStatuses" (session_id); + +CREATE TABLE IF NOT EXISTS "RbacRoles" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT UNIQUE NOT NULL, + kind TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "RbacRoleRules" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + role_id INT NOT NULL, + api_groups TEXT NOT NULL, + resources TEXT NOT NULL, + verbs TEXT NOT NULL, + resource_names TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (role_id) REFERENCES "RbacRoles" (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "RbacRoleBindings" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT UNIQUE NOT NULL, + kind TEXT, + role_ref TEXT NOT NULL, + subjects TEXT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + role_id INTEGER +); + +CREATE TABLE IF NOT EXISTS "RbacServiceAccounts" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + name TEXT NOT NULL, + role_ref TEXT, + role_id INT REFERENCES "RbacRoles" (id), + microservice_uuid VARCHAR(36) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + application_id INT REFERENCES "Applications" (id) ON DELETE SET NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_rbac_role_rules_role_id ON "RbacRoleRules" (role_id); +CREATE INDEX idx_rbac_roles_name ON "RbacRoles" (name); +CREATE INDEX idx_rbac_role_bindings_name ON "RbacRoleBindings" (name); +CREATE INDEX idx_rbac_service_accounts_name ON "RbacServiceAccounts" (name); + +CREATE TABLE IF NOT EXISTS "RbacCacheVersion" ( + id INT PRIMARY KEY DEFAULT 1, + version BIGINT NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + CONSTRAINT single_row CHECK (id = 1) +); + +CREATE INDEX idx_rbac_role_bindings_role_id ON "RbacRoleBindings" (role_id); + +CREATE INDEX idx_rbac_service_accounts_role_id ON "RbacServiceAccounts" (role_id); +CREATE UNIQUE INDEX idx_rbac_service_accounts_microservice_uuid_unique ON "RbacServiceAccounts" (microservice_uuid) WHERE microservice_uuid IS NOT NULL; +CREATE UNIQUE INDEX idx_rbac_service_accounts_application_id_name_unique ON "RbacServiceAccounts" (application_id, name); + +CREATE TABLE IF NOT EXISTS "ClusterControllers" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + host VARCHAR(255), + process_id INT, + last_heartbeat TIMESTAMPTZ(0), + is_active BOOLEAN DEFAULT true, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_cluster_controllers_uuid ON "ClusterControllers" (uuid); +CREATE INDEX idx_cluster_controllers_host ON "ClusterControllers" (host); +CREATE INDEX idx_cluster_controllers_active ON "ClusterControllers" (is_active, last_heartbeat); + +CREATE TABLE IF NOT EXISTS "NatsOperators" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "NatsAccounts" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + is_system BOOLEAN DEFAULT false, + is_leaf_system BOOLEAN DEFAULT false, + operator_id INT NOT NULL, + application_id INT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (operator_id) REFERENCES "NatsOperators" (id) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES "Applications" (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "NatsUsers" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + creds_secret_name TEXT NOT NULL, + is_bearer BOOLEAN DEFAULT false, + account_id INT NOT NULL, + microservice_uuid VARCHAR(36), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + nats_user_rule_id INTEGER, + FOREIGN KEY (account_id) REFERENCES "NatsAccounts" (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS "NatsInstances" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + iofog_uuid VARCHAR(36), + is_leaf BOOLEAN DEFAULT true, + is_hub BOOLEAN DEFAULT false, + host TEXT, + server_port INT, + leaf_port INT, + cluster_port INT, + mqtt_port INT, + http_port INT, + configmap_name TEXT, + jwt_dir_mount_name TEXT, + cert_secret_name TEXT, + js_storage_size TEXT, + js_memory_store_size TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (iofog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "NatsConnections" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + source_nats INT NOT NULL, + dest_nats INT NOT NULL, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (source_nats) REFERENCES "NatsInstances" (id) ON DELETE CASCADE, + FOREIGN KEY (dest_nats) REFERENCES "NatsInstances" (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "NatsReconcileTasks" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + reason VARCHAR(64) NOT NULL, + application_id INT, + account_rule_id INT, + user_rule_id INT, + fog_uuids TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at TIMESTAMPTZ(0), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "NatsAccountRules" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL, + description TEXT, + info_url TEXT, + max_connections INT, + max_leaf_node_connections INT, + max_data BIGINT, + max_exports INT, + max_imports INT, + max_msg_payload INT, + max_subscriptions INT, + exports_allow_wildcards BOOLEAN DEFAULT true, + disallow_bearer BOOLEAN, + response_permissions TEXT, + resp_max INT, + resp_ttl BIGINT, + imports TEXT, + exports TEXT, + mem_storage BIGINT, + disk_storage BIGINT, + streams INT, + consumer INT, + max_ack_pending INT, + mem_max_stream_bytes BIGINT, + disk_max_stream_bytes BIGINT, + max_bytes_required BOOLEAN, + tiered_limits TEXT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "NatsUserRules" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT UNIQUE NOT NULL, + description TEXT, + max_subscriptions INT, + max_payload INT, + max_data BIGINT, + bearer_token BOOLEAN DEFAULT false, + proxy_required BOOLEAN, + allowed_connection_types TEXT, + src TEXT, + times TEXT, + times_location TEXT, + resp_max INT, + resp_ttl BIGINT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + tags TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE UNIQUE INDEX idx_nats_accounts_application_id_unique ON "NatsAccounts" (application_id) WHERE application_id IS NOT NULL; +CREATE INDEX idx_nats_accounts_application_id ON "NatsAccounts" (application_id); +CREATE UNIQUE INDEX idx_nats_users_account_id_name ON "NatsUsers" (account_id, name); +CREATE INDEX idx_nats_users_account_id ON "NatsUsers" (account_id); +CREATE INDEX idx_nats_users_microservice_uuid ON "NatsUsers" (microservice_uuid); +CREATE INDEX idx_nats_users_nats_user_rule_id ON "NatsUsers" (nats_user_rule_id); +CREATE UNIQUE INDEX idx_nats_instances_iofog_uuid_unique ON "NatsInstances" (iofog_uuid); +CREATE INDEX idx_nats_instances_iofog_uuid ON "NatsInstances" (iofog_uuid); +CREATE UNIQUE INDEX idx_nats_connections_source_dest_unique ON "NatsConnections" (source_nats, dest_nats); +CREATE INDEX idx_nats_connections_source_nats ON "NatsConnections" (source_nats); +CREATE INDEX idx_nats_connections_dest_nats ON "NatsConnections" (dest_nats); +CREATE INDEX idx_nats_account_rules_name ON "NatsAccountRules" (name); +CREATE INDEX idx_nats_user_rules_name ON "NatsUserRules" (name); +CREATE INDEX idx_nats_reconcile_tasks_status_claimed ON "NatsReconcileTasks" (status, claimed_at); + +CREATE INDEX idx_applications_nats_rule_id ON "Applications" (nats_rule_id); +CREATE INDEX idx_microservices_nats_rule_id ON "Microservices" (nats_rule_id); +CREATE INDEX idx_microservices_nats_account_id ON "Microservices" (nats_account_id); +CREATE INDEX idx_microservices_nats_user_id ON "Microservices" (nats_user_id); + +CREATE TABLE IF NOT EXISTS "AuthUsers" ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + password_history_hashes TEXT, + must_change_password BOOLEAN DEFAULT false, + is_bootstrap BOOLEAN DEFAULT false, + failed_attempts INT DEFAULT 0, + locked_until TIMESTAMPTZ(0), + deleted_at TIMESTAMPTZ(0), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_auth_users_email ON "AuthUsers" (email); +CREATE INDEX idx_auth_users_deleted_at ON "AuthUsers" (deleted_at); + +CREATE TABLE IF NOT EXISTS "AuthGroups" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + is_system BOOLEAN DEFAULT false, + mfa_required BOOLEAN NOT NULL DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "AuthUserGroups" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + user_id VARCHAR(36) NOT NULL, + group_id INT NOT NULL, + created_at TIMESTAMPTZ(0), + FOREIGN KEY (user_id) REFERENCES "AuthUsers" (id) ON DELETE CASCADE, + FOREIGN KEY (group_id) REFERENCES "AuthGroups" (id) ON DELETE CASCADE, + UNIQUE (user_id, group_id) +); + +CREATE INDEX idx_auth_user_groups_user_id ON "AuthUserGroups" (user_id); +CREATE INDEX idx_auth_user_groups_group_id ON "AuthUserGroups" (group_id); + +CREATE TABLE IF NOT EXISTS "AuthMfa" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + user_id VARCHAR(36) NOT NULL UNIQUE, + totp_secret_encrypted TEXT, + enabled BOOLEAN DEFAULT false, + recovery_codes_hash TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (user_id) REFERENCES "AuthUsers" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_mfa_user_id ON "AuthMfa" (user_id); + +CREATE TABLE IF NOT EXISTS "AuthPasswordResetSessions" ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + user_id VARCHAR(36) NOT NULL, + expires_at TIMESTAMPTZ(0) NOT NULL, + created_at TIMESTAMPTZ(0), + FOREIGN KEY (user_id) REFERENCES "AuthUsers" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_password_reset_sessions_user_id ON "AuthPasswordResetSessions" (user_id); +CREATE INDEX idx_auth_password_reset_sessions_expires_at ON "AuthPasswordResetSessions" (expires_at); + +CREATE TABLE IF NOT EXISTS "AuthRefreshTokens" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + token_hash VARCHAR(255) NOT NULL, + user_id VARCHAR(36) NOT NULL, + family_id VARCHAR(36) NOT NULL, + expires_at TIMESTAMPTZ(0) NOT NULL, + revoked BOOLEAN DEFAULT false, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (user_id) REFERENCES "AuthUsers" (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_refresh_tokens_token_hash ON "AuthRefreshTokens" (token_hash); +CREATE INDEX idx_auth_refresh_tokens_user_id ON "AuthRefreshTokens" (user_id); +CREATE INDEX idx_auth_refresh_tokens_family_id ON "AuthRefreshTokens" (family_id); +CREATE INDEX idx_auth_refresh_tokens_expires_at ON "AuthRefreshTokens" (expires_at); + +CREATE TABLE IF NOT EXISTS "AuthOidcKeys" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + kid VARCHAR(255) NOT NULL UNIQUE, + key_material_encrypted TEXT, + vault_ref TEXT, + active BOOLEAN DEFAULT true, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE INDEX idx_auth_oidc_keys_active ON "AuthOidcKeys" (active); + +CREATE TABLE IF NOT EXISTS "AuthOidcClients" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + client_id VARCHAR(255) NOT NULL UNIQUE, + secret_ref TEXT, + client_type VARCHAR(32) NOT NULL DEFAULT 'confidential', + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "AuthOidcProviderStates" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + model VARCHAR(64) NOT NULL, + record_id VARCHAR(255) NOT NULL, + payload TEXT NOT NULL, + expires_at TIMESTAMPTZ(0), + grant_id VARCHAR(255), + uid VARCHAR(255), + user_code VARCHAR(255), + consumed BOOLEAN DEFAULT false, + consumed_at TIMESTAMPTZ(0), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + UNIQUE (model, record_id) +); + +CREATE INDEX idx_auth_oidc_provider_states_grant_id ON "AuthOidcProviderStates" (grant_id); +CREATE INDEX idx_auth_oidc_provider_states_uid ON "AuthOidcProviderStates" (uid); +CREATE INDEX idx_auth_oidc_provider_states_user_code ON "AuthOidcProviderStates" (user_code); +CREATE INDEX idx_auth_oidc_provider_states_expires_at ON "AuthOidcProviderStates" (expires_at); + +CREATE TABLE IF NOT EXISTS "AuthBffSessions" ( + sid VARCHAR(255) PRIMARY KEY NOT NULL, + data TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ +); + +CREATE INDEX idx_auth_bff_sessions_expires_at ON "AuthBffSessions" (expires_at); + +CREATE TABLE IF NOT EXISTS "AuthInteractionStates" ( + uid VARCHAR(255) PRIMARY KEY NOT NULL, + payload TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ +); + +CREATE INDEX idx_auth_interaction_states_expires_at ON "AuthInteractionStates" (expires_at); + +CREATE TABLE IF NOT EXISTS "AuthBootstrapMeta" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + completed_at TIMESTAMPTZ(0), + bootstrap_admin_user_id VARCHAR(36), + session_secret_ref TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (bootstrap_admin_user_id) REFERENCES "AuthUsers" (id) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS "AuthPolicy" ( + id INT PRIMARY KEY NOT NULL CHECK (id = 1), + min_password_length INT DEFAULT 12, + require_uppercase BOOLEAN DEFAULT true, + require_lowercase BOOLEAN DEFAULT true, + require_digit BOOLEAN DEFAULT true, + password_max_age_days INT DEFAULT 0, + password_history_count INT DEFAULT 5, + max_failed_attempts INT DEFAULT 5, + lockout_duration_minutes INT DEFAULT 15, + access_token_ttl_seconds INT DEFAULT 900, + refresh_token_ttl_seconds INT DEFAULT 3600, + refresh_rotation BOOLEAN DEFAULT true, + max_concurrent_sessions INT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "FogPlatformSpecs" ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + spec_json TEXT NOT NULL, + generation INT NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "FogPlatformStatuses" ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + observed_generation INT NOT NULL DEFAULT 0, + phase VARCHAR(32) NOT NULL DEFAULT 'Pending', + last_error TEXT, + last_transition_at TIMESTAMPTZ(0), + conditions_json TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "FogPlatformReconcileTasks" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36) NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_generation INT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at TIMESTAMPTZ(0), + next_attempt_at TIMESTAMPTZ(0), + attempts INT NOT NULL DEFAULT 0, + last_error TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX idx_fog_platform_reconcile_tasks_active_fog_uuid ON "FogPlatformReconcileTasks" (fog_uuid) WHERE status IN ('pending', 'in_progress'); +CREATE INDEX idx_fog_platform_reconcile_tasks_status_claimed ON "FogPlatformReconcileTasks" (status, claimed_at); +CREATE INDEX idx_fog_platform_reconcile_tasks_next_attempt ON "FogPlatformReconcileTasks" (next_attempt_at); + +CREATE TABLE IF NOT EXISTS "ServicePlatformReconcileTasks" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + service_name TEXT NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_snapshot TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at TIMESTAMPTZ(0), + next_attempt_at TIMESTAMPTZ(0), + attempts INT NOT NULL DEFAULT 0, + last_error TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE UNIQUE INDEX idx_service_platform_reconcile_tasks_active_service_name ON "ServicePlatformReconcileTasks" (service_name) WHERE status IN ('pending', 'in_progress'); +CREATE INDEX idx_service_platform_reconcile_tasks_status_claimed ON "ServicePlatformReconcileTasks" (status, claimed_at); +CREATE INDEX idx_service_platform_reconcile_tasks_next_attempt ON "ServicePlatformReconcileTasks" (next_attempt_at); + +CREATE TABLE IF NOT EXISTS "ReconcileOutbox" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + kind VARCHAR(32) NOT NULL, + payload TEXT NOT NULL, + idempotency_key VARCHAR(255) NOT NULL UNIQUE, + created_at TIMESTAMPTZ(0), + processed_at TIMESTAMPTZ(0), + last_error TEXT +); + +CREATE INDEX idx_reconcile_outbox_unprocessed ON "ReconcileOutbox" (processed_at, id); + +CREATE TABLE IF NOT EXISTS "HubRouterConfigLocks" ( + id INT PRIMARY KEY NOT NULL CHECK (id = 1), + leader_uuid VARCHAR(36), + claimed_at TIMESTAMPTZ(0), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); +CREATE INDEX IF NOT EXISTS idx_registries_url_type ON "Registries" (url, type); + +CREATE TABLE IF NOT EXISTS "Models" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_models_registry_id ON "Models" (registry_id); +CREATE INDEX IF NOT EXISTS idx_models_name ON "Models" (name); + +CREATE TABLE IF NOT EXISTS "RuntimeClasses" ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS "FogModels" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES "Models" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_models_fog_uuid ON "FogModels" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_models_model_uuid ON "FogModels" (model_uuid); + +CREATE TABLE IF NOT EXISTS "FogRuntimeClasses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES "RuntimeClasses" (name) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_fog_uuid ON "FogRuntimeClasses" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_runtime_class_name ON "FogRuntimeClasses" (runtime_class_name); + +CREATE TABLE IF NOT EXISTS "MicroserviceTemplates" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "MicroserviceTemplateVariables" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_template_id INT NOT NULL, + key TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_template_id) REFERENCES "MicroserviceTemplates" (id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservicetemplatevariables_microservice_template_id ON "MicroserviceTemplateVariables" (microservice_template_id); + +CREATE TABLE IF NOT EXISTS "MicroserviceEntrypoints" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_entrypoints_microserviceUuid ON "MicroserviceEntrypoints" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceDevices" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_devices_microserviceUuid ON "MicroserviceDevices" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceTmpfs" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + container_path TEXT, + size DOUBLE PRECISION, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_tmpfs_microserviceUuid ON "MicroserviceTmpfs" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceUlimits" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + soft DOUBLE PRECISION, + hard DOUBLE PRECISION, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_ulimits_microserviceUuid ON "MicroserviceUlimits" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceModels" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "MicroserviceModelItems" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES "Models" (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON "MicroserviceModelItems" (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON "MicroserviceModelItems" (name); + +CREATE TABLE IF NOT EXISTS "Knowledge" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_registry_id ON "Knowledge" (registry_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_name ON "Knowledge" (name); + +CREATE TABLE IF NOT EXISTS "FogKnowledge" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES "Knowledge" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_fog_uuid ON "FogKnowledge" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_knowledge_uuid ON "FogKnowledge" (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceKnowledge" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "MicroserviceKnowledgeItems" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES "Knowledge" (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_microserviceUuid ON "MicroserviceKnowledgeItems" (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_name ON "MicroserviceKnowledgeItems" (name); diff --git a/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql new file mode 100644 index 000000000..fb3e494b1 --- /dev/null +++ b/src/data/migrations/postgres/db_migration_pg_v3.9.0.sql @@ -0,0 +1,238 @@ +ALTER TABLE "Architectures" DROP COLUMN IF EXISTS hal_catalog_item_id CASCADE; +ALTER TABLE "Architectures" DROP COLUMN IF EXISTS bluetooth_catalog_item_id CASCADE; +DROP INDEX IF EXISTS idx_architecture_hal_catalog_item_id; +DROP INDEX IF EXISTS idx_architecture_bluetooth_catalog_item_id; + +DELETE FROM "Microservices" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemImages" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemInputTypes" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemOutputTypes" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue'); + +ALTER TABLE "Registries" ADD COLUMN IF NOT EXISTS type VARCHAR(16) NOT NULL DEFAULT 'oci'; +ALTER TABLE "Registries" ADD COLUMN IF NOT EXISTS ca TEXT; +ALTER TABLE "Registries" ADD COLUMN IF NOT EXISTS insecure BOOLEAN NOT NULL DEFAULT false; + +CREATE INDEX IF NOT EXISTS idx_registries_url_type ON "Registries" (url, type); + +CREATE TABLE IF NOT EXISTS "Models" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_models_registry_id ON "Models" (registry_id); +CREATE INDEX IF NOT EXISTS idx_models_name ON "Models" (name); + +CREATE TABLE IF NOT EXISTS "RuntimeClasses" ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS "FogModels" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES "Models" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_models_fog_uuid ON "FogModels" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_models_model_uuid ON "FogModels" (model_uuid); + +CREATE TABLE IF NOT EXISTS "FogRuntimeClasses" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES "RuntimeClasses" (name) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_fog_uuid ON "FogRuntimeClasses" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_runtime_class_name ON "FogRuntimeClasses" (runtime_class_name); + +CREATE TABLE IF NOT EXISTS "MicroserviceTemplates" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0) +); + +CREATE TABLE IF NOT EXISTS "MicroserviceTemplateVariables" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + microservice_template_id INT NOT NULL, + key TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (microservice_template_id) REFERENCES "MicroserviceTemplates" (id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservicetemplatevariables_microservice_template_id ON "MicroserviceTemplateVariables" (microservice_template_id); + +ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS models BOOLEAN DEFAULT false; +ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS runtime_classes BOOLEAN DEFAULT false; +ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS microservice_models BOOLEAN DEFAULT false; + +ALTER TABLE "Fogs" DROP COLUMN IF EXISTS device_scan_frequency; +ALTER TABLE "Fogs" DROP COLUMN IF EXISTS bluetooth; +ALTER TABLE "Fogs" DROP COLUMN IF EXISTS hal; + +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS model_status TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS runtime_classes TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS available_cdi_devices TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS active_models BIGINT DEFAULT 0; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS model_last_update BIGINT DEFAULT 0; + +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS run_as_group TEXT; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS read_only_root_filesystem BOOLEAN DEFAULT false; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS working_dir TEXT; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS cpus DOUBLE PRECISION; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS memory_reservation DOUBLE PRECISION; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS memory_swap DOUBLE PRECISION; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS shm_size DOUBLE PRECISION; +ALTER TABLE "Microservices" ADD COLUMN IF NOT EXISTS sysctls TEXT; + +CREATE TABLE IF NOT EXISTS "MicroserviceEntrypoints" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_entrypoints_microserviceUuid ON "MicroserviceEntrypoints" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceDevices" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_devices_microserviceUuid ON "MicroserviceDevices" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceTmpfs" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + container_path TEXT, + size DOUBLE PRECISION, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_tmpfs_microserviceUuid ON "MicroserviceTmpfs" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceUlimits" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name TEXT NOT NULL, + soft DOUBLE PRECISION, + hard DOUBLE PRECISION, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_ulimits_microserviceUuid ON "MicroserviceUlimits" (microservice_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceModels" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "MicroserviceModelItems" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES "Models" (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON "MicroserviceModelItems" (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON "MicroserviceModelItems" (name); + +ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS pod_id TEXT DEFAULT ''; +ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS last_error TEXT DEFAULT ''; +ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS last_error_at BIGINT DEFAULT 0; +ALTER TABLE "MicroserviceStatuses" ADD COLUMN IF NOT EXISTS restart_count INTEGER DEFAULT 0; + +ALTER TABLE "VolumeMappings" ADD COLUMN IF NOT EXISTS scope VARCHAR(16) NOT NULL DEFAULT 'private'; + +DROP TABLE IF EXISTS "HWInfos"; +DROP TABLE IF EXISTS "USBInfos"; + +CREATE TABLE IF NOT EXISTS "Knowledge" ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at TIMESTAMPTZ(0), + updated_at TIMESTAMPTZ(0), + FOREIGN KEY (registry_id) REFERENCES "Registries" (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_registry_id ON "Knowledge" (registry_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_name ON "Knowledge" (name); + +CREATE TABLE IF NOT EXISTS "FogKnowledge" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES "Fogs" (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES "Knowledge" (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_fog_uuid ON "FogKnowledge" (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_knowledge_uuid ON "FogKnowledge" (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS "MicroserviceKnowledge" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS "MicroserviceKnowledgeItems" ( + id INT GENERATED ALWAYS AS IDENTITY PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES "Microservices" (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES "Knowledge" (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_microserviceUuid ON "MicroserviceKnowledgeItems" (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_name ON "MicroserviceKnowledgeItems" (name); + +ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS knowledge BOOLEAN DEFAULT false; +ALTER TABLE "ChangeTrackings" ADD COLUMN IF NOT EXISTS microservice_knowledge BOOLEAN DEFAULT false; + +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS knowledge_status TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS active_knowledge BIGINT DEFAULT 0; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS knowledge_last_update BIGINT DEFAULT 0; + +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_cpus BIGINT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_total_memory BIGINT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_total_disk BIGINT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_os TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_os_version TEXT; +ALTER TABLE "Fogs" ADD COLUMN IF NOT EXISTS system_kernel_version TEXT; diff --git a/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql new file mode 100644 index 000000000..dd4392129 --- /dev/null +++ b/src/data/migrations/sqlite/db_migration_sqlite_baseline_v3.9.0.sql @@ -0,0 +1,1406 @@ +CREATE TABLE IF NOT EXISTS Applications ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE, + description VARCHAR(255) DEFAULT '', + is_activated BOOLEAN DEFAULT false, + is_system BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + nats_access BOOLEAN DEFAULT false, + nats_rule_id INTEGER +); + +CREATE TABLE IF NOT EXISTS Registries ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + url VARCHAR(255), + is_public BOOLEAN, + user_name TEXT, + password TEXT, + user_email TEXT, + type VARCHAR(16) DEFAULT 'oci', + ca TEXT, + insecure BOOLEAN DEFAULT false +); + +CREATE TABLE IF NOT EXISTS CatalogItems ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE, + description VARCHAR(255), + category TEXT, + config_example VARCHAR(255) DEFAULT '{}', + publisher TEXT, + disk_required BIGINT DEFAULT 0, + ram_required BIGINT DEFAULT 0, + picture VARCHAR(255) DEFAULT 'images/shared/default.png', + is_public BOOLEAN DEFAULT false, + registry_id INT, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX idx_catalog_item_registry_id ON CatalogItems (registry_id); + +CREATE TABLE IF NOT EXISTS Architectures ( + id INT PRIMARY KEY, + name TEXT, + image TEXT, + description TEXT, + network_catalog_item_id INT, + FOREIGN KEY (network_catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_architecture_network_catalog_item_id ON Architectures (network_catalog_item_id); + +CREATE TABLE IF NOT EXISTS Fogs ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) DEFAULT 'Unnamed ioFog 1', + location TEXT, + gps_mode TEXT, + latitude FLOAT, + longitude FLOAT, + description TEXT, + last_active BIGINT, + daemon_status VARCHAR(36) DEFAULT 'NOT_PROVISIONED', + daemon_operating_duration BIGINT DEFAULT 0, + daemon_last_start BIGINT, + memory_usage FLOAT DEFAULT 0.000, + disk_usage FLOAT DEFAULT 0.000, + cpu_usage FLOAT DEFAULT 0.00, + memory_violation TEXT, + disk_violation TEXT, + cpu_violation TEXT, + system_available_disk BIGINT, + system_available_memory BIGINT, + system_total_cpu FLOAT, + system_cpus BIGINT, + system_total_memory BIGINT, + system_total_disk BIGINT, + system_os TEXT, + system_os_version TEXT, + system_kernel_version TEXT, + security_status VARCHAR(36) DEFAULT 'OK', + security_violation_info VARCHAR(36) DEFAULT 'No violation', + catalog_item_status TEXT, + repository_count BIGINT DEFAULT 0, + repository_status TEXT, + system_time BIGINT, + last_status_time BIGINT, + ip_address VARCHAR(36) DEFAULT '0.0.0.0', + ip_address_external VARCHAR(36) DEFAULT '0.0.0.0', + host VARCHAR(36), + catalog_item_message_counts TEXT, + last_command_time BIGINT, + network_interface VARCHAR(36) DEFAULT 'dynamic', + docker_url VARCHAR(255) DEFAULT 'unix:///run/edgelet/contaienrd.sock', + disk_limit FLOAT DEFAULT 50, + disk_directory VARCHAR(255) DEFAULT '/var/lib/iofog-agent/', + memory_limit FLOAT DEFAULT 4096, + cpu_limit FLOAT DEFAULT 80, + log_limit FLOAT DEFAULT 10, + log_directory VARCHAR(255) DEFAULT '/var/log/iofog-agent/', + log_file_count BIGINT DEFAULT 10, + `version` TEXT, + is_ready_to_upgrade BOOLEAN DEFAULT TRUE, + is_ready_to_rollback BOOLEAN DEFAULT FALSE, + status_frequency INT DEFAULT 10, + change_frequency INT DEFAULT 20, + tunnel VARCHAR(255) DEFAULT '', + isolated_docker_container BOOLEAN DEFAULT FALSE, + docker_pruning_freq INT DEFAULT 0, + available_disk_threshold FLOAT DEFAULT 20, + log_level VARCHAR(10) DEFAULT 'INFO', + is_system BOOLEAN DEFAULT FALSE, + router_id INT DEFAULT 0, + time_zone VARCHAR(36) DEFAULT 'Etc/UTC', + created_at DATETIME, + updated_at DATETIME, + arch_id INT DEFAULT 0, + container_engine VARCHAR(36), + deployment_type VARCHAR(36), + active_volume_mounts BIGINT DEFAULT 0, + volume_mount_last_update BIGINT DEFAULT 0, + warning_message TEXT, + gps_device VARCHAR(36), + gps_scan_frequency INT DEFAULT 60, + edge_guard_frequency INT DEFAULT 0, + gps_status VARCHAR(32), + nats_id INT, + available_runtimes TEXT, + model_status TEXT, + runtime_classes TEXT, + available_cdi_devices TEXT, + active_models BIGINT DEFAULT 0, + model_last_update BIGINT DEFAULT 0, + knowledge_status TEXT, + active_knowledge BIGINT DEFAULT 0, + knowledge_last_update BIGINT DEFAULT 0, + FOREIGN KEY (arch_id) REFERENCES Architectures (id) +); + +CREATE INDEX idx_fog_arch_id ON Fogs (arch_id); + +CREATE TABLE IF NOT EXISTS ChangeTrackings ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_config BOOLEAN DEFAULT false, + reboot BOOLEAN DEFAULT false, + deletenode BOOLEAN DEFAULT false, + version BOOLEAN DEFAULT false, + microservice_list BOOLEAN DEFAULT false, + config BOOLEAN DEFAULT false, + registries BOOLEAN DEFAULT false, + tunnel BOOLEAN DEFAULT false, + router_changed BOOLEAN DEFAULT false, + prune BOOLEAN DEFAULT false, + last_updated VARCHAR(255) DEFAULT false, + iofog_uuid VARCHAR(36), + volume_mounts BOOLEAN DEFAULT false, + exec_sessions BOOLEAN DEFAULT false, + microservice_logs BOOLEAN DEFAULT false, + fog_logs BOOLEAN DEFAULT false, + models BOOLEAN DEFAULT false, + runtime_classes BOOLEAN DEFAULT false, + microservice_models BOOLEAN DEFAULT false, + knowledge BOOLEAN DEFAULT false, + microservice_knowledge BOOLEAN DEFAULT false, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_change_tracking_iofog_uuid ON ChangeTrackings (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogProvisionKeys ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + provisioning_string VARCHAR(100), + expiration_time BIGINT, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_provision_keys_iofogUuid ON FogProvisionKeys (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogVersionCommands ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + version_command VARCHAR(100), + semver VARCHAR(100), + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_version_commands_iofogUuid ON FogVersionCommands (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Tunnels ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + username TEXT, + password TEXT, + host TEXT, + remote_port INT, + local_port INT DEFAULT 22, + rsa_key TEXT, + closed BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_tunnels_iofogUuid ON Tunnels (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Microservices ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + config TEXT, + name VARCHAR(255) DEFAULT 'New Microservice', + config_last_updated BIGINT, + rebuild BOOLEAN DEFAULT false, + log_size BIGINT DEFAULT 0, + `delete` BOOLEAN DEFAULT false, + delete_with_cleanup BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + catalog_item_id INT, + registry_id INT DEFAULT 1, + iofog_uuid VARCHAR(36), + application_id INT, + run_as_user TEXT, + platform TEXT, + runtime TEXT, + annotations TEXT, + pid_mode VARCHAR(36), + ipc_mode VARCHAR(36), + schedule INT DEFAULT 50, + cpu_set_cpus TEXT, + memory_limit FLOAT, + is_activated BOOLEAN DEFAULT true, + host_network_mode BOOLEAN DEFAULT false, + is_privileged BOOLEAN DEFAULT false, + nats_rule_id INTEGER, + nats_access BOOLEAN DEFAULT false, + nats_account_id INTEGER, + nats_user_id INTEGER, + nats_creds_secret_name TEXT, + is_controller BOOLEAN DEFAULT false, + run_as_group TEXT, + read_only_root_filesystem BOOLEAN DEFAULT false, + working_dir TEXT, + cpus FLOAT, + memory_reservation FLOAT, + memory_swap FLOAT, + shm_size FLOAT, + sysctls TEXT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES Applications (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservices_catalogItemId ON Microservices (catalog_item_id); +CREATE INDEX idx_microservices_registryId ON Microservices (registry_id); +CREATE INDEX idx_microservices_iofogUuid ON Microservices (iofog_uuid); +CREATE INDEX idx_microservices_applicationId ON Microservices (application_id); + +CREATE TABLE IF NOT EXISTS MicroserviceArgs ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + cmd TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_args_microserviceUuid ON MicroserviceArgs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceEnvs ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + `key` TEXT, + `value` TEXT, + microservice_uuid VARCHAR(36), + value_from_secret TEXT, + value_from_config_map TEXT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_envs_microserviceUuid ON MicroserviceEnvs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceExtraHost ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + template_type TEXT, + name TEXT, + template TEXT, + `value` TEXT, + microservice_uuid VARCHAR(36), + target_microservice_uuid VARCHAR(36), + target_fog_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (target_fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_extra_host_microserviceUuid ON MicroserviceExtraHost (microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetMicroserviceUuid ON MicroserviceExtraHost (target_microservice_uuid); +CREATE INDEX idx_microservice_extra_host_targetFogUuid ON MicroserviceExtraHost (target_fog_uuid); + +CREATE TABLE IF NOT EXISTS MicroservicePorts ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + port_internal INT, + port_external INT, + is_udp BOOLEAN, + created_at DATETIME, + updated_at DATETIME, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_port_microserviceUuid ON MicroservicePorts (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceStatuses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + status VARCHAR(255) DEFAULT 'QUEUED', + operating_duration BIGINT DEFAULT 0, + start_time BIGINT DEFAULT 0, + cpu_usage FLOAT DEFAULT 0.000, + memory_usage BIGINT DEFAULT 0, + container_id VARCHAR(255) DEFAULT '', + percentage FLOAT DEFAULT 0.00, + error_message TEXT, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + ip_address TEXT, + exec_session_ids TEXT, + health_status TEXT, + pod_id TEXT DEFAULT '', + last_error TEXT DEFAULT '', + last_error_at BIGINT DEFAULT 0, + restart_count INTEGER DEFAULT 0, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_status_microserviceUuid ON MicroserviceStatuses (microservice_uuid); + +CREATE TABLE IF NOT EXISTS VolumeMappings ( + uuid INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + host_destination TEXT, + container_destination TEXT, + access_mode TEXT, + type TEXT, + scope TEXT NOT NULL DEFAULT 'private', + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mappings_microserviceUuid ON VolumeMappings (microservice_uuid); + +CREATE TABLE IF NOT EXISTS CatalogItemImages ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + container_image TEXT, + catalog_item_id INT, + microservice_uuid VARCHAR(36), + arch_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (arch_id) REFERENCES Architectures (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_image_catalog_item_id ON CatalogItemImages (catalog_item_id); +CREATE INDEX idx_catalog_item_image_microservice_uuid ON CatalogItemImages (microservice_uuid); +CREATE INDEX idx_catalog_item_image_arch_id ON CatalogItemImages (arch_id); + +CREATE TABLE IF NOT EXISTS CatalogItemInputTypes ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_input_type_catalog_item_id ON CatalogItemInputTypes (catalog_item_id); + +CREATE TABLE IF NOT EXISTS CatalogItemOutputTypes ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + info_type TEXT, + info_format TEXT, + catalog_item_id INT, + FOREIGN KEY (catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +CREATE INDEX idx_catalog_item_output_type_catalog_item_id ON CatalogItemOutputTypes (catalog_item_id); + +CREATE TABLE IF NOT EXISTS Routers ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + is_edge BOOLEAN DEFAULT true, + messaging_port INT DEFAULT 5671, + edge_router_port INT, + inter_router_port INT, + host TEXT, + is_default BOOLEAN DEFAULT false, + iofog_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_router_iofogUuid ON Routers (iofog_uuid); + +CREATE TABLE RouterConnections ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + source_router INT, + dest_router INT, + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL, + FOREIGN KEY (source_router) REFERENCES Routers(id) ON DELETE CASCADE, + FOREIGN KEY (dest_router) REFERENCES Routers(id) ON DELETE CASCADE +); + +CREATE INDEX idx_routerconnections_sourceRouter ON RouterConnections (source_router); +CREATE INDEX idx_routerconnections_destRouter ON RouterConnections (dest_router); + +CREATE TABLE IF NOT EXISTS Config ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + `key` VARCHAR(255) NOT NULL UNIQUE, + value VARCHAR(255) NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_config_key ON Config (`key`); + +CREATE TABLE IF NOT EXISTS Tags ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + value VARCHAR(255) UNIQUE NOT NULL +); + +CREATE TABLE IF NOT EXISTS IofogTags ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_iofogtags_fog_uuid ON IofogTags (fog_uuid); +CREATE INDEX idx_iofogtags_tag_id ON IofogTags (tag_id); + +CREATE TABLE IF NOT EXISTS ApplicationTemplates ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL DEFAULT 'new-application', + description VARCHAR(255) DEFAULT '', + schema_version VARCHAR(255) DEFAULT '', + application_json LONGTEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS ApplicationTemplateVariables ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + application_template_id INT NOT NULL, + `key` TEXT, + description VARCHAR(255) DEFAULT '', + default_value VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (application_template_id) REFERENCES ApplicationTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX idx_applicationtemplatevariables_application_template_id ON ApplicationTemplateVariables (application_template_id); + +CREATE TABLE IF NOT EXISTS MicroserviceCdiDevices ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + cdi_devices TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_cdiDevices_microserviceUuid ON MicroserviceCdiDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroservicePubTags ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceSubTags ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_uuid VARCHAR(36), + tag_id INT, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_microservicepubtags_microservice_uuid ON MicroservicePubTags (microservice_uuid); +CREATE INDEX idx_microservicesubtags_microservice_uuid ON MicroservicesubTags (microservice_uuid); +CREATE INDEX idx_microservicepubtags_tag_id ON MicroservicePubTags (tag_id); +CREATE INDEX idx_microservicesubtags_tag_id ON MicroservicesubTags (tag_id); + +CREATE TABLE IF NOT EXISTS MicroserviceCapAdd ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + cap_add TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capAdd_microserviceUuid ON MicroserviceCapAdd (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceCapDrop ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + cap_drop TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_capDrop_microserviceUuid ON MicroserviceCapDrop (microservice_uuid); + +CREATE TABLE IF NOT EXISTS FogPublicKeys ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + public_key TEXT, + iofog_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_public_keys_iofogUuid ON FogPublicKeys (iofog_uuid); + +CREATE TABLE IF NOT EXISTS FogUsedTokens ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + jti VARCHAR(255) NOT NULL, + iofog_uuid VARCHAR(36), + expiry_time BIGINT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_used_tokens_iofogUuid ON FogUsedTokens (iofog_uuid); + +CREATE TABLE IF NOT EXISTS Secrets ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + type VARCHAR(50) NOT NULL CHECK (type IN ('Opaque', 'tls')), + data TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_secrets_name ON Secrets (name); + +CREATE TABLE IF NOT EXISTS Certificates ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + subject TEXT NOT NULL, + is_ca BOOLEAN DEFAULT false, + signed_by_id INTEGER, + hosts TEXT, + valid_from DATETIME NOT NULL, + valid_to DATETIME NOT NULL, + serial_number TEXT NOT NULL, + secret_id INTEGER, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (signed_by_id) REFERENCES Certificates (id) ON DELETE SET NULL, + FOREIGN KEY (secret_id) REFERENCES Secrets (id) ON DELETE CASCADE +); + +CREATE INDEX idx_certificates_name ON Certificates (name); +CREATE INDEX idx_certificates_valid_to ON Certificates (valid_to); +CREATE INDEX idx_certificates_is_ca ON Certificates (is_ca); +CREATE INDEX idx_certificates_signed_by_id ON Certificates (signed_by_id); +CREATE INDEX idx_certificates_secret_id ON Certificates (secret_id); + +CREATE TABLE IF NOT EXISTS Services ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + type TEXT NOT NULL, + resource TEXT NOT NULL, + target_port INTEGER NOT NULL, + service_port INTEGER, + k8s_type TEXT, + bridge_port INTEGER, + default_bridge TEXT, + service_endpoint TEXT, + created_at DATETIME, + updated_at DATETIME, + provisioning_status VARCHAR(36) DEFAULT 'pending', + provisioning_error TEXT +); + +CREATE INDEX idx_services_id ON Services (id); +CREATE INDEX idx_services_name ON Services (name); + +CREATE TABLE IF NOT EXISTS ServiceTags ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + service_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (service_id) REFERENCES Services (id) ON DELETE CASCADE, + FOREIGN KEY (tag_id) REFERENCES Tags (id) ON DELETE CASCADE +); + +CREATE INDEX idx_service_tags_service_id ON ServiceTags (service_id); +CREATE INDEX idx_service_tags_tag_id ON ServiceTags (tag_id); + +CREATE TABLE IF NOT EXISTS ConfigMaps ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + immutable BOOLEAN DEFAULT false, + data TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + use_vault BOOLEAN DEFAULT true +); + +CREATE INDEX idx_config_maps_name ON ConfigMaps (name); + +CREATE TABLE IF NOT EXISTS VolumeMounts ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + config_map_name VARCHAR(255), + secret_name VARCHAR(255), + version INTEGER DEFAULT 1, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (config_map_name) REFERENCES ConfigMaps (name) ON DELETE CASCADE, + FOREIGN KEY (secret_name) REFERENCES Secrets (name) ON DELETE CASCADE +); + +CREATE INDEX idx_volume_mounts_uuid ON VolumeMounts (uuid); +CREATE INDEX idx_volume_mounts_config_map_name ON VolumeMounts (config_map_name); +CREATE INDEX idx_volume_mounts_secret_name ON VolumeMounts (secret_name); + +CREATE TABLE IF NOT EXISTS FogVolumeMounts ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + volume_mount_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (volume_mount_uuid) REFERENCES VolumeMounts (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_volume_mounts_fog_uuid ON FogVolumeMounts (fog_uuid); +CREATE INDEX idx_fog_volume_mounts_volume_mount_uuid ON FogVolumeMounts (volume_mount_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceExecStatuses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + status VARCHAR(255) DEFAULT 'INACTIVE', + exec_session_id VARCHAR(255), + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_status_microservice_uuid ON MicroserviceExecStatuses (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceHealthChecks ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + test TEXT, + interval FLOAT, + timeout FLOAT, + start_period FLOAT, + start_interval FLOAT, + retries INT, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_health_check_microservice_uuid ON MicroserviceHealthChecks (microservice_uuid); + +CREATE TABLE IF NOT EXISTS Events ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + timestamp BIGINT NOT NULL, + event_type VARCHAR(20) NOT NULL, + endpoint_type VARCHAR(10) NOT NULL, + actor_id VARCHAR(255), + method VARCHAR(10), + resource_type VARCHAR(50), + resource_id VARCHAR(255), + endpoint_path TEXT NOT NULL, + ip_address VARCHAR(45), + status VARCHAR(20) NOT NULL, + status_code INTEGER, + status_message TEXT, + request_id VARCHAR(255), + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX IF NOT EXISTS idx_events_timestamp ON Events (timestamp); +CREATE INDEX IF NOT EXISTS idx_events_endpoint_type ON Events (endpoint_type); +CREATE INDEX IF NOT EXISTS idx_events_actor_id ON Events (actor_id); +CREATE INDEX IF NOT EXISTS idx_events_resource_type ON Events (resource_type); +CREATE INDEX IF NOT EXISTS idx_events_status ON Events (status); +CREATE INDEX IF NOT EXISTS idx_events_method ON Events (method); +CREATE INDEX IF NOT EXISTS idx_events_event_type ON Events (event_type); +CREATE INDEX IF NOT EXISTS idx_events_created_at ON Events (created_at); + +CREATE TABLE IF NOT EXISTS MicroserviceLogStatuses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_uuid VARCHAR(36), + log_session_id TEXT, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_log_status_microservice_uuid ON MicroserviceLogStatuses (microservice_uuid); +CREATE INDEX idx_microservice_log_status_session_id ON MicroserviceLogStatuses (session_id); + +CREATE TABLE IF NOT EXISTS MicroserviceExecSessions ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + user_connected BOOLEAN DEFAULT false, + agent_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_microservice_exec_sessions_microservice_uuid ON MicroserviceExecSessions (microservice_uuid); +CREATE INDEX idx_microservice_exec_sessions_session_id ON MicroserviceExecSessions (session_id); + +CREATE TABLE IF NOT EXISTS FogLogStatuses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + iofog_uuid VARCHAR(36), + log_session_id TEXT, + session_id TEXT UNIQUE NOT NULL, + status TEXT, + tail_config TEXT, + agent_connected BOOLEAN DEFAULT false, + user_connected BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE INDEX idx_fog_log_status_iofog_uuid ON FogLogStatuses (iofog_uuid); +CREATE INDEX idx_fog_log_status_session_id ON FogLogStatuses (session_id); + +CREATE TABLE IF NOT EXISTS RbacRoles ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + kind TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS RbacRoleRules ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + role_id INTEGER NOT NULL, + api_groups TEXT NOT NULL, + resources TEXT NOT NULL, + verbs TEXT NOT NULL, + resource_names TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (role_id) REFERENCES RbacRoles (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS RbacRoleBindings ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + kind TEXT, + role_ref TEXT NOT NULL, + subjects TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME, + role_id INTEGER +); + +CREATE TABLE IF NOT EXISTS RbacServiceAccounts ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + role_ref TEXT, + role_id INTEGER, + microservice_uuid VARCHAR(36), + application_id INTEGER, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_rbac_role_rules_role_id ON RbacRoleRules (role_id); +CREATE INDEX idx_rbac_roles_name ON RbacRoles (name); +CREATE INDEX idx_rbac_role_bindings_name ON RbacRoleBindings (name); +CREATE INDEX idx_rbac_service_accounts_name ON RbacServiceAccounts (name); + +CREATE TABLE IF NOT EXISTS RbacCacheVersion ( + id INTEGER PRIMARY KEY DEFAULT 1 CHECK (id = 1), + version INTEGER NOT NULL DEFAULT 1, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_rbac_role_bindings_role_id ON RbacRoleBindings (role_id); + +CREATE INDEX idx_rbac_service_accounts_role_id ON RbacServiceAccounts (role_id); +CREATE UNIQUE INDEX idx_rbac_service_accounts_microservice_uuid_unique ON RbacServiceAccounts (microservice_uuid) WHERE microservice_uuid IS NOT NULL; +CREATE UNIQUE INDEX idx_rbac_service_accounts_application_id_name_unique ON RbacServiceAccounts (application_id, name); + +CREATE TABLE IF NOT EXISTS ClusterControllers ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + host VARCHAR(255), + process_id INTEGER, + last_heartbeat DATETIME, + is_active BOOLEAN DEFAULT true, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_cluster_controllers_uuid ON ClusterControllers (uuid); +CREATE INDEX idx_cluster_controllers_host ON ClusterControllers (host); +CREATE INDEX idx_cluster_controllers_active ON ClusterControllers (is_active, last_heartbeat); + +CREATE TABLE IF NOT EXISTS NatsOperators ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS NatsAccounts ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + seed_secret_name TEXT NOT NULL, + is_system BOOLEAN DEFAULT false, + is_leaf_system BOOLEAN DEFAULT false, + operator_id INTEGER NOT NULL, + application_id INTEGER, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (operator_id) REFERENCES NatsOperators (id) ON DELETE CASCADE, + FOREIGN KEY (application_id) REFERENCES Applications (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsUsers ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + public_key TEXT NOT NULL, + jwt TEXT NOT NULL, + creds_secret_name TEXT NOT NULL, + is_bearer BOOLEAN DEFAULT false, + account_id INTEGER NOT NULL, + microservice_uuid VARCHAR(36), + created_at DATETIME, + updated_at DATETIME, + nats_user_rule_id INTEGER, + FOREIGN KEY (account_id) REFERENCES NatsAccounts (id) ON DELETE CASCADE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS NatsInstances ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + iofog_uuid VARCHAR(36), + is_leaf BOOLEAN DEFAULT true, + is_hub BOOLEAN DEFAULT false, + host TEXT, + server_port INTEGER, + leaf_port INTEGER, + cluster_port INTEGER, + mqtt_port INTEGER, + http_port INTEGER, + configmap_name TEXT, + jwt_dir_mount_name TEXT, + cert_secret_name TEXT, + js_storage_size TEXT, + js_memory_store_size TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (iofog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsConnections ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + source_nats INTEGER NOT NULL, + dest_nats INTEGER NOT NULL, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (source_nats) REFERENCES NatsInstances (id) ON DELETE CASCADE, + FOREIGN KEY (dest_nats) REFERENCES NatsInstances (id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS NatsReconcileTasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + reason TEXT NOT NULL, + application_id INTEGER, + account_rule_id INTEGER, + user_rule_id INTEGER, + fog_uuids TEXT, + status TEXT NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS NatsAccountRules ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + description TEXT, + info_url TEXT, + max_connections INTEGER, + max_leaf_node_connections INTEGER, + max_data BIGINT, + max_exports INTEGER, + max_imports INTEGER, + max_msg_payload INTEGER, + max_subscriptions INTEGER, + exports_allow_wildcards BOOLEAN DEFAULT true, + disallow_bearer BOOLEAN, + response_permissions TEXT, + resp_max INTEGER, + resp_ttl BIGINT, + imports TEXT, + exports TEXT, + mem_storage BIGINT, + disk_storage BIGINT, + streams INTEGER, + consumer INTEGER, + max_ack_pending INTEGER, + mem_max_stream_bytes BIGINT, + disk_max_stream_bytes BIGINT, + max_bytes_required BOOLEAN, + tiered_limits TEXT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS NatsUserRules ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT UNIQUE NOT NULL, + description TEXT, + max_subscriptions INTEGER, + max_payload INTEGER, + max_data BIGINT, + bearer_token BOOLEAN DEFAULT false, + proxy_required BOOLEAN, + allowed_connection_types TEXT, + src TEXT, + times TEXT, + times_location TEXT, + resp_max INTEGER, + resp_ttl BIGINT, + pub_allow TEXT, + pub_deny TEXT, + sub_allow TEXT, + sub_deny TEXT, + tags TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE UNIQUE INDEX idx_nats_accounts_application_id_unique ON NatsAccounts (application_id) WHERE application_id IS NOT NULL; +CREATE INDEX idx_nats_accounts_application_id ON NatsAccounts (application_id); +CREATE UNIQUE INDEX idx_nats_users_account_id_name ON NatsUsers (account_id, name); +CREATE INDEX idx_nats_users_account_id ON NatsUsers (account_id); +CREATE INDEX idx_nats_users_microservice_uuid ON NatsUsers (microservice_uuid); +CREATE INDEX idx_nats_users_nats_user_rule_id ON NatsUsers (nats_user_rule_id); +CREATE UNIQUE INDEX idx_nats_instances_iofog_uuid_unique ON NatsInstances (iofog_uuid); +CREATE INDEX idx_nats_instances_iofog_uuid ON NatsInstances (iofog_uuid); +CREATE UNIQUE INDEX idx_nats_connections_source_dest_unique ON NatsConnections (source_nats, dest_nats); +CREATE INDEX idx_nats_connections_source_nats ON NatsConnections (source_nats); +CREATE INDEX idx_nats_connections_dest_nats ON NatsConnections (dest_nats); +CREATE INDEX idx_nats_account_rules_name ON NatsAccountRules (name); +CREATE INDEX idx_nats_user_rules_name ON NatsUserRules (name); +CREATE INDEX idx_nats_reconcile_tasks_status_claimed ON NatsReconcileTasks (status, claimed_at); + +CREATE INDEX idx_applications_nats_rule_id ON Applications (nats_rule_id); +CREATE INDEX idx_microservices_nats_rule_id ON Microservices (nats_rule_id); +CREATE INDEX idx_microservices_nats_account_id ON Microservices (nats_account_id); +CREATE INDEX idx_microservices_nats_user_id ON Microservices (nats_user_id); + +CREATE TABLE IF NOT EXISTS AuthUsers ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + password_history_hashes TEXT, + must_change_password BOOLEAN DEFAULT false, + is_bootstrap BOOLEAN DEFAULT false, + failed_attempts INT DEFAULT 0, + locked_until DATETIME, + deleted_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_users_email ON AuthUsers (email); +CREATE INDEX idx_auth_users_deleted_at ON AuthUsers (deleted_at); + +CREATE TABLE IF NOT EXISTS AuthGroups ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + is_system BOOLEAN DEFAULT false, + mfa_required BOOLEAN NOT NULL DEFAULT false, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS AuthUserGroups ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + user_id VARCHAR(36) NOT NULL, + group_id INT NOT NULL, + created_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE, + FOREIGN KEY (group_id) REFERENCES AuthGroups (id) ON DELETE CASCADE, + UNIQUE (user_id, group_id) +); + +CREATE INDEX idx_auth_user_groups_user_id ON AuthUserGroups (user_id); +CREATE INDEX idx_auth_user_groups_group_id ON AuthUserGroups (group_id); + +CREATE TABLE IF NOT EXISTS AuthMfa ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + user_id VARCHAR(36) NOT NULL UNIQUE, + totp_secret_encrypted TEXT, + enabled BOOLEAN DEFAULT false, + recovery_codes_hash TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_mfa_user_id ON AuthMfa (user_id); + +CREATE TABLE IF NOT EXISTS AuthPasswordResetSessions ( + id VARCHAR(36) PRIMARY KEY NOT NULL, + user_id VARCHAR(36) NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_password_reset_sessions_user_id ON AuthPasswordResetSessions (user_id); +CREATE INDEX idx_auth_password_reset_sessions_expires_at ON AuthPasswordResetSessions (expires_at); + +CREATE TABLE IF NOT EXISTS AuthRefreshTokens ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + token_hash VARCHAR(255) NOT NULL, + user_id VARCHAR(36) NOT NULL, + family_id VARCHAR(36) NOT NULL, + expires_at DATETIME NOT NULL, + revoked BOOLEAN DEFAULT false, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (user_id) REFERENCES AuthUsers (id) ON DELETE CASCADE +); + +CREATE INDEX idx_auth_refresh_tokens_token_hash ON AuthRefreshTokens (token_hash); +CREATE INDEX idx_auth_refresh_tokens_user_id ON AuthRefreshTokens (user_id); +CREATE INDEX idx_auth_refresh_tokens_family_id ON AuthRefreshTokens (family_id); +CREATE INDEX idx_auth_refresh_tokens_expires_at ON AuthRefreshTokens (expires_at); + +CREATE TABLE IF NOT EXISTS AuthOidcKeys ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + kid VARCHAR(255) NOT NULL UNIQUE, + key_material_encrypted TEXT, + vault_ref TEXT, + active BOOLEAN DEFAULT true, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_oidc_keys_active ON AuthOidcKeys (active); + +CREATE TABLE IF NOT EXISTS AuthOidcClients ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + client_id VARCHAR(255) NOT NULL UNIQUE, + secret_ref TEXT, + client_type VARCHAR(32) NOT NULL DEFAULT 'confidential', + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS AuthOidcProviderStates ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + model VARCHAR(64) NOT NULL, + record_id VARCHAR(255) NOT NULL, + payload TEXT NOT NULL, + expires_at DATETIME, + grant_id VARCHAR(255), + uid VARCHAR(255), + user_code VARCHAR(255), + consumed BOOLEAN DEFAULT false, + consumed_at DATETIME, + created_at DATETIME, + updated_at DATETIME, + UNIQUE (model, record_id) +); + +CREATE INDEX idx_auth_oidc_provider_states_grant_id ON AuthOidcProviderStates (grant_id); +CREATE INDEX idx_auth_oidc_provider_states_uid ON AuthOidcProviderStates (uid); +CREATE INDEX idx_auth_oidc_provider_states_user_code ON AuthOidcProviderStates (user_code); +CREATE INDEX idx_auth_oidc_provider_states_expires_at ON AuthOidcProviderStates (expires_at); + +CREATE TABLE IF NOT EXISTS AuthBffSessions ( + sid VARCHAR(255) PRIMARY KEY NOT NULL, + data TEXT NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_bff_sessions_expires_at ON AuthBffSessions (expires_at); + +CREATE TABLE IF NOT EXISTS AuthInteractionStates ( + uid VARCHAR(255) PRIMARY KEY NOT NULL, + payload TEXT NOT NULL, + expires_at DATETIME NOT NULL, + created_at DATETIME, + updated_at DATETIME +); + +CREATE INDEX idx_auth_interaction_states_expires_at ON AuthInteractionStates (expires_at); + +CREATE TABLE IF NOT EXISTS AuthBootstrapMeta ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + completed_at DATETIME, + bootstrap_admin_user_id VARCHAR(36), + session_secret_ref TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (bootstrap_admin_user_id) REFERENCES AuthUsers (id) ON DELETE SET NULL +); + +CREATE TABLE IF NOT EXISTS AuthPolicy ( + id INT PRIMARY KEY NOT NULL CHECK (id = 1), + min_password_length INT DEFAULT 12, + require_uppercase BOOLEAN DEFAULT true, + require_lowercase BOOLEAN DEFAULT true, + require_digit BOOLEAN DEFAULT true, + password_max_age_days INT DEFAULT 0, + password_history_count INT DEFAULT 5, + max_failed_attempts INT DEFAULT 5, + lockout_duration_minutes INT DEFAULT 15, + access_token_ttl_seconds INT DEFAULT 900, + refresh_token_ttl_seconds INT DEFAULT 3600, + refresh_rotation BOOLEAN DEFAULT true, + max_concurrent_sessions INT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS FogPlatformSpecs ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + spec_json TEXT NOT NULL, + generation INTEGER NOT NULL DEFAULT 1, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS FogPlatformStatuses ( + fog_uuid VARCHAR(36) PRIMARY KEY NOT NULL, + observed_generation INTEGER NOT NULL DEFAULT 0, + phase VARCHAR(32) NOT NULL DEFAULT 'Pending', + last_error TEXT, + last_transition_at DATETIME, + conditions_json TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS FogPlatformReconcileTasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36) NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_generation INTEGER, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + next_attempt_at DATETIME, + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE +); + +CREATE UNIQUE INDEX idx_fog_platform_reconcile_tasks_active_fog_uuid ON FogPlatformReconcileTasks (fog_uuid) WHERE status IN ('pending', 'in_progress'); +CREATE INDEX idx_fog_platform_reconcile_tasks_status_claimed ON FogPlatformReconcileTasks (status, claimed_at); +CREATE INDEX idx_fog_platform_reconcile_tasks_next_attempt ON FogPlatformReconcileTasks (next_attempt_at); + +CREATE TABLE IF NOT EXISTS ServicePlatformReconcileTasks ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + service_name TEXT NOT NULL, + reason VARCHAR(64) NOT NULL, + spec_snapshot TEXT, + status VARCHAR(32) NOT NULL DEFAULT 'pending', + leader_uuid VARCHAR(36), + claimed_at DATETIME, + next_attempt_at DATETIME, + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE UNIQUE INDEX idx_service_platform_reconcile_tasks_active_service_name ON ServicePlatformReconcileTasks (service_name) WHERE status IN ('pending', 'in_progress'); +CREATE INDEX idx_service_platform_reconcile_tasks_status_claimed ON ServicePlatformReconcileTasks (status, claimed_at); +CREATE INDEX idx_service_platform_reconcile_tasks_next_attempt ON ServicePlatformReconcileTasks (next_attempt_at); + +CREATE TABLE IF NOT EXISTS ReconcileOutbox ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + kind VARCHAR(32) NOT NULL, + payload TEXT NOT NULL, + idempotency_key VARCHAR(255) NOT NULL UNIQUE, + created_at DATETIME, + processed_at DATETIME, + last_error TEXT +); + +CREATE INDEX idx_reconcile_outbox_unprocessed ON ReconcileOutbox (processed_at, id); + +CREATE TABLE IF NOT EXISTS HubRouterConfigLocks ( + id INTEGER PRIMARY KEY NOT NULL CHECK (id = 1), + leader_uuid VARCHAR(36), + claimed_at DATETIME, + created_at DATETIME, + updated_at DATETIME +); +CREATE INDEX IF NOT EXISTS idx_registries_url_type ON Registries (url, type); + +CREATE TABLE IF NOT EXISTS Models ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_models_registry_id ON Models (registry_id); +CREATE INDEX IF NOT EXISTS idx_models_name ON Models (name); + +CREATE TABLE IF NOT EXISTS RuntimeClasses ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS FogModels ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES Models (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_models_fog_uuid ON FogModels (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_models_model_uuid ON FogModels (model_uuid); + +CREATE TABLE IF NOT EXISTS FogRuntimeClasses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES RuntimeClasses (name) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_fog_uuid ON FogRuntimeClasses (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_runtime_class_name ON FogRuntimeClasses (runtime_class_name); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplates ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplateVariables ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_template_id INT NOT NULL, + `key` TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_template_id) REFERENCES MicroserviceTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservicetemplatevariables_microservice_template_id ON MicroserviceTemplateVariables (microservice_template_id); + +CREATE TABLE IF NOT EXISTS MicroserviceEntrypoints ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_entrypoints_microserviceUuid ON MicroserviceEntrypoints (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceDevices ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_devices_microserviceUuid ON MicroserviceDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceTmpfs ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + container_path TEXT, + size FLOAT, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_tmpfs_microserviceUuid ON MicroserviceTmpfs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceUlimits ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + soft FLOAT, + hard FLOAT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_ulimits_microserviceUuid ON MicroserviceUlimits (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceModels ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceModelItems ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Models (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON MicroserviceModelItems (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON MicroserviceModelItems (name); + +CREATE TABLE IF NOT EXISTS Knowledge ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_registry_id ON Knowledge (registry_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_name ON Knowledge (name); + +CREATE TABLE IF NOT EXISTS FogKnowledge ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES Knowledge (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_fog_uuid ON FogKnowledge (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_knowledge_uuid ON FogKnowledge (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledge ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledgeItems ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Knowledge (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_microserviceUuid ON MicroserviceKnowledgeItems (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_name ON MicroserviceKnowledgeItems (name); diff --git a/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql new file mode 100644 index 000000000..9393fb36f --- /dev/null +++ b/src/data/migrations/sqlite/db_migration_sqlite_v3.9.0.sql @@ -0,0 +1,255 @@ +DROP INDEX IF EXISTS idx_architecture_hal_catalog_item_id; +DROP INDEX IF EXISTS idx_architecture_bluetooth_catalog_item_id; +DROP INDEX IF EXISTS idx_architecture_network_catalog_item_id; + +CREATE TABLE Architectures_new ( + id INT PRIMARY KEY, + name TEXT, + image TEXT, + description TEXT, + network_catalog_item_id INT, + FOREIGN KEY (network_catalog_item_id) REFERENCES CatalogItems (id) ON DELETE CASCADE +); + +INSERT INTO Architectures_new (id, name, image, description, network_catalog_item_id) +SELECT id, name, image, description, network_catalog_item_id FROM Architectures; + +DROP TABLE Architectures; + +ALTER TABLE Architectures_new RENAME TO Architectures; + +CREATE INDEX idx_architecture_network_catalog_item_id ON Architectures (network_catalog_item_id); + +DELETE FROM Microservices WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemImages WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemInputTypes WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemOutputTypes WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue'); + +ALTER TABLE Registries ADD COLUMN type VARCHAR(16) DEFAULT 'oci'; +ALTER TABLE Registries ADD COLUMN ca TEXT; +ALTER TABLE Registries ADD COLUMN insecure BOOLEAN DEFAULT false; + +CREATE INDEX IF NOT EXISTS idx_registries_url_type ON Registries (url, type); + +CREATE TABLE IF NOT EXISTS Models ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_models_registry_id ON Models (registry_id); +CREATE INDEX IF NOT EXISTS idx_models_name ON Models (name); + +CREATE TABLE IF NOT EXISTS RuntimeClasses ( + name VARCHAR(255) PRIMARY KEY NOT NULL, + handler VARCHAR(255) NOT NULL +); + +CREATE TABLE IF NOT EXISTS FogModels ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + model_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (model_uuid) REFERENCES Models (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_models_fog_uuid ON FogModels (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_models_model_uuid ON FogModels (model_uuid); + +CREATE TABLE IF NOT EXISTS FogRuntimeClasses ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + runtime_class_name VARCHAR(255), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (runtime_class_name) REFERENCES RuntimeClasses (name) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_fog_uuid ON FogRuntimeClasses (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_runtime_classes_runtime_class_name ON FogRuntimeClasses (runtime_class_name); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplates ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) UNIQUE NOT NULL, + description TEXT DEFAULT '', + microservice_json TEXT, + created_at DATETIME, + updated_at DATETIME +); + +CREATE TABLE IF NOT EXISTS MicroserviceTemplateVariables ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + microservice_template_id INT NOT NULL, + `key` TEXT, + description TEXT DEFAULT '', + default_value TEXT, + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (microservice_template_id) REFERENCES MicroserviceTemplates (id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservicetemplatevariables_microservice_template_id ON MicroserviceTemplateVariables (microservice_template_id); + +ALTER TABLE ChangeTrackings ADD COLUMN models BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN runtime_classes BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN microservice_models BOOLEAN DEFAULT false; + +ALTER TABLE Fogs DROP COLUMN device_scan_frequency; +ALTER TABLE Fogs DROP COLUMN bluetooth; +ALTER TABLE Fogs DROP COLUMN hal; + +ALTER TABLE Fogs ADD COLUMN model_status TEXT; +ALTER TABLE Fogs ADD COLUMN runtime_classes TEXT; +ALTER TABLE Fogs ADD COLUMN available_cdi_devices TEXT; +ALTER TABLE Fogs ADD COLUMN active_models BIGINT DEFAULT 0; +ALTER TABLE Fogs ADD COLUMN model_last_update BIGINT DEFAULT 0; + +ALTER TABLE Microservices ADD COLUMN run_as_group TEXT; +ALTER TABLE Microservices ADD COLUMN read_only_root_filesystem BOOLEAN DEFAULT false; +ALTER TABLE Microservices ADD COLUMN working_dir TEXT; +ALTER TABLE Microservices ADD COLUMN cpus FLOAT; +ALTER TABLE Microservices ADD COLUMN memory_reservation FLOAT; +ALTER TABLE Microservices ADD COLUMN memory_swap FLOAT; +ALTER TABLE Microservices ADD COLUMN shm_size FLOAT; +ALTER TABLE Microservices ADD COLUMN sysctls TEXT; + +CREATE TABLE IF NOT EXISTS MicroserviceEntrypoints ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + entrypoint TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_entrypoints_microserviceUuid ON MicroserviceEntrypoints (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceDevices ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + host_path TEXT, + container_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_devices_microserviceUuid ON MicroserviceDevices (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceTmpfs ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + container_path TEXT, + size FLOAT, + mode TEXT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_microservice_tmpfs_microserviceUuid ON MicroserviceTmpfs (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceUlimits ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name TEXT NOT NULL, + soft FLOAT, + hard FLOAT, + microservice_uuid VARCHAR(36), + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_ulimits_microserviceUuid ON MicroserviceUlimits (microservice_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceModels ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceModelItems ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Models (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_microserviceUuid ON MicroserviceModelItems (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_model_items_name ON MicroserviceModelItems (name); + +ALTER TABLE MicroserviceStatuses ADD COLUMN pod_id TEXT DEFAULT ''; +ALTER TABLE MicroserviceStatuses ADD COLUMN last_error TEXT DEFAULT ''; +ALTER TABLE MicroserviceStatuses ADD COLUMN last_error_at BIGINT DEFAULT 0; +ALTER TABLE MicroserviceStatuses ADD COLUMN restart_count INTEGER DEFAULT 0; + +ALTER TABLE VolumeMappings ADD COLUMN scope TEXT NOT NULL DEFAULT 'private'; + +DROP TABLE IF EXISTS HWInfos; +DROP TABLE IF EXISTS USBInfos; + +CREATE TABLE IF NOT EXISTS Knowledge ( + uuid VARCHAR(36) PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL UNIQUE, + repo TEXT, + revision TEXT, + registry_id INT, + files TEXT, + format VARCHAR(255), + created_at DATETIME, + updated_at DATETIME, + FOREIGN KEY (registry_id) REFERENCES Registries (id) ON DELETE SET NULL +); + +CREATE INDEX IF NOT EXISTS idx_knowledge_registry_id ON Knowledge (registry_id); +CREATE INDEX IF NOT EXISTS idx_knowledge_name ON Knowledge (name); + +CREATE TABLE IF NOT EXISTS FogKnowledge ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + fog_uuid VARCHAR(36), + knowledge_uuid VARCHAR(36), + FOREIGN KEY (fog_uuid) REFERENCES Fogs (uuid) ON DELETE CASCADE, + FOREIGN KEY (knowledge_uuid) REFERENCES Knowledge (uuid) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_fog_uuid ON FogKnowledge (fog_uuid); +CREATE INDEX IF NOT EXISTS idx_fog_knowledge_knowledge_uuid ON FogKnowledge (knowledge_uuid); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledge ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + bind_path TEXT, + permissions TEXT, + microservice_uuid VARCHAR(36) NOT NULL UNIQUE, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS MicroserviceKnowledgeItems ( + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + name VARCHAR(255) NOT NULL, + microservice_uuid VARCHAR(36) NOT NULL, + FOREIGN KEY (microservice_uuid) REFERENCES Microservices (uuid) ON DELETE CASCADE, + FOREIGN KEY (name) REFERENCES Knowledge (name) ON DELETE RESTRICT, + UNIQUE (microservice_uuid, name) +); + +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_microserviceUuid ON MicroserviceKnowledgeItems (microservice_uuid); +CREATE INDEX IF NOT EXISTS idx_microservice_knowledge_items_name ON MicroserviceKnowledgeItems (name); + +ALTER TABLE ChangeTrackings ADD COLUMN knowledge BOOLEAN DEFAULT false; +ALTER TABLE ChangeTrackings ADD COLUMN microservice_knowledge BOOLEAN DEFAULT false; + +ALTER TABLE Fogs ADD COLUMN knowledge_status TEXT; +ALTER TABLE Fogs ADD COLUMN active_knowledge BIGINT DEFAULT 0; +ALTER TABLE Fogs ADD COLUMN knowledge_last_update BIGINT DEFAULT 0; + +ALTER TABLE Fogs ADD COLUMN system_cpus BIGINT; +ALTER TABLE Fogs ADD COLUMN system_total_memory BIGINT; +ALTER TABLE Fogs ADD COLUMN system_total_disk BIGINT; +ALTER TABLE Fogs ADD COLUMN system_os TEXT; +ALTER TABLE Fogs ADD COLUMN system_os_version TEXT; +ALTER TABLE Fogs ADD COLUMN system_kernel_version TEXT; diff --git a/src/data/models/architecture.js b/src/data/models/architecture.js index 8ec11768e..b3a53a56d 100644 --- a/src/data/models/architecture.js +++ b/src/data/models/architecture.js @@ -33,22 +33,6 @@ module.exports = (sequelize, DataTypes) => { }, as: 'networkCatalogItem' }) - - Architecture.belongsTo(models.CatalogItem, { - foreignKey: { - name: 'halCatalogItemId', - field: 'hal_catalog_item_id' - }, - as: 'halCatalogItem' - }) - - Architecture.belongsTo(models.CatalogItem, { - foreignKey: { - name: 'bluetoothCatalogItemId', - field: 'bluetooth_catalog_item_id' - }, - as: 'bluetoothCatalogItem' - }) } return Architecture } diff --git a/src/data/models/changetracking.js b/src/data/models/changetracking.js index e131dfc45..e1170f66b 100644 --- a/src/data/models/changetracking.js +++ b/src/data/models/changetracking.js @@ -78,6 +78,31 @@ module.exports = (sequelize, DataTypes) => { field: 'fog_logs', defaultValue: false }, + models: { + type: DataTypes.BOOLEAN, + field: 'models', + defaultValue: false + }, + runtimeClasses: { + type: DataTypes.BOOLEAN, + field: 'runtime_classes', + defaultValue: false + }, + microserviceModels: { + type: DataTypes.BOOLEAN, + field: 'microservice_models', + defaultValue: false + }, + knowledge: { + type: DataTypes.BOOLEAN, + field: 'knowledge', + defaultValue: false + }, + microserviceKnowledge: { + type: DataTypes.BOOLEAN, + field: 'microservice_knowledge', + defaultValue: false + }, lastUpdated: { type: DataTypes.STRING, field: 'last_updated', diff --git a/src/data/models/fleet-knowledge.js b/src/data/models/fleet-knowledge.js new file mode 100644 index 000000000..2e91ece29 --- /dev/null +++ b/src/data/models/fleet-knowledge.js @@ -0,0 +1,78 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const FleetKnowledge = sequelize.define('FleetKnowledge', { + uuid: { + type: DataTypes.STRING(36), + primaryKey: true, + allowNull: false, + field: 'uuid' + }, + name: { + type: DataTypes.STRING, + allowNull: false, + unique: true, + field: 'name' + }, + repo: { + type: DataTypes.TEXT, + field: 'repo' + }, + revision: { + type: DataTypes.TEXT, + field: 'revision', + defaultValue: '' + }, + files: { + type: DataTypes.TEXT, + field: 'files', + defaultValue: '[]', + get () { + const value = this.getDataValue('files') + if (value == null || value === '') { + return [] + } + try { + return JSON.parse(value) + } catch (e) { + return [] + } + }, + set (value) { + if (value == null) { + this.setDataValue('files', '[]') + return + } + this.setDataValue('files', typeof value === 'string' ? value : JSON.stringify(value)) + } + }, + format: { + type: DataTypes.STRING, + field: 'format' + } + }, { + tableName: 'Knowledge', + timestamps: true, + underscored: true + }) + + FleetKnowledge.associate = function (models) { + FleetKnowledge.belongsTo(models.Registry, { + foreignKey: { + name: 'registryId', + field: 'registry_id' + }, + as: 'registry', + onDelete: 'set null' + }) + + FleetKnowledge.belongsToMany(models.Fog, { + through: models.FogKnowledge, + as: 'fogs', + foreignKey: 'knowledge_uuid', + otherKey: 'fog_uuid' + }) + } + + return FleetKnowledge +} diff --git a/src/data/models/fleet-model.js b/src/data/models/fleet-model.js new file mode 100644 index 000000000..92bada651 --- /dev/null +++ b/src/data/models/fleet-model.js @@ -0,0 +1,78 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const FleetModel = sequelize.define('FleetModel', { + uuid: { + type: DataTypes.STRING(36), + primaryKey: true, + allowNull: false, + field: 'uuid' + }, + name: { + type: DataTypes.STRING, + allowNull: false, + unique: true, + field: 'name' + }, + repo: { + type: DataTypes.TEXT, + field: 'repo' + }, + revision: { + type: DataTypes.TEXT, + field: 'revision', + defaultValue: '' + }, + files: { + type: DataTypes.TEXT, + field: 'files', + defaultValue: '[]', + get () { + const value = this.getDataValue('files') + if (value == null || value === '') { + return [] + } + try { + return JSON.parse(value) + } catch (e) { + return [] + } + }, + set (value) { + if (value == null) { + this.setDataValue('files', '[]') + return + } + this.setDataValue('files', typeof value === 'string' ? value : JSON.stringify(value)) + } + }, + format: { + type: DataTypes.STRING, + field: 'format' + } + }, { + tableName: 'Models', + timestamps: true, + underscored: true + }) + + FleetModel.associate = function (models) { + FleetModel.belongsTo(models.Registry, { + foreignKey: { + name: 'registryId', + field: 'registry_id' + }, + as: 'registry', + onDelete: 'set null' + }) + + FleetModel.belongsToMany(models.Fog, { + through: models.FogModels, + as: 'fogs', + foreignKey: 'model_uuid', + otherKey: 'fog_uuid' + }) + } + + return FleetModel +} diff --git a/src/data/models/fog-knowledge.js b/src/data/models/fog-knowledge.js new file mode 100644 index 000000000..dbf83460e --- /dev/null +++ b/src/data/models/fog-knowledge.js @@ -0,0 +1,10 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const FogKnowledge = sequelize.define('FogKnowledge', {}, { + tableName: 'FogKnowledge', + timestamps: false, + underscored: true + }) + return FogKnowledge +} diff --git a/src/data/models/fog-models.js b/src/data/models/fog-models.js new file mode 100644 index 000000000..fdfb78d31 --- /dev/null +++ b/src/data/models/fog-models.js @@ -0,0 +1,10 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const FogModels = sequelize.define('FogModels', {}, { + tableName: 'FogModels', + timestamps: false, + underscored: true + }) + return FogModels +} diff --git a/src/data/models/fog-runtime-classes.js b/src/data/models/fog-runtime-classes.js new file mode 100644 index 000000000..1f14c40dd --- /dev/null +++ b/src/data/models/fog-runtime-classes.js @@ -0,0 +1,10 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const FogRuntimeClasses = sequelize.define('FogRuntimeClasses', {}, { + tableName: 'FogRuntimeClasses', + timestamps: false, + underscored: true + }) + return FogRuntimeClasses +} diff --git a/src/data/models/fog.js b/src/data/models/fog.js index 303d790cb..c34908676 100644 --- a/src/data/models/fog.js +++ b/src/data/models/fog.js @@ -118,6 +118,39 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.FLOAT, field: 'system_total_cpu' }, + systemCpus: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('systemCpus')) + }, + field: 'system_cpus' + }, + systemTotalMemory: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('systemTotalMemory')) + }, + field: 'system_total_memory' + }, + systemTotalDisk: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('systemTotalDisk')) + }, + field: 'system_total_disk' + }, + systemOs: { + type: DataTypes.TEXT, + field: 'system_os' + }, + systemOsVersion: { + type: DataTypes.TEXT, + field: 'system_os_version' + }, + systemKernelVersion: { + type: DataTypes.TEXT, + field: 'system_kernel_version' + }, securityStatus: { type: DataTypes.TEXT, defaultValue: 'OK', @@ -237,16 +270,6 @@ module.exports = (sequelize, DataTypes) => { defaultValue: '/var/log/iofog/', field: 'log_directory' }, - bluetoothEnabled: { - type: DataTypes.BOOLEAN, - defaultValue: false, - field: 'bluetooth' - }, - abstractedHardwareEnabled: { - type: DataTypes.BOOLEAN, - defaultValue: false, - field: 'hal' - }, logFileCount: { type: DataTypes.BIGINT, get () { @@ -279,11 +302,6 @@ module.exports = (sequelize, DataTypes) => { defaultValue: 20, field: 'change_frequency' }, - deviceScanFrequency: { - type: DataTypes.INTEGER, - defaultValue: 20, - field: 'device_scan_frequency' - }, tunnel: { type: DataTypes.TEXT, defaultValue: '', @@ -343,6 +361,52 @@ module.exports = (sequelize, DataTypes) => { }, field: 'volume_mount_last_update' }, + modelStatus: { + type: DataTypes.TEXT, + field: 'model_status' + }, + runtimeClasses: { + type: DataTypes.TEXT, + field: 'runtime_classes' + }, + availableCdiDevices: { + type: DataTypes.TEXT, + field: 'available_cdi_devices' + }, + activeModels: { + type: DataTypes.BIGINT, + defaultValue: 0, + get () { + return convertToInt(this.getDataValue('activeModels'), 0) + }, + field: 'active_models' + }, + modelLastUpdate: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('modelLastUpdate'), 0) + }, + field: 'model_last_update' + }, + knowledgeStatus: { + type: DataTypes.TEXT, + field: 'knowledge_status' + }, + activeKnowledge: { + type: DataTypes.BIGINT, + defaultValue: 0, + get () { + return convertToInt(this.getDataValue('activeKnowledge'), 0) + }, + field: 'active_knowledge' + }, + knowledgeLastUpdate: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('knowledgeLastUpdate'), 0) + }, + field: 'knowledge_last_update' + }, warningMessage: { type: DataTypes.TEXT, field: 'warning_message', @@ -409,6 +473,24 @@ module.exports = (sequelize, DataTypes) => { Fog.belongsToMany(models.Tags, { through: 'IofogTags', as: 'tags' }) Fog.belongsToMany(models.VolumeMount, { through: 'FogVolumeMounts', as: 'volumeMounts' }) + Fog.belongsToMany(models.FleetModel, { + through: models.FogModels, + as: 'models', + foreignKey: 'fog_uuid', + otherKey: 'model_uuid' + }) + Fog.belongsToMany(models.FleetKnowledge, { + through: models.FogKnowledge, + as: 'knowledge', + foreignKey: 'fog_uuid', + otherKey: 'knowledge_uuid' + }) + Fog.belongsToMany(models.RuntimeClass, { + through: models.FogRuntimeClasses, + as: 'runtimeClassLinks', + foreignKey: 'fog_uuid', + otherKey: 'runtime_class_name' + }) } return Fog diff --git a/src/data/models/hwinfo.js b/src/data/models/hwinfo.js deleted file mode 100644 index 10e73a1e3..000000000 --- a/src/data/models/hwinfo.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict' -module.exports = (sequelize, DataTypes) => { - const HWInfo = sequelize.define('HWInfo', { - id: { - type: DataTypes.INTEGER, - primaryKey: true, - autoIncrement: true, - allowNull: false, - field: 'id' - }, - info: { - type: DataTypes.TEXT, - defaultValue: ' ', - field: 'info' - } - }, { - // add the timestamp attributes (updatedAt, createdAt) - tableName: 'HWInfos', - timestamps: true, - underscored: true - }) - HWInfo.associate = function (models) { - HWInfo.belongsTo(models.Fog, { - foreignKey: { - name: 'iofogUuid', - field: 'iofog_uuid' - }, - as: 'iofog', - onDelete: 'cascade' - }) - } - return HWInfo -} diff --git a/src/data/models/index.js b/src/data/models/index.js index ccfa2a9ce..e3685109e 100644 --- a/src/data/models/index.js +++ b/src/data/models/index.js @@ -63,16 +63,13 @@ db.initDB = async (isStart) => { if (databaseProvider instanceof require('../providers/sqlite')) { const sqliteDbPath = databaseProvider.sequelize.options.storage logger.info('Running SQLite database migrations and seeders...') - await databaseProvider.runMigrationSQLite(sqliteDbPath) - await databaseProvider.runSeederSQLite(sqliteDbPath) + await databaseProvider.runVersionChainSQLite(sqliteDbPath) } else if (databaseProvider instanceof require('../providers/mysql')) { logger.info('Running MySQL database migrations and seeders...') - await databaseProvider.runMigrationMySQL(databaseProvider.sequelize) - await databaseProvider.runSeederMySQL(databaseProvider.sequelize) + await databaseProvider.runVersionChainMySQL(databaseProvider.sequelize) } else if (databaseProvider instanceof require('../providers/postgres')) { logger.info('Running PostgreSQL database migrations and seeders...') - await databaseProvider.runMigrationPostgres(databaseProvider.sequelize) - await databaseProvider.runSeederPostgres(databaseProvider.sequelize) + await databaseProvider.runVersionChainPostgres(databaseProvider.sequelize) } // Initialize RBAC cache version if it doesn't exist diff --git a/src/data/models/microservice-device.js b/src/data/models/microservice-device.js new file mode 100644 index 000000000..56ea88a48 --- /dev/null +++ b/src/data/models/microservice-device.js @@ -0,0 +1,39 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceDevice = sequelize.define('MicroserviceDevice', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + hostPath: { + type: DataTypes.TEXT, + field: 'host_path' + }, + containerPath: { + type: DataTypes.TEXT, + field: 'container_path' + }, + permissions: { + type: DataTypes.TEXT, + field: 'permissions' + } + }, { + tableName: 'MicroserviceDevices', + timestamps: false, + underscored: true + }) + MicroserviceDevice.associate = function (models) { + MicroserviceDevice.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceDevice +} diff --git a/src/data/models/microservice-entrypoint.js b/src/data/models/microservice-entrypoint.js new file mode 100644 index 000000000..9799efb1f --- /dev/null +++ b/src/data/models/microservice-entrypoint.js @@ -0,0 +1,31 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceEntrypoint = sequelize.define('MicroserviceEntrypoint', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + entrypoint: { + type: DataTypes.TEXT, + field: 'entrypoint' + } + }, { + tableName: 'MicroserviceEntrypoints', + timestamps: false, + underscored: true + }) + MicroserviceEntrypoint.associate = function (models) { + MicroserviceEntrypoint.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceEntrypoint +} diff --git a/src/data/models/microservice-knowledge-item.js b/src/data/models/microservice-knowledge-item.js new file mode 100644 index 000000000..53c29de7d --- /dev/null +++ b/src/data/models/microservice-knowledge-item.js @@ -0,0 +1,41 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceKnowledgeItem = sequelize.define('MicroserviceKnowledgeItem', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + name: { + type: DataTypes.STRING, + allowNull: false, + field: 'name' + } + }, { + tableName: 'MicroserviceKnowledgeItems', + timestamps: false, + underscored: true + }) + MicroserviceKnowledgeItem.associate = function (models) { + MicroserviceKnowledgeItem.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + MicroserviceKnowledgeItem.belongsTo(models.FleetKnowledge, { + foreignKey: { + name: 'name', + field: 'name' + }, + targetKey: 'name', + as: 'knowledge', + onDelete: 'restrict' + }) + } + return MicroserviceKnowledgeItem +} diff --git a/src/data/models/microservice-knowledge.js b/src/data/models/microservice-knowledge.js new file mode 100644 index 000000000..22d438cb9 --- /dev/null +++ b/src/data/models/microservice-knowledge.js @@ -0,0 +1,35 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceKnowledge = sequelize.define('MicroserviceKnowledge', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + bindPath: { + type: DataTypes.TEXT, + field: 'bind_path' + }, + permissions: { + type: DataTypes.TEXT, + field: 'permissions' + } + }, { + tableName: 'MicroserviceKnowledge', + timestamps: false, + underscored: true + }) + MicroserviceKnowledge.associate = function (models) { + MicroserviceKnowledge.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceKnowledge +} diff --git a/src/data/models/microservice-model-item.js b/src/data/models/microservice-model-item.js new file mode 100644 index 000000000..018e1c5a9 --- /dev/null +++ b/src/data/models/microservice-model-item.js @@ -0,0 +1,41 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceModelItem = sequelize.define('MicroserviceModelItem', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + name: { + type: DataTypes.STRING, + allowNull: false, + field: 'name' + } + }, { + tableName: 'MicroserviceModelItems', + timestamps: false, + underscored: true + }) + MicroserviceModelItem.associate = function (models) { + MicroserviceModelItem.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + MicroserviceModelItem.belongsTo(models.FleetModel, { + foreignKey: { + name: 'name', + field: 'name' + }, + targetKey: 'name', + as: 'model', + onDelete: 'restrict' + }) + } + return MicroserviceModelItem +} diff --git a/src/data/models/microservice-model.js b/src/data/models/microservice-model.js new file mode 100644 index 000000000..701ba057d --- /dev/null +++ b/src/data/models/microservice-model.js @@ -0,0 +1,35 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceModel = sequelize.define('MicroserviceModel', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + bindPath: { + type: DataTypes.TEXT, + field: 'bind_path' + }, + permissions: { + type: DataTypes.TEXT, + field: 'permissions' + } + }, { + tableName: 'MicroserviceModels', + timestamps: false, + underscored: true + }) + MicroserviceModel.associate = function (models) { + MicroserviceModel.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceModel +} diff --git a/src/data/models/microservice-template-variable.js b/src/data/models/microservice-template-variable.js new file mode 100644 index 000000000..d317fb24c --- /dev/null +++ b/src/data/models/microservice-template-variable.js @@ -0,0 +1,43 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const MicroserviceTemplateVariable = sequelize.define('MicroserviceTemplateVariable', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + key: { + type: DataTypes.TEXT, + field: 'key' + }, + description: { + type: DataTypes.TEXT, + field: 'description', + defaultValue: '' + }, + defaultValue: { + type: DataTypes.TEXT, + field: 'default_value' + } + }, { + tableName: 'MicroserviceTemplateVariables', + timestamps: true, + underscored: true + }) + + MicroserviceTemplateVariable.associate = function (models) { + MicroserviceTemplateVariable.belongsTo(models.MicroserviceTemplate, { + foreignKey: { + name: 'microserviceTemplateId', + field: 'microservice_template_id' + }, + as: 'microserviceTemplate', + onDelete: 'cascade' + }) + } + + return MicroserviceTemplateVariable +} diff --git a/src/data/models/microservice-template.js b/src/data/models/microservice-template.js new file mode 100644 index 000000000..f53db1fa5 --- /dev/null +++ b/src/data/models/microservice-template.js @@ -0,0 +1,45 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const MicroserviceTemplate = sequelize.define('MicroserviceTemplate', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + name: { + type: DataTypes.STRING, + allowNull: false, + unique: true, + field: 'name' + }, + description: { + type: DataTypes.TEXT, + field: 'description', + defaultValue: '' + }, + microserviceJSON: { + type: DataTypes.TEXT, + field: 'microservice_json', + defaultValue: '{}' + } + }, { + tableName: 'MicroserviceTemplates', + timestamps: true, + underscored: true + }) + + MicroserviceTemplate.associate = function (models) { + MicroserviceTemplate.hasMany(models.MicroserviceTemplateVariable, { + foreignKey: { + name: 'microserviceTemplateId', + field: 'microservice_template_id' + }, + as: 'variables' + }) + } + + return MicroserviceTemplate +} diff --git a/src/data/models/microservice-tmpfs.js b/src/data/models/microservice-tmpfs.js new file mode 100644 index 000000000..0553f6a18 --- /dev/null +++ b/src/data/models/microservice-tmpfs.js @@ -0,0 +1,39 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceTmpfs = sequelize.define('MicroserviceTmpfs', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + containerPath: { + type: DataTypes.TEXT, + field: 'container_path' + }, + size: { + type: DataTypes.FLOAT, + field: 'size' + }, + mode: { + type: DataTypes.TEXT, + field: 'mode' + } + }, { + tableName: 'MicroserviceTmpfs', + timestamps: false, + underscored: true + }) + MicroserviceTmpfs.associate = function (models) { + MicroserviceTmpfs.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceTmpfs +} diff --git a/src/data/models/microservice-ulimit.js b/src/data/models/microservice-ulimit.js new file mode 100644 index 000000000..3f8b5c857 --- /dev/null +++ b/src/data/models/microservice-ulimit.js @@ -0,0 +1,40 @@ +'use strict' +module.exports = (sequelize, DataTypes) => { + const MicroserviceUlimit = sequelize.define('MicroserviceUlimit', { + id: { + type: DataTypes.INTEGER, + primaryKey: true, + autoIncrement: true, + allowNull: false, + field: 'id' + }, + name: { + type: DataTypes.TEXT, + field: 'name', + allowNull: false + }, + soft: { + type: DataTypes.FLOAT, + field: 'soft' + }, + hard: { + type: DataTypes.FLOAT, + field: 'hard' + } + }, { + tableName: 'MicroserviceUlimits', + timestamps: false, + underscored: true + }) + MicroserviceUlimit.associate = function (models) { + MicroserviceUlimit.belongsTo(models.Microservice, { + foreignKey: { + name: 'microserviceUuid', + field: 'microservice_uuid' + }, + as: 'microservice', + onDelete: 'cascade' + }) + } + return MicroserviceUlimit +} diff --git a/src/data/models/microservice.js b/src/data/models/microservice.js index 017a3821c..4ed5a1b39 100644 --- a/src/data/models/microservice.js +++ b/src/data/models/microservice.js @@ -138,6 +138,41 @@ module.exports = (sequelize, DataTypes) => { type: DataTypes.BOOLEAN, field: 'is_controller', defaultValue: false + }, + runAsGroup: { + type: DataTypes.TEXT, + field: 'run_as_group', + defaultValue: '' + }, + readOnlyRootFilesystem: { + type: DataTypes.BOOLEAN, + field: 'read_only_root_filesystem', + defaultValue: false + }, + workingDir: { + type: DataTypes.TEXT, + field: 'working_dir', + defaultValue: '' + }, + cpus: { + type: DataTypes.FLOAT, + field: 'cpus' + }, + memoryReservation: { + type: DataTypes.FLOAT, + field: 'memory_reservation' + }, + memorySwap: { + type: DataTypes.FLOAT, + field: 'memory_swap' + }, + shmSize: { + type: DataTypes.FLOAT, + field: 'shm_size' + }, + sysctls: { + type: DataTypes.TEXT, + field: 'sysctls' } }, { tableName: 'Microservices', @@ -227,6 +262,46 @@ module.exports = (sequelize, DataTypes) => { as: 'cmd' }) + Microservice.hasMany(models.MicroserviceEntrypoint, { + foreignKey: 'microservice_uuid', + as: 'entrypoint' + }) + + Microservice.hasMany(models.MicroserviceDevice, { + foreignKey: 'microservice_uuid', + as: 'devices' + }) + + Microservice.hasMany(models.MicroserviceTmpfs, { + foreignKey: 'microservice_uuid', + as: 'tmpfs' + }) + + Microservice.hasMany(models.MicroserviceUlimit, { + foreignKey: 'microservice_uuid', + as: 'ulimits' + }) + + Microservice.hasOne(models.MicroserviceModel, { + foreignKey: 'microservice_uuid', + as: 'microserviceModel' + }) + + Microservice.hasMany(models.MicroserviceModelItem, { + foreignKey: 'microservice_uuid', + as: 'modelItems' + }) + + Microservice.hasOne(models.MicroserviceKnowledge, { + foreignKey: 'microservice_uuid', + as: 'microserviceKnowledge' + }) + + Microservice.hasMany(models.MicroserviceKnowledgeItem, { + foreignKey: 'microservice_uuid', + as: 'knowledgeItems' + }) + Microservice.hasMany(models.MicroserviceCdiDev, { foreignKey: 'microservice_uuid', as: 'cdiDevices' diff --git a/src/data/models/microservicestatus.js b/src/data/models/microservicestatus.js index b9f7a3cbe..5d1797051 100644 --- a/src/data/models/microservicestatus.js +++ b/src/data/models/microservicestatus.js @@ -51,6 +51,11 @@ module.exports = (sequelize, DataTypes) => { defaultValue: '', field: 'container_id' }, + podId: { + type: DataTypes.TEXT, + defaultValue: '', + field: 'pod_id' + }, percentage: { type: DataTypes.FLOAT, defaultValue: 0.00, @@ -61,6 +66,24 @@ module.exports = (sequelize, DataTypes) => { defaultValue: '', field: 'error_message' }, + lastError: { + type: DataTypes.TEXT, + defaultValue: '', + field: 'last_error' + }, + lastErrorAt: { + type: DataTypes.BIGINT, + get () { + return convertToInt(this.getDataValue('lastErrorAt')) + }, + defaultValue: 0, + field: 'last_error_at' + }, + restartCount: { + type: DataTypes.INTEGER, + defaultValue: 0, + field: 'restart_count' + }, ipAddress: { type: DataTypes.TEXT, defaultValue: '', diff --git a/src/data/models/registry.js b/src/data/models/registry.js index 5a721d2c8..47d25c170 100644 --- a/src/data/models/registry.js +++ b/src/data/models/registry.js @@ -62,6 +62,21 @@ module.exports = (sequelize, DataTypes) => { userEmail: { type: DataTypes.TEXT, field: 'user_email' + }, + type: { + type: DataTypes.STRING(16), + field: 'type', + defaultValue: 'oci' + }, + ca: { + type: DataTypes.TEXT, + field: 'ca', + allowNull: true + }, + insecure: { + type: DataTypes.BOOLEAN, + field: 'insecure', + defaultValue: false } }, { tableName: 'Registries', diff --git a/src/data/models/runtime-class.js b/src/data/models/runtime-class.js new file mode 100644 index 000000000..94a40c299 --- /dev/null +++ b/src/data/models/runtime-class.js @@ -0,0 +1,32 @@ +'use strict' + +module.exports = (sequelize, DataTypes) => { + const RuntimeClass = sequelize.define('RuntimeClass', { + name: { + type: DataTypes.STRING, + primaryKey: true, + allowNull: false, + field: 'name' + }, + handler: { + type: DataTypes.STRING, + allowNull: false, + field: 'handler' + } + }, { + tableName: 'RuntimeClasses', + timestamps: false, + underscored: true + }) + + RuntimeClass.associate = function (models) { + RuntimeClass.belongsToMany(models.Fog, { + through: models.FogRuntimeClasses, + as: 'fogs', + foreignKey: 'runtime_class_name', + otherKey: 'fog_uuid' + }) + } + + return RuntimeClass +} diff --git a/src/data/models/usbinfo.js b/src/data/models/usbinfo.js deleted file mode 100644 index 6458bc907..000000000 --- a/src/data/models/usbinfo.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict' -module.exports = (sequelize, DataTypes) => { - const USBInfo = sequelize.define('USBInfo', { - id: { - type: DataTypes.INTEGER, - primaryKey: true, - autoIncrement: true, - allowNull: false, - field: 'id' - }, - info: { - type: DataTypes.TEXT, - defaultValue: ' ', - field: 'info' - } - }, { - tableName: 'USBInfos', - // add the timestamp attributes (updatedAt, createdAt) - timestamps: true, - underscored: true - }) - USBInfo.associate = function (models) { - USBInfo.belongsTo(models.Fog, { - foreignKey: { - name: 'iofogUuid', - field: 'iofog_uuid' - }, - as: 'iofog', - onDelete: 'cascade' - }) - } - return USBInfo -} diff --git a/src/data/models/volumemapping.js b/src/data/models/volumemapping.js index a9ddce3b9..4ef92d371 100644 --- a/src/data/models/volumemapping.js +++ b/src/data/models/volumemapping.js @@ -23,6 +23,12 @@ module.exports = (sequelize, DataTypes) => { type: { type: DataTypes.TEXT, field: 'type' + }, + scope: { + type: DataTypes.ENUM('private', 'shared'), + allowNull: false, + defaultValue: 'private', + field: 'scope' } }, { tableName: 'VolumeMappings', diff --git a/src/data/providers/database-provider.js b/src/data/providers/database-provider.js index 331f60eaa..eaf03fc28 100644 --- a/src/data/providers/database-provider.js +++ b/src/data/providers/database-provider.js @@ -21,6 +21,133 @@ function sqliteClose (db) { }) } +const SCHEMA_VERSIONS = ['3.8.0', '3.9.0'] +const BASELINE_VERSION = '3.9.0' + +function compareVersions (left, right) { + const leftParts = String(left || '0').split('.').map((part) => parseInt(part, 10) || 0) + const rightParts = String(right || '0').split('.').map((part) => parseInt(part, 10) || 0) + const length = Math.max(leftParts.length, rightParts.length) + for (let i = 0; i < length; i++) { + const delta = (leftParts[i] || 0) - (rightParts[i] || 0) + if (delta !== 0) { + return delta + } + } + return 0 +} + +function isVersionLessThan (current, target) { + return current == null || compareVersions(current, target) < 0 +} + +function versionsToApply (untilVersion) { + if (!untilVersion) { + return SCHEMA_VERSIONS.slice() + } + return SCHEMA_VERSIONS.filter((version) => compareVersions(version, untilVersion) <= 0) +} + +function shouldApplyBaseline (currentSchemaVersion, untilVersion) { + return currentSchemaVersion == null && untilVersion == null +} + +function splitSqlStatements (sql) { + return sql.split(';').map((query) => query.trim()).filter(Boolean) +} + +function isTransactionControl (query) { + const normalized = query.replace(/;$/, '').trim().toUpperCase() + return normalized === 'START TRANSACTION' || + normalized === 'BEGIN' || + normalized === 'BEGIN TRANSACTION' || + normalized === 'COMMIT' || + normalized === 'ROLLBACK' +} + +function migrationSqlPath (provider, version) { + const relative = { + sqlite: `sqlite/db_migration_sqlite_v${version}.sql`, + mysql: `mysql/db_migration_mysql_v${version}.sql`, + postgres: `postgres/db_migration_pg_v${version}.sql` + }[provider] + return path.resolve(__dirname, '../migrations', relative) +} + +function seederSqlPath (provider, version) { + const relative = { + sqlite: `sqlite/db_seeder_sqlite_v${version}.sql`, + mysql: `mysql/db_seeder_mysql_v${version}.sql`, + postgres: `postgres/db_seeder_pg_v${version}.sql` + }[provider] + return path.resolve(__dirname, '../seeders', relative) +} + +function baselineMigrationSqlPath (provider) { + const relative = { + sqlite: 'sqlite/db_migration_sqlite_baseline_v3.9.0.sql', + mysql: 'mysql/db_migration_mysql_baseline_v3.9.0.sql', + postgres: 'postgres/db_migration_pg_baseline_v3.9.0.sql' + }[provider] + return path.resolve(__dirname, '../migrations', relative) +} + +function baselineSeederSqlPath (provider) { + const relative = { + sqlite: 'sqlite/db_seeder_sqlite_baseline_v3.9.0.sql', + mysql: 'mysql/db_seeder_mysql_baseline_v3.9.0.sql', + postgres: 'postgres/db_seeder_pg_baseline_v3.9.0.sql' + }[provider] + return path.resolve(__dirname, '../seeders', relative) +} + +function readSqlFile (filePath) { + if (!fs.existsSync(filePath)) { + logger.error(`SQL file not found: ${filePath}`) + throw new Error('SQL file not found') + } + return fs.readFileSync(filePath).toString() +} + +function isIgnorableSqliteError (err) { + const message = (err && err.message) || '' + return message.includes('already exists') || message.includes('duplicate') +} + +function isIgnorableMysqlError (err) { + const errorToCheck = (err && err.parent) || err || {} + return errorToCheck.code === 'ER_TABLE_EXISTS_ERROR' || + errorToCheck.code === 'ER_DUP_FIELDNAME' || + errorToCheck.code === 'ER_DUP_KEYNAME' || + errorToCheck.code === 'ER_BLOB_KEY_WITHOUT_LENGTH' || + errorToCheck.code === 'ER_CANT_DROP_FIELD_OR_KEY' || + errorToCheck.code === 'ER_DUP_ENTRY' || + errorToCheck.code === 'ER_DUP_KEY' || + errorToCheck.code === 'duplicate_key' || + errorToCheck.code === 'already_exists' || + errorToCheck.errno === 1091 || + errorToCheck.errno === 1061 || + errorToCheck.errno === 1170 +} + +function isIgnorablePostgresError (err) { + const errorToCheck = (err && err.parent) || err || {} + const message = errorToCheck.message || '' + return errorToCheck.code === '42P07' || + errorToCheck.code === '42701' || + errorToCheck.code === '42P06' || + errorToCheck.code === '23505' || + errorToCheck.code === '23503' || + errorToCheck.code === '42P01' || + errorToCheck.code === '42703' || + errorToCheck.code === '42P16' || + errorToCheck.code === '42P17' || + errorToCheck.code === '42P18' || + message.includes('already exists') || + message.includes('duplicate key') || + message.includes('does not exist') +} + class DatabaseProvider { constructor () { this.basename = path.basename(__filename) @@ -271,375 +398,418 @@ class DatabaseProvider { } } - // SQLite migration — greenfield v3.8.0 (see src/data/migrations/README.md) - async runMigrationSQLite (dbName) { - const migrationSqlPath = path.resolve(__dirname, '../migrations/sqlite/db_migration_sqlite_v3.8.0.sql') - const migrationVersion = '3.8.0' - - if (!fs.existsSync(migrationSqlPath)) { - logger.error(`Migration file not found: ${migrationSqlPath}`) - throw new Error('Migration file not found') + async applySqliteStatements (db, sql, { ignoreDuplicate } = { ignoreDuplicate: true }) { + for (const raw of splitSqlStatements(sql)) { + if (isTransactionControl(raw)) { + continue + } + const query = raw + ';' + try { + await sqliteRun(db, query) + } catch (err) { + if (ignoreDuplicate && isIgnorableSqliteError(err)) { + logger.warn(`Ignored error: ${err.message}`) + } else { + throw err + } + } } + } - const migrationSql = fs.readFileSync(migrationSqlPath).toString() - const dataArr = migrationSql.split(';') + async applySqliteMigrationIfNeeded (db, version) { + const currentVersion = await this.checkMigrationVersion(db, 'sqlite') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`SQLite schema ${currentVersion || 'none'} already includes ${version}, skipping migration`) + return + } - const db = new sqlite3.Database(dbName, (err) => { - if (err) { - logger.error(err.message) - throw err + const sql = readSqlFile(migrationSqlPath('sqlite', version)) + await sqliteRun(db, 'BEGIN TRANSACTION') + try { + await this.applySqliteStatements(db, sql) + await this.updateMigrationVersion(db, version, 'sqlite') + await sqliteRun(db, 'COMMIT') + logger.info(`SQLite migration ${version} completed successfully.`) + } catch (err) { + try { + await sqliteRun(db, 'ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } - logger.info('Connected to the SQLite database for migration.') - }) + logger.error(`SQLite migration ${version} failed:`, err) + throw err + } + } - try { - await this.createSchemaVersionTable(db, 'sqlite') - const currentVersion = await this.checkMigrationVersion(db, 'sqlite') + async applySqliteSeederIfNeeded (db, version) { + const currentVersion = await this.checkSeederVersion(db, 'sqlite') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`SQLite seeder ${currentVersion || 'none'} already includes ${version}, skipping seeder`) + return + } - if (currentVersion === migrationVersion) { - logger.info('Migration already up to date, skipping...') - return + const sql = readSqlFile(seederSqlPath('sqlite', version)) + await sqliteRun(db, 'BEGIN TRANSACTION') + try { + await this.applySqliteStatements(db, sql) + await this.updateSeederVersion(db, version, 'sqlite') + await sqliteRun(db, 'COMMIT') + logger.info(`SQLite seeder ${version} completed successfully.`) + } catch (err) { + try { + await sqliteRun(db, 'ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`SQLite seeder ${version} failed:`, err) + throw err + } + } - await sqliteRun(db, 'PRAGMA foreign_keys=OFF') - await sqliteRun(db, 'BEGIN TRANSACTION') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await sqliteRun(db, query) - } catch (err) { - if (err.message.includes('already exists') || err.message.includes('duplicate')) { - logger.warn(`Ignored error: ${err.message}`) - } else { - throw err - } - } - } + async applySqliteBaseline (db) { + logger.info(`Applying empty-database baseline schema ${BASELINE_VERSION}`) + const migrationSql = readSqlFile(baselineMigrationSqlPath('sqlite')) + await sqliteRun(db, 'BEGIN TRANSACTION') + try { + await this.applySqliteStatements(db, migrationSql) + await this.updateMigrationVersion(db, BASELINE_VERSION, 'sqlite') + await sqliteRun(db, 'COMMIT') + logger.info(`SQLite baseline migration ${BASELINE_VERSION} completed successfully.`) + } catch (err) { + try { + await sqliteRun(db, 'ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`SQLite baseline migration ${BASELINE_VERSION} failed:`, err) + throw err + } - await this.updateMigrationVersion(db, migrationVersion, 'sqlite') + logger.info(`Applying empty-database baseline seed ${BASELINE_VERSION}`) + const seederSql = readSqlFile(baselineSeederSqlPath('sqlite')) + await sqliteRun(db, 'BEGIN TRANSACTION') + try { + await this.applySqliteStatements(db, seederSql) + await this.updateSeederVersion(db, BASELINE_VERSION, 'sqlite') await sqliteRun(db, 'COMMIT') - logger.info('Migration completed successfully.') + logger.info(`SQLite baseline seeder ${BASELINE_VERSION} completed successfully.`) } catch (err) { try { await sqliteRun(db, 'ROLLBACK') } catch (rollbackErr) { // No active transaction to roll back. } - logger.error('Migration failed:', err) + logger.error(`SQLite baseline seeder ${BASELINE_VERSION} failed:`, err) throw err + } + } + + async runVersionChainSQLite (dbName, options = {}) { + const db = new sqlite3.Database(dbName, (err) => { + if (err) { + logger.error(err.message) + throw err + } + logger.info('Connected to the SQLite database for schema updates.') + }) + + try { + await this.createSchemaVersionTable(db, 'sqlite') + await sqliteRun(db, 'PRAGMA foreign_keys=OFF') + const currentVersion = await this.checkMigrationVersion(db, 'sqlite') + if (shouldApplyBaseline(currentVersion, options.untilVersion)) { + await this.applySqliteBaseline(db) + return + } + for (const version of versionsToApply(options.untilVersion)) { + await this.applySqliteMigrationIfNeeded(db, version) + await this.applySqliteSeederIfNeeded(db, version) + } } finally { try { await sqliteClose(db) - logger.info('Database connection closed after migration.') + logger.info('Database connection closed after schema updates.') } catch (closeErr) { logger.error('Error closing database connection:', closeErr.message) } } } - // MySQL migration — greenfield v3.8.0 (see src/data/migrations/README.md) - async runMigrationMySQL (db) { - const migrationSqlPath = path.resolve(__dirname, '../migrations/mysql/db_migration_mysql_v3.8.0.sql') - const migrationVersion = '3.8.0' - - if (!fs.existsSync(migrationSqlPath)) { - logger.error(`Migration file not found: ${migrationSqlPath}`) - throw new Error('Migration file not found') - } - - const migrationSql = fs.readFileSync(migrationSqlPath).toString() - const dataArr = migrationSql.split(';') - - try { - await this.createSchemaVersionTable(db, 'mysql') - const currentVersion = await this.checkMigrationVersion(db, 'mysql') - - if (currentVersion === migrationVersion) { - logger.info('Migration already up to date, skipping...') - return + async applyMysqlStatements (db, sql) { + for (const raw of splitSqlStatements(sql)) { + if (isTransactionControl(raw)) { + continue } - - await db.query('START TRANSACTION') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await db.query(query) - } catch (err) { - const errorToCheck = err.parent || err - if (errorToCheck.code === 'ER_TABLE_EXISTS_ERROR' || - errorToCheck.code === 'ER_DUP_FIELDNAME' || - errorToCheck.code === 'ER_DUP_KEYNAME' || - errorToCheck.code === 'ER_BLOB_KEY_WITHOUT_LENGTH' || - errorToCheck.code === 'ER_CANT_DROP_FIELD_OR_KEY' || - errorToCheck.code === 'duplicate_key' || - errorToCheck.code === 'already_exists' || - errorToCheck.errno === 1091 || - errorToCheck.errno === 1061 || - errorToCheck.errno === 1170) { - logger.warn(`Ignored MySQL error: ${errorToCheck.message}`) - } else { - await db.query('ROLLBACK') - throw err - } - } + const query = raw + ';' + try { + await db.query(query) + } catch (err) { + const errorToCheck = err.parent || err + if (errorToCheck.code === '25P02') { + throw err + } + if (isIgnorableMysqlError(err)) { + logger.warn(`Ignored MySQL error: ${errorToCheck.message}`) + } else { + throw err } } + } + } - await this.updateMigrationVersion(db, migrationVersion, 'mysql') + async applyMysqlMigrationIfNeeded (db, version) { + const currentVersion = await this.checkMigrationVersion(db, 'mysql') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`MySQL schema ${currentVersion || 'none'} already includes ${version}, skipping migration`) + return + } + + const sql = readSqlFile(migrationSqlPath('mysql', version)) + await db.query('START TRANSACTION') + try { + await this.applyMysqlStatements(db, sql) + await this.updateMigrationVersion(db, version, 'mysql') await db.query('COMMIT') - logger.info('Migration completed successfully.') + logger.info(`MySQL migration ${version} completed successfully.`) } catch (err) { - await db.query('ROLLBACK') - logger.error('Migration failed:', err) + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. + } + logger.error(`MySQL migration ${version} failed:`, err) throw err } } - // PostgreSQL migration — greenfield v3.8.0 (see src/data/migrations/README.md) - async runMigrationPostgres (db) { - const migrationSqlPath = path.resolve(__dirname, '../migrations/postgres/db_migration_pg_v3.8.0.sql') - const migrationVersion = '3.8.0' - - if (!fs.existsSync(migrationSqlPath)) { - logger.error(`Migration file not found: ${migrationSqlPath}`) - throw new Error('Migration file not found') + async applyMysqlSeederIfNeeded (db, version) { + const currentVersion = await this.checkSeederVersion(db, 'mysql') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`MySQL seeder ${currentVersion || 'none'} already includes ${version}, skipping seeder`) + return } - const migrationSql = fs.readFileSync(migrationSqlPath).toString() - const dataArr = migrationSql.split(';') - + const sql = readSqlFile(seederSqlPath('mysql', version)) + await db.query('START TRANSACTION') try { - await this.createSchemaVersionTable(db, 'postgres') - const currentVersion = await this.checkMigrationVersion(db, 'postgres') - - if (currentVersion === migrationVersion) { - logger.info('Migration already up to date, skipping...') - return + await this.applyMysqlStatements(db, sql) + await this.updateSeederVersion(db, version, 'mysql') + await db.query('COMMIT') + logger.info(`MySQL seeder ${version} completed successfully.`) + } catch (err) { + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`MySQL seeder ${version} failed:`, err) + throw err + } + } - await db.query('BEGIN') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await db.query(query) - } catch (err) { - const errorToCheck = err.parent || err - - if (errorToCheck.code === '25P02') { - logger.warn('Transaction aborted, rolling back and starting new transaction...') - await db.query('ROLLBACK') - await db.query('BEGIN') - continue - } - - if (errorToCheck.code === '42P07' || - errorToCheck.code === '42701' || - errorToCheck.code === '42P06' || - errorToCheck.code === '23505' || - errorToCheck.code === '23503' || - errorToCheck.code === '42P01' || - errorToCheck.code === '42703' || - errorToCheck.code === '42P16' || - errorToCheck.code === '42P17' || - errorToCheck.code === '42P18' || - (errorToCheck.message && ( - errorToCheck.message.includes('already exists') || - errorToCheck.message.includes('duplicate key') || - errorToCheck.message.includes('relation') - ))) { - logger.warn(`Ignored PostgreSQL error: ${errorToCheck.message}`) - } else { - await db.query('ROLLBACK') - throw err - } - } - } + async applyMysqlBaseline (db) { + logger.info(`Applying empty-database baseline schema ${BASELINE_VERSION}`) + const migrationSql = readSqlFile(baselineMigrationSqlPath('mysql')) + await db.query('START TRANSACTION') + try { + await this.applyMysqlStatements(db, migrationSql) + await this.updateMigrationVersion(db, BASELINE_VERSION, 'mysql') + await db.query('COMMIT') + logger.info(`MySQL baseline migration ${BASELINE_VERSION} completed successfully.`) + } catch (err) { + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`MySQL baseline migration ${BASELINE_VERSION} failed:`, err) + throw err + } - await this.updateMigrationVersion(db, migrationVersion, 'postgres') + logger.info(`Applying empty-database baseline seed ${BASELINE_VERSION}`) + const seederSql = readSqlFile(baselineSeederSqlPath('mysql')) + await db.query('START TRANSACTION') + try { + await this.applyMysqlStatements(db, seederSql) + await this.updateSeederVersion(db, BASELINE_VERSION, 'mysql') await db.query('COMMIT') - logger.info('Migration completed successfully.') + logger.info(`MySQL baseline seeder ${BASELINE_VERSION} completed successfully.`) } catch (err) { - await db.query('ROLLBACK') - logger.error('Migration failed:', err) + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. + } + logger.error(`MySQL baseline seeder ${BASELINE_VERSION} failed:`, err) throw err } } - // SQLite seeder — greenfield v3.8.0 (see src/data/migrations/README.md) - async runSeederSQLite (dbName) { - const seederSqlPath = path.resolve(__dirname, '../seeders/sqlite/db_seeder_sqlite_v3.8.0.sql') - const seederVersion = '3.8.0' - - if (!fs.existsSync(seederSqlPath)) { - logger.error(`Seeder file not found: ${seederSqlPath}`) - throw new Error('Seeder file not found') + async runVersionChainMySQL (db, options = {}) { + await this.createSchemaVersionTable(db, 'mysql') + const currentVersion = await this.checkMigrationVersion(db, 'mysql') + if (shouldApplyBaseline(currentVersion, options.untilVersion)) { + await this.applyMysqlBaseline(db) + return } + for (const version of versionsToApply(options.untilVersion)) { + await this.applyMysqlMigrationIfNeeded(db, version) + await this.applyMysqlSeederIfNeeded(db, version) + } + } - const seederSql = fs.readFileSync(seederSqlPath).toString() - const dataArr = seederSql.split(';') - - const db = new sqlite3.Database(dbName, (err) => { - if (err) { - logger.error(err.message) - throw err - } - logger.info('Connected to the SQLite database for seeding.') - }) - - try { - const currentVersion = await this.checkSeederVersion(db, 'sqlite') - - if (currentVersion === seederVersion) { - logger.info('Seeder already up to date, skipping...') - return + async applyPostgresStatements (db, sql) { + for (const raw of splitSqlStatements(sql)) { + if (isTransactionControl(raw)) { + continue } - - await sqliteRun(db, 'PRAGMA foreign_keys=OFF') - await sqliteRun(db, 'BEGIN TRANSACTION') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await sqliteRun(db, query) - } catch (err) { - if (err.message.includes('already exists') || err.message.includes('duplicate')) { - logger.warn(`Ignored error: ${err.message}`) - } else { - throw err - } - } + const query = raw + ';' + try { + await db.query(query) + } catch (err) { + const errorToCheck = err.parent || err + if (errorToCheck.code === '25P02') { + logger.warn('Transaction aborted, rolling back and starting new transaction...') + await db.query('ROLLBACK') + await db.query('BEGIN') + continue + } + if (isIgnorablePostgresError(err)) { + logger.warn(`Ignored PostgreSQL error: ${errorToCheck.message}`) + } else { + throw err } } + } + } - await this.updateSeederVersion(db, seederVersion, 'sqlite') - await sqliteRun(db, 'COMMIT') - logger.info('Seeding completed successfully.') + async applyPostgresMigrationIfNeeded (db, version) { + const currentVersion = await this.checkMigrationVersion(db, 'postgres') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`PostgreSQL schema ${currentVersion || 'none'} already includes ${version}, skipping migration`) + return + } + + const sql = readSqlFile(migrationSqlPath('postgres', version)) + await db.query('BEGIN') + try { + await this.applyPostgresStatements(db, sql) + await this.updateMigrationVersion(db, version, 'postgres') + await db.query('COMMIT') + logger.info(`PostgreSQL migration ${version} completed successfully.`) } catch (err) { try { - await sqliteRun(db, 'ROLLBACK') + await db.query('ROLLBACK') } catch (rollbackErr) { // No active transaction to roll back. } - logger.error('Seeding failed:', err) + logger.error(`PostgreSQL migration ${version} failed:`, err) throw err - } finally { - try { - await sqliteClose(db) - logger.info('Database connection closed after seeding.') - } catch (closeErr) { - logger.error('Error closing database connection:', closeErr.message) - } } } - // MySQL seeder — greenfield v3.8.0 (see src/data/migrations/README.md) - async runSeederMySQL (db) { - const seederSqlPath = path.resolve(__dirname, '../seeders/mysql/db_seeder_mysql_v3.8.0.sql') - const seederVersion = '3.8.0' - - if (!fs.existsSync(seederSqlPath)) { - logger.error(`Seeder file not found: ${seederSqlPath}`) - throw new Error('Seeder file not found') + async applyPostgresSeederIfNeeded (db, version) { + const currentVersion = await this.checkSeederVersion(db, 'postgres') + if (!isVersionLessThan(currentVersion, version)) { + logger.info(`PostgreSQL seeder ${currentVersion || 'none'} already includes ${version}, skipping seeder`) + return } - const seederSql = fs.readFileSync(seederSqlPath).toString() - const dataArr = seederSql.split(';') - + const sql = readSqlFile(seederSqlPath('postgres', version)) + await db.query('BEGIN') try { - const currentVersion = await this.checkSeederVersion(db, 'mysql') - - if (currentVersion === seederVersion) { - logger.info('Seeder already up to date, skipping...') - return + await this.applyPostgresStatements(db, sql) + await this.updateSeederVersion(db, version, 'postgres') + await db.query('COMMIT') + logger.info(`PostgreSQL seeder ${version} completed successfully.`) + } catch (err) { + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`PostgreSQL seeder ${version} failed:`, err) + throw err + } + } - await db.query('START TRANSACTION') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await db.query(query) - } catch (err) { - if (err.code === 'ER_DUP_ENTRY' || - err.code === 'ER_DUP_KEY') { - logger.warn(`Ignored MySQL error: ${err.message}`) - } else { - await db.query('ROLLBACK') - throw err - } - } - } + async applyPostgresBaseline (db) { + logger.info(`Applying empty-database baseline schema ${BASELINE_VERSION}`) + const migrationSql = readSqlFile(baselineMigrationSqlPath('postgres')) + await db.query('BEGIN') + try { + await this.applyPostgresStatements(db, migrationSql) + await this.updateMigrationVersion(db, BASELINE_VERSION, 'postgres') + await db.query('COMMIT') + logger.info(`PostgreSQL baseline migration ${BASELINE_VERSION} completed successfully.`) + } catch (err) { + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. } + logger.error(`PostgreSQL baseline migration ${BASELINE_VERSION} failed:`, err) + throw err + } - await this.updateSeederVersion(db, seederVersion, 'mysql') + logger.info(`Applying empty-database baseline seed ${BASELINE_VERSION}`) + const seederSql = readSqlFile(baselineSeederSqlPath('postgres')) + await db.query('BEGIN') + try { + await this.applyPostgresStatements(db, seederSql) + await this.updateSeederVersion(db, BASELINE_VERSION, 'postgres') await db.query('COMMIT') - logger.info('Seeding completed successfully.') + logger.info(`PostgreSQL baseline seeder ${BASELINE_VERSION} completed successfully.`) } catch (err) { - await db.query('ROLLBACK') - logger.error('Seeding failed:', err) + try { + await db.query('ROLLBACK') + } catch (rollbackErr) { + // No active transaction to roll back. + } + logger.error(`PostgreSQL baseline seeder ${BASELINE_VERSION} failed:`, err) throw err } } - // PostgreSQL seeder — greenfield v3.8.0 (see src/data/migrations/README.md) - async runSeederPostgres (db) { - const seederSqlPath = path.resolve(__dirname, '../seeders/postgres/db_seeder_pg_v3.8.0.sql') - const seederVersion = '3.8.0' - - if (!fs.existsSync(seederSqlPath)) { - logger.error(`Seeder file not found: ${seederSqlPath}`) - throw new Error('Seeder file not found') + async runVersionChainPostgres (db, options = {}) { + await this.createSchemaVersionTable(db, 'postgres') + const currentVersion = await this.checkMigrationVersion(db, 'postgres') + if (shouldApplyBaseline(currentVersion, options.untilVersion)) { + await this.applyPostgresBaseline(db) + return + } + for (const version of versionsToApply(options.untilVersion)) { + await this.applyPostgresMigrationIfNeeded(db, version) + await this.applyPostgresSeederIfNeeded(db, version) } + } - const seederSql = fs.readFileSync(seederSqlPath).toString() - const dataArr = seederSql.split(';') + async runMigrationSQLite (dbName, options) { + return this.runVersionChainSQLite(dbName, options) + } - try { - const currentVersion = await this.checkSeederVersion(db, 'postgres') + async runMigrationMySQL (db, options) { + return this.runVersionChainMySQL(db, options) + } - if (currentVersion === seederVersion) { - logger.info('Seeder already up to date, skipping...') - return - } + async runMigrationPostgres (db, options) { + return this.runVersionChainPostgres(db, options) + } - await db.query('BEGIN') - - for (let query of dataArr) { - if (query.trim()) { - query = query.trim() + ';' - try { - await db.query(query) - } catch (err) { - if (err.code === '23505' || - err.code === '23503') { - logger.warn(`Ignored PostgreSQL error: ${err.message}`) - } else { - await db.query('ROLLBACK') - throw err - } - } - } - } + async runSeederSQLite (dbName, options) { + return this.runVersionChainSQLite(dbName, options) + } - await this.updateSeederVersion(db, seederVersion, 'postgres') - await db.query('COMMIT') - logger.info('Seeding completed successfully.') - } catch (err) { - await db.query('ROLLBACK') - logger.error('Seeding failed:', err) - throw err - } + async runSeederMySQL (db, options) { + return this.runVersionChainMySQL(db, options) + } + + async runSeederPostgres (db, options) { + return this.runVersionChainPostgres(db, options) } } +DatabaseProvider.SCHEMA_VERSIONS = SCHEMA_VERSIONS +DatabaseProvider.BASELINE_VERSION = BASELINE_VERSION + module.exports = DatabaseProvider diff --git a/src/data/providers/sqlite.js b/src/data/providers/sqlite.js index 53693e1cb..20e802324 100644 --- a/src/data/providers/sqlite.js +++ b/src/data/providers/sqlite.js @@ -17,7 +17,9 @@ class SqliteDatabaseProvider extends DatabaseProvider { if (!sqliteConfig.databaseName.endsWith('.sqlite')) { sqliteConfig.databaseName += '.sqlite' } - const storageFolder = path.resolve(__dirname, '../sqlite_files') + const storageFolder = process.env.SQLITE_STORAGE_DIR + ? path.resolve(process.env.SQLITE_STORAGE_DIR) + : path.resolve(__dirname, '../sqlite_files') sqliteConfig.storage = path.resolve(storageFolder, sqliteConfig.databaseName) if (!fs.existsSync(storageFolder)) { fs.mkdirSync(storageFolder) diff --git a/src/data/seeders/mysql/db_seeder_mysql_baseline_v3.9.0.sql b/src/data/seeders/mysql/db_seeder_mysql_baseline_v3.9.0.sql new file mode 100644 index 000000000..93ad07a0a --- /dev/null +++ b/src/data/seeders/mysql/db_seeder_mysql_baseline_v3.9.0.sql @@ -0,0 +1,69 @@ +START TRANSACTION; + +INSERT INTO `Registries` (url, is_public, user_name, password, user_email, type, ca, insecure) +VALUES + ('docker.io', true, '', '', '', 'oci', NULL, false), + ('from_cache', true, '', '', '', 'oci', NULL, false), + ('https://huggingface.co', true, '', '', '', 'hf', NULL, false); + +INSERT INTO `CatalogItems` (name, description, category, publisher, disk_required, ram_required, picture, config_example, is_public, registry_id) +VALUES + ('Router', 'The built-in router for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('Debug', 'The built-in debugger for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('NATs', 'NATs server microservice for Edgelet', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, true, 1); + +INSERT INTO `Architectures` (id, name, image, description, network_catalog_item_id) +VALUES + (0, 'auto', 'iointegrator0.png', 'Architecture will be selected on provision', 1), + (1, 'amd64', 'iointegrator1.png', 'Standard x86_64 Linux. Compatible with common Linux distributions such as Ubuntu, Red Hat, and CentOS.', 1), + (2, 'arm64', 'iointegrator2.png', '64-bit ARM Linux (aarch64). Microservices for this architecture are built for ARM64 systems.', 1), + (3, 'riscv64', 'iointegrator3.png', 'RISC-V 64-bit Linux. Microservices for this architecture are built for RISC-V systems.', 1), + (4, 'arm', 'iointegrator4.png', '32-bit ARM Linux. Microservices for this architecture are built for 32-bit ARM systems.', 1); + +UPDATE `Fogs` +SET arch_id = 0 +WHERE arch_id IS NULL; + +INSERT INTO `CatalogItemImages` (catalog_item_id, arch_id, container_image) +VALUES + (1, 1, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 2, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 3, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 4, 'ghcr.io/eclipse-iofog/router:latest'), + (2, 1, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 2, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 3, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 4, 'ghcr.io/eclipse-iofog/debugger:latest'), + (3, 1, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 2, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 3, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 4, 'ghcr.io/eclipse-iofog/nats:latest'); + +INSERT IGNORE INTO AuthPolicy ( + id, + min_password_length, + require_uppercase, + require_lowercase, + require_digit, + password_max_age_days, + password_history_count, + max_failed_attempts, + lockout_duration_minutes, + access_token_ttl_seconds, + refresh_token_ttl_seconds, + refresh_rotation, + max_concurrent_sessions +) +VALUES (1, 12, true, true, true, 0, 5, 5, 15, 900, 3600, true, NULL); + +INSERT IGNORE INTO AuthGroups (name, is_system, mfa_required) +VALUES + ('admin', true, false), + ('sre', true, false), + ('developer', true, false), + ('viewer', true, false); + +INSERT IGNORE INTO AuthBootstrapMeta (id, completed_at) +VALUES (1, NULL); + +COMMIT; diff --git a/src/data/seeders/mysql/db_seeder_mysql_v3.9.0.sql b/src/data/seeders/mysql/db_seeder_mysql_v3.9.0.sql new file mode 100644 index 000000000..a52bca809 --- /dev/null +++ b/src/data/seeders/mysql/db_seeder_mysql_v3.9.0.sql @@ -0,0 +1,12 @@ +INSERT INTO Registries (url, is_public, user_name, password, user_email, type, ca, insecure) +SELECT 'https://huggingface.co', true, '', '', '', 'hf', NULL, false +FROM DUAL +WHERE NOT EXISTS ( + SELECT 1 FROM Registries WHERE url = 'https://huggingface.co' AND type = 'hf' +); + +DELETE FROM Microservices WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemImages WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemInputTypes WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItemOutputTypes WHERE catalog_item_id IN (SELECT id FROM (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')) AS catalog_ids); +DELETE FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue'); diff --git a/src/data/seeders/postgres/db_seeder_pg_baseline_v3.9.0.sql b/src/data/seeders/postgres/db_seeder_pg_baseline_v3.9.0.sql new file mode 100644 index 000000000..a8c5cf9a3 --- /dev/null +++ b/src/data/seeders/postgres/db_seeder_pg_baseline_v3.9.0.sql @@ -0,0 +1,73 @@ +START TRANSACTION; + +INSERT INTO "Registries" (url, is_public, user_name, password, user_email, type, ca, insecure) +VALUES + ('docker.io', true, '', '', '', 'oci', NULL, false), + ('from_cache', true, '', '', '', 'oci', NULL, false), + ('https://huggingface.co', true, '', '', '', 'hf', NULL, false); + +INSERT INTO "CatalogItems" (name, description, category, publisher, disk_required, ram_required, picture, config_example, is_public, registry_id) +VALUES + ('Router', 'The built-in router for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('Debug', 'The built-in debugger for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('NATs', 'NATs server microservice for Edgelet', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, true, 1); + +INSERT INTO "Architectures" (id, name, image, description, network_catalog_item_id) +VALUES + (0, 'auto', 'iointegrator0.png', 'Architecture will be selected on provision', 1), + (1, 'amd64', 'iointegrator1.png', 'Standard x86_64 Linux. Compatible with common Linux distributions such as Ubuntu, Red Hat, and CentOS.', 1), + (2, 'arm64', 'iointegrator2.png', '64-bit ARM Linux (aarch64). Microservices for this architecture are built for ARM64 systems.', 1), + (3, 'riscv64', 'iointegrator3.png', 'RISC-V 64-bit Linux. Microservices for this architecture are built for RISC-V systems.', 1), + (4, 'arm', 'iointegrator4.png', '32-bit ARM Linux. Microservices for this architecture are built for 32-bit ARM systems.', 1); + +UPDATE "Fogs" +SET arch_id = 0 +WHERE arch_id IS NULL; + +INSERT INTO "CatalogItemImages" (catalog_item_id, arch_id, container_image) +VALUES + (1, 1, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 2, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 3, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 4, 'ghcr.io/eclipse-iofog/router:latest'), + (2, 1, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 2, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 3, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 4, 'ghcr.io/eclipse-iofog/debugger:latest'), + (3, 1, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 2, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 3, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 4, 'ghcr.io/eclipse-iofog/nats:latest'); + +INSERT INTO "AuthPolicy" ( + id, + min_password_length, + require_uppercase, + require_lowercase, + require_digit, + password_max_age_days, + password_history_count, + max_failed_attempts, + lockout_duration_minutes, + access_token_ttl_seconds, + refresh_token_ttl_seconds, + refresh_rotation, + max_concurrent_sessions +) +VALUES (1, 12, true, true, true, 0, 5, 5, 15, 900, 3600, true, NULL) +ON CONFLICT (id) DO NOTHING; + +INSERT INTO "AuthGroups" (name, is_system, mfa_required) +VALUES + ('admin', true, false), + ('sre', true, false), + ('developer', true, false), + ('viewer', true, false) +ON CONFLICT (name) DO NOTHING; + +INSERT INTO "AuthBootstrapMeta" (id, completed_at) +OVERRIDING SYSTEM VALUE +VALUES (1, NULL) +ON CONFLICT (id) DO NOTHING; + +COMMIT; diff --git a/src/data/seeders/postgres/db_seeder_pg_v3.9.0.sql b/src/data/seeders/postgres/db_seeder_pg_v3.9.0.sql new file mode 100644 index 000000000..d9d13de39 --- /dev/null +++ b/src/data/seeders/postgres/db_seeder_pg_v3.9.0.sql @@ -0,0 +1,11 @@ +INSERT INTO "Registries" (url, is_public, user_name, password, user_email, type, ca, insecure) +SELECT 'https://huggingface.co', true, '', '', '', 'hf', NULL, false +WHERE NOT EXISTS ( + SELECT 1 FROM "Registries" WHERE url = 'https://huggingface.co' AND type = 'hf' +); + +DELETE FROM "Microservices" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemImages" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemInputTypes" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItemOutputTypes" WHERE catalog_item_id IN (SELECT id FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM "CatalogItems" WHERE name IN ('HAL', 'RESTBlue'); diff --git a/src/data/seeders/sqlite/db_seeder_sqlite_baseline_v3.9.0.sql b/src/data/seeders/sqlite/db_seeder_sqlite_baseline_v3.9.0.sql new file mode 100644 index 000000000..01cae2f34 --- /dev/null +++ b/src/data/seeders/sqlite/db_seeder_sqlite_baseline_v3.9.0.sql @@ -0,0 +1,65 @@ +INSERT INTO `Registries` (url, is_public, user_name, password, user_email, type, ca, insecure) +VALUES + ('docker.io', true, '', '', '', 'oci', NULL, false), + ('from_cache', true, '', '', '', 'oci', NULL, false), + ('https://huggingface.co', true, '', '', '', 'hf', NULL, false); + +INSERT INTO `CatalogItems` (name, description, category, publisher, disk_required, ram_required, picture, config_example, is_public, registry_id) +VALUES + ('Router', 'The built-in router for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('Debug', 'The built-in debugger for Edgelet.', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, false, 1), + ('NATs', 'NATs server microservice for Edgelet', 'SYSTEM', 'Eclipse ioFog', 0, 0, 'none.png', NULL, true, 1); + +INSERT INTO `Architectures` (id, name, image, description, network_catalog_item_id) +VALUES + (0, 'auto', 'iointegrator0.png', 'Architecture will be selected on provision', 1), + (1, 'amd64', 'iointegrator1.png', 'Standard x86_64 Linux. Compatible with common Linux distributions such as Ubuntu, Red Hat, and CentOS.', 1), + (2, 'arm64', 'iointegrator2.png', '64-bit ARM Linux (aarch64). Microservices for this architecture are built for ARM64 systems.', 1), + (3, 'riscv64', 'iointegrator3.png', 'RISC-V 64-bit Linux. Microservices for this architecture are built for RISC-V systems.', 1), + (4, 'arm', 'iointegrator4.png', '32-bit ARM Linux. Microservices for this architecture are built for 32-bit ARM systems.', 1); + +UPDATE `Fogs` +SET arch_id = 0 +WHERE arch_id IS NULL; + +INSERT INTO `CatalogItemImages` (catalog_item_id, arch_id, container_image) +VALUES + (1, 1, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 2, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 3, 'ghcr.io/eclipse-iofog/router:latest'), + (1, 4, 'ghcr.io/eclipse-iofog/router:latest'), + (2, 1, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 2, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 3, 'ghcr.io/eclipse-iofog/debugger:latest'), + (2, 4, 'ghcr.io/eclipse-iofog/debugger:latest'), + (3, 1, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 2, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 3, 'ghcr.io/eclipse-iofog/nats:latest'), + (3, 4, 'ghcr.io/eclipse-iofog/nats:latest'); + +INSERT OR IGNORE INTO AuthPolicy ( + id, + min_password_length, + require_uppercase, + require_lowercase, + require_digit, + password_max_age_days, + password_history_count, + max_failed_attempts, + lockout_duration_minutes, + access_token_ttl_seconds, + refresh_token_ttl_seconds, + refresh_rotation, + max_concurrent_sessions +) +VALUES (1, 12, true, true, true, 0, 5, 5, 15, 900, 3600, true, NULL); + +INSERT OR IGNORE INTO AuthGroups (name, is_system, mfa_required) +VALUES + ('admin', true, false), + ('sre', true, false), + ('developer', true, false), + ('viewer', true, false); + +INSERT OR IGNORE INTO AuthBootstrapMeta (id, completed_at) +VALUES (1, NULL); diff --git a/src/data/seeders/sqlite/db_seeder_sqlite_v3.9.0.sql b/src/data/seeders/sqlite/db_seeder_sqlite_v3.9.0.sql new file mode 100644 index 000000000..f212e2cf0 --- /dev/null +++ b/src/data/seeders/sqlite/db_seeder_sqlite_v3.9.0.sql @@ -0,0 +1,11 @@ +INSERT INTO Registries (url, is_public, user_name, password, user_email, type, ca, insecure) +SELECT 'https://huggingface.co', true, '', '', '', 'hf', NULL, false +WHERE NOT EXISTS ( + SELECT 1 FROM Registries WHERE url = 'https://huggingface.co' AND type = 'hf' +); + +DELETE FROM Microservices WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemImages WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemInputTypes WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItemOutputTypes WHERE catalog_item_id IN (SELECT id FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')); +DELETE FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue'); diff --git a/src/helpers/constants.js b/src/helpers/constants.js index 694dfa92f..1e0c04417 100644 --- a/src/helpers/constants.js +++ b/src/helpers/constants.js @@ -55,6 +55,7 @@ module.exports = { HTTP_CODE_NOT_IMPLEMENTED: 501, READINESS_RETRY_AFTER_SECONDS: 5, HTTP_CODE_DUPLICATE_PROPERTY: 409, + HTTP_CODE_CONFLICT: 409, HTTP_CODE_INTERNAL_ERROR: 500, DEFAULT_ROUTER_NAME: 'default-router', diff --git a/src/helpers/error-messages.js b/src/helpers/error-messages.js index 9f4f7af18..6552df0c3 100644 --- a/src/helpers/error-messages.js +++ b/src/helpers/error-messages.js @@ -57,6 +57,10 @@ module.exports = { USER_ALREADY_ACTIVATED: 'User is already activated.', USER_NOT_ACTIVATED_YET: 'User is not activated yet.', REGISTRY_IS_SYSTEM: 'Registry is system and can\'t be updated or deleted', + REGISTRY_PRIVATE_OCI_CREDENTIALS_REQUIRED: 'Private OCI registries require username and password', + REGISTRY_PRIVATE_HF_TOKEN_REQUIRED: 'Private Hugging Face registries require a password token', + REGISTRY_INVALID_CA: 'Registry CA must be valid base64 when provided', + REGISTRY_NOT_OCI_FOR_IMAGE: 'Registry \'{}\' cannot be used for container images; type must be oci', INVALID_CLUSTER_CONTROLLER_UUID: 'Invalid cluster controller UUID \'{}\'', CLUSTER_CONTROLLER_NOT_FOUND: 'Cluster controller not found', CLUSTER_CONTROLLER_REGISTRATION_FAILED: 'Cluster controller registration failed', @@ -132,5 +136,65 @@ module.exports = { CONFIGMAP_IMMUTABLE: 'Configmap {} is immutable and cannot be updated. If you want to update it, please delete it and create a new configmap.', VOLUME_MOUNT_NOT_FOUND: 'Volume mount with name {} not found', INVALID_VOLUME_MOUNT_REFERENCE_FOR_VOLUME_MAPPING: 'Invalid volume mount reference for volume mapping: {}', - SECRET_TYPE_MISMATCH: 'Secret type mismatch. Secret {} is of type {} but trying to update with type {}' + SECRET_TYPE_MISMATCH: 'Secret type mismatch. Secret {} is of type {} but trying to update with type {}', + MODEL_NOT_FOUND: 'Model with name {} not found', + MODEL_ALREADY_EXISTS: 'Model with name {} already exists', + MODEL_IN_USE: 'Cannot delete model \'{}\': bound by microservices {}', + MODEL_UNLINK_IN_USE: 'Cannot unlink model \'{}\' from agent {}: bound by microservices {}', + MODEL_HF_FILES_REQUIRED: 'Hugging Face models require a non-empty files list', + MODEL_OCI_FILES_FORBIDDEN: 'OCI models must not specify files', + KNOWLEDGE_NOT_FOUND: 'Knowledge with name {} not found', + KNOWLEDGE_ALREADY_EXISTS: 'Knowledge with name {} already exists', + KNOWLEDGE_IN_USE: 'Cannot delete knowledge \'{}\': bound by microservices {}', + KNOWLEDGE_UNLINK_IN_USE: 'Cannot unlink knowledge \'{}\' from agent {}: bound by microservices {}', + KNOWLEDGE_OCI_FILES_FORBIDDEN: 'OCI knowledge must not specify files', + RUNTIME_CLASS_NOT_FOUND: 'RuntimeClass with name {} not found', + RUNTIME_CLASS_ALREADY_EXISTS: 'RuntimeClass with name {} already exists', + RUNTIME_CLASS_IN_USE: 'Cannot delete RuntimeClass \'{}\': pinned by microservices {}', + RUNTIME_CLASS_UNLINK_IN_USE: 'Cannot unlink RuntimeClass \'{}\' from agent {}: pinned by microservices {}', + RUNTIME_CLASS_LINK_REQUIRES_EDGELET: 'Cannot link RuntimeClass to agent {}: container engine must be edgelet', + RUNTIME_CLASS_LINK_NOT_AVAILABLE: 'Cannot link RuntimeClass \'{}\' to agent \'{}\': runtime is not in availableRuntimes', + RUNTIME_CLASS_MISSING: 'RuntimeClass \'{}\' does not exist', + MICROSERVICE_CATALOG_BIND_PATH_REQUIRED: 'Model catalog bindPath is required when items are present', + MICROSERVICE_CATALOG_BIND_PATH_REQUIRES_ITEMS: 'Model catalog bindPath requires at least one item', + MICROSERVICE_CATALOG_BIND_PATH_ABSOLUTE: 'Model catalog bindPath must be an absolute container path', + MICROSERVICE_CATALOG_PERMISSIONS: 'Model catalog permissions must be ro or rw', + MICROSERVICE_CATALOG_DUPLICATE_ITEM: 'Model catalog item name \'{}\' is duplicated', + MICROSERVICE_CATALOG_ITEM_NAME: 'Model catalog item name \'{}\' is invalid', + MICROSERVICE_CATALOG_PATH_COLLISION: 'Model catalog path \'{}\' collides with a volume, tmpfs, or knowledge catalog mount', + MICROSERVICE_CATALOG_MODEL_NOT_FOUND: 'Model catalog item \'{}\' does not match a fleet model', + MICROSERVICE_KNOWLEDGE_BIND_PATH_REQUIRED: 'Knowledge catalog bindPath is required when items are present', + MICROSERVICE_KNOWLEDGE_BIND_PATH_REQUIRES_ITEMS: 'Knowledge catalog bindPath requires at least one item', + MICROSERVICE_KNOWLEDGE_BIND_PATH_ABSOLUTE: 'Knowledge catalog bindPath must be an absolute container path', + MICROSERVICE_KNOWLEDGE_PERMISSIONS: 'Knowledge catalog permissions must be ro or rw', + MICROSERVICE_KNOWLEDGE_DUPLICATE_ITEM: 'Knowledge catalog item name \'{}\' is duplicated', + MICROSERVICE_KNOWLEDGE_ITEM_NAME: 'Knowledge catalog item name \'{}\' is invalid', + MICROSERVICE_KNOWLEDGE_PATH_COLLISION: 'Knowledge catalog path \'{}\' collides with a volume, tmpfs, or model catalog mount', + MICROSERVICE_KNOWLEDGE_NOT_FOUND: 'Knowledge catalog item \'{}\' does not match fleet knowledge', + MICROSERVICE_KNOWLEDGE_SYSTEM_FORBIDDEN: 'System microservices cannot set a knowledge catalog', + MICROSERVICE_RUN_AS_USER_GROUP_CONFLICT: 'runAsUser cannot contain \':\' when runAsGroup is set', + MICROSERVICE_MEMORY_SWAP_REQUIRES_LIMIT: 'memorySwap requires memoryLimit unless it is -1', + MICROSERVICE_MEMORY_SWAP_VALUE: 'memorySwap must be -1 or a positive MiB total', + MICROSERVICE_WORKING_DIR_ABSOLUTE: 'workingDir must be an absolute container path', + MICROSERVICE_POSITIVE_NUMBER: 'Field \'{}\' must be greater than 0', + MICROSERVICE_ARGV_TYPE: 'Field \'{}\' must be an array of strings', + MICROSERVICE_SYSCTL_UNKNOWN: 'sysctl \'{}\' is not allowed', + MICROSERVICE_SYSCTL_SHAPE: 'sysctls must be an object of string values', + MICROSERVICE_SYSCTL_HOST_NETWORK: 'net.* sysctls cannot be set when hostNetworkMode is true', + MICROSERVICE_SYSCTL_HOST_IPC: 'IPC sysctls cannot be set when ipcMode is host', + MICROSERVICE_ULIMIT_UNKNOWN: 'ulimit \'{}\' is not allowed', + MICROSERVICE_ULIMIT_SHAPE: 'ulimit \'{}\' must be an object with soft and hard', + MICROSERVICE_ULIMIT_SHAPE_ROOT: 'ulimits must be an object of {soft, hard} entries', + MICROSERVICE_ULIMIT_RANGE: 'ulimit \'{}\' soft must be <= hard unless unlimited', + MICROSERVICE_DEVICES_SHAPE: 'devices must be an array of {hostPath, containerPath, permissions} objects', + MICROSERVICE_DEVICE_HOST_PATH: 'device hostPath must be an absolute path under /dev', + MICROSERVICE_DEVICE_CONTAINER_PATH: 'device containerPath is required', + MICROSERVICE_DEVICE_PERMISSIONS: 'device permissions must be a combination of r, w, and m', + MICROSERVICE_TMPFS_SHAPE: 'tmpfs must be an array of {containerPath, size?, mode?} objects', + MICROSERVICE_TMPFS_PATH: 'tmpfs containerPath must be an absolute container path', + MICROSERVICE_TMPFS_SIZE: 'tmpfs size must be greater than 0', + MICROSERVICE_TEMPLATE_NOT_FOUND: 'Microservice template with name {} not found', + MICROSERVICE_TEMPLATE_ALREADY_EXISTS: 'Microservice template with name {} already exists', + MICROSERVICE_TEMPLATE_INVALID: 'Microservice template {} is invalid', + MICROSERVICE_TEMPLATE_FILE_NOT_FOUND: 'Microservice template file missing' } diff --git a/src/helpers/knowledge-format.js b/src/helpers/knowledge-format.js new file mode 100644 index 000000000..34f255c69 --- /dev/null +++ b/src/helpers/knowledge-format.js @@ -0,0 +1,36 @@ +const KNOWN_FORMATS = new Set([ + 'markdown', + 'pdf', + 'jsonl', + 'parquet', + 'arrow', + 'sqlite', + 'faiss', + 'chroma', + 'lance' +]) + +/** + * Soft hint for a Knowledge artifact. Never throws. + * Omit, null, and empty string stay unset. Known tokens persist lowercase. + * Any other value persists "unknown". + * @param {*} value + * @returns {string|null} + */ +function normalizeKnowledgeFormat (value) { + if (value == null || value === '') { + return null + } + if (typeof value !== 'string') { + return 'unknown' + } + const token = value.toLowerCase() + if (KNOWN_FORMATS.has(token)) { + return token + } + return 'unknown' +} + +module.exports = { + normalizeKnowledgeFormat +} diff --git a/src/helpers/microservice-container-catalog.js b/src/helpers/microservice-container-catalog.js new file mode 100644 index 000000000..ab57c3190 --- /dev/null +++ b/src/helpers/microservice-container-catalog.js @@ -0,0 +1,681 @@ +'use strict' + +const Errors = require('./errors') +const ErrorMessages = require('./error-messages') +const AppHelper = require('./app-helper') +const { serviceNameRegex } = require('../schemas/utils/utils') + +const SERVICE_NAME_RE = new RegExp(`^${serviceNameRegex}$`) + +const SYSCTL_ALLOWLIST = new Set([ + 'kernel.shm_rmid_forced', + 'net.ipv4.ip_local_port_range', + 'net.ipv4.tcp_syncookies', + 'net.ipv4.ping_group_range', + 'net.ipv4.ip_unprivileged_port_start', + 'net.ipv4.ip_local_reserved_ports', + 'net.ipv4.tcp_keepalive_time', + 'net.ipv4.tcp_fin_timeout', + 'net.ipv4.tcp_keepalive_intvl', + 'net.ipv4.tcp_keepalive_probes', + 'net.ipv4.tcp_rmem', + 'net.ipv4.tcp_wmem', + 'net.ipv4.tcp_slow_start_after_idle', + 'net.ipv4.tcp_notsent_lowat' +]) + +const ULIMIT_ALLOWLIST = new Set([ + 'core', 'cpu', 'data', 'fsize', 'locks', 'memlock', 'msgqueue', 'nice', + 'nofile', 'nproc', 'rss', 'rtprio', 'rttime', 'sigpending', 'stack' +]) + +function parseJsonField (raw) { + if (raw == null || raw === '') { + return null + } + if (typeof raw === 'object') { + return raw + } + if (typeof raw !== 'string') { + return null + } + try { + return JSON.parse(raw) + } catch (error) { + return null + } +} + +function serializeJsonField (value) { + if (value == null) { + return null + } + return JSON.stringify(value) +} + +function parseArgv (raw) { + const parsed = Array.isArray(raw) ? raw : parseJsonField(raw) + if (!Array.isArray(parsed)) { + return [] + } + return parsed.filter((item) => typeof item === 'string') +} + +function serializeArgv (argv) { + if (!Array.isArray(argv) || argv.length === 0) { + return null + } + return JSON.stringify(argv) +} + +function resolveProcessArgv (data) { + if (!data || typeof data !== 'object') { + return undefined + } + if (data.commands !== undefined) { + return data.commands + } + if (data.cmd !== undefined) { + return data.cmd + } + return undefined +} + +function assertArgv (name, argv) { + if (argv === undefined || argv === null) { + return + } + if (!Array.isArray(argv) || argv.some((item) => typeof item !== 'string')) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ARGV_TYPE, name)) + } +} + +function isCatalogEmpty (catalog) { + return !catalog || !Array.isArray(catalog.items) || catalog.items.length === 0 +} + +function parseCatalog (raw) { + return parseJsonField(raw) +} + +function normalizeCatalog (input) { + if (input == null) { + return null + } + const items = Array.isArray(input.items) + ? input.items.map((item) => ({ name: item && item.name })) + : [] + const permissions = input.permissions == null || input.permissions === '' + ? 'ro' + : input.permissions + return { + bindPath: input.bindPath, + permissions, + items + } +} + +function serializeCatalog (catalog) { + if (catalog == null) { + return null + } + return JSON.stringify({ + bindPath: catalog.bindPath || '', + permissions: catalog.permissions || 'ro', + items: Array.isArray(catalog.items) ? catalog.items : [] + }) +} + +function catalogItemNames (catalog) { + if (isCatalogEmpty(catalog)) { + return [] + } + return catalog.items.map((item) => item && item.name).filter(Boolean) +} + +function catalogRequiresRebuild (previousRaw, nextCatalog) { + const previous = previousRaw && typeof previousRaw === 'object' && !Array.isArray(previousRaw) + ? previousRaw + : parseCatalog(previousRaw) + const previousEmpty = isCatalogEmpty(previous) + const nextEmpty = isCatalogEmpty(nextCatalog) + if (previousEmpty !== nextEmpty) { + return true + } + if (previousEmpty && nextEmpty) { + return false + } + const previousBind = (previous && previous.bindPath) || '' + const nextBind = (nextCatalog && nextCatalog.bindPath) || '' + const previousPermissions = (previous && previous.permissions) || 'ro' + const nextPermissions = (nextCatalog && nextCatalog.permissions) || 'ro' + return previousBind !== nextBind || previousPermissions !== nextPermissions +} + +function isAbsoluteContainerPath (value) { + return typeof value === 'string' && value.startsWith('/') && value.length > 1 +} + +function joinCatalogItemPath (bindPath, name) { + const base = (bindPath || '').replace(/\/+$/, '') + return `${base}/${name}/` +} + +function collectMountPaths (volumeMappings, tmpfs) { + const paths = [] + if (Array.isArray(volumeMappings)) { + for (const mapping of volumeMappings) { + if (mapping && mapping.containerDestination) { + paths.push(mapping.containerDestination) + } + } + } + const tmpfsList = Array.isArray(tmpfs) ? tmpfs : parseJsonField(tmpfs) + if (Array.isArray(tmpfsList)) { + for (const entry of tmpfsList) { + if (entry && entry.containerPath) { + paths.push(entry.containerPath) + } + } + } + return paths +} + +function catalogMountPaths (catalog) { + if (isCatalogEmpty(catalog)) { + return [] + } + const paths = [] + if (catalog.bindPath) { + paths.push(catalog.bindPath) + } + for (const item of catalog.items) { + if (item && item.name) { + paths.push(joinCatalogItemPath(catalog.bindPath, item.name)) + } + } + return paths +} + +function catalogValidationMessages (kind) { + if (kind === 'knowledge') { + return { + permissions: ErrorMessages.MICROSERVICE_KNOWLEDGE_PERMISSIONS, + bindRequired: ErrorMessages.MICROSERVICE_KNOWLEDGE_BIND_PATH_REQUIRED, + bindRequiresItems: ErrorMessages.MICROSERVICE_KNOWLEDGE_BIND_PATH_REQUIRES_ITEMS, + bindAbsolute: ErrorMessages.MICROSERVICE_KNOWLEDGE_BIND_PATH_ABSOLUTE, + itemName: ErrorMessages.MICROSERVICE_KNOWLEDGE_ITEM_NAME, + duplicate: ErrorMessages.MICROSERVICE_KNOWLEDGE_DUPLICATE_ITEM, + collision: ErrorMessages.MICROSERVICE_KNOWLEDGE_PATH_COLLISION + } + } + return { + permissions: ErrorMessages.MICROSERVICE_CATALOG_PERMISSIONS, + bindRequired: ErrorMessages.MICROSERVICE_CATALOG_BIND_PATH_REQUIRED, + bindRequiresItems: ErrorMessages.MICROSERVICE_CATALOG_BIND_PATH_REQUIRES_ITEMS, + bindAbsolute: ErrorMessages.MICROSERVICE_CATALOG_BIND_PATH_ABSOLUTE, + itemName: ErrorMessages.MICROSERVICE_CATALOG_ITEM_NAME, + duplicate: ErrorMessages.MICROSERVICE_CATALOG_DUPLICATE_ITEM, + collision: ErrorMessages.MICROSERVICE_CATALOG_PATH_COLLISION + } +} + +function pathsCollide (left, right) { + if (!left || !right) { + return false + } + const a = left.endsWith('/') ? left.slice(0, -1) : left + const b = right.endsWith('/') ? right.slice(0, -1) : right + return a === b +} + +function validateCatalog (catalog, { volumeMappings, tmpfs, otherCatalog, kind } = {}) { + if (catalog == null) { + return + } + const messages = catalogValidationMessages(kind) + if (catalog.permissions != null && catalog.permissions !== 'ro' && catalog.permissions !== 'rw') { + throw new Errors.ValidationError(messages.permissions) + } + const items = Array.isArray(catalog.items) ? catalog.items : [] + if (items.length === 0) { + if (typeof catalog.bindPath === 'string' && catalog.bindPath.trim() !== '') { + throw new Errors.ValidationError(messages.bindRequiresItems) + } + return + } + if (!catalog.bindPath) { + throw new Errors.ValidationError(messages.bindRequired) + } + if (!isAbsoluteContainerPath(catalog.bindPath) && catalog.bindPath !== '/') { + throw new Errors.ValidationError(messages.bindAbsolute) + } + + const seen = new Set() + for (const item of items) { + const name = item && item.name + if (!name || !SERVICE_NAME_RE.test(name)) { + throw new Errors.ValidationError(AppHelper.formatMessage(messages.itemName, name || '')) + } + if (seen.has(name)) { + throw new Errors.ValidationError(AppHelper.formatMessage(messages.duplicate, name)) + } + seen.add(name) + } + + const mountPaths = collectMountPaths(volumeMappings, tmpfs).concat(catalogMountPaths(otherCatalog)) + const catalogPaths = [catalog.bindPath, ...items.map((item) => joinCatalogItemPath(catalog.bindPath, item.name))] + for (const catalogPath of catalogPaths) { + for (const mountPath of mountPaths) { + if (pathsCollide(catalogPath, mountPath)) { + throw new Errors.ValidationError(AppHelper.formatMessage(messages.collision, catalogPath)) + } + } + } +} + +function pickValue (spec, existing, key) { + if (spec && spec[key] !== undefined) { + return spec[key] + } + return existing ? existing[key] : undefined +} + +function assertPositiveNumber (name, value) { + if (value == null || value === '') { + return + } + if (typeof value !== 'number' || Number.isNaN(value) || value <= 0) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_POSITIVE_NUMBER, name)) + } +} + +function isIpcSysctl (name) { + return name.startsWith('kernel.shm') || + name.startsWith('kernel.msg') || + name.startsWith('kernel.sem') || + name.startsWith('fs.mqueue.') +} + +function coerceSysctlValue (value) { + if (typeof value === 'string') { + return value + } + if (typeof value === 'number' && Number.isFinite(value)) { + return String(value) + } + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_SYSCTL_SHAPE) +} + +function normalizeSysctls (sysctls) { + if (sysctls == null || sysctls === '') { + return sysctls + } + if (typeof sysctls !== 'object' || Array.isArray(sysctls)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_SYSCTL_SHAPE) + } + const normalized = {} + for (const [key, value] of Object.entries(sysctls)) { + normalized[key] = coerceSysctlValue(value) + } + return normalized +} + +function validateSysctls (sysctls, hostNetworkMode, ipcMode) { + if (sysctls == null || sysctls === '') { + return + } + if (typeof sysctls !== 'object' || Array.isArray(sysctls)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_SYSCTL_SHAPE) + } + for (const key of Object.keys(sysctls)) { + if (!SYSCTL_ALLOWLIST.has(key)) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_SYSCTL_UNKNOWN, key)) + } + if (hostNetworkMode === true && key.startsWith('net.')) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_SYSCTL_HOST_NETWORK) + } + if (ipcMode === 'host' && isIpcSysctl(key)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_SYSCTL_HOST_IPC) + } + coerceSysctlValue(sysctls[key]) + } +} + +function validateUlimits (ulimits) { + if (ulimits == null || ulimits === '') { + return + } + if (typeof ulimits !== 'object' || Array.isArray(ulimits)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_ULIMIT_SHAPE_ROOT) + } + for (const [name, value] of Object.entries(ulimits)) { + if (!ULIMIT_ALLOWLIST.has(name)) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ULIMIT_UNKNOWN, name)) + } + if (!value || typeof value !== 'object' || Array.isArray(value) || + !Object.prototype.hasOwnProperty.call(value, 'soft') || + !Object.prototype.hasOwnProperty.call(value, 'hard')) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ULIMIT_SHAPE, name)) + } + const soft = value.soft + const hard = value.hard + if (typeof soft !== 'number' || typeof hard !== 'number') { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ULIMIT_SHAPE, name)) + } + if (soft === -1 && hard !== -1) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ULIMIT_RANGE, name)) + } + if (soft !== -1 && hard !== -1 && soft > hard) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_ULIMIT_RANGE, name)) + } + } +} + +function normalizeDevicePermissions (permissions) { + if (permissions == null || permissions === '') { + return 'rwm' + } + if (typeof permissions !== 'string' || !/^[rwm]+$/.test(permissions)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_DEVICE_PERMISSIONS) + } + return permissions +} + +function validateDevices (devices) { + if (devices == null || devices === '') { + return + } + if (!Array.isArray(devices)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_DEVICES_SHAPE) + } + for (const device of devices) { + if (!device || typeof device !== 'object') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_DEVICES_SHAPE) + } + if (typeof device.hostPath !== 'string' || !device.hostPath.startsWith('/dev/')) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_DEVICE_HOST_PATH) + } + if (typeof device.containerPath !== 'string' || device.containerPath.length === 0) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_DEVICE_CONTAINER_PATH) + } + device.permissions = normalizeDevicePermissions(device.permissions) + } +} + +function validateTmpfs (tmpfs) { + if (tmpfs == null || tmpfs === '') { + return + } + if (!Array.isArray(tmpfs)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_TMPFS_SHAPE) + } + for (const entry of tmpfs) { + if (!entry || typeof entry !== 'object') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_TMPFS_SHAPE) + } + if (!isAbsoluteContainerPath(entry.containerPath) && entry.containerPath !== '/') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_TMPFS_PATH) + } + if (entry.size != null && (typeof entry.size !== 'number' || entry.size <= 0)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_TMPFS_SIZE) + } + } +} + +function validateContainerFields (spec, existing = {}) { + const data = spec || {} + assertArgv('entrypoint', data.entrypoint) + const argv = resolveProcessArgv(data) + assertArgv('commands', argv) + + const runAsUser = pickValue(data, existing, 'runAsUser') + const runAsGroup = pickValue(data, existing, 'runAsGroup') + if (typeof runAsUser === 'string' && runAsUser.includes(':') && runAsGroup) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_RUN_AS_USER_GROUP_CONFLICT) + } + + const workingDir = data.workingDir + if (workingDir != null && workingDir !== '' && !isAbsoluteContainerPath(workingDir) && workingDir !== '/') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_WORKING_DIR_ABSOLUTE) + } + + assertPositiveNumber('cpus', data.cpus) + assertPositiveNumber('memoryReservation', data.memoryReservation) + assertPositiveNumber('shmSize', data.shmSize) + + const memoryLimit = pickValue(data, existing, 'memoryLimit') + if (data.memorySwap != null && data.memorySwap !== '') { + if (typeof data.memorySwap !== 'number' || Number.isNaN(data.memorySwap)) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_MEMORY_SWAP_VALUE) + } + if (data.memorySwap !== -1) { + if (data.memorySwap <= 0) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_MEMORY_SWAP_VALUE) + } + if (memoryLimit == null || memoryLimit === '') { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_MEMORY_SWAP_REQUIRES_LIMIT) + } + } + } + + const hostNetworkMode = pickValue(data, existing, 'hostNetworkMode') + const ipcMode = pickValue(data, existing, 'ipcMode') + if (data.sysctls !== undefined) { + data.sysctls = normalizeSysctls(data.sysctls) + } + validateSysctls(data.sysctls, hostNetworkMode, ipcMode) + validateUlimits(data.ulimits) + validateDevices(data.devices) + validateTmpfs(data.tmpfs) +} + +function serializeContainerColumns (spec) { + const data = spec || {} + const columns = {} + if (data.runAsGroup !== undefined) { + columns.runAsGroup = data.runAsGroup + } + if (data.readOnlyRootFilesystem !== undefined) { + columns.readOnlyRootFilesystem = data.readOnlyRootFilesystem + } + if (data.workingDir !== undefined) { + columns.workingDir = data.workingDir + } + if (data.cpus !== undefined) { + columns.cpus = data.cpus + } + if (data.memoryReservation !== undefined) { + columns.memoryReservation = data.memoryReservation + } + if (data.memorySwap !== undefined) { + columns.memorySwap = data.memorySwap + } + if (data.shmSize !== undefined) { + columns.shmSize = data.shmSize + } + if (data.sysctls !== undefined) { + const sysctls = normalizeSysctls(data.sysctls) + columns.sysctls = sysctls && Object.keys(sysctls).length > 0 + ? serializeJsonField(sysctls) + : null + } + return columns +} + +function containerFieldsRequireRebuild (spec, existing) { + if (!spec) { + return false + } + const scalarKeys = [ + 'runAsGroup', 'readOnlyRootFilesystem', 'workingDir', 'cpus', + 'memoryReservation', 'memorySwap', 'shmSize' + ] + for (const key of scalarKeys) { + if (spec[key] !== undefined && spec[key] !== existing[key]) { + return true + } + } + if (spec.entrypoint !== undefined) { + return true + } + if (resolveProcessArgv(spec) !== undefined) { + return true + } + if (spec.sysctls !== undefined || spec.ulimits !== undefined || + spec.devices !== undefined || spec.tmpfs !== undefined) { + return true + } + if (spec.models !== undefined && catalogRequiresRebuild(existing.models, normalizeCatalog(spec.models))) { + return true + } + if (spec.knowledge !== undefined && catalogRequiresRebuild(existing.knowledge, normalizeCatalog(spec.knowledge))) { + return true + } + return false +} + +function sortRowsById (rows) { + return Array.isArray(rows) ? rows.slice().sort((left, right) => (left.id || 0) - (right.id || 0)) : [] +} + +function tokensFromRows (rows, field) { + return sortRowsById(rows).map((row) => row[field]).filter((token) => typeof token === 'string') +} + +function devicesFromRows (rows) { + return sortRowsById(rows).map((row) => ({ + hostPath: row.hostPath, + containerPath: row.containerPath, + permissions: row.permissions || 'rwm' + })) +} + +function tmpfsFromRows (rows) { + return sortRowsById(rows).map((row) => { + const entry = { containerPath: row.containerPath } + if (row.size != null) { + entry.size = row.size + } + if (row.mode) { + entry.mode = row.mode + } + return entry + }) +} + +function ulimitsFromRows (rows) { + const ulimits = {} + for (const row of sortRowsById(rows)) { + ulimits[row.name] = { soft: row.soft, hard: row.hard } + } + return ulimits +} + +function catalogFromRows (catalogRow, itemRows) { + const items = sortRowsById(itemRows).map((row) => ({ name: row.name })) + if (!catalogRow && items.length === 0) { + return null + } + return { + bindPath: catalogRow ? catalogRow.bindPath : '', + permissions: (catalogRow && catalogRow.permissions) || 'ro', + items + } +} + +function hydrateJsonColumns (row) { + if (!row) { + return row + } + const hydrated = { ...row } + hydrated.sysctls = parseJsonField(row.sysctls) + return hydrated +} + +function applyAgentContainerFields (response, microservice, extras = {}) { + const cmdFromArgs = Array.isArray(extras) ? extras : extras.cmd + const entrypoint = extras.entrypoint || [] + if (entrypoint.length > 0) { + response.entrypoint = entrypoint + } + + const argv = extras.commands || (Array.isArray(cmdFromArgs) ? cmdFromArgs : []) + if (argv.length > 0) { + response.commands = argv + response.cmd = argv + } else if (Array.isArray(cmdFromArgs)) { + response.cmd = cmdFromArgs + } + + if (microservice.runAsGroup) { + response.runAsGroup = microservice.runAsGroup + } + if (microservice.readOnlyRootFilesystem) { + response.readOnlyRootFilesystem = true + } + if (microservice.workingDir) { + response.workingDir = microservice.workingDir + } + if (microservice.cpus != null) { + response.cpus = microservice.cpus + } + if (microservice.memoryReservation != null) { + response.memoryReservation = microservice.memoryReservation + } + if (microservice.memorySwap != null) { + response.memorySwap = microservice.memorySwap + } + if (microservice.shmSize != null) { + response.shmSize = microservice.shmSize + } + + const sysctls = parseJsonField(microservice.sysctls) + if (sysctls && typeof sysctls === 'object' && Object.keys(sysctls).length > 0) { + response.sysctls = sysctls + } + const ulimits = extras.ulimits + if (ulimits && typeof ulimits === 'object' && Object.keys(ulimits).length > 0) { + response.ulimits = ulimits + } + if (Array.isArray(extras.devices) && extras.devices.length > 0) { + response.devices = extras.devices + } + if (Array.isArray(extras.tmpfs) && extras.tmpfs.length > 0) { + response.tmpfs = extras.tmpfs + } + if (!isCatalogEmpty(extras.models)) { + response.models = extras.models + } + if (!isCatalogEmpty(extras.knowledge)) { + response.knowledge = extras.knowledge + } + + return response +} + +module.exports = { + parseJsonField, + serializeJsonField, + parseArgv, + serializeArgv, + resolveProcessArgv, + isCatalogEmpty, + parseCatalog, + normalizeCatalog, + serializeCatalog, + catalogItemNames, + catalogRequiresRebuild, + validateCatalog, + validateContainerFields, + serializeContainerColumns, + containerFieldsRequireRebuild, + hydrateJsonColumns, + applyAgentContainerFields, + normalizeSysctls, + tokensFromRows, + devicesFromRows, + tmpfsFromRows, + ulimitsFromRows, + catalogFromRows +} diff --git a/src/helpers/microservice-runtime-metrics.js b/src/helpers/microservice-runtime-metrics.js new file mode 100644 index 000000000..b0b809371 --- /dev/null +++ b/src/helpers/microservice-runtime-metrics.js @@ -0,0 +1,131 @@ +const LIVE_RUNTIME_STATUS = 'RUNNING' + +function isLiveRuntimeStatus (status) { + return status === LIVE_RUNTIME_STATUS +} + +function zeroRuntimeMetrics () { + return { + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + } +} + +function coerceRuntimeMetric (value) { + if (value === undefined || value === null || value === '') { + return 0 + } + const numeric = Number(value) + return Number.isFinite(numeric) ? numeric : 0 +} + +function applyRuntimeMetrics (patch, observedStatus) { + if (!patch) { + return patch + } + if (!isLiveRuntimeStatus(observedStatus)) { + Object.assign(patch, zeroRuntimeMetrics()) + } + return patch +} + +function _cloneStatusRow (statusRow) { + if (statusRow == null) { + return statusRow + } + if (typeof statusRow.get === 'function') { + return statusRow.get({ plain: true }) + } + if (statusRow.dataValues) { + return Object.assign({}, statusRow.dataValues) + } + return Object.assign({}, statusRow) +} + +function coerceCrashText (value) { + if (value === undefined || value === null) { + return undefined + } + if (typeof value === 'string') { + return value + } + if (typeof value === 'number' && Number.isFinite(value)) { + return String(value) + } + return undefined +} + +function coerceOptionalNonNegativeInt (value) { + if (value === undefined || value === null || value === '') { + return undefined + } + const numeric = Number(value) + if (!Number.isFinite(numeric) || numeric < 0) { + return undefined + } + return Math.floor(numeric) +} + +function applyCrashExtras (patch, reported) { + if (!patch || !reported) { + return patch + } + const lastErrorText = coerceCrashText(reported.lastError) + const lastErrorPresent = lastErrorText !== undefined && lastErrorText.length > 0 + const lastErrorAt = coerceOptionalNonNegativeInt(reported.lastErrorAt) + const lastErrorAtPresent = lastErrorAt !== undefined && lastErrorAt > 0 + const restartCountFieldSent = reported.restartCount !== undefined && + reported.restartCount !== null && reported.restartCount !== '' + const restartCount = coerceOptionalNonNegativeInt(reported.restartCount) + + if (lastErrorPresent) { + patch.lastError = lastErrorText + } + if (lastErrorAtPresent) { + patch.lastErrorAt = lastErrorAt + } + if (restartCountFieldSent) { + if (restartCount !== undefined) { + patch.restartCount = restartCount + } + } else if (lastErrorPresent || lastErrorAtPresent) { + patch.restartCount = 0 + } + return patch +} + +function projectCrashExtras (projected) { + if (!projected) { + return projected + } + if (projected.lastError == null) { + projected.lastError = '' + } + const lastErrorAt = Number(projected.lastErrorAt) + projected.lastErrorAt = Number.isFinite(lastErrorAt) ? lastErrorAt : 0 + const restartCount = Number(projected.restartCount) + projected.restartCount = Number.isFinite(restartCount) ? restartCount : 0 + return projected +} + +function projectStatusForApi (statusRow) { + const projected = _cloneStatusRow(statusRow) + if (!projected) { + return projected + } + applyRuntimeMetrics(projected, projected.status) + projectCrashExtras(projected) + return projected +} + +module.exports = { + isLiveRuntimeStatus, + zeroRuntimeMetrics, + coerceRuntimeMetric, + applyRuntimeMetrics, + applyCrashExtras, + projectCrashExtras, + projectStatusForApi +} diff --git a/src/helpers/system-naming.js b/src/helpers/system-naming.js index 3570ff601..c7a92b1d7 100644 --- a/src/helpers/system-naming.js +++ b/src/helpers/system-naming.js @@ -3,7 +3,7 @@ const Errors = require('./errors') const ApplicationManager = require('../data/managers/application-manager') const MicroserviceManager = require('../data/managers/microservice-manager') -const SYSTEM_MICROSERVICE_TYPES = ['router', 'hal', 'ble', 'debug', 'nats'] +const SYSTEM_MICROSERVICE_TYPES = ['router', 'debug', 'nats'] const getSystemAppName = (fogName) => `system-${fogName}` diff --git a/src/helpers/volume-mapping-scope.js b/src/helpers/volume-mapping-scope.js new file mode 100644 index 000000000..6030b64b3 --- /dev/null +++ b/src/helpers/volume-mapping-scope.js @@ -0,0 +1,86 @@ +'use strict' + +const path = require('path') + +const Errors = require('./errors') + +const PRIVATE_SCOPE = 'private' +const SHARED_SCOPE = 'shared' +const VOLUME_TYPE = 'volume' +const BIND_TYPE = 'bind' +const VOLUME_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9_.-]*$/ +const WINDOWS_ABS_HOST_PATH = /^[a-zA-Z]:[\\/].+/ + +const INVALID_VOLUME_NAME_MESSAGE = 'hostDestination includes invalid characters for a local volume name, only ' + + '"[a-zA-Z0-9][a-zA-Z0-9_.-]" are allowed. If you intended to pass a host directory, use type: bind' + +const INVALID_BIND_HOST_PATH_MESSAGE = + 'hostDestination must be an absolute host path when type is bind' + +function isValidBindHostDestination (hostDestination) { + if (typeof hostDestination !== 'string') { + return false + } + const trimmed = hostDestination.trim() + if (trimmed === '') { + return false + } + return path.isAbsolute(trimmed) || WINDOWS_ABS_HOST_PATH.test(trimmed) +} + +function applyVolumeMappingScope (mapping, options = {}) { + if (!mapping) { + return mapping + } + + if (mapping.type === BIND_TYPE) { + if (!isValidBindHostDestination(mapping.hostDestination)) { + throw new Errors.InvalidArgumentError(INVALID_BIND_HOST_PATH_MESSAGE) + } + mapping.scope = PRIVATE_SCOPE + return mapping + } + + if (mapping.type !== VOLUME_TYPE) { + mapping.scope = PRIVATE_SCOPE + return mapping + } + + if (typeof mapping.hostDestination !== 'string' || !VOLUME_NAME_PATTERN.test(mapping.hostDestination)) { + throw new Errors.InvalidArgumentError(INVALID_VOLUME_NAME_MESSAGE) + } + + const raw = mapping.scope + if (raw == null) { + mapping.scope = PRIVATE_SCOPE + } else if (typeof raw !== 'string') { + throw new Errors.ValidationError(`Unknown volume mapping scope '${raw}'`) + } else { + const trimmed = raw.trim() + if (trimmed === '') { + mapping.scope = PRIVATE_SCOPE + } else { + const normalized = trimmed.toLowerCase() + if (normalized !== PRIVATE_SCOPE && normalized !== SHARED_SCOPE) { + throw new Errors.ValidationError(`Unknown volume mapping scope '${raw}'`) + } + mapping.scope = normalized + } + } + + if (options.rejectShared && mapping.scope === SHARED_SCOPE) { + throw new Errors.ValidationError( + 'Shared volume mapping scope is not allowed on controller or system microservices' + ) + } + + return mapping +} + +module.exports = { + applyVolumeMappingScope, + isValidBindHostDestination, + PRIVATE_SCOPE, + SHARED_SCOPE, + VOLUME_NAME_PATTERN +} diff --git a/src/jobs/fog-status-job.js b/src/jobs/fog-status-job.js index 118c01431..d42b896a2 100644 --- a/src/jobs/fog-status-job.js +++ b/src/jobs/fog-status-job.js @@ -1,102 +1,161 @@ +const { Op } = require('sequelize') const TransactionDecorator = require('../decorators/transaction-decorator') const { PRIORITY_BACKGROUND } = require('../helpers/transaction-runner') const FogManager = require('../data/managers/iofog-manager') +const ApplicationManager = require('../data/managers/application-manager') const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') -const MicroserviceService = require('../services/microservices-service') const { microserviceState, microserviceExecState } = require('../enums/microservice-state') const FogStates = require('../enums/fog-state') +const { zeroRuntimeMetrics } = require('../helpers/microservice-runtime-metrics') const Config = require('../config') const logger = require('../logger') +const MIN_FOG_STATUS_FREQUENCY_SEC = 10 +const LIVENESS_DAEMON_STATUSES = [FogStates.RUNNING, FogStates.WARNING] const scheduleTime = Config.get('settings.fogStatusUpdateInterval') * 1000 +let inFlight = false + +function clampedStatusFrequencySec (statusFrequency) { + const numeric = Number(statusFrequency) + const freqSec = Number.isFinite(numeric) ? numeric : 0 + return Math.max(freqSec, MIN_FOG_STATUS_FREQUENCY_SEC) +} + +function isFogStatusStale (fog, tolerance, nowMs) { + const deadlineMs = fog.lastStatusTime + clampedStatusFrequencySec(fog.statusFrequency) * 1000 * tolerance + return nowMs > deadlineMs +} + +function isDesiredInactive (microservice, application) { + return microservice.isActivated === false || (application && application.isActivated === false) +} + +function shouldMarkUnknownOnQuietFog (microservice, application) { + if (!microservice.microserviceStatus) { + return false + } + if (isDesiredInactive(microservice, application)) { + return false + } + return true +} + async function run () { + if (inFlight) { + return + } + inFlight = true try { - const _updateFogsConnectionStatus = TransactionDecorator.generateTransaction( - updateFogsConnectionStatus, - { priority: PRIORITY_BACKGROUND, label: 'fogStatus.updateConnection' } - ) - await _updateFogsConnectionStatus() + await module.exports.runLivenessPass() } catch (error) { logger.error('Error during fog status update:', error) } finally { + inFlight = false setTimeout(run, scheduleTime) } } -async function updateFogsConnectionStatus (transaction) { - const unknownFogUuids = await _updateFogStatus(transaction) - const microservices = await _updateMicroserviceStatus(unknownFogUuids, transaction) - await _deleteNotRunningMicroservices(microservices, transaction) -} +async function runLivenessPass () { + const chunkSize = Math.max(1, Number(Config.get('settings.fogStatusLivenessChunkSize', 50)) || 50) + const tolerance = Config.get('settings.fogStatusUpdateTolerance') + const processChunk = TransactionDecorator.generateTransaction( + processLivenessChunk, + { priority: PRIORITY_BACKGROUND, label: 'fogStatus.updateConnection' } + ) -async function _updateFogStatus (transaction) { - const statusUpdateTolerance = Config.get('settings.fogStatusUpdateTolerance') - const fogs = [ - ...await FogManager.findAll({ daemonStatus: FogStates.RUNNING }, transaction), - ...await FogManager.findAll({ daemonStatus: FogStates.WARNING }, transaction) - ] - const unknownFogUuids = fogs - .filter((fog) => { - const statusUpdateToleranceMs = fog.statusFrequency * 1000 * statusUpdateTolerance - return (Date.now() - fog.lastStatusTime) > statusUpdateToleranceMs - }) - .map((fog) => fog.uuid) - - const where = { uuid: unknownFogUuids } - const data = { daemonStatus: FogStates.UNKNOWN } - await FogManager.update(where, data, transaction) - return unknownFogUuids + let afterUuid = null + while (true) { + const result = await processChunk(afterUuid, chunkSize, tolerance) + if (!result || !result.nextCursor) { + break + } + afterUuid = result.nextCursor + } } -async function _updateMicroserviceStatus (unknownFogUuids, transaction) { - const microservices = await MicroserviceManager.findAllWithStatuses({ iofogUuid: unknownFogUuids }, transaction) - - // Filter out inactive microservices that are already STOPPED - they should not be updated to UNKNOWN - const microserviceStatusIds = microservices - .filter((microservice) => { - // Skip if no microservice status - if (!microservice.microserviceStatus) { - return false - } - // Skip if microservice is inactive and already STOPPED - if (!microservice.isActivated && microservice.microserviceStatus.status === microserviceState.STOPPED) { - return false - } - return true - }) - .map((microservice) => microservice.microserviceStatus.id) - - const microserviceExecStatusIds = microservices - .filter((microservice) => { - // Skip if no exec status - if (!microservice.microserviceExecStatus) { - return false - } - // Skip if microservice is inactive and already STOPPED - if (!microservice.isActivated && microservice.microserviceStatus && microservice.microserviceStatus.status === microserviceState.STOPPED) { - return false - } - return true - }) - .map((microservice) => microservice.microserviceExecStatus.id) - - await MicroserviceStatusManager.update({ id: microserviceStatusIds }, { status: microserviceState.UNKNOWN }, transaction) - await MicroserviceExecStatusManager.update({ id: microserviceExecStatusIds }, { execSesssionId: '', status: microserviceExecState.INACTIVE }, transaction) - return microservices +async function processLivenessChunk (afterUuid, chunkSize, tolerance, transaction) { + const nowMs = Date.now() + const heardBefore = nowMs - MIN_FOG_STATUS_FREQUENCY_SEC * 1000 * tolerance + const candidates = await FogManager.findStatusLivenessCandidates({ + afterUuid, + limit: chunkSize, + heardBefore, + daemonStatuses: LIVENESS_DAEMON_STATUSES + }, transaction) + + if (!candidates.length) { + return { nextCursor: null, staleCount: 0 } + } + + const nextCursor = candidates[candidates.length - 1].uuid + const staleFogs = candidates.filter((fog) => isFogStatusStale(fog, tolerance, nowMs)) + if (!staleFogs.length) { + return { nextCursor, staleCount: 0 } + } + + const unknownFogUuids = staleFogs.map((fog) => fog.uuid) + await FogManager.update({ uuid: unknownFogUuids }, { daemonStatus: FogStates.UNKNOWN }, transaction) + await markDesiredActiveUnknown(unknownFogUuids, transaction) + return { nextCursor, staleCount: staleFogs.length } } -async function _deleteNotRunningMicroservices (microservices, transaction) { +async function markDesiredActiveUnknown (unknownFogUuids, transaction) { + if (!unknownFogUuids.length) { + return + } + + const microservices = await MicroserviceManager.findAllWithStatuses({ + iofogUuid: { [Op.in]: unknownFogUuids } + }, transaction) + if (!microservices.length) { + return + } + + const applicationIds = [...new Set(microservices.map((ms) => ms.applicationId).filter((id) => id != null))] + const applications = applicationIds.length + ? await ApplicationManager.findAll({ id: applicationIds }, transaction) + : [] + const applicationById = new Map((applications || []).map((app) => [app.id, app])) + + const statusIds = [] + const execIds = [] for (const microservice of microservices) { - if (microservice.delete) { - await MicroserviceService.deleteMicroserviceWithRoutesAndPortMappings(microservice, transaction) + const application = applicationById.get(microservice.applicationId) + if (!shouldMarkUnknownOnQuietFog(microservice, application)) { + continue } + statusIds.push(microservice.microserviceStatus.id) + if (microservice.microserviceExecStatus) { + execIds.push(microservice.microserviceExecStatus.id) + } + } + + if (statusIds.length) { + await MicroserviceStatusManager.update( + { id: statusIds }, + Object.assign({ status: microserviceState.UNKNOWN }, zeroRuntimeMetrics()), + transaction + ) + } + if (execIds.length) { + await MicroserviceExecStatusManager.update( + { id: execIds }, + { execSessionId: '', status: microserviceExecState.INACTIVE }, + transaction + ) } } module.exports = { - run + run, + runLivenessPass, + processLivenessChunk, + isFogStatusStale, + shouldMarkUnknownOnQuietFog, + clampedStatusFrequencySec, + MIN_FOG_STATUS_FREQUENCY_SEC } diff --git a/src/jobs/stopped-app-status-job.js b/src/jobs/stopped-app-status-job.js index 00d752c75..7346464bd 100644 --- a/src/jobs/stopped-app-status-job.js +++ b/src/jobs/stopped-app-status-job.js @@ -1,3 +1,4 @@ +const { Op } = require('sequelize') const TransactionDecorator = require('../decorators/transaction-decorator') const { PRIORITY_BACKGROUND } = require('../helpers/transaction-runner') @@ -5,6 +6,7 @@ const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') const { microserviceState, microserviceExecState } = require('../enums/microservice-state') +const { zeroRuntimeMetrics } = require('../helpers/microservice-runtime-metrics') const Config = require('../config') const ApplicationManager = require('../data/managers/application-manager') @@ -12,55 +14,68 @@ const logger = require('../logger') const scheduleTime = Config.get('settings.fogStatusUpdateInterval') * 1000 +const STALE_DESIRED_INACTIVE_STATUSES = new Set([ + microserviceState.DELETED, + microserviceState.DELETING, + microserviceState.RUNNING, + microserviceState.STOPPING +]) + +let inFlight = false + +function shouldForceObservedStopped (microservice) { + const status = microservice.microserviceStatus && microservice.microserviceStatus.status + return STALE_DESIRED_INACTIVE_STATUSES.has(status) +} + async function run () { + if (inFlight) { + return + } + inFlight = true try { - const _updateStoppedApplicationMicroserviceStatus = TransactionDecorator.generateTransaction( - updateStoppedApplicationMicroserviceStatus, - { priority: PRIORITY_BACKGROUND, label: 'stoppedAppStatus.application' } - ) - const _updateStoppedMicroserviceStatus = TransactionDecorator.generateTransaction( - updateStoppedMicroserviceStatus, - { priority: PRIORITY_BACKGROUND, label: 'stoppedAppStatus.microservice' } - ) - - // Handle microservices from deactivated applications - await _updateStoppedApplicationMicroserviceStatus() - // Handle individually deactivated microservices - await _updateStoppedMicroserviceStatus() + await module.exports.runSafetyNetPass() } catch (error) { logger.error('Error during stopped application status update:', error) } finally { + inFlight = false setTimeout(run, scheduleTime) } } -async function updateStoppedApplicationMicroserviceStatus (transaction) { - // Get all deactivated applications +async function runSafetyNetPass () { + const updateStoppedApplicationMicroserviceStatus = TransactionDecorator.generateTransaction( + updateApplicationMicroservices, + { priority: PRIORITY_BACKGROUND, label: 'stoppedAppStatus.application' } + ) + const updateStoppedMicroserviceStatus = TransactionDecorator.generateTransaction( + updateDeactivatedMicroservices, + { priority: PRIORITY_BACKGROUND, label: 'stoppedAppStatus.microservice' } + ) + + await updateStoppedApplicationMicroserviceStatus() + await updateStoppedMicroserviceStatus() +} + +async function updateApplicationMicroservices (transaction) { const stoppedApplications = await ApplicationManager.findAllWithAttributes({ isActivated: false }, ['id'], transaction) if (stoppedApplications.length === 0) { return } - // Get all microservices from these applications const applicationIds = stoppedApplications.map(app => app.id) - const { Op } = require('sequelize') const stoppedMicroservices = await MicroserviceManager.findAllWithStatuses({ applicationId: { [Op.in]: applicationIds } }, transaction) - await _updateMicroserviceStatusStopped(stoppedMicroservices, transaction) + await updateMicroserviceStatusStopped(stoppedMicroservices, transaction) } -async function updateStoppedMicroserviceStatus (transaction) { - // Get all individually deactivated microservices (where microservice isActivated = false but parent application is still active) - const { Op } = require('sequelize') - - // First get all active applications +async function updateDeactivatedMicroservices (transaction) { const activeApplications = await ApplicationManager.findAllWithAttributes({ isActivated: true }, ['id'], transaction) if (activeApplications.length === 0) { return } - // Then get microservices that are individually deactivated but belong to active applications const activeApplicationIds = activeApplications.map(app => app.id) const stoppedMicroservices = await MicroserviceManager.findAllWithStatuses({ isActivated: false, @@ -71,27 +86,38 @@ async function updateStoppedMicroserviceStatus (transaction) { return } - await _updateMicroserviceStatusStopped(stoppedMicroservices, transaction) + await updateMicroserviceStatusStopped(stoppedMicroservices, transaction) } -async function _updateMicroserviceStatusStopped (stoppedMicroservices, transaction) { - const microserviceStatusIds = stoppedMicroservices - .filter((microservice) => microservice.microserviceStatus && (microservice.microserviceStatus.status === microserviceState.DELETED || - microservice.microserviceStatus.status === microserviceState.DELETING)) - .map((microservice) => microservice.microserviceStatus.id) - const microserviceExecStatusIds = stoppedMicroservices - .filter((microservice) => - microservice.microserviceStatus && - (microservice.microserviceStatus.status === microserviceState.DELETED || - microservice.microserviceStatus.status === microserviceState.DELETING) && - microservice.microserviceExecStatus - ) +async function updateMicroserviceStatusStopped (stoppedMicroservices, transaction) { + const toStop = (stoppedMicroservices || []).filter(shouldForceObservedStopped) + const microserviceStatusIds = toStop.map((microservice) => microservice.microserviceStatus.id) + const microserviceExecStatusIds = toStop + .filter((microservice) => microservice.microserviceExecStatus) .map((microservice) => microservice.microserviceExecStatus.id) - await MicroserviceStatusManager.update({ id: microserviceStatusIds }, { status: microserviceState.STOPPED }, transaction) - await MicroserviceExecStatusManager.update({ id: microserviceExecStatusIds }, { execSesssionId: '', status: microserviceExecState.INACTIVE }, transaction) - return stoppedMicroservices + + if (microserviceStatusIds.length) { + await MicroserviceStatusManager.update( + { id: microserviceStatusIds }, + Object.assign({ status: microserviceState.STOPPED }, zeroRuntimeMetrics()), + transaction + ) + } + if (microserviceExecStatusIds.length) { + await MicroserviceExecStatusManager.update( + { id: microserviceExecStatusIds }, + { execSessionId: '', status: microserviceExecState.INACTIVE }, + transaction + ) + } + return toStop } module.exports = { - run + run, + runSafetyNetPass, + updateApplicationMicroservices, + updateDeactivatedMicroservices, + updateMicroserviceStatusStopped, + shouldForceObservedStopped } diff --git a/src/routes/agent.js b/src/routes/agent.js index e78340dfa..2d438d18f 100644 --- a/src/routes/agent.js +++ b/src/routes/agent.js @@ -300,7 +300,7 @@ module.exports = [ }, { method: 'get', - path: '/api/v3/agent/microservices', + path: '/api/v3/agent/models', middleware: async (req, res) => { logger.apiReq(req) @@ -316,9 +316,9 @@ module.exports = [ } ] - const getAgentMicroservicesEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentMicroservicesEndPoint, + const getAgentLinkedModelsEndpoint = ResponseDecorator.handleErrors(AgentController.getAgentLinkedModelsEndpoint, successCode, errorCodes) - const responseObject = await getAgentMicroservicesEndPoint(req) + const responseObject = await getAgentLinkedModelsEndpoint(req) res .status(responseObject.code) @@ -329,16 +329,12 @@ module.exports = [ }, { method: 'get', - path: '/api/v3/agent/microservices/:microserviceUuid', + path: '/api/v3/agent/knowledge', middleware: async (req, res) => { logger.apiReq(req) const successCode = constants.HTTP_CODE_SUCCESS const errorCodes = [ - { - code: constants.HTTP_CODE_NOT_FOUND, - errors: [Errors.NotFoundError] - }, { code: constants.HTTP_CODE_UNAUTHORIZED, errors: [Errors.AgentAuthenticationError, Errors.AuthenticationError] @@ -349,9 +345,9 @@ module.exports = [ } ] - const getAgentMicroserviceEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentMicroserviceEndPoint, + const getAgentLinkedKnowledgeEndpoint = ResponseDecorator.handleErrors(AgentController.getAgentLinkedKnowledgeEndpoint, successCode, errorCodes) - const responseObject = await getAgentMicroserviceEndPoint(req) + const responseObject = await getAgentLinkedKnowledgeEndpoint(req) res .status(responseObject.code) @@ -361,8 +357,8 @@ module.exports = [ } }, { - method: 'post', - path: '/api/v3/agent/controller/register', + method: 'get', + path: '/api/v3/agent/runtimeClasses', middleware: async (req, res) => { logger.apiReq(req) @@ -375,23 +371,15 @@ module.exports = [ { code: constants.HTTP_CODE_SERVICE_UNAVAILABLE, errors: [Errors.ServiceUnavailableError] - }, - { - code: constants.HTTP_CODE_FORBIDDEN, - errors: [Errors.ForbiddenError] - }, - { - code: constants.HTTP_CODE_BAD_REQUEST, - errors: [Errors.ValidationError] } ] - const registerControllerMicroserviceEndPoint = ResponseDecorator.handleErrors( - AgentController.registerControllerMicroserviceEndPoint, + const getAgentLinkedRuntimeClassesEndpoint = ResponseDecorator.handleErrors( + AgentController.getAgentLinkedRuntimeClassesEndpoint, successCode, errorCodes ) - const responseObject = await registerControllerMicroserviceEndPoint(req) + const responseObject = await getAgentLinkedRuntimeClassesEndpoint(req) res .status(responseObject.code) @@ -402,7 +390,7 @@ module.exports = [ }, { method: 'get', - path: '/api/v3/agent/registries', + path: '/api/v3/agent/microservices', middleware: async (req, res) => { logger.apiReq(req) @@ -418,9 +406,9 @@ module.exports = [ } ] - const getAgentRegistriesEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentRegistriesEndPoint, + const getAgentMicroservicesEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentMicroservicesEndPoint, successCode, errorCodes) - const responseObject = await getAgentRegistriesEndPoint(req) + const responseObject = await getAgentMicroservicesEndPoint(req) res .status(responseObject.code) @@ -431,7 +419,7 @@ module.exports = [ }, { method: 'get', - path: '/api/v3/agent/tunnel', + path: '/api/v3/agent/microservices/:microserviceUuid', middleware: async (req, res) => { logger.apiReq(req) @@ -451,9 +439,9 @@ module.exports = [ } ] - const getAgentTunnelEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentTunnelEndPoint, + const getAgentMicroserviceEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentMicroserviceEndPoint, successCode, errorCodes) - const responseObject = await getAgentTunnelEndPoint(req) + const responseObject = await getAgentMicroserviceEndPoint(req) res .status(responseObject.code) @@ -463,8 +451,8 @@ module.exports = [ } }, { - method: 'get', - path: '/api/v3/agent/version', + method: 'post', + path: '/api/v3/agent/controller/register', middleware: async (req, res) => { logger.apiReq(req) @@ -479,14 +467,21 @@ module.exports = [ errors: [Errors.ServiceUnavailableError] }, { - code: constants.HTTP_CODE_NOT_FOUND, - errors: [Errors.NotFoundError] + code: constants.HTTP_CODE_FORBIDDEN, + errors: [Errors.ForbiddenError] + }, + { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] } ] - const getAgentChangeVersionCommandEndPoint = ResponseDecorator.handleErrors( - AgentController.getAgentChangeVersionCommandEndPoint, successCode, errorCodes) - const responseObject = await getAgentChangeVersionCommandEndPoint(req) + const registerControllerMicroserviceEndPoint = ResponseDecorator.handleErrors( + AgentController.registerControllerMicroserviceEndPoint, + successCode, + errorCodes + ) + const responseObject = await registerControllerMicroserviceEndPoint(req) res .status(responseObject.code) @@ -496,12 +491,12 @@ module.exports = [ } }, { - method: 'put', - path: '/api/v3/agent/hal/hw', + method: 'get', + path: '/api/v3/agent/registries', middleware: async (req, res) => { logger.apiReq(req) - const successCode = constants.HTTP_CODE_NO_CONTENT + const successCode = constants.HTTP_CODE_SUCCESS const errorCodes = [ { code: constants.HTTP_CODE_UNAUTHORIZED, @@ -510,16 +505,45 @@ module.exports = [ { code: constants.HTTP_CODE_SERVICE_UNAVAILABLE, errors: [Errors.ServiceUnavailableError] + } + ] + + const getAgentRegistriesEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentRegistriesEndPoint, + successCode, errorCodes) + const responseObject = await getAgentRegistriesEndPoint(req) + + res + .status(responseObject.code) + .send(responseObject.body) + + logger.apiRes({ req, res, responseObject }) + } + }, + { + method: 'get', + path: '/api/v3/agent/tunnel', + middleware: async (req, res) => { + logger.apiReq(req) + + const successCode = constants.HTTP_CODE_SUCCESS + const errorCodes = [ + { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] }, { - code: constants.HTTP_CODE_BAD_REQUEST, - errors: [Errors.ValidationError] + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AgentAuthenticationError, Errors.AuthenticationError] + }, + { + code: constants.HTTP_CODE_SERVICE_UNAVAILABLE, + errors: [Errors.ServiceUnavailableError] } ] - const updateHalHardwareInfoEndPoint = ResponseDecorator.handleErrors(AgentController.updateHalHardwareInfoEndPoint, + const getAgentTunnelEndPoint = ResponseDecorator.handleErrors(AgentController.getAgentTunnelEndPoint, successCode, errorCodes) - const responseObject = await updateHalHardwareInfoEndPoint(req) + const responseObject = await getAgentTunnelEndPoint(req) res .status(responseObject.code) @@ -529,12 +553,12 @@ module.exports = [ } }, { - method: 'put', - path: '/api/v3/agent/hal/usb', + method: 'get', + path: '/api/v3/agent/version', middleware: async (req, res) => { logger.apiReq(req) - const successCode = constants.HTTP_CODE_NO_CONTENT + const successCode = constants.HTTP_CODE_SUCCESS const errorCodes = [ { code: constants.HTTP_CODE_UNAUTHORIZED, @@ -545,14 +569,14 @@ module.exports = [ errors: [Errors.ServiceUnavailableError] }, { - code: constants.HTTP_CODE_BAD_REQUEST, - errors: [Errors.ValidationError] + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] } ] - const updateHalUsbInfoEndPoint = ResponseDecorator.handleErrors(AgentController.updateHalUsbInfoEndPoint, - successCode, errorCodes) - const responseObject = await updateHalUsbInfoEndPoint(req) + const getAgentChangeVersionCommandEndPoint = ResponseDecorator.handleErrors( + AgentController.getAgentChangeVersionCommandEndPoint, successCode, errorCodes) + const responseObject = await getAgentChangeVersionCommandEndPoint(req) res .status(responseObject.code) diff --git a/src/routes/iofog.js b/src/routes/iofog.js index dce4c412c..3dd4e9d1d 100644 --- a/src/routes/iofog.js +++ b/src/routes/iofog.js @@ -302,67 +302,6 @@ module.exports = [ }) } }, - { - method: 'get', - path: '/api/v3/iofog/:uuid/hal/hw', - middleware: async (req, res) => { - logger.apiReq(req) - - const successCode = constants.HTTP_CODE_SUCCESS - const errCodes = [ - { - code: 401, - errors: [Errors.AuthenticationError] - }, - { - code: 404, - errors: [Errors.NotFoundError] - } - ] - - await rbacMiddleware.protect()(req, res, async () => { - const getHalHardwareInfo = ResponseDecorator.handleErrors(FogController.getHalHardwareInfoEndPoint, - successCode, errCodes) - const responseObject = await getHalHardwareInfo(req) - const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' - res - .status(responseObject.code) - .send(responseObject.body) - - logger.apiRes({ req, user, res, responseObject }) - }) - } - }, - { - method: 'get', - path: '/api/v3/iofog/:uuid/hal/usb', - middleware: async (req, res) => { - logger.apiReq(req) - - const successCode = constants.HTTP_CODE_SUCCESS - const errCodes = [ - { - code: 401, - errors: [Errors.AuthenticationError] - }, - { - code: 404, - errors: [Errors.NotFoundError] - } - ] - - await rbacMiddleware.protect()(req, res, async () => { - const getHalUsbInfo = ResponseDecorator.handleErrors(FogController.getHalUsbInfoEndPoint, successCode, errCodes) - const responseObject = await getHalUsbInfo(req) - const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' - res - .status(responseObject.code) - .send(responseObject.body) - - logger.apiRes({ req, user, res, responseObject }) - }) - } - }, { method: 'post', path: '/api/v3/iofog/:uuid/prune', diff --git a/src/routes/knowledge.js b/src/routes/knowledge.js new file mode 100644 index 000000000..6c0c31ab0 --- /dev/null +++ b/src/routes/knowledge.js @@ -0,0 +1,187 @@ +const constants = require('../helpers/constants') +const KnowledgeController = require('../controllers/knowledge-controller') +const ResponseDecorator = require('../decorators/response-decorator') +const logger = require('../logger') +const Errors = require('../helpers/errors') +const rbacMiddleware = require('../lib/rbac/middleware') + +const unauthorized = { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] +} + +const notFound = { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] +} + +const badRequest = { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] +} + +const conflict = { + code: constants.HTTP_CODE_CONFLICT, + errors: [Errors.ConflictError] +} + +function preferredUsername (req) { + return req.kauth && req.kauth.grant && req.kauth.grant.access_token + ? req.kauth.grant.access_token.content.preferred_username + : 'system' +} + +async function sendHandled (req, res, handler, successCode, errorCodes) { + const endpoint = ResponseDecorator.handleErrors(handler, successCode, errorCodes) + const responseObject = await endpoint(req) + const user = preferredUsername(req) + res + .status(responseObject.code) + .send(responseObject.body) + logger.apiRes({ req, user, res, responseObject }) +} + +module.exports = [ + { + method: 'get', + path: '/api/v3/knowledge', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.listKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [unauthorized]) + }) + } + }, + { + method: 'get', + path: '/api/v3/knowledge/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.getKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound + ]) + }) + } + }, + { + method: 'patch', + path: '/api/v3/knowledge/:name', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.updateKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/knowledge/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.deleteKnowledgeEndpoint, constants.HTTP_CODE_ACCEPTED, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/knowledge', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.createKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/knowledge/yaml', + fileInput: 'knowledge', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.createKnowledgeYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'put', + path: '/api/v3/knowledge/yaml/:name', + fileInput: 'knowledge', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.upsertKnowledgeYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'get', + path: '/api/v3/knowledge/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.getKnowledgeLinkEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/knowledge/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.linkKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/knowledge/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, KnowledgeController.unlinkKnowledgeEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + } +] diff --git a/src/routes/microserviceTemplates.js b/src/routes/microserviceTemplates.js new file mode 100644 index 000000000..1a97d6b0c --- /dev/null +++ b/src/routes/microserviceTemplates.js @@ -0,0 +1,140 @@ +const constants = require('../helpers/constants') +const MicroserviceTemplateController = require('../controllers/microservice-template-controller') +const ResponseDecorator = require('../decorators/response-decorator') +const logger = require('../logger') +const Errors = require('../helpers/errors') +const rbacMiddleware = require('../lib/rbac/middleware') + +const unauthorized = { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] +} + +const notFound = { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] +} + +const badRequest = { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] +} + +const conflict = { + code: constants.HTTP_CODE_CONFLICT, + errors: [Errors.ConflictError] +} + +function preferredUsername (req) { + return req.kauth && req.kauth.grant && req.kauth.grant.access_token + ? req.kauth.grant.access_token.content.preferred_username + : 'system' +} + +async function sendHandled (req, res, handler, successCode, errorCodes) { + const endpoint = ResponseDecorator.handleErrors(handler, successCode, errorCodes) + const responseObject = await endpoint(req) + const user = preferredUsername(req) + res + .status(responseObject.code) + .send(responseObject.body) + logger.apiRes({ req, user, res, responseObject }) +} + +module.exports = [ + { + method: 'get', + path: '/api/v3/microserviceTemplates', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.listMicroserviceTemplatesEndpoint, constants.HTTP_CODE_SUCCESS, [unauthorized]) + }) + } + }, + { + method: 'get', + path: '/api/v3/microserviceTemplates/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.getMicroserviceTemplateEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound + ]) + }) + } + }, + { + method: 'patch', + path: '/api/v3/microserviceTemplates/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.updateMicroserviceTemplateEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/microserviceTemplates/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.deleteMicroserviceTemplateEndpoint, constants.HTTP_CODE_ACCEPTED, [ + unauthorized, + notFound + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/microserviceTemplates', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.createMicroserviceTemplateEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/microserviceTemplates/yaml', + fileInput: 'template', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.createMicroserviceTemplateYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'put', + path: '/api/v3/microserviceTemplates/yaml/:name', + fileInput: 'template', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, MicroserviceTemplateController.upsertMicroserviceTemplateYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + } +] diff --git a/src/routes/microservices.js b/src/routes/microservices.js index c9aa31720..d45a68d8f 100644 --- a/src/routes/microservices.js +++ b/src/routes/microservices.js @@ -261,6 +261,78 @@ module.exports = [ }) } }, + { + method: 'patch', + path: '/api/v3/microservices/:uuid/models', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + + const successCode = constants.HTTP_CODE_NO_CONTENT + const errorCodes = [ + { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] + }, + { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] + }, + { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] + } + ] + + await rbacMiddleware.protect()(req, res, async () => { + const updateMicroserviceCatalogEndPoint = ResponseDecorator.handleErrors(MicroservicesController.updateMicroserviceCatalogEndPoint, + successCode, errorCodes) + const responseObject = await updateMicroserviceCatalogEndPoint(req) + const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' + res + .status(responseObject.code) + .send(responseObject.body) + + logger.apiRes({ req, user, res, responseObject }) + }) + } + }, + { + method: 'patch', + path: '/api/v3/microservices/:uuid/knowledge', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + + const successCode = constants.HTTP_CODE_NO_CONTENT + const errorCodes = [ + { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] + }, + { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] + }, + { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] + } + ] + + await rbacMiddleware.protect()(req, res, async () => { + const updateMicroserviceKnowledgeEndPoint = ResponseDecorator.handleErrors(MicroservicesController.updateMicroserviceKnowledgeEndPoint, + successCode, errorCodes) + const responseObject = await updateMicroserviceKnowledgeEndPoint(req) + const user = req.kauth && req.kauth.grant && req.kauth.grant.access_token ? req.kauth.grant.access_token.content.preferred_username : 'system' + res + .status(responseObject.code) + .send(responseObject.body) + + logger.apiRes({ req, user, res, responseObject }) + }) + } + }, { method: 'patch', path: '/api/v3/microservices/:uuid/rebuild', diff --git a/src/routes/models.js b/src/routes/models.js new file mode 100644 index 000000000..acc4ab32b --- /dev/null +++ b/src/routes/models.js @@ -0,0 +1,187 @@ +const constants = require('../helpers/constants') +const ModelController = require('../controllers/model-controller') +const ResponseDecorator = require('../decorators/response-decorator') +const logger = require('../logger') +const Errors = require('../helpers/errors') +const rbacMiddleware = require('../lib/rbac/middleware') + +const unauthorized = { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] +} + +const notFound = { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] +} + +const badRequest = { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] +} + +const conflict = { + code: constants.HTTP_CODE_CONFLICT, + errors: [Errors.ConflictError] +} + +function preferredUsername (req) { + return req.kauth && req.kauth.grant && req.kauth.grant.access_token + ? req.kauth.grant.access_token.content.preferred_username + : 'system' +} + +async function sendHandled (req, res, handler, successCode, errorCodes) { + const endpoint = ResponseDecorator.handleErrors(handler, successCode, errorCodes) + const responseObject = await endpoint(req) + const user = preferredUsername(req) + res + .status(responseObject.code) + .send(responseObject.body) + logger.apiRes({ req, user, res, responseObject }) +} + +module.exports = [ + { + method: 'get', + path: '/api/v3/models', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.listModelsEndpoint, constants.HTTP_CODE_SUCCESS, [unauthorized]) + }) + } + }, + { + method: 'get', + path: '/api/v3/models/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.getModelEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound + ]) + }) + } + }, + { + method: 'patch', + path: '/api/v3/models/:name', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.updateModelEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/models/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.deleteModelEndpoint, constants.HTTP_CODE_ACCEPTED, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/models', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.createModelEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/models/yaml', + fileInput: 'model', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.createModelYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'put', + path: '/api/v3/models/yaml/:name', + fileInput: 'model', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.upsertModelYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'get', + path: '/api/v3/models/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.getModelLinkEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/models/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.linkModelEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/models/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, ModelController.unlinkModelEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + } +] diff --git a/src/routes/runtimeClasses.js b/src/routes/runtimeClasses.js new file mode 100644 index 000000000..cc070ed7b --- /dev/null +++ b/src/routes/runtimeClasses.js @@ -0,0 +1,187 @@ +const constants = require('../helpers/constants') +const RuntimeClassController = require('../controllers/runtime-class-controller') +const ResponseDecorator = require('../decorators/response-decorator') +const logger = require('../logger') +const Errors = require('../helpers/errors') +const rbacMiddleware = require('../lib/rbac/middleware') + +const unauthorized = { + code: constants.HTTP_CODE_UNAUTHORIZED, + errors: [Errors.AuthenticationError] +} + +const notFound = { + code: constants.HTTP_CODE_NOT_FOUND, + errors: [Errors.NotFoundError] +} + +const badRequest = { + code: constants.HTTP_CODE_BAD_REQUEST, + errors: [Errors.ValidationError] +} + +const conflict = { + code: constants.HTTP_CODE_CONFLICT, + errors: [Errors.ConflictError] +} + +function preferredUsername (req) { + return req.kauth && req.kauth.grant && req.kauth.grant.access_token + ? req.kauth.grant.access_token.content.preferred_username + : 'system' +} + +async function sendHandled (req, res, handler, successCode, errorCodes) { + const endpoint = ResponseDecorator.handleErrors(handler, successCode, errorCodes) + const responseObject = await endpoint(req) + const user = preferredUsername(req) + res + .status(responseObject.code) + .send(responseObject.body) + logger.apiRes({ req, user, res, responseObject }) +} + +module.exports = [ + { + method: 'get', + path: '/api/v3/runtimeClasses', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.listRuntimeClassesEndpoint, constants.HTTP_CODE_SUCCESS, [unauthorized]) + }) + } + }, + { + method: 'get', + path: '/api/v3/runtimeClasses/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.getRuntimeClassEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound + ]) + }) + } + }, + { + method: 'patch', + path: '/api/v3/runtimeClasses/:name', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.updateRuntimeClassEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/runtimeClasses/:name', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.deleteRuntimeClassEndpoint, constants.HTTP_CODE_ACCEPTED, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/runtimeClasses', + supportSubstitution: true, + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.createRuntimeClassEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/runtimeClasses/yaml', + fileInput: 'runtimeClass', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.createRuntimeClassYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'put', + path: '/api/v3/runtimeClasses/yaml/:name', + fileInput: 'runtimeClass', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.upsertRuntimeClassYamlEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + }, + { + method: 'get', + path: '/api/v3/runtimeClasses/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.getRuntimeClassLinkEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'post', + path: '/api/v3/runtimeClasses/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.linkRuntimeClassEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest + ]) + }) + } + }, + { + method: 'delete', + path: '/api/v3/runtimeClasses/:name/link', + middleware: async (req, res) => { + logger.apiReq(req) + await rbacMiddleware.protect()(req, res, async () => { + await sendHandled(req, res, RuntimeClassController.unlinkRuntimeClassEndpoint, constants.HTTP_CODE_SUCCESS, [ + unauthorized, + notFound, + badRequest, + conflict + ]) + }) + } + } +] diff --git a/src/schemas/agent.js b/src/schemas/agent.js index 4b5ffbcdb..7decec2b2 100644 --- a/src/schemas/agent.js +++ b/src/schemas/agent.js @@ -31,13 +31,12 @@ const updateAgentConfig = { diskLimit: { type: 'integer', minimum: 0 }, diskDirectory: { type: 'string' }, memoryLimit: { type: 'integer', minimum: 0 }, - cpuLimit: { type: 'integer', minimum: 0 }, + cpuLimit: { type: 'integer', minimum: 5, maximum: 400 }, logLimit: { type: 'integer', minimum: 0 }, logDirectory: { type: 'string' }, logFileCount: { type: 'integer', minimum: 0 }, statusFrequency: { type: 'integer', minimum: 0 }, changeFrequency: { type: 'integer', minimum: 0 }, - deviceScanFrequency: { type: 'integer', minimum: 0 }, watchdogEnabled: { type: 'boolean' }, latitude: { type: 'number', minimum: -90, maximum: 90 }, longitude: { type: 'number', minimum: -180, maximum: 180 }, @@ -80,6 +79,12 @@ const updateAgentStatus = { systemAvailableDisk: { type: 'integer' }, systemAvailableMemory: { type: 'integer' }, systemTotalCpu: { type: 'number' }, + systemCpus: { type: 'integer', minimum: 0 }, + systemTotalMemory: { type: 'integer', minimum: 0 }, + systemTotalDisk: { type: 'integer', minimum: 0 }, + systemOs: { type: 'string' }, + systemOsVersion: { type: 'string' }, + systemKernelVersion: { type: 'string' }, securityStatus: { type: 'string' }, securityViolationInfo: { type: 'string' }, microserviceStatus: { type: 'string' }, @@ -94,6 +99,20 @@ const updateAgentStatus = { type: 'array', items: { type: 'string' } }, + runtimeClasses: { + type: 'array', + items: { type: 'object' } + }, + availableCdiDevices: { + type: 'array', + items: { type: 'string' } + }, + modelStatus: { type: 'string' }, + activeModels: { type: 'integer', minimum: 0 }, + modelLastUpdate: { type: 'integer', minimum: 0 }, + knowledgeStatus: { type: ['string', 'array'] }, + activeKnowledge: { type: 'integer', minimum: 0 }, + knowledgeLastUpdate: { type: 'integer', minimum: 0 }, runtimeAgentPhase: { type: 'string' }, controlPlaneQuiesced: { type: 'boolean' }, gpsMode: { type: 'string' }, @@ -123,34 +142,18 @@ const microserviceStatus = { memoryUsage: { type: 'number' }, ipAddress: { type: 'string' }, ipAddressExternal: { type: 'string' }, - execSessionIds: { type: 'array', items: { type: 'string' } } + execSessionIds: { type: 'array', items: { type: 'string' } }, + podId: { type: 'string' }, + errorMessage: { type: 'string' }, + lastError: { type: 'string' }, + lastErrorAt: { type: 'integer' }, + restartCount: { type: 'integer' } }, required: ['id'], additionalProperties: true } -const updateHardwareInfo = { - id: '/updateHardwareInfo', - type: 'object', - properties: { - info: { type: 'string' } - }, - required: ['info'], - additionalProperties: true -} - -const updateUsbInfo = { - id: '/updateUsbInfo', - type: 'object', - properties: { - info: { type: 'string' } - }, - required: ['info'], - additionalProperties: true -} - module.exports = { - mainSchemas: [agentProvision, agentDeprovision, updateAgentConfig, updateAgentGps, updateAgentStatus, - updateHardwareInfo, updateUsbInfo], + mainSchemas: [agentProvision, agentDeprovision, updateAgentConfig, updateAgentGps, updateAgentStatus], innerSchemas: [microserviceStatus] } diff --git a/src/schemas/fog-platform-spec.js b/src/schemas/fog-platform-spec.js index 84c09c394..30b038f3e 100644 --- a/src/schemas/fog-platform-spec.js +++ b/src/schemas/fog-platform-spec.js @@ -54,8 +54,6 @@ const fogPlatformSpec = { jsMemoryStoreSize: { type: 'string', maxLength: 32 }, networkRouter: { type: ['string', 'null'] }, containerEngine: { type: 'string', enum: ['edgelet', 'docker', 'podman'] }, - bluetoothEnabled: { type: 'boolean' }, - abstractedHardwareEnabled: { type: 'boolean' }, tags: { type: 'array', items: { @@ -95,14 +93,23 @@ function serializeJson (object, label) { return json } +function formatFogPlatformSpecValidationError (error) { + let property = (error.property || '').replace(/^instance\.?/, '') + if (!property || property === 'instance') { + const match = error.message && error.message.match(/additional property "([^"]+)"/) + if (match) { + property = match[1] + } + } + return property + ? `Invalid fog platform spec field '${property}'` + : 'Invalid fog platform spec' +} + async function validateFogPlatformSpec (object) { const response = validator.validate(object || {}, fogPlatformSpec) if (!response.valid) { - const error = response.errors[0] - const property = (error.property || '').replace('instance.', '') - throw new Errors.ValidationError( - property ? `Invalid fog platform spec field '${property}'` : 'Invalid fog platform spec' - ) + throw new Errors.ValidationError(formatFogPlatformSpecValidationError(response.errors[0])) } return object } @@ -159,9 +166,7 @@ const FOG_PLATFORM_SPEC_SCALAR_FIELDS = [ 'jsStorageSize', 'jsMemoryStoreSize', 'networkRouter', - 'containerEngine', - 'bluetoothEnabled', - 'abstractedHardwareEnabled' + 'containerEngine' ] const FOG_PLATFORM_SPEC_ARRAY_FIELDS = [ @@ -177,18 +182,26 @@ function normalizeSpecTags (tags) { return tags.map((tag) => (typeof tag === 'string' ? { value: tag } : tag)) } -function buildPlatformSpecFromFogData (fogData, options = {}) { +function pickPlatformSpecFields (source) { + if (!source) { + return {} + } const spec = {} for (const field of FOG_PLATFORM_SPEC_SCALAR_FIELDS) { - if (fogData[field] !== undefined) { - spec[field] = fogData[field] + if (source[field] !== undefined) { + spec[field] = source[field] } } for (const field of FOG_PLATFORM_SPEC_ARRAY_FIELDS) { - if (fogData[field] !== undefined) { - spec[field] = field === 'tags' ? normalizeSpecTags(fogData[field]) : fogData[field] + if (source[field] !== undefined) { + spec[field] = field === 'tags' ? normalizeSpecTags(source[field]) : source[field] } } + return spec +} + +function buildPlatformSpecFromFogData (fogData, options = {}) { + const spec = pickPlatformSpecFields(fogData) if (options.applyCreateDefaults) { if (spec.routerMode === undefined) { spec.routerMode = 'edge' @@ -201,7 +214,7 @@ function buildPlatformSpecFromFogData (fogData, options = {}) { } function mergePlatformSpecPatch (existingSpec, patchFogData) { - const merged = { ...(existingSpec || {}) } + const merged = pickPlatformSpecFields(existingSpec) for (const field of FOG_PLATFORM_SPEC_SCALAR_FIELDS) { if (patchFogData[field] !== undefined) { merged[field] = patchFogData[field] @@ -232,6 +245,7 @@ module.exports = { serializeConditionsJson, FOG_PLATFORM_SPEC_SCALAR_FIELDS, FOG_PLATFORM_SPEC_ARRAY_FIELDS, + pickPlatformSpecFields, buildPlatformSpecFromFogData, mergePlatformSpecPatch } diff --git a/src/schemas/iofog.js b/src/schemas/iofog.js index 3a3bc130e..de6ef7e5d 100644 --- a/src/schemas/iofog.js +++ b/src/schemas/iofog.js @@ -16,16 +16,13 @@ const iofogCreate = { diskLimit: { type: 'integer', minimum: 0 }, diskDirectory: { type: 'string' }, memoryLimit: { type: 'integer', minimum: 0 }, - cpuLimit: { type: 'integer', minimum: 0 }, + cpuLimit: { type: 'integer', minimum: 5, maximum: 400 }, logLimit: { type: 'integer', minimum: 0 }, logDirectory: { type: 'string' }, logFileCount: { type: 'integer', minimum: 0 }, statusFrequency: { type: 'integer', minimum: 0 }, changeFrequency: { type: 'integer', minimum: 0 }, - deviceScanFrequency: { type: 'integer', minimum: 0 }, - bluetoothEnabled: { type: 'boolean' }, watchdogEnabled: { type: 'boolean' }, - abstractedHardwareEnabled: { type: 'boolean' }, archId: { type: 'integer', minimum: 0, maximum: 4 }, pruningFrequency: { type: 'integer', minimum: 0 }, availableDiskThreshold: { type: 'integer', minimum: 0 }, @@ -93,16 +90,13 @@ const iofogUpdate = { diskLimit: { type: 'integer', minimum: 0 }, diskDirectory: { type: 'string' }, memoryLimit: { type: 'integer', minimum: 0 }, - cpuLimit: { type: 'integer', minimum: 0 }, + cpuLimit: { type: 'integer', minimum: 5, maximum: 400 }, logLimit: { type: 'integer', minimum: 0 }, logDirectory: { type: 'string' }, logFileCount: { type: 'integer', minimum: 0 }, statusFrequency: { type: 'integer', minimum: 0 }, changeFrequency: { type: 'integer', minimum: 0 }, - deviceScanFrequency: { type: 'integer', minimum: 0 }, - bluetoothEnabled: { type: 'boolean' }, watchdogEnabled: { type: 'boolean' }, - abstractedHardwareEnabled: { type: 'boolean' }, archId: { type: 'integer', minimum: 0, maximum: 4 }, pruningFrequency: { type: 'integer', minimum: 0 }, availableDiskThreshold: { type: 'integer', minimum: 0 }, @@ -232,16 +226,6 @@ const filter = { additionalProperties: true } -const halGet = { - id: '/halGet', - type: 'object', - properties: { - uuid: { type: 'string' } - }, - required: ['uuid'], - additionalProperties: true -} - const iofogPrune = { id: '/iofogPrune', type: 'object', @@ -294,6 +278,6 @@ const disableNodeExec = { module.exports = { mainSchemas: [iofogCreate, iofogUpdate, iofogDelete, iofogGet, iofogGenerateProvision, iofogSetVersionCommand, - iofogReboot, iofogFilters, halGet, iofogPrune, defaultRouterCreate, iofogTag, enableNodeExec, disableNodeExec], + iofogReboot, iofogFilters, iofogPrune, defaultRouterCreate, iofogTag, enableNodeExec, disableNodeExec], innerSchemas: [filter, iofogTag] } diff --git a/src/schemas/knowledge.js b/src/schemas/knowledge.js new file mode 100644 index 000000000..6930dc7ac --- /dev/null +++ b/src/schemas/knowledge.js @@ -0,0 +1,97 @@ +const { serviceNameRegex } = require('./utils/utils') + +const knowledgeCreate = { + id: '/knowledgeCreate', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + }, + repo: { + type: 'string', + minLength: 1 + }, + revision: { + type: 'string' + }, + registryId: { + type: 'integer', + minimum: 1 + }, + files: { + type: 'array', + items: { type: 'string', minLength: 1 } + }, + format: { + type: ['string', 'null'] + } + }, + required: ['name', 'repo', 'registryId'], + additionalProperties: false +} + +const knowledgeUpdate = { + id: '/knowledgeUpdate', + type: 'object', + properties: { + repo: { + type: 'string', + minLength: 1 + }, + revision: { + type: 'string' + }, + registryId: { + type: 'integer', + minimum: 1 + }, + files: { + type: 'array', + items: { type: 'string', minLength: 1 } + }, + format: { + type: ['string', 'null'] + } + }, + additionalProperties: false +} + +const knowledgeLink = { + id: '/knowledgeLink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +const knowledgeUnlink = { + id: '/knowledgeUnlink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +module.exports = { + mainSchemas: [knowledgeCreate, knowledgeUpdate, knowledgeLink, knowledgeUnlink], + innerSchemas: [knowledgeCreate, knowledgeUpdate] +} diff --git a/src/schemas/microservice-container-spec.js b/src/schemas/microservice-container-spec.js index 6cf86b63c..e427f0546 100644 --- a/src/schemas/microservice-container-spec.js +++ b/src/schemas/microservice-container-spec.js @@ -8,9 +8,73 @@ const microserviceContainerSpecProperties = { pidMode: { type: 'string' }, ipcMode: { type: 'string' }, runAsUser: { type: 'string' }, + runAsGroup: { type: 'string' }, + readOnlyRootFilesystem: { type: 'boolean' }, platform: { type: 'string' }, cpuSetCpus: { type: 'string' }, memoryLimit: { type: 'integer' }, + memoryReservation: { type: 'integer', minimum: 1 }, + memorySwap: { type: 'integer' }, + shmSize: { type: 'integer', minimum: 1 }, + cpus: { type: 'number' }, + workingDir: { type: 'string' }, + entrypoint: { + type: 'array', + items: { type: 'string' } + }, + commands: { + type: 'array', + items: { type: 'string' } + }, + sysctls: { + type: 'object', + additionalProperties: false, + properties: { + 'kernel.shm_rmid_forced': { type: 'string' }, + 'net.ipv4.ip_local_port_range': { type: 'string' }, + 'net.ipv4.tcp_syncookies': { type: 'string' }, + 'net.ipv4.ping_group_range': { type: 'string' }, + 'net.ipv4.ip_unprivileged_port_start': { type: 'string' }, + 'net.ipv4.ip_local_reserved_ports': { type: 'string' }, + 'net.ipv4.tcp_keepalive_time': { type: 'string' }, + 'net.ipv4.tcp_fin_timeout': { type: 'string' }, + 'net.ipv4.tcp_keepalive_intvl': { type: 'string' }, + 'net.ipv4.tcp_keepalive_probes': { type: 'string' }, + 'net.ipv4.tcp_rmem': { type: 'string' }, + 'net.ipv4.tcp_wmem': { type: 'string' }, + 'net.ipv4.tcp_slow_start_after_idle': { type: 'string' }, + 'net.ipv4.tcp_notsent_lowat': { type: 'string' } + } + }, + ulimits: { + type: 'object', + additionalProperties: false, + properties: { + core: { $ref: '/containerUlimit' }, + cpu: { $ref: '/containerUlimit' }, + data: { $ref: '/containerUlimit' }, + fsize: { $ref: '/containerUlimit' }, + locks: { $ref: '/containerUlimit' }, + memlock: { $ref: '/containerUlimit' }, + msgqueue: { $ref: '/containerUlimit' }, + nice: { $ref: '/containerUlimit' }, + nofile: { $ref: '/containerUlimit' }, + nproc: { $ref: '/containerUlimit' }, + rss: { $ref: '/containerUlimit' }, + rtprio: { $ref: '/containerUlimit' }, + rttime: { $ref: '/containerUlimit' }, + sigpending: { $ref: '/containerUlimit' }, + stack: { $ref: '/containerUlimit' } + } + }, + devices: { + type: 'array', + items: { $ref: '/containerDevice' } + }, + tmpfs: { + type: 'array', + items: { $ref: '/containerTmpfs' } + }, ports: { type: 'array', items: { $ref: '/ports' } diff --git a/src/schemas/microservice-template.js b/src/schemas/microservice-template.js new file mode 100644 index 000000000..154e3c8d0 --- /dev/null +++ b/src/schemas/microservice-template.js @@ -0,0 +1,98 @@ +const { nameRegex, serviceNameRegex } = require('./utils/utils') + +const microserviceTemplateVariable = { + id: '/microserviceTemplateVariable', + type: 'object', + properties: { + key: { + type: 'string', + minLength: 1, + pattern: nameRegex + }, + description: { type: 'string' } + }, + additionalProperties: true, + required: ['key'] +} + +const microserviceTemplateCreate = { + id: '/microserviceTemplateCreate', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + }, + description: { type: 'string' }, + variables: { + type: 'array', + items: { $ref: '/microserviceTemplateVariable' } + }, + microservice: { + type: 'object' + } + }, + required: ['name', 'microservice'], + additionalProperties: false +} + +const microserviceTemplateUpdate = { + id: '/microserviceTemplateUpdate', + type: 'object', + properties: { + description: { type: 'string' }, + variables: { + type: 'array', + items: { $ref: '/microserviceTemplateVariable' } + }, + microservice: { + type: 'object' + } + }, + additionalProperties: false +} + +const microserviceTemplateDeploy = { + id: '/microserviceTemplateDeploy', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + }, + variables: { + oneOf: [ + { + type: 'object' + }, + { + type: 'array', + items: { + type: 'object', + properties: { + key: { type: 'string' }, + value: {} + }, + required: ['key'] + } + } + ] + } + }, + required: ['name'], + additionalProperties: true +} + +module.exports = { + mainSchemas: [ + microserviceTemplateCreate, + microserviceTemplateVariable, + microserviceTemplateUpdate, + microserviceTemplateDeploy + ], + innerSchemas: [microserviceTemplateVariable] +} diff --git a/src/schemas/microservice.js b/src/schemas/microservice.js index 76dc34bb0..a877b2296 100644 --- a/src/schemas/microservice.js +++ b/src/schemas/microservice.js @@ -1,4 +1,4 @@ -const { nameRegex } = require('./utils/utils') +const { nameRegex, serviceNameRegex } = require('./utils/utils') const microserviceCreate = { id: '/microserviceCreate', @@ -66,10 +66,76 @@ const microserviceCreate = { items: { type: 'string' } }, runAsUser: { type: 'string' }, + runAsGroup: { type: 'string' }, + readOnlyRootFilesystem: { type: 'boolean' }, platform: { type: 'string' }, runtime: { type: 'string' }, cpuSetCpus: { type: 'string' }, memoryLimit: { type: 'integer' }, + memoryReservation: { type: 'integer', minimum: 1 }, + memorySwap: { type: 'integer' }, + shmSize: { type: 'integer', minimum: 1 }, + cpus: { type: 'number' }, + workingDir: { type: 'string' }, + entrypoint: { + type: 'array', + items: { type: 'string' } + }, + commands: { + type: 'array', + items: { type: 'string' } + }, + sysctls: { + type: 'object', + additionalProperties: false, + properties: { + 'kernel.shm_rmid_forced': { type: 'string' }, + 'net.ipv4.ip_local_port_range': { type: 'string' }, + 'net.ipv4.tcp_syncookies': { type: 'string' }, + 'net.ipv4.ping_group_range': { type: 'string' }, + 'net.ipv4.ip_unprivileged_port_start': { type: 'string' }, + 'net.ipv4.ip_local_reserved_ports': { type: 'string' }, + 'net.ipv4.tcp_keepalive_time': { type: 'string' }, + 'net.ipv4.tcp_fin_timeout': { type: 'string' }, + 'net.ipv4.tcp_keepalive_intvl': { type: 'string' }, + 'net.ipv4.tcp_keepalive_probes': { type: 'string' }, + 'net.ipv4.tcp_rmem': { type: 'string' }, + 'net.ipv4.tcp_wmem': { type: 'string' }, + 'net.ipv4.tcp_slow_start_after_idle': { type: 'string' }, + 'net.ipv4.tcp_notsent_lowat': { type: 'string' } + } + }, + ulimits: { + type: 'object', + additionalProperties: false, + properties: { + core: { $ref: '/containerUlimit' }, + cpu: { $ref: '/containerUlimit' }, + data: { $ref: '/containerUlimit' }, + fsize: { $ref: '/containerUlimit' }, + locks: { $ref: '/containerUlimit' }, + memlock: { $ref: '/containerUlimit' }, + msgqueue: { $ref: '/containerUlimit' }, + nice: { $ref: '/containerUlimit' }, + nofile: { $ref: '/containerUlimit' }, + nproc: { $ref: '/containerUlimit' }, + rss: { $ref: '/containerUlimit' }, + rtprio: { $ref: '/containerUlimit' }, + rttime: { $ref: '/containerUlimit' }, + sigpending: { $ref: '/containerUlimit' }, + stack: { $ref: '/containerUlimit' } + } + }, + devices: { + type: 'array', + items: { $ref: '/containerDevice' } + }, + tmpfs: { + type: 'array', + items: { $ref: '/containerTmpfs' } + }, + models: { $ref: '/microserviceCatalog' }, + knowledge: { $ref: '/microserviceCatalog' }, natsConfig: { $ref: '/microserviceNatsConfig' }, healthCheck: { type: 'object', @@ -155,10 +221,76 @@ const microserviceUpdate = { items: { type: 'string' } }, runAsUser: { type: 'string' }, + runAsGroup: { type: 'string' }, + readOnlyRootFilesystem: { type: 'boolean' }, platform: { type: 'string' }, runtime: { type: 'string' }, cpuSetCpus: { type: 'string' }, memoryLimit: { type: 'integer' }, + memoryReservation: { type: 'integer', minimum: 1 }, + memorySwap: { type: 'integer' }, + shmSize: { type: 'integer', minimum: 1 }, + cpus: { type: 'number' }, + workingDir: { type: 'string' }, + entrypoint: { + type: 'array', + items: { type: 'string' } + }, + commands: { + type: 'array', + items: { type: 'string' } + }, + sysctls: { + type: 'object', + additionalProperties: false, + properties: { + 'kernel.shm_rmid_forced': { type: 'string' }, + 'net.ipv4.ip_local_port_range': { type: 'string' }, + 'net.ipv4.tcp_syncookies': { type: 'string' }, + 'net.ipv4.ping_group_range': { type: 'string' }, + 'net.ipv4.ip_unprivileged_port_start': { type: 'string' }, + 'net.ipv4.ip_local_reserved_ports': { type: 'string' }, + 'net.ipv4.tcp_keepalive_time': { type: 'string' }, + 'net.ipv4.tcp_fin_timeout': { type: 'string' }, + 'net.ipv4.tcp_keepalive_intvl': { type: 'string' }, + 'net.ipv4.tcp_keepalive_probes': { type: 'string' }, + 'net.ipv4.tcp_rmem': { type: 'string' }, + 'net.ipv4.tcp_wmem': { type: 'string' }, + 'net.ipv4.tcp_slow_start_after_idle': { type: 'string' }, + 'net.ipv4.tcp_notsent_lowat': { type: 'string' } + } + }, + ulimits: { + type: 'object', + additionalProperties: false, + properties: { + core: { $ref: '/containerUlimit' }, + cpu: { $ref: '/containerUlimit' }, + data: { $ref: '/containerUlimit' }, + fsize: { $ref: '/containerUlimit' }, + locks: { $ref: '/containerUlimit' }, + memlock: { $ref: '/containerUlimit' }, + msgqueue: { $ref: '/containerUlimit' }, + nice: { $ref: '/containerUlimit' }, + nofile: { $ref: '/containerUlimit' }, + nproc: { $ref: '/containerUlimit' }, + rss: { $ref: '/containerUlimit' }, + rtprio: { $ref: '/containerUlimit' }, + rttime: { $ref: '/containerUlimit' }, + sigpending: { $ref: '/containerUlimit' }, + stack: { $ref: '/containerUlimit' } + } + }, + devices: { + type: 'array', + items: { $ref: '/containerDevice' } + }, + tmpfs: { + type: 'array', + items: { $ref: '/containerTmpfs' } + }, + models: { $ref: '/microserviceCatalog' }, + knowledge: { $ref: '/microserviceCatalog' }, natsConfig: { $ref: '/microserviceNatsConfig' }, healthCheck: { type: 'object', @@ -260,7 +392,8 @@ const volumeMappings = { hostDestination: { type: 'string' }, containerDestination: { type: 'string' }, accessMode: { type: 'string' }, - type: { enum: ['volume', 'bind', 'volumeMount', 'serviceAccount'] } + type: { enum: ['volume', 'bind', 'volumeMount', 'serviceAccount'] }, + scope: { type: 'string' } }, required: ['hostDestination', 'containerDestination', 'accessMode'], additionalProperties: true @@ -294,7 +427,85 @@ const microserviceNatsConfig = { additionalProperties: false } +const microserviceCatalogItem = { + id: '/microserviceCatalogItem', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + } + }, + required: ['name'], + additionalProperties: false +} + +const microserviceCatalog = { + id: '/microserviceCatalog', + type: 'object', + properties: { + bindPath: { type: 'string' }, + permissions: { enum: ['ro', 'rw'] }, + items: { + type: 'array', + items: { $ref: '/microserviceCatalogItem' } + } + }, + additionalProperties: false +} + +const microserviceCatalogPatch = { + id: '/microserviceCatalogPatch', + type: 'object', + properties: { + bindPath: { type: 'string' }, + permissions: { enum: ['ro', 'rw'] }, + items: { + type: 'array', + items: { $ref: '/microserviceCatalogItem' } + } + }, + additionalProperties: false +} + +const containerDevice = { + id: '/containerDevice', + type: 'object', + properties: { + hostPath: { type: 'string', pattern: '^/dev/' }, + containerPath: { type: 'string', minLength: 1 }, + permissions: { type: 'string', pattern: '^[rwm]+$' } + }, + required: ['hostPath', 'containerPath'], + additionalProperties: false +} + +const containerTmpfs = { + id: '/containerTmpfs', + type: 'object', + properties: { + containerPath: { type: 'string', minLength: 1 }, + size: { type: 'integer', minimum: 1 }, + mode: { type: 'string' } + }, + required: ['containerPath'], + additionalProperties: false +} + +const containerUlimit = { + id: '/containerUlimit', + type: 'object', + properties: { + soft: { type: 'integer' }, + hard: { type: 'integer' } + }, + required: ['soft', 'hard'], + additionalProperties: false +} + module.exports = { - mainSchemas: [microserviceCreate, microserviceUpdate, env, ports, extraHosts, portsCreate, microserviceDelete, volumeMappings, microserviceHealthCheck], - innerSchemas: [volumeMappings, ports, env, extraHosts, microserviceCreate, microserviceHealthCheck, microserviceNatsConfig] + mainSchemas: [microserviceCreate, microserviceUpdate, env, ports, extraHosts, portsCreate, microserviceDelete, volumeMappings, microserviceHealthCheck, microserviceCatalogPatch], + innerSchemas: [volumeMappings, ports, env, extraHosts, microserviceCreate, microserviceHealthCheck, microserviceNatsConfig, microserviceCatalog, microserviceCatalogItem, containerDevice, containerTmpfs, containerUlimit] } diff --git a/src/schemas/model.js b/src/schemas/model.js new file mode 100644 index 000000000..8c455ff2b --- /dev/null +++ b/src/schemas/model.js @@ -0,0 +1,101 @@ +const { serviceNameRegex } = require('./utils/utils') + +const modelFormats = ['gguf', 'safetensors', 'onnx', 'pytorch', 'tensorrt', 'unknown'] + +const modelCreate = { + id: '/modelCreate', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + }, + repo: { + type: 'string', + minLength: 1 + }, + revision: { + type: 'string' + }, + registryId: { + type: 'integer', + minimum: 1 + }, + files: { + type: 'array', + items: { type: 'string', minLength: 1 } + }, + format: { + type: 'string', + enum: modelFormats + } + }, + required: ['name', 'repo', 'registryId'], + additionalProperties: false +} + +const modelUpdate = { + id: '/modelUpdate', + type: 'object', + properties: { + repo: { + type: 'string', + minLength: 1 + }, + revision: { + type: 'string' + }, + registryId: { + type: 'integer', + minimum: 1 + }, + files: { + type: 'array', + items: { type: 'string', minLength: 1 } + }, + format: { + type: 'string', + enum: modelFormats + } + }, + additionalProperties: false +} + +const modelLink = { + id: '/modelLink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +const modelUnlink = { + id: '/modelUnlink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +module.exports = { + mainSchemas: [modelCreate, modelUpdate, modelLink, modelUnlink], + innerSchemas: [modelCreate, modelUpdate] +} diff --git a/src/schemas/registry.js b/src/schemas/registry.js index e07a46d7b..2b19b76f0 100644 --- a/src/schemas/registry.js +++ b/src/schemas/registry.js @@ -1,18 +1,23 @@ +const emailPattern = '^$|^(([^<>()\\[\\]\\\\.,;:\\s@"]+(\\.[^<>()\\[\\]\\\\.,;:\\s@"]+)*)|(".+"))@((\\[[0-9]{1,3}\\.[0-9]{1,3}' + + '\\.[0-9]{1,3}\\.[0-9]{1,3}])|(([a-zA-Z\\-0-9]+\\.)+[a-zA-Z]{2,}))$' + const registryCreate = { id: '/registryCreate', type: 'object', properties: { url: { type: 'string', minLength: 1 }, isPublic: { type: 'boolean' }, - username: { type: 'string', minLength: 1 }, + username: { type: 'string' }, password: { type: 'string' }, email: { type: 'string', - pattern: '^(([^<>()\\[\\]\\\\.,;:\\s@"]+(\\.[^<>()\\[\\]\\\\.,;:\\s@"]+)*)|(".+"))@((\\[[0-9]{1,3}\\.[0-9]{1,3}' + - '\\.[0-9]{1,3}\\.[0-9]{1,3}])|(([a-zA-Z\\-0-9]+\\.)+[a-zA-Z]{2,}))$' - } + pattern: emailPattern + }, + type: { type: 'string', enum: ['oci', 'hf'] }, + ca: { type: 'string' }, + insecure: { type: 'boolean' } }, - required: ['url', 'isPublic', 'username', 'password', 'email'], + required: ['url', 'isPublic'], additionalProperties: true } @@ -32,13 +37,15 @@ const registryUpdate = { properties: { url: { type: 'string', minLength: 1 }, isPublic: { type: 'boolean' }, - username: { type: 'string', minLength: 1 }, + username: { type: 'string' }, password: { type: 'string' }, email: { type: 'string', - pattern: '^(([^<>()\\[\\]\\\\.,;:\\s@"]+(\\.[^<>()\\[\\]\\\\.,;:\\s@"]+)*)|(".+"))@((\\[[0-9]{1,3}\\.[0-9]{1,3}' + - '\\.[0-9]{1,3}\\.[0-9]{1,3}])|(([a-zA-Z\\-0-9]+\\.)+[a-zA-Z]{2,}))$' - } + pattern: emailPattern + }, + type: { type: 'string', enum: ['oci', 'hf'] }, + ca: { type: 'string' }, + insecure: { type: 'boolean' } }, additionalProperties: true } diff --git a/src/schemas/runtime-class.js b/src/schemas/runtime-class.js new file mode 100644 index 000000000..3d1fe25e3 --- /dev/null +++ b/src/schemas/runtime-class.js @@ -0,0 +1,73 @@ +const { serviceNameRegex, nameRegex } = require('./utils/utils') + +const runtimeClassCreate = { + id: '/runtimeClassCreate', + type: 'object', + properties: { + name: { + type: 'string', + pattern: serviceNameRegex, + minLength: 1, + maxLength: 63 + }, + handler: { + type: 'string', + pattern: nameRegex, + minLength: 1, + maxLength: 253 + } + }, + required: ['name', 'handler'], + additionalProperties: false +} + +const runtimeClassUpdate = { + id: '/runtimeClassUpdate', + type: 'object', + properties: { + handler: { + type: 'string', + pattern: nameRegex, + minLength: 1, + maxLength: 253 + } + }, + additionalProperties: false +} + +const runtimeClassLink = { + id: '/runtimeClassLink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +const runtimeClassUnlink = { + id: '/runtimeClassUnlink', + type: 'object', + properties: { + fogUuids: { + type: 'array', + items: { + type: 'string' + }, + minItems: 1 + } + }, + required: ['fogUuids'], + additionalProperties: false +} + +module.exports = { + mainSchemas: [runtimeClassCreate, runtimeClassUpdate, runtimeClassLink, runtimeClassUnlink], + innerSchemas: [runtimeClassCreate, runtimeClassUpdate] +} diff --git a/src/services/agent-service.js b/src/services/agent-service.js index ffd4f65e2..a934247d9 100644 --- a/src/services/agent-service.js +++ b/src/services/agent-service.js @@ -2,7 +2,7 @@ const config = require('../config') // const Sequelize = require('sequelize') const moment = require('moment') // const Op = Sequelize.Op -// const logger = require('../logger') +const logger = require('../logger') const TransactionDecorator = require('../decorators/transaction-decorator') const FogProvisionKeyManager = require('../data/managers/iofog-provision-key-manager') @@ -13,6 +13,7 @@ const ChangeTrackingService = require('./change-tracking-service') const FogVersionCommandManager = require('../data/managers/iofog-version-command-manager') const { refreshProvisionKeyForFog } = require('./iofog-service') const RegistryManager = require('../data/managers/registry-manager') +const { toAgentRegistry } = require('./registry-service') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') const { microserviceState, microserviceExecState } = require('../enums/microservice-state') @@ -21,13 +22,18 @@ const Validator = require('../schemas') const Errors = require('../helpers/errors') const AppHelper = require('../helpers/app-helper') const ErrorMessages = require('../helpers/error-messages') -const HWInfoManager = require('../data/managers/hw-info-manager') -const USBInfoManager = require('../data/managers/usb-info-manager') const TunnelManager = require('../data/managers/tunnel-manager') const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceService = require('../services/microservices-service') const ApplicationManager = require('../data/managers/application-manager') const constants = require('../helpers/constants') +const CatalogContainer = require('../helpers/microservice-container-catalog') +const { + applyCrashExtras, + applyRuntimeMetrics, + coerceRuntimeMetric, + zeroRuntimeMetrics +} = require('../helpers/microservice-runtime-metrics') const SecretManager = require('../data/managers/secret-manager') const ConfigMapManager = require('../data/managers/config-map-manager') const MicroserviceLogStatusManager = require('../data/managers/microservice-log-status-manager') @@ -35,12 +41,41 @@ const MicroserviceExecSessionManager = require('../data/managers/microservice-ex const FogLogStatusManager = require('../data/managers/fog-log-status-manager') const RbacRoleManager = require('../data/managers/rbac-role-manager') +const AGENT_LAST_STATUS_SKEW_MS = 5 * 60 * 1000 + const CHANGE_TRACKING_DEFAULT = {} -const CHANGE_TRACKING_KEYS = ['config', 'version', 'reboot', 'deleteNode', 'microserviceList', 'microserviceConfig', 'registries', 'tunnel', 'prune', 'routerChanged', 'volumeMounts', 'execSessions', 'microserviceLogs', 'fogLogs'] +const CHANGE_TRACKING_KEYS = [ + 'config', + 'version', + 'reboot', + 'deleteNode', + 'microserviceList', + 'microserviceConfig', + 'registries', + 'tunnel', + 'prune', + 'routerChanged', + 'volumeMounts', + 'execSessions', + 'microserviceLogs', + 'fogLogs', + 'models', + 'runtimeClasses', + 'microserviceModels', + 'knowledge', + 'microserviceKnowledge' +] for (const key of CHANGE_TRACKING_KEYS) { CHANGE_TRACKING_DEFAULT[key] = false } +function _stringifyStatusField (value) { + if (value == null) { + return undefined + } + return typeof value === 'string' ? value : JSON.stringify(value) +} + const agentProvision = async function (provisionData, transaction) { await Validator.validate(provisionData, Validator.schemas.agentProvision) @@ -153,7 +188,6 @@ const getAgentConfig = async function (fog, transaction) { edgeGuardFrequency: fogData.edgeGuardFrequency, statusFrequency: fogData.statusFrequency, changeFrequency: fogData.changeFrequency, - deviceScanFrequency: fogData.deviceScanFrequency, watchdogEnabled: fogData.watchdogEnabled, latitude: fogData.latitude, longitude: fogData.longitude, @@ -180,7 +214,6 @@ const updateAgentConfig = async function (updateData, fog, transaction) { logFileCount: updateData.logFileCount, statusFrequency: updateData.statusFrequency, changeFrequency: updateData.changeFrequency, - deviceScanFrequency: updateData.deviceScanFrequency, watchdogEnabled: updateData.watchdogEnabled, latitude: updateData.latitude, longitude: updateData.longitude, @@ -249,12 +282,17 @@ const updateAgentStatus = async function (agentStatus, fog, transaction) { systemAvailableDisk: agentStatus.systemAvailableDisk, systemAvailableMemory: agentStatus.systemAvailableMemory, systemTotalCpu: agentStatus.systemTotalCpu, + systemCpus: agentStatus.systemCpus, + systemTotalMemory: agentStatus.systemTotalMemory, + systemTotalDisk: agentStatus.systemTotalDisk, + systemOs: agentStatus.systemOs, + systemOsVersion: agentStatus.systemOsVersion, + systemKernelVersion: agentStatus.systemKernelVersion, securityStatus: agentStatus.securityStatus, securityViolationInfo: agentStatus.securityViolationInfo, repositoryCount: agentStatus.repositoryCount, repositoryStatus: agentStatus.repositoryStatus, systemTime: agentStatus.systemTime, - lastStatusTime: agentStatus.lastStatusTime, ipAddress: agentStatus.ipAddress, ipAddressExternal: agentStatus.ipAddressExternal, availableRuntimes: agentStatus.availableRuntimes != null @@ -267,11 +305,28 @@ const updateAgentStatus = async function (agentStatus, fog, transaction) { isReadyToRollback: agentStatus.isReadyToRollback, activeVolumeMounts: agentStatus.activeVolumeMounts, volumeMountLastUpdate: agentStatus.volumeMountLastUpdate, - gpsStatus: agentStatus.gpsStatus + gpsStatus: agentStatus.gpsStatus, + runtimeClasses: _stringifyStatusField(agentStatus.runtimeClasses), + availableCdiDevices: _stringifyStatusField(agentStatus.availableCdiDevices), + modelStatus: _stringifyStatusField(agentStatus.modelStatus), + activeModels: agentStatus.activeModels, + modelLastUpdate: agentStatus.modelLastUpdate, + knowledgeStatus: _stringifyStatusField(agentStatus.knowledgeStatus), + activeKnowledge: agentStatus.activeKnowledge, + knowledgeLastUpdate: agentStatus.knowledgeLastUpdate } fogStatus = AppHelper.deleteUndefinedFields(fogStatus) + const receiptTime = Date.now() + const agentLastStatusTime = Number(agentStatus.lastStatusTime) + if (Number.isFinite(agentLastStatusTime) && Math.abs(receiptTime - agentLastStatusTime) > AGENT_LAST_STATUS_SKEW_MS) { + logger.warn( + `Agent lastStatusTime ${agentLastStatusTime} differs from Controller receipt time ${receiptTime} for fog ${fog.uuid}` + ) + } + fogStatus.lastStatusTime = receiptTime + const existingFog = await FogManager.findOne({ uuid: fog.uuid }, transaction) @@ -304,30 +359,85 @@ const updateAgentStatus = async function (agentStatus, fog, transaction) { } const _updateMicroserviceStatuses = async function (microserviceStatus, fog, transaction) { - for (const status of microserviceStatus) { - let microserviceStatus = { + const reported = Array.isArray(microserviceStatus) ? microserviceStatus : [] + for (const status of reported) { + let observedStatus = { containerId: status.containerId, status: status.status, healthStatus: status.healthStatus, startTime: status.startTime, operatingDuration: status.operatingDuration, - cpuUsage: status.cpuUsage, - memoryUsage: status.memoryUsage, + cpuUsage: coerceRuntimeMetric(status.cpuUsage), + memoryUsage: coerceRuntimeMetric(status.memoryUsage), percentage: status.percentage, errorMessage: status.errorMessage, ipAddress: status.ipAddress, - execSessionIds: status.execSessionIds + execSessionIds: status.execSessionIds, + podId: status.podId } - microserviceStatus = AppHelper.deleteUndefinedFields(microserviceStatus) + applyRuntimeMetrics(observedStatus, observedStatus.status) + applyCrashExtras(observedStatus, status) + observedStatus = AppHelper.deleteUndefinedFields(observedStatus) const microservice = await MicroserviceManager.findOne({ uuid: status.id }, transaction) if (microservice && fog.uuid === microservice.iofogUuid) { await MicroserviceStatusManager.update({ microserviceUuid: status.id - }, microserviceStatus, transaction) + }, observedStatus, transaction) + } + } + await _reconcileOmittedMicroserviceStatuses(reported, fog, transaction) +} + +const _reconcileOmittedMicroserviceStatuses = async function (reportedStatuses, fog, transaction) { + const reportedIds = new Set(reportedStatuses.map((item) => item.id).filter(Boolean)) + const assigned = await MicroserviceManager.findAll({ iofogUuid: fog.uuid }, transaction) + const omitted = (assigned || []).filter((ms) => !reportedIds.has(ms.uuid)) + if (!omitted.length) { + return + } + + const applicationIds = [...new Set(omitted.map((ms) => ms.applicationId).filter((id) => id != null))] + const applications = applicationIds.length + ? await ApplicationManager.findAll({ id: applicationIds }, transaction) + : [] + const applicationById = new Map((applications || []).map((app) => [app.id, app])) + + const stoppedUuids = [] + const unknownUuids = [] + for (const ms of omitted) { + const application = applicationById.get(ms.applicationId) + const desiredInactive = !ms.isActivated || (application && !application.isActivated) + if (desiredInactive) { + stoppedUuids.push(ms.uuid) + } else { + unknownUuids.push(ms.uuid) } } + + const zeros = zeroRuntimeMetrics() + if (stoppedUuids.length) { + await MicroserviceStatusManager.update( + { microserviceUuid: stoppedUuids }, + Object.assign({ status: microserviceState.STOPPED }, zeros), + transaction + ) + } + if (unknownUuids.length) { + await MicroserviceStatusManager.update( + { microserviceUuid: unknownUuids }, + Object.assign({ status: microserviceState.UNKNOWN }, zeros), + transaction + ) + } + + const omittedUuids = stoppedUuids.concat(unknownUuids) + await MicroserviceExecStatusManager.update( + { microserviceUuid: omittedUuids }, + { status: microserviceExecState.INACTIVE, execSessionId: '' }, + transaction + ) } const _mapExtraHost = function (extraHost) { @@ -470,6 +580,16 @@ const getAgentMicroservices = async function (fog, transaction) { schedule: microservice.schedule } + CatalogContainer.applyAgentContainerFields(responseMicroservice, microservice, { + cmd, + entrypoint: CatalogContainer.tokensFromRows(microservice.entrypoint, 'entrypoint'), + devices: CatalogContainer.devicesFromRows(microservice.devices), + tmpfs: CatalogContainer.tmpfsFromRows(microservice.tmpfs), + ulimits: CatalogContainer.ulimitsFromRows(microservice.ulimits), + models: CatalogContainer.catalogFromRows(microservice.microserviceModel, microservice.modelItems), + knowledge: CatalogContainer.catalogFromRows(microservice.microserviceKnowledge, microservice.knowledgeItems) + }) + // Resolve service account with rules from relationship const serviceAccountData = await _resolveServiceAccountRules(microservice.serviceAccount, transaction) if (serviceAccountData) { @@ -514,7 +634,7 @@ const getAgentMicroservice = async function (microserviceUuid, fog, transaction) const getAgentRegistries = async function (fog, transaction) { const registries = await RegistryManager.findAll({}, transaction) return { - registries + registries: registries.map(toAgentRegistry) } } @@ -555,26 +675,6 @@ const getAgentChangeVersionCommand = async function (fog, transaction) { return response } -const updateHalHardwareInfo = async function (hardwareData, fog, transaction) { - await Validator.validate(hardwareData, Validator.schemas.updateHardwareInfo) - - hardwareData.iofogUuid = fog.uuid - - await HWInfoManager.updateOrCreate({ - iofogUuid: fog.uuid - }, hardwareData, transaction) -} - -const updateHalUsbInfo = async function (usbData, fog, transaction) { - await Validator.validate(usbData, Validator.schemas.updateUsbInfo) - - usbData.iofogUuid = fog.uuid - - await USBInfoManager.updateOrCreate({ - iofogUuid: fog.uuid - }, usbData, transaction) -} - const deleteNode = async function (fog, transaction) { await FogManager.delete({ uuid: fog.uuid @@ -775,6 +875,119 @@ const getAgentLinkedVolumeMounts = async function (fog, transaction) { return volumeMounts } +function _toPlain (row) { + if (!row) { + return null + } + return typeof row.toJSON === 'function' ? row.toJSON() : row +} + +const getAgentLinkedModels = async function (fog, transaction) { + if (!fog || typeof fog.getModels !== 'function') { + return [] + } + + const resources = await fog.getModels({ + attributes: ['uuid', 'name', 'repo', 'revision', 'files', 'format', 'registryId'], + joinTableAttributes: [], + transaction + }) + + const models = [] + for (const resource of resources) { + const row = _toPlain(resource) + if (!row || !row.uuid || !row.name) { + continue + } + models.push({ + uuid: row.uuid, + name: row.name, + repo: row.repo, + revision: row.revision != null ? row.revision : '', + registryId: row.registryId, + files: Array.isArray(row.files) ? row.files : [], + format: row.format + }) + } + return models +} + +function _knowledgeFiles (value) { + if (Array.isArray(value)) { + return value + } + if (value == null || value === '') { + return [] + } + if (typeof value === 'string') { + try { + const parsed = JSON.parse(value) + return Array.isArray(parsed) ? parsed : [] + } catch (e) { + return [] + } + } + return [] +} + +const getAgentLinkedKnowledge = async function (fog, transaction) { + if (!fog || typeof fog.getKnowledge !== 'function') { + return [] + } + + const resources = await fog.getKnowledge({ + attributes: ['uuid', 'name', 'repo', 'revision', 'files', 'format', 'registryId'], + joinTableAttributes: [], + transaction + }) + + const knowledge = [] + for (const resource of resources) { + const row = _toPlain(resource) + if (!row || !row.uuid || !row.name) { + continue + } + const item = { + uuid: row.uuid, + name: row.name, + repo: row.repo, + revision: row.revision != null ? row.revision : '', + registryId: row.registryId, + files: _knowledgeFiles(row.files) + } + if (row.format != null && row.format !== '') { + item.format = row.format + } + knowledge.push(item) + } + return knowledge +} + +const getAgentLinkedRuntimeClasses = async function (fog, transaction) { + if (!fog || typeof fog.getRuntimeClassLinks !== 'function') { + return [] + } + + const resources = await fog.getRuntimeClassLinks({ + attributes: ['name', 'handler'], + joinTableAttributes: [], + transaction + }) + + const runtimeClasses = [] + for (const resource of resources) { + const row = _toPlain(resource) + if (!row || !row.name || !row.handler) { + continue + } + runtimeClasses.push({ + name: row.name, + handler: row.handler + }) + } + return runtimeClasses +} + module.exports = { agentProvision: TransactionDecorator.generateTransaction(agentProvision), agentDeprovision: TransactionDecorator.generateTransaction(agentDeprovision), @@ -789,10 +1002,11 @@ module.exports = { getAgentRegistries: TransactionDecorator.generateTransaction(getAgentRegistries), getAgentTunnel: TransactionDecorator.generateTransaction(getAgentTunnel), getAgentChangeVersionCommand: TransactionDecorator.generateTransaction(getAgentChangeVersionCommand), - updateHalHardwareInfo: TransactionDecorator.generateTransaction(updateHalHardwareInfo), - updateHalUsbInfo: TransactionDecorator.generateTransaction(updateHalUsbInfo), deleteNode: TransactionDecorator.generateTransaction(deleteNode), getAgentLinkedVolumeMounts: TransactionDecorator.generateTransaction(getAgentLinkedVolumeMounts), + getAgentLinkedModels: TransactionDecorator.generateTransaction(getAgentLinkedModels), + getAgentLinkedKnowledge: TransactionDecorator.generateTransaction(getAgentLinkedKnowledge), + getAgentLinkedRuntimeClasses: TransactionDecorator.generateTransaction(getAgentLinkedRuntimeClasses), getControllerCA, getAgentLogSessions: TransactionDecorator.generateTransaction(getAgentLogSessions), getAgentExecSessions: TransactionDecorator.generateTransaction(getAgentExecSessions) diff --git a/src/services/application-service.js b/src/services/application-service.js index c55254e9e..72a195b7e 100644 --- a/src/services/application-service.js +++ b/src/services/application-service.js @@ -189,6 +189,13 @@ const patchApplicationEndPoint = async function (applicationData, conditions, is } if (oldApplication.isActivated !== applicationData.isActivated) { + if (oldApplication.isActivated && applicationData.isActivated === false) { + const microservices = await ApplicationManager.findApplicationMicroservices(conditions, transaction) + await MicroserviceService.setMicroservicesObservedStopping( + (microservices || []).map((ms) => ms.uuid), + transaction + ) + } await _updateChangeTrackingsAndDeleteMicroservicesByApplicationId(conditions, false, transaction) } } @@ -257,6 +264,13 @@ const updateApplicationEndPoint = async function (applicationData, name, isCLI, await _updateMicroservices(application.name, applicationData.microservices, isCLI, transaction) } if (oldApplication.isActivated !== applicationData.isActivated) { + if (oldApplication.isActivated && applicationData.isActivated === false) { + const microservices = await ApplicationManager.findApplicationMicroservices({ name }, transaction) + await MicroserviceService.setMicroservicesObservedStopping( + (microservices || []).map((ms) => ms.uuid), + transaction + ) + } await _updateChangeTrackingsAndDeleteMicroservicesByApplicationId({ name }, false, transaction) } } diff --git a/src/services/application-template-service.js b/src/services/application-template-service.js index 7b1e14b82..6328dd1ef 100644 --- a/src/services/application-template-service.js +++ b/src/services/application-template-service.js @@ -11,6 +11,32 @@ const ApplicationTemplateVariableManager = require('../data/managers/application const TransactionDecorator = require('../decorators/transaction-decorator') const Validator = require('../schemas') +function _parseStoredDefaultValue (defaultValue) { + if (defaultValue === undefined || defaultValue === null) { + return undefined + } + if (typeof defaultValue !== 'string') { + return defaultValue + } + try { + return JSON.parse(defaultValue) + } catch (error) { + return defaultValue + } +} + +function _buildDefaultVariableMap (variables) { + return (variables || []).reduce((acc, variable) => { + if (variable && variable.key != null) { + const parsed = _parseStoredDefaultValue(variable.defaultValue) + if (parsed !== undefined) { + acc[variable.key] = parsed + } + } + return acc + }, {}) +} + const createApplicationTemplateEndPoint = async function (applicationTemplateData, isCLI, transaction) { // Add a name field to pass schema validation using the applicationCreate schema applicationTemplateData.application = { ...applicationTemplateData.application, name: 'validation' } @@ -157,9 +183,16 @@ const _buildGetApplicationObj = function (applicationDBObj) { const JSONData = applicationDBObj.toJSON() JSONData.application = JSON.parse(JSONData.applicationJSON || null) JSONData.variables = (JSONData.variables || []).map(v => { - if (v.defaultValue === undefined) { return v } + if (v.defaultValue === undefined || v.defaultValue === null) { + return v + } + const parsed = _parseStoredDefaultValue(v.defaultValue) + if (parsed === undefined) { + return v + } return { - ...v, defaultValue: JSON.parse(v.defaultValue) + ...v, + defaultValue: parsed } }) delete JSONData.applicationJSON @@ -208,9 +241,7 @@ const getApplicationDataFromTemplate = async function (deploymentData, isCLI, tr const newApplication = applicationTemplate.application // Replace variables - const defaultVariablesValues = (applicationTemplate.variables || []).reduce((acc, v) => { - return { ...acc, [v.key]: JSON.parse(v.defaultValue) } - }, {}) + const defaultVariablesValues = _buildDefaultVariableMap(applicationTemplate.variables) const userProvidedVariables = (deploymentData.variables || []).reduce((acc, v) => { return { ...acc, [v.key]: v.value } }, {}) diff --git a/src/services/catalog-service.js b/src/services/catalog-service.js index 603df1fc5..59ce930ed 100644 --- a/src/services/catalog-service.js +++ b/src/services/catalog-service.js @@ -157,24 +157,6 @@ async function getDebugCatalogItem (transaction) { }, transaction) } -async function getBluetoothCatalogItem (transaction) { - return CatalogItemManager.findOne({ - name: 'RESTBlue', - category: 'SYSTEM', - publisher: 'Eclipse ioFog', - registry_id: 1 - }, transaction) -} - -async function getHalCatalogItem (transaction) { - return CatalogItemManager.findOne({ - name: 'HAL', - category: 'SYSTEM', - publisher: 'Eclipse ioFog', - registry_id: 1 - }, transaction) -} - const _checkForDuplicateName = async function (name, item, transaction) { if (name) { const where = (item && item.id) @@ -371,8 +353,6 @@ module.exports = { getCatalogItem, getSystemCatalogItem, getNatsCatalogItem, - getBluetoothCatalogItem, - getHalCatalogItem, getRouterCatalogItem, getDebugCatalogItem } diff --git a/src/services/change-tracking-service.js b/src/services/change-tracking-service.js index 3174fd744..7deb24217 100644 --- a/src/services/change-tracking-service.js +++ b/src/services/change-tracking-service.js @@ -16,15 +16,22 @@ const events = Object.freeze({ prune: false, routerChanged: false, volumeMounts: false, - execSessions: false + execSessions: false, + models: false, + runtimeClasses: false, + microserviceModels: false, + knowledge: false, + microserviceKnowledge: false }, microserviceFull: { microserviceConfig: true, - microserviceList: true + microserviceList: true, + microserviceModels: true }, microserviceCommon: { microserviceConfig: true, - microserviceList: true + microserviceList: true, + microserviceModels: true }, microserviceList: { microserviceList: true @@ -67,6 +74,21 @@ const events = Object.freeze({ }, fogLogs: { fogLogs: true + }, + models: { + models: true + }, + runtimeClasses: { + runtimeClasses: true + }, + microserviceModels: { + microserviceModels: true + }, + knowledge: { + knowledge: true + }, + microserviceKnowledge: { + microserviceKnowledge: true } }) diff --git a/src/services/controller-ms-service.js b/src/services/controller-ms-service.js index af414d849..d4f9ef927 100644 --- a/src/services/controller-ms-service.js +++ b/src/services/controller-ms-service.js @@ -19,6 +19,7 @@ const WorkloadSpec = require('./microservice-workload-spec') const isEqual = require('lodash/isEqual') const Op = require('sequelize').Op const { VOLUME_MAPPING_DEFAULT } = require('../helpers/constants') +const { applyVolumeMappingScope, PRIVATE_SCOPE } = require('../helpers/volume-mapping-scope') const CONTROLLER_MS_NAME = 'controller' const SERVICE_ACCOUNT_VOLUME_TYPE = 'serviceAccount' @@ -78,10 +79,7 @@ function _validateVolumeMappingFields (volumeMappings) { } for (const mapping of volumeMappings) { mapping.type = mapping.type || VOLUME_MAPPING_DEFAULT - if (mapping.type === 'volume' && (!/^[a-zA-Z0-9_.-]/.test(mapping.hostDestination))) { - throw new Errors.InvalidArgumentError('hostDestination includes invalid characters for a local volume name, only ' + - '"[a-zA-Z0-9][a-zA-Z0-9_.-]" are allowed. If you intended to pass a host directory, use type: bind') - } + applyVolumeMappingScope(mapping, { rejectShared: true }) if (mapping.type === 'volumeMount') { if (!mapping.hostDestination || mapping.hostDestination === '') { throw new Errors.ValidationError('hostDestination is required when type is volumeMount') @@ -211,7 +209,8 @@ async function _createVolumeMappings (microserviceUuid, volumeMappings, transact hostDestination: volumeMapping.hostDestination, containerDestination: volumeMapping.containerDestination, accessMode: volumeMapping.accessMode, - type: volumeMapping.type || VOLUME_MAPPING_DEFAULT + type: volumeMapping.type || VOLUME_MAPPING_DEFAULT, + scope: volumeMapping.scope || PRIVATE_SCOPE })) await VolumeMappingManager.bulkCreate(mappings, transaction) } diff --git a/src/services/event-service.js b/src/services/event-service.js index cdf269509..6eef22506 100644 --- a/src/services/event-service.js +++ b/src/services/event-service.js @@ -29,6 +29,10 @@ function extractResourceType (path) { { pattern: /^\/api\/v3\/nats/, type: 'nats' }, { pattern: /^\/api\/v3\/registries/, type: 'registry' }, { pattern: /^\/api\/v3\/volumeMounts/, type: 'volumeMount' }, + { pattern: /^\/api\/v3\/models/, type: 'model' }, + { pattern: /^\/api\/v3\/knowledge/, type: 'knowledge' }, + { pattern: /^\/api\/v3\/runtimeClasses/, type: 'runtimeClass' }, + { pattern: /^\/api\/v3\/microserviceTemplates/, type: 'microserviceTemplate' }, { pattern: /^\/api\/v3\/configMaps/, type: 'configMap' }, { pattern: /^\/api\/v3\/applicationTemplates/, type: 'applicationTemplate' }, { pattern: /^\/api\/v3\/catalog/, type: 'catalog' }, diff --git a/src/services/fog-platform-service.js b/src/services/fog-platform-service.js index 9b9c2049d..f2f73b1c8 100644 --- a/src/services/fog-platform-service.js +++ b/src/services/fog-platform-service.js @@ -45,10 +45,6 @@ function buildFogDataFromSpecAndFog (fog, spec) { jsMemoryStoreSize: spec.jsMemoryStoreSize, networkRouter: spec.networkRouter, containerEngine: spec.containerEngine || fog.containerEngine, - bluetoothEnabled: spec.bluetoothEnabled != null ? spec.bluetoothEnabled : fog.bluetoothEnabled, - abstractedHardwareEnabled: spec.abstractedHardwareEnabled != null - ? spec.abstractedHardwareEnabled - : fog.abstractedHardwareEnabled, tags: fogTags.length > 0 ? fogTags : specTags } } @@ -389,22 +385,6 @@ async function reconcileFogPlatform (fogUuid, prep, transaction) { await IofogService._updateMicroserviceExtraHosts(fogUuid, spec.host, transaction) } - if (fog.abstractedHardwareEnabled === true && spec.abstractedHardwareEnabled === false) { - await IofogService._deleteHalMicroserviceByFog(fogData, transaction) - await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.microserviceCommon, transaction) - } else if (fog.abstractedHardwareEnabled === false && spec.abstractedHardwareEnabled === true) { - await IofogService._createHalMicroserviceForFog(fogData, fog, transaction) - await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.microserviceCommon, transaction) - } - - if (fog.bluetoothEnabled === true && spec.bluetoothEnabled === false) { - await IofogService._deleteBluetoothMicroserviceByFog(fogData, transaction) - await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.microserviceCommon, transaction) - } else if (fog.bluetoothEnabled === false && spec.bluetoothEnabled === true) { - await IofogService._createBluetoothMicroserviceForFog(fogData, fog, transaction) - await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.microserviceCommon, transaction) - } - if (prep.isFirstReconcile) { await ChangeTrackingService.create(fogUuid, transaction) } diff --git a/src/services/iofog-service.js b/src/services/iofog-service.js index affde2fa0..c602d6a12 100644 --- a/src/services/iofog-service.js +++ b/src/services/iofog-service.js @@ -10,8 +10,6 @@ const NatsService = require('./nats-service') const Errors = require('../helpers/errors') const ErrorMessages = require('../helpers/error-messages') const Validator = require('../schemas') -const HWInfoManager = require('../data/managers/hw-info-manager') -const USBInfoManager = require('../data/managers/usb-info-manager') const CatalogService = require('./catalog-service') const MicroserviceManager = require('../data/managers/microservice-manager') const ApplicationManager = require('../data/managers/application-manager') @@ -355,10 +353,7 @@ async function createFogEndPoint (fogData, isCLI, transaction) { logFileCount: fogData.logFileCount, statusFrequency: fogData.statusFrequency, changeFrequency: fogData.changeFrequency, - deviceScanFrequency: fogData.deviceScanFrequency, - bluetoothEnabled: fogData.bluetoothEnabled, watchdogEnabled: fogData.watchdogEnabled, - abstractedHardwareEnabled: fogData.abstractedHardwareEnabled, archId: _resolveArchId(fogData), logLevel: fogData.logLevel, edgeGuardFrequency: fogData.edgeGuardFrequency, @@ -370,7 +365,7 @@ async function createFogEndPoint (fogData, isCLI, transaction) { timeZone: fogData.timeZone } - if ((fogData.latitude || fogData.longitude) && (fogData.gpsMode !== 'dynamic' && fogData.gpsMode !== 'off')) { + if ((fogData.latitude || fogData.longitude) && (fogData.gpsMode !== 'dynamic' && fogData.gpsMode !== 'off' && fogData.gpsMode !== 'auto')) { createFogData.gpsMode = 'manual' } else if (fogData.gpsMode === 'dynamic' && fogData.gpsDevice) { createFogData.gpsMode = fogData.gpsMode @@ -484,11 +479,8 @@ async function updateFogEndPoint (fogData, isCLI, transaction) { logFileCount: fogData.logFileCount, statusFrequency: fogData.statusFrequency, changeFrequency: fogData.changeFrequency, - deviceScanFrequency: fogData.deviceScanFrequency, - bluetoothEnabled: fogData.bluetoothEnabled, watchdogEnabled: fogData.watchdogEnabled, isSystem: fogData.isSystem, - abstractedHardwareEnabled: fogData.abstractedHardwareEnabled, archId: _resolveArchId(fogData), logLevel: fogData.logLevel, pruningFrequency: fogData.pruningFrequency, @@ -498,7 +490,7 @@ async function updateFogEndPoint (fogData, isCLI, transaction) { timeZone: fogData.timeZone } - if ((fogData.latitude || fogData.longitude) && (fogData.gpsMode !== 'dynamic' && fogData.gpsMode !== 'off')) { + if ((fogData.latitude || fogData.longitude) && (fogData.gpsMode !== 'dynamic' && fogData.gpsMode !== 'off' && fogData.gpsMode !== 'auto')) { updateFogData.gpsMode = 'manual' } else if (fogData.gpsMode === 'dynamic' && fogData.gpsDevice) { updateFogData.gpsMode = fogData.gpsMode @@ -1014,36 +1006,6 @@ async function setFogRebootCommandEndPoint (fogData, isCLI, transaction) { await ChangeTrackingService.update(fogData.uuid, ChangeTrackingService.events.reboot, transaction) } -async function getHalHardwareInfoEndPoint (uuidObj, isCLI, transaction) { - await Validator.validate(uuidObj, Validator.schemas.halGet) - - const fog = await FogManager.findOne({ - uuid: uuidObj.uuid - }, transaction) - if (!fog) { - throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_IOFOG_UUID, uuidObj.uuid)) - } - - return HWInfoManager.findOne({ - iofogUuid: uuidObj.uuid - }, transaction) -} - -async function getHalUsbInfoEndPoint (uuidObj, isCLI, transaction) { - await Validator.validate(uuidObj, Validator.schemas.halGet) - - const fog = await FogManager.findOne({ - uuid: uuidObj.uuid - }, transaction) - if (!fog) { - throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_IOFOG_UUID, uuidObj.uuid)) - } - - return USBInfoManager.findOne({ - iofogUuid: uuidObj.uuid - }, transaction) -} - function _filterFogs (fogs, filters) { if (!filters) { return fogs @@ -1107,60 +1069,6 @@ async function _processDeleteCommand (fog, transaction) { await FogManager.delete({ uuid: fog.uuid }, transaction) } -async function _createHalMicroserviceForFog (fogData, oldFog, transaction) { - const halItem = await CatalogService.getHalCatalogItem(transaction) - const systemMicroserviceName = getSystemMicroserviceName('hal') - const fogForName = (fogData && fogData.name) ? fogData : oldFog - - const halMicroserviceData = { - uuid: AppHelper.generateUUID(), - name: systemMicroserviceName, - config: '{}', - catalogItemId: halItem.id, - iofogUuid: fogData.uuid, - hostNetworkMode: true, - isPrivileged: true, - logSize: Constants.MICROSERVICE_DEFAULT_LOG_SIZE, - schedule: 1, - configLastUpdated: Date.now() - } - - const application = await ensureSystemApplication(fogForName, transaction) - halMicroserviceData.applicationId = application.id - const existingMicroservice = await MicroserviceManager.findOne({ - name: systemMicroserviceName, - applicationId: application.id - }, transaction) - if (!existingMicroservice) { - await MicroserviceManager.create(halMicroserviceData, transaction) - await MicroserviceStatusManager.create({ microserviceUuid: halMicroserviceData.uuid }, transaction) - await MicroserviceExecStatusManager.create({ microserviceUuid: halMicroserviceData.uuid }, transaction) - } -} - -async function _deleteHalMicroserviceByFog (fogData, transaction) { - const halItem = await CatalogService.getHalCatalogItem(transaction) - const deleteHalMicroserviceData = { - iofogUuid: fogData.uuid, - catalogItemId: halItem.id - } - - const fog = await FogManager.findOne({ uuid: fogData.uuid }, transaction) - if (!fog) { - throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_IOFOG_UUID, fogData.uuid)) - } - const systemAppName = getSystemAppName(fog.name) - const legacySystemAppName = getLegacySystemAppName(fog.uuid) - let application = await ApplicationManager.findOne({ name: systemAppName }, transaction) - if (!application) { - application = await ApplicationManager.findOne({ name: legacySystemAppName }, transaction) - } - if (application) { - deleteHalMicroserviceData.applicationId = application.id - await MicroserviceManager.delete(deleteHalMicroserviceData, transaction) - } -} - async function _deleteNatsMicroserviceByFog (fogData, transaction) { const natsItem = await CatalogService.getNatsCatalogItem(transaction) if (!natsItem) return @@ -1185,59 +1093,6 @@ async function _deleteNatsMicroserviceByFog (fogData, transaction) { } } -async function _createBluetoothMicroserviceForFog (fogData, oldFog, transaction) { - const bluetoothItem = await CatalogService.getBluetoothCatalogItem(transaction) - const systemMicroserviceName = getSystemMicroserviceName('ble') - const fogForName = (fogData && fogData.name) ? fogData : oldFog - - const bluetoothMicroserviceData = { - uuid: AppHelper.generateUUID(), - name: systemMicroserviceName, - config: '{}', - catalogItemId: bluetoothItem.id, - iofogUuid: fogData.uuid, - hostNetworkMode: true, - isPrivileged: true, - logSize: Constants.MICROSERVICE_DEFAULT_LOG_SIZE, - schedule: 1, - configLastUpdated: Date.now() - } - - const application = await ensureSystemApplication(fogForName, transaction) - bluetoothMicroserviceData.applicationId = application.id - const existingMicroservice = await MicroserviceManager.findOne({ - name: systemMicroserviceName, - applicationId: application.id - }, transaction) - if (!existingMicroservice) { - await MicroserviceManager.create(bluetoothMicroserviceData, transaction) - await MicroserviceStatusManager.create({ microserviceUuid: bluetoothMicroserviceData.uuid }, transaction) - await MicroserviceExecStatusManager.create({ microserviceUuid: bluetoothMicroserviceData.uuid }, transaction) - } -} - -async function _deleteBluetoothMicroserviceByFog (fogData, transaction) { - const bluetoothItem = await CatalogService.getBluetoothCatalogItem(transaction) - const deleteBluetoothMicroserviceData = { - iofogUuid: fogData.uuid, - catalogItemId: bluetoothItem.id - } - const fog = await FogManager.findOne({ uuid: fogData.uuid }, transaction) - if (!fog) { - throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_IOFOG_UUID, fogData.uuid)) - } - const systemAppName = getSystemAppName(fog.name) - const legacySystemAppName = getLegacySystemAppName(fog.uuid) - let application = await ApplicationManager.findOne({ name: systemAppName }, transaction) - if (!application) { - application = await ApplicationManager.findOne({ name: legacySystemAppName }, transaction) - } - if (application) { - deleteBluetoothMicroserviceData.applicationId = application.id - await MicroserviceManager.delete(deleteBluetoothMicroserviceData, transaction) - } -} - async function setFogPruneCommandEndPoint (fogData, isCLI, transaction) { await Validator.validate(fogData, Validator.schemas.iofogPrune) @@ -1545,8 +1400,6 @@ module.exports = { generateProvisioningKeyEndPoint: TransactionDecorator.generateTransaction(generateProvisioningKeyEndPoint), setFogVersionCommandEndPoint: TransactionDecorator.generateTransaction(setFogVersionCommandEndPoint), setFogRebootCommandEndPoint: TransactionDecorator.generateTransaction(setFogRebootCommandEndPoint), - getHalHardwareInfoEndPoint: TransactionDecorator.generateTransaction(getHalHardwareInfoEndPoint), - getHalUsbInfoEndPoint: TransactionDecorator.generateTransaction(getHalUsbInfoEndPoint), getFog, refreshProvisionKeyForFog, setFogPruneCommandEndPoint: TransactionDecorator.generateTransaction(setFogPruneCommandEndPoint), @@ -1565,9 +1418,5 @@ module.exports = { _processDeleteCommand, _reconcileNatsCertificatesOnHostChange, _deleteNatsMicroserviceByFog, - _createHalMicroserviceForFog, - _deleteHalMicroserviceByFog, - _createBluetoothMicroserviceForFog, - _deleteBluetoothMicroserviceByFog, _updateMicroserviceExtraHosts } diff --git a/src/services/knowledge-service.js b/src/services/knowledge-service.js new file mode 100644 index 000000000..047befb04 --- /dev/null +++ b/src/services/knowledge-service.js @@ -0,0 +1,293 @@ +const Errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const AppHelper = require('../helpers/app-helper') +const { normalizeKnowledgeFormat } = require('../helpers/knowledge-format') +const FleetKnowledgeManager = require('../data/managers/fleet-knowledge-manager') +const RegistryManager = require('../data/managers/registry-manager') +const MicroserviceManager = require('../data/managers/microservice-manager') +const MicroserviceKnowledgeItemManager = require('../data/managers/microservice-knowledge-item-manager') +const ApplicationManager = require('../data/managers/application-manager') +const FogManager = require('../data/managers/iofog-manager') +const ChangeTrackingService = require('./change-tracking-service') +const TransactionDecorator = require('../decorators/transaction-decorator') +const Validator = require('../schemas') + +function _toPlain (row) { + if (!row) { + return null + } + return typeof row.toJSON === 'function' ? row.toJSON() : row +} + +function toWireKnowledge (row) { + const plain = _toPlain(row) + if (!plain) { + return null + } + const item = { + uuid: plain.uuid, + name: plain.name, + repo: plain.repo, + revision: plain.revision != null ? plain.revision : '', + registryId: plain.registryId, + files: Array.isArray(plain.files) ? plain.files : [] + } + if (plain.format != null && plain.format !== '') { + item.format = plain.format + } + return item +} + +async function findLinkedFogUuids (knowledge, transaction) { + if (!knowledge || typeof knowledge.getFogs !== 'function') { + return [] + } + const fogs = await knowledge.getFogs({ transaction }) + return fogs.map((fog) => fog.uuid) +} + +async function _updateChangeTrackingForFogs (fogUuids, transaction) { + for (const fogUuid of fogUuids) { + await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.knowledge, transaction) + } +} + +async function _isUserMicroservice (microservice, transaction) { + if (!microservice || microservice.isController) { + return false + } + if (!microservice.applicationId) { + return true + } + const application = await ApplicationManager.findOne({ id: microservice.applicationId }, transaction) + return !application || !application.isSystem +} + +async function findMicroservicesBindingKnowledgeName (knowledgeName, options, transaction) { + const items = await MicroserviceKnowledgeItemManager.findAll({ name: knowledgeName }, transaction) + const blocking = [] + const userOnly = Boolean(options && options.userOnly) + for (const item of items) { + if (userOnly || (options && options.fogUuid)) { + const microservice = await MicroserviceManager.findOne({ uuid: item.microserviceUuid }, transaction) + if (!microservice) { + continue + } + if (options && options.fogUuid && microservice.iofogUuid !== options.fogUuid) { + continue + } + if (userOnly && !(await _isUserMicroservice(microservice, transaction))) { + continue + } + blocking.push(microservice.uuid) + continue + } + blocking.push(item.microserviceUuid) + } + return blocking +} + +async function _validateRegistryAndFiles (registryId, files, transaction) { + const registry = await RegistryManager.findOne({ id: registryId }, transaction) + if (!registry) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_REGISTRY_ID, registryId)) + } + const type = registry.type || 'oci' + const fileList = Array.isArray(files) ? files : [] + if (type !== 'hf' && fileList.length > 0) { + throw new Errors.ValidationError(ErrorMessages.KNOWLEDGE_OCI_FILES_FORBIDDEN) + } + return registry +} + +async function getKnowledgeByName (name, transaction) { + const knowledge = await FleetKnowledgeManager.findOne({ name }, transaction) + if (!knowledge) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.KNOWLEDGE_NOT_FOUND, name)) + } + return knowledge +} + +async function listKnowledgeEndpoint (transaction) { + const knowledge = await FleetKnowledgeManager.findAll({}, transaction) + return { + knowledge: knowledge.map(toWireKnowledge) + } +} + +async function getKnowledgeEndpoint (name, transaction) { + const knowledge = await getKnowledgeByName(name, transaction) + return toWireKnowledge(knowledge) +} + +async function upsertKnowledgeEndpoint (name, data, transaction) { + const existing = await FleetKnowledgeManager.findOne({ name }, transaction) + if (existing) { + return updateKnowledgeEndpoint(name, data, transaction) + } + return createKnowledgeEndpoint({ ...data, name }, transaction) +} + +async function createKnowledgeEndpoint (data, transaction) { + await Validator.validate(data, Validator.schemas.knowledgeCreate) + + const existing = await FleetKnowledgeManager.findOne({ name: data.name }, transaction) + if (existing) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.KNOWLEDGE_ALREADY_EXISTS, data.name)) + } + + const files = Array.isArray(data.files) ? data.files : [] + await _validateRegistryAndFiles(data.registryId, files, transaction) + + const created = await FleetKnowledgeManager.create({ + uuid: AppHelper.generateUUID(), + name: data.name, + repo: data.repo, + revision: data.revision != null ? data.revision : '', + registryId: data.registryId, + files, + format: normalizeKnowledgeFormat(data.format) + }, transaction) + + return toWireKnowledge(created) +} + +async function updateKnowledgeEndpoint (name, data, transaction) { + await Validator.validate(data, Validator.schemas.knowledgeUpdate) + const knowledge = await getKnowledgeByName(name, transaction) + + const nextRepo = data.repo !== undefined ? data.repo : knowledge.repo + const nextRevision = data.revision !== undefined ? data.revision : (knowledge.revision != null ? knowledge.revision : '') + const nextRegistryId = data.registryId !== undefined ? data.registryId : knowledge.registryId + const nextFiles = data.files !== undefined ? data.files : (Array.isArray(knowledge.files) ? knowledge.files : []) + const nextFormat = data.format !== undefined ? normalizeKnowledgeFormat(data.format) : knowledge.format + + await _validateRegistryAndFiles(nextRegistryId, nextFiles, transaction) + + const linkedFogUuids = await findLinkedFogUuids(knowledge, transaction) + + await FleetKnowledgeManager.update({ name }, { + repo: nextRepo, + revision: nextRevision, + registryId: nextRegistryId, + files: nextFiles, + format: nextFormat + }, transaction) + + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + + return getKnowledgeEndpoint(name, transaction) +} + +async function deleteKnowledgeEndpoint (name, transaction) { + const knowledge = await getKnowledgeByName(name, transaction) + const blocking = await findMicroservicesBindingKnowledgeName(name, {}, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.KNOWLEDGE_IN_USE, name, blocking.join(', '))) + } + + const linkedFogUuids = await findLinkedFogUuids(knowledge, transaction) + await FleetKnowledgeManager.delete({ name }, transaction) + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + return {} +} + +async function getKnowledgeLinkEndpoint (name, transaction) { + const knowledge = await getKnowledgeByName(name, transaction) + return { + fogUuids: await findLinkedFogUuids(knowledge, transaction) + } +} + +async function linkKnowledgeEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.knowledgeLink) + const knowledge = await getKnowledgeByName(name, transaction) + const alreadyLinked = new Set(await findLinkedFogUuids(knowledge, transaction)) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + await agent.addKnowledge(knowledge, { transaction }) + } + + const newlyLinked = fogUuids.filter((uuid) => !alreadyLinked.has(uuid)) + if (newlyLinked.length > 0) { + await _updateChangeTrackingForFogs(newlyLinked, transaction) + } + + return getKnowledgeEndpoint(name, transaction) +} + +async function unlinkKnowledgeEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.knowledgeUnlink) + const knowledge = await getKnowledgeByName(name, transaction) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + const blocking = await findMicroservicesBindingKnowledgeName(name, { fogUuid, userOnly: true }, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.KNOWLEDGE_UNLINK_IN_USE, name, fogUuid, blocking.join(', '))) + } + await agent.removeKnowledge(knowledge, { transaction }) + } + + await _updateChangeTrackingForFogs(fogUuids, transaction) + return {} +} + +async function ensureKnowledgeLinkedToFog (fogUuid, names, transaction) { + const knowledgeNames = [...new Set((names || []).filter((name) => typeof name === 'string' && name.length > 0))] + if (knowledgeNames.length === 0) { + return [] + } + + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + + const newlyLinked = [] + for (const name of knowledgeNames) { + const knowledge = await FleetKnowledgeManager.findOne({ name }, transaction) + if (!knowledge) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.KNOWLEDGE_NOT_FOUND, name)) + } + const alreadyLinked = typeof agent.hasKnowledge === 'function' + ? await agent.hasKnowledge(knowledge, { transaction }) + : false + if (alreadyLinked) { + continue + } + await agent.addKnowledge(knowledge, { transaction }) + newlyLinked.push(name) + } + + if (newlyLinked.length > 0) { + await _updateChangeTrackingForFogs([fogUuid], transaction) + } + + return newlyLinked +} + +module.exports = { + listKnowledgeEndpoint: TransactionDecorator.generateTransaction(listKnowledgeEndpoint), + getKnowledgeEndpoint: TransactionDecorator.generateTransaction(getKnowledgeEndpoint), + createKnowledgeEndpoint: TransactionDecorator.generateTransaction(createKnowledgeEndpoint), + updateKnowledgeEndpoint: TransactionDecorator.generateTransaction(updateKnowledgeEndpoint), + deleteKnowledgeEndpoint: TransactionDecorator.generateTransaction(deleteKnowledgeEndpoint), + getKnowledgeLinkEndpoint: TransactionDecorator.generateTransaction(getKnowledgeLinkEndpoint), + linkKnowledgeEndpoint: TransactionDecorator.generateTransaction(linkKnowledgeEndpoint), + unlinkKnowledgeEndpoint: TransactionDecorator.generateTransaction(unlinkKnowledgeEndpoint), + upsertKnowledgeEndpoint: TransactionDecorator.generateTransaction(upsertKnowledgeEndpoint), + ensureKnowledgeLinkedToFog: TransactionDecorator.generateTransaction(ensureKnowledgeLinkedToFog), + toWireKnowledge +} diff --git a/src/services/microservice-template-service.js b/src/services/microservice-template-service.js new file mode 100644 index 000000000..56359f683 --- /dev/null +++ b/src/services/microservice-template-service.js @@ -0,0 +1,279 @@ +const Errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const AppHelper = require('../helpers/app-helper') +const MicroserviceTemplateManager = require('../data/managers/microservice-template-manager') +const MicroserviceTemplateVariableManager = require('../data/managers/microservice-template-variable-manager') +const TransactionDecorator = require('../decorators/transaction-decorator') +const Validator = require('../schemas') + +const INSTANCE_FIELDS = ['name', 'iofogUuid', 'flowId', 'template'] + +function _toPlain (row) { + if (!row) { + return null + } + return typeof row.toJSON === 'function' ? row.toJSON() : row +} + +function _cloneJson (value, fallback) { + if (value == null || value === '') { + return fallback + } + if (typeof value === 'object') { + return JSON.parse(JSON.stringify(value)) + } + try { + return JSON.parse(value) + } catch (error) { + return fallback + } +} + +function _parseStoredDefaultValue (defaultValue) { + if (defaultValue === undefined) { + return undefined + } + if (typeof defaultValue !== 'string') { + return defaultValue + } + try { + return JSON.parse(defaultValue) + } catch (error) { + return defaultValue + } +} + +function stripInstanceFields (microservice) { + const clone = { ...(microservice || {}) } + for (const field of INSTANCE_FIELDS) { + delete clone[field] + } + return clone +} + +function toWireVariable (variable) { + const plain = _toPlain(variable) || variable + const wire = { + key: plain.key + } + if (plain.description !== undefined) { + wire.description = plain.description + } + if (plain.defaultValue !== undefined) { + wire.defaultValue = _parseStoredDefaultValue(plain.defaultValue) + } + return wire +} + +function toWireTemplate (row) { + const plain = _toPlain(row) + if (!plain) { + return null + } + return { + name: plain.name, + description: plain.description != null ? plain.description : '', + variables: (plain.variables || []).map(toWireVariable), + microservice: _cloneJson(plain.microserviceJSON, {}) + } +} + +function _buildUserVariableMap (userVariables) { + if (Array.isArray(userVariables)) { + return userVariables.reduce((acc, variable) => { + if (variable && variable.key != null) { + acc[variable.key] = variable.value + } + return acc + }, {}) + } + if (userVariables && typeof userVariables === 'object') { + return { ...userVariables } + } + return {} +} + +function _buildDefaultVariableMap (schemaVariables) { + return (schemaVariables || []).reduce((acc, variable) => { + if (variable && variable.key != null && variable.defaultValue !== undefined) { + acc[variable.key] = variable.defaultValue + } + return acc + }, {}) +} + +function _parseJsonObjectField (value) { + if (typeof value !== 'string' || value === '') { + return undefined + } + const trimmed = value.trim() + if (!trimmed.startsWith('{') && !trimmed.startsWith('[')) { + return undefined + } + try { + const parsed = JSON.parse(value) + if (parsed && typeof parsed === 'object') { + return parsed + } + } catch (error) { + return undefined + } + return undefined +} + +async function _createVariable (microserviceTemplateId, variableData, transaction) { + const newVariable = { + ...variableData + } + if (newVariable.defaultValue !== undefined) { + newVariable.defaultValue = JSON.stringify(newVariable.defaultValue) + } + return MicroserviceTemplateVariableManager.create({ + ...newVariable, + microserviceTemplateId + }, transaction) +} + +async function _updateVariables (microserviceTemplateId, variables, transaction) { + await MicroserviceTemplateVariableManager.delete({ microserviceTemplateId }, transaction) + for (const variableData of variables || []) { + await _createVariable(microserviceTemplateId, variableData, transaction) + } +} + +async function getTemplateByName (name, transaction) { + const template = await MicroserviceTemplateManager.findOnePopulated({ name }, transaction) + if (!template) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_TEMPLATE_NOT_FOUND, name)) + } + return template +} + +async function listMicroserviceTemplatesEndpoint (transaction) { + const templates = await MicroserviceTemplateManager.findAllPopulated({}, transaction) + return { + microserviceTemplates: templates.map(toWireTemplate) + } +} + +async function getMicroserviceTemplateEndpoint (name, transaction) { + const template = await getTemplateByName(name, transaction) + return toWireTemplate(template) +} + +async function createMicroserviceTemplateEndpoint (data, transaction) { + await Validator.validate(data, Validator.schemas.microserviceTemplateCreate) + + const existing = await MicroserviceTemplateManager.findOne({ name: data.name }, transaction) + if (existing) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_TEMPLATE_ALREADY_EXISTS, data.name)) + } + + const created = await MicroserviceTemplateManager.create({ + name: data.name, + description: data.description != null ? data.description : '', + microserviceJSON: JSON.stringify(stripInstanceFields(data.microservice)) + }, transaction) + + try { + if (data.variables) { + for (const variableData of data.variables) { + await _createVariable(created.id, variableData, transaction) + } + } + return getMicroserviceTemplateEndpoint(created.name, transaction) + } catch (error) { + await deleteMicroserviceTemplateEndpoint(created.name, transaction) + throw error + } +} + +async function updateMicroserviceTemplateEndpoint (name, data, transaction) { + await Validator.validate(data, Validator.schemas.microserviceTemplateUpdate) + const existing = await getTemplateByName(name, transaction) + const current = toWireTemplate(existing) + + const nextMicroservice = data.microservice !== undefined + ? stripInstanceFields(data.microservice) + : current.microservice + const nextDescription = data.description !== undefined ? data.description : current.description + + await MicroserviceTemplateManager.update({ name }, { + description: nextDescription, + microserviceJSON: JSON.stringify(nextMicroservice) + }, transaction) + + if (data.variables !== undefined) { + await _updateVariables(existing.id, data.variables, transaction) + } + + return getMicroserviceTemplateEndpoint(name, transaction) +} + +async function upsertMicroserviceTemplateEndpoint (name, data, transaction) { + const existing = await MicroserviceTemplateManager.findOne({ name }, transaction) + if (existing) { + return updateMicroserviceTemplateEndpoint(name, data, transaction) + } + return createMicroserviceTemplateEndpoint({ ...data, name }, transaction) +} + +async function deleteMicroserviceTemplateEndpoint (name, transaction) { + await getTemplateByName(name, transaction) + await MicroserviceTemplateManager.delete({ name }, transaction) + return {} +} + +async function getMicroserviceDataFromTemplate (deploymentData, isCLI, transaction) { + await Validator.validate(deploymentData, Validator.schemas.microserviceTemplateDeploy) + + const templateRow = await MicroserviceTemplateManager.findOnePopulated({ name: deploymentData.name }, transaction) + if (!templateRow) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_TEMPLATE_NOT_FOUND, deploymentData.name)) + } + + const template = toWireTemplate(templateRow) + if (!template.microservice || typeof template.microservice !== 'object' || Array.isArray(template.microservice)) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_TEMPLATE_INVALID, deploymentData.name)) + } + + const spec = stripInstanceFields(template.microservice) + const parsedConfig = _parseJsonObjectField(spec.config) + if (parsedConfig !== undefined) { + spec.config = parsedConfig + } + const parsedAnnotations = _parseJsonObjectField(spec.annotations) + if (parsedAnnotations !== undefined) { + spec.annotations = parsedAnnotations + } + + const { rvaluesVarSubstition } = require('../helpers/template-helper') + const variableMap = { + ..._buildDefaultVariableMap(template.variables), + ..._buildUserVariableMap(deploymentData.variables), + self: spec + } + await rvaluesVarSubstition(spec, variableMap) + + if (spec.config && typeof spec.config === 'object' && !Array.isArray(spec.config)) { + spec.config = JSON.stringify(spec.config) + } + if (spec.annotations && typeof spec.annotations === 'object' && !Array.isArray(spec.annotations)) { + spec.annotations = JSON.stringify(spec.annotations) + } + + return spec +} + +module.exports = { + listMicroserviceTemplatesEndpoint: TransactionDecorator.generateTransaction(listMicroserviceTemplatesEndpoint), + getMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(getMicroserviceTemplateEndpoint), + createMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(createMicroserviceTemplateEndpoint), + updateMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(updateMicroserviceTemplateEndpoint), + upsertMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(upsertMicroserviceTemplateEndpoint), + deleteMicroserviceTemplateEndpoint: TransactionDecorator.generateTransaction(deleteMicroserviceTemplateEndpoint), + getMicroserviceDataFromTemplate, + toWireTemplate, + stripInstanceFields, + stripIdentityFields: stripInstanceFields +} diff --git a/src/services/microservice-workload-spec.js b/src/services/microservice-workload-spec.js index 599ab1194..636e5e19a 100644 --- a/src/services/microservice-workload-spec.js +++ b/src/services/microservice-workload-spec.js @@ -2,10 +2,15 @@ const AppHelper = require('../helpers/app-helper') const Errors = require('../helpers/errors') const ErrorMessages = require('../helpers/error-messages') const constants = require('../helpers/constants') +const CatalogContainer = require('../helpers/microservice-container-catalog') const FogManager = require('../data/managers/iofog-manager') const ApplicationManager = require('../data/managers/application-manager') const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceArgManager = require('../data/managers/microservice-arg-manager') +const MicroserviceEntrypointManager = require('../data/managers/microservice-entrypoint-manager') +const MicroserviceDeviceManager = require('../data/managers/microservice-device-manager') +const MicroserviceTmpfsManager = require('../data/managers/microservice-tmpfs-manager') +const MicroserviceUlimitManager = require('../data/managers/microservice-ulimit-manager') const MicroserviceCdiDevManager = require('../data/managers/microservice-cdi-device-manager') const MicroserviceCapAddManager = require('../data/managers/microservice-cap-add-manager') const MicroserviceCapDropManager = require('../data/managers/microservice-cap-drop-manager') @@ -64,7 +69,8 @@ function buildScalarColumns (spec, { defaultLogSize = constants.MICROSERVICE_DEF runAsUser: spec.runAsUser, platform: spec.platform, cpuSetCpus: spec.cpuSetCpus, - memoryLimit: spec.memoryLimit + memoryLimit: spec.memoryLimit, + ...CatalogContainer.serializeContainerColumns(spec) }) } @@ -255,6 +261,61 @@ async function _updateArg (arg, microserviceUuid, transaction) { } } +async function _replaceEntrypoint (entrypoint, microserviceUuid, transaction) { + await MicroserviceEntrypointManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(entrypoint)) { + return + } + for (const token of entrypoint) { + await MicroserviceEntrypointManager.create({ entrypoint: token, microserviceUuid }, transaction) + } +} + +async function _replaceDevices (devices, microserviceUuid, transaction) { + await MicroserviceDeviceManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(devices)) { + return + } + for (const device of devices) { + await MicroserviceDeviceManager.create({ + hostPath: device.hostPath, + containerPath: device.containerPath, + permissions: device.permissions || 'rwm', + microserviceUuid + }, transaction) + } +} + +async function _replaceTmpfs (tmpfs, microserviceUuid, transaction) { + await MicroserviceTmpfsManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(tmpfs)) { + return + } + for (const entry of tmpfs) { + await MicroserviceTmpfsManager.create({ + containerPath: entry.containerPath, + size: entry.size, + mode: entry.mode, + microserviceUuid + }, transaction) + } +} + +async function _replaceUlimits (ulimits, microserviceUuid, transaction) { + await MicroserviceUlimitManager.delete({ microserviceUuid }, transaction) + if (!ulimits || typeof ulimits !== 'object') { + return + } + for (const [name, value] of Object.entries(ulimits)) { + await MicroserviceUlimitManager.create({ + name, + soft: value.soft, + hard: value.hard, + microserviceUuid + }, transaction) + } +} + async function _createCdiDevice (microserviceUuid, cdiDevice, transaction) { await MicroserviceCdiDevManager.create({ cdiDevices: cdiDevice, microserviceUuid }, transaction) } @@ -306,6 +367,7 @@ async function _updateHealthCheck (microserviceUuid, healthCheck, transaction) { } async function createWorkloadRelations (microservice, spec, { validatedExtraHosts, transaction }) { + CatalogContainer.validateContainerFields(spec, {}) const microserviceUuid = microservice.uuid if (validatedExtraHosts && validatedExtraHosts.length) { @@ -320,12 +382,26 @@ async function createWorkloadRelations (microservice, spec, { validatedExtraHost } } - if (spec.cmd) { - for (const arg of spec.cmd) { + const argv = CatalogContainer.resolveProcessArgv(spec) + if (argv) { + for (const arg of argv) { await _createArg(microserviceUuid, arg, transaction) } } + if (spec.entrypoint !== undefined) { + await _replaceEntrypoint(spec.entrypoint, microserviceUuid, transaction) + } + if (spec.devices !== undefined) { + await _replaceDevices(spec.devices, microserviceUuid, transaction) + } + if (spec.tmpfs !== undefined) { + await _replaceTmpfs(spec.tmpfs, microserviceUuid, transaction) + } + if (spec.ulimits !== undefined) { + await _replaceUlimits(spec.ulimits, microserviceUuid, transaction) + } + if (spec.cdiDevices) { for (const cdiDevice of spec.cdiDevices) { await _createCdiDevice(microserviceUuid, cdiDevice, transaction) @@ -350,6 +426,8 @@ async function createWorkloadRelations (microservice, spec, { validatedExtraHost } async function updateWorkloadRelations (microserviceUuid, spec, { validatedExtraHosts, transaction }) { + CatalogContainer.validateContainerFields(spec, {}) + if (validatedExtraHosts) { await _updateExtraHosts(validatedExtraHosts, microserviceUuid, transaction) } @@ -358,8 +436,22 @@ async function updateWorkloadRelations (microserviceUuid, spec, { validatedExtra await _updateEnv(spec.env, microserviceUuid, transaction) } - if (spec.cmd) { - await _updateArg(spec.cmd, microserviceUuid, transaction) + const argv = CatalogContainer.resolveProcessArgv(spec) + if (argv !== undefined) { + await _updateArg(argv, microserviceUuid, transaction) + } + + if (spec.entrypoint !== undefined) { + await _replaceEntrypoint(spec.entrypoint, microserviceUuid, transaction) + } + if (spec.devices !== undefined) { + await _replaceDevices(spec.devices, microserviceUuid, transaction) + } + if (spec.tmpfs !== undefined) { + await _replaceTmpfs(spec.tmpfs, microserviceUuid, transaction) + } + if (spec.ulimits !== undefined) { + await _replaceUlimits(spec.ulimits, microserviceUuid, transaction) } if (spec.cdiDevices) { @@ -400,10 +492,23 @@ function shouldRebuildForWorkloadChange (existing, spec, { config, annotations, spec.ports || spec.extraHosts || spec.cmd || + spec.commands || + spec.entrypoint || spec.cdiDevices || spec.capAdd || spec.capDrop || - spec.healthCheck + spec.healthCheck || + spec.runAsGroup || + spec.readOnlyRootFilesystem || + spec.workingDir || + spec.cpus || + spec.memoryReservation || + spec.memorySwap || + spec.shmSize || + spec.sysctls || + spec.ulimits || + spec.devices || + spec.tmpfs ) } diff --git a/src/services/microservices-service.js b/src/services/microservices-service.js index 00c64cd9f..8ac4d0bb7 100644 --- a/src/services/microservices-service.js +++ b/src/services/microservices-service.js @@ -3,6 +3,14 @@ const MicroserviceManager = require('../data/managers/microservice-manager') const MicroserviceStatusManager = require('../data/managers/microservice-status-manager') const MicroserviceExecStatusManager = require('../data/managers/microservice-exec-status-manager') const MicroserviceArgManager = require('../data/managers/microservice-arg-manager') +const MicroserviceEntrypointManager = require('../data/managers/microservice-entrypoint-manager') +const MicroserviceDeviceManager = require('../data/managers/microservice-device-manager') +const MicroserviceTmpfsManager = require('../data/managers/microservice-tmpfs-manager') +const MicroserviceUlimitManager = require('../data/managers/microservice-ulimit-manager') +const MicroserviceModelManager = require('../data/managers/microservice-model-manager') +const MicroserviceModelItemManager = require('../data/managers/microservice-model-item-manager') +const MicroserviceKnowledgeManager = require('../data/managers/microservice-knowledge-manager') +const MicroserviceKnowledgeItemManager = require('../data/managers/microservice-knowledge-item-manager') const MicroserviceCdiDevManager = require('../data/managers/microservice-cdi-device-manager') const MicroserviceCapAddManager = require('../data/managers/microservice-cap-add-manager') const MicroserviceCapDropManager = require('../data/managers/microservice-cap-drop-manager') @@ -13,6 +21,7 @@ const CatalogItemImageManager = require('../data/managers/catalog-item-image-man const RegistryManager = require('../data/managers/registry-manager') // const RouterManager = require('../data/managers/router-manager') const MicroserviceStates = require('../enums/microservice-state') +const { microserviceState, microserviceExecState } = MicroserviceStates const VolumeMappingManager = require('../data/managers/volume-mapping-manager') const ChangeTrackingService = require('./change-tracking-service') const AppHelper = require('../helpers/app-helper') @@ -33,6 +42,13 @@ const ServiceServices = require('./services-service') const ConfigMapManager = require('../data/managers/config-map-manager') const SecretManager = require('../data/managers/secret-manager') const VolumeMountService = require('./volume-mount-service') +const RuntimeClassService = require('./runtime-class-service') +const ModelService = require('./model-service') +const KnowledgeService = require('./knowledge-service') +const RegistryService = require('./registry-service') +const FleetModelManager = require('../data/managers/fleet-model-manager') +const FleetKnowledgeManager = require('../data/managers/fleet-knowledge-manager') +const CatalogContainer = require('../helpers/microservice-container-catalog') const RbacServiceAccountManager = require('../data/managers/rbac-service-account-manager') const RbacRoleManager = require('../data/managers/rbac-role-manager') const NatsAuthService = require('./nats-auth-service') @@ -40,17 +56,40 @@ const NatsUserRuleManager = require('../data/managers/nats-user-rule-manager') const NatsRuleJwtValidation = require('../helpers/nats-rule-jwt-validation') const NatsInstanceManager = require('../data/managers/nats-instance-manager') const { resolveNatsServerUrl } = require('../helpers/nats-server-url') +const { + applyRuntimeMetrics, + projectStatusForApi, + zeroRuntimeMetrics +} = require('../helpers/microservice-runtime-metrics') const Op = require('sequelize').Op const FogManager = require('../data/managers/iofog-manager') const MicroserviceExtraHostManager = require('../data/managers/microservice-extra-host-manager') const { VOLUME_MAPPING_DEFAULT } = require('../helpers/constants') +const { applyVolumeMappingScope, PRIVATE_SCOPE } = require('../helpers/volume-mapping-scope') const constants = require('../helpers/constants') const logger = require('../logger') const SERVICE_ACCOUNT_VOLUME_TYPE = 'serviceAccount' const SERVICE_ACCOUNT_VOLUME_CONTAINER_DESTINATION = '/var/run/secrets/edgelet.iofog.org/serviceaccount' const SERVICE_ACCOUNT_VOLUME_ACCESS_MODE = 'ro' + +async function _resolveServiceAccountForUserResponse (microservice, transaction) { + let serviceAccount = microservice.serviceAccount + if (!serviceAccount && microservice.uuid) { + serviceAccount = await RbacServiceAccountManager.findOneByMicroserviceUuid(microservice.uuid, transaction) + } + if (!serviceAccount) { + return null + } + const plain = serviceAccount.get ? serviceAccount.get({ plain: true }) : serviceAccount + if (!plain.roleRef) { + return null + } + return { + roleRef: plain.roleRef + } +} const NATS_CREDS_PATH_ENV = 'NATS_CREDS_PATH' const NATS_SERVER_URL_ENV = 'NATS_SERVER_URL' @@ -425,7 +464,7 @@ function _parseFogAvailableRuntimes (fog) { } } -function _validateMicroserviceRuntime (runtime, fog) { +async function _validateMicroserviceRuntime (runtime, fog, transaction) { if (runtime == null || runtime === '') { return } @@ -436,6 +475,272 @@ function _validateMicroserviceRuntime (runtime, fog) { `Runtime '${runtime}' is not available on agent '${agentLabel}'` ) } + await RuntimeClassService.ensureRuntimeClassLinkedToFog(fog.uuid, runtime, transaction) +} + +async function _assertOciRegistryForImage (registryId, transaction) { + if (registryId == null) { + return + } + await RegistryService.assertOciRegistryForImage(registryId, transaction) +} + +async function _assertCatalogModelsExist (catalog, transaction) { + const names = CatalogContainer.catalogItemNames(catalog) + for (const name of names) { + const model = await FleetModelManager.findOne({ name }, transaction) + if (!model) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_CATALOG_MODEL_NOT_FOUND, name)) + } + } +} + +async function _autoAttachCatalogModels (fogUuid, catalog, transaction) { + if (!fogUuid) { + return + } + const names = CatalogContainer.catalogItemNames(catalog) + if (names.length === 0) { + return + } + await ModelService.ensureModelsLinkedToFog(fogUuid, names, transaction) +} + +async function _assertCatalogKnowledgeExist (catalog, transaction) { + const names = CatalogContainer.catalogItemNames(catalog) + for (const name of names) { + const knowledge = await FleetKnowledgeManager.findOne({ name }, transaction) + if (!knowledge) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_KNOWLEDGE_NOT_FOUND, name)) + } + } +} + +async function _autoAttachCatalogKnowledge (fogUuid, catalog, transaction) { + if (!fogUuid) { + return + } + const names = CatalogContainer.catalogItemNames(catalog) + if (names.length === 0) { + return + } + await KnowledgeService.ensureKnowledgeLinkedToFog(fogUuid, names, transaction) +} + +function _assertKnowledgeCatalogAllowed (microserviceData, microservice) { + if (!microserviceData || microserviceData.knowledge === undefined) { + return + } + const catalogItem = microservice && microservice.catalogItem + const isSystem = catalogItem && catalogItem.category === 'SYSTEM' + const isController = microservice && microservice.isController + if (isSystem || isController) { + throw new Errors.ValidationError(ErrorMessages.MICROSERVICE_KNOWLEDGE_SYSTEM_FORBIDDEN) + } +} + +async function _loadCatalog (microserviceUuid, transaction) { + const catalogRow = await MicroserviceModelManager.findOne({ microserviceUuid }, transaction) + const itemRows = await MicroserviceModelItemManager.findAll({ microserviceUuid }, transaction) + return CatalogContainer.catalogFromRows(catalogRow, itemRows) +} + +async function _loadKnowledgeCatalog (microserviceUuid, transaction) { + const catalogRow = await MicroserviceKnowledgeManager.findOne({ microserviceUuid }, transaction) + const itemRows = await MicroserviceKnowledgeItemManager.findAll({ microserviceUuid }, transaction) + return CatalogContainer.catalogFromRows(catalogRow, itemRows) +} + +async function _loadTmpfs (microserviceUuid, transaction) { + const rows = await MicroserviceTmpfsManager.findAll({ microserviceUuid }, transaction) + return CatalogContainer.tmpfsFromRows(rows) +} + +function _catalogMountContext (microserviceData, existing, volumeMappings, tmpfs) { + return { + volumeMappings: volumeMappings !== undefined ? volumeMappings : existing.volumeMappings, + tmpfs: tmpfs !== undefined ? tmpfs : (microserviceData.tmpfs !== undefined ? microserviceData.tmpfs : existing.tmpfs) + } +} + +function _prepareCatalogAndContainerFields (microserviceData, existing = {}, volumeMappings, tmpfs) { + CatalogContainer.validateContainerFields(microserviceData, existing) + const mountContext = _catalogMountContext(microserviceData, existing, volumeMappings, tmpfs) + const nextModels = microserviceData.models !== undefined + ? CatalogContainer.normalizeCatalog(microserviceData.models) + : existing.models + const nextKnowledge = microserviceData.knowledge !== undefined + ? CatalogContainer.normalizeCatalog(microserviceData.knowledge) + : existing.knowledge + if (microserviceData.models !== undefined) { + CatalogContainer.validateCatalog(nextModels, { + ...mountContext, + otherCatalog: nextKnowledge, + kind: 'model' + }) + } + if (microserviceData.knowledge !== undefined) { + CatalogContainer.validateCatalog(nextKnowledge, { + ...mountContext, + otherCatalog: nextModels, + kind: 'knowledge' + }) + } +} + +function _assignContainerColumns (target, microserviceData) { + Object.assign(target, CatalogContainer.serializeContainerColumns(microserviceData)) +} + +async function _replaceEntrypoint (entrypoint, microserviceUuid, transaction) { + await MicroserviceEntrypointManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(entrypoint)) { + return + } + for (const token of entrypoint) { + await MicroserviceEntrypointManager.create({ + entrypoint: token, + microserviceUuid + }, transaction) + } +} + +async function _replaceDevices (devices, microserviceUuid, transaction) { + await MicroserviceDeviceManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(devices)) { + return + } + for (const device of devices) { + await MicroserviceDeviceManager.create({ + hostPath: device.hostPath, + containerPath: device.containerPath, + permissions: device.permissions || 'rwm', + microserviceUuid + }, transaction) + } +} + +async function _replaceTmpfs (tmpfs, microserviceUuid, transaction) { + await MicroserviceTmpfsManager.delete({ microserviceUuid }, transaction) + if (!Array.isArray(tmpfs)) { + return + } + for (const entry of tmpfs) { + await MicroserviceTmpfsManager.create({ + containerPath: entry.containerPath, + size: entry.size, + mode: entry.mode, + microserviceUuid + }, transaction) + } +} + +async function _replaceUlimits (ulimits, microserviceUuid, transaction) { + await MicroserviceUlimitManager.delete({ microserviceUuid }, transaction) + if (!ulimits || typeof ulimits !== 'object') { + return + } + for (const [name, value] of Object.entries(ulimits)) { + await MicroserviceUlimitManager.create({ + name, + soft: value.soft, + hard: value.hard, + microserviceUuid + }, transaction) + } +} + +async function _replaceCatalog (catalogInput, microserviceUuid, transaction) { + await MicroserviceModelItemManager.delete({ microserviceUuid }, transaction) + await MicroserviceModelManager.delete({ microserviceUuid }, transaction) + const catalog = CatalogContainer.normalizeCatalog(catalogInput) + if (CatalogContainer.isCatalogEmpty(catalog)) { + return + } + await MicroserviceModelManager.create({ + bindPath: catalog.bindPath, + permissions: catalog.permissions || 'ro', + microserviceUuid + }, transaction) + for (const item of catalog.items) { + await MicroserviceModelItemManager.create({ + name: item.name, + microserviceUuid + }, transaction) + } +} + +async function _replaceKnowledgeCatalog (catalogInput, microserviceUuid, transaction) { + await MicroserviceKnowledgeItemManager.delete({ microserviceUuid }, transaction) + await MicroserviceKnowledgeManager.delete({ microserviceUuid }, transaction) + const catalog = CatalogContainer.normalizeCatalog(catalogInput) + if (CatalogContainer.isCatalogEmpty(catalog)) { + return + } + await MicroserviceKnowledgeManager.create({ + bindPath: catalog.bindPath, + permissions: catalog.permissions || 'ro', + microserviceUuid + }, transaction) + for (const item of catalog.items) { + await MicroserviceKnowledgeItemManager.create({ + name: item.name, + microserviceUuid + }, transaction) + } +} + +async function _createContainerChildRows (microservice, microserviceData, transaction) { + if (microserviceData.entrypoint !== undefined) { + await _replaceEntrypoint(microserviceData.entrypoint, microservice.uuid, transaction) + } + if (microserviceData.devices !== undefined) { + await _replaceDevices(microserviceData.devices, microservice.uuid, transaction) + } + if (microserviceData.tmpfs !== undefined) { + await _replaceTmpfs(microserviceData.tmpfs, microservice.uuid, transaction) + } + if (microserviceData.ulimits !== undefined) { + await _replaceUlimits(microserviceData.ulimits, microservice.uuid, transaction) + } + if (microserviceData.models !== undefined) { + await _replaceCatalog(microserviceData.models, microservice.uuid, transaction) + } + if (microserviceData.knowledge !== undefined) { + await _replaceKnowledgeCatalog(microserviceData.knowledge, microservice.uuid, transaction) + } +} + +async function _applyCatalogContainerUpdate (microserviceData, microservice, microserviceDataUpdate, fogUuid, transaction) { + const volumeMappings = microserviceData.volumeMappings !== undefined + ? microserviceData.volumeMappings + : await VolumeMappingManager.findAll({ microserviceUuid: microservice.uuid }, transaction) + const tmpfs = microserviceData.tmpfs !== undefined + ? microserviceData.tmpfs + : await _loadTmpfs(microservice.uuid, transaction) + const existingCatalog = await _loadCatalog(microservice.uuid, transaction) + const existingKnowledge = await _loadKnowledgeCatalog(microservice.uuid, transaction) + microservice.models = existingCatalog + microservice.knowledge = existingKnowledge + _assertKnowledgeCatalogAllowed(microserviceData, microservice) + _prepareCatalogAndContainerFields(microserviceData, microservice, volumeMappings, tmpfs) + _assignContainerColumns(microserviceDataUpdate, microserviceData) + const argv = CatalogContainer.resolveProcessArgv(microserviceData) + if (argv !== undefined) { + microserviceDataUpdate.cmd = argv + } + await _assertOciRegistryForImage(microserviceDataUpdate.registryId, transaction) + if (microserviceData.models !== undefined) { + const catalog = CatalogContainer.normalizeCatalog(microserviceData.models) + await _assertCatalogModelsExist(catalog, transaction) + await _autoAttachCatalogModels(fogUuid, catalog, transaction) + } + if (microserviceData.knowledge !== undefined) { + const catalog = CatalogContainer.normalizeCatalog(microserviceData.knowledge) + await _assertCatalogKnowledgeExist(catalog, transaction) + await _autoAttachCatalogKnowledge(fogUuid, catalog, transaction) + } + await _createContainerChildRows(microservice, microserviceData, transaction) } function _isServiceAccountVolumeType (type) { @@ -520,7 +825,48 @@ async function _normalizeMicroserviceNatsConfig (microserviceData, transaction, } } +function _identityOverlayFromRequest (request, existing = null) { + const overlay = {} + if (request.name !== undefined) { + overlay.name = request.name + } else if (existing?.name) { + overlay.name = existing.name + } + if (request.application !== undefined) { + overlay.application = request.application + } + if (request.iofogUuid !== undefined) { + overlay.iofogUuid = request.iofogUuid + } else if (existing?.iofogUuid) { + overlay.iofogUuid = existing.iofogUuid + } + if (request.agentName !== undefined) { + overlay.agentName = request.agentName + } + return overlay +} + +async function _applyMicroserviceTemplateOverlay (microserviceData, isCLI, transaction, existing = null) { + if (!microserviceData.template || !microserviceData.template.name) { + return microserviceData + } + + const MicroserviceTemplateService = require('./microservice-template-service') + const identity = _identityOverlayFromRequest(microserviceData, existing) + const fromTemplate = await MicroserviceTemplateService.getMicroserviceDataFromTemplate( + microserviceData.template, + isCLI, + transaction + ) + return { + ...fromTemplate, + ...identity + } +} + async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) { + microserviceData = await _applyMicroserviceTemplateOverlay(microserviceData, isCLI, transaction) + // API Retro compatibility if (!microserviceData.application) { microserviceData.application = microserviceData.flowId @@ -537,12 +883,13 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) // Set fog uuid for further reference microserviceData.iofogUuid = fog.uuid - _validateMicroserviceRuntime(microserviceData.runtime, fog) + await _validateMicroserviceRuntime(microserviceData.runtime, fog, transaction) // validate images + let catalogItem = null if (microserviceData.catalogItemId) { // validate catalog item - const catalogItem = await CatalogService.getCatalogItem(microserviceData.catalogItemId, isCLI, transaction) + catalogItem = await CatalogService.getCatalogItem(microserviceData.catalogItemId, isCLI, transaction) validateImagesAgainstCatalog(catalogItem, microserviceData.images || []) microserviceData.images = catalogItem.images _validateImageArch(microserviceData, fog, catalogItem.images) @@ -559,6 +906,8 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.MICROSERVICE_DOES_NOT_HAVE_IMAGES, microserviceData.name)) } + await _assertOciRegistryForImage(microserviceData.registryId || 1, transaction) + // validate extraHosts const extraHosts = await _validateExtraHosts(microserviceData, fog.uuid, transaction) @@ -566,6 +915,15 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) _validateVolumeMappings(microserviceData.volumeMappings) + _prepareCatalogAndContainerFields(microserviceData, {}, microserviceData.volumeMappings, microserviceData.tmpfs) + _assertKnowledgeCatalogAllowed(microserviceData, { catalogItem }) + if (microserviceData.models !== undefined) { + await _assertCatalogModelsExist(CatalogContainer.normalizeCatalog(microserviceData.models), transaction) + } + if (microserviceData.knowledge !== undefined) { + await _assertCatalogKnowledgeExist(CatalogContainer.normalizeCatalog(microserviceData.knowledge), transaction) + } + const microservice = await _createMicroservice({ ...microserviceData, iofogUuid: fog.uuid }, isCLI, transaction) if (!microserviceData.catalogItemId) { @@ -605,8 +963,9 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) await _createEnv(microservice, env, transaction) } } - if (microserviceData.cmd) { - for (const arg of microserviceData.cmd) { + const createArgv = CatalogContainer.resolveProcessArgv(microserviceData) + if (createArgv) { + for (const arg of createArgv) { await _createArg(microservice, arg, transaction) } } @@ -638,7 +997,17 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) await _injectServiceAccountVolume(microservice, transaction) + await _createContainerChildRows(microservice, microserviceData, transaction) + + if (microserviceData.models !== undefined) { + await _autoAttachCatalogModels(fog.uuid, CatalogContainer.normalizeCatalog(microserviceData.models), transaction) + } + if (microserviceData.knowledge !== undefined) { + await _autoAttachCatalogKnowledge(fog.uuid, CatalogContainer.normalizeCatalog(microserviceData.knowledge), transaction) + } + if (microserviceData.iofogUuid) { + // Full create always reloads the microservice list on the agent. await _updateChangeTracking(false, microserviceData.iofogUuid, transaction) } @@ -681,15 +1050,12 @@ async function createMicroserviceEndPoint (microserviceData, isCLI, transaction) return res } -function _validateVolumeMappings (volumeMappings) { +function _validateVolumeMappings (volumeMappings, options = {}) { _stripUserServiceAccountVolumeMappings(volumeMappings) if (volumeMappings) { for (const mapping of volumeMappings) { mapping.type = mapping.type || VOLUME_MAPPING_DEFAULT - if (mapping.type === 'volume' && (!/^[a-zA-Z0-9_.-]/.test(mapping.hostDestination))) { - throw new Errors.InvalidArgumentError('hostDestination includes invalid characters for a local volume name, only ' + - '"[a-zA-Z0-9][a-zA-Z0-9_.-]" are allowed. If you intended to pass a host directory, use type: bind') - } + applyVolumeMappingScope(mapping, options) if (mapping.type === 'volumeMount') { if (!mapping.hostDestination || mapping.hostDestination === '') { throw new Errors.ValidationError('hostDestination is required when type is volumeMount') @@ -699,6 +1065,14 @@ function _validateVolumeMappings (volumeMappings) { } } +async function _rejectsSharedVolumeScope (microservice, transaction) { + if (microservice.isController) { + return true + } + const app = await ApplicationManager.findOne({ id: microservice.applicationId }, transaction) + return !!(app && app.isSystem === true) +} + function _validateKeyPath (data, keyPath, resourceName, resourceType, volumeMountName) { if (!keyPath || keyPath === '') { return true // No key path to validate @@ -936,6 +1310,14 @@ async function updateSystemMicroserviceEndPoint (microserviceUuid, microserviceD microserviceDataUpdate.registryId = microservice.registryId } + await _applyCatalogContainerUpdate( + microserviceData, + microservice, + microserviceDataUpdate, + microserviceDataUpdate.iofogUuid || microservice.iofogUuid, + transaction + ) + if (microserviceDataUpdate.ports) { await _updateSystemPorts(microserviceDataUpdate.ports, microservice, transaction) } @@ -1023,7 +1405,7 @@ async function updateSystemMicroserviceEndPoint (microserviceUuid, microserviceD const runtimeToValidate = microserviceDataUpdate.runtime !== undefined ? microserviceDataUpdate.runtime : microservice.runtime - _validateMicroserviceRuntime(runtimeToValidate, fog) + await _validateMicroserviceRuntime(runtimeToValidate, fog, transaction) } } @@ -1048,7 +1430,7 @@ async function updateSystemMicroserviceEndPoint (microserviceUuid, microserviceD microserviceDataUpdate.ports || (microserviceDataUpdate.schedule !== undefined && microserviceDataUpdate.schedule !== microservice.schedule) || extraHosts - ) + ) || CatalogContainer.containerFieldsRequireRebuild(microserviceData, microservice) const updatedMicroservice = await MicroserviceManager.updateAndFind(query, microserviceDataUpdate, transaction) if (extraHosts) { @@ -1152,6 +1534,7 @@ async function updateSystemMicroserviceEndPoint (microserviceUuid, microserviceD async function updateMicroserviceEndPoint (microserviceUuid, microserviceData, isCLI, transaction, changeTrackingEnabled = true) { const current = await MicroserviceManager.findOne({ uuid: microserviceUuid }, transaction) + microserviceData = await _applyMicroserviceTemplateOverlay(microserviceData, isCLI, transaction, current) await _normalizeMicroserviceNatsConfig(microserviceData, transaction, current) await Validator.validate(microserviceData, Validator.schemas.microserviceUpdate) let needStatusReset = false @@ -1327,10 +1710,18 @@ async function updateMicroserviceEndPoint (microserviceUuid, microserviceData, i const runtimeToValidate = microserviceDataUpdate.runtime !== undefined ? microserviceDataUpdate.runtime : microservice.runtime - _validateMicroserviceRuntime(runtimeToValidate, fog) + await _validateMicroserviceRuntime(runtimeToValidate, fog, transaction) } } + await _applyCatalogContainerUpdate( + microserviceData, + microservice, + microserviceDataUpdate, + microserviceDataUpdate.iofogUuid || microservice.iofogUuid, + transaction + ) + // Set rebuild flag if needed microserviceDataUpdate.rebuild = microserviceDataUpdate.rebuild || !!( (microserviceDataUpdate.hostNetworkMode !== undefined && microservice.hostNetworkMode !== microserviceDataUpdate.hostNetworkMode) || @@ -1352,7 +1743,7 @@ async function updateMicroserviceEndPoint (microserviceUuid, microserviceData, i microserviceDataUpdate.ports || (microserviceDataUpdate.schedule !== undefined && microserviceDataUpdate.schedule !== microservice.schedule) || extraHosts - ) + ) || CatalogContainer.containerFieldsRequireRebuild(microserviceData, microservice) const updatedMicroservice = await MicroserviceManager.updateAndFind(query, microserviceDataUpdate, transaction) if (extraHosts) { @@ -1469,6 +1860,12 @@ async function updateMicroserviceEndPoint (microserviceUuid, microserviceData, i if (changeTrackingEnabled) { await _updateChangeTracking(true, microservice.iofogUuid, transaction) await _updateChangeTracking(true, updatedMicroservice.iofogUuid, transaction) + if (microserviceData.knowledge !== undefined) { + await _flagMicroserviceKnowledge(microservice.iofogUuid, transaction) + if (updatedMicroservice.iofogUuid !== microservice.iofogUuid) { + await _flagMicroserviceKnowledge(updatedMicroservice.iofogUuid, transaction) + } + } } else { return { microserviceIofogUuid: microservice.iofogUuid, @@ -1501,6 +1898,101 @@ async function updateMicroserviceConfigEndPoint (microserviceUuid, config, isCLI } } +async function updateMicroserviceCatalogEndPoint (microserviceUuid, catalogData, isCLI, transaction) { + await Validator.validate(catalogData, Validator.schemas.microserviceCatalogPatch) + + const query = { uuid: microserviceUuid } + const microservice = await MicroserviceManager.findOneWithCategory(query, transaction) + if (!microservice) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_MICROSERVICE_UUID, microserviceUuid)) + } + if (microservice.catalogItem && microservice.catalogItem.category === 'SYSTEM') { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) + } + if (microservice.isController) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) + } + if (!isCLI) { + await _validateMicroserviceOnGet(microserviceUuid, transaction) + } + + const catalog = CatalogContainer.normalizeCatalog(catalogData) + const volumeMappings = await VolumeMappingManager.findAll({ microserviceUuid }, transaction) + const tmpfs = await _loadTmpfs(microserviceUuid, transaction) + const knowledgeCatalog = await _loadKnowledgeCatalog(microserviceUuid, transaction) + CatalogContainer.validateCatalog(catalog, { + volumeMappings, + tmpfs, + otherCatalog: knowledgeCatalog, + kind: 'model' + }) + await _assertCatalogModelsExist(catalog, transaction) + await _autoAttachCatalogModels(microservice.iofogUuid, catalog, transaction) + + const previousCatalog = await _loadCatalog(microserviceUuid, transaction) + const rebuild = CatalogContainer.catalogRequiresRebuild(previousCatalog, catalog) + await _replaceCatalog(catalog, microserviceUuid, transaction) + await MicroserviceManager.update(query, { + rebuild: rebuild || !!microservice.rebuild + }, transaction) + + if (microservice.iofogUuid) { + const event = rebuild + ? ChangeTrackingService.events.microserviceList + : ChangeTrackingService.events.microserviceModels + await ChangeTrackingService.update(microservice.iofogUuid, event, transaction) + } + + return { + uuid: microserviceUuid + } +} + +async function updateMicroserviceKnowledgeEndPoint (microserviceUuid, catalogData, isCLI, transaction) { + await Validator.validate(catalogData, Validator.schemas.microserviceCatalogPatch) + + const query = { uuid: microserviceUuid } + const microservice = await MicroserviceManager.findOneWithCategory(query, transaction) + if (!microservice) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_MICROSERVICE_UUID, microserviceUuid)) + } + if (microservice.catalogItem && microservice.catalogItem.category === 'SYSTEM') { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) + } + if (microservice.isController) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.SYSTEM_MICROSERVICE_UPDATE, microserviceUuid)) + } + if (!isCLI) { + await _validateMicroserviceOnGet(microserviceUuid, transaction) + } + + const catalog = CatalogContainer.normalizeCatalog(catalogData) + const volumeMappings = await VolumeMappingManager.findAll({ microserviceUuid }, transaction) + const tmpfs = await _loadTmpfs(microserviceUuid, transaction) + const modelsCatalog = await _loadCatalog(microserviceUuid, transaction) + CatalogContainer.validateCatalog(catalog, { + volumeMappings, + tmpfs, + otherCatalog: modelsCatalog, + kind: 'knowledge' + }) + await _assertCatalogKnowledgeExist(catalog, transaction) + await _autoAttachCatalogKnowledge(microservice.iofogUuid, catalog, transaction) + + const previousCatalog = await _loadKnowledgeCatalog(microserviceUuid, transaction) + const rebuild = CatalogContainer.catalogRequiresRebuild(previousCatalog, catalog) + await _replaceKnowledgeCatalog(catalog, microserviceUuid, transaction) + await MicroserviceManager.update(query, { + rebuild: rebuild || !!microservice.rebuild + }, transaction) + + await _flagMicroserviceKnowledge(microservice.iofogUuid, transaction) + + return { + uuid: microserviceUuid + } +} + async function getMicroserviceConfigEndPoint (microserviceUuid, isCLI, transaction) { const query = isCLI ? { @@ -1709,13 +2201,22 @@ async function deleteMicroserviceEndPoint (microserviceUuid, microserviceData, i async function deleteNotRunningMicroservices (fog, transaction) { const microservices = await MicroserviceManager.findAllWithStatuses({ iofogUuid: fog.uuid }, transaction) - microservices - .filter((microservice) => microservice.delete) - .filter((microservice) => microservice.microserviceStatus.status === MicroserviceStates.UNKNOWN || - microservice.microserviceStatus.status === MicroserviceStates.STOPPING || - microservice.microserviceStatus.status === MicroserviceStates.DELETING || - microservice.microserviceStatus.status === MicroserviceStates.MARKED_FOR_DELETION) - .forEach(async (microservice) => { await deleteMicroserviceWithRoutesAndPortMappings(microservice, transaction) }) + const toDelete = (microservices || []).filter((microservice) => { + if (!microservice.delete) { + return false + } + const status = microservice.microserviceStatus && microservice.microserviceStatus.status + if (!status) { + return false + } + return status === microserviceState.UNKNOWN || + status === microserviceState.STOPPING || + status === microserviceState.DELETING || + status === microserviceState.MARKED_FOR_DELETION + }) + for (const microservice of toDelete) { + await module.exports.deleteMicroserviceWithRoutesAndPortMappings(microservice, transaction) + } } async function createPortMappingEndPoint (microserviceUuid, portMappingData, isCLI, transaction) { @@ -1911,6 +2412,13 @@ async function isMicroserviceNats (microservice, transaction) { return !!(app && app.isSystem === true) } +async function _applyVolumeMappingChangeTracking (microservice, transaction) { + await MicroserviceManager.update({ uuid: microservice.uuid }, { rebuild: true }, transaction) + if (microservice.iofogUuid) { + await _updateChangeTracking(true, microservice.iofogUuid, transaction) + } +} + async function createVolumeMappingEndPoint (microserviceUuid, volumeMappingData, isCLI, transaction) { await Validator.validate(volumeMappingData, Validator.schemas.volumeMappings) @@ -1941,7 +2449,9 @@ async function createVolumeMappingEndPoint (microserviceUuid, volumeMappingData, throw new Errors.ValidationError(ErrorMessages.VOLUME_MAPPING_ALREADY_EXISTS) } - _validateVolumeMappings([volumeMappingData]) + _validateVolumeMappings([volumeMappingData], { + rejectShared: await _rejectsSharedVolumeScope(microservice, transaction) + }) // Validate volume mount references before creating mapping // When type is 'volumeMount', validates that the volume mount exists and is linked to the fog node @@ -1954,10 +2464,13 @@ async function createVolumeMappingEndPoint (microserviceUuid, volumeMappingData, hostDestination: volumeMappingData.hostDestination, containerDestination: volumeMappingData.containerDestination, accessMode: volumeMappingData.accessMode, - type + type: volumeMappingData.type || type, + scope: volumeMappingData.scope || PRIVATE_SCOPE } - return VolumeMappingManager.create(volumeMappingObj, transaction) + const created = await VolumeMappingManager.create(volumeMappingObj, transaction) + await _applyVolumeMappingChangeTracking(microservice, transaction) + return created } async function createSystemVolumeMappingEndPoint (microserviceUuid, volumeMappingData, isCLI, transaction) { @@ -1990,7 +2503,9 @@ async function createSystemVolumeMappingEndPoint (microserviceUuid, volumeMappin throw new Errors.ValidationError(ErrorMessages.VOLUME_MAPPING_ALREADY_EXISTS) } - _validateVolumeMappings([volumeMappingData]) + _validateVolumeMappings([volumeMappingData], { + rejectShared: await _rejectsSharedVolumeScope(microservice, transaction) + }) // Validate volume mount references before creating mapping // When type is 'volumeMount', validates that the volume mount exists and is linked to the fog node @@ -2003,10 +2518,13 @@ async function createSystemVolumeMappingEndPoint (microserviceUuid, volumeMappin hostDestination: volumeMappingData.hostDestination, containerDestination: volumeMappingData.containerDestination, accessMode: volumeMappingData.accessMode, - type + type: volumeMappingData.type || type, + scope: volumeMappingData.scope || PRIVATE_SCOPE } - return VolumeMappingManager.create(volumeMappingObj, transaction) + const created = await VolumeMappingManager.create(volumeMappingObj, transaction) + await _applyVolumeMappingChangeTracking(microservice, transaction) + return created } async function deleteVolumeMappingEndPoint (microserviceUuid, volumeMappingUuid, isCLI, transaction) { @@ -2039,6 +2557,8 @@ async function deleteVolumeMappingEndPoint (microserviceUuid, volumeMappingUuid, if (affectedRows === 0) { throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_VOLUME_MAPPING_UUID, volumeMappingUuid)) } + + await _applyVolumeMappingChangeTracking(microservice, transaction) } async function deleteSystemVolumeMappingEndPoint (microserviceUuid, volumeMappingUuid, isCLI, transaction) { @@ -2071,6 +2591,8 @@ async function deleteSystemVolumeMappingEndPoint (microserviceUuid, volumeMappin if (affectedRows === 0) { throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.INVALID_VOLUME_MAPPING_UUID, volumeMappingUuid)) } + + await _applyVolumeMappingChangeTracking(microservice, transaction) } async function listVolumeMappingsEndPoint (microserviceUuid, isCLI, transaction) { @@ -2176,6 +2698,8 @@ async function _createMicroservice (microserviceData, isCLI, transaction) { natsRuleId: microserviceData.natsRuleId } + _assignContainerColumns(newMicroservice, microserviceData) + newMicroservice = AppHelper.deleteUndefinedFields(newMicroservice) if (newMicroservice.registryId) { @@ -2303,23 +2827,30 @@ async function _createVolumeMappings (microservice, volumeMappings, transaction) const mappings = [] for (const volumeMapping of volumeMappings) { - const mapping = Object.assign({}, volumeMapping) - mapping.microserviceUuid = microservice.uuid - mappings.push(mapping) + mappings.push({ + microserviceUuid: microservice.uuid, + hostDestination: volumeMapping.hostDestination, + containerDestination: volumeMapping.containerDestination, + accessMode: volumeMapping.accessMode, + type: volumeMapping.type || VOLUME_MAPPING_DEFAULT, + scope: volumeMapping.scope || PRIVATE_SCOPE + }) } await VolumeMappingManager.bulkCreate(mappings, transaction) } async function _updateVolumeMappings (volumeMappings, microserviceUuid, transaction) { - _validateVolumeMappings(volumeMappings) - // Get microservice to find fogUuid for volume mount validation const microservice = await MicroserviceManager.findOne({ uuid: microserviceUuid }, transaction) if (!microservice) { throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_MICROSERVICE_UUID, microserviceUuid)) } + _validateVolumeMappings(volumeMappings, { + rejectShared: await _rejectsSharedVolumeScope(microservice, transaction) + }) + // Validate volume mount references before updating mappings // When type is 'volumeMount', validates that the volume mount exists and is linked to the fog node if (volumeMappings && microservice.iofogUuid) { @@ -2342,7 +2873,8 @@ async function _updateVolumeMappings (volumeMappings, microserviceUuid, transact hostDestination: volumeMapping.hostDestination, containerDestination: volumeMapping.containerDestination, accessMode: volumeMapping.accessMode, - type + type, + scope: volumeMapping.scope || PRIVATE_SCOPE } await VolumeMappingManager.create(volumeMappingObj, transaction) @@ -2501,8 +3033,16 @@ async function _updateSystemPorts (newPortMappings, microservice, transaction) { } } +async function _flagMicroserviceKnowledge (fogUuid, transaction) { + if (!fogUuid) { + return + } + await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.microserviceKnowledge, transaction) +} + async function _updateChangeTracking (configUpdated, fogNodeUuid, transaction) { if (configUpdated) { + // Spec updates other than a catalog-only PATCH reload the full microservice list. await ChangeTrackingService.update(fogNodeUuid, ChangeTrackingService.events.microserviceCommon, transaction) } else { await ChangeTrackingService.update(fogNodeUuid, ChangeTrackingService.events.microserviceList, transaction) @@ -2579,8 +3119,18 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { const images = await CatalogItemImageManager.findAll({ microserviceUuid }, transaction) const volumeMappings = await VolumeMappingManager.findAll({ microserviceUuid }, transaction) const env = await MicroserviceEnvManager.findAllExcludeFields({ microserviceUuid }, transaction) - const cmd = await MicroserviceArgManager.findAllExcludeFields({ microserviceUuid }, transaction) - const arg = cmd.map((it) => it.cmd) + const cmdRows = await MicroserviceArgManager.findAll({ microserviceUuid }, transaction) + const arg = CatalogContainer.tokensFromRows(cmdRows, 'cmd') + const entrypointRows = await MicroserviceEntrypointManager.findAll({ microserviceUuid }, transaction) + const entrypoint = CatalogContainer.tokensFromRows(entrypointRows, 'entrypoint') + const deviceRows = await MicroserviceDeviceManager.findAll({ microserviceUuid }, transaction) + const devices = CatalogContainer.devicesFromRows(deviceRows) + const tmpfsRows = await MicroserviceTmpfsManager.findAll({ microserviceUuid }, transaction) + const tmpfs = CatalogContainer.tmpfsFromRows(tmpfsRows) + const ulimitRows = await MicroserviceUlimitManager.findAll({ microserviceUuid }, transaction) + const ulimits = CatalogContainer.ulimitsFromRows(ulimitRows) + const catalog = await _loadCatalog(microserviceUuid, transaction) + const knowledge = await _loadKnowledgeCatalog(microserviceUuid, transaction) const cdiDevices = await MicroserviceCdiDevManager.findAllExcludeFields({ microserviceUuid }, transaction) const cdiDevs = cdiDevices.map((it) => it.cdiDevices) const capAdd = await MicroserviceCapAddManager.findAllExcludeFields({ microserviceUuid }, transaction) @@ -2601,13 +3151,31 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { res.volumeMappings = volumeMappings.map((vm) => vm.dataValues) res.env = env res.cmd = arg + const hydrated = CatalogContainer.hydrateJsonColumns(res) + res.sysctls = hydrated.sysctls + if (entrypoint.length > 0) { + res.entrypoint = entrypoint + } else { + delete res.entrypoint + } + if (arg.length > 0) { + res.commands = arg + } else { + delete res.commands + } + res.ulimits = Object.keys(ulimits).length > 0 ? ulimits : null + res.devices = devices + res.tmpfs = tmpfs + res.models = catalog + res.knowledge = knowledge + res.runAsGroup = res.runAsGroup || '' res.cdiDevices = cdiDevs res.capAdd = capAdds res.capDrop = capDrops res.extraHosts = extraHosts.map(eH => ({ name: eH.name, address: eH.template, value: eH.value })) res.images = images.map(i => ({ containerImage: i.containerImage, archId: i.archId })) if (status && status.length) { - res.status = status[0] + res.status = projectStatusForApi(status[0]) } if (execStatus && execStatus.length) { res.execStatus = execStatus[0] @@ -2679,9 +3247,30 @@ async function _buildGetMicroserviceResponse (microservice, transaction) { } delete res.natsRuleId + delete res.serviceAccount + res.serviceAccount = await _resolveServiceAccountForUserResponse(microservice, transaction) + return res } +async function setMicroservicesObservedStopping (microserviceUuids, transaction) { + const uuids = (microserviceUuids || []).filter(Boolean) + if (!uuids.length) { + return + } + const observedStatus = applyRuntimeMetrics( + Object.assign({ status: microserviceState.STOPPING }, zeroRuntimeMetrics()), + microserviceState.STOPPING + ) + await MicroserviceStatusManager.update({ microserviceUuid: uuids }, observedStatus, transaction) + await MicroserviceExecStatusManager.update({ + microserviceUuid: uuids + }, { + status: microserviceExecState.INACTIVE, + execSessionId: '' + }, transaction) +} + async function startMicroserviceEndPoint (microserviceUuid, isCLI, transaction) { const microservice = await MicroserviceManager.findOneWithCategory({ uuid: microserviceUuid }, transaction) if (!microservice) { @@ -2716,6 +3305,7 @@ async function stopMicroserviceEndPoint (microserviceUuid, isCLI, transaction) { } await MicroserviceManager.update({ uuid: microservice.uuid }, { isActivated: false }, transaction) + await setMicroservicesObservedStopping([microservice.uuid], transaction) await ChangeTrackingService.update(microservice.iofogUuid, ChangeTrackingService.events.microserviceList, transaction) return { @@ -2769,6 +3359,8 @@ module.exports = { updateMicroserviceEndPoint: TransactionDecorator.generateTransaction(updateMicroserviceEndPoint), updateSystemMicroserviceEndPoint: TransactionDecorator.generateTransaction(updateSystemMicroserviceEndPoint), updateMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(updateMicroserviceConfigEndPoint), + updateMicroserviceCatalogEndPoint: TransactionDecorator.generateTransaction(updateMicroserviceCatalogEndPoint), + updateMicroserviceKnowledgeEndPoint: TransactionDecorator.generateTransaction(updateMicroserviceKnowledgeEndPoint), getMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(getMicroserviceConfigEndPoint), getSystemMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(getSystemMicroserviceConfigEndPoint), deleteMicroserviceConfigEndPoint: TransactionDecorator.generateTransaction(deleteMicroserviceConfigEndPoint), @@ -2780,6 +3372,7 @@ module.exports = { updateChangeTracking: _updateChangeTracking, startMicroserviceEndPoint: TransactionDecorator.generateTransaction(startMicroserviceEndPoint), stopMicroserviceEndPoint: TransactionDecorator.generateTransaction(stopMicroserviceEndPoint), + setMicroservicesObservedStopping, reconcileNatsForApplication: TransactionDecorator.generateTransaction(reconcileNatsForApplication), injectServiceAccountVolume: _injectServiceAccountVolume, stripUserServiceAccountVolumeMappings: _stripUserServiceAccountVolumeMappings, diff --git a/src/services/model-service.js b/src/services/model-service.js new file mode 100644 index 000000000..10457a685 --- /dev/null +++ b/src/services/model-service.js @@ -0,0 +1,271 @@ +const Errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const AppHelper = require('../helpers/app-helper') +const FleetModelManager = require('../data/managers/fleet-model-manager') +const RegistryManager = require('../data/managers/registry-manager') +const MicroserviceManager = require('../data/managers/microservice-manager') +const MicroserviceModelItemManager = require('../data/managers/microservice-model-item-manager') +const FogManager = require('../data/managers/iofog-manager') +const ChangeTrackingService = require('./change-tracking-service') +const TransactionDecorator = require('../decorators/transaction-decorator') +const Validator = require('../schemas') + +function _toPlain (row) { + if (!row) { + return null + } + return typeof row.toJSON === 'function' ? row.toJSON() : row +} + +function toWireModel (row) { + const plain = _toPlain(row) + if (!plain) { + return null + } + return { + uuid: plain.uuid, + name: plain.name, + repo: plain.repo, + revision: plain.revision != null ? plain.revision : '', + registryId: plain.registryId, + files: Array.isArray(plain.files) ? plain.files : [], + format: plain.format + } +} + +async function findLinkedFogUuids (model, transaction) { + if (!model || typeof model.getFogs !== 'function') { + return [] + } + const fogs = await model.getFogs({ transaction }) + return fogs.map((fog) => fog.uuid) +} + +async function _updateChangeTrackingForFogs (fogUuids, transaction) { + for (const fogUuid of fogUuids) { + await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.models, transaction) + } +} + +async function findMicroservicesBindingModelName (modelName, options, transaction) { + const items = await MicroserviceModelItemManager.findAll({ name: modelName }, transaction) + const blocking = [] + for (const item of items) { + if (options && options.fogUuid) { + const microservice = await MicroserviceManager.findOne({ uuid: item.microserviceUuid }, transaction) + if (!microservice || microservice.iofogUuid !== options.fogUuid) { + continue + } + blocking.push(microservice.uuid) + continue + } + blocking.push(item.microserviceUuid) + } + return blocking +} + +async function _validateRegistryAndFiles (registryId, files, transaction) { + const registry = await RegistryManager.findOne({ id: registryId }, transaction) + if (!registry) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_REGISTRY_ID, registryId)) + } + const type = registry.type || 'oci' + const fileList = Array.isArray(files) ? files : [] + if (type === 'hf') { + if (fileList.length === 0) { + throw new Errors.ValidationError(ErrorMessages.MODEL_HF_FILES_REQUIRED) + } + } else if (fileList.length > 0) { + throw new Errors.ValidationError(ErrorMessages.MODEL_OCI_FILES_FORBIDDEN) + } + return registry +} + +async function getModelByName (name, transaction) { + const model = await FleetModelManager.findOne({ name }, transaction) + if (!model) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.MODEL_NOT_FOUND, name)) + } + return model +} + +async function listModelsEndpoint (transaction) { + const models = await FleetModelManager.findAll({}, transaction) + return { + models: models.map(toWireModel) + } +} + +async function getModelEndpoint (name, transaction) { + const model = await getModelByName(name, transaction) + return toWireModel(model) +} + +async function upsertModelEndpoint (name, data, transaction) { + const existing = await FleetModelManager.findOne({ name }, transaction) + if (existing) { + return updateModelEndpoint(name, data, transaction) + } + return createModelEndpoint({ ...data, name }, transaction) +} + +async function createModelEndpoint (data, transaction) { + await Validator.validate(data, Validator.schemas.modelCreate) + + const existing = await FleetModelManager.findOne({ name: data.name }, transaction) + if (existing) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.MODEL_ALREADY_EXISTS, data.name)) + } + + const files = Array.isArray(data.files) ? data.files : [] + await _validateRegistryAndFiles(data.registryId, files, transaction) + + const created = await FleetModelManager.create({ + uuid: AppHelper.generateUUID(), + name: data.name, + repo: data.repo, + revision: data.revision != null ? data.revision : '', + registryId: data.registryId, + files, + format: data.format + }, transaction) + + return toWireModel(created) +} + +async function updateModelEndpoint (name, data, transaction) { + await Validator.validate(data, Validator.schemas.modelUpdate) + const model = await getModelByName(name, transaction) + + const nextRepo = data.repo !== undefined ? data.repo : model.repo + const nextRevision = data.revision !== undefined ? data.revision : (model.revision != null ? model.revision : '') + const nextRegistryId = data.registryId !== undefined ? data.registryId : model.registryId + const nextFiles = data.files !== undefined ? data.files : (Array.isArray(model.files) ? model.files : []) + const nextFormat = data.format !== undefined ? data.format : model.format + + await _validateRegistryAndFiles(nextRegistryId, nextFiles, transaction) + + const linkedFogUuids = await findLinkedFogUuids(model, transaction) + + await FleetModelManager.update({ name }, { + repo: nextRepo, + revision: nextRevision, + registryId: nextRegistryId, + files: nextFiles, + format: nextFormat + }, transaction) + + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + + return getModelEndpoint(name, transaction) +} + +async function deleteModelEndpoint (name, transaction) { + const model = await getModelByName(name, transaction) + const blocking = await findMicroservicesBindingModelName(name, {}, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.MODEL_IN_USE, name, blocking.join(', '))) + } + + const linkedFogUuids = await findLinkedFogUuids(model, transaction) + await FleetModelManager.delete({ name }, transaction) + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + return {} +} + +async function getModelLinkEndpoint (name, transaction) { + const model = await getModelByName(name, transaction) + return { + fogUuids: await findLinkedFogUuids(model, transaction) + } +} + +async function linkModelEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.modelLink) + const model = await getModelByName(name, transaction) + const alreadyLinked = new Set(await findLinkedFogUuids(model, transaction)) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + await agent.addModel(model, { transaction }) + } + + const newlyLinked = fogUuids.filter((uuid) => !alreadyLinked.has(uuid)) + if (newlyLinked.length > 0) { + await _updateChangeTrackingForFogs(newlyLinked, transaction) + } + + return getModelEndpoint(name, transaction) +} + +async function unlinkModelEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.modelUnlink) + const model = await getModelByName(name, transaction) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + const blocking = await findMicroservicesBindingModelName(name, { fogUuid }, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.MODEL_UNLINK_IN_USE, name, fogUuid, blocking.join(', '))) + } + await agent.removeModel(model, { transaction }) + } + + await _updateChangeTrackingForFogs(fogUuids, transaction) + return {} +} + +async function ensureModelsLinkedToFog (fogUuid, modelNames, transaction) { + const names = [...new Set((modelNames || []).filter((name) => typeof name === 'string' && name.length > 0))] + if (names.length === 0) { + return [] + } + + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + + const newlyLinked = [] + for (const name of names) { + const model = await getModelByName(name, transaction) + const alreadyLinked = typeof agent.hasModel === 'function' + ? await agent.hasModel(model, { transaction }) + : false + if (alreadyLinked) { + continue + } + await agent.addModel(model, { transaction }) + newlyLinked.push(name) + } + + if (newlyLinked.length > 0) { + await _updateChangeTrackingForFogs([fogUuid], transaction) + } + + return newlyLinked +} + +module.exports = { + listModelsEndpoint: TransactionDecorator.generateTransaction(listModelsEndpoint), + getModelEndpoint: TransactionDecorator.generateTransaction(getModelEndpoint), + createModelEndpoint: TransactionDecorator.generateTransaction(createModelEndpoint), + updateModelEndpoint: TransactionDecorator.generateTransaction(updateModelEndpoint), + deleteModelEndpoint: TransactionDecorator.generateTransaction(deleteModelEndpoint), + getModelLinkEndpoint: TransactionDecorator.generateTransaction(getModelLinkEndpoint), + linkModelEndpoint: TransactionDecorator.generateTransaction(linkModelEndpoint), + unlinkModelEndpoint: TransactionDecorator.generateTransaction(unlinkModelEndpoint), + upsertModelEndpoint: TransactionDecorator.generateTransaction(upsertModelEndpoint), + ensureModelsLinkedToFog: TransactionDecorator.generateTransaction(ensureModelsLinkedToFog), + toWireModel +} diff --git a/src/services/registry-service.js b/src/services/registry-service.js index 4f94b0ba6..a82fd1c78 100644 --- a/src/services/registry-service.js +++ b/src/services/registry-service.js @@ -10,26 +10,180 @@ const ErrorMessages = require('../helpers/error-messages') const ReconcileOutboxManager = require('../data/managers/reconcile-outbox-manager') const TransactionDecorator = require('../decorators/transaction-decorator') const MicroserviceManager = require('../data/managers/microservice-manager') -// const Sequelize = require('sequelize') -// const Op = Sequelize.Op const AppHelper = require('../helpers/app-helper') +const REGISTRY_TYPE_OCI = 'oci' +const REGISTRY_TYPE_HF = 'hf' +const HUB_REGISTRY_URL = 'https://huggingface.co' +const HUB_REGISTRY_TYPE = REGISTRY_TYPE_HF +const LEGACY_SYSTEM_REGISTRY_IDS = new Set([1, 2]) + function isPasswordEmpty (password) { return password == null || (typeof password === 'string' && password.trim() === '') } +function isBlank (value) { + return value == null || (typeof value === 'string' && value.trim() === '') +} + +function normalizeRegistryType (type) { + if (type == null || type === '') { + return REGISTRY_TYPE_OCI + } + return type +} + +function isHubRegistry (registry) { + if (!registry) { + return false + } + return registry.url === HUB_REGISTRY_URL && normalizeRegistryType(registry.type) === HUB_REGISTRY_TYPE +} + +function isLegacySystemRegistryId (id) { + return LEGACY_SYSTEM_REGISTRY_IDS.has(parseInt(id, 10)) +} + +function toPlainRegistry (registry) { + if (!registry) { + return null + } + return typeof registry.toJSON === 'function' ? registry.toJSON() : { ...registry } +} + +function toAgentRegistry (registry) { + const plain = toPlainRegistry(registry) || {} + return { + id: plain.id, + url: plain.url, + isPublic: !!plain.isPublic, + username: plain.username || '', + password: plain.password || '', + userEmail: plain.userEmail || '', + type: normalizeRegistryType(plain.type), + ca: plain.ca || '', + insecure: !!plain.insecure + } +} + +function isValidBase64 (value) { + if (value == null || value === '') { + return true + } + if (typeof value !== 'string') { + return false + } + const normalized = value.replace(/\s/g, '') + if (!normalized) { + return true + } + if (normalized.length % 4 !== 0) { + return false + } + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(normalized)) { + return false + } + try { + Buffer.from(normalized, 'base64') + return true + } catch (err) { + return false + } +} + +function normalizeCaForStorage (ca) { + if (ca === undefined || ca === null || (typeof ca === 'string' && ca.trim() === '')) { + return null + } + if (!isValidBase64(ca)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_INVALID_CA) + } + return ca +} + +function normalizeCaForWire (ca) { + if (ca == null || (typeof ca === 'string' && ca.trim() === '')) { + return null + } + return ca +} + +function assertCredentialRules (data, existing) { + const type = normalizeRegistryType(data.type !== undefined ? data.type : existing && existing.type) + const isPublic = data.isPublic !== undefined ? data.isPublic : !!(existing && existing.isPublic) + const username = data.username !== undefined ? data.username : (existing && existing.username) + const password = data.password !== undefined ? data.password : (existing && existing.password) + + if (isPublic) { + return + } + if (type === REGISTRY_TYPE_OCI) { + if (isBlank(username) || isPasswordEmpty(password)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_PRIVATE_OCI_CREDENTIALS_REQUIRED) + } + return + } + if (type === REGISTRY_TYPE_HF && isPasswordEmpty(password)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_PRIVATE_HF_TOKEN_REQUIRED) + } +} + +function assertSystemRegistryProtection (existing, patch) { + if (isLegacySystemRegistryId(existing.id)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } + if (!isHubRegistry(existing)) { + return + } + if (patch.url !== undefined && patch.url !== existing.url) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } + if (patch.type !== undefined && normalizeRegistryType(patch.type) !== HUB_REGISTRY_TYPE) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } +} + +function mergedRegistryIdentity (existing, patch) { + return { + url: patch.url !== undefined ? patch.url : existing && existing.url, + type: normalizeRegistryType(patch.type !== undefined ? patch.type : existing && existing.type) + } +} + const createRegistry = async function (registry, transaction) { await Validator.validate(registry, Validator.schemas.registryCreate) + const type = normalizeRegistryType(registry.type) + const ca = normalizeCaForStorage(registry.ca) + assertCredentialRules({ + type, + isPublic: registry.isPublic, + username: registry.username, + password: registry.password + }, null) + + if (isHubRegistry({ url: registry.url, type })) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } + let registryCreate = { url: registry.url, username: registry.username, password: registry.password, isPublic: registry.isPublic, - userEmail: registry.email + userEmail: registry.email, + type, + ca, + insecure: registry.insecure === true } registryCreate = AppHelper.deleteUndefinedFields(registryCreate) + if (registryCreate.type === undefined) { + registryCreate.type = REGISTRY_TYPE_OCI + } + if (registryCreate.insecure === undefined) { + registryCreate.insecure = false + } const createdRegistry = await RegistryManager.create(registryCreate, transaction) @@ -70,7 +224,15 @@ const findRegistries = async function (isCLI, transaction) { const registries = await RegistryManager.findAllWithAttributes(queryRegistry, { exclude: ['password'] }, transaction) return { - registries + registries: registries.map((registry) => { + const plain = toPlainRegistry(registry) + return { + ...plain, + type: normalizeRegistryType(plain.type), + ca: normalizeCaForWire(plain.ca), + insecure: !!plain.insecure + } + }) } } @@ -79,15 +241,17 @@ const deleteRegistry = async function (registryData, isCLI, transaction) { const queryData = isCLI ? { id: registryData.id } : { id: registryData.id } - // Convert registryId to number to handle string IDs from URL parameters const id = parseInt(registryData.id, 10) - if (id === 1 || id === 2) { + if (isLegacySystemRegistryId(id)) { throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) } const registry = await RegistryManager.findOne(queryData, transaction) if (!registry) { throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_REGISTRY_ID, registryData.id)) } + if (isHubRegistry(registry)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } const microservices = await MicroserviceManager.findAllWithStatuses({ registryId: registryData.id }, transaction) if (microservices.length > 0) { throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_IN_USE) @@ -103,9 +267,8 @@ const deleteRegistry = async function (registryData, isCLI, transaction) { const updateRegistry = async function (registry, registryId, isCLI, transaction) { await Validator.validate(registry, Validator.schemas.registryUpdate) - // Convert registryId to number to handle string IDs from URL parameters const id = parseInt(registryId, 10) - if (id === 1 || id === 2) { + if (isLegacySystemRegistryId(id)) { throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) } const existingRegistry = await RegistryManager.findOne({ @@ -116,12 +279,24 @@ const updateRegistry = async function (registry, registryId, isCLI, transaction) throw new Errors.NotFoundError(ErrorMessages.REGISTRY_NOT_FOUND) } + assertSystemRegistryProtection(existingRegistry, registry) + + const mergedIdentity = mergedRegistryIdentity(existingRegistry, registry) + if (!isHubRegistry(existingRegistry) && isHubRegistry(mergedIdentity)) { + throw new Errors.ValidationError(ErrorMessages.REGISTRY_IS_SYSTEM) + } + + assertCredentialRules(registry, existingRegistry) + let registryUpdate = { url: registry.url, username: registry.username, password: registry.password, isPublic: registry.isPublic, - userEmail: registry.email + userEmail: registry.email, + type: registry.type, + ca: registry.ca !== undefined ? normalizeCaForStorage(registry.ca) : undefined, + insecure: registry.insecure } registryUpdate = AppHelper.deleteUndefinedFields(registryUpdate) @@ -160,6 +335,21 @@ const getRegistry = async function (registryId, isCLI, transaction) { if (!registry) { throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.INVALID_REGISTRY_ID, registryId)) } + const plain = toPlainRegistry(registry) + return { + ...plain, + type: normalizeRegistryType(plain.type), + ca: normalizeCaForWire(plain.ca), + insecure: !!plain.insecure + } +} + +const assertOciRegistryForImage = async function (registryId, transaction) { + const id = parseInt(registryId, 10) + const registry = Number.isFinite(id) ? await RegistryManager.findOne({ id }, transaction) : null + if (!registry || normalizeRegistryType(registry.type) !== REGISTRY_TYPE_OCI) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.REGISTRY_NOT_OCI_FOR_IMAGE, registryId)) + } return registry } @@ -168,5 +358,7 @@ module.exports = { findRegistries: TransactionDecorator.generateTransaction(findRegistries), deleteRegistry: TransactionDecorator.generateTransaction(deleteRegistry), updateRegistry: TransactionDecorator.generateTransaction(updateRegistry), - getRegistry: TransactionDecorator.generateTransaction(getRegistry) + getRegistry: TransactionDecorator.generateTransaction(getRegistry), + assertOciRegistryForImage: TransactionDecorator.generateTransaction(assertOciRegistryForImage), + toAgentRegistry } diff --git a/src/services/runtime-class-service.js b/src/services/runtime-class-service.js new file mode 100644 index 000000000..274f50714 --- /dev/null +++ b/src/services/runtime-class-service.js @@ -0,0 +1,277 @@ +const Errors = require('../helpers/errors') +const ErrorMessages = require('../helpers/error-messages') +const AppHelper = require('../helpers/app-helper') +const RuntimeClassManager = require('../data/managers/runtime-class-manager') +const MicroserviceManager = require('../data/managers/microservice-manager') +const FogManager = require('../data/managers/iofog-manager') +const ChangeTrackingService = require('./change-tracking-service') +const TransactionDecorator = require('../decorators/transaction-decorator') +const Validator = require('../schemas') +const logger = require('../logger') + +const CATALOG_HANDLERS = new Set([ + 'spin', + 'edgelet-wasmtime', + 'wasmtime', + 'wasmedge', + 'nvidia-cdi' +]) + +function _toPlain (row) { + if (!row) { + return null + } + return typeof row.toJSON === 'function' ? row.toJSON() : row +} + +function toWireRuntimeClass (row) { + const plain = _toPlain(row) + if (!plain) { + return null + } + return { + name: plain.name, + handler: plain.handler + } +} + +function _warnIfUnknownHandler (name, handler) { + if (!handler || CATALOG_HANDLERS.has(handler)) { + return + } + logger.warn( + `RuntimeClass '${name}' handler '${handler}' is not in the known catalog; applying it may restart the data plane` + ) +} + +function _parseFogAvailableRuntimes (fog) { + if (!fog || fog.availableRuntimes == null || fog.availableRuntimes === '') { + return [] + } + if (Array.isArray(fog.availableRuntimes)) { + return fog.availableRuntimes + } + try { + const parsed = JSON.parse(fog.availableRuntimes) + return Array.isArray(parsed) ? parsed : [] + } catch (error) { + return [] + } +} + +function _assertRuntimeClassAvailableOnFog (runtimeClassName, agent) { + const availableRuntimes = _parseFogAvailableRuntimes(agent) + if (!availableRuntimes.includes(runtimeClassName)) { + const agentLabel = agent.name || agent.uuid || 'agent' + throw new Errors.ValidationError( + AppHelper.formatMessage( + ErrorMessages.RUNTIME_CLASS_LINK_NOT_AVAILABLE, + runtimeClassName, + agentLabel + ) + ) + } +} + +async function findLinkedFogUuids (runtimeClass, transaction) { + if (!runtimeClass || typeof runtimeClass.getFogs !== 'function') { + return [] + } + const fogs = await runtimeClass.getFogs({ transaction }) + return fogs.map((fog) => fog.uuid) +} + +async function _updateChangeTrackingForFogs (fogUuids, transaction) { + for (const fogUuid of fogUuids) { + await ChangeTrackingService.update(fogUuid, ChangeTrackingService.events.runtimeClasses, transaction) + } +} + +async function findMicroservicesPinningRuntime (className, options, transaction) { + const where = { runtime: className } + if (options && options.fogUuid) { + where.iofogUuid = options.fogUuid + } + const microservices = await MicroserviceManager.findAll(where, transaction) + return microservices.map((microservice) => microservice.uuid) +} + +async function getRuntimeClassByName (name, transaction) { + const runtimeClass = await RuntimeClassManager.findOne({ name }, transaction) + if (!runtimeClass) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_NOT_FOUND, name)) + } + return runtimeClass +} + +async function listRuntimeClassesEndpoint (transaction) { + const runtimeClasses = await RuntimeClassManager.findAll({}, transaction) + return { + runtimeClasses: runtimeClasses.map(toWireRuntimeClass) + } +} + +async function getRuntimeClassEndpoint (name, transaction) { + const runtimeClass = await getRuntimeClassByName(name, transaction) + return toWireRuntimeClass(runtimeClass) +} + +async function upsertRuntimeClassEndpoint (name, data, transaction) { + const existing = await RuntimeClassManager.findOne({ name }, transaction) + if (existing) { + return updateRuntimeClassEndpoint(name, data, transaction) + } + return createRuntimeClassEndpoint({ ...data, name }, transaction) +} + +async function createRuntimeClassEndpoint (data, transaction) { + await Validator.validate(data, Validator.schemas.runtimeClassCreate) + + const existing = await RuntimeClassManager.findOne({ name: data.name }, transaction) + if (existing) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_ALREADY_EXISTS, data.name)) + } + + const created = await RuntimeClassManager.create({ + name: data.name, + handler: data.handler + }, transaction) + + return toWireRuntimeClass(created) +} + +async function updateRuntimeClassEndpoint (name, data, transaction) { + await Validator.validate(data, Validator.schemas.runtimeClassUpdate) + const runtimeClass = await getRuntimeClassByName(name, transaction) + + const nextHandler = data.handler !== undefined ? data.handler : runtimeClass.handler + const linkedFogUuids = await findLinkedFogUuids(runtimeClass, transaction) + + await RuntimeClassManager.update({ name }, { + handler: nextHandler + }, transaction) + + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + + return getRuntimeClassEndpoint(name, transaction) +} + +async function deleteRuntimeClassEndpoint (name, transaction) { + const runtimeClass = await getRuntimeClassByName(name, transaction) + const blocking = await findMicroservicesPinningRuntime(name, {}, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError(AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_IN_USE, name, blocking.join(', '))) + } + + const linkedFogUuids = await findLinkedFogUuids(runtimeClass, transaction) + await RuntimeClassManager.delete({ name }, transaction) + if (linkedFogUuids.length > 0) { + await _updateChangeTrackingForFogs(linkedFogUuids, transaction) + } + return {} +} + +async function getRuntimeClassLinkEndpoint (name, transaction) { + const runtimeClass = await getRuntimeClassByName(name, transaction) + return { + fogUuids: await findLinkedFogUuids(runtimeClass, transaction) + } +} + +async function linkRuntimeClassEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.runtimeClassLink) + const runtimeClass = await getRuntimeClassByName(name, transaction) + const alreadyLinked = new Set(await findLinkedFogUuids(runtimeClass, transaction)) + + _warnIfUnknownHandler(name, runtimeClass.handler) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + if (agent.containerEngine !== 'edgelet') { + throw new Errors.ValidationError( + AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_LINK_REQUIRES_EDGELET, fogUuid) + ) + } + _assertRuntimeClassAvailableOnFog(name, agent) + await runtimeClass.addFog(agent, { transaction }) + } + + const newlyLinked = fogUuids.filter((uuid) => !alreadyLinked.has(uuid)) + if (newlyLinked.length > 0) { + await _updateChangeTrackingForFogs(newlyLinked, transaction) + } + + return getRuntimeClassEndpoint(name, transaction) +} + +async function unlinkRuntimeClassEndpoint (name, fogUuids, transaction) { + await Validator.validate({ fogUuids }, Validator.schemas.runtimeClassUnlink) + const runtimeClass = await getRuntimeClassByName(name, transaction) + + for (const fogUuid of fogUuids) { + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + const blocking = await findMicroservicesPinningRuntime(name, { fogUuid }, transaction) + if (blocking.length > 0) { + throw new Errors.ConflictError( + AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_UNLINK_IN_USE, name, fogUuid, blocking.join(', ')) + ) + } + await runtimeClass.removeFog(agent, { transaction }) + } + + await _updateChangeTrackingForFogs(fogUuids, transaction) + return {} +} + +async function ensureRuntimeClassLinkedToFog (fogUuid, className, transaction) { + if (typeof className !== 'string' || className.length === 0) { + return false + } + + const agent = await FogManager.findOne({ uuid: fogUuid }, transaction) + if (!agent) { + throw new Errors.NotFoundError(AppHelper.formatMessage(ErrorMessages.NOT_FOUND_AGENT_NAME, fogUuid)) + } + + const runtimeClass = await RuntimeClassManager.findOne({ name: className }, transaction) + if (!runtimeClass) { + throw new Errors.ValidationError(AppHelper.formatMessage(ErrorMessages.RUNTIME_CLASS_MISSING, className)) + } + + if (agent.containerEngine !== 'edgelet') { + return false + } + + const alreadyLinked = typeof runtimeClass.hasFog === 'function' + ? await runtimeClass.hasFog(agent, { transaction }) + : false + if (alreadyLinked) { + return false + } + + await runtimeClass.addFog(agent, { transaction }) + await _updateChangeTrackingForFogs([fogUuid], transaction) + return true +} + +module.exports = { + listRuntimeClassesEndpoint: TransactionDecorator.generateTransaction(listRuntimeClassesEndpoint), + getRuntimeClassEndpoint: TransactionDecorator.generateTransaction(getRuntimeClassEndpoint), + createRuntimeClassEndpoint: TransactionDecorator.generateTransaction(createRuntimeClassEndpoint), + updateRuntimeClassEndpoint: TransactionDecorator.generateTransaction(updateRuntimeClassEndpoint), + deleteRuntimeClassEndpoint: TransactionDecorator.generateTransaction(deleteRuntimeClassEndpoint), + getRuntimeClassLinkEndpoint: TransactionDecorator.generateTransaction(getRuntimeClassLinkEndpoint), + linkRuntimeClassEndpoint: TransactionDecorator.generateTransaction(linkRuntimeClassEndpoint), + unlinkRuntimeClassEndpoint: TransactionDecorator.generateTransaction(unlinkRuntimeClassEndpoint), + upsertRuntimeClassEndpoint: TransactionDecorator.generateTransaction(upsertRuntimeClassEndpoint), + ensureRuntimeClassLinkedToFog: TransactionDecorator.generateTransaction(ensureRuntimeClassLinkedToFog), + toWireRuntimeClass +} diff --git a/src/services/yaml-parser-service.js b/src/services/yaml-parser-service.js index cda0c3c9a..bd678ab59 100644 --- a/src/services/yaml-parser-service.js +++ b/src/services/yaml-parser-service.js @@ -1,4 +1,5 @@ const Errors = require('../helpers/errors') +const CatalogContainer = require('../helpers/microservice-container-catalog') const lget = require('lodash/get') const yaml = require('js-yaml') @@ -236,93 +237,155 @@ const { mapYamlImagesToArchList } = require('../helpers/arch-images') const mapImages = (images) => mapYamlImagesToArchList(images) -const parseMicroserviceImages = async (fileImages) => { +const REGISTRY_BY_NAME = { + remote: 1, + local: 2 +} + +function isTemplatePlaceholder (value) { + return typeof value === 'string' && value.includes('{{') +} + +function asIdOrPlaceholder (value, registryByName = REGISTRY_BY_NAME) { + if (value == null || value === '') { + return undefined + } + if (typeof value === 'number' && Number.isFinite(value)) { + return value + } + if (typeof value === 'string') { + const trimmed = value.trim() + if (isTemplatePlaceholder(trimmed)) { + return trimmed + } + if (Object.hasOwn(registryByName, trimmed)) { + return registryByName[trimmed] + } + const asNumber = Number(trimmed) + if (trimmed !== '' && Number.isFinite(asNumber) && String(asNumber) === trimmed) { + return asNumber + } + return trimmed + } + const asNumber = Number(value) + if (Number.isFinite(asNumber)) { + return asNumber + } + return value +} + +const parseMicroserviceImages = async (fileImages, options = {}) => { + const { templateMode = false } = options // Could be undefined if patch call if (!fileImages) { return { registryId: undefined, images: undefined, catalogItemId: undefined } } - if (fileImages.catalogId > 0) { - return { registryId: undefined, images: undefined, catalogItemId: fileImages.catalogId } - } - const registryByName = { - remote: 1, - local: 2 + if (fileImages.catalogId != null && fileImages.catalogId !== '') { + const catalogItemId = asIdOrPlaceholder(fileImages.catalogId) + if (templateMode && typeof catalogItemId === 'string') { + return { registryId: undefined, images: undefined, catalogItemId } + } + if (typeof catalogItemId === 'number' && catalogItemId > 0) { + return { registryId: undefined, images: undefined, catalogItemId } + } } const images = mapImages(fileImages) - const registryId = fileImages.registry != null ? registryByName[fileImages.registry] || Number(fileImages.registry) : 1 + let registryId + if (fileImages.registry != null && fileImages.registry !== '') { + registryId = asIdOrPlaceholder(fileImages.registry) + } else if (!templateMode) { + registryId = 1 + } return { registryId, catalogItemId: undefined, images } } -const parseMicroserviceYAML = async (microservice) => { - const { registryId, catalogItemId, images } = await parseMicroserviceImages(microservice.images) - const container = microservice.container || {} +function parseEnvVariables (envArray) { + if (!envArray || !Array.isArray(envArray)) { + return [] + } - // Parse environment variables with support for value, valueFromSecret, and valueFromConfigMap - const parseEnvVariables = (envArray) => { - if (!envArray || !Array.isArray(envArray)) { - return [] + return envArray.map(env => { + if (!env || typeof env !== 'object') { + throw new Errors.ValidationError('Invalid environment variable format') } - return envArray.map(env => { - if (!env || typeof env !== 'object') { - throw new Errors.ValidationError('Invalid environment variable format') - } + if (!env.key) { + throw new Errors.ValidationError('Environment variable must have a key') + } - if (!env.key) { - throw new Errors.ValidationError('Environment variable must have a key') - } + const envVar = { + key: env.key.toString() + } - const envVar = { - key: env.key.toString() - } + const hasValue = Object.hasOwn(env, 'value') + const hasValueFromSecret = Object.hasOwn(env, 'valueFromSecret') + const hasValueFromConfigMap = Object.hasOwn(env, 'valueFromConfigMap') - // Check that exactly one of value, valueFromSecret, or valueFromConfigMap is provided - const hasValue = Object.hasOwn(env, 'value') - const hasValueFromSecret = Object.hasOwn(env, 'valueFromSecret') - const hasValueFromConfigMap = Object.hasOwn(env, 'valueFromConfigMap') + const valueCount = [hasValue, hasValueFromSecret, hasValueFromConfigMap].filter(Boolean).length - const valueCount = [hasValue, hasValueFromSecret, hasValueFromConfigMap].filter(Boolean).length + if (valueCount === 0) { + throw new Errors.ValidationError(`Environment variable '${env.key}' must have either value, valueFromSecret, or valueFromConfigMap`) + } - if (valueCount === 0) { - throw new Errors.ValidationError(`Environment variable '${env.key}' must have either value, valueFromSecret, or valueFromConfigMap`) - } + if (valueCount > 1) { + throw new Errors.ValidationError(`Environment variable '${env.key}' can only have one of: value, valueFromSecret, or valueFromConfigMap`) + } - if (valueCount > 1) { - throw new Errors.ValidationError(`Environment variable '${env.key}' can only have one of: value, valueFromSecret, or valueFromConfigMap`) - } + if (hasValue) { + envVar.value = env.value.toString() + } - // Handle simple value - if (hasValue) { - envVar.value = env.value.toString() + if (hasValueFromSecret) { + if (typeof env.valueFromSecret !== 'string') { + throw new Errors.ValidationError(`valueFromSecret for environment variable '${env.key}' must be a string`) } - - // Handle valueFromSecret - if (hasValueFromSecret) { - if (typeof env.valueFromSecret !== 'string') { - throw new Errors.ValidationError(`valueFromSecret for environment variable '${env.key}' must be a string`) - } - const parts = env.valueFromSecret.split('/') - if (parts.length !== 2 || !parts[0] || !parts[1]) { - throw new Errors.ValidationError(`valueFromSecret for environment variable '${env.key}' must be in format 'secret-name/key'`) - } - envVar.valueFromSecret = env.valueFromSecret + const parts = env.valueFromSecret.split('/') + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Errors.ValidationError(`valueFromSecret for environment variable '${env.key}' must be in format 'secret-name/key'`) } + envVar.valueFromSecret = env.valueFromSecret + } - // Handle valueFromConfigMap - if (hasValueFromConfigMap) { - if (typeof env.valueFromConfigMap !== 'string') { - throw new Errors.ValidationError(`valueFromConfigMap for environment variable '${env.key}' must be a string`) - } - const parts = env.valueFromConfigMap.split('/') - if (parts.length !== 2 || !parts[0] || !parts[1]) { - throw new Errors.ValidationError(`valueFromConfigMap for environment variable '${env.key}' must be in format 'configmap-name/key'`) - } - envVar.valueFromConfigMap = env.valueFromConfigMap + if (hasValueFromConfigMap) { + if (typeof env.valueFromConfigMap !== 'string') { + throw new Errors.ValidationError(`valueFromConfigMap for environment variable '${env.key}' must be a string`) } + const parts = env.valueFromConfigMap.split('/') + if (parts.length !== 2 || !parts[0] || !parts[1]) { + throw new Errors.ValidationError(`valueFromConfigMap for environment variable '${env.key}' must be in format 'configmap-name/key'`) + } + envVar.valueFromConfigMap = env.valueFromConfigMap + } + + return envVar + }) +} - return envVar - }) +function parseMicroserviceTemplateRef (template) { + if (template == null) { + return undefined + } + if (typeof template !== 'object' || Array.isArray(template)) { + throw new Errors.ValidationError('Invalid template format') + } + if (!template.name) { + throw new Errors.ValidationError('template.name is required when template is specified') } + const result = { name: template.name } + if (template.variables !== undefined) { + result.variables = template.variables + } + return result +} + +const parseMicroserviceYAML = async (microservice) => { + const { registryId, catalogItemId, images } = await parseMicroserviceImages(microservice.images) + const container = microservice.container || {} + + const commands = container.commands !== undefined ? container.commands : container.cmd + const sysctls = container.sysctls !== undefined + ? CatalogContainer.normalizeSysctls(container.sysctls) + : undefined const microserviceData = { config: microservice.config != null ? JSON.stringify(microservice.config) : undefined, @@ -330,7 +393,6 @@ const parseMicroserviceYAML = async (microservice) => { catalogItemId, agentName: lget(microservice, 'agent.name'), registryId, - ...container, hostNetworkMode: lget(microservice, 'container.hostNetworkMode', false), isPrivileged: lget(microservice, 'container.isPrivileged', false), pidMode: lget(microservice, 'container.pidMode', ''), @@ -343,15 +405,35 @@ const parseMicroserviceYAML = async (microservice) => { capDrop: lget(microservice, 'container.capDrop', []), ports: (lget(microservice, 'container.ports', [])), volumeMappings: lget(microservice, 'container.volumes', []), - cmd: lget(microservice, 'container.commands', []), + commands, + cmd: commands, env: parseEnvVariables(lget(microservice, 'container.env', [])), images, extraHosts: lget(microservice, 'container.extraHosts', []), application: microservice.application, + models: microservice.models, + knowledge: microservice.knowledge, + template: parseMicroserviceTemplateRef(microservice.template), schedule: lget(microservice, 'schedule', 50), serviceAccount: lget(microservice, 'serviceAccount', undefined), natsEnabled: lget(microservice, 'natsEnabled', undefined), - natsConfig: lget(microservice, 'natsConfig', undefined) + natsConfig: lget(microservice, 'natsConfig', undefined), + runAsUser: lget(microservice, 'container.runAsUser', undefined), + runAsGroup: lget(microservice, 'container.runAsGroup', undefined), + readOnlyRootFilesystem: lget(microservice, 'container.readOnlyRootFilesystem', undefined), + platform: lget(microservice, 'container.platform', undefined), + runtime: lget(microservice, 'container.runtime', undefined), + cdiDevices: lget(microservice, 'container.cdiDevices', undefined), + sysctls, + ulimits: lget(microservice, 'container.ulimits', undefined), + cpus: lget(microservice, 'container.cpus', undefined), + memoryReservation: lget(microservice, 'container.memoryReservation', undefined), + memorySwap: lget(microservice, 'container.memorySwap', undefined), + shmSize: lget(microservice, 'container.shmSize', undefined), + devices: lget(microservice, 'container.devices', undefined), + tmpfs: lget(microservice, 'container.tmpfs', undefined), + workingDir: lget(microservice, 'container.workingDir', undefined), + entrypoint: lget(microservice, 'container.entrypoint', undefined) } _deleteUndefinedFields(microserviceData) return microserviceData @@ -648,6 +730,303 @@ async function parseNatsUserRuleFile (fileContent, options = {}) { } } +async function parseModelFile (fileContent, options = {}) { + try { + const doc = yaml.load(fileContent) + if (!doc || !doc.kind) { + throw new Errors.ValidationError('Invalid YAML format: missing kind field') + } + if (doc.kind !== 'Model') { + throw new Errors.ValidationError(`Invalid kind ${doc.kind}`) + } + if (doc.metadata == null || doc.spec == null) { + throw new Errors.ValidationError('Invalid YAML format: missing metadata or spec') + } + + const allowedApiVersions = new Set(['iofog.org/v3', 'datasance.com/v3']) + if (doc.apiVersion && !allowedApiVersions.has(doc.apiVersion)) { + throw new Errors.ValidationError(`Invalid apiVersion ${doc.apiVersion}`) + } + + if (options.isUpdate && options.modelName) { + if (doc.metadata.name !== options.modelName) { + throw new Errors.ValidationError(`Model name in YAML (${doc.metadata.name}) doesn't match endpoint path (${options.modelName})`) + } + } + + const spec = doc.spec || {} + const registryId = spec.registryId != null ? spec.registryId : spec.registry + const result = { + name: lget(doc, 'metadata.name', undefined), + repo: spec.repo, + revision: spec.revision, + registryId, + files: spec.files, + format: spec.format + } + _deleteUndefinedFields(result) + + if (options.isUpdate && options.modelName) { + delete result.name + } + + return result + } catch (error) { + if (error instanceof Errors.ValidationError) { + throw error + } + throw new Errors.ValidationError(`Error parsing YAML: ${error.message}`) + } +} + +async function parseKnowledgeFile (fileContent, options = {}) { + try { + const doc = yaml.load(fileContent) + if (!doc || !doc.kind) { + throw new Errors.ValidationError('Invalid YAML format: missing kind field') + } + if (doc.kind !== 'Knowledge') { + throw new Errors.ValidationError(`Invalid kind ${doc.kind}`) + } + if (doc.metadata == null || doc.spec == null) { + throw new Errors.ValidationError('Invalid YAML format: missing metadata or spec') + } + + const allowedApiVersions = new Set(['iofog.org/v3', 'datasance.com/v3']) + if (doc.apiVersion && !allowedApiVersions.has(doc.apiVersion)) { + throw new Errors.ValidationError(`Invalid apiVersion ${doc.apiVersion}`) + } + + if (options.isUpdate && options.knowledgeName) { + if (doc.metadata.name !== options.knowledgeName) { + throw new Errors.ValidationError(`Knowledge name in YAML (${doc.metadata.name}) doesn't match endpoint path (${options.knowledgeName})`) + } + } + + const spec = doc.spec || {} + const registryId = spec.registryId != null ? spec.registryId : spec.registry + const result = { + name: lget(doc, 'metadata.name', undefined), + repo: spec.repo, + revision: spec.revision, + registryId, + files: spec.files, + format: spec.format + } + _deleteUndefinedFields(result) + + if (options.isUpdate && options.knowledgeName) { + delete result.name + } + + return result + } catch (error) { + if (error instanceof Errors.ValidationError) { + throw error + } + throw new Errors.ValidationError(`Error parsing YAML: ${error.message}`) + } +} + +async function parseRuntimeClassFile (fileContent, options = {}) { + try { + const doc = yaml.load(fileContent) + if (!doc || !doc.kind) { + throw new Errors.ValidationError('Invalid YAML format: missing kind field') + } + if (doc.kind !== 'RuntimeClass') { + throw new Errors.ValidationError(`Invalid kind ${doc.kind}`) + } + if (doc.metadata == null) { + throw new Errors.ValidationError('Invalid YAML format: missing metadata') + } + + const allowedApiVersions = new Set(['iofog.org/v3', 'datasance.com/v3']) + if (doc.apiVersion && !allowedApiVersions.has(doc.apiVersion)) { + throw new Errors.ValidationError(`Invalid apiVersion ${doc.apiVersion}`) + } + + if (options.isUpdate && options.runtimeClassName) { + if (doc.metadata.name !== options.runtimeClassName) { + throw new Errors.ValidationError(`RuntimeClass name in YAML (${doc.metadata.name}) doesn't match endpoint path (${options.runtimeClassName})`) + } + } + + const handler = doc.handler != null && doc.handler !== '' + ? doc.handler + : lget(doc, 'spec.handler', undefined) + if (handler == null || handler === '') { + throw new Errors.ValidationError('Invalid YAML format: handler is required') + } + + const result = { + name: lget(doc, 'metadata.name', undefined), + handler + } + _deleteUndefinedFields(result) + + if (options.isUpdate && options.runtimeClassName) { + delete result.name + } + + return result + } catch (error) { + if (error instanceof Errors.ValidationError) { + throw error + } + throw new Errors.ValidationError(`Error parsing YAML: ${error.message}`) + } +} + +function setContainerFieldIfPresent (data, container, dataKey, containerKey, transform) { + if (!Object.hasOwn(container, containerKey)) { + return + } + data[dataKey] = transform ? transform(container[containerKey]) : container[containerKey] +} + +async function parseMicroserviceTemplateYAML (microservice) { + const { registryId, catalogItemId, images } = await parseMicroserviceImages(microservice.images, { templateMode: true }) + const container = microservice.container || {} + const microserviceData = {} + + if (microservice.config != null) { + microserviceData.config = JSON.stringify(microservice.config) + } + if (microservice.application !== undefined) { + microserviceData.application = microservice.application + } + if (lget(microservice, 'agent.name') !== undefined) { + microserviceData.agentName = lget(microservice, 'agent.name') + } + if (catalogItemId !== undefined) { + microserviceData.catalogItemId = catalogItemId + } + if (registryId !== undefined) { + microserviceData.registryId = registryId + } + if (images && images.length > 0) { + microserviceData.images = images + } + if (microservice.models !== undefined) { + microserviceData.models = microservice.models + } + if (microservice.knowledge !== undefined) { + microserviceData.knowledge = microservice.knowledge + } + if (microservice.schedule !== undefined) { + microserviceData.schedule = microservice.schedule + } + if (microservice.serviceAccount !== undefined) { + microserviceData.serviceAccount = microservice.serviceAccount + } + if (microservice.natsEnabled !== undefined) { + microserviceData.natsEnabled = microservice.natsEnabled + } + if (microservice.natsConfig !== undefined) { + microserviceData.natsConfig = microservice.natsConfig + } + + setContainerFieldIfPresent(microserviceData, container, 'hostNetworkMode', 'hostNetworkMode') + setContainerFieldIfPresent(microserviceData, container, 'isPrivileged', 'isPrivileged') + setContainerFieldIfPresent(microserviceData, container, 'pidMode', 'pidMode') + setContainerFieldIfPresent(microserviceData, container, 'ipcMode', 'ipcMode') + setContainerFieldIfPresent(microserviceData, container, 'cpuSetCpus', 'cpuSetCpus') + setContainerFieldIfPresent(microserviceData, container, 'memoryLimit', 'memoryLimit') + setContainerFieldIfPresent(microserviceData, container, 'runAsUser', 'runAsUser') + setContainerFieldIfPresent(microserviceData, container, 'runAsGroup', 'runAsGroup') + setContainerFieldIfPresent(microserviceData, container, 'readOnlyRootFilesystem', 'readOnlyRootFilesystem') + setContainerFieldIfPresent(microserviceData, container, 'platform', 'platform') + setContainerFieldIfPresent(microserviceData, container, 'runtime', 'runtime') + setContainerFieldIfPresent(microserviceData, container, 'cdiDevices', 'cdiDevices') + setContainerFieldIfPresent(microserviceData, container, 'cpus', 'cpus') + setContainerFieldIfPresent(microserviceData, container, 'memoryReservation', 'memoryReservation') + setContainerFieldIfPresent(microserviceData, container, 'memorySwap', 'memorySwap') + setContainerFieldIfPresent(microserviceData, container, 'shmSize', 'shmSize') + setContainerFieldIfPresent(microserviceData, container, 'workingDir', 'workingDir') + setContainerFieldIfPresent(microserviceData, container, 'entrypoint', 'entrypoint') + setContainerFieldIfPresent(microserviceData, container, 'ulimits', 'ulimits') + setContainerFieldIfPresent(microserviceData, container, 'devices', 'devices') + setContainerFieldIfPresent(microserviceData, container, 'tmpfs', 'tmpfs') + setContainerFieldIfPresent(microserviceData, container, 'capAdd', 'capAdd') + setContainerFieldIfPresent(microserviceData, container, 'capDrop', 'capDrop') + setContainerFieldIfPresent(microserviceData, container, 'ports', 'ports') + setContainerFieldIfPresent(microserviceData, container, 'volumes', 'volumeMappings') + setContainerFieldIfPresent(microserviceData, container, 'extraHosts', 'extraHosts') + setContainerFieldIfPresent(microserviceData, container, 'healthCheck', 'healthCheck') + if (container.annotations != null) { + microserviceData.annotations = JSON.stringify(container.annotations) + } + if (container.sysctls !== undefined) { + microserviceData.sysctls = CatalogContainer.normalizeSysctls(container.sysctls) + } + if (Object.hasOwn(container, 'commands') || Object.hasOwn(container, 'cmd')) { + microserviceData.commands = container.commands !== undefined ? container.commands : container.cmd + } + if (Object.hasOwn(container, 'env')) { + microserviceData.env = parseEnvVariables(container.env) + } + + _deleteUndefinedFields(microserviceData) + return microserviceData +} + +async function parseMicroserviceTemplateFile (fileContent, options = {}) { + try { + const doc = yaml.load(fileContent) + if (!doc || !doc.kind) { + throw new Errors.ValidationError('Invalid YAML format: missing kind field') + } + if (doc.kind !== 'MicroserviceTemplate') { + throw new Errors.ValidationError(`Invalid kind ${doc.kind}`) + } + if (doc.metadata == null || doc.spec == null) { + throw new Errors.ValidationError('Invalid YAML format: missing metadata or spec') + } + + const allowedApiVersions = new Set(['iofog.org/v3', 'datasance.com/v3']) + if (doc.apiVersion && !allowedApiVersions.has(doc.apiVersion)) { + throw new Errors.ValidationError(`Invalid apiVersion ${doc.apiVersion}`) + } + + if (options.isUpdate && options.templateName) { + if (doc.metadata.name !== options.templateName) { + throw new Errors.ValidationError(`MicroserviceTemplate name in YAML (${doc.metadata.name}) doesn't match endpoint path (${options.templateName})`) + } + } + + const spec = doc.spec || {} + if (!spec.microservice || typeof spec.microservice !== 'object' || Array.isArray(spec.microservice)) { + throw new Errors.ValidationError('MicroserviceTemplate spec.microservice is required') + } + + const microservice = await parseMicroserviceTemplateYAML(spec.microservice) + delete microservice.name + delete microservice.iofogUuid + delete microservice.flowId + delete microservice.template + + const result = { + name: lget(doc, 'metadata.name', undefined), + description: spec.description, + variables: spec.variables, + microservice + } + _deleteUndefinedFields(result) + + if (options.isUpdate && options.templateName) { + delete result.name + } + + return result + } catch (error) { + if (error instanceof Errors.ValidationError) { + throw error + } + throw new Errors.ValidationError(`Error parsing YAML: ${error.message}`) + } +} + module.exports = { parseAppTemplateFile, parseAppFile, @@ -661,5 +1040,9 @@ module.exports = { parseServiceFile, parseRoleFile, parseRoleBindingFile, - parseServiceAccountFile + parseServiceAccountFile, + parseModelFile, + parseKnowledgeFile, + parseRuntimeClassFile, + parseMicroserviceTemplateFile } diff --git a/test/src/config/rbac-audit.test.js b/test/src/config/rbac-audit.test.js new file mode 100644 index 000000000..bb5384b3d --- /dev/null +++ b/test/src/config/rbac-audit.test.js @@ -0,0 +1,53 @@ +'use strict' + +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') +const yaml = require('js-yaml') +const { auditRbac } = require('../../../scripts/rbac-audit') + +const CATALOG_PATH = path.join(__dirname, '../../../src/config/rbac-resources.yaml') + +describe('RBAC route catalog', () => { + it('registers every live Express route in rbac-resources.yaml', () => { + const result = auditRbac() + expect(result.gaps, JSON.stringify(result.gaps, null, 2)).to.deep.equal([]) + expect(result.banned).to.deep.equal([]) + expect(result.missing).to.deep.equal([]) + }) + + it('registers fleet model, Knowledge, RuntimeClass, and microservice template routes', () => { + const catalog = yaml.load(fs.readFileSync(CATALOG_PATH, 'utf8')) + + expect(catalog.resources.models).to.be.an('object') + expect(catalog.resources.knowledge).to.be.an('object') + expect(catalog.resources.runtimeClasses).to.be.an('object') + expect(catalog.resources.microserviceTemplates).to.be.an('object') + expect(catalog.resources.microservices.routes.some((route) => { + return route.path === '/api/v3/microservices/:uuid/models' + })).to.equal(true) + expect(catalog.resources.microservices.routes.some((route) => { + return route.path === '/api/v3/microservices/:uuid/knowledge' + })).to.equal(true) + expect(catalog.resources.agent.routes.some((route) => route.path === '/api/v3/agent/models')).to.equal(true) + expect(catalog.resources.agent.routes.some((route) => route.path === '/api/v3/agent/knowledge')).to.equal(true) + expect(catalog.resources.knowledge.routes.some((route) => String(route.path).includes('/agent/'))).to.equal(false) + expect(catalog.resources.agent.routes.some((route) => route.path === '/api/v3/agent/runtimeClasses')).to.equal(true) + }) + + it('does not leave stale yaml entries for fleet model, Knowledge, or RuntimeClass routes', () => { + const result = auditRbac() + const fleetOrphans = result.orphans.filter((orphan) => { + return orphan.resource === 'models' || + orphan.resource === 'knowledge' || + orphan.resource === 'runtimeClasses' || + orphan.resource === 'microserviceTemplates' || + orphan.path.includes('/agent/models') || + orphan.path.includes('/agent/knowledge') || + orphan.path.includes('/agent/runtimeClasses') || + orphan.path.includes('/microservices/:uuid/models') || + orphan.path.includes('/microservices/:uuid/knowledge') + }) + expect(fleetOrphans, JSON.stringify(fleetOrphans, null, 2)).to.deep.equal([]) + }) +}) diff --git a/test/src/controllers/agent-controller.test.js b/test/src/controllers/agent-controller.test.js index 17082bb3d..46796702c 100644 --- a/test/src/controllers/agent-controller.test.js +++ b/test/src/controllers/agent-controller.test.js @@ -139,7 +139,6 @@ describe('Agent Controller', () => { def('logFileCount', () => 5) def('statusFrequency', () => 60) def('changeFrequency', () => 30) - def('deviceScanFrequency', () => 40) def('watchdogEnabled', () => true) def('latitude', () => 30) def('longitude', () => 40) @@ -158,7 +157,6 @@ describe('Agent Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - devicesScanFrequency: $deviceScanFrequency, watchdogEnabled: $watchdogEnabled, latitude: $latitude, longitude: $longitude, @@ -187,7 +185,6 @@ describe('Agent Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - devicesScanFrequency: $deviceScanFrequency, watchdogEnabled: $watchdogEnabled, latitude: $latitude, longitude: $longitude, @@ -535,85 +532,57 @@ describe('Agent Controller', () => { }) }) - describe('updateHalHardwareInfoEndPoint()', () => { + describe('getAgentLinkedModelsEndpoint()', () => { def('fog', () => 'fog!') - - def('info', () => 'testInfo') - - def('req', () => ({ - body: { - info: $info, - }, - })) - def('response', () => Promise.resolve()) - def('subject', () => $subject.updateHalHardwareInfoEndPoint($req, $fog)) + def('req', () => ({ body: {} })) + def('response', () => Promise.resolve([])) + def('subject', () => $subject.getAgentLinkedModelsEndpoint($req, $fog)) beforeEach(() => { - $sandbox.stub(AgentService, 'updateHalHardwareInfo').returns($response) - }) - - it('calls AgentService.updateHalHardwareInfo with correct args', async () => { - await $subject - expect(AgentService.updateHalHardwareInfo).to.have.been.calledWith({ - info: $info, - }, $fog) + $sandbox.stub(AgentService, 'getAgentLinkedModels').returns($response) }) - context('when AgentService#updateHalHardwareInfo fails', () => { - const error = 'Error!' - - def('response', () => Promise.reject(error)) - - it(`fails with "${error}"`, () => { - return expect($subject).to.be.rejectedWith(error) - }) - }) - - context('when AgentService#updateHalHardwareInfo succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.eventually.equal(undefined) - }) + it('returns linked models for the requesting fog', async () => { + await expect($subject).to.eventually.eql({ models: [] }) + expect(AgentService.getAgentLinkedModels).to.have.been.calledWith($fog) }) }) - describe('updateHalUsbInfoEndPoint()', () => { + describe('getAgentLinkedKnowledgeEndpoint()', () => { def('fog', () => 'fog!') - - def('info', () => 'testInfo') - - def('req', () => ({ - body: { - info: $info, - }, - })) - def('response', () => Promise.resolve()) - def('subject', () => $subject.updateHalUsbInfoEndPoint($req, $fog)) + def('req', () => ({ body: {} })) + def('response', () => Promise.resolve([])) + def('subject', () => $subject.getAgentLinkedKnowledgeEndpoint($req, $fog)) beforeEach(() => { - $sandbox.stub(AgentService, 'updateHalUsbInfo').returns($response) + $sandbox.stub(AgentService, 'getAgentLinkedKnowledge').returns($response) }) - it('calls AgentService.updateHalUsbInfo with correct args', async () => { - await $subject - expect(AgentService.updateHalUsbInfo).to.have.been.calledWith({ - info: $info, - }, $fog) + it('returns linked knowledge for the requesting fog', async () => { + await expect($subject).to.eventually.eql({ knowledge: [] }) + expect(AgentService.getAgentLinkedKnowledge).to.have.been.calledWith($fog) }) - context('when AgentService#updateHalUsbInfo fails', () => { - const error = 'Error!' + it('keeps linked rows under the knowledge key', async () => { + const rows = [{ uuid: 'k-1', name: 'product-docs' }] + AgentService.getAgentLinkedKnowledge.resolves(rows) + await expect($subject).to.eventually.eql({ knowledge: rows }) + }) + }) - def('response', () => Promise.reject(error)) + describe('getAgentLinkedRuntimeClassesEndpoint()', () => { + def('fog', () => 'fog!') + def('req', () => ({ body: {} })) + def('response', () => Promise.resolve([])) + def('subject', () => $subject.getAgentLinkedRuntimeClassesEndpoint($req, $fog)) - it(`fails with "${error}"`, () => { - return expect($subject).to.be.rejectedWith(error) - }) + beforeEach(() => { + $sandbox.stub(AgentService, 'getAgentLinkedRuntimeClasses').returns($response) }) - context('when AgentService#updateHalUsbInfo succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.eventually.equal(undefined) - }) + it('returns linked runtime classes for the requesting fog', async () => { + await expect($subject).to.eventually.eql({ runtimeClasses: [] }) + expect(AgentService.getAgentLinkedRuntimeClasses).to.have.been.calledWith($fog) }) }) diff --git a/test/src/controllers/iofog-controller.test.js b/test/src/controllers/iofog-controller.test.js index 187212359..691a14211 100644 --- a/test/src/controllers/iofog-controller.test.js +++ b/test/src/controllers/iofog-controller.test.js @@ -30,10 +30,7 @@ describe('ioFog Controller', () => { def('logFileCount', () => 8) def('statusFrequency', 6) def('changeFrequency', 18) - def('deviceScanFrequency', 28) - def('bluetoothEnabled', () => false) def('watchdogEnabled', () => true) - def('abstractedHardwareEnabled', () => false) def('archId', () => 0) def('req', () => ({ @@ -53,10 +50,7 @@ describe('ioFog Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - deviceScanFrequency: $deviceScanFrequency, - bluetoothEnabled: $bluetoothEnabled, watchdogEnabled: $watchdogEnabled, - abstractedHardwareEnabled: $abstractedHardwareEnabled, archId: $archId, }, })) @@ -86,10 +80,7 @@ describe('ioFog Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - deviceScanFrequency: $deviceScanFrequency, - bluetoothEnabled: $bluetoothEnabled, watchdogEnabled: $watchdogEnabled, - abstractedHardwareEnabled: $abstractedHardwareEnabled, archId: $archId, }, false) }) @@ -130,10 +121,7 @@ describe('ioFog Controller', () => { def('logFileCount', () => 8) def('statusFrequency', 6) def('changeFrequency', 18) - def('deviceScanFrequency', 28) - def('bluetoothEnabled', () => false) def('watchdogEnabled', () => true) - def('abstractedHardwareEnabled', () => false) def('archId', () => 0) def('req', () => ({ @@ -156,10 +144,7 @@ describe('ioFog Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - deviceScanFrequency: $deviceScanFrequency, - bluetoothEnabled: $bluetoothEnabled, watchdogEnabled: $watchdogEnabled, - abstractedHardwareEnabled: $abstractedHardwareEnabled, archId: $archId, }, })) @@ -189,10 +174,7 @@ describe('ioFog Controller', () => { logFileCount: $logFileCount, statusFrequency: $statusFrequency, changeFrequency: $changeFrequency, - deviceScanFrequency: $deviceScanFrequency, - bluetoothEnabled: $bluetoothEnabled, watchdogEnabled: $watchdogEnabled, - abstractedHardwareEnabled: $abstractedHardwareEnabled, archId: $archId, }, false) }) @@ -490,80 +472,4 @@ describe('ioFog Controller', () => { }) }) }) - - describe('getHalHardwareInfoEndPoint()', () => { - def('user', () => 'user!') - def('uuid', () => 'testUuid') - - def('req', () => ({ - params: { - uuid: $uuid, - }, - })) - def('response', () => Promise.resolve()) - def('subject', () => $subject.getHalHardwareInfoEndPoint($req)) - - beforeEach(() => { - $sandbox.stub(ioFogService, 'getHalHardwareInfoEndPoint').returns($response) - }) - - it('calls ioFogService.getHalHardwareInfoEndPoint with correct args', async () => { - await $subject - expect(ioFogService.getHalHardwareInfoEndPoint).to.have.been.calledWith({ uuid: $uuid }, false) - }) - - context('when ioFogService#getHalHardwareInfoEndPoint fails', () => { - const error = 'Error!' - - def('response', () => Promise.reject(error)) - - it(`fails with "${error}"`, () => { - return expect($subject).to.be.rejectedWith(error) - }) - }) - - context('when ioFogService#getHalHardwareInfoEndPoint succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.eventually.equal(undefined) - }) - }) - }) - - describe('getHalUsbInfoEndPoint.()', () => { - def('user', () => 'user!') - def('uuid', () => 'testUuid') - - def('req', () => ({ - params: { - uuid: $uuid, - }, - })) - def('response', () => Promise.resolve({ info: undefined })) - def('subject', () => $subject.getHalUsbInfoEndPoint($req)) - - beforeEach(() => { - $sandbox.stub(ioFogService, 'getHalUsbInfoEndPoint').returns($response) - }) - - it('calls ioFogService.getHalUsbInfoEndPoint with correct args', async () => { - await $subject - expect(ioFogService.getHalUsbInfoEndPoint).to.have.been.calledWith({ uuid: $uuid }, false) - }) - - context('when ioFogService#getHalUsbInfoEndPoint fails', () => { - const error = 'Error!' - - def('response', () => Promise.reject(error)) - - it(`fails with "${error}"`, () => { - return expect($subject).to.be.rejectedWith(error) - }) - }) - - context('when ioFogService#getHalUsbInfoEndPoint succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.eventually.have.property('info') - }) - }) - }) }) diff --git a/test/src/data/fog-platform-managers.test.js b/test/src/data/fog-platform-managers.test.js index f7e8e4fdb..a159e19d0 100644 --- a/test/src/data/fog-platform-managers.test.js +++ b/test/src/data/fog-platform-managers.test.js @@ -7,7 +7,8 @@ const { parseSpecJson, serializeSpecJson, parseSpecSnapshot, - serializeSpecSnapshot + serializeSpecSnapshot, + mergePlatformSpecPatch } = require('../../../src/schemas/fog-platform-spec') const FogPlatformReconcileTaskManager = require('../../../src/data/managers/fog-platform-reconcile-task-manager') const ServicePlatformReconcileTaskManager = require('../../../src/data/managers/service-platform-reconcile-task-manager') @@ -28,9 +29,29 @@ describe('Fog platform spec schema', () => { throw new Error('expected validation to fail') } catch (error) { expect(error.name).to.equal('ValidationError') + expect(error.message).to.equal("Invalid fog platform spec field 'unknownField'") } }) + it('mergePlatformSpecPatch drops removed fields from existing spec', async () => { + const merged = mergePlatformSpecPatch({ + routerMode: 'edge', + natsMode: 'leaf', + host: '1.2.3.4', + bluetoothEnabled: false, + abstractedHardwareEnabled: false + }, { + host: '5.6.7.8' + }) + + expect(merged).to.eql({ + routerMode: 'edge', + natsMode: 'leaf', + host: '5.6.7.8' + }) + await validateFogPlatformSpec(merged) + }) + it('round-trips spec JSON through parse and serialize', () => { const spec = { routerMode: 'interior', diff --git a/test/src/data/schema-version-chain-sqlite.test.js b/test/src/data/schema-version-chain-sqlite.test.js new file mode 100644 index 000000000..608fa6603 --- /dev/null +++ b/test/src/data/schema-version-chain-sqlite.test.js @@ -0,0 +1,382 @@ +'use strict' + +const { expect } = require('chai') +const fs = require('fs') +const os = require('os') +const path = require('path') +const sqlite3 = require('sqlite3').verbose() + +const DatabaseProvider = require('../../../src/data/providers/database-provider') + +function sqliteGet (db, sql, params = []) { + return new Promise((resolve, reject) => { + db.get(sql, params, (err, row) => { + if (err) reject(err) + else resolve(row) + }) + }) +} + +function sqliteAll (db, sql, params = []) { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => { + if (err) reject(err) + else resolve(rows) + }) + }) +} + +function sqliteClose (db) { + return new Promise((resolve, reject) => { + db.close((err) => { + if (err) reject(err) + else resolve() + }) + }) +} + +function openSqlite (dbPath) { + return new Promise((resolve, reject) => { + const db = new sqlite3.Database(dbPath, (err) => { + if (err) reject(err) + else resolve(db) + }) + }) +} + +function cleanupSqlite (dbPath) { + for (const suffix of ['', '-wal', '-shm']) { + try { + fs.unlinkSync(dbPath + suffix) + } catch (_) { /* ignore */ } + } +} + +function normalizeDefault (value) { + if (value == null) { + return null + } + const text = String(value).trim().replace(/^['"]|['"]$/g, '').toLowerCase() + if (text === 'false' || text === '0') { + return '0' + } + if (text === 'true' || text === '1') { + return '1' + } + return text +} + +function normalizeIndexSql (sql) { + if (!sql) { + return '' + } + return sql + .replace(/\s+/g, ' ') + .replace(/IF NOT EXISTS /ig, '') + .replace(/"/g, '') + .trim() + .toLowerCase() +} + +async function dumpNormalizedSchema (db) { + const tables = await sqliteAll(db, ` + SELECT name FROM sqlite_master + WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND name != 'SchemaVersion' + ORDER BY name + `) + + const tableDump = [] + for (const { name } of tables) { + const cols = await sqliteAll(db, `PRAGMA table_info("${name}")`) + const fks = await sqliteAll(db, `PRAGMA foreign_key_list("${name}")`) + tableDump.push({ + table: name, + columns: cols + .map((col) => ({ + name: col.name, + type: String(col.type || '').toUpperCase(), + notnull: col.notnull, + dflt_value: normalizeDefault(col.dflt_value), + pk: col.pk + })) + .sort((left, right) => left.name.localeCompare(right.name)), + foreignKeys: fks + .map((fk) => ({ + table: fk.table, + from: fk.from, + to: fk.to, + on_delete: String(fk.on_delete || '').toUpperCase() + })) + .sort((left, right) => { + const key = (row) => `${row.table}|${row.from}|${row.to}|${row.on_delete}` + return key(left).localeCompare(key(right)) + }) + }) + } + + const indexes = await sqliteAll(db, ` + SELECT name, tbl_name, sql FROM sqlite_master + WHERE type = 'index' + AND name NOT LIKE 'sqlite_%' + AND tbl_name != 'SchemaVersion' + ORDER BY name + `) + + return JSON.stringify({ + tables: tableDump, + indexes: indexes.map((index) => ({ + name: index.name, + table: index.tbl_name, + sql: normalizeIndexSql(index.sql) + })) + }, null, 2) +} + +describe('schema version chain (sqlite)', function () { + this.timeout(60000) + + let dbPath + const provider = new DatabaseProvider() + + beforeEach(() => { + dbPath = path.join(os.tmpdir(), `controller-schema-chain-${Date.now()}-${Math.random()}.sqlite`) + }) + + afterEach(() => { + cleanupSqlite(dbPath) + }) + + it('fresh install reaches 3.9.0 with Hub registry and system catalog only', async () => { + await provider.runVersionChainSQLite(dbPath) + + const db = await openSqlite(dbPath) + try { + const versions = await sqliteAll(db, 'SELECT migration_version, seeder_version FROM SchemaVersion ORDER BY id') + expect(versions).to.have.length(1) + expect(versions[0].migration_version).to.equal('3.9.0') + expect(versions[0].seeder_version).to.equal('3.9.0') + + const authGroups = await sqliteAll(db, 'SELECT name FROM AuthGroups ORDER BY name') + expect(authGroups.map((row) => row.name)).to.deep.equal(['admin', 'developer', 'sre', 'viewer']) + const authPolicy = await sqliteGet(db, 'SELECT id FROM AuthPolicy WHERE id = 1') + expect(authPolicy).to.not.equal(undefined) + const authBootstrap = await sqliteGet(db, 'SELECT id FROM AuthBootstrapMeta WHERE id = 1') + expect(authBootstrap).to.not.equal(undefined) + + const hub = await sqliteGet(db, "SELECT id, url, type FROM Registries WHERE url = 'https://huggingface.co' AND type = 'hf'") + expect(hub).to.not.equal(undefined) + expect(hub.id).to.equal(3) + expect(hub.type).to.equal('hf') + + const catalog = await sqliteAll(db, "SELECT name FROM CatalogItems WHERE category = 'SYSTEM' ORDER BY name") + expect(catalog.map((row) => row.name)).to.deep.equal(['Debug', 'NATs', 'Router']) + + const tables = await sqliteAll(db, "SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('Models', 'RuntimeClasses', 'FogModels', 'FogRuntimeClasses', 'MicroserviceTemplates', 'MicroserviceTemplateVariables', 'MicroserviceEntrypoints', 'MicroserviceDevices', 'MicroserviceTmpfs', 'MicroserviceUlimits', 'MicroserviceModels', 'MicroserviceModelItems', 'HWInfos', 'USBInfos')") + const tableNames = tables.map((row) => row.name).sort() + expect(tableNames).to.deep.equal([ + 'FogModels', + 'FogRuntimeClasses', + 'MicroserviceDevices', + 'MicroserviceEntrypoints', + 'MicroserviceModelItems', + 'MicroserviceModels', + 'MicroserviceTemplateVariables', + 'MicroserviceTemplates', + 'MicroserviceTmpfs', + 'MicroserviceUlimits', + 'Models', + 'RuntimeClasses' + ]) + + const templateCols = await sqliteAll(db, 'PRAGMA table_info(MicroserviceTemplates)') + expect(templateCols.map((col) => col.name)).to.not.include('variables_json') + expect(templateCols.map((col) => col.name)).to.include('microservice_json') + + const archInfo = await sqliteGet(db, 'PRAGMA table_info(Architectures)') + const archCols = await sqliteAll(db, 'PRAGMA table_info(Architectures)') + expect(archInfo).to.not.equal(undefined) + expect(archCols.map((col) => col.name)).to.not.include('hal_catalog_item_id') + expect(archCols.map((col) => col.name)).to.not.include('bluetooth_catalog_item_id') + expect(archCols.map((col) => col.name)).to.include('network_catalog_item_id') + + const runtimeClasses = await sqliteAll(db, 'SELECT name FROM RuntimeClasses') + expect(runtimeClasses).to.deep.equal([]) + } finally { + await sqliteClose(db) + } + }) + + it('loads sequelize models against the migrated schema', async () => { + await provider.runVersionChainSQLite(dbPath) + + const Sequelize = require('sequelize') + const sequelize = new Sequelize({ + dialect: 'sqlite', + storage: dbPath, + logging: false + }) + const modelsDir = path.join(__dirname, '../../../src/data/models') + const db = {} + fs.readdirSync(modelsDir) + .filter((file) => file.endsWith('.js') && file !== 'index.js') + .forEach((file) => { + const model = require(path.join(modelsDir, file))(sequelize, Sequelize.DataTypes) + db[model.name] = model + }) + Object.keys(db).forEach((name) => { + if (db[name].associate) { + db[name].associate(db) + } + }) + + try { + const architectures = await db.Architecture.findAll({ order: [['id', 'ASC']] }) + expect(architectures).to.have.length(5) + expect(architectures[0].networkCatalogItemId).to.not.equal(undefined) + expect(architectures[0].halCatalogItemId).to.equal(undefined) + + const hub = await db.Registry.findOne({ where: { url: 'https://huggingface.co', type: 'hf' } }) + expect(hub).to.not.equal(null) + expect(hub.type).to.equal('hf') + expect(hub.insecure).to.equal(false) + + const fleetModels = await db.FleetModel.findAll() + expect(fleetModels).to.have.length(0) + const runtimeClasses = await db.RuntimeClass.findAll() + expect(runtimeClasses).to.have.length(0) + const microserviceTemplates = await db.MicroserviceTemplate.findAll() + expect(microserviceTemplates).to.have.length(0) + const microserviceTemplateVariables = await db.MicroserviceTemplateVariable.findAll() + expect(microserviceTemplateVariables).to.have.length(0) + + const fogs = await db.Fog.findAll() + expect(fogs).to.have.length(0) + } finally { + await sequelize.close() + } + }) + + it('upgrade from 3.8.0 applies 3.9.0 without dropping architecture rows', async () => { + await provider.runVersionChainSQLite(dbPath, { untilVersion: '3.8.0' }) + + const before = await openSqlite(dbPath) + let architectureNames + try { + const schema = await sqliteGet(before, 'SELECT migration_version, seeder_version FROM SchemaVersion ORDER BY id DESC LIMIT 1') + expect(schema.migration_version).to.equal('3.8.0') + expect(schema.seeder_version).to.equal('3.8.0') + + const catalog = await sqliteAll(before, "SELECT name FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue') ORDER BY name") + expect(catalog.map((row) => row.name)).to.deep.equal(['HAL', 'RESTBlue']) + + const hub = await sqliteGet(before, "SELECT id FROM Registries WHERE url = 'https://huggingface.co'") + expect(hub).to.equal(undefined) + + const architectures = await sqliteAll(before, 'SELECT id, name FROM Architectures ORDER BY id') + expect(architectures).to.have.length(5) + architectureNames = architectures.map((row) => row.name) + } finally { + await sqliteClose(before) + } + + await provider.runVersionChainSQLite(dbPath) + + const after = await openSqlite(dbPath) + try { + const versions = await sqliteAll(after, 'SELECT migration_version, seeder_version FROM SchemaVersion ORDER BY id') + expect(versions).to.have.length(2) + expect(versions[0].migration_version).to.equal('3.8.0') + expect(versions[0].seeder_version).to.equal('3.8.0') + expect(versions[1].migration_version).to.equal('3.9.0') + expect(versions[1].seeder_version).to.equal('3.9.0') + + const architectures = await sqliteAll(after, 'SELECT id, name FROM Architectures ORDER BY id') + expect(architectures).to.have.length(5) + expect(architectures.map((row) => row.name)).to.deep.equal(architectureNames) + + const catalog = await sqliteAll(after, "SELECT name FROM CatalogItems WHERE name IN ('HAL', 'RESTBlue')") + expect(catalog).to.deep.equal([]) + + const hub = await sqliteGet(after, "SELECT id, url, type FROM Registries WHERE url = 'https://huggingface.co' AND type = 'hf'") + expect(hub).to.not.equal(undefined) + expect(hub.type).to.equal('hf') + expect(hub.id).to.not.equal(1) + expect(hub.id).to.not.equal(2) + + const fogCols = await sqliteAll(after, 'PRAGMA table_info(Fogs)') + const fogColNames = fogCols.map((col) => col.name) + expect(fogColNames).to.not.include('device_scan_frequency') + expect(fogColNames).to.not.include('bluetooth') + expect(fogColNames).to.not.include('hal') + expect(fogColNames).to.include('model_status') + expect(fogColNames).to.include('runtime_classes') + expect(fogColNames).to.include('available_cdi_devices') + + const msCols = await sqliteAll(after, 'PRAGMA table_info(Microservices)') + const msColNames = msCols.map((col) => col.name) + expect(msColNames).to.include('sysctls') + expect(msColNames).to.include('run_as_group') + expect(msColNames).to.not.include('entrypoint') + expect(msColNames).to.not.include('commands') + expect(msColNames).to.not.include('ulimits') + expect(msColNames).to.not.include('devices') + expect(msColNames).to.not.include('tmpfs') + expect(msColNames).to.not.include('models') + + const tables = await sqliteAll(after, "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name") + const tableNames = tables.map((row) => row.name) + expect(tableNames).to.include.members([ + 'MicroserviceEntrypoints', + 'MicroserviceDevices', + 'MicroserviceTmpfs', + 'MicroserviceUlimits', + 'MicroserviceModels', + 'MicroserviceModelItems' + ]) + } finally { + await sqliteClose(after) + } + }) + + it('fresh baseline schema matches upgrade from 3.8.0', async () => { + const freshPath = dbPath + const upgradePath = dbPath + '.upgrade' + cleanupSqlite(upgradePath) + try { + await provider.runVersionChainSQLite(freshPath) + await provider.runVersionChainSQLite(upgradePath, { untilVersion: '3.8.0' }) + await provider.runVersionChainSQLite(upgradePath) + + const freshDb = await openSqlite(freshPath) + const upgradeDb = await openSqlite(upgradePath) + try { + const freshDump = await dumpNormalizedSchema(freshDb) + const upgradeDump = await dumpNormalizedSchema(upgradeDb) + expect(freshDump).to.equal(upgradeDump) + } finally { + await sqliteClose(freshDb) + await sqliteClose(upgradeDb) + } + } finally { + cleanupSqlite(upgradePath) + } + }) + + it('is idempotent when the version chain runs twice on a baseline database', async () => { + await provider.runVersionChainSQLite(dbPath) + await provider.runVersionChainSQLite(dbPath) + + const db = await openSqlite(dbPath) + try { + const versions = await sqliteAll(db, 'SELECT migration_version, seeder_version FROM SchemaVersion ORDER BY id') + expect(versions).to.have.length(1) + expect(versions[0].migration_version).to.equal('3.9.0') + expect(versions[0].seeder_version).to.equal('3.9.0') + + const catalog = await sqliteAll(db, "SELECT name FROM CatalogItems WHERE category = 'SYSTEM' ORDER BY name") + expect(catalog.map((row) => row.name)).to.deep.equal(['Debug', 'NATs', 'Router']) + } finally { + await sqliteClose(db) + } + }) +}) diff --git a/test/src/helpers/microservice-container-catalog.test.js b/test/src/helpers/microservice-container-catalog.test.js new file mode 100644 index 000000000..6e5f6e459 --- /dev/null +++ b/test/src/helpers/microservice-container-catalog.test.js @@ -0,0 +1,244 @@ +'use strict' + +const { expect } = require('chai') + +const CatalogContainer = require('../../../src/helpers/microservice-container-catalog') +const Errors = require('../../../src/helpers/errors') + +describe('Microservice container catalog helpers', () => { + describe('.catalogRequiresRebuild()', () => { + const populated = { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + + it('rebuilds empty to non-empty', () => { + expect(CatalogContainer.catalogRequiresRebuild(null, populated)).to.equal(true) + }) + + it('rebuilds non-empty to empty', () => { + expect(CatalogContainer.catalogRequiresRebuild(populated, { items: [] })).to.equal(true) + }) + + it('rebuilds bindPath changes', () => { + expect(CatalogContainer.catalogRequiresRebuild(populated, { + ...populated, + bindPath: '/opt/models' + })).to.equal(true) + }) + + it('rebuilds permissions changes', () => { + expect(CatalogContainer.catalogRequiresRebuild(populated, { + ...populated, + permissions: 'rw' + })).to.equal(true) + }) + + it('does not rebuild item add/remove with the same bindPath and permissions', () => { + expect(CatalogContainer.catalogRequiresRebuild(populated, { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }, { name: 'qwen3-8-27b' }] + })).to.equal(false) + }) + }) + + describe('.resolveProcessArgv()', () => { + it('prefers commands over cmd', () => { + expect(CatalogContainer.resolveProcessArgv({ + commands: ['a'], + cmd: ['b'] + })).to.eql(['a']) + }) + + it('accepts cmd as an alias', () => { + expect(CatalogContainer.resolveProcessArgv({ cmd: ['b'] })).to.eql(['b']) + }) + }) + + describe('.serializeArgv()', () => { + it('omits empty arrays', () => { + expect(CatalogContainer.serializeArgv([])).to.equal(null) + expect(CatalogContainer.serializeArgv(undefined)).to.equal(null) + }) + }) + + describe('.validateCatalog()', () => { + it('requires bindPath when items are present', () => { + expect(() => CatalogContainer.validateCatalog({ + items: [{ name: 'test-model' }] + })).to.throw(Errors.ValidationError) + }) + + it('rejects a model bindPath when there are no items', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/models', + items: [] + })).to.throw(Errors.ValidationError, /requires at least one item/) + }) + + it('rejects a knowledge bindPath when items are omitted', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/knowledge' + }, { kind: 'knowledge' })).to.throw(Errors.ValidationError, /requires at least one item/) + }) + + it('accepts an empty catalog without bindPath', () => { + expect(() => CatalogContainer.validateCatalog({ items: [] })).to.not.throw() + expect(() => CatalogContainer.validateCatalog({ + bindPath: '', + items: [] + }, { kind: 'knowledge' })).to.not.throw() + }) + + it('rejects duplicate item names', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/models', + items: [{ name: 'test-model' }, { name: 'test-model' }] + })).to.throw(Errors.ValidationError) + }) + + it('rejects bindPath collisions with volumes', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/data', + items: [{ name: 'test-model' }] + }, { + volumeMappings: [{ containerDestination: '/data' }] + })).to.throw(Errors.ValidationError) + }) + + it('rejects a knowledge bindPath that matches the models catalog', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }, { + kind: 'knowledge', + otherCatalog: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + })).to.throw(Errors.ValidationError, /Knowledge catalog path/) + }) + + it('rejects a knowledge item path that matches a volume destination', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + }, { + kind: 'knowledge', + volumeMappings: [{ containerDestination: '/knowledge/product-docs' }] + })).to.throw(Errors.ValidationError, /Knowledge catalog path/) + }) + + it('rejects a knowledge bindPath that matches a tmpfs mount', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/run/knowledge', + items: [{ name: 'product-docs' }] + }, { + kind: 'knowledge', + tmpfs: [{ containerPath: '/run/knowledge' }] + })).to.throw(Errors.ValidationError, /Knowledge catalog path/) + }) + + it('rejects duplicate knowledge item names', () => { + expect(() => CatalogContainer.validateCatalog({ + bindPath: '/knowledge', + items: [{ name: 'product-docs' }, { name: 'product-docs' }] + }, { kind: 'knowledge' })).to.throw(Errors.ValidationError, /duplicated/) + }) + }) + + describe('.validateContainerFields()', () => { + it('rejects runAsUser with colon when runAsGroup is set', () => { + expect(() => CatalogContainer.validateContainerFields({ + runAsUser: '1000:1000', + runAsGroup: '1000' + })).to.throw(Errors.ValidationError) + }) + + it('rejects memorySwap without memoryLimit unless -1', () => { + expect(() => CatalogContainer.validateContainerFields({ + memorySwap: 512 + })).to.throw(Errors.ValidationError) + expect(() => CatalogContainer.validateContainerFields({ + memorySwap: -1 + })).to.not.throw() + }) + + it('rejects unknown sysctls', () => { + expect(() => CatalogContainer.validateContainerFields({ + sysctls: { 'kernel.unprivileged_bpf_disabled': '1' } + })).to.throw(Errors.ValidationError) + }) + + it('coerces numeric sysctl values to strings', () => { + const data = { sysctls: { 'net.ipv4.tcp_syncookies': 1 } } + expect(() => CatalogContainer.validateContainerFields(data)).to.not.throw() + expect(data.sysctls['net.ipv4.tcp_syncookies']).to.equal('1') + }) + + it('rejects net sysctls when hostNetworkMode is true', () => { + expect(() => CatalogContainer.validateContainerFields({ + hostNetworkMode: true, + sysctls: { 'net.ipv4.tcp_syncookies': '1' } + })).to.throw(Errors.ValidationError) + }) + + it('rejects unknown ulimit keys and scalar values', () => { + expect(() => CatalogContainer.validateContainerFields({ + ulimits: { nofile: 1024 } + })).to.throw(Errors.ValidationError) + expect(() => CatalogContainer.validateContainerFields({ + ulimits: { as: { soft: 1, hard: 1 } } + })).to.throw(Errors.ValidationError) + }) + + it('rejects hostPath devices outside /dev', () => { + expect(() => CatalogContainer.validateContainerFields({ + devices: [{ hostPath: '/etc/passwd', containerPath: '/dev/foo' }] + })).to.throw(Errors.ValidationError) + }) + }) + + describe('.applyAgentContainerFields()', () => { + it('omits empty argv arrays and includes a non-empty catalog', () => { + const response = { uuid: 'ms' } + CatalogContainer.applyAgentContainerFields(response, { + sysctls: null + }, { + cmd: [], + entrypoint: [], + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }, + knowledge: { + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + } + }) + + expect(response).to.not.have.property('entrypoint') + expect(response).to.not.have.property('commands') + expect(response.cmd).to.eql([]) + expect(response.models.items).to.eql([{ name: 'test-model' }]) + expect(response.knowledge.items).to.eql([{ name: 'product-docs' }]) + expect(response.knowledge.items[0]).to.not.have.property('uuid') + }) + + it('emits commands from MicroserviceArgs rows', () => { + const response = { uuid: 'ms' } + CatalogContainer.applyAgentContainerFields(response, {}, { + cmd: ['python', 'app.py'] + }) + + expect(response.commands).to.eql(['python', 'app.py']) + expect(response.cmd).to.eql(['python', 'app.py']) + }) + }) +}) diff --git a/test/src/helpers/microservice-runtime-metrics.test.js b/test/src/helpers/microservice-runtime-metrics.test.js new file mode 100644 index 000000000..b451bda02 --- /dev/null +++ b/test/src/helpers/microservice-runtime-metrics.test.js @@ -0,0 +1,205 @@ +const { expect } = require('chai') + +const { + isLiveRuntimeStatus, + zeroRuntimeMetrics, + coerceRuntimeMetric, + applyRuntimeMetrics, + applyCrashExtras, + projectStatusForApi +} = require('../../../src/helpers/microservice-runtime-metrics') + +describe('helpers/microservice-runtime-metrics', () => { + describe('isLiveRuntimeStatus()', () => { + it('is true only for RUNNING', () => { + expect(isLiveRuntimeStatus('RUNNING')).to.equal(true) + expect(isLiveRuntimeStatus('STOPPING')).to.equal(false) + expect(isLiveRuntimeStatus('STOPPED')).to.equal(false) + expect(isLiveRuntimeStatus('UNKNOWN')).to.equal(false) + }) + }) + + describe('zeroRuntimeMetrics()', () => { + it('returns zero cpu, memory, startTime, and operatingDuration', () => { + expect(zeroRuntimeMetrics()).to.eql({ + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + }) + }) + }) + + describe('coerceRuntimeMetric()', () => { + it('treats omitted values as 0', () => { + expect(coerceRuntimeMetric(undefined)).to.equal(0) + expect(coerceRuntimeMetric(null)).to.equal(0) + expect(coerceRuntimeMetric('')).to.equal(0) + }) + + it('parses numeric strings', () => { + expect(coerceRuntimeMetric('12.5')).to.equal(12.5) + expect(coerceRuntimeMetric('8')).to.equal(8) + }) + + it('treats non-numeric values as 0', () => { + expect(coerceRuntimeMetric('n/a')).to.equal(0) + }) + }) + + describe('applyRuntimeMetrics()', () => { + it('leaves live RUNNING meters unchanged', () => { + const patch = { status: 'RUNNING', cpuUsage: 12.5, memoryUsage: 64, startTime: 9, operatingDuration: 10 } + expect(applyRuntimeMetrics(patch, 'RUNNING')).to.equal(patch) + expect(patch).to.include({ cpuUsage: 12.5, memoryUsage: 64, startTime: 9, operatingDuration: 10 }) + }) + + it('zeros meters for any non-RUNNING status', () => { + const patch = { + status: 'STOPPED', + cpuUsage: 12.5, + memoryUsage: 64, + startTime: 9, + operatingDuration: 10, + containerId: 'ctr-1', + errorMessage: 'exit 1', + podId: 'pod-1', + healthStatus: 'healthy' + } + applyRuntimeMetrics(patch, 'STOPPED') + expect(patch).to.include({ + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + containerId: 'ctr-1', + errorMessage: 'exit 1', + podId: 'pod-1', + healthStatus: 'healthy' + }) + }) + }) + + describe('projectStatusForApi()', () => { + it('zeros meters on a non-RUNNING row without dropping identity fields', () => { + const projected = projectStatusForApi({ + status: 'FAILED', + cpuUsage: 22, + memoryUsage: 100, + startTime: 1, + operatingDuration: 2, + containerId: 'ctr-keep', + errorMessage: 'boom', + podId: 'pod-keep', + healthStatus: 'unhealthy' + }) + expect(projected).to.include({ + status: 'FAILED', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + containerId: 'ctr-keep', + errorMessage: 'boom', + podId: 'pod-keep', + healthStatus: 'unhealthy' + }) + }) + + it('does not mutate the original row', () => { + const row = { status: 'STOPPED', cpuUsage: 9, containerId: 'ctr-1' } + const projected = projectStatusForApi(row) + expect(projected.cpuUsage).to.equal(0) + expect(row.cpuUsage).to.equal(9) + }) + + it('clones Sequelize-style rows via get({ plain: true })', () => { + const row = { + get (options) { + expect(options).to.eql({ plain: true }) + return { status: 'STOPPING', cpuUsage: 4, memoryUsage: 8, containerId: 'ctr-2' } + } + } + expect(projectStatusForApi(row)).to.include({ + status: 'STOPPING', + cpuUsage: 0, + memoryUsage: 0, + containerId: 'ctr-2' + }) + }) + + it('always returns lastError, lastErrorAt, and restartCount', () => { + const projected = projectStatusForApi({ + status: 'RUNNING', + cpuUsage: 1, + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123, + restartCount: 4 + }) + expect(projected).to.include({ + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123, + restartCount: 4 + }) + }) + + it('fills empty last crash fields when the row omits them', () => { + expect(projectStatusForApi({ status: 'RUNNING', cpuUsage: 1 })).to.include({ + lastError: '', + lastErrorAt: 0, + restartCount: 0 + }) + }) + }) + + describe('applyCrashExtras()', () => { + it('persists lastError, lastErrorAt, and restartCount when present', () => { + const patch = { status: 'RUNNING' } + applyCrashExtras(patch, { + lastError: 'exitCode=1 oomKilled=false error=config missing', + lastErrorAt: 1726660000123, + restartCount: 4 + }) + expect(patch).to.include({ + lastError: 'exitCode=1 oomKilled=false error=config missing', + lastErrorAt: 1726660000123, + restartCount: 4 + }) + }) + + it('writes restartCount 0 when lastError is present and restartCount is omitted', () => { + const patch = { status: 'STARTING' } + applyCrashExtras(patch, { + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123 + }) + expect(patch.lastError).to.equal('exitCode=1 oomKilled=false') + expect(patch.lastErrorAt).to.equal(1726660000123) + expect(patch.restartCount).to.equal(0) + }) + + it('leaves last crash fields unset when all extras are omitted', () => { + const patch = { status: 'RUNNING', cpuUsage: 1 } + applyCrashExtras(patch, { status: 'RUNNING' }) + expect(patch).to.not.have.property('lastError') + expect(patch).to.not.have.property('lastErrorAt') + expect(patch).to.not.have.property('restartCount') + }) + + it('does not treat empty lastError as a wipe', () => { + const patch = { status: 'RUNNING' } + applyCrashExtras(patch, { lastError: '', lastErrorAt: 0 }) + expect(patch).to.not.have.property('lastError') + expect(patch).to.not.have.property('lastErrorAt') + expect(patch).to.not.have.property('restartCount') + }) + + it('ignores malformed extras instead of throwing', () => { + const patch = { status: 'RUNNING' } + applyCrashExtras(patch, { lastError: { nested: true }, lastErrorAt: 'nope', restartCount: 'n/a' }) + expect(patch).to.not.have.property('lastError') + expect(patch).to.not.have.property('lastErrorAt') + expect(patch).to.not.have.property('restartCount') + }) + }) +}) diff --git a/test/src/helpers/transaction-grep-gates.test.js b/test/src/helpers/transaction-grep-gates.test.js index 05ba022ed..d8f13ed7a 100644 --- a/test/src/helpers/transaction-grep-gates.test.js +++ b/test/src/helpers/transaction-grep-gates.test.js @@ -29,6 +29,21 @@ function grepSrc (pattern, extraArgs = [], searchPath = 'src/') { } } +function grepPaths (pattern, extraArgs, searchPaths) { + const hits = [] + for (const searchPath of searchPaths) { + const resolved = path.join(REPO_ROOT, searchPath) + if (!fs.existsSync(resolved)) { + continue + } + const match = grepSrc(pattern, extraArgs, searchPath) + if (match) { + hits.push(match) + } + } + return hits.join('\n') +} + /** * Plan 19-I-C: K8s I/O must run outside runInTransaction callback bodies. * Allowed external helpers (called after tx commit / between phases): @@ -167,6 +182,17 @@ describe('grep gates', () => { expect(volumeMountSource).to.not.match(/removeVolumeMount\(volumeMount\.uuid, transaction\)/) }) + it('passes Sequelize transaction inside options for model association calls', () => { + const modelSource = fs.readFileSync( + path.join(REPO_ROOT, 'src/services/model-service.js'), + 'utf8' + ) + expect(modelSource).to.match(/getFogs\(\{ transaction \}\)/) + expect(modelSource).to.match(/addModel\(model, \{ transaction \}\)/) + expect(modelSource).to.match(/removeModel\(model, \{ transaction \}\)/) + expect(modelSource).to.match(/hasModel\(model, \{ transaction \}\)/) + }) + it('keeps vault HTTP out of secret/configmap/registry transaction bodies', () => { const secretSource = fs.readFileSync( path.join(REPO_ROOT, 'src/services/secret-service.js'), @@ -261,6 +287,46 @@ describe('grep gates', () => { expect(grepSrc('ChangeTrackingService', [], 'src/services/registry-service.js')).to.equal('') }) + it('has zero plan or RFC rule tokens in application code, tests, swagger, migrations, and seeders', () => { + const codeHits = grepPaths( + 'R1[5-7][0-9]|plan.?23|BR-', + ['-E', '-i', '--include=*.js', '--exclude=transaction-grep-gates.test.js'], + ['src/', 'test/'] + ) + const sqlHits = grepPaths( + 'R1[5-7][0-9]|plan.?23|BR-', + ['-E', '-i', '--include=*.sql'], + ['src/data/migrations/', 'src/data/seeders/'] + ) + const swaggerHits = grepSrc( + 'R1[5-7][0-9]|plan.?23|BR-', + ['-E', '-i'], + 'docs/swagger.yaml' + ) + expect([codeHits, sqlHits, swaggerHits].filter(Boolean).join('\n')).to.equal('') + }) + + it('has zero hardware inventory and bluetooth catalog leftovers in application JS', () => { + const hits = grepSrc( + 'deviceScanFrequency|abstractedHardwareEnabled|bluetoothEnabled|getHalCatalogItem|getBluetoothCatalogItem|RESTBlue', + ['-E', '--exclude-dir=cli'] + ) + expect(hits).to.equal('') + }) + + it('does not register hardware inventory HTTP routes', () => { + expect(grepSrc('/hal/', [], 'src/routes/')).to.equal('') + const yamlHits = grepSrc('/agent/hal/|/iofog/.*/hal/', [ + '-E', + '--include=*.yaml' + ], 'src/config') + expect(yamlHits).to.equal('') + }) + + it('keeps Edge Guard frequency on fog config', () => { + expect(grepSrc('edgeGuardFrequency')).to.not.equal('') + }) + it('keeps MicroserviceManager.findAllWithStatuses fan-out out of catalog/registry propagation paths', () => { const catalogSource = fs.readFileSync( path.join(REPO_ROOT, 'src/services/catalog-service.js'), diff --git a/test/src/helpers/volume-mapping-scope.test.js b/test/src/helpers/volume-mapping-scope.test.js new file mode 100644 index 000000000..dedadb899 --- /dev/null +++ b/test/src/helpers/volume-mapping-scope.test.js @@ -0,0 +1,119 @@ +'use strict' + +const { expect } = require('chai') + +const { + applyVolumeMappingScope, + isValidBindHostDestination +} = require('../../../src/helpers/volume-mapping-scope') +const Errors = require('../../../src/helpers/errors') + +function volumeMapping (overrides = {}) { + return { + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + ...overrides + } +} + +describe('volume mapping scope helper', () => { + describe('type volume', () => { + it('stores private when scope is omitted', () => { + const mapping = volumeMapping() + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('private') + }) + + it('stores private when scope is empty, null, or whitespace', () => { + for (const scope of ['', null, ' ']) { + const mapping = volumeMapping({ scope }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('private') + } + }) + + it('stores lowercase shared when scope is Shared', () => { + const mapping = volumeMapping({ scope: 'Shared' }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('shared') + }) + + it('rejects unknown scope values', () => { + expect(() => applyVolumeMappingScope(volumeMapping({ scope: 'Shareed' }))) + .to.throw(Errors.ValidationError, /Unknown volume mapping scope/) + }) + + it('accepts nodered-config as a volume name', () => { + const mapping = volumeMapping({ hostDestination: 'nodered-config' }) + applyVolumeMappingScope(mapping) + expect(mapping.hostDestination).to.equal('nodered-config') + expect(mapping.scope).to.equal('private') + }) + + it('rejects volume names that start with a dot, underscore, or path traversal', () => { + for (const hostDestination of ['../x', '.hidden', '_scratch']) { + expect(() => applyVolumeMappingScope(volumeMapping({ hostDestination }))) + .to.throw(Errors.InvalidArgumentError, /invalid characters/) + } + }) + + it('persists shared with read-only access', () => { + const mapping = volumeMapping({ scope: 'shared', accessMode: 'ro' }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('shared') + expect(mapping.accessMode).to.equal('ro') + }) + + it('rejects explicit shared when the microservice is controller or system', () => { + expect(() => applyVolumeMappingScope(volumeMapping({ scope: 'shared' }), { rejectShared: true })) + .to.throw(Errors.ValidationError, /not allowed on controller or system/) + }) + + it('stores private on controller or system when shared is not sent', () => { + const mapping = volumeMapping() + applyVolumeMappingScope(mapping, { rejectShared: true }) + expect(mapping.scope).to.equal('private') + }) + }) + + describe('type bind', () => { + it('accepts absolute unix and windows host paths', () => { + for (const hostDestination of ['/var/lib/data', 'C:\\data\\app']) { + expect(isValidBindHostDestination(hostDestination)).to.equal(true) + const mapping = volumeMapping({ + type: 'bind', + hostDestination, + scope: 'shared' + }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('private') + } + }) + + it('rejects relative host paths and volume-style names on bind', () => { + for (const hostDestination of ['relative/path', 'nodered-config', '../x']) { + expect(isValidBindHostDestination(hostDestination)).to.equal(false) + expect(() => applyVolumeMappingScope(volumeMapping({ type: 'bind', hostDestination }))) + .to.throw(Errors.InvalidArgumentError, /absolute host path/) + } + }) + }) + + describe('non-volume types', () => { + it('stores private when type is omitted and scope is shared', () => { + const mapping = volumeMapping({ type: undefined, scope: 'shared' }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('private') + }) + + it('stores private and does not reject shared on volumeMount or serviceAccount', () => { + for (const type of ['volumeMount', 'serviceAccount']) { + const mapping = volumeMapping({ type, scope: 'shared' }) + applyVolumeMappingScope(mapping) + expect(mapping.scope).to.equal('private') + } + }) + }) +}) diff --git a/test/src/integration/first-fog-reconcile-sqlite.test.js b/test/src/integration/first-fog-reconcile-sqlite.test.js index 96fba5811..fe851bc6a 100644 --- a/test/src/integration/first-fog-reconcile-sqlite.test.js +++ b/test/src/integration/first-fog-reconcile-sqlite.test.js @@ -57,9 +57,7 @@ describe('first-fog reconcile sqlite (R133)', function () { name: 'hub-edge', host: '127.0.0.1', archId: 1, - containerEngine: 'edgelet', - bluetoothEnabled: false, - abstractedHardwareEnabled: false + containerEngine: 'edgelet' } let fogUuid diff --git a/test/src/integration/fleet-models-runtime-sqlite.test.js b/test/src/integration/fleet-models-runtime-sqlite.test.js new file mode 100644 index 000000000..951a2fc2a --- /dev/null +++ b/test/src/integration/fleet-models-runtime-sqlite.test.js @@ -0,0 +1,458 @@ +'use strict' + +/** + * Fleet models, RuntimeClass, and registry gates — sqlite integration paths. + * + * Skipped in default `npm test` unless RUN_INTEGRATION=1 (full DB migrate + init). + * + * RUN_INTEGRATION=1 npm run test:integration:fleet-models + */ + +describe('fleet models and runtime sqlite integration', function () { + this.timeout(60000) + + let harness + let runTx + let systemFog + let testCounter = 0 + + function uniqueSuffix () { + testCounter += 1 + return `${Date.now()}-${testCounter}` + } + + before(function () { + if (process.env.RUN_INTEGRATION !== '1') { + this.skip() + } + }) + + before(async function () { + const { + createFirstFogSqliteHarness, + driveReconcileUntilReady + } = require('../../support/first-fog-sqlite-harness') + + harness = await createFirstFogSqliteHarness() + + const { runInTransaction, PRIORITY_INTERACTIVE } = require('../../../src/helpers/transaction-runner') + runTx = (fn, label) => runInTransaction(fn, { + priority: PRIORITY_INTERACTIVE, + label: label || 'integration.fleetModelsRuntime' + }) + + systemFog = await createSystemFog('shared') + + const FogPlatformStatusManager = require('../../../src/data/managers/fog-platform-status-manager') + const { drainOnce } = require('../../../src/jobs/reconcile-outbox-drainer-job') + const PlatformReconcileWorkerJob = require('../../../src/jobs/platform-reconcile-worker-job') + await driveReconcileUntilReady(systemFog.uuid, { + drainOnce, + processNextFogTask: PlatformReconcileWorkerJob.processNextFogTask, + getStatus: (uuid) => FogPlatformStatusManager.getParsedStatus(uuid) + }) + }) + + after(async function () { + if (harness) { + await harness.teardown() + } + }) + + async function createSystemFog (suffix) { + return runTx( + (transaction) => require('../../../src/services/iofog-service').createFogEndPoint({ + name: `hub-fleet-${suffix}`, + host: '127.0.0.1', + archId: 1, + containerEngine: 'edgelet' + }, false, transaction), + 'integration.createSystemFog' + ) + } + + async function createEdgeFog (suffix) { + return runTx( + (transaction) => require('../../../src/services/iofog-service').createFogEndPoint({ + name: `edge-fleet-${suffix}`, + host: '127.0.0.1', + archId: 1, + containerEngine: 'edgelet', + routerMode: 'edge', + natsMode: 'leaf', + upstreamRouters: [systemFog.uuid] + }, false, transaction), + 'integration.createEdgeFog' + ) + } + + async function createDockerFog (suffix) { + return runTx( + (transaction) => require('../../../src/services/iofog-service').createFogEndPoint({ + name: `docker-fleet-${suffix}`, + host: '127.0.0.1', + archId: 1, + containerEngine: 'docker', + routerMode: 'edge', + natsMode: 'leaf', + upstreamRouters: [systemFog.uuid] + }, false, transaction), + 'integration.createDockerFog' + ) + } + + async function loadFog (uuid) { + const FogManager = require('../../../src/data/managers/iofog-manager') + return runTx((transaction) => FogManager.findOne({ uuid }, transaction), 'integration.loadFog') + } + + it('returns linked models on agent GET after model create and fog link', async function () { + const { expect } = require('chai') + const ModelService = require('../../../src/services/model-service') + const AgentService = require('../../../src/services/agent-service') + + const suffix = uniqueSuffix() + const modelName = `test-model-${suffix}` + const edgeFog = await createEdgeFog(suffix) + + await runTx( + (transaction) => ModelService.createModelEndpoint({ + name: modelName, + repo: 'org/repo', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + }, transaction), + 'integration.createModel' + ) + + await runTx( + (transaction) => ModelService.linkModelEndpoint(modelName, [edgeFog.uuid], transaction), + 'integration.linkModel' + ) + + const fog = await loadFog(edgeFog.uuid) + const models = await runTx( + (transaction) => AgentService.getAgentLinkedModels(fog, transaction), + 'integration.agentModels' + ) + + expect(models).to.have.length(1) + expect(models[0]).to.include({ + name: modelName, + repo: 'org/repo', + registryId: 3, + format: 'gguf' + }) + expect(models[0].files).to.eql(['file.gguf']) + }) + + it('returns linked runtime class on agent GET after link on edgelet fog', async function () { + const { expect } = require('chai') + const RuntimeClassService = require('../../../src/services/runtime-class-service') + const AgentService = require('../../../src/services/agent-service') + const FogManager = require('../../../src/data/managers/iofog-manager') + + const suffix = uniqueSuffix() + const edgeFog = await createEdgeFog(suffix) + const className = `spin-${suffix}` + + await runTx( + (transaction) => RuntimeClassService.createRuntimeClassEndpoint({ + name: className, + handler: 'spin' + }, transaction), + 'integration.createRuntimeClass' + ) + + await runTx( + (transaction) => FogManager.update( + { uuid: edgeFog.uuid }, + { availableRuntimes: JSON.stringify([className]) }, + transaction + ), + 'integration.setAvailableRuntimesForLink' + ) + + await runTx( + (transaction) => RuntimeClassService.linkRuntimeClassEndpoint(className, [edgeFog.uuid], transaction), + 'integration.linkRuntimeClass' + ) + + const fog = await loadFog(edgeFog.uuid) + const runtimeClasses = await runTx( + (transaction) => AgentService.getAgentLinkedRuntimeClasses(fog, transaction), + 'integration.agentRuntimeClasses' + ) + + expect(runtimeClasses).to.eql([{ name: className, handler: 'spin' }]) + }) + + it('refuses runtime class link on docker fog with validation error', async function () { + const { expect } = require('chai') + const RuntimeClassService = require('../../../src/services/runtime-class-service') + const Errors = require('../../../src/helpers/errors') + + const suffix = uniqueSuffix() + const dockerFog = await createDockerFog(suffix) + const className = `spin-docker-${suffix}` + + await runTx( + (transaction) => RuntimeClassService.createRuntimeClassEndpoint({ + name: className, + handler: 'spin' + }, transaction), + 'integration.createRuntimeClassDocker' + ) + + await expect(runTx( + (transaction) => RuntimeClassService.linkRuntimeClassEndpoint(className, [dockerFog.uuid], transaction), + 'integration.linkRuntimeClassDocker' + )).to.be.rejectedWith(Errors.ValidationError, /edgelet/) + }) + + it('sets microserviceModels change flag and rebuild on catalog PATCH', async function () { + const { expect } = require('chai') + const ApplicationService = require('../../../src/services/application-service') + const MicroserviceService = require('../../../src/services/microservices-service') + const ModelService = require('../../../src/services/model-service') + const AgentService = require('../../../src/services/agent-service') + const MicroserviceManager = require('../../../src/data/managers/microservice-manager') + + const suffix = uniqueSuffix() + const edgeFog = await createEdgeFog(suffix) + const modelName = `catalog-model-${suffix}` + const appName = `fleet-catalog-app-${suffix}` + + await runTx( + (transaction) => ModelService.createModelEndpoint({ + name: modelName, + repo: 'org/repo', + registryId: 3, + files: ['weights.gguf'] + }, transaction), + 'integration.createCatalogModel' + ) + + await runTx( + (transaction) => ApplicationService.createApplicationEndPoint({ + name: appName, + description: 'catalog patch integration' + }, false, transaction), + 'integration.createCatalogApp' + ) + + const microservice = await runTx( + (transaction) => MicroserviceService.createMicroserviceEndPoint({ + name: `catalog-ms-${suffix}`, + application: appName, + iofogUuid: edgeFog.uuid, + images: [{ containerImage: 'demo:latest', archId: 1 }], + registryId: 1 + }, false, transaction), + 'integration.createCatalogMs' + ) + + await runTx( + (transaction) => MicroserviceService.updateMicroserviceCatalogEndPoint(microservice.uuid, { + bindPath: '/models', + permissions: 'ro', + items: [{ name: modelName }] + }, false, transaction), + 'integration.patchCatalog' + ) + + const updated = await runTx( + (transaction) => MicroserviceManager.findOne({ uuid: microservice.uuid }, transaction), + 'integration.loadCatalogMs' + ) + expect(updated.rebuild).to.equal(true) + + const MicroserviceModelManager = require('../../../src/data/managers/microservice-model-manager') + const MicroserviceModelItemManager = require('../../../src/data/managers/microservice-model-item-manager') + const catalogRow = await runTx( + (transaction) => MicroserviceModelManager.findOne({ microserviceUuid: microservice.uuid }, transaction), + 'integration.loadCatalogRow' + ) + const catalogItems = await runTx( + (transaction) => MicroserviceModelItemManager.findAll({ microserviceUuid: microservice.uuid }, transaction), + 'integration.loadCatalogItems' + ) + expect(catalogRow.bindPath).to.equal('/models') + expect(catalogItems.map((item) => item.name)).to.eql([modelName]) + + const fog = await loadFog(edgeFog.uuid) + const changes = await runTx( + (transaction) => AgentService.getAgentConfigChanges(fog, transaction), + 'integration.catalogChanges' + ) + expect(changes.microserviceList).to.equal(true) + }) + + it('auto-links model and runtime class rows when microservice deploy references them', async function () { + const { expect } = require('chai') + const ApplicationService = require('../../../src/services/application-service') + const MicroserviceService = require('../../../src/services/microservices-service') + const ModelService = require('../../../src/services/model-service') + const RuntimeClassService = require('../../../src/services/runtime-class-service') + const FogManager = require('../../../src/data/managers/iofog-manager') + + const suffix = uniqueSuffix() + const className = `spin-${suffix}` + const modelName = `deploy-model-${suffix}` + const appName = `fleet-deploy-app-${suffix}` + const edgeFog = await createEdgeFog(suffix) + + await runTx( + (transaction) => FogManager.update( + { uuid: edgeFog.uuid }, + { availableRuntimes: JSON.stringify([className]) }, + transaction + ), + 'integration.setAvailableRuntimes' + ) + + await runTx( + (transaction) => ModelService.createModelEndpoint({ + name: modelName, + repo: 'org/repo', + registryId: 3, + files: ['weights.gguf'] + }, transaction), + 'integration.createDeployModel' + ) + + await runTx( + (transaction) => RuntimeClassService.createRuntimeClassEndpoint({ + name: className, + handler: 'spin' + }, transaction), + 'integration.createDeployRuntimeClass' + ) + + await runTx( + (transaction) => ApplicationService.createApplicationEndPoint({ + name: appName, + description: 'deploy auto-link integration' + }, false, transaction), + 'integration.createDeployApp' + ) + + await runTx( + (transaction) => MicroserviceService.createMicroserviceEndPoint({ + name: `deploy-ms-${suffix}`, + application: appName, + iofogUuid: edgeFog.uuid, + images: [{ containerImage: 'demo:latest', archId: 1 }], + registryId: 1, + runtime: className, + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: modelName }] + } + }, false, transaction), + 'integration.createDeployMs' + ) + + const fog = await loadFog(edgeFog.uuid) + const linkedModels = await runTx( + (transaction) => fog.getModels({ transaction }), + 'integration.loadModelLinks' + ) + const linkedRuntimeClasses = await runTx( + (transaction) => fog.getRuntimeClassLinks({ transaction }), + 'integration.loadRuntimeLinks' + ) + + expect(linkedModels).to.have.length(1) + expect(linkedModels[0].name).to.equal(modelName) + expect(linkedRuntimeClasses).to.have.length(1) + expect(linkedRuntimeClasses[0].name).to.equal(className) + }) + + it('rejects Hugging Face registry for microservice images', async function () { + const { expect } = require('chai') + const ApplicationService = require('../../../src/services/application-service') + const MicroserviceService = require('../../../src/services/microservices-service') + const AppHelper = require('../../../src/helpers/app-helper') + const ErrorMessages = require('../../../src/helpers/error-messages') + + const suffix = uniqueSuffix() + const appName = `fleet-registry-app-${suffix}` + const edgeFog = await createEdgeFog(suffix) + + await runTx( + (transaction) => ApplicationService.createApplicationEndPoint({ + name: appName, + description: 'registry gate integration' + }, false, transaction), + 'integration.createRegistryApp' + ) + + await expect(runTx( + (transaction) => MicroserviceService.createMicroserviceEndPoint({ + name: `registry-ms-${suffix}`, + application: appName, + iofogUuid: edgeFog.uuid, + images: [{ containerImage: 'demo:latest', archId: 1 }], + registryId: 3 + }, false, transaction), + 'integration.createRegistryMs' + )).to.be.rejectedWith(AppHelper.formatMessage(ErrorMessages.REGISTRY_NOT_OCI_FOR_IMAGE, 3)) + }) + + it('persists modelStatus from agent PUT and returns parseable JSON on user GET', async function () { + const { expect } = require('chai') + const AgentService = require('../../../src/services/agent-service') + const IofogService = require('../../../src/services/iofog-service') + + const modelStatusPayload = [{ name: 'test-model', state: 'ready' }] + const modelStatusJson = JSON.stringify(modelStatusPayload) + + await runTx( + (transaction) => AgentService.updateAgentStatus({ + daemonStatus: 'RUNNING', + warningMessage: '', + daemonOperatingDuration: 1, + daemonLastStart: 1, + memoryUsage: 1, + diskUsage: 1, + cpuUsage: 1, + memoryViolation: false, + diskViolation: false, + cpuViolation: false, + systemAvailableDisk: 1, + systemAvailableMemory: 1, + systemTotalCpu: 1, + repositoryCount: 0, + repositoryStatus: '[]', + systemTime: 1, + lastStatusTime: 1, + ipAddress: '127.0.0.1', + ipAddressExternal: '127.0.0.1', + lastCommandTime: 1, + tunnelStatus: '{}', + version: '1.1.0', + isReadyToUpgrade: false, + isReadyToRollback: false, + gpsStatus: 'OK', + microserviceStatus: '[]', + modelStatus: modelStatusJson, + activeModels: 1, + modelLastUpdate: 1710000000 + }, { uuid: systemFog.uuid }, transaction), + 'integration.agentStatusModel' + ) + + const fog = await runTx( + (transaction) => IofogService.getFogEndPoint({ uuid: systemFog.uuid }, false, transaction), + 'integration.userGetFog' + ) + + expect(fog.modelStatus).to.equal(modelStatusJson) + expect(fog.activeModels).to.equal(1) + expect(fog.modelLastUpdate).to.equal(1710000000) + expect(JSON.parse(fog.modelStatus)).to.eql(modelStatusPayload) + }) +}) diff --git a/test/src/jobs/fog-status-job.test.js b/test/src/jobs/fog-status-job.test.js new file mode 100644 index 000000000..76139cf36 --- /dev/null +++ b/test/src/jobs/fog-status-job.test.js @@ -0,0 +1,203 @@ +const { expect } = require('chai') +const fs = require('fs') +const path = require('path') +const sinon = require('sinon') + +const FogManager = require('../../../src/data/managers/iofog-manager') +const ApplicationManager = require('../../../src/data/managers/application-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const MicroserviceStatusManager = require('../../../src/data/managers/microservice-status-manager') +const MicroserviceExecStatusManager = require('../../../src/data/managers/microservice-exec-status-manager') +const FogStatusJob = require('../../../src/jobs/fog-status-job') +const FogStates = require('../../../src/enums/fog-state') + +const JOB_SOURCE = fs.readFileSync(path.join(__dirname, '../../../src/jobs/fog-status-job.js'), 'utf8') + +describe('fog-status-job', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + + afterEach(() => $sandbox.restore()) + + describe('clampedStatusFrequencySec()', () => { + it('uses a 10s floor when frequency is 0', () => { + expect(FogStatusJob.clampedStatusFrequencySec(0)).to.equal(10) + }) + + it('keeps a frequency above the floor', () => { + expect(FogStatusJob.clampedStatusFrequencySec(30)).to.equal(30) + }) + }) + + describe('isFogStatusStale()', () => { + it('does not treat frequency 0 as an immediate miss', () => { + const nowMs = 1_000_000 + const fog = { lastStatusTime: nowMs - 15_000, statusFrequency: 0 } + expect(FogStatusJob.isFogStatusStale(fog, 3, nowMs)).to.equal(false) + }) + + it('marks a fog stale after the clamped deadline', () => { + const nowMs = 1_000_000 + const fog = { lastStatusTime: nowMs - 31_000, statusFrequency: 0 } + expect(FogStatusJob.isFogStatusStale(fog, 3, nowMs)).to.equal(true) + }) + + it('keeps a just-heard fog live even when an agent clock of 0 would miss the deadline', () => { + const nowMs = 1_700_000_000_000 + expect(FogStatusJob.isFogStatusStale({ lastStatusTime: nowMs, statusFrequency: 10 }, 3, nowMs)).to.equal(false) + expect(FogStatusJob.isFogStatusStale({ lastStatusTime: 0, statusFrequency: 10 }, 3, nowMs)).to.equal(true) + }) + }) + + describe('shouldMarkUnknownOnQuietFog()', () => { + it('skips desired-inactive STOPPED microservices', () => { + expect(FogStatusJob.shouldMarkUnknownOnQuietFog({ + isActivated: false, + microserviceStatus: { id: 1, status: 'STOPPED' } + }, { isActivated: true })).to.equal(false) + }) + + it('skips microservices on a deactivated application', () => { + expect(FogStatusJob.shouldMarkUnknownOnQuietFog({ + isActivated: true, + microserviceStatus: { id: 1, status: 'RUNNING' } + }, { isActivated: false })).to.equal(false) + }) + + it('marks desired-active workloads unknown', () => { + expect(FogStatusJob.shouldMarkUnknownOnQuietFog({ + isActivated: true, + microserviceStatus: { id: 1, status: 'RUNNING' } + }, { isActivated: true })).to.equal(true) + }) + }) + + describe('processLivenessChunk()', () => { + beforeEach(() => { + $sandbox.stub(FogManager, 'findStatusLivenessCandidates') + $sandbox.stub(FogManager, 'update') + $sandbox.stub(MicroserviceManager, 'findAllWithStatuses') + $sandbox.stub(ApplicationManager, 'findAll') + $sandbox.stub(MicroserviceStatusManager, 'update') + $sandbox.stub(MicroserviceExecStatusManager, 'update') + }) + + it('does not update when no quiet fogs are found', async () => { + FogManager.findStatusLivenessCandidates.resolves([]) + + const result = await FogStatusJob.processLivenessChunk(null, 50, 3, transaction) + + expect(result).to.eql({ nextCursor: null, staleCount: 0 }) + expect(FogManager.update).to.not.have.been.called + expect(MicroserviceManager.findAllWithStatuses).to.not.have.been.called + }) + + it('does not update when candidates are still within their deadline', async () => { + const nowMs = Date.now() + FogManager.findStatusLivenessCandidates.resolves([{ + uuid: 'fog-fresh', + statusFrequency: 0, + lastStatusTime: nowMs - 1_000, + daemonStatus: FogStates.RUNNING + }]) + + const result = await FogStatusJob.processLivenessChunk(null, 50, 3, transaction) + + expect(result.nextCursor).to.equal('fog-fresh') + expect(result.staleCount).to.equal(0) + expect(FogManager.update).to.not.have.been.called + expect(MicroserviceManager.findAllWithStatuses).to.not.have.been.called + }) + + it('marks a quiet desired-active microservice UNKNOWN with zero meters', async () => { + const nowMs = Date.now() + FogManager.findStatusLivenessCandidates.resolves([{ + uuid: 'fog-quiet', + statusFrequency: 10, + lastStatusTime: nowMs - 60_000, + daemonStatus: FogStates.RUNNING + }]) + MicroserviceManager.findAllWithStatuses.resolves([{ + applicationId: 7, + isActivated: true, + microserviceStatus: { id: 11, status: 'RUNNING' }, + microserviceExecStatus: { id: 21 } + }]) + ApplicationManager.findAll.resolves([{ id: 7, isActivated: true }]) + FogManager.update.resolves() + MicroserviceStatusManager.update.resolves() + MicroserviceExecStatusManager.update.resolves() + + const result = await FogStatusJob.processLivenessChunk(null, 50, 3, transaction) + + expect(result.staleCount).to.equal(1) + expect(FogManager.update).to.have.been.calledWith( + { uuid: ['fog-quiet'] }, + { daemonStatus: FogStates.UNKNOWN }, + transaction + ) + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { id: [11] }, + sinon.match({ + status: 'UNKNOWN', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { id: [21] }, + { execSessionId: '', status: 'INACTIVE' }, + transaction + ) + }) + + it('does not mark desired-inactive STOPPED microservices UNKNOWN', async () => { + const nowMs = Date.now() + FogManager.findStatusLivenessCandidates.resolves([{ + uuid: 'fog-quiet', + statusFrequency: 10, + lastStatusTime: nowMs - 60_000, + daemonStatus: FogStates.WARNING + }]) + MicroserviceManager.findAllWithStatuses.resolves([{ + applicationId: 7, + isActivated: false, + microserviceStatus: { id: 11, status: 'STOPPED' }, + microserviceExecStatus: { id: 21 } + }]) + ApplicationManager.findAll.resolves([{ id: 7, isActivated: true }]) + FogManager.update.resolves() + + await FogStatusJob.processLivenessChunk(null, 50, 3, transaction) + + expect(FogManager.update).to.have.been.calledOnce + expect(MicroserviceStatusManager.update).to.not.have.been.called + expect(MicroserviceExecStatusManager.update).to.not.have.been.called + }) + }) + + describe('run()', () => { + it('runs inner work once when a second pass overlaps', async () => { + $sandbox.useFakeTimers() + let release + const blocked = new Promise((resolve) => { release = resolve }) + $sandbox.stub(FogStatusJob, 'runLivenessPass').callsFake(() => blocked) + + const first = FogStatusJob.run() + await Promise.resolve() + await FogStatusJob.run() + expect(FogStatusJob.runLivenessPass).to.have.been.calledOnce + + release() + await first + }) + }) + + it('does not delete microservices from the liveness job', () => { + expect(JOB_SOURCE).to.not.include('deleteMicroserviceWithRoutesAndPortMappings') + expect(JOB_SOURCE).to.not.include('deleteNotRunningMicroservices') + }) +}) diff --git a/test/src/jobs/stopped-app-status-job.test.js b/test/src/jobs/stopped-app-status-job.test.js new file mode 100644 index 000000000..1404ac763 --- /dev/null +++ b/test/src/jobs/stopped-app-status-job.test.js @@ -0,0 +1,122 @@ +const { expect } = require('chai') +const sinon = require('sinon') + +const ApplicationManager = require('../../../src/data/managers/application-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const MicroserviceStatusManager = require('../../../src/data/managers/microservice-status-manager') +const MicroserviceExecStatusManager = require('../../../src/data/managers/microservice-exec-status-manager') +const StoppedAppStatusJob = require('../../../src/jobs/stopped-app-status-job') + +describe('stopped-app-status-job', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + + afterEach(() => $sandbox.restore()) + + describe('shouldForceObservedStopped()', () => { + it('matches leftover observed statuses on desired-inactive workloads', () => { + expect(StoppedAppStatusJob.shouldForceObservedStopped({ + microserviceStatus: { status: 'RUNNING' } + })).to.equal(true) + expect(StoppedAppStatusJob.shouldForceObservedStopped({ + microserviceStatus: { status: 'STOPPING' } + })).to.equal(true) + expect(StoppedAppStatusJob.shouldForceObservedStopped({ + microserviceStatus: { status: 'DELETED' } + })).to.equal(true) + expect(StoppedAppStatusJob.shouldForceObservedStopped({ + microserviceStatus: { status: 'DELETING' } + })).to.equal(true) + }) + + it('leaves already STOPPED rows alone', () => { + expect(StoppedAppStatusJob.shouldForceObservedStopped({ + microserviceStatus: { status: 'STOPPED' } + })).to.equal(false) + }) + }) + + describe('updateMicroserviceStatusStopped()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() + }) + + it('does not update when there are no leftover statuses', async () => { + await StoppedAppStatusJob.updateMicroserviceStatusStopped([{ + microserviceStatus: { id: 1, status: 'STOPPED' }, + microserviceExecStatus: { id: 2 } + }], transaction) + + expect(MicroserviceStatusManager.update).to.not.have.been.called + expect(MicroserviceExecStatusManager.update).to.not.have.been.called + }) + + it('sets desired-inactive RUNNING to STOPPED with zero meters', async () => { + await StoppedAppStatusJob.updateMicroserviceStatusStopped([{ + isActivated: false, + microserviceStatus: { id: 11, status: 'RUNNING' }, + microserviceExecStatus: { id: 21 } + }], transaction) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { id: [11] }, + sinon.match({ + status: 'STOPPED', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { id: [21] }, + { execSessionId: '', status: 'INACTIVE' }, + transaction + ) + }) + }) + + describe('updateDeactivatedMicroservices()', () => { + it('skips status updates when no individually deactivated microservices exist', async () => { + $sandbox.stub(ApplicationManager, 'findAllWithAttributes').resolves([{ id: 3 }]) + $sandbox.stub(MicroserviceManager, 'findAllWithStatuses').resolves([]) + $sandbox.stub(MicroserviceStatusManager, 'update') + + await StoppedAppStatusJob.updateDeactivatedMicroservices(transaction) + + expect(MicroserviceStatusManager.update).to.not.have.been.called + }) + }) + + describe('updateApplicationMicroservices()', () => { + it('skips status updates when no deactivated applications exist', async () => { + $sandbox.stub(ApplicationManager, 'findAllWithAttributes').resolves([]) + $sandbox.stub(MicroserviceManager, 'findAllWithStatuses') + $sandbox.stub(MicroserviceStatusManager, 'update') + + await StoppedAppStatusJob.updateApplicationMicroservices(transaction) + + expect(MicroserviceManager.findAllWithStatuses).to.not.have.been.called + expect(MicroserviceStatusManager.update).to.not.have.been.called + }) + }) + + describe('run()', () => { + it('runs inner work once when a second pass overlaps', async () => { + $sandbox.useFakeTimers() + let release + const blocked = new Promise((resolve) => { release = resolve }) + $sandbox.stub(StoppedAppStatusJob, 'runSafetyNetPass').callsFake(() => blocked) + + const first = StoppedAppStatusJob.run() + await Promise.resolve() + await StoppedAppStatusJob.run() + expect(StoppedAppStatusJob.runSafetyNetPass).to.have.been.calledOnce + + release() + await first + }) + }) +}) diff --git a/test/src/lib/rbac/authorizer.test.js b/test/src/lib/rbac/authorizer.test.js index cda8343e9..276f63fa8 100644 --- a/test/src/lib/rbac/authorizer.test.js +++ b/test/src/lib/rbac/authorizer.test.js @@ -120,6 +120,30 @@ describe('RBAC authorizer fast path', () => { }) describe('authorize() full path', () => { + it('allows viewer get and list on knowledge and denies writes', async () => { + const getResult = await authorizer.authorize( + VIEWER_SUBJECTS, '', 'knowledge', 'get', null, $fakeTransaction + ) + const listResult = await authorizer.authorize( + VIEWER_SUBJECTS, '', 'knowledge', 'list', null, $fakeTransaction + ) + const createResult = await authorizer.authorize( + VIEWER_SUBJECTS, '', 'knowledge', 'create', null, $fakeTransaction + ) + const patchResult = await authorizer.authorize( + VIEWER_SUBJECTS, '', 'knowledge', 'patch', null, $fakeTransaction + ) + const deleteResult = await authorizer.authorize( + VIEWER_SUBJECTS, '', 'knowledge', 'delete', null, $fakeTransaction + ) + + expect(getResult.allowed).to.equal(true) + expect(listResult.allowed).to.equal(true) + expect(createResult.allowed).to.equal(false) + expect(patchResult.allowed).to.equal(false) + expect(deleteResult.allowed).to.equal(false) + }) + it('allows viewer system role for microservices get', async () => { const result = await authorizer.authorize( VIEWER_SUBJECTS, diff --git a/test/src/routes/agent-routes.test.js b/test/src/routes/agent-routes.test.js new file mode 100644 index 000000000..e164f5004 --- /dev/null +++ b/test/src/routes/agent-routes.test.js @@ -0,0 +1,17 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('Agent routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/agent.js'), 'utf8') + + it('registers fog-token model and runtime class list routes', () => { + expect(source).to.include("path: '/api/v3/agent/models'") + expect(source).to.include("path: '/api/v3/agent/runtimeClasses'") + expect(source).to.include("path: '/api/v3/agent/knowledge'") + }) + + it('does not register agent hardware inventory routes', () => { + expect(source).to.not.include('/agent/hal/') + }) +}) diff --git a/test/src/routes/iofog-routes.test.js b/test/src/routes/iofog-routes.test.js new file mode 100644 index 000000000..1979dd634 --- /dev/null +++ b/test/src/routes/iofog-routes.test.js @@ -0,0 +1,11 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('ioFog routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/iofog.js'), 'utf8') + + it('does not register hardware inventory user routes', () => { + expect(source).to.not.include('/hal/') + }) +}) diff --git a/test/src/routes/knowledge-routes.test.js b/test/src/routes/knowledge-routes.test.js new file mode 100644 index 000000000..4f58404d8 --- /dev/null +++ b/test/src/routes/knowledge-routes.test.js @@ -0,0 +1,32 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('Knowledge routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/knowledge.js'), 'utf8') + + it('registers user knowledge CRUD, YAML, and link routes', () => { + expect(source).to.include("path: '/api/v3/knowledge'") + expect(source).to.include("path: '/api/v3/knowledge/:name'") + expect(source).to.include("path: '/api/v3/knowledge/yaml'") + expect(source).to.include("path: '/api/v3/knowledge/yaml/:name'") + expect(source).to.include("path: '/api/v3/knowledge/:name/link'") + expect(source).to.include("fileInput: 'knowledge'") + expect(source).to.include("method: 'put'") + expect(source).to.include("method: 'patch'") + expect(source).to.include("method: 'delete'") + expect(source).to.include("method: 'post'") + expect(source).to.not.include('/api/v3/knowledges') + }) + + it('documents knowledge and model status clocks as Unix milliseconds', () => { + const swagger = fs.readFileSync(path.resolve(__dirname, '../../../docs/swagger.yaml'), 'utf8') + for (const field of ['knowledgeLastUpdate:', 'modelLastUpdate:']) { + const blocks = swagger.split(field).slice(1) + expect(blocks.length).to.be.at.least(1) + for (const block of blocks) { + expect(block.slice(0, 500)).to.match(/Unix milliseconds/) + } + } + }) +}) diff --git a/test/src/routes/microserviceTemplates-routes.test.js b/test/src/routes/microserviceTemplates-routes.test.js new file mode 100644 index 000000000..0cdcf11e5 --- /dev/null +++ b/test/src/routes/microserviceTemplates-routes.test.js @@ -0,0 +1,18 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('Microservice template routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/microserviceTemplates.js'), 'utf8') + + it('registers template CRUD and YAML routes', () => { + expect(source).to.include("path: '/api/v3/microserviceTemplates'") + expect(source).to.include("path: '/api/v3/microserviceTemplates/:name'") + expect(source).to.include("path: '/api/v3/microserviceTemplates/yaml'") + expect(source).to.include("path: '/api/v3/microserviceTemplates/yaml/:name'") + expect(source).to.include("method: 'put'") + expect(source).to.include("method: 'patch'") + expect(source).to.include("method: 'delete'") + expect(source).to.include("method: 'post'") + }) +}) diff --git a/test/src/routes/microservices-routes.test.js b/test/src/routes/microservices-routes.test.js new file mode 100644 index 000000000..be56eca8c --- /dev/null +++ b/test/src/routes/microservices-routes.test.js @@ -0,0 +1,15 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('Microservice routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/microservices.js'), 'utf8') + + it('registers catalog-only PATCH on a microservice', () => { + expect(source).to.include("path: '/api/v3/microservices/:uuid/models'") + expect(source).to.include('updateMicroserviceCatalogEndPoint') + expect(source).to.include("path: '/api/v3/microservices/:uuid/knowledge'") + expect(source).to.include('updateMicroserviceKnowledgeEndPoint') + expect(source).to.not.include("path: '/api/v3/microservices/system/:uuid/knowledge'") + }) +}) diff --git a/test/src/routes/models-routes.test.js b/test/src/routes/models-routes.test.js new file mode 100644 index 000000000..e3db8e482 --- /dev/null +++ b/test/src/routes/models-routes.test.js @@ -0,0 +1,19 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('Model routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/models.js'), 'utf8') + + it('registers user model CRUD, YAML, and link routes', () => { + expect(source).to.include("path: '/api/v3/models'") + expect(source).to.include("path: '/api/v3/models/:name'") + expect(source).to.include("path: '/api/v3/models/yaml'") + expect(source).to.include("path: '/api/v3/models/yaml/:name'") + expect(source).to.include("path: '/api/v3/models/:name/link'") + expect(source).to.include("method: 'put'") + expect(source).to.include("method: 'patch'") + expect(source).to.include("method: 'delete'") + expect(source).to.include("method: 'post'") + }) +}) diff --git a/test/src/routes/runtimeClasses-routes.test.js b/test/src/routes/runtimeClasses-routes.test.js new file mode 100644 index 000000000..1d793aac9 --- /dev/null +++ b/test/src/routes/runtimeClasses-routes.test.js @@ -0,0 +1,19 @@ +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +describe('RuntimeClass routes', () => { + const source = fs.readFileSync(path.resolve(__dirname, '../../../src/routes/runtimeClasses.js'), 'utf8') + + it('registers user runtime class CRUD, YAML, and link routes', () => { + expect(source).to.include("path: '/api/v3/runtimeClasses'") + expect(source).to.include("path: '/api/v3/runtimeClasses/:name'") + expect(source).to.include("path: '/api/v3/runtimeClasses/yaml'") + expect(source).to.include("path: '/api/v3/runtimeClasses/yaml/:name'") + expect(source).to.include("path: '/api/v3/runtimeClasses/:name/link'") + expect(source).to.include("method: 'put'") + expect(source).to.include("method: 'patch'") + expect(source).to.include("method: 'delete'") + expect(source).to.include("method: 'post'") + }) +}) diff --git a/test/src/services/agent-service.test.js b/test/src/services/agent-service.test.js index 8163d1108..9c9203553 100644 --- a/test/src/services/agent-service.test.js +++ b/test/src/services/agent-service.test.js @@ -18,8 +18,6 @@ const RegistryManager = require('../../../src/data/managers/registry-manager') const TunnelManager = require('../../../src/data/managers/tunnel-manager') const ioFogVersionCommandManager = require('../../../src/data/managers/iofog-version-command-manager') const ioFogProvisionKeyManager = require('../../../src/data/managers/iofog-provision-key-manager') -const HWInfoManager = require('../../../src/data/managers/hw-info-manager') -const USBInfoManager = require('../../../src/data/managers/usb-info-manager') const Sequelize = require('sequelize') const Op = Sequelize.Op const path = require('path') @@ -28,6 +26,7 @@ const FogStates = require('../../../src/enums/fog-state') const constants = require('../../../src/helpers/constants') const config = require('../../../src/config') const Errors = require('../../../src/helpers/errors') +const logger = require('../../../src/logger') global.appRoot = path.resolve(__dirname) @@ -450,7 +449,6 @@ describe('Agent Service', () => { logFileCount: agentConfig.logFileCount, statusFrequency: agentConfig.statusFrequency, changeFrequency: agentConfig.changeFrequency, - deviceScanFrequency: agentConfig.deviceScanFrequency, watchdogEnabled: agentConfig.watchdogEnabled, latitude: agentConfig.latitude, longitude: agentConfig.longitude, @@ -486,6 +484,7 @@ describe('Agent Service', () => { expect(ioFogManager.update).to.have.been.calledWith({ uuid: $uuid, }, expectedFogUpdate, transaction) + expect(expectedFogUpdate).to.not.have.property('deviceScanFrequency') }) context('when ioFogManager#update fails', () => { @@ -507,21 +506,51 @@ describe('Agent Service', () => { }) }) - describe('.getAgentConfigChanges()', () => { - const configChanges = { - config: undefined, - version: undefined, - reboot: undefined, - deleteNode: undefined, - microserviceList: undefined, - microserviceConfig: undefined, - routing: undefined, - registries: undefined, - tunnel: undefined, - routerChanged: undefined, - prune: undefined, + describe('.getAgentConfig()', () => { + const transaction = {} + const fogData = { + networkInterface: 'eth0', + containerEngineUrl: 'unix:///run/edgelet/containerd.sock', + diskLimit: 50, + diskDirectory: '/var/lib/iofog/', + memoryLimit: 4096, + cpuLimit: 80, + logLimit: 10, + logDirectory: '/var/log/iofog/', + logFileCount: 10, + gpsMode: 'auto', + gpsDevice: '', + gpsScanFrequency: 60, + edgeGuardFrequency: 0, + statusFrequency: 10, + changeFrequency: 20, + watchdogEnabled: false, + latitude: 0, + longitude: 0, + logLevel: 'INFO', + availableDiskThreshold: 20, + pruningFrequency: 0, + timeZone: 'UTC', + deviceScanFrequency: 50, } + def('uuid', () => 'testUuid') + def('fog', () => ({ uuid: $uuid })) + def('subject', () => $subject.getAgentConfig($fog, transaction)) + + beforeEach(() => { + $sandbox.stub(ioFogManager, 'findOne').resolves(fogData) + }) + + it('omits deviceScanFrequency from the agent config payload', async () => { + const result = await $subject + expect(result).to.not.have.property('deviceScanFrequency') + expect(result.statusFrequency).to.equal(fogData.statusFrequency) + expect(result.edgeGuardFrequency).to.equal(fogData.edgeGuardFrequency) + }) + }) + + describe('.getAgentConfigChanges()', () => { const transaction = {} const error = 'Error!' @@ -535,7 +564,7 @@ describe('Agent Service', () => { def('subject', () => $subject.getAgentConfigChanges($fog, transaction)) - def('getByFogIdResponse', () => 'getByFogIdResponse') + def('getByFogIdResponse', () => []) def('updateIfChangedResponse', () => Promise.resolve()) beforeEach(() => { @@ -548,6 +577,16 @@ describe('Agent Service', () => { expect(ChangeTrackingService.getByIoFogUuid).to.have.been.calledWith($uuid, transaction) }) + it('includes additive change flags as false by default', async () => { + const result = await $subject + expect(result.models).to.equal(false) + expect(result.runtimeClasses).to.equal(false) + expect(result.microserviceModels).to.equal(false) + expect(result.knowledge).to.equal(false) + expect(result.microserviceKnowledge).to.equal(false) + expect(result.prune).to.equal(false) + }) + context('when ChangeTrackingService#getByIoFogUuid() fails', () => { def('getByFogIdResponse', () => Promise.reject(error)) @@ -682,6 +721,7 @@ describe('Agent Service', () => { def('platformStatusResponse', () => Promise.resolve(null)) beforeEach(() => { + $sandbox.useFakeTimers({ now: fogStatus.lastStatusTime, toFake: ['Date'] }) $sandbox.stub(Validator, 'validate').returns($validatorResponse) $sandbox.spy(AppHelper, 'deleteUndefinedFields') $sandbox.stub(ioFogManager, 'findOne').returns($findOneResponse) @@ -691,6 +731,7 @@ describe('Agent Service', () => { $sandbox.stub(MicroserviceStatusManager, 'update').returns($updateMicroserviceStatusesResponse) $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').returns($deleteNotRunningResponse) $sandbox.stub(MicroserviceManager, 'findOne').returns($findMicroservice) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) }) it('calls Validator#validate() with correct args', async () => { @@ -843,10 +884,10 @@ describe('Agent Service', () => { const expectedMicroserviceUpdate = { containerId: microserviceStatus.containerId, status: microserviceStatus.status, - startTime: microserviceStatus.startTime, - operatingDuration: microserviceStatus.operatingDuration, - cpuUsage: microserviceStatus.cpuUsage, - memoryUsage: microserviceStatus.memoryUsage, + startTime: 0, + operatingDuration: 0, + cpuUsage: 0, + memoryUsage: 0, percentage: microserviceStatus.percentage, errorMessage: microserviceStatus.errorMessage, } @@ -945,6 +986,7 @@ describe('Agent Service', () => { def('platformStatusResponse', () => Promise.resolve(null)) beforeEach(() => { + $sandbox.useFakeTimers({ now: fogStatus.lastStatusTime, toFake: ['Date'] }) $sandbox.stub(Validator, 'validate').returns($validatorResponse) $sandbox.spy(AppHelper, 'deleteUndefinedFields') $sandbox.stub(ioFogManager, 'findOne').returns($findOneResponse) @@ -954,6 +996,7 @@ describe('Agent Service', () => { $sandbox.stub(MicroserviceStatusManager, 'update').returns($updateMicroserviceStatusesResponse) $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').returns($deleteNotRunningResponse) $sandbox.stub(MicroserviceManager, 'findOne').returns($findMicroservice) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) }) it('calls Validator#validate() with correct args', async () => { @@ -1119,6 +1162,7 @@ describe('Agent Service', () => { def('subject', () => AgentService.updateAgentStatus(baseAgentStatus, $fog, transaction)) beforeEach(() => { + $sandbox.stub(logger, 'warn') $sandbox.stub(Validator, 'validate').resolves(true) $sandbox.stub(ioFogManager, 'findOne').resolves({ warningMessage: $existingWarningMessage }) $sandbox.stub(FogPlatformStatusManager, 'getParsedStatus').resolves( @@ -1127,6 +1171,7 @@ describe('Agent Service', () => { $sandbox.stub(ioFogManager, 'update').resolves() $sandbox.stub(JSON, 'parse').returns([]) $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').resolves() + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) }) it('forces daemonStatus WARNING when platform phase is Progressing', async () => { @@ -1349,11 +1394,72 @@ describe('Agent Service', () => { expect(msvc.application).to.equal('testApp') expect(msvc.registryId).to.equal(microserviceResponse.microservices[0].registryId) expect(msvc.cmd).to.deep.equal(microserviceResponse.microservices[0].cmd) + expect(msvc.commands).to.deep.equal(microserviceResponse.microservices[0].cmd) expect(msvc.extraHosts).to.deep.equal(microserviceResponse.microservices[0].extraHosts) expect(msvc.isController).to.equal(true) expect(msvc.isRouter).to.equal(false) expect(msvc.isNats).to.equal(false) }) + + it('includes volume mapping scope from the persisted mapping', async () => { + const volumeMappings = [{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }] + MicroserviceManager.findAllActiveApplicationMicroservices.resolves([{ + ...microserviceWithValidImage, + volumeMappings + }]) + + const result = await $subject + expect(result.microservices[0].volumeMappings).to.eql(volumeMappings) + expect(result.microservices[0].volumeMappings[0].scope).to.equal('shared') + }) + + it('emits catalog models and omits empty argv arrays', async () => { + MicroserviceManager.findAllActiveApplicationMicroservices.resolves([{ + ...microserviceWithValidImage, + cmd: [], + entrypoint: [], + microserviceModel: { bindPath: '/models', permissions: 'ro' }, + modelItems: [{ id: 1, name: 'test-model' }], + cpus: 1.5, + workingDir: '/app' + }]) + + const result = await $subject + const msvc = result.microservices[0] + expect(msvc).to.not.have.property('entrypoint') + expect(msvc).to.not.have.property('commands') + expect(msvc.cmd).to.eql([]) + expect(msvc.models).to.eql({ + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }) + expect(msvc).to.not.have.property('knowledge') + expect(msvc.cpus).to.equal(1.5) + expect(msvc.workingDir).to.equal('/app') + }) + + it('emits knowledge item names and omits an empty catalog', async () => { + MicroserviceManager.findAllActiveApplicationMicroservices.resolves([{ + ...microserviceWithValidImage, + microserviceKnowledge: { bindPath: '/knowledge', permissions: 'ro' }, + knowledgeItems: [{ id: 1, name: 'product-docs' }] + }]) + + const result = await $subject + expect(result.microservices[0].knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }) + expect(result.microservices[0].knowledge.items[0]).to.not.have.property('uuid') + }) }) }) }) @@ -1437,7 +1543,7 @@ describe('Agent Service', () => { def('subject', () => $subject.getAgentRegistries($fog, transaction)) - def('getAgentRegistriesResponse', () => Promise.resolve()) + def('getAgentRegistriesResponse', () => Promise.resolve([])) beforeEach(() => { $sandbox.stub(RegistryManager, 'findAll').returns($getAgentRegistriesResponse) @@ -1460,6 +1566,55 @@ describe('Agent Service', () => { it(`succeeds`, () => { return expect($subject).to.eventually.have.property('registries') }) + + context('with registry rows', () => { + def('getAgentRegistriesResponse', () => Promise.resolve([{ + id: 3, + url: 'https://huggingface.co', + isPublic: true, + username: '', + password: '', + userEmail: '', + type: 'hf', + ca: null, + insecure: false + }, { + id: 4, + url: 'https://registry.example.com', + isPublic: false, + username: 'user', + password: 'secret', + userEmail: 'user@example.com', + type: 'oci', + ca: 'Y2VydA==', + insecure: true + }])) + + it('includes type, ca, and insecure on every registry', async () => { + const result = await $subject + expect(result.registries).to.eql([{ + id: 3, + url: 'https://huggingface.co', + isPublic: true, + username: '', + password: '', + userEmail: '', + type: 'hf', + ca: '', + insecure: false + }, { + id: 4, + url: 'https://registry.example.com', + isPublic: false, + username: 'user', + password: 'secret', + userEmail: 'user@example.com', + type: 'oci', + ca: 'Y2VydA==', + insecure: true + }]) + }) + }) }) }) @@ -1612,139 +1767,719 @@ describe('Agent Service', () => { }) }) - describe('.updateHalHardwareInfo()', () => { + describe('.getAgentLinkedModels()', () => { const transaction = {} - const error = 'Error!' def('uuid', () => 'testUuid') - + def('getModels', () => $sandbox.stub().resolves([])) def('fog', () => ({ uuid: $uuid, + getModels: $getModels, })) + def('subject', () => $subject.getAgentLinkedModels($fog, transaction)) + + it('returns an empty list when the fog has no linked models', async () => { + await expect($subject).to.eventually.eql([]) + expect($getModels).to.have.been.calledWith({ + attributes: ['uuid', 'name', 'repo', 'revision', 'files', 'format', 'registryId'], + joinTableAttributes: [], + transaction, + }) + }) + + it('maps linked model rows to the wire object', async () => { + $getModels.resolves([{ + uuid: '3f2c8a1e-2b64-4c0d-9f11-0a1b2c3d4e5f', + name: 'test-model', + repo: 'org/repo', + revision: null, + registryId: 3, + files: ['file.gguf'], + format: 'gguf', + toJSON () { + return { + uuid: this.uuid, + name: this.name, + repo: this.repo, + revision: this.revision, + registryId: this.registryId, + files: this.files, + format: this.format + } + } + }]) + + await expect($subject).to.eventually.eql([{ + uuid: '3f2c8a1e-2b64-4c0d-9f11-0a1b2c3d4e5f', + name: 'test-model', + repo: 'org/repo', + revision: '', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + }]) + }) + }) + + describe('.getAgentLinkedKnowledge()', () => { + const transaction = {} - def('info', () => 'testInfo') - def('hardwareData', () => ({ - info: $info, + def('uuid', () => 'testUuid') + def('getKnowledge', () => $sandbox.stub().resolves([])) + def('fog', () => ({ + uuid: $uuid, + getKnowledge: $getKnowledge, })) + def('subject', () => $subject.getAgentLinkedKnowledge($fog, transaction)) + + it('returns an empty list when the fog has no linked knowledge', async () => { + await expect($subject).to.eventually.eql([]) + expect($getKnowledge).to.have.been.calledWith({ + attributes: ['uuid', 'name', 'repo', 'revision', 'files', 'format', 'registryId'], + joinTableAttributes: [], + transaction, + }) + }) - def('response', () => ({ - versionCommand: $versionCommandLine, - provisionKey: $provisionKey, - expirationTime: $expirationTime, + it('maps linked knowledge and omits an unset format', async () => { + $getKnowledge.resolves([ + { + toJSON () { + return { + uuid: '3f2c1111-2222-3333-4444-555566667777', + name: 'product-docs', + repo: 'acme/product-manuals', + revision: null, + registryId: 3, + files: '["data/**/*.jsonl"]', + format: 'jsonl' + } + } + }, + { + toJSON () { + return { + uuid: '', + name: 'missing-uuid' + } + } + }, + { + toJSON () { + return { + uuid: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + name: 'wiki', + repo: 'acme/wiki', + revision: 'main', + registryId: 1, + files: ['index.md'], + format: '' + } + } + } + ]) + + await expect($subject).to.eventually.eql([ + { + uuid: '3f2c1111-2222-3333-4444-555566667777', + name: 'product-docs', + repo: 'acme/product-manuals', + revision: '', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'jsonl' + }, + { + uuid: 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + name: 'wiki', + repo: 'acme/wiki', + revision: 'main', + registryId: 1, + files: ['index.md'] + } + ]) + }) + }) + + describe('.getAgentLinkedRuntimeClasses()', () => { + const transaction = {} + + def('uuid', () => 'testUuid') + def('getRuntimeClassLinks', () => $sandbox.stub().resolves([])) + def('fog', () => ({ + uuid: $uuid, + getRuntimeClassLinks: $getRuntimeClassLinks, })) + def('subject', () => $subject.getAgentLinkedRuntimeClasses($fog, transaction)) + + it('returns an empty list when the fog has no linked runtime classes', async () => { + await expect($subject).to.eventually.eql([]) + expect($getRuntimeClassLinks).to.have.been.calledWith({ + attributes: ['name', 'handler'], + joinTableAttributes: [], + transaction, + }) + }) - def('subject', () => $subject.updateHalHardwareInfo($hardwareData, $fog, transaction)) + it('maps linked runtime class rows to the wire object', async () => { + $getRuntimeClassLinks.resolves([{ + name: 'spin', + handler: 'spin', + toJSON () { + return { + name: this.name, + handler: this.handler + } + } + }]) - def('validatorResponse', () => Promise.resolve(true)) - def('hwResponse', () => Promise.resolve()) + await expect($subject).to.eventually.eql([{ + name: 'spin', + handler: 'spin' + }]) + }) + }) + + describe('.updateAgentStatus() additive fields', () => { + const transaction = {} + const microserviceStatus = '[{"id":"testUuid","containerId":"ctr-1","status":"RUNNING","podId":"pod-1"}]' + const agentStatus = { + daemonStatus: 'RUNNING', + daemonOperatingDuration: 25, + daemonLastStart: 1, + warningMessage: '', + memoryUsage: 1, + diskUsage: 1, + cpuUsage: 1, + memoryViolation: false, + diskViolation: false, + cpuViolation: false, + systemAvailableDisk: 1, + systemAvailableMemory: 1, + systemTotalCpu: 1, + repositoryCount: 0, + repositoryStatus: '[]', + systemTime: 1, + lastStatusTime: 1, + ipAddress: '127.0.0.1', + ipAddressExternal: '127.0.0.1', + lastCommandTime: 1, + tunnelStatus: '{}', + version: '1.1.0', + isReadyToUpgrade: false, + isReadyToRollback: false, + gpsStatus: 'OK', + microserviceStatus, + runtimeClasses: [{ name: 'spin', handler: 'spin', source: 'managed' }], + availableCdiDevices: ['nvidia.com/gpu=0'], + modelStatus: '[]', + activeModels: 2, + modelLastUpdate: 1710000000, + } + + def('uuid', () => 'testUuid') + def('fog', () => ({ uuid: $uuid })) + def('subject', () => $subject.updateAgentStatus(agentStatus, $fog, transaction)) beforeEach(() => { - $sandbox.stub(Validator, 'validate').returns($validatorResponse) - $sandbox.stub(HWInfoManager, 'updateOrCreate').returns($hwResponse) + $sandbox.stub(logger, 'warn') + $sandbox.stub(Validator, 'validate').resolves(true) + $sandbox.spy(AppHelper, 'deleteUndefinedFields') + $sandbox.stub(ioFogManager, 'findOne').resolves({ warningMessage: '' }) + $sandbox.stub(FogPlatformStatusManager, 'getParsedStatus').resolves(null) + $sandbox.stub(ioFogManager, 'update').resolves() + $sandbox.stub(MicroserviceStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').resolves() + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: $uuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) }) - it('calls Validator#validate() with correct args', async () => { + it('persists additive fog status fields and microservice podId', async () => { await $subject - expect(Validator.validate).to.have.been.calledWith($hardwareData, Validator.schemas.updateHardwareInfo) + expect(ioFogManager.update).to.have.been.calledWith( + { uuid: $uuid }, + sinon.match({ + runtimeClasses: JSON.stringify(agentStatus.runtimeClasses), + availableCdiDevices: JSON.stringify(agentStatus.availableCdiDevices), + modelStatus: '[]', + activeModels: 2, + modelLastUpdate: 1710000000, + }), + transaction + ) + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: 'testUuid' }, + sinon.match({ podId: 'pod-1', containerId: 'ctr-1' }), + transaction + ) }) - context('when Validator#validate() fails', () => { - def('validatorResponse', () => Promise.reject(error)) + it('persists host capacity and OS identity from agent status', async () => { + const status = { + ...agentStatus, + systemCpus: 8, + systemTotalMemory: 17179869184, + systemTotalDisk: 494384795648, + systemOs: 'linux', + systemOsVersion: 'Ubuntu 22.04', + systemKernelVersion: '6.8.0-45-generic' + } + await AgentService.updateAgentStatus(status, $fog, transaction) + expect(ioFogManager.update).to.have.been.calledWith( + { uuid: $uuid }, + sinon.match({ + systemCpus: 8, + systemTotalMemory: 17179869184, + systemTotalDisk: 494384795648, + systemOs: 'linux', + systemOsVersion: 'Ubuntu 22.04', + systemKernelVersion: '6.8.0-45-generic' + }), + transaction + ) + }) - it(`fails with ${error}`, () => { - return expect($subject).to.be.rejectedWith(error) - }) + it('stringifies a knowledge status array and keeps explicit zeros', async () => { + const status = { + ...agentStatus, + knowledgeStatus: [{ name: 'product-docs', state: 'Ready', source: 'managed' }], + activeKnowledge: 0, + knowledgeLastUpdate: 1710000000000 + } + await AgentService.updateAgentStatus(status, $fog, transaction) + expect(ioFogManager.update).to.have.been.calledWith( + { uuid: $uuid }, + sinon.match({ + knowledgeStatus: JSON.stringify(status.knowledgeStatus), + activeKnowledge: 0, + knowledgeLastUpdate: 1710000000000 + }), + transaction + ) }) - context('when Validator#validate() succeeds', () => { - it('calls HWInfoManager#updateOrCreate() with correct args', async () => { - await $subject - expect(HWInfoManager.updateOrCreate).to.have.been.calledWith({ - iofogUuid: $uuid, - }, $hardwareData, transaction) + it('leaves stored knowledge status unchanged when the keys are omitted', async () => { + await $subject + const update = ioFogManager.update.firstCall.args[1] + expect(update).to.not.have.property('knowledgeStatus') + expect(update).to.not.have.property('activeKnowledge') + expect(update).to.not.have.property('knowledgeLastUpdate') + }) + + it('stores the managed knowledge count separately from the status list length', async () => { + const status = { + ...agentStatus, + knowledgeStatus: [ + { name: 'product-docs', source: 'managed', uuid: '3f2c1111-2222-3333-4444-555566667777' }, + { name: 'local-notes', source: 'local' } + ], + activeKnowledge: 1, + knowledgeLastUpdate: 1710000000000 + } + await AgentService.updateAgentStatus(status, $fog, transaction) + const update = ioFogManager.update.firstCall.args[1] + expect(update.knowledgeStatus).to.equal(JSON.stringify(status.knowledgeStatus)) + expect(update.activeKnowledge).to.equal(1) + expect(JSON.parse(update.knowledgeStatus)).to.have.length(2) + expect(update.activeKnowledge).to.not.equal(JSON.parse(update.knowledgeStatus).length) + }) + + it('ignores unknown status fields', async () => { + const status = { + ...agentStatus, + knowledgeStatus: '[]', + activeKnowledge: 0, + knowledgeLastUpdate: 0, + futureKnowledgeHint: 'ignore-me' + } + await AgentService.updateAgentStatus(status, $fog, transaction) + const update = ioFogManager.update.firstCall.args[1] + expect(update).to.not.have.property('futureKnowledgeHint') + expect(update.knowledgeStatus).to.equal('[]') + expect(update.modelLastUpdate).to.equal(1710000000) + }) + }) - context('when HWInfoManager#updateOrCreate() fails', () => { - def('hwResponse', () => Promise.reject(error)) + describe('agent status schema', () => { + it('accepts a knowledge status string, an array, and omitted keys', async () => { + await expect(Validator.validate({ + knowledgeStatus: '[]', + activeKnowledge: 0, + knowledgeLastUpdate: 0 + }, Validator.schemas.updateAgentStatus)).to.be.fulfilled - it(`fails with ${error}`, () => { - return expect($subject).to.be.equal(undefined) - }) - }) + await expect(Validator.validate({ + knowledgeStatus: [{ name: 'product-docs' }] + }, Validator.schemas.updateAgentStatus)).to.be.fulfilled - context('when HWInfoManager#updateOrCreate() succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.equal(undefined) - }) - }) - }) + await expect(Validator.validate({ + daemonStatus: 'RUNNING' + }, Validator.schemas.updateAgentStatus)).to.be.fulfilled + + await expect(Validator.validate({ + modelLastUpdate: 1710000000000, + knowledgeLastUpdate: 1710000000000, + unexpectedStatusExtra: true + }, Validator.schemas.updateAgentStatus)).to.be.fulfilled + }) + + it('rejects a negative managed knowledge count', async () => { + await expect(Validator.validate({ + activeKnowledge: -1 + }, Validator.schemas.updateAgentStatus)).to.be.rejected }) }) - describe('.updateHalUsbInfo()', () => { + describe('.updateAgentStatus() observed runtime', () => { const transaction = {} - const error = 'Error!' + const receiptTime = 1700000000000 + const fogUuid = 'fog-1' + const reportedUuid = 'ms-reported' + const omittedUuid = 'ms-omitted' + + function buildAgentStatus (microserviceStatusItems, extra = {}) { + return { + daemonStatus: 'RUNNING', + daemonOperatingDuration: 25, + daemonLastStart: 1, + warningMessage: '', + memoryUsage: 1, + diskUsage: 1, + cpuUsage: 1, + memoryViolation: false, + diskViolation: false, + cpuViolation: false, + systemAvailableDisk: 1, + systemAvailableMemory: 1, + systemTotalCpu: 1, + repositoryCount: 0, + repositoryStatus: '[]', + systemTime: 1, + lastStatusTime: 1, + ipAddress: '127.0.0.1', + ipAddressExternal: '127.0.0.1', + lastCommandTime: 1, + tunnelStatus: '{}', + version: '1.1.0', + isReadyToUpgrade: false, + isReadyToRollback: false, + gpsStatus: 'OK', + microserviceStatus: JSON.stringify(microserviceStatusItems), + ...extra, + } + } - def('uuid', () => 'testUuid') + beforeEach(() => { + $sandbox.useFakeTimers({ now: receiptTime, toFake: ['Date'] }) + $sandbox.stub(logger, 'warn') + $sandbox.stub(Validator, 'validate').resolves(true) + $sandbox.stub(ioFogManager, 'findOne').resolves({ warningMessage: '' }) + $sandbox.stub(FogPlatformStatusManager, 'getParsedStatus').resolves(null) + $sandbox.stub(ioFogManager, 'update').resolves() + $sandbox.stub(MicroserviceStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceService, 'deleteNotRunningMicroservices').resolves() + $sandbox.stub(ApplicationManager, 'findAll').resolves([]) + }) - def('fog', () => ({ - uuid: $uuid, - })) + it('stamps fog lastStatusTime with Controller receipt time', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) - def('info', () => 'testInfo') - def('usbData', () => ({ - info: $info, - })) + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }]), + { uuid: fogUuid }, + transaction + ) - def('response', () => ({ - versionCommand: $versionCommandLine, - provisionKey: $provisionKey, - expirationTime: $expirationTime, - })) + expect(ioFogManager.update).to.have.been.calledWith( + { uuid: fogUuid }, + sinon.match({ lastStatusTime: receiptTime }), + transaction + ) + expect(ioFogManager.update.firstCall.args[1].lastStatusTime).to.not.equal(1) + }) - def('subject', () => $subject.updateHalUsbInfo($usbData, $fog, transaction)) + it('stamps fog lastStatusTime with Controller receipt time when the agent sends 0', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) - def('validatorResponse', () => Promise.resolve(true)) - def('usbResponse', () => Promise.resolve()) + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }], { lastStatusTime: 0 }), + { uuid: fogUuid }, + transaction + ) - beforeEach(() => { - $sandbox.stub(Validator, 'validate').returns($validatorResponse) - $sandbox.stub(USBInfoManager, 'updateOrCreate').returns($usbResponse) + expect(ioFogManager.update.firstCall.args[1].lastStatusTime).to.equal(receiptTime) }) - it('calls Validator#validate() with correct args', async () => { - await $subject - expect(Validator.validate).to.have.been.calledWith($usbData, Validator.schemas.updateUsbInfo) + it('stamps fog lastStatusTime with Controller receipt time when the agent clock is hours behind', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus( + [{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }], + { lastStatusTime: receiptTime - (6 * 60 * 60 * 1000) } + ), + { uuid: fogUuid }, + transaction + ) + + expect(ioFogManager.update.firstCall.args[1].lastStatusTime).to.equal(receiptTime) }) - context('when Validator#validate() fails', () => { - def('validatorResponse', () => Promise.reject(error)) + it('stores STOPPED meters as 0 when the agent omits cpu/memory', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) - it(`fails with ${error}`, () => { - return expect($subject).to.be.rejectedWith(error) - }) + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'STOPPED', containerId: 'ctr-stop' }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: reportedUuid }, + sinon.match({ + status: 'STOPPED', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + containerId: 'ctr-stop', + }), + transaction + ) }) - context('when Validator#validate() succeeds', () => { - it('calls USBInfoManager#updateOrCreate() with correct args', async () => { - await $subject - expect(USBInfoManager.updateOrCreate).to.have.been.calledWith({ - iofogUuid: $uuid, - }, $usbData, transaction) + it('stores STOPPED leftover cpu as 0', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) - context('when USBInfoManager#updateOrCreate() fails', () => { - def('usbResponse', () => Promise.reject(error)) + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'STOPPED', cpuUsage: 12.5, memoryUsage: 64 }]), + { uuid: fogUuid }, + transaction + ) - it(`fails with ${error}`, () => { - return expect($subject).to.be.equal(undefined) - }) - }) + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: reportedUuid }, + sinon.match({ status: 'STOPPED', cpuUsage: 0, memoryUsage: 0 }), + transaction + ) + }) - context('when USBInfoManager#updateOrCreate() succeeds', () => { - it(`succeeds`, () => { - return expect($subject).to.equal(undefined) - }) - }) - }) + it('stores RUNNING cpu 12.5', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 12.5, memoryUsage: 64 }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: reportedUuid }, + sinon.match({ status: 'RUNNING', cpuUsage: 12.5, memoryUsage: 64 }), + transaction + ) + }) + + it('stores lastError, lastErrorAt, and restartCount from the agent', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ + id: reportedUuid, + status: 'RUNNING', + cpuUsage: 1, + errorMessage: '', + lastError: 'exitCode=1 oomKilled=false error=config missing', + lastErrorAt: 1726660000123, + restartCount: 4 + }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: reportedUuid }, + sinon.match({ + lastError: 'exitCode=1 oomKilled=false error=config missing', + lastErrorAt: 1726660000123, + restartCount: 4 + }), + transaction + ) + }) + + it('stores restartCount 0 when lastError is present and restartCount is omitted', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ + id: reportedUuid, + status: 'STARTING', + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123 + }]), + { uuid: fogUuid }, + transaction + ) + + const patch = MicroserviceStatusManager.update.firstCall.args[1] + expect(patch.lastError).to.equal('exitCode=1 oomKilled=false') + expect(patch.lastErrorAt).to.equal(1726660000123) + expect(patch.restartCount).to.equal(0) + }) + + it('does not write last crash fields when the agent omits all extras', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }]), + { uuid: fogUuid }, + transaction + ) + + const patch = MicroserviceStatusManager.update.firstCall.args[1] + expect(patch).to.not.have.property('lastError') + expect(patch).to.not.have.property('lastErrorAt') + expect(patch).to.not.have.property('restartCount') + }) + + it('does not fail the PUT when extras are malformed', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ + id: reportedUuid, + status: 'RUNNING', + cpuUsage: 1, + lastError: { nested: true }, + lastErrorAt: 'bogus', + restartCount: 'n/a' + }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledOnce + const patch = MicroserviceStatusManager.update.firstCall.args[1] + expect(patch).to.not.have.property('lastError') + expect(patch.status).to.equal('RUNNING') + }) + + it('treats omitted RUNNING cpu as 0 for this sample', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING' }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: reportedUuid }, + sinon.match({ status: 'RUNNING', cpuUsage: 0, memoryUsage: 0 }), + transaction + ) + }) + + it('marks an omitted desired-active microservice UNKNOWN with zeros', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([{ + uuid: omittedUuid, + iofogUuid: fogUuid, + isActivated: true, + applicationId: 7, + }]) + ApplicationManager.findAll.resolves([{ id: 7, isActivated: true }]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [omittedUuid] }, + sinon.match({ + status: microserviceState.UNKNOWN, + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [omittedUuid] }, + sinon.match({ status: 'INACTIVE', execSessionId: '' }), + transaction + ) + }) + + it('marks an omitted desired-inactive microservice STOPPED with zeros', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([{ + uuid: omittedUuid, + iofogUuid: fogUuid, + isActivated: true, + applicationId: 7, + }]) + ApplicationManager.findAll.resolves([{ id: 7, isActivated: false }]) + + await AgentService.updateAgentStatus( + buildAgentStatus([{ id: reportedUuid, status: 'RUNNING', cpuUsage: 1 }]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [omittedUuid] }, + sinon.match({ + status: microserviceState.STOPPED, + cpuUsage: 0, + memoryUsage: 0, + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [omittedUuid] }, + sinon.match({ status: 'INACTIVE', execSessionId: '' }), + transaction + ) + }) + + it('marks an omitted microservice STOPPED when the microservice itself is inactive', async () => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ iofogUuid: fogUuid }) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([{ + uuid: omittedUuid, + iofogUuid: fogUuid, + isActivated: false, + applicationId: 7, + }]) + ApplicationManager.findAll.resolves([{ id: 7, isActivated: true }]) + + await AgentService.updateAgentStatus( + buildAgentStatus([]), + { uuid: fogUuid }, + transaction + ) + + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [omittedUuid] }, + sinon.match({ status: microserviceState.STOPPED, cpuUsage: 0 }), + transaction + ) }) }) diff --git a/test/src/services/application-service.test.js b/test/src/services/application-service.test.js index 6be461429..f3572220f 100644 --- a/test/src/services/application-service.test.js +++ b/test/src/services/application-service.test.js @@ -8,6 +8,8 @@ const Validator = require('../../../src/schemas') const ChangeTrackingService = require('../../../src/services/change-tracking-service') const MicroserviceService = require('../../../src/services/microservices-service') const NatsAuthService = require('../../../src/services/nats-auth-service') +const MicroserviceStatusManager = require('../../../src/data/managers/microservice-status-manager') +const MicroserviceExecStatusManager = require('../../../src/data/managers/microservice-exec-status-manager') const Errors = require('../../../src/helpers/errors') const transaction = {} @@ -256,6 +258,41 @@ describe('Application Service', () => { it('rejects rename attempts', () => expect($subject).to.be.rejectedWith('Application Resource Name is immutable')) }) + + context('when deactivating', () => { + const members = [ + { uuid: 'ms-active', iofogUuid: 'fog-1', isActivated: true }, + { uuid: 'ms-already-off', iofogUuid: 'fog-1', isActivated: false } + ] + def('patchData', () => ({ isActivated: false })) + + beforeEach(() => { + $sandbox.stub(ApplicationManager, 'findApplicationMicroservices').resolves(members) + $sandbox.stub(MicroserviceStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('sets observed STOPPING and zero meters on all member microservices', async () => { + await $subject + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: ['ms-active', 'ms-already-off'] }, + sinon.match({ + status: 'STOPPING', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { microserviceUuid: ['ms-active', 'ms-already-off'] }, + sinon.match({ status: 'INACTIVE', execSessionId: '' }), + transaction + ) + }) + }) }) describe('.getUserApplicationsEndPoint()', () => { diff --git a/test/src/services/catalog-service.test.js b/test/src/services/catalog-service.test.js index 86efad673..700106aee 100644 --- a/test/src/services/catalog-service.test.js +++ b/test/src/services/catalog-service.test.js @@ -381,25 +381,5 @@ describe('Catalog Service', () => { registry_id: 1 }, transaction) }) - - it('.getBluetoothCatalogItem() queries RESTBlue system item', async () => { - await CatalogService.getBluetoothCatalogItem(transaction) - expect(CatalogItemManager.findOne).to.have.been.calledWith({ - name: 'RESTBlue', - category: 'SYSTEM', - publisher: 'Eclipse ioFog', - registry_id: 1 - }, transaction) - }) - - it('.getHalCatalogItem() queries HAL system item', async () => { - await CatalogService.getHalCatalogItem(transaction) - expect(CatalogItemManager.findOne).to.have.been.calledWith({ - name: 'HAL', - category: 'SYSTEM', - publisher: 'Eclipse ioFog', - registry_id: 1 - }, transaction) - }) }) }) diff --git a/test/src/services/change-tracking-service.test.js b/test/src/services/change-tracking-service.test.js index 09d34082d..70bfdd1f4 100644 --- a/test/src/services/change-tracking-service.test.js +++ b/test/src/services/change-tracking-service.test.js @@ -198,4 +198,20 @@ describe('ChangeTracking Service', () => { }) }) }) + + describe('events', () => { + it('includes additive model and runtime class flags', () => { + expect(ChangeTrackingService.events.clean.models).to.equal(false) + expect(ChangeTrackingService.events.clean.runtimeClasses).to.equal(false) + expect(ChangeTrackingService.events.clean.microserviceModels).to.equal(false) + expect(ChangeTrackingService.events.clean.knowledge).to.equal(false) + expect(ChangeTrackingService.events.clean.microserviceKnowledge).to.equal(false) + expect(ChangeTrackingService.events.models).to.eql({ models: true }) + expect(ChangeTrackingService.events.runtimeClasses).to.eql({ runtimeClasses: true }) + expect(ChangeTrackingService.events.microserviceModels).to.eql({ microserviceModels: true }) + expect(ChangeTrackingService.events.knowledge).to.eql({ knowledge: true }) + expect(ChangeTrackingService.events.microserviceKnowledge).to.eql({ microserviceKnowledge: true }) + expect(ChangeTrackingService.events.prune).to.eql({ prune: true }) + }) + }) }) diff --git a/test/src/services/controller-ms-service.test.js b/test/src/services/controller-ms-service.test.js index c5841bc7a..72c127a97 100644 --- a/test/src/services/controller-ms-service.test.js +++ b/test/src/services/controller-ms-service.test.js @@ -166,6 +166,27 @@ describe('Controller MS Service', () => { expect(MicroservicesService.createOrUpdateServiceAccountForMicroservice).to.not.have.been.called }) + context('when register sends a shared volume mapping', () => { + def('body', () => ({ + ...registerData, + volumeMappings: [{ + hostDestination: 'db', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }] + })) + + it('rejects shared scope on controller volumes', async () => { + await expect($subject).to.be.rejectedWith( + Errors.ValidationError, + /not allowed on controller or system/ + ) + expect(VolumeMappingManager.bulkCreate).to.not.have.been.called + }) + }) + context('when container workload fields are sent on create', () => { def('body', () => ({ ...registerData, diff --git a/test/src/services/fog-platform-service.test.js b/test/src/services/fog-platform-service.test.js index 87385414b..698e423a7 100644 --- a/test/src/services/fog-platform-service.test.js +++ b/test/src/services/fog-platform-service.test.js @@ -61,8 +61,6 @@ describe('Fog platform service', () => { name: 'edge-a', isSystem: false, host: '10.0.0.5', - bluetoothEnabled: false, - abstractedHardwareEnabled: false, containerEngine: 'edgelet', tags: [] } @@ -71,9 +69,7 @@ describe('Fog platform service', () => { natsMode: 'leaf', host: '10.0.0.5', messagingPort: 5671, - containerEngine: 'edgelet', - bluetoothEnabled: false, - abstractedHardwareEnabled: false + containerEngine: 'edgelet' } const parsedSpec = { fogUuid, generation: 2, spec } const router = { diff --git a/test/src/services/iofog-service.test.js b/test/src/services/iofog-service.test.js index ff4660a44..d7b6b54ee 100644 --- a/test/src/services/iofog-service.test.js +++ b/test/src/services/iofog-service.test.js @@ -12,7 +12,6 @@ const NatsConnectionManager = require('../../../src/data/managers/nats-connectio const AppHelper = require('../../../src/helpers/app-helper') const Validator = require('../../../src/schemas') const ChangeTrackingService = require('../../../src/services/change-tracking-service') -const CatalogService = require('../../../src/services/catalog-service') const MicroserviceManager = require('../../../src/data/managers/microservice-manager') const MicroserviceService = require('../../../src/services/microservices-service') const ApplicationManager = require('../../../src/data/managers/application-manager') @@ -21,8 +20,6 @@ const FogPublicKeyManager = require('../../../src/data/managers/iofog-public-key const TagsManager = require('../../../src/data/managers/tags-manager') const ioFogProvisionKeyManager = require('../../../src/data/managers/iofog-provision-key-manager') const ioFogVersionCommandManager = require('../../../src/data/managers/iofog-version-command-manager') -const HWInfoManager = require('../../../src/data/managers/hw-info-manager') -const USBInfoManager = require('../../../src/data/managers/usb-info-manager') const Errors = require('../../../src/helpers/errors') const config = require('../../../src/config') const FogPlatformSpecManager = require('../../../src/data/managers/fog-platform-spec-manager') @@ -137,9 +134,7 @@ describe('ioFog Service', () => { pruningFrequency: 10, availableDiskThreshold: 20, logLevel: 'INFO', - routerMode: 'edge', - abstractedHardwareEnabled: false, - bluetoothEnabled: false + routerMode: 'edge' } def('subject', () => $subject.createFogEndPoint(fogData, isCLI, transaction)) @@ -187,14 +182,6 @@ describe('ioFog Service', () => { expect(NatsService.ensureNatsForFog).to.not.have.been.called }) - it('does not run HAL/Bluetooth catalog work on the synchronous path', async () => { - $sandbox.stub(CatalogService, 'getHalCatalogItem').resolves({ id: 1 }) - $sandbox.stub(CatalogService, 'getBluetoothCatalogItem').resolves({ id: 2 }) - await $subject - expect(CatalogService.getHalCatalogItem).to.not.have.been.called - expect(CatalogService.getBluetoothCatalogItem).to.not.have.been.called - }) - context('when validation fails', () => { const validationError = new Error('validation failed') @@ -560,6 +547,55 @@ describe('ioFog Service', () => { expect(result).to.have.property('platformStatus', null) }) + context('when knowledge status is stored as a JSON string', () => { + const knowledgeStatusJson = JSON.stringify([{ name: 'product-docs', source: 'managed' }]) + + beforeEach(() => { + ioFogManager.findOneWithTags.resolves(buildFogModel({ + uuid, + name: 'testName', + knowledgeStatus: knowledgeStatusJson, + activeKnowledge: 1, + knowledgeLastUpdate: 1710000000000 + })) + }) + + it('returns knowledgeStatus as the stored JSON string on user GET', async () => { + const result = await $subject + expect(result.knowledgeStatus).to.equal(knowledgeStatusJson) + expect(JSON.parse(result.knowledgeStatus)).to.eql([{ name: 'product-docs', source: 'managed' }]) + expect(result.activeKnowledge).to.equal(1) + expect(result.knowledgeLastUpdate).to.equal(1710000000000) + }) + }) + + context('when the status list is longer than the managed count', () => { + const knowledgeStatusJson = JSON.stringify([ + { name: 'product-docs', source: 'managed', uuid: '3f2c1111-2222-3333-4444-555566667777' }, + { name: 'local-notes', source: 'local' } + ]) + + beforeEach(() => { + ioFogManager.findOneWithTags.resolves(buildFogModel({ + uuid, + name: 'testName', + knowledgeStatus: knowledgeStatusJson, + activeKnowledge: 1, + knowledgeLastUpdate: 1710000000000 + })) + }) + + it('keeps activeKnowledge as the managed count', async () => { + const result = await $subject + const rows = JSON.parse(result.knowledgeStatus) + expect(result.knowledgeStatus).to.equal(knowledgeStatusJson) + expect(rows).to.have.length(2) + expect(result.activeKnowledge).to.equal(1) + expect(result.activeKnowledge).to.not.equal(rows.length) + expect(rows[1]).to.not.have.property('uuid') + }) + }) + context('when platform status exists', () => { const lastTransitionAt = new Date('2026-06-24T12:00:00.000Z') @@ -871,42 +907,4 @@ describe('ioFog Service', () => { expect(ChangeTrackingService.update).to.have.been.calledWith(uuid, ChangeTrackingService.events.reboot, transaction) }) }) - - describe('.getHalHardwareInfoEndPoint()', () => { - const uuidObj = { uuid: 'testUuid' } - const hwInfo = { cpu: 'arm64' } - - def('subject', () => $subject.getHalHardwareInfoEndPoint(uuidObj, isCLI, transaction)) - - beforeEach(() => { - $sandbox.stub(Validator, 'validate').resolves(true) - $sandbox.stub(ioFogManager, 'findOne').resolves({ uuid: uuidObj.uuid }) - $sandbox.stub(HWInfoManager, 'findOne').resolves(hwInfo) - }) - - it('returns HAL hardware info', async () => { - const result = await $subject - expect(result).to.equal(hwInfo) - expect(HWInfoManager.findOne).to.have.been.calledWith({ iofogUuid: uuidObj.uuid }, transaction) - }) - }) - - describe('.getHalUsbInfoEndPoint()', () => { - const uuidObj = { uuid: 'testUuid' } - const usbInfo = { devices: [] } - - def('subject', () => $subject.getHalUsbInfoEndPoint(uuidObj, isCLI, transaction)) - - beforeEach(() => { - $sandbox.stub(Validator, 'validate').resolves(true) - $sandbox.stub(ioFogManager, 'findOne').resolves({ uuid: uuidObj.uuid }) - $sandbox.stub(USBInfoManager, 'findOne').resolves(usbInfo) - }) - - it('returns HAL USB info', async () => { - const result = await $subject - expect(result).to.equal(usbInfo) - expect(USBInfoManager.findOne).to.have.been.calledWith({ iofogUuid: uuidObj.uuid }, transaction) - }) - }) }) diff --git a/test/src/services/knowledge-service.test.js b/test/src/services/knowledge-service.test.js new file mode 100644 index 000000000..b22e0fb41 --- /dev/null +++ b/test/src/services/knowledge-service.test.js @@ -0,0 +1,303 @@ +'use strict' + +const { expect } = require('chai') +const sinon = require('sinon') + +const FleetKnowledgeManager = require('../../../src/data/managers/fleet-knowledge-manager') +const FleetModelManager = require('../../../src/data/managers/fleet-model-manager') +const RegistryManager = require('../../../src/data/managers/registry-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const MicroserviceKnowledgeItemManager = require('../../../src/data/managers/microservice-knowledge-item-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') +const ChangeTrackingService = require('../../../src/services/change-tracking-service') +const KnowledgeService = require('../../../src/services/knowledge-service') +const { normalizeKnowledgeFormat } = require('../../../src/helpers/knowledge-format') +const AppHelper = require('../../../src/helpers/app-helper') +const Errors = require('../../../src/helpers/errors') + +describe('Knowledge Service', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + const knowledgeUuid = '3f2c1111-2222-3333-4444-555566667777' + const fogUuid = 'fog-1' + + afterEach(() => { + $sandbox.restore() + }) + + function stubKnowledgeRow (overrides = {}) { + return { + uuid: knowledgeUuid, + name: 'product-docs', + repo: 'acme/product-manuals', + revision: '', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'jsonl', + getFogs: $sandbox.stub().resolves([]), + ...overrides + } + } + + describe('format mapping', () => { + it('lowercases a known token and stores unknown for anything else', () => { + expect(normalizeKnowledgeFormat('JSONL')).to.equal('jsonl') + expect(normalizeKnowledgeFormat('PDF')).to.equal('pdf') + expect(normalizeKnowledgeFormat('docx')).to.equal('unknown') + expect(normalizeKnowledgeFormat('gguf')).to.equal('unknown') + expect(normalizeKnowledgeFormat('txt')).to.equal('unknown') + expect(normalizeKnowledgeFormat('Markdown ')).to.equal('unknown') + }) + + it('leaves an omitted, null, or empty format unset', () => { + expect(normalizeKnowledgeFormat(undefined)).to.equal(null) + expect(normalizeKnowledgeFormat(null)).to.equal(null) + expect(normalizeKnowledgeFormat('')).to.equal(null) + }) + }) + + describe('.createKnowledgeEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(AppHelper, 'generateUUID').returns(knowledgeUuid) + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(null) + $sandbox.stub(FleetKnowledgeManager, 'create').callsFake(async (data) => data) + $sandbox.stub(RegistryManager, 'findOne').resolves({ id: 3, type: 'hf' }) + }) + + it('persists a Hugging Face artifact with lowercase format and a server uuid', async () => { + const result = await KnowledgeService.createKnowledgeEndpoint({ + name: 'product-docs', + repo: 'acme/product-manuals', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'JSONL' + }, transaction) + + expect(result).to.eql({ + uuid: knowledgeUuid, + name: 'product-docs', + repo: 'acme/product-manuals', + revision: '', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'jsonl' + }) + expect(FleetKnowledgeManager.create).to.have.been.calledWith( + sinon.match({ uuid: knowledgeUuid, format: 'jsonl' }), + transaction + ) + }) + + it('stores unknown for an unrecognized format and does not reject the request', async () => { + for (const format of ['docx', 'gguf', 'txt']) { + FleetKnowledgeManager.create.resetHistory() + const result = await KnowledgeService.createKnowledgeEndpoint({ + name: 'product-docs', + repo: 'acme/product-manuals', + registryId: 3, + format + }, transaction) + expect(result.format).to.equal('unknown') + } + }) + + it('omits format when the field is left out', async () => { + const result = await KnowledgeService.createKnowledgeEndpoint({ + name: 'product-docs', + repo: 'acme/product-manuals', + registryId: 3, + files: ['data/**/*.jsonl'] + }, transaction) + + expect(result).to.not.have.property('format') + expect(FleetKnowledgeManager.create).to.have.been.calledWith( + sinon.match({ format: null }), + transaction + ) + }) + + it('rejects non-empty files on an OCI registry', async () => { + RegistryManager.findOne.resolves({ id: 1, type: 'oci' }) + + await expect(KnowledgeService.createKnowledgeEndpoint({ + name: 'oci-docs', + repo: 'acme/manuals', + registryId: 1, + files: ['index.md'] + }, transaction)).to.be.rejectedWith(Errors.ValidationError, /must not specify files/) + expect(FleetKnowledgeManager.create).to.not.have.been.called + }) + + it('accepts an OCI registry when files are empty or omitted', async () => { + RegistryManager.findOne.resolves({ id: 1, type: 'oci' }) + + const withEmpty = await KnowledgeService.createKnowledgeEndpoint({ + name: 'oci-docs', + repo: 'acme/manuals', + registryId: 1, + files: [] + }, transaction) + expect(withEmpty.files).to.eql([]) + expect(withEmpty).to.not.have.property('format') + + const omitted = await KnowledgeService.createKnowledgeEndpoint({ + name: 'oci-docs', + repo: 'acme/manuals', + registryId: 1 + }, transaction) + expect(omitted.files).to.eql([]) + }) + + it('rejects a duplicate knowledge name', async () => { + FleetKnowledgeManager.findOne.resolves(stubKnowledgeRow()) + + await expect(KnowledgeService.createKnowledgeEndpoint({ + name: 'product-docs', + repo: 'acme/product-manuals', + registryId: 3 + }, transaction)).to.be.rejectedWith(Errors.ConflictError, /already exists/) + }) + + it('allows a knowledge name that already belongs to a model', async () => { + $sandbox.stub(FleetModelManager, 'findOne').resolves({ name: 'product-docs', uuid: 'model-uuid' }) + + const result = await KnowledgeService.createKnowledgeEndpoint({ + name: 'product-docs', + repo: 'acme/product-manuals', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'jsonl' + }, transaction) + + expect(result.name).to.equal('product-docs') + expect(result.uuid).to.equal(knowledgeUuid) + expect(FleetModelManager.findOne).to.not.have.been.called + expect(FleetKnowledgeManager.create).to.have.been.calledOnce + }) + }) + + describe('.getKnowledgeEndpoint() and .getKnowledgeLinkEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(stubKnowledgeRow({ + getFogs: $sandbox.stub().resolves([{ uuid: fogUuid }]), + fogUuids: [fogUuid] + })) + }) + + it('returns the artifact spec without linked fogs', async () => { + const row = await FleetKnowledgeManager.findOne() + const result = await KnowledgeService.getKnowledgeEndpoint('product-docs', transaction) + expect(result).to.eql({ + uuid: knowledgeUuid, + name: 'product-docs', + repo: 'acme/product-manuals', + revision: '', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'jsonl' + }) + expect(result).to.not.have.property('fogUuids') + expect(result).to.not.have.property('fogs') + expect(row.getFogs).to.not.have.been.called + }) + + it('returns linked fog uuids from the link endpoint', async () => { + const result = await KnowledgeService.getKnowledgeLinkEndpoint('product-docs', transaction) + expect(result).to.eql({ fogUuids: [fogUuid] }) + }) + }) + + describe('.updateKnowledgeEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(stubKnowledgeRow()) + $sandbox.stub(FleetKnowledgeManager, 'update').resolves() + $sandbox.stub(RegistryManager, 'findOne').resolves({ id: 3, type: 'hf' }) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('rejects a name change because the name is immutable', async () => { + await expect(KnowledgeService.updateKnowledgeEndpoint('product-docs', { + name: 'renamed-docs', + repo: 'acme/other' + }, transaction)).to.be.rejectedWith(Errors.ValidationError, /not allowed/) + expect(FleetKnowledgeManager.update).to.not.have.been.called + }) + + it('updates spec fields without writing the name', async () => { + await KnowledgeService.updateKnowledgeEndpoint('product-docs', { + revision: 'main', + files: ['index/faiss.index'] + }, transaction) + + const patch = FleetKnowledgeManager.update.firstCall.args[1] + expect(patch).to.not.have.property('name') + expect(patch.revision).to.equal('main') + expect(FleetKnowledgeManager.update.firstCall.args[0]).to.eql({ name: 'product-docs' }) + }) + }) + + describe('.deleteKnowledgeEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(stubKnowledgeRow()) + $sandbox.stub(FleetKnowledgeManager, 'delete').resolves() + $sandbox.stub(MicroserviceKnowledgeItemManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('refuses delete while a microservice binds the name and names that microservice', async () => { + MicroserviceKnowledgeItemManager.findAll.resolves([ + { microserviceUuid: 'ms-1', name: 'product-docs' } + ]) + + await expect(KnowledgeService.deleteKnowledgeEndpoint('product-docs', transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-1/) + expect(FleetKnowledgeManager.delete).to.not.have.been.called + }) + }) + + describe('link and unlink', () => { + let knowledge + let agent + + beforeEach(() => { + knowledge = stubKnowledgeRow() + agent = { + uuid: fogUuid, + addKnowledge: $sandbox.stub().resolves(), + removeKnowledge: $sandbox.stub().resolves() + } + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(knowledge) + $sandbox.stub(FogManager, 'findOne').resolves(agent) + $sandbox.stub(MicroserviceManager, 'findOne') + $sandbox.stub(MicroserviceKnowledgeItemManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('links a fog and sets the knowledge change flag', async () => { + const result = await KnowledgeService.linkKnowledgeEndpoint('product-docs', [fogUuid], transaction) + expect(agent.addKnowledge).to.have.been.calledWith(knowledge, { transaction }) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.knowledge, + transaction + ) + expect(result.name).to.equal('product-docs') + expect(result).to.not.have.property('fogUuids') + }) + + it('refuses unlink while a microservice on that fog binds the name', async () => { + MicroserviceKnowledgeItemManager.findAll.resolves([ + { microserviceUuid: 'ms-bound', name: 'product-docs' } + ]) + MicroserviceManager.findOne.resolves({ + uuid: 'ms-bound', + iofogUuid: fogUuid + }) + + await expect(KnowledgeService.unlinkKnowledgeEndpoint('product-docs', [fogUuid], transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-bound/) + expect(agent.removeKnowledge).to.not.have.been.called + expect(ChangeTrackingService.update).to.not.have.been.called + }) + }) +}) diff --git a/test/src/services/microservice-template-service.test.js b/test/src/services/microservice-template-service.test.js new file mode 100644 index 000000000..39e87aa3b --- /dev/null +++ b/test/src/services/microservice-template-service.test.js @@ -0,0 +1,316 @@ +'use strict' + +const { expect } = require('chai') +const sinon = require('sinon') + +const MicroserviceTemplateManager = require('../../../src/data/managers/microservice-template-manager') +const MicroserviceTemplateVariableManager = require('../../../src/data/managers/microservice-template-variable-manager') +const MicroserviceTemplateService = require('../../../src/services/microservice-template-service') +const Errors = require('../../../src/helpers/errors') + +describe('Microservice Template Service', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + + afterEach(() => { + $sandbox.restore() + }) + + function templateSpec () { + return { + images: [{ containerImage: 'nginx:alpine', archId: 1 }], + registryId: 1, + commands: ['nginx', '-g', 'daemon off;'], + env: [{ key: 'PORT', value: '{{ port }}' }], + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }, + knowledge: { + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + } + } + } + + function stubTemplateRow (overrides = {}) { + return { + id: 1, + name: 'nginx-edge', + description: 'nginx at the edge', + microserviceJSON: JSON.stringify(templateSpec()), + variables: [ + { key: 'port', description: 'listen port', defaultValue: JSON.stringify('80') } + ], + ...overrides + } + } + + describe('.createMicroserviceTemplateEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceTemplateManager, 'findOne').resolves(null) + $sandbox.stub(MicroserviceTemplateManager, 'create').callsFake(async (data) => ({ id: 1, ...data })) + $sandbox.stub(MicroserviceTemplateVariableManager, 'create').resolves() + $sandbox.stub(MicroserviceTemplateManager, 'findOnePopulated').resolves(stubTemplateRow()) + }) + + it('creates a template and strips deploy instance fields only', async () => { + const result = await MicroserviceTemplateService.createMicroserviceTemplateEndpoint({ + name: 'nginx-edge', + description: 'nginx at the edge', + variables: [{ key: 'port', defaultValue: '80' }], + microservice: { + ...templateSpec(), + name: 'should-not-store', + application: '{{ app-name }}', + agentName: '{{ agent-name }}', + iofogUuid: 'fog-uuid' + } + }, transaction) + + expect(result.name).to.equal('nginx-edge') + const storedMicroservice = JSON.parse(MicroserviceTemplateManager.create.getCall(0).args[0].microserviceJSON) + expect(storedMicroservice).to.not.have.property('name') + expect(storedMicroservice.application).to.equal('{{ app-name }}') + expect(storedMicroservice.agentName).to.equal('{{ agent-name }}') + expect(storedMicroservice).to.not.have.property('iofogUuid') + expect(storedMicroservice.models.items).to.eql([{ name: 'test-model' }]) + expect(storedMicroservice.knowledge.items).to.eql([{ name: 'product-docs' }]) + expect(result.microservice.commands).to.eql(['nginx', '-g', 'daemon off;']) + expect(result.variables[0].key).to.equal('port') + expect(result.variables[0].defaultValue).to.equal('80') + expect(MicroserviceTemplateManager.create).to.have.been.calledOnce + expect(MicroserviceTemplateVariableManager.create).to.have.been.calledWith( + sinon.match({ + key: 'port', + defaultValue: JSON.stringify('80'), + microserviceTemplateId: 1 + }), + transaction + ) + }) + + it('rejects a duplicate template name', async () => { + MicroserviceTemplateManager.findOne.resolves(stubTemplateRow()) + + await expect(MicroserviceTemplateService.createMicroserviceTemplateEndpoint({ + name: 'nginx-edge', + microservice: templateSpec() + }, transaction)).to.be.rejectedWith(Errors.ConflictError, /already exists/) + }) + }) + + describe('.listMicroserviceTemplatesEndpoint() and .getMicroserviceTemplateEndpoint()', () => { + const row = stubTemplateRow() + + beforeEach(() => { + $sandbox.stub(MicroserviceTemplateManager, 'findAllPopulated').resolves([row]) + $sandbox.stub(MicroserviceTemplateManager, 'findOnePopulated').resolves(row) + }) + + it('lists templates with parsed microservice and variables', async () => { + const result = await MicroserviceTemplateService.listMicroserviceTemplatesEndpoint(transaction) + expect(result.microserviceTemplates).to.have.length(1) + expect(result.microserviceTemplates[0].name).to.equal('nginx-edge') + expect(result.microserviceTemplates[0].microservice.models.bindPath).to.equal('/models') + expect(result.microserviceTemplates[0].microservice.knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }) + expect(result.microserviceTemplates[0].variables[0].defaultValue).to.equal('80') + }) + + it('returns a template by name', async () => { + const result = await MicroserviceTemplateService.getMicroserviceTemplateEndpoint('nginx-edge', transaction) + expect(result.name).to.equal('nginx-edge') + expect(result.microservice.commands).to.eql(['nginx', '-g', 'daemon off;']) + }) + }) + + describe('.updateMicroserviceTemplateEndpoint() and .deleteMicroserviceTemplateEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceTemplateManager, 'findOnePopulated').resolves(stubTemplateRow()) + $sandbox.stub(MicroserviceTemplateManager, 'update').resolves() + $sandbox.stub(MicroserviceTemplateManager, 'delete').resolves() + $sandbox.stub(MicroserviceTemplateVariableManager, 'delete').resolves() + $sandbox.stub(MicroserviceTemplateVariableManager, 'create').resolves() + }) + + it('patches description and microservice spec', async () => { + MicroserviceTemplateManager.findOnePopulated.onSecondCall().resolves(stubTemplateRow({ + description: 'updated', + microserviceJSON: JSON.stringify({ + images: [{ containerImage: 'nginx:stable', archId: 1 }], + registryId: 1 + }) + })) + + const result = await MicroserviceTemplateService.updateMicroserviceTemplateEndpoint('nginx-edge', { + description: 'updated', + microservice: { + images: [{ containerImage: 'nginx:stable', archId: 1 }], + registryId: 1 + } + }, transaction) + + expect(MicroserviceTemplateManager.update).to.have.been.calledOnce + expect(MicroserviceTemplateVariableManager.delete).to.not.have.been.called + expect(result.description).to.equal('updated') + expect(result.microservice.images[0].containerImage).to.equal('nginx:stable') + }) + + it('replaces variables when they are sent on update', async () => { + MicroserviceTemplateManager.findOnePopulated.onSecondCall().resolves(stubTemplateRow({ + variables: [ + { key: 'port', defaultValue: JSON.stringify('8080') } + ] + })) + + const result = await MicroserviceTemplateService.updateMicroserviceTemplateEndpoint('nginx-edge', { + variables: [{ key: 'port', defaultValue: '8080' }] + }, transaction) + + expect(MicroserviceTemplateVariableManager.delete).to.have.been.calledWith( + { microserviceTemplateId: 1 }, + transaction + ) + expect(MicroserviceTemplateVariableManager.create).to.have.been.calledOnce + expect(result.variables[0].defaultValue).to.equal('8080') + }) + + it('deletes a template by name', async () => { + const result = await MicroserviceTemplateService.deleteMicroserviceTemplateEndpoint('nginx-edge', transaction) + expect(result).to.eql({}) + expect(MicroserviceTemplateManager.delete).to.have.been.calledWith({ name: 'nginx-edge' }, transaction) + }) + + it('rejects get/update/delete when the template is missing', async () => { + MicroserviceTemplateManager.findOnePopulated.resolves(null) + await expect(MicroserviceTemplateService.getMicroserviceTemplateEndpoint('missing', transaction)) + .to.be.rejectedWith(Errors.NotFoundError, /not found/) + }) + }) + + describe('.getMicroserviceDataFromTemplate()', () => { + beforeEach(() => { + $sandbox.stub(MicroserviceTemplateManager, 'findOnePopulated').resolves(stubTemplateRow()) + }) + + it('substitutes variables and keeps catalog and container fields', async () => { + const result = await MicroserviceTemplateService.getMicroserviceDataFromTemplate({ + name: 'nginx-edge', + variables: { port: '8080' } + }, false, transaction) + + expect(result.env).to.eql([{ key: 'PORT', value: '8080' }]) + expect(result.commands).to.eql(['nginx', '-g', 'daemon off;']) + expect(result.models).to.eql({ + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }) + expect(result).to.not.have.property('name') + expect(result).to.not.have.property('application') + expect(result).to.not.have.property('iofogUuid') + }) + + it('uses default variable values when the deploy overlay omits them', async () => { + const result = await MicroserviceTemplateService.getMicroserviceDataFromTemplate({ + name: 'nginx-edge' + }, false, transaction) + + expect(result.env).to.eql([{ key: 'PORT', value: '80' }]) + }) + + it('substitutes typed default values into the microservice spec', async () => { + MicroserviceTemplateManager.findOnePopulated.resolves(stubTemplateRow({ + microserviceJSON: JSON.stringify({ + application: '{{application}}', + agentName: '{{agent-name}}', + registryId: '{{registry-id}}', + schedule: '{{schedule}}', + shmSize: '{{shm-size}}', + natsConfig: { + natsAccess: '{{nats-access}}', + natsRule: '{{nats-rule}}' + }, + models: { + bindPath: '{{bind-path}}', + permissions: '{{permissions}}', + items: [ + { name: '{{model1}}' }, + { name: '{{model2}}' }, + { name: '{{model3}}' } + ] + }, + images: [ + { archId: 1, containerImage: '{{amd64-image}}' }, + { archId: 2, containerImage: '{{arm64-image}}' } + ] + }), + variables: [ + { key: 'application', defaultValue: JSON.stringify('test-app') }, + { key: 'agent-name', defaultValue: JSON.stringify('lima') }, + { key: 'nats-access', defaultValue: JSON.stringify(true) }, + { key: 'nats-rule', defaultValue: JSON.stringify('default-user') }, + { key: 'schedule', defaultValue: JSON.stringify(50) }, + { key: 'bind-path', defaultValue: JSON.stringify('/models') }, + { key: 'permissions', defaultValue: JSON.stringify('ro') }, + { key: 'model1', defaultValue: JSON.stringify('model1') }, + { key: 'model2', defaultValue: JSON.stringify('model2') }, + { key: 'model3', defaultValue: JSON.stringify('') }, + { key: 'registry-id', defaultValue: JSON.stringify(5) }, + { key: 'arm64-image', defaultValue: JSON.stringify('dhi.io/clickhouse-server:26.7') }, + { key: 'amd64-image', defaultValue: JSON.stringify('dhi.io/clickhouse-server:26.7') }, + { key: 'shm-size', defaultValue: JSON.stringify(1024) } + ] + })) + + const result = await MicroserviceTemplateService.getMicroserviceDataFromTemplate({ + name: 'nginx-edge' + }, false, transaction) + + expect(result.application).to.equal('test-app') + expect(result.agentName).to.equal('lima') + expect(result.registryId).to.equal(5) + expect(result.schedule).to.equal(50) + expect(result.shmSize).to.equal(1024) + expect(result.natsConfig).to.eql({ + natsAccess: true, + natsRule: 'default-user' + }) + expect(result.models).to.eql({ + bindPath: '/models', + permissions: 'ro', + items: [ + { name: 'model1' }, + { name: 'model2' }, + { name: '' } + ] + }) + expect(result.images).to.eql([ + { archId: 1, containerImage: 'dhi.io/clickhouse-server:26.7' }, + { archId: 2, containerImage: 'dhi.io/clickhouse-server:26.7' } + ]) + }) + + it('accepts application-style array variables', async () => { + const result = await MicroserviceTemplateService.getMicroserviceDataFromTemplate({ + name: 'nginx-edge', + variables: [{ key: 'port', value: '9090' }] + }, false, transaction) + + expect(result.env[0].value).to.equal('9090') + }) + + it('rejects a missing template name', async () => { + MicroserviceTemplateManager.findOnePopulated.resolves(null) + await expect(MicroserviceTemplateService.getMicroserviceDataFromTemplate({ + name: 'missing' + }, false, transaction)).to.be.rejectedWith(Errors.NotFoundError, /not found/) + }) + }) +}) diff --git a/test/src/services/microservices-service.test.js b/test/src/services/microservices-service.test.js index c036f818f..aaee32193 100644 --- a/test/src/services/microservices-service.test.js +++ b/test/src/services/microservices-service.test.js @@ -18,12 +18,27 @@ const VolumeMappingManager = require('../../../src/data/managers/volume-mapping- const MicroserviceExtraHostManager = require('../../../src/data/managers/microservice-extra-host-manager') const MicroserviceEnvManager = require('../../../src/data/managers/microservice-env-manager') const MicroserviceArgManager = require('../../../src/data/managers/microservice-arg-manager') +const MicroserviceEntrypointManager = require('../../../src/data/managers/microservice-entrypoint-manager') +const MicroserviceDeviceManager = require('../../../src/data/managers/microservice-device-manager') +const MicroserviceTmpfsManager = require('../../../src/data/managers/microservice-tmpfs-manager') +const MicroserviceUlimitManager = require('../../../src/data/managers/microservice-ulimit-manager') +const MicroserviceModelManager = require('../../../src/data/managers/microservice-model-manager') +const MicroserviceModelItemManager = require('../../../src/data/managers/microservice-model-item-manager') +const MicroserviceKnowledgeManager = require('../../../src/data/managers/microservice-knowledge-manager') +const MicroserviceKnowledgeItemManager = require('../../../src/data/managers/microservice-knowledge-item-manager') const MicroserviceCdiDevManager = require('../../../src/data/managers/microservice-cdi-device-manager') const MicroserviceCapAddManager = require('../../../src/data/managers/microservice-cap-add-manager') const MicroserviceCapDropManager = require('../../../src/data/managers/microservice-cap-drop-manager') const MicroserviceHealthCheckManager = require('../../../src/data/managers/microservice-healthcheck-manager') const VolumeMountService = require('../../../src/services/volume-mount-service') +const RuntimeClassService = require('../../../src/services/runtime-class-service') +const ModelService = require('../../../src/services/model-service') +const KnowledgeService = require('../../../src/services/knowledge-service') +const MicroserviceTemplateService = require('../../../src/services/microservice-template-service') +const FleetModelManager = require('../../../src/data/managers/fleet-model-manager') +const FleetKnowledgeManager = require('../../../src/data/managers/fleet-knowledge-manager') const RbacRoleManager = require('../../../src/data/managers/rbac-role-manager') +const ErrorMessages = require('../../../src/helpers/error-messages') const RbacServiceAccountManager = require('../../../src/data/managers/rbac-service-account-manager') const NatsAuthService = require('../../../src/services/nats-auth-service') const Errors = require('../../../src/helpers/errors') @@ -47,19 +62,51 @@ function buildMicroserviceRecord (fields = {}) { } } +function stubChildTableDeps (sandbox) { + sandbox.stub(MicroserviceArgManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceArgManager, 'findAllExcludeFields').resolves([]) + sandbox.stub(MicroserviceArgManager, 'create').resolves() + sandbox.stub(MicroserviceArgManager, 'delete').resolves() + sandbox.stub(MicroserviceEntrypointManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceEntrypointManager, 'create').resolves() + sandbox.stub(MicroserviceEntrypointManager, 'delete').resolves() + sandbox.stub(MicroserviceDeviceManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceDeviceManager, 'create').resolves() + sandbox.stub(MicroserviceDeviceManager, 'delete').resolves() + sandbox.stub(MicroserviceTmpfsManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceTmpfsManager, 'create').resolves() + sandbox.stub(MicroserviceTmpfsManager, 'delete').resolves() + sandbox.stub(MicroserviceUlimitManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceUlimitManager, 'create').resolves() + sandbox.stub(MicroserviceUlimitManager, 'delete').resolves() + sandbox.stub(MicroserviceModelManager, 'findOne').resolves(null) + sandbox.stub(MicroserviceModelManager, 'create').resolves() + sandbox.stub(MicroserviceModelManager, 'delete').resolves() + sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceModelItemManager, 'create').resolves() + sandbox.stub(MicroserviceModelItemManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeManager, 'findOne').resolves(null) + sandbox.stub(MicroserviceKnowledgeManager, 'create').resolves() + sandbox.stub(MicroserviceKnowledgeManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceKnowledgeItemManager, 'create').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'delete').resolves() +} + function stubBuildGetResponseDeps (sandbox) { sandbox.stub(MicroservicePortService, 'getPortMappings').resolves([]) sandbox.stub(MicroserviceExtraHostManager, 'findAll').resolves([]) sandbox.stub(CatalogItemImageManager, 'findAll').resolves([]) sandbox.stub(VolumeMappingManager, 'findAll').resolves([]) sandbox.stub(MicroserviceEnvManager, 'findAllExcludeFields').resolves([]) - sandbox.stub(MicroserviceArgManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceCdiDevManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceCapAddManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceCapDropManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceStatusManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceExecStatusManager, 'findAllExcludeFields').resolves([]) sandbox.stub(MicroserviceHealthCheckManager, 'findAllExcludeFields').resolves([]) + sandbox.stub(RbacServiceAccountManager, 'findOneByMicroserviceUuid').resolves(null) + stubChildTableDeps(sandbox) } function stubServiceAccountDeps (sandbox) { @@ -97,12 +144,14 @@ function stubCreateMicroserviceDeps (sandbox, { msvcUuid = 'msvc-uuid', appId = return Promise.resolve(null) }) sandbox.stub(MicroservicePortService, 'validatePortMappings').resolves() + sandbox.stub(MicroservicePortService, 'switchOnUpdateFlagsForMicroservicesForPortMapping').resolves() sandbox.stub(CatalogItemImageManager, 'bulkCreate').resolves() sandbox.stub(MicroserviceStatusManager, 'create').resolves() sandbox.stub(MicroserviceExecStatusManager, 'create').resolves() sandbox.stub(ChangeTrackingService, 'update').resolves() sandbox.stub(VolumeMappingManager, 'bulkCreate').resolves() stubServiceAccountDeps(sandbox) + stubChildTableDeps(sandbox) } function stubUpdateMicroserviceDeps (sandbox, existing) { @@ -112,6 +161,7 @@ function stubUpdateMicroserviceDeps (sandbox, existing) { sandbox.stub(Validator, 'validate').resolves(true) sandbox.stub(AppHelper, 'deleteUndefinedFields').callsFake((value) => value) sandbox.stub(FogManager, 'findOne').resolves(fog) + sandbox.stub(RegistryManager, 'findOne').resolves({ id: existing.registryId || 1, type: 'oci' }) sandbox.stub(MicroserviceManager, 'findOneWithCategory').resolves({ ...existing, catalogItem: existing.catalogItem || null, @@ -125,7 +175,9 @@ function stubUpdateMicroserviceDeps (sandbox, existing) { sandbox.stub(ServiceManager, 'findOne').resolves(null) sandbox.stub(ChangeTrackingService, 'update').resolves() sandbox.stub(VolumeMappingManager, 'delete').resolves() + sandbox.stub(VolumeMappingManager, 'findAll').resolves([]) stubServiceAccountDeps(sandbox) + stubChildTableDeps(sandbox) } describe('Microservices Service', () => { @@ -215,6 +267,81 @@ describe('Microservices Service', () => { }) }) + describe('.buildGetMicroserviceResponse()', () => { + const msvcUuid = 'msvc-uuid' + const roleRef = { + kind: 'Role', + name: 'custom-role', + apiGroup: 'edgelet.iofog.org/v1' + } + + def('subject', () => $service.buildGetMicroserviceResponse($microservice, transaction)) + + beforeEach(() => { + stubBuildGetResponseDeps($sandbox) + $sandbox.stub(ApplicationManager, 'findOne').resolves({ name: 'my-app' }) + }) + + context('when serviceAccount is preloaded on the microservice', () => { + def('microservice', () => ({ + uuid: msvcUuid, + name: 'test-msvc', + applicationId: 42, + logSize: 1024, + serviceAccount: { name: 'test-msvc', roleRef } + })) + + it('returns shaped serviceAccount with roleRef only', async () => { + const result = await $subject + expect(RbacServiceAccountManager.findOneByMicroserviceUuid).to.not.have.been.called + expect(result.serviceAccount).to.eql({ roleRef }) + }) + + it('does not leak raw serviceAccount database fields', async () => { + const result = await $subject + expect(result.serviceAccount).to.not.have.property('id') + expect(result.serviceAccount).to.not.have.property('microserviceUuid') + expect(result.serviceAccount).to.not.have.property('roleId') + }) + }) + + context('when serviceAccount is not preloaded', () => { + def('microservice', () => ({ + uuid: msvcUuid, + name: 'test-msvc', + applicationId: 42, + logSize: 1024 + })) + + beforeEach(() => { + RbacServiceAccountManager.findOneByMicroserviceUuid.resolves({ + name: 'test-msvc', + roleRef + }) + }) + + it('loads serviceAccount by microservice uuid', async () => { + const result = await $subject + expect(RbacServiceAccountManager.findOneByMicroserviceUuid).to.have.been.calledWith(msvcUuid, transaction) + expect(result.serviceAccount).to.eql({ roleRef }) + }) + }) + + context('when microservice has no service account', () => { + def('microservice', () => ({ + uuid: msvcUuid, + name: 'test-msvc', + applicationId: 42, + logSize: 1024 + })) + + it('returns null serviceAccount', async () => { + const result = await $subject + expect(result.serviceAccount).to.equal(null) + }) + }) + }) + describe('.createMicroserviceEndPoint()', () => { const msvcUuid = 'new-msvc-uuid' const microserviceData = { @@ -287,6 +414,230 @@ describe('Microservices Service', () => { expect(VolumeMappingManager.create).to.have.been.called }) }) + + context('when volumeMappings include scope', () => { + const basePayload = { + name: 'new-msvc', + application: 'my-app', + iofogUuid: 'fog-uuid', + images: [{ containerImage: 'demo:latest', archId: 1 }], + registryId: 1 + } + + it('stores private when type is volume and scope is omitted', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0].scope).to.equal('private') + }) + + it('stores private when type is volume and scope is empty or null', async () => { + for (const scope of ['', null]) { + VolumeMappingManager.bulkCreate.resetHistory() + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0].scope).to.equal('private') + } + }) + + it('stores shared when type is volume and scope is Shared', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'Shared' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0]).to.include({ + hostDestination: 'nodered-config', + type: 'volume', + scope: 'shared' + }) + }) + + it('rejects unknown scope on type volume', async () => { + await expect($service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'Shareed' + }] + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError, /Unknown volume mapping scope/) + }) + + it('stores private when type is omitted and scope is shared', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: '/host/data', + containerDestination: '/data', + accessMode: 'rw', + scope: 'shared' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0]).to.include({ type: 'bind', scope: 'private' }) + }) + + it('stores private when type is bind and scope is unknown', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: '/host/data', + containerDestination: '/data', + accessMode: 'rw', + type: 'bind', + scope: 'nope' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0]).to.include({ type: 'bind', scope: 'private' }) + }) + + it('persists both a private and a shared mapping with the same volume name', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'config', + containerDestination: '/private-config', + accessMode: 'rw', + type: 'volume' + }, { + hostDestination: 'config', + containerDestination: '/shared-config', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings).to.have.length(2) + expect(mappings[0]).to.include({ + hostDestination: 'config', + containerDestination: '/private-config', + scope: 'private' + }) + expect(mappings[1]).to.include({ + hostDestination: 'config', + containerDestination: '/shared-config', + scope: 'shared' + }) + }) + + it('persists shared with read-only access', async () => { + await $service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'ro', + type: 'volume', + scope: 'shared' + }] + }, isCLI, transaction) + + const [mappings] = VolumeMappingManager.bulkCreate.firstCall.args + expect(mappings[0]).to.include({ + accessMode: 'ro', + scope: 'shared' + }) + }) + + it('rejects a volume name that is not a valid local volume name', async () => { + await expect($service.createMicroserviceEndPoint({ + ...basePayload, + volumeMappings: [{ + hostDestination: '../x', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume' + }] + }, isCLI, transaction)).to.be.rejectedWith(Errors.InvalidArgumentError, /invalid characters/) + }) + }) + + context('when runtime is set', () => { + it('auto-links the runtime class on an edgelet fog', async () => { + FogManager.findOne.resolves({ + uuid: 'fog-uuid', + archId: 1, + name: 'edge-1', + availableRuntimes: ['spin'], + containerEngine: 'edgelet' + }) + const ensure = $sandbox.stub(RuntimeClassService, 'ensureRuntimeClassLinkedToFog').resolves(true) + + await $service.createMicroserviceEndPoint({ + ...microserviceData, + runtime: 'spin' + }, isCLI, transaction) + + expect(ensure).to.have.been.calledWith('fog-uuid', 'spin', transaction) + }) + + it('rejects when the runtime is not in availableRuntimes', async () => { + FogManager.findOne.resolves({ + uuid: 'fog-uuid', + archId: 1, + name: 'edge-1', + availableRuntimes: ['runc'], + containerEngine: 'edgelet' + }) + const ensure = $sandbox.stub(RuntimeClassService, 'ensureRuntimeClassLinkedToFog').resolves(true) + + await expect($service.createMicroserviceEndPoint({ + ...microserviceData, + runtime: 'spin' + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError, /not available/) + expect(ensure).to.not.have.been.called + }) + + it('rejects when the runtime class row is missing', async () => { + FogManager.findOne.resolves({ + uuid: 'fog-uuid', + archId: 1, + name: 'edge-1', + availableRuntimes: ['spin'], + containerEngine: 'edgelet' + }) + $sandbox.stub(RuntimeClassService, 'ensureRuntimeClassLinkedToFog') + .rejects(new Errors.ValidationError("RuntimeClass 'spin' does not exist")) + + await expect($service.createMicroserviceEndPoint({ + ...microserviceData, + runtime: 'spin' + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError, /does not exist/) + }) + }) }) describe('.updateMicroserviceEndPoint()', () => { @@ -310,12 +661,88 @@ describe('Microservices Service', () => { expect(ChangeTrackingService.update).to.have.been.called }) + context('when runtime is set', () => { + def('updateData', () => ({ runtime: 'spin' })) + + it('auto-links the runtime class on an edgelet fog', async () => { + FogManager.findOne.resolves({ + uuid: existing.iofogUuid, + archId: 1, + name: 'edge-1', + availableRuntimes: ['spin'], + containerEngine: 'edgelet' + }) + const ensure = $sandbox.stub(RuntimeClassService, 'ensureRuntimeClassLinkedToFog').resolves(true) + + await $subject + expect(ensure).to.have.been.calledWith(existing.iofogUuid, 'spin', transaction) + }) + }) + context('when renaming', () => { def('updateData', () => ({ name: 'new-name' })) it('rejects rename attempts', () => expect($subject).to.be.rejectedWith('Microservice Resource Name is immutable')) }) + context('when rebasing from a template', () => { + def('updateData', () => ({ + template: { + name: 'nginx-edge', + variables: { port: '9090' } + } + })) + + beforeEach(() => { + MicroserviceManager.findOne.callsFake((where) => { + if (where && where.name && where.uuid) { + return Promise.resolve(null) + } + return Promise.resolve(existing) + }) + $sandbox.stub(CatalogItemImageManager, 'delete').resolves() + $sandbox.stub(CatalogItemImageManager, 'bulkCreate').resolves() + $sandbox.stub(FleetModelManager, 'findOne').resolves({ uuid: 'model-uuid', name: 'test-model' }) + $sandbox.stub(ModelService, 'ensureModelsLinkedToFog').resolves(['test-model']) + $sandbox.stub(MicroserviceTemplateService, 'getMicroserviceDataFromTemplate').resolves({ + images: [{ containerImage: 'nginx:alpine', archId: 1 }], + registryId: 1, + commands: ['nginx', '-g', 'daemon off;'], + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + }) + }) + + it('expands the template and preserves identity from the existing microservice', async () => { + await $subject + + expect(MicroserviceTemplateService.getMicroserviceDataFromTemplate).to.have.been.calledWith( + { name: 'nginx-edge', variables: { port: '9090' } }, + isCLI, + transaction + ) + expect(Validator.validate).to.have.been.calledWith( + sinon.match({ + name: existing.name, + iofogUuid: existing.iofogUuid, + images: [{ containerImage: 'nginx:alpine', archId: 1 }] + }), + Validator.schemas.microserviceUpdate + ) + expect(MicroserviceModelItemManager.create).to.have.been.calledWith( + sinon.match({ name: 'test-model' }), + transaction + ) + expect(MicroserviceArgManager.create).to.have.been.calledWith( + sinon.match({ cmd: 'nginx' }), + transaction + ) + }) + }) + context('when microservice is controller', () => { beforeEach(() => { MicroserviceManager.findOneWithCategory.resolves({ @@ -421,6 +848,41 @@ describe('Microservices Service', () => { }) }) }) + + context('when volumeMappings change scope', () => { + def('updateData', () => ({ + volumeMappings: [{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }] + })) + + it('marks the microservice for rebuild and notifies the node', async () => { + await $subject + expect(MicroserviceManager.updateAndFind).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(VolumeMappingManager.create).to.have.been.calledWith( + sinon.match({ + microserviceUuid: msvcUuid, + hostDestination: 'nodered-config', + type: 'volume', + scope: 'shared' + }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + }) }) describe('.updateSystemMicroserviceEndPoint()', () => { @@ -462,6 +924,25 @@ describe('Microservices Service', () => { ) }) + context('when a controller microservice is sent a shared volume', () => { + def('updateData', () => ({ + volumeMappings: [{ + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }] + })) + + it('rejects shared scope on a controller microservice', () => { + return expect($subject).to.be.rejectedWith( + Errors.ValidationError, + /not allowed on controller or system/ + ) + }) + }) + context('when microservice is not controller', () => { def('updateData', () => ({ schedule: 4 })) @@ -617,8 +1098,11 @@ describe('Microservices Service', () => { beforeEach(() => { $sandbox.stub(Validator, 'validate').resolves(true) $sandbox.stub(MicroserviceManager, 'findMicroserviceOnGet').resolves(microservice) + $sandbox.stub(ApplicationManager, 'findOne').resolves({ id: microservice.applicationId, isSystem: false }) $sandbox.stub(VolumeMappingManager, 'findOne').resolves(null) $sandbox.stub(VolumeMappingManager, 'create').resolves({ uuid: 'vol-uuid' }) + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() }) it('creates a user volume mapping', async () => { @@ -626,6 +1110,175 @@ describe('Microservices Service', () => { expect(VolumeMappingManager.create).to.have.been.calledOnce }) + it('persists default private scope and marks the microservice for rebuild', async () => { + await $subject + expect(VolumeMappingManager.create).to.have.been.calledWith( + sinon.match({ + microserviceUuid: msvcUuid, + hostDestination: 'data', + type: 'volume', + scope: 'private' + }), + transaction + ) + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + microservice.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + + it('persists shared scope on type volume', async () => { + await $service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'Shared' + }, isCLI, transaction) + + expect(VolumeMappingManager.create).to.have.been.calledWith( + sinon.match({ + hostDestination: 'nodered-config', + type: 'volume', + scope: 'shared' + }), + transaction + ) + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + microservice.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + + it('rejects bind when hostDestination is not an absolute path', () => { + return expect($service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'bind' + }, isCLI, transaction)).to.be.rejectedWith(Errors.InvalidArgumentError, /absolute host path/) + }) + + it('stores private when type is bind and scope is shared', async () => { + await $service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: '/host/data', + containerDestination: '/data', + accessMode: 'rw', + type: 'bind', + scope: 'shared' + }, isCLI, transaction) + + expect(VolumeMappingManager.create).to.have.been.calledWith( + sinon.match({ + type: 'bind', + scope: 'private' + }), + transaction + ) + }) + + it('rejects unknown scope on type volume', () => { + return expect($service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'Shareed' + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError, /Unknown volume mapping scope/) + }) + + it('rejects shared scope on a system microservice', () => { + ApplicationManager.findOne.resolves({ id: microservice.applicationId, isSystem: true }) + return expect($service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }, isCLI, transaction)).to.be.rejectedWith( + Errors.ValidationError, + /not allowed on controller or system/ + ) + }) + + it('rejects shared scope on a controller microservice', () => { + MicroserviceManager.findMicroserviceOnGet.resolves(buildMicroserviceRecord({ + uuid: msvcUuid, + isController: true + })) + return expect($service.createVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }, isCLI, transaction)).to.be.rejectedWith( + Errors.ValidationError, + /not allowed on controller or system/ + ) + }) + + it('allows the same shared volume name on two user microservices on one node', async () => { + const otherMs = buildMicroserviceRecord({ uuid: 'other-msvc', iofogUuid: 'fog-uuid' }) + MicroserviceManager.findMicroserviceOnGet.callsFake((where) => { + if (where && where.uuid === 'other-msvc') { + return Promise.resolve(otherMs) + } + return Promise.resolve(microservice) + }) + const mapping = { + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + } + + await $service.createVolumeMappingEndPoint(msvcUuid, mapping, isCLI, transaction) + await $service.createVolumeMappingEndPoint('other-msvc', { ...mapping }, isCLI, transaction) + + expect(VolumeMappingManager.create).to.have.been.calledTwice + expect(VolumeMappingManager.create.firstCall.args[0]).to.include({ + microserviceUuid: msvcUuid, + hostDestination: 'nodered-config', + scope: 'shared' + }) + expect(VolumeMappingManager.create.secondCall.args[0]).to.include({ + microserviceUuid: 'other-msvc', + hostDestination: 'nodered-config', + scope: 'shared' + }) + }) + + it('skips fog change tracking when the microservice has no node', async () => { + MicroserviceManager.findMicroserviceOnGet.resolves(buildMicroserviceRecord({ + uuid: msvcUuid, + iofogUuid: null + })) + + await $subject + + expect(VolumeMappingManager.create).to.have.been.calledOnce + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.called + }) + context('when type is serviceAccount', () => { def('volumeMappingData', () => ({ hostDestination: 'immutable-name', @@ -641,4 +1294,905 @@ describe('Microservices Service', () => { }) }) }) + + describe('.createSystemVolumeMappingEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const microservice = buildMicroserviceRecord({ uuid: msvcUuid }) + + def('subject', () => $service.createSystemVolumeMappingEndPoint(msvcUuid, $volumeMappingData, isCLI, transaction)) + def('volumeMappingData', () => ({ + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume' + })) + + beforeEach(() => { + $sandbox.stub(Validator, 'validate').resolves(true) + $sandbox.stub(MicroserviceManager, 'findOne').resolves(microservice) + $sandbox.stub(ApplicationManager, 'findOne').resolves({ id: microservice.applicationId, isSystem: true }) + $sandbox.stub(VolumeMappingManager, 'findOne').resolves(null) + $sandbox.stub(VolumeMappingManager, 'create').resolves({ uuid: 'vol-uuid' }) + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('persists private scope and marks the system microservice for rebuild', async () => { + await $subject + expect(VolumeMappingManager.create).to.have.been.calledWith( + sinon.match({ + microserviceUuid: msvcUuid, + type: 'volume', + scope: 'private' + }), + transaction + ) + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + microservice.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + + it('rejects explicit shared scope on a system microservice volume', () => { + return expect($service.createSystemVolumeMappingEndPoint(msvcUuid, { + hostDestination: 'data', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }, isCLI, transaction)).to.be.rejectedWith( + Errors.ValidationError, + /not allowed on controller or system/ + ) + }) + }) + + describe('.deleteVolumeMappingEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const mappingUuid = 'vol-uuid' + const microservice = buildMicroserviceRecord({ uuid: msvcUuid }) + + def('subject', () => $service.deleteVolumeMappingEndPoint(msvcUuid, mappingUuid, isCLI, transaction)) + + beforeEach(() => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves(microservice) + $sandbox.stub(VolumeMappingManager, 'findOne').resolves({ + uuid: mappingUuid, + microserviceUuid: msvcUuid, + type: 'volume' + }) + $sandbox.stub(VolumeMappingManager, 'delete').resolves(1) + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('deletes the mapping and marks the microservice for rebuild', async () => { + await $subject + expect(VolumeMappingManager.delete).to.have.been.calledWith( + { uuid: mappingUuid, microserviceUuid: msvcUuid }, + transaction + ) + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + microservice.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + + it('rejects delete of system-managed serviceAccount mappings', () => { + VolumeMappingManager.findOne.resolves({ + uuid: mappingUuid, + microserviceUuid: msvcUuid, + type: 'serviceAccount' + }) + return expect($subject).to.be.rejectedWith( + 'Volume mappings of type serviceAccount are system-managed and cannot be deleted by users' + ) + }) + }) + + describe('.deleteSystemVolumeMappingEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const mappingUuid = 'vol-uuid' + const microservice = buildMicroserviceRecord({ uuid: msvcUuid }) + + beforeEach(() => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves(microservice) + $sandbox.stub(VolumeMappingManager, 'findOne').resolves({ + uuid: mappingUuid, + microserviceUuid: msvcUuid, + type: 'volume' + }) + $sandbox.stub(VolumeMappingManager, 'delete').resolves(1) + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('deletes the mapping and marks the system microservice for rebuild', async () => { + await $service.deleteSystemVolumeMappingEndPoint(msvcUuid, mappingUuid, isCLI, transaction) + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { rebuild: true }, + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + microservice.iofogUuid, + ChangeTrackingService.events.microserviceCommon, + transaction + ) + }) + }) + + describe('.listVolumeMappingsEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const microservice = buildMicroserviceRecord({ uuid: msvcUuid }) + const mappings = [{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + id: 7, + type: 'volume', + scope: 'shared' + }] + + beforeEach(() => { + $sandbox.stub(MicroserviceManager, 'findOne').resolves(microservice) + $sandbox.stub(VolumeMappingManager, 'findAll').resolves(mappings) + }) + + it('returns stored mapping scope', async () => { + const result = await $service.listVolumeMappingsEndPoint(msvcUuid, isCLI, transaction) + expect(VolumeMappingManager.findAll).to.have.been.calledWith({ microserviceUuid: msvcUuid }, transaction) + expect(result).to.eql(mappings) + expect(result[0].scope).to.equal('shared') + }) + }) + + describe('.createMicroserviceEndPoint() catalog and registry', () => { + const msvcUuid = 'new-msvc-uuid' + const microserviceData = { + name: 'new-msvc', + application: 'my-app', + iofogUuid: 'fog-uuid', + images: [{ containerImage: 'demo:latest', archId: 1 }], + registryId: 1 + } + + beforeEach(() => { + stubCreateMicroserviceDeps($sandbox, { msvcUuid }) + }) + + it('rejects a Hugging Face registry for microservice images', async () => { + RegistryManager.findOne.resolves({ id: 3, type: 'hf' }) + + return expect($service.createMicroserviceEndPoint({ + ...microserviceData, + registryId: 3 + }, isCLI, transaction)).to.be.rejectedWith( + AppHelper.formatMessage(ErrorMessages.REGISTRY_NOT_OCI_FOR_IMAGE, 3) + ) + }) + + it('auto-attaches catalog model names to the target agent', async () => { + $sandbox.stub(FleetModelManager, 'findOne').resolves({ uuid: 'model-uuid', name: 'test-model' }) + const attach = $sandbox.stub(ModelService, 'ensureModelsLinkedToFog').resolves(['test-model']) + + await $service.createMicroserviceEndPoint({ + ...microserviceData, + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + }, isCLI, transaction) + + expect(attach).to.have.been.calledWith('fog-uuid', ['test-model'], transaction) + expect(MicroserviceModelManager.create).to.have.been.calledWith( + sinon.match({ bindPath: '/models', permissions: 'ro' }), + transaction + ) + expect(MicroserviceModelItemManager.create).to.have.been.calledWith( + sinon.match({ name: 'test-model' }), + transaction + ) + const created = MicroserviceManager.create.firstCall.args[0] + expect(created).to.not.have.property('models') + expect(created).to.not.have.property('commands') + }) + + it('persists a knowledge catalog and auto-attaches names to the target agent', async () => { + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves({ uuid: 'knowledge-uuid', name: 'product-docs' }) + const attach = $sandbox.stub(KnowledgeService, 'ensureKnowledgeLinkedToFog').resolves(['product-docs']) + + await $service.createMicroserviceEndPoint({ + ...microserviceData, + knowledge: { + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + } + }, isCLI, transaction) + + expect(attach).to.have.been.calledWith('fog-uuid', ['product-docs'], transaction) + expect(MicroserviceKnowledgeManager.create).to.have.been.calledWith( + sinon.match({ bindPath: '/knowledge', permissions: 'ro' }), + transaction + ) + expect(MicroserviceKnowledgeItemManager.create).to.have.been.calledWith( + sinon.match({ name: 'product-docs' }), + transaction + ) + const created = MicroserviceManager.create.firstCall.args[0] + expect(created).to.not.have.property('knowledge') + }) + + it('rejects an unknown knowledge catalog item', async () => { + $sandbox.stub(FleetKnowledgeManager, 'findOne').resolves(null) + + return expect($service.createMicroserviceEndPoint({ + ...microserviceData, + knowledge: { + bindPath: '/knowledge', + items: [{ name: 'missing-docs' }] + } + }, isCLI, transaction)).to.be.rejectedWith( + AppHelper.formatMessage(ErrorMessages.MICROSERVICE_KNOWLEDGE_NOT_FOUND, 'missing-docs') + ) + }) + + it('rejects knowledge on a system catalog microservice', async () => { + const CatalogService = require('../../../src/services/catalog-service') + $sandbox.stub(CatalogService, 'getCatalogItem').resolves({ + category: 'SYSTEM', + images: microserviceData.images, + registryId: 1 + }) + + return expect($service.createMicroserviceEndPoint({ + ...microserviceData, + catalogItemId: 9, + knowledge: { + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + } + }, isCLI, transaction)).to.be.rejectedWith(ErrorMessages.MICROSERVICE_KNOWLEDGE_SYSTEM_FORBIDDEN) + }) + + it('persists commands when cmd is sent as an alias', async () => { + await $service.createMicroserviceEndPoint({ + ...microserviceData, + cmd: ['python', 'app.py'] + }, isCLI, transaction) + + const created = MicroserviceManager.create.firstCall.args[0] + expect(created).to.not.have.property('commands') + expect(MicroserviceArgManager.create).to.have.been.calledWith( + sinon.match({ cmd: 'python' }), + transaction + ) + expect(MicroserviceArgManager.create).to.have.been.calledWith( + sinon.match({ cmd: 'app.py' }), + transaction + ) + }) + + it('deploys from a template with identity overlay and merged catalog/container fields', async () => { + $sandbox.stub(FleetModelManager, 'findOne').resolves({ uuid: 'model-uuid', name: 'test-model' }) + $sandbox.stub(ModelService, 'ensureModelsLinkedToFog').resolves(['test-model']) + $sandbox.stub(MicroserviceTemplateService, 'getMicroserviceDataFromTemplate').resolves({ + images: [{ containerImage: 'nginx:alpine', archId: 1 }], + registryId: 1, + commands: ['nginx', '-g', 'daemon off;'], + models: { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + }) + + await $service.createMicroserviceEndPoint({ + name: 'my-instance', + application: 'my-app', + iofogUuid: 'fog-uuid', + template: { + name: 'nginx-edge', + variables: { port: '8080' } + } + }, isCLI, transaction) + + expect(MicroserviceTemplateService.getMicroserviceDataFromTemplate).to.have.been.calledWith( + { name: 'nginx-edge', variables: { port: '8080' } }, + isCLI, + transaction + ) + expect(Validator.validate).to.have.been.calledWith( + sinon.match({ + name: 'my-instance', + application: 'my-app', + iofogUuid: 'fog-uuid', + images: [{ containerImage: 'nginx:alpine', archId: 1 }] + }), + Validator.schemas.microserviceCreate + ) + const created = MicroserviceManager.create.firstCall.args[0] + expect(created.name).to.equal('my-instance') + expect(created).to.not.have.property('models') + expect(created).to.not.have.property('commands') + expect(MicroserviceModelItemManager.create).to.have.been.calledWith( + sinon.match({ name: 'test-model' }), + transaction + ) + expect(MicroserviceArgManager.create).to.have.been.calledWith( + sinon.match({ cmd: 'nginx' }), + transaction + ) + }) + }) + + describe('.updateMicroserviceCatalogEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const existingCatalog = { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + } + + function stubCatalogPatch (sandbox, existing, previousCatalog = null) { + sandbox.stub(MicroserviceManager, 'findOneWithCategory').resolves(existing) + sandbox.stub(VolumeMappingManager, 'findAll').resolves([]) + sandbox.stub(FleetModelManager, 'findOne').resolves({ uuid: 'model-uuid', name: 'test-model' }) + sandbox.stub(ModelService, 'ensureModelsLinkedToFog').resolves([]) + sandbox.stub(MicroserviceManager, 'update').resolves() + sandbox.stub(ChangeTrackingService, 'update').resolves() + sandbox.stub(MicroserviceTmpfsManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceModelManager, 'findOne').resolves( + previousCatalog && previousCatalog.bindPath + ? { bindPath: previousCatalog.bindPath, permissions: previousCatalog.permissions || 'ro' } + : null + ) + sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves( + previousCatalog && Array.isArray(previousCatalog.items) + ? previousCatalog.items.map((item, id) => ({ id, name: item.name })) + : [] + ) + sandbox.stub(MicroserviceModelManager, 'delete').resolves() + sandbox.stub(MicroserviceModelManager, 'create').resolves() + sandbox.stub(MicroserviceModelItemManager, 'delete').resolves() + sandbox.stub(MicroserviceModelItemManager, 'create').resolves() + sandbox.stub(MicroserviceKnowledgeManager, 'findOne').resolves(null) + sandbox.stub(MicroserviceKnowledgeItemManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceKnowledgeManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeManager, 'create').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'create').resolves() + } + + it('rebuilds when the catalog goes from empty to non-empty', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing) + + await $service.updateMicroserviceCatalogEndPoint(msvcUuid, { + bindPath: '/models', + items: [{ name: 'test-model' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceModels, + transaction + ) + expect(MicroserviceModelItemManager.create).to.have.been.calledWith( + sinon.match({ name: 'test-model' }), + transaction + ) + }) + + it('rebuilds when the catalog goes from non-empty to empty', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceCatalogEndPoint(msvcUuid, { + items: [] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + }) + + it('rebuilds when bindPath changes', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceCatalogEndPoint(msvcUuid, { + bindPath: '/opt/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + }) + + it('rebuilds when permissions change', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceCatalogEndPoint(msvcUuid, { + bindPath: '/models', + permissions: 'rw', + items: [{ name: 'test-model' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + }) + + it('does not rebuild when only items change on a non-empty catalog', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing, existingCatalog) + FleetModelManager.findOne.callsFake(async (where) => ({ uuid: 'model-uuid', name: where.name })) + + await $service.updateMicroserviceCatalogEndPoint(msvcUuid, { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }, { name: 'qwen3-8-27b' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: false }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceModels, + transaction + ) + }) + + it('rejects a model bindPath that matches the knowledge catalog', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubCatalogPatch($sandbox, existing, existingCatalog) + MicroserviceKnowledgeManager.findOne.resolves({ bindPath: '/models', permissions: 'ro' }) + MicroserviceKnowledgeItemManager.findAll.resolves([{ id: 1, name: 'product-docs' }]) + + return expect($service.updateMicroserviceCatalogEndPoint(msvcUuid, { + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError) + }) + }) + + describe('.updateMicroserviceKnowledgeEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const existingCatalog = { + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + } + + function stubKnowledgePatch (sandbox, existing, previousCatalog = null) { + sandbox.stub(MicroserviceManager, 'findOneWithCategory').resolves(existing) + sandbox.stub(VolumeMappingManager, 'findAll').resolves([]) + sandbox.stub(FleetKnowledgeManager, 'findOne').resolves({ uuid: 'knowledge-uuid', name: 'product-docs' }) + sandbox.stub(KnowledgeService, 'ensureKnowledgeLinkedToFog').resolves([]) + sandbox.stub(MicroserviceManager, 'update').resolves() + sandbox.stub(ChangeTrackingService, 'update').resolves() + sandbox.stub(MicroserviceTmpfsManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceModelManager, 'findOne').resolves(null) + sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves([]) + sandbox.stub(MicroserviceKnowledgeManager, 'findOne').resolves( + previousCatalog && previousCatalog.bindPath + ? { bindPath: previousCatalog.bindPath, permissions: previousCatalog.permissions || 'ro' } + : null + ) + sandbox.stub(MicroserviceKnowledgeItemManager, 'findAll').resolves( + previousCatalog && Array.isArray(previousCatalog.items) + ? previousCatalog.items.map((item, id) => ({ id, name: item.name })) + : [] + ) + sandbox.stub(MicroserviceKnowledgeManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeManager, 'create').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'delete').resolves() + sandbox.stub(MicroserviceKnowledgeItemManager, 'create').resolves() + } + + it('rebuilds when the catalog goes from empty to non-empty and flags knowledge only', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubKnowledgePatch($sandbox, existing) + + await $service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceKnowledge, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + }) + + it('rebuilds when the last item is removed', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubKnowledgePatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + items: [] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(MicroserviceKnowledgeManager.create).to.not.have.been.called + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceKnowledge, + transaction + ) + }) + + it('does not rebuild when only items change on a non-empty catalog', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubKnowledgePatch($sandbox, existing, existingCatalog) + FleetKnowledgeManager.findOne.callsFake(async (where) => ({ uuid: 'knowledge-uuid', name: where.name })) + + await $service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }, { name: 'wiki-faiss' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: false }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceKnowledge, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + expect(KnowledgeService.ensureKnowledgeLinkedToFog).to.have.been.calledWith( + existing.iofogUuid, + ['product-docs', 'wiki-faiss'], + transaction + ) + }) + + it('rebuilds when bindPath changes and flags knowledge only', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubKnowledgePatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/data', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceKnowledge, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + }) + + it('rebuilds when permissions change and flags knowledge only', async () => { + const existing = buildMicroserviceRecord({ rebuild: false }) + stubKnowledgePatch($sandbox, existing, existingCatalog) + + await $service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/knowledge', + permissions: 'rw', + items: [{ name: 'product-docs' }] + }, isCLI, transaction) + + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + sinon.match({ rebuild: true }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledOnceWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceKnowledge, + transaction + ) + expect(ChangeTrackingService.update).to.not.have.been.calledWith( + existing.iofogUuid, + ChangeTrackingService.events.microserviceList, + transaction + ) + }) + + it('rejects a system microservice', async () => { + stubKnowledgePatch($sandbox, buildMicroserviceRecord({ + catalogItem: { category: 'SYSTEM' } + })) + + return expect($service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError) + }) + + it('rejects a controller microservice', async () => { + stubKnowledgePatch($sandbox, buildMicroserviceRecord({ isController: true })) + + return expect($service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + bindPath: '/knowledge', + items: [{ name: 'product-docs' }] + }, isCLI, transaction)).to.be.rejectedWith(Errors.ValidationError) + }) + + it('returns not found when the microservice is missing', async () => { + $sandbox.stub(MicroserviceManager, 'findOneWithCategory').resolves(null) + + return expect($service.updateMicroserviceKnowledgeEndPoint(msvcUuid, { + items: [] + }, isCLI, transaction)).to.be.rejectedWith(Errors.NotFoundError) + }) + }) + + describe('.buildGetMicroserviceResponse() observed status', () => { + def('subject', () => $service.buildGetMicroserviceResponse($microservice, transaction)) + def('microservice', () => ({ + uuid: 'msvc-uuid', + name: 'test-msvc', + applicationId: 42, + logSize: 1024 + })) + + beforeEach(() => { + stubBuildGetResponseDeps($sandbox) + $sandbox.stub(ApplicationManager, 'findOne').resolves({ name: 'my-app' }) + }) + + it('returns the knowledge catalog with item names only', async () => { + MicroserviceKnowledgeManager.findOne.resolves({ bindPath: '/knowledge', permissions: 'rw' }) + MicroserviceKnowledgeItemManager.findAll.resolves([{ id: 2, name: 'product-docs' }]) + + const result = await $subject + expect(result.knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'rw', + items: [{ name: 'product-docs' }] + }) + expect(result.knowledge.items[0]).to.not.have.property('uuid') + }) + + it('zeros CPU when status is STOPPED', async () => { + MicroserviceStatusManager.findAllExcludeFields.resolves([{ + status: 'STOPPED', + cpuUsage: 12.5, + memoryUsage: 64, + startTime: 9, + operatingDuration: 10, + containerId: 'ctr-keep' + }]) + + const result = await $subject + expect(result.status).to.include({ + status: 'STOPPED', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + containerId: 'ctr-keep' + }) + }) + + it('zeros CPU when status is STOPPING', async () => { + MicroserviceStatusManager.findAllExcludeFields.resolves([{ + status: 'STOPPING', + cpuUsage: 12.5, + memoryUsage: 64, + startTime: 9, + operatingDuration: 10, + containerId: 'ctr-keep' + }]) + + const result = await $subject + expect(result.status).to.include({ + status: 'STOPPING', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0, + containerId: 'ctr-keep' + }) + }) + + it('keeps RUNNING meters', async () => { + MicroserviceStatusManager.findAllExcludeFields.resolves([{ + status: 'RUNNING', + cpuUsage: 12.5, + memoryUsage: 64, + startTime: 9, + operatingDuration: 10 + }]) + + const result = await $subject + expect(result.status).to.include({ + status: 'RUNNING', + cpuUsage: 12.5, + memoryUsage: 64 + }) + }) + + it('returns lastError, lastErrorAt, and restartCount on GET', async () => { + MicroserviceStatusManager.findAllExcludeFields.resolves([{ + status: 'RUNNING', + cpuUsage: 1, + errorMessage: '', + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123, + restartCount: 4 + }]) + + const result = await $subject + expect(result.status).to.include({ + errorMessage: '', + lastError: 'exitCode=1 oomKilled=false', + lastErrorAt: 1726660000123, + restartCount: 4 + }) + }) + + it('returns empty last crash fields when the row has none', async () => { + MicroserviceStatusManager.findAllExcludeFields.resolves([{ + status: 'STOPPED', + cpuUsage: 0, + memoryUsage: 0 + }]) + + const result = await $subject + expect(result.status).to.include({ + lastError: '', + lastErrorAt: 0, + restartCount: 0 + }) + }) + }) + + describe('.stopMicroserviceEndPoint()', () => { + const msvcUuid = 'msvc-uuid' + const microservice = buildMicroserviceRecord({ + uuid: msvcUuid, + catalogItem: { category: 'USER' } + }) + + def('subject', () => $service.stopMicroserviceEndPoint(msvcUuid, isCLI, transaction)) + + beforeEach(() => { + $sandbox.stub(MicroserviceManager, 'findOneWithCategory').resolves(microservice) + $sandbox.stub(MicroserviceManager, 'update').resolves() + $sandbox.stub(MicroserviceStatusManager, 'update').resolves() + $sandbox.stub(MicroserviceExecStatusManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('sets observed STOPPING with zero meters and inactive exec', async () => { + const result = await $subject + expect(MicroserviceManager.update).to.have.been.calledWith( + { uuid: msvcUuid }, + { isActivated: false }, + transaction + ) + expect(MicroserviceStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [msvcUuid] }, + sinon.match({ + status: 'STOPPING', + cpuUsage: 0, + memoryUsage: 0, + startTime: 0, + operatingDuration: 0 + }), + transaction + ) + expect(MicroserviceExecStatusManager.update).to.have.been.calledWith( + { microserviceUuid: [msvcUuid] }, + sinon.match({ status: 'INACTIVE', execSessionId: '' }), + transaction + ) + expect(result).to.eql({ uuid: msvcUuid, isActivated: false }) + }) + }) + + describe('.deleteNotRunningMicroservices()', () => { + def('subject', () => $service.deleteNotRunningMicroservices({ uuid: 'fog-uuid' }, transaction)) + + beforeEach(() => { + $sandbox.stub(MicroserviceManager, 'findAllWithStatuses') + $sandbox.stub($service, 'deleteMicroserviceWithRoutesAndPortMappings').resolves() + }) + + it('awaits deletes for not-running flagged microservices', async () => { + MicroserviceManager.findAllWithStatuses.resolves([ + { delete: true, microserviceStatus: { status: 'UNKNOWN' } }, + { delete: true, microserviceStatus: { status: 'STOPPING' } }, + { delete: true, microserviceStatus: { status: 'RUNNING' } }, + { delete: false, microserviceStatus: { status: 'DELETING' } }, + { delete: true, microserviceStatus: null } + ]) + + await $subject + + expect($service.deleteMicroserviceWithRoutesAndPortMappings).to.have.been.calledTwice + expect($service.deleteMicroserviceWithRoutesAndPortMappings.firstCall).to.have.been.calledWith( + sinon.match({ microserviceStatus: { status: 'UNKNOWN' } }), + transaction + ) + expect($service.deleteMicroserviceWithRoutesAndPortMappings.secondCall).to.have.been.calledWith( + sinon.match({ microserviceStatus: { status: 'STOPPING' } }), + transaction + ) + }) + }) }) diff --git a/test/src/services/model-service.test.js b/test/src/services/model-service.test.js new file mode 100644 index 000000000..bebc0494f --- /dev/null +++ b/test/src/services/model-service.test.js @@ -0,0 +1,327 @@ +'use strict' + +const { expect } = require('chai') +const sinon = require('sinon') + +const FleetModelManager = require('../../../src/data/managers/fleet-model-manager') +const RegistryManager = require('../../../src/data/managers/registry-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const MicroserviceModelItemManager = require('../../../src/data/managers/microservice-model-item-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') +const ChangeTrackingService = require('../../../src/services/change-tracking-service') +const ModelService = require('../../../src/services/model-service') +const AppHelper = require('../../../src/helpers/app-helper') +const Errors = require('../../../src/helpers/errors') + +describe('Model Service', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + const modelUuid = '3f2c8a1e-2b64-4c0d-9f11-0a1b2c3d4e5f' + const fogUuid = 'fog-1' + + afterEach(() => { + $sandbox.restore() + }) + + function stubModelRow (overrides = {}) { + const row = { + uuid: modelUuid, + name: 'test-model', + repo: 'org/repo', + revision: '', + registryId: 3, + files: ['file.gguf'], + format: 'gguf', + getFogs: $sandbox.stub().resolves([]), + ...overrides + } + return row + } + + describe('.createModelEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(AppHelper, 'generateUUID').returns(modelUuid) + $sandbox.stub(FleetModelManager, 'findOne').resolves(null) + $sandbox.stub(FleetModelManager, 'create').callsFake(async (data) => data) + $sandbox.stub(RegistryManager, 'findOne').resolves({ id: 3, type: 'hf' }) + }) + + it('creates a model with a server-generated uuid', async () => { + const result = await ModelService.createModelEndpoint({ + name: 'test-model', + repo: 'org/repo', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + }, transaction) + + expect(result).to.eql({ + uuid: modelUuid, + name: 'test-model', + repo: 'org/repo', + revision: '', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + }) + expect(FleetModelManager.create).to.have.been.calledOnce + }) + + it('rejects a duplicate name', async () => { + FleetModelManager.findOne.resolves(stubModelRow()) + + await expect(ModelService.createModelEndpoint({ + name: 'test-model', + repo: 'org/repo', + registryId: 3, + files: ['file.gguf'] + }, transaction)).to.be.rejectedWith(Errors.ConflictError, /already exists/) + }) + + it('requires files for a Hugging Face registry', async () => { + await expect(ModelService.createModelEndpoint({ + name: 'test-model', + repo: 'org/repo', + registryId: 3 + }, transaction)).to.be.rejectedWith(Errors.ValidationError, /non-empty files list/) + }) + + it('rejects files for an OCI registry', async () => { + RegistryManager.findOne.resolves({ id: 1, type: 'oci' }) + + await expect(ModelService.createModelEndpoint({ + name: 'oci-model', + repo: 'org/repo', + registryId: 1, + files: ['weights.bin'] + }, transaction)).to.be.rejectedWith(Errors.ValidationError, /must not specify files/) + }) + }) + + describe('.listModelsEndpoint() and .getModelEndpoint()', () => { + const row = { + uuid: modelUuid, + name: 'test-model', + repo: 'org/repo', + revision: 'abc', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + } + + beforeEach(() => { + $sandbox.stub(FleetModelManager, 'findAll').resolves([row]) + $sandbox.stub(FleetModelManager, 'findOne').resolves(row) + }) + + it('lists all fleet models', async () => { + const result = await ModelService.listModelsEndpoint(transaction) + expect(result).to.eql({ models: [row] }) + }) + + it('returns spec only without linked fogs', async () => { + const result = await ModelService.getModelEndpoint('test-model', transaction) + expect(result).to.eql(row) + expect(result).to.not.have.property('fogUuids') + expect(result).to.not.have.property('fogs') + }) + + it('throws when the model is missing', async () => { + FleetModelManager.findOne.resolves(null) + await expect(ModelService.getModelEndpoint('missing', transaction)) + .to.be.rejectedWith(Errors.NotFoundError, /missing/) + }) + }) + + describe('.updateModelEndpoint()', () => { + beforeEach(() => { + const row = stubModelRow() + $sandbox.stub(FleetModelManager, 'findOne').resolves(row) + $sandbox.stub(FleetModelManager, 'update').resolves() + $sandbox.stub(RegistryManager, 'findOne').resolves({ id: 3, type: 'hf' }) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('updates spec fields and notifies linked agents', async () => { + FleetModelManager.findOne.resolves(stubModelRow({ + getFogs: $sandbox.stub().resolves([{ uuid: fogUuid }]) + })) + + await ModelService.updateModelEndpoint('test-model', { + revision: 'main', + files: ['other.gguf'] + }, transaction) + + expect(FleetModelManager.update).to.have.been.calledWith( + { name: 'test-model' }, + sinon.match({ + repo: 'org/repo', + revision: 'main', + registryId: 3, + files: ['other.gguf'], + format: 'gguf' + }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.models, + transaction + ) + }) + }) + + describe('.deleteModelEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(FleetModelManager, 'findOne').resolves(stubModelRow()) + $sandbox.stub(FleetModelManager, 'delete').resolves() + $sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('deletes when no microservice binds the name', async () => { + const result = await ModelService.deleteModelEndpoint('test-model', transaction) + expect(result).to.eql({}) + expect(FleetModelManager.delete).to.have.been.calledWith({ name: 'test-model' }, transaction) + }) + + it('refuses delete when any microservice binds the name', async () => { + MicroserviceModelItemManager.findAll.resolves([ + { microserviceUuid: 'ms-1', name: 'test-model' } + ]) + + await expect(ModelService.deleteModelEndpoint('test-model', transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-1/) + expect(FleetModelManager.delete).to.not.have.been.called + }) + }) + + describe('link and unlink', () => { + let model + let agent + + beforeEach(() => { + model = stubModelRow() + agent = { + uuid: fogUuid, + addModel: $sandbox.stub().resolves(), + removeModel: $sandbox.stub().resolves() + } + $sandbox.stub(FleetModelManager, 'findOne').resolves(model) + $sandbox.stub(FogManager, 'findOne').resolves(agent) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + $sandbox.stub(MicroserviceModelItemManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('links a fog and sets the models change flag', async () => { + const result = await ModelService.linkModelEndpoint('test-model', [fogUuid], transaction) + expect(agent.addModel).to.have.been.calledWith(model, { transaction }) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.models, + transaction + ) + expect(result.name).to.equal('test-model') + }) + + it('does not set the change flag when the fog is already linked', async () => { + model.getFogs.resolves([{ uuid: fogUuid }]) + await ModelService.linkModelEndpoint('test-model', [fogUuid], transaction) + expect(ChangeTrackingService.update).to.not.have.been.called + }) + + it('returns linked fog uuids', async () => { + model.getFogs.resolves([{ uuid: fogUuid }]) + const result = await ModelService.getModelLinkEndpoint('test-model', transaction) + expect(result).to.eql({ fogUuids: [fogUuid] }) + }) + + it('unlinks a fog and sets the models change flag', async () => { + await ModelService.unlinkModelEndpoint('test-model', [fogUuid], transaction) + expect(agent.removeModel).to.have.been.calledWith(model, { transaction }) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.models, + transaction + ) + }) + + it('refuses unlink when a microservice on that fog binds the name', async () => { + MicroserviceModelItemManager.findAll.resolves([ + { microserviceUuid: 'ms-bound', name: 'test-model' } + ]) + $sandbox.stub(MicroserviceManager, 'findOne').resolves({ + uuid: 'ms-bound', + iofogUuid: fogUuid + }) + + await expect(ModelService.unlinkModelEndpoint('test-model', [fogUuid], transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-bound/) + expect(agent.removeModel).to.not.have.been.called + }) + }) + + describe('.upsertModelEndpoint()', () => { + it('creates when the name is new and updates when it already exists', async () => { + $sandbox.stub(AppHelper, 'generateUUID').returns(modelUuid) + $sandbox.stub(FleetModelManager, 'findOne').resolves(null) + $sandbox.stub(FleetModelManager, 'create').callsFake(async (data) => data) + $sandbox.stub(RegistryManager, 'findOne').resolves({ id: 3, type: 'hf' }) + + const created = await ModelService.upsertModelEndpoint('test-model', { + repo: 'org/repo', + registryId: 3, + files: ['file.gguf'] + }, transaction) + expect(created.uuid).to.equal(modelUuid) + expect(FleetModelManager.create).to.have.been.calledOnce + + const existing = stubModelRow({ + getFogs: $sandbox.stub().resolves([]) + }) + FleetModelManager.findOne.resolves(existing) + $sandbox.stub(FleetModelManager, 'update').resolves() + + await ModelService.upsertModelEndpoint('test-model', { + repo: 'org/other', + files: ['file.gguf'] + }, transaction) + expect(FleetModelManager.update).to.have.been.calledOnce + expect(FleetModelManager.create).to.have.been.calledOnce + }) + }) + + describe('.ensureModelsLinkedToFog()', () => { + let model + let agent + + beforeEach(() => { + model = stubModelRow() + agent = { + uuid: fogUuid, + hasModel: $sandbox.stub().resolves(false), + addModel: $sandbox.stub().resolves() + } + $sandbox.stub(FogManager, 'findOne').resolves(agent) + $sandbox.stub(FleetModelManager, 'findOne').resolves(model) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('links missing models and is idempotent when already linked', async () => { + const first = await ModelService.ensureModelsLinkedToFog(fogUuid, ['test-model'], transaction) + expect(first).to.eql(['test-model']) + expect(agent.addModel).to.have.been.calledOnce + expect(ChangeTrackingService.update).to.have.been.calledOnce + + agent.hasModel.resolves(true) + ChangeTrackingService.update.resetHistory() + agent.addModel.resetHistory() + + const second = await ModelService.ensureModelsLinkedToFog(fogUuid, ['test-model'], transaction) + expect(second).to.eql([]) + expect(agent.addModel).to.not.have.been.called + expect(ChangeTrackingService.update).to.not.have.been.called + }) + }) +}) diff --git a/test/src/services/registry-service.test.js b/test/src/services/registry-service.test.js index dae3da1a1..65fadb36d 100644 --- a/test/src/services/registry-service.test.js +++ b/test/src/services/registry-service.test.js @@ -23,6 +23,9 @@ function buildRegistryRecord (fields = {}) { password: 'encrypted-secret', isPublic: false, userEmail: 'user@example.com', + type: 'oci', + ca: null, + insecure: false, ...fields } } @@ -65,7 +68,10 @@ describe('Registry Service', () => { expect(RegistryManager.create).to.have.been.calledWithMatch({ url: registryData.url, username: registryData.username, - userEmail: registryData.email + userEmail: registryData.email, + type: 'oci', + ca: null, + insecure: false }, transaction) expect(SecretHelper.encryptSecretInternal).to.have.been.calledWith( { value: registryData.password }, @@ -100,6 +106,75 @@ describe('Registry Service', () => { expect(RegistryManager.update).to.not.have.been.called }) }) + + it('rejects private OCI registries without username and password', () => { + return expect($service.createRegistry({ + url: 'https://registry.example.com', + isPublic: false, + type: 'oci' + }, transaction)).to.be.rejectedWith(ErrorMessages.REGISTRY_PRIVATE_OCI_CREDENTIALS_REQUIRED) + }) + + it('creates a private Hugging Face registry with a token', async () => { + const hfRegistry = { + url: 'https://hf.example.com', + isPublic: false, + type: 'hf', + password: 'hf_token' + } + RegistryManager.create.resolves(buildRegistryRecord({ + id: 18, + url: hfRegistry.url, + type: 'hf', + password: hfRegistry.password, + isPublic: false, + username: '' + })) + + const result = await $service.createRegistry(hfRegistry, transaction) + + expect(RegistryManager.create).to.have.been.calledWithMatch({ + url: hfRegistry.url, + type: 'hf', + password: hfRegistry.password, + insecure: false + }, transaction) + expect(result).to.eql({ id: 18 }) + }) + + it('rejects creating a second Hub registry', () => { + return expect($service.createRegistry({ + url: 'https://huggingface.co', + isPublic: true, + type: 'hf' + }, transaction)).to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) + }) + + it('rejects an invalid CA bundle', () => { + return expect($service.createRegistry({ + url: 'https://registry.example.com', + isPublic: true, + ca: 'not-valid-base64!!!' + }, transaction)).to.be.rejectedWith(ErrorMessages.REGISTRY_INVALID_CA) + }) + + it('stores empty ca as null', async () => { + await $service.createRegistry({ + url: 'https://registry.example.com', + isPublic: true, + ca: '' + }, transaction) + expect(RegistryManager.create).to.have.been.calledWithMatch({ ca: null }, transaction) + }) + + it('stores a valid base64 ca bundle', async () => { + await $service.createRegistry({ + url: 'https://registry.example.com', + isPublic: true, + ca: 'Y2VydA==' + }, transaction) + expect(RegistryManager.create).to.have.been.calledWithMatch({ ca: 'Y2VydA==' }, transaction) + }) }) describe('.findRegistries()', () => { @@ -111,14 +186,24 @@ describe('Registry Service', () => { $sandbox.stub(RegistryManager, 'findAllWithAttributes').resolves(registries) }) - it('returns registries without password field', async () => { + it('returns registries without password field and with type, ca, and insecure defaults', async () => { const result = await $subject expect(RegistryManager.findAllWithAttributes).to.have.been.calledWith( {}, { exclude: ['password'] }, transaction ) - expect(result.registries).to.equal(registries) + expect(result.registries).to.eql([{ + id: 16, + url: 'https://registry.example.com', + username: 'user', + password: 'encrypted-secret', + isPublic: false, + userEmail: 'user@example.com', + type: 'oci', + ca: null, + insecure: false + }]) }) }) @@ -151,6 +236,19 @@ describe('Registry Service', () => { .to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) }) + it('rejects the Hub registry by url and type regardless of id', () => { + RegistryManager.findOne.resolves(buildRegistryRecord({ + id: 9, + url: 'https://huggingface.co', + type: 'hf', + isPublic: true, + username: '', + password: '' + })) + return expect($service.deleteRegistry({ id: 9 }, isCLI, transaction)) + .to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) + }) + context('when registry is missing', () => { beforeEach(() => { RegistryManager.findOne.resolves(null) @@ -209,6 +307,44 @@ describe('Registry Service', () => { .to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) }) + it('rejects Hub url or type changes that would break identity', async () => { + RegistryManager.findOne.resolves(buildRegistryRecord({ + id: 9, + url: 'https://huggingface.co', + type: 'hf', + isPublic: true + })) + await expect($service.updateRegistry({ url: 'https://example.com' }, 9, isCLI, transaction)) + .to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) + await expect($service.updateRegistry({ type: 'oci' }, 9, isCLI, transaction)) + .to.be.rejectedWith(ErrorMessages.REGISTRY_IS_SYSTEM) + }) + + it('clears ca to null when an empty string is provided', async () => { + await $service.updateRegistry({ ca: '' }, registryId, isCLI, transaction) + expect(RegistryManager.update).to.have.been.calledWith( + { id: registryId }, + sinon.match({ ca: null }), + transaction + ) + }) + + it('allows Hub token updates without changing identity fields', async () => { + RegistryManager.findOne.resolves(buildRegistryRecord({ + id: 9, + url: 'https://huggingface.co', + type: 'hf', + isPublic: true, + password: '' + })) + await $service.updateRegistry({ password: 'hf_token' }, 9, isCLI, transaction) + expect(RegistryManager.update).to.have.been.calledWith( + { id: 9 }, + sinon.match({ password: 'hf_token' }), + transaction + ) + }) + context('when registry is missing', () => { beforeEach(() => { RegistryManager.findOne.resolves(null) @@ -243,7 +379,7 @@ describe('Registry Service', () => { context('when password is cleared and vault reference exists', () => { beforeEach(() => { $sandbox.stub(vaultManager, 'isEnabled').returns(true) - RegistryManager.findOne.resolves({ ...existing, password: 'vault:ref' }) + RegistryManager.findOne.resolves({ ...existing, password: 'vault:ref', isPublic: true }) $sandbox.stub(SecretHelper, 'isVaultReference').returns(true) $sandbox.stub(SecretHelper, 'deleteSecret').resolves() }) @@ -268,10 +404,15 @@ describe('Registry Service', () => { $sandbox.stub(RegistryManager, 'findOne').resolves(registry) }) - it('returns the registry record', async () => { + it('returns the registry record with type, ca, and insecure', async () => { const result = await $subject expect(RegistryManager.findOne).to.have.been.calledWith({ id: registryId }, transaction) - expect(result).to.equal(registry) + expect(result).to.include({ + id: registryId, + type: 'oci', + ca: null, + insecure: false + }) }) context('when registry is missing', () => { @@ -282,4 +423,32 @@ describe('Registry Service', () => { it('rejects with NotFoundError', () => expect($subject).to.be.rejectedWith(Errors.NotFoundError)) }) }) + + describe('.assertOciRegistryForImage()', () => { + beforeEach(() => { + $sandbox.stub(RegistryManager, 'findOne') + }) + + it('accepts an OCI registry', async () => { + RegistryManager.findOne.resolves(buildRegistryRecord({ id: 4, type: 'oci' })) + const registry = await $service.assertOciRegistryForImage(4, transaction) + expect(registry.type).to.equal('oci') + }) + + it('rejects a Hugging Face registry id', () => { + RegistryManager.findOne.resolves(buildRegistryRecord({ + id: 3, + url: 'https://huggingface.co', + type: 'hf' + })) + return expect($service.assertOciRegistryForImage(3, transaction)) + .to.be.rejectedWith(AppHelper.formatMessage(ErrorMessages.REGISTRY_NOT_OCI_FOR_IMAGE, 3)) + }) + + it('rejects a missing registry', () => { + RegistryManager.findOne.resolves(null) + return expect($service.assertOciRegistryForImage(99, transaction)) + .to.be.rejectedWith(AppHelper.formatMessage(ErrorMessages.REGISTRY_NOT_OCI_FOR_IMAGE, 99)) + }) + }) }) diff --git a/test/src/services/runtime-class-service.test.js b/test/src/services/runtime-class-service.test.js new file mode 100644 index 000000000..4d99421ce --- /dev/null +++ b/test/src/services/runtime-class-service.test.js @@ -0,0 +1,294 @@ +'use strict' + +const { expect } = require('chai') +const sinon = require('sinon') + +const RuntimeClassManager = require('../../../src/data/managers/runtime-class-manager') +const MicroserviceManager = require('../../../src/data/managers/microservice-manager') +const FogManager = require('../../../src/data/managers/iofog-manager') +const ChangeTrackingService = require('../../../src/services/change-tracking-service') +const RuntimeClassService = require('../../../src/services/runtime-class-service') +const logger = require('../../../src/logger') +const Errors = require('../../../src/helpers/errors') + +describe('RuntimeClass Service', () => { + def('sandbox', () => sinon.createSandbox()) + const transaction = {} + const fogUuid = 'fog-1' + + afterEach(() => { + $sandbox.restore() + }) + + function stubRuntimeClassRow (overrides = {}) { + return { + name: 'spin', + handler: 'spin', + getFogs: $sandbox.stub().resolves([]), + addFog: $sandbox.stub().resolves(), + removeFog: $sandbox.stub().resolves(), + hasFog: $sandbox.stub().resolves(false), + ...overrides + } + } + + describe('.createRuntimeClassEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(null) + $sandbox.stub(RuntimeClassManager, 'create').callsFake(async (data) => data) + }) + + it('creates a runtime class by name', async () => { + const result = await RuntimeClassService.createRuntimeClassEndpoint({ + name: 'spin', + handler: 'spin' + }, transaction) + + expect(result).to.eql({ name: 'spin', handler: 'spin' }) + expect(RuntimeClassManager.create).to.have.been.calledOnce + }) + + it('rejects a duplicate name', async () => { + RuntimeClassManager.findOne.resolves(stubRuntimeClassRow()) + + await expect(RuntimeClassService.createRuntimeClassEndpoint({ + name: 'spin', + handler: 'spin' + }, transaction)).to.be.rejectedWith(Errors.ConflictError, /already exists/) + }) + }) + + describe('.listRuntimeClassesEndpoint() and .getRuntimeClassEndpoint()', () => { + const row = { name: 'spin', handler: 'spin' } + + beforeEach(() => { + $sandbox.stub(RuntimeClassManager, 'findAll').resolves([row]) + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(row) + }) + + it('lists all runtime classes', async () => { + const result = await RuntimeClassService.listRuntimeClassesEndpoint(transaction) + expect(result).to.eql({ runtimeClasses: [row] }) + }) + + it('returns spec only without linked fogs', async () => { + const result = await RuntimeClassService.getRuntimeClassEndpoint('spin', transaction) + expect(result).to.eql(row) + expect(result).to.not.have.property('fogUuids') + expect(result).to.not.have.property('fogs') + }) + + it('throws when the runtime class is missing', async () => { + RuntimeClassManager.findOne.resolves(null) + await expect(RuntimeClassService.getRuntimeClassEndpoint('missing', transaction)) + .to.be.rejectedWith(Errors.NotFoundError, /missing/) + }) + }) + + describe('.updateRuntimeClassEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(stubRuntimeClassRow()) + $sandbox.stub(RuntimeClassManager, 'update').resolves() + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('updates handler and notifies linked agents', async () => { + RuntimeClassManager.findOne.resolves(stubRuntimeClassRow({ + getFogs: $sandbox.stub().resolves([{ uuid: fogUuid }]) + })) + + await RuntimeClassService.updateRuntimeClassEndpoint('spin', { + handler: 'wasmtime' + }, transaction) + + expect(RuntimeClassManager.update).to.have.been.calledWith( + { name: 'spin' }, + sinon.match({ handler: 'wasmtime' }), + transaction + ) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.runtimeClasses, + transaction + ) + }) + }) + + describe('.deleteRuntimeClassEndpoint()', () => { + beforeEach(() => { + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(stubRuntimeClassRow()) + $sandbox.stub(RuntimeClassManager, 'delete').resolves() + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('deletes when no microservice pins the runtime name', async () => { + const result = await RuntimeClassService.deleteRuntimeClassEndpoint('spin', transaction) + expect(result).to.eql({}) + expect(RuntimeClassManager.delete).to.have.been.calledWith({ name: 'spin' }, transaction) + }) + + it('refuses delete when any microservice pins the runtime name', async () => { + MicroserviceManager.findAll.resolves([ + { uuid: 'ms-1', runtime: 'spin' } + ]) + + await expect(RuntimeClassService.deleteRuntimeClassEndpoint('spin', transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-1/) + expect(RuntimeClassManager.delete).to.not.have.been.called + }) + }) + + describe('link and unlink', () => { + let runtimeClass + let agent + + beforeEach(() => { + runtimeClass = stubRuntimeClassRow() + agent = { + uuid: fogUuid, + name: 'edge-1', + containerEngine: 'edgelet', + availableRuntimes: ['spin'] + } + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(runtimeClass) + $sandbox.stub(FogManager, 'findOne').resolves(agent) + $sandbox.stub(MicroserviceManager, 'findAll').resolves([]) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + $sandbox.stub(logger, 'warn') + }) + + it('links an edgelet fog and sets the runtimeClasses change flag', async () => { + const result = await RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction) + expect(runtimeClass.addFog).to.have.been.calledWith(agent, { transaction }) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.runtimeClasses, + transaction + ) + expect(result.name).to.equal('spin') + }) + + it('refuses link when the fog is not using edgelet', async () => { + agent.containerEngine = 'docker' + await expect(RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction)) + .to.be.rejectedWith(Errors.ValidationError, /edgelet/) + expect(runtimeClass.addFog).to.not.have.been.called + }) + + it('refuses link when the runtime is not in availableRuntimes', async () => { + agent.availableRuntimes = ['runc'] + await expect(RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction)) + .to.be.rejectedWith(Errors.ValidationError, /availableRuntimes/) + expect(runtimeClass.addFog).to.not.have.been.called + }) + + it('warns when the handler is not in the known catalog', async () => { + runtimeClass.handler = 'custom-oci' + await RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction) + expect(logger.warn).to.have.been.calledOnce + }) + + it('does not set the change flag when the fog is already linked', async () => { + runtimeClass.getFogs.resolves([{ uuid: fogUuid }]) + await RuntimeClassService.linkRuntimeClassEndpoint('spin', [fogUuid], transaction) + expect(ChangeTrackingService.update).to.not.have.been.called + }) + + it('returns linked fog uuids', async () => { + runtimeClass.getFogs.resolves([{ uuid: fogUuid }]) + const result = await RuntimeClassService.getRuntimeClassLinkEndpoint('spin', transaction) + expect(result).to.eql({ fogUuids: [fogUuid] }) + }) + + it('unlinks a fog and sets the runtimeClasses change flag', async () => { + await RuntimeClassService.unlinkRuntimeClassEndpoint('spin', [fogUuid], transaction) + expect(runtimeClass.removeFog).to.have.been.calledWith(agent, { transaction }) + expect(ChangeTrackingService.update).to.have.been.calledWith( + fogUuid, + ChangeTrackingService.events.runtimeClasses, + transaction + ) + }) + + it('refuses unlink when a microservice on that fog pins the runtime name', async () => { + MicroserviceManager.findAll.resolves([ + { uuid: 'ms-bound', iofogUuid: fogUuid, runtime: 'spin' } + ]) + + await expect(RuntimeClassService.unlinkRuntimeClassEndpoint('spin', [fogUuid], transaction)) + .to.be.rejectedWith(Errors.ConflictError, /ms-bound/) + expect(runtimeClass.removeFog).to.not.have.been.called + }) + }) + + describe('.upsertRuntimeClassEndpoint()', () => { + it('creates when the name is new and updates when it already exists', async () => { + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(null) + $sandbox.stub(RuntimeClassManager, 'create').callsFake(async (data) => data) + + const created = await RuntimeClassService.upsertRuntimeClassEndpoint('spin', { + handler: 'spin' + }, transaction) + expect(created.name).to.equal('spin') + expect(RuntimeClassManager.create).to.have.been.calledOnce + + const existing = stubRuntimeClassRow({ + getFogs: $sandbox.stub().resolves([]) + }) + RuntimeClassManager.findOne.resolves(existing) + $sandbox.stub(RuntimeClassManager, 'update').resolves() + + await RuntimeClassService.upsertRuntimeClassEndpoint('spin', { + handler: 'wasmtime' + }, transaction) + expect(RuntimeClassManager.update).to.have.been.calledOnce + expect(RuntimeClassManager.create).to.have.been.calledOnce + }) + }) + + describe('.ensureRuntimeClassLinkedToFog()', () => { + let runtimeClass + let agent + + beforeEach(() => { + runtimeClass = stubRuntimeClassRow() + agent = { + uuid: fogUuid, + containerEngine: 'edgelet' + } + $sandbox.stub(FogManager, 'findOne').resolves(agent) + $sandbox.stub(RuntimeClassManager, 'findOne').resolves(runtimeClass) + $sandbox.stub(ChangeTrackingService, 'update').resolves() + }) + + it('links a missing class on an edgelet fog and is idempotent when already linked', async () => { + const first = await RuntimeClassService.ensureRuntimeClassLinkedToFog(fogUuid, 'spin', transaction) + expect(first).to.equal(true) + expect(runtimeClass.addFog).to.have.been.calledOnce + expect(ChangeTrackingService.update).to.have.been.calledOnce + + runtimeClass.hasFog.resolves(true) + ChangeTrackingService.update.resetHistory() + runtimeClass.addFog.resetHistory() + + const second = await RuntimeClassService.ensureRuntimeClassLinkedToFog(fogUuid, 'spin', transaction) + expect(second).to.equal(false) + expect(runtimeClass.addFog).to.not.have.been.called + expect(ChangeTrackingService.update).to.not.have.been.called + }) + + it('does not link when the fog is not using edgelet', async () => { + agent.containerEngine = 'docker' + const linked = await RuntimeClassService.ensureRuntimeClassLinkedToFog(fogUuid, 'spin', transaction) + expect(linked).to.equal(false) + expect(runtimeClass.addFog).to.not.have.been.called + }) + + it('rejects when the runtime class row is missing', async () => { + RuntimeClassManager.findOne.resolves(null) + await expect(RuntimeClassService.ensureRuntimeClassLinkedToFog(fogUuid, 'spin', transaction)) + .to.be.rejectedWith(Errors.ValidationError, /does not exist/) + }) + }) +}) diff --git a/test/src/services/yaml-parser-service.test.js b/test/src/services/yaml-parser-service.test.js index 7c6fc0f64..fd9e5f15e 100644 --- a/test/src/services/yaml-parser-service.test.js +++ b/test/src/services/yaml-parser-service.test.js @@ -21,6 +21,57 @@ spec: natsRule: 'app-rule' }) }) + + it('parses nested microservice serviceAccount', async () => { + const yaml = ` +kind: Application +metadata: + name: app-a +spec: + microservices: + - name: ms-a + serviceAccount: + roleRef: + kind: Role + name: custom-role + apiGroup: edgelet.iofog.org/v1 + container: + env: [] +` + const result = await YamlParserService.parseAppFile(yaml) + expect(result.microservices).to.have.length(1) + expect(result.microservices[0].serviceAccount).to.eql({ + roleRef: { + kind: 'Role', + name: 'custom-role', + apiGroup: 'edgelet.iofog.org/v1' + } + }) + }) + + it('copies knowledge from an inline microservice', async () => { + const yaml = ` +kind: Application +metadata: + name: app-a +spec: + microservices: + - name: ms-a + knowledge: + bindPath: /knowledge + permissions: rw + items: + - name: product-docs + container: + env: [] +` + const result = await YamlParserService.parseAppFile(yaml) + expect(result.microservices[0].knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'rw', + items: [{ name: 'product-docs' }] + }) + }) }) describe('.parseMicroserviceFile()', () => { @@ -45,6 +96,30 @@ spec: expect(result.name).to.eql('ms-a') }) + it('parses microservice serviceAccount', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-a +spec: + serviceAccount: + roleRef: + kind: Role + name: custom-role + apiGroup: edgelet.iofog.org/v1 + container: + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.serviceAccount).to.eql({ + roleRef: { + kind: 'Role', + name: 'custom-role', + apiGroup: 'edgelet.iofog.org/v1' + } + }) + }) + it('parses microservice natsEnabled', async () => { const yaml = ` kind: Microservice @@ -84,6 +159,128 @@ spec: ]) }) + it('maps spec.models and prefers container.commands over cmd', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-models +spec: + models: + bindPath: /models + permissions: ro + items: + - name: test-model + container: + commands: + - python + - app.py + cmd: + - ignored + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.models).to.eql({ + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }) + expect(result.commands).to.eql(['python', 'app.py']) + expect(result.cmd).to.eql(['python', 'app.py']) + expect(result).to.not.have.property('volumes') + }) + + it('maps spec.knowledge next to models', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-knowledge +spec: + knowledge: + bindPath: /knowledge + permissions: ro + items: + - name: product-docs + container: + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }) + }) + + it('maps container devices, tmpfs, sysctls, and ulimits', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-container +spec: + container: + sysctls: + net.ipv4.tcp_syncookies: 1 + ulimits: + nofile: + soft: 65536 + hard: 65536 + devices: + - hostPath: /dev/null + containerPath: /dev/null + permissions: rwm + tmpfs: + - containerPath: /tmp + size: 64 + volumes: + - hostDestination: /data + containerDestination: /data + accessMode: rw + type: bind + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.sysctls).to.eql({ 'net.ipv4.tcp_syncookies': '1' }) + expect(result.ulimits).to.eql({ nofile: { soft: 65536, hard: 65536 } }) + expect(result.devices).to.eql([{ + hostPath: '/dev/null', + containerPath: '/dev/null', + permissions: 'rwm' + }]) + expect(result.tmpfs).to.eql([{ containerPath: '/tmp', size: 64 }]) + expect(result.volumeMappings).to.eql([{ + hostDestination: '/data', + containerDestination: '/data', + accessMode: 'rw', + type: 'bind' + }]) + expect(result).to.not.have.property('volumes') + }) + + it('passes container volume scope through to volumeMappings', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-shared-volume +spec: + container: + volumes: + - hostDestination: nodered-config + containerDestination: /data + accessMode: rw + type: volume + scope: shared + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.volumeMappings).to.eql([{ + hostDestination: 'nodered-config', + containerDestination: '/data', + accessMode: 'rw', + type: 'volume', + scope: 'shared' + }]) + }) + it('does not map deprecated top-level natsAccess', async () => { const yaml = ` kind: Microservice @@ -97,6 +294,72 @@ spec: const result = await YamlParserService.parseMicroserviceFile(yaml) expect(result.natsAccess).to.eql(undefined) }) + + it('maps spec.template with object variables', async () => { + const yaml = ` +kind: Microservice +metadata: + name: whisper-instance-1 +spec: + application: test-app + agent: + name: edge-node-1 + template: + name: whisper-infer + variables: + model: llama-7b + container: + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.name).to.eql('whisper-instance-1') + expect(result.application).to.eql('test-app') + expect(result.agentName).to.eql('edge-node-1') + expect(result.template).to.eql({ + name: 'whisper-infer', + variables: { model: 'llama-7b' } + }) + }) + + it('maps spec.template with array variables', async () => { + const yaml = ` +kind: Microservice +metadata: + name: test-app/whisper-instance-2 +spec: + template: + name: whisper-infer + variables: + - key: model + value: llama-7b + container: + env: [] +` + const result = await YamlParserService.parseMicroserviceFile(yaml) + expect(result.name).to.eql('whisper-instance-2') + expect(result.application).to.eql('test-app') + expect(result.template).to.eql({ + name: 'whisper-infer', + variables: [{ key: 'model', value: 'llama-7b' }] + }) + }) + + it('rejects spec.template without name', async () => { + const yaml = ` +kind: Microservice +metadata: + name: app-a/ms-template +spec: + template: + variables: + model: llama-7b + container: + env: [] +` + await expect(YamlParserService.parseMicroserviceFile(yaml)).to.be.rejectedWith( + 'template.name is required when template is specified' + ) + }) }) describe('NATS rule YAML parsing', () => { @@ -136,4 +399,376 @@ spec: }) }) }) + + describe('.parseModelFile()', () => { + it('parses metadata.name and spec fields matching the wire object', async () => { + const yaml = ` +apiVersion: datasance.com/v3 +kind: Model +metadata: + name: test-model +spec: + repo: org/repo + revision: abc123 + registryId: 3 + files: + - file.gguf + format: gguf +` + const result = await YamlParserService.parseModelFile(yaml) + expect(result).to.eql({ + name: 'test-model', + repo: 'org/repo', + revision: 'abc123', + registryId: 3, + files: ['file.gguf'], + format: 'gguf' + }) + }) + + it('accepts spec.registry as an alias for registryId', async () => { + const yaml = ` +apiVersion: iofog.org/v3 +kind: Model +metadata: + name: test-model +spec: + repo: org/repo + registry: 3 + files: + - file.gguf +` + const result = await YamlParserService.parseModelFile(yaml) + expect(result.registryId).to.equal(3) + }) + + it('omits name on update and rejects a path mismatch', async () => { + const yaml = ` +kind: Model +metadata: + name: test-model +spec: + repo: org/repo + registryId: 3 +` + const updated = await YamlParserService.parseModelFile(yaml, { + isUpdate: true, + modelName: 'test-model' + }) + expect(updated).to.not.have.property('name') + expect(updated.repo).to.equal('org/repo') + + await expect(YamlParserService.parseModelFile(yaml, { + isUpdate: true, + modelName: 'other-model' + })).to.be.rejectedWith(/doesn't match endpoint path/) + }) + }) + + describe('.parseKnowledgeFile()', () => { + it('parses kind Knowledge into the wire object', async () => { + const yaml = ` +apiVersion: datasance.com/v3 +kind: Knowledge +metadata: + name: product-docs +spec: + repo: acme/product-manuals + revision: abc123 + registryId: 3 + files: + - data/**/*.jsonl + format: JSONL +` + const result = await YamlParserService.parseKnowledgeFile(yaml) + expect(result).to.eql({ + name: 'product-docs', + repo: 'acme/product-manuals', + revision: 'abc123', + registryId: 3, + files: ['data/**/*.jsonl'], + format: 'JSONL' + }) + }) + + it('accepts spec.registry as an alias for registryId', async () => { + const yaml = ` +apiVersion: iofog.org/v3 +kind: Knowledge +metadata: + name: product-docs +spec: + repo: acme/product-manuals + registry: 3 + files: + - data/**/*.jsonl +` + const result = await YamlParserService.parseKnowledgeFile(yaml) + expect(result.registryId).to.equal(3) + expect(result).to.not.have.property('labels') + }) + + it('omits name on upsert and rejects a path mismatch', async () => { + const yaml = ` +kind: Knowledge +metadata: + name: product-docs +spec: + repo: acme/product-manuals + registryId: 3 +` + const updated = await YamlParserService.parseKnowledgeFile(yaml, { + isUpdate: true, + knowledgeName: 'product-docs' + }) + expect(updated).to.not.have.property('name') + expect(updated.repo).to.equal('acme/product-manuals') + + await expect(YamlParserService.parseKnowledgeFile(yaml, { + isUpdate: true, + knowledgeName: 'other-docs' + })).to.be.rejectedWith(/doesn't match endpoint path/) + }) + }) + + describe('.parseRuntimeClassFile()', () => { + it('parses metadata.name and top-level handler matching the wire object', async () => { + const yaml = ` +apiVersion: datasance.com/v3 +kind: RuntimeClass +metadata: + name: spin +handler: spin +` + const result = await YamlParserService.parseRuntimeClassFile(yaml) + expect(result).to.eql({ + name: 'spin', + handler: 'spin' + }) + }) + + it('accepts iofog.org/v3 apiVersion', async () => { + const yaml = ` +apiVersion: iofog.org/v3 +kind: RuntimeClass +metadata: + name: nvidia-cdi +handler: nvidia-cdi +` + const result = await YamlParserService.parseRuntimeClassFile(yaml) + expect(result.handler).to.equal('nvidia-cdi') + }) + + it('accepts spec.handler as a fallback', async () => { + const yaml = ` +kind: RuntimeClass +metadata: + name: spin +spec: + handler: spin +` + const result = await YamlParserService.parseRuntimeClassFile(yaml) + expect(result).to.eql({ name: 'spin', handler: 'spin' }) + }) + + it('rejects YAML without handler', async () => { + const yaml = ` +kind: RuntimeClass +metadata: + name: spin +` + await expect(YamlParserService.parseRuntimeClassFile(yaml)) + .to.be.rejectedWith(/handler is required/) + }) + + it('omits name on update and rejects a path mismatch', async () => { + const yaml = ` +kind: RuntimeClass +metadata: + name: spin +handler: wasmtime +` + const updated = await YamlParserService.parseRuntimeClassFile(yaml, { + isUpdate: true, + runtimeClassName: 'spin' + }) + expect(updated).to.not.have.property('name') + expect(updated.handler).to.equal('wasmtime') + + await expect(YamlParserService.parseRuntimeClassFile(yaml, { + isUpdate: true, + runtimeClassName: 'other-class' + })).to.be.rejectedWith(/doesn't match endpoint path/) + }) + }) + + describe('.parseMicroserviceTemplateFile()', () => { + it('parses kind, variables, and microservice catalog/container fields', async () => { + const yaml = ` +apiVersion: datasance.com/v3 +kind: MicroserviceTemplate +metadata: + name: nginx-edge +spec: + description: nginx at the edge + variables: + - key: port + description: listen port + defaultValue: "8080" + microservice: + images: + x86: nginx:alpine + models: + bindPath: /models + permissions: ro + items: + - name: test-model + knowledge: + bindPath: /knowledge + permissions: ro + items: + - name: product-docs + container: + commands: + - nginx + - -g + - daemon off; + env: + - key: PORT + value: "{{ port }}" +` + const result = await YamlParserService.parseMicroserviceTemplateFile(yaml) + expect(result.name).to.equal('nginx-edge') + expect(result.description).to.equal('nginx at the edge') + expect(result.variables).to.eql([ + { key: 'port', description: 'listen port', defaultValue: '8080' } + ]) + expect(result.microservice.images).to.eql([ + { archId: 1, containerImage: 'nginx:alpine' } + ]) + expect(result.microservice.models).to.eql({ + bindPath: '/models', + permissions: 'ro', + items: [{ name: 'test-model' }] + }) + expect(result.microservice.knowledge).to.eql({ + bindPath: '/knowledge', + permissions: 'ro', + items: [{ name: 'product-docs' }] + }) + expect(result.microservice.commands).to.eql(['nginx', '-g', 'daemon off;']) + expect(result.microservice.env).to.eql([{ key: 'PORT', value: '{{ port }}' }]) + expect(result.microservice).to.not.have.property('name') + expect(result.microservice).to.not.have.property('pidMode') + expect(result.microservice).to.not.have.property('healthCheck') + }) + + it('preserves template placeholders and typed variable defaults', async () => { + const yaml = ` +apiVersion: datasance.com/v3 +kind: MicroserviceTemplate +metadata: + name: ms-template2 +spec: + description: Template for creating microservices + variables: + - key: application + defaultValue: test-app + - key: agent-name + defaultValue: lima + - key: nats-access + defaultValue: true + - key: registry-id + defaultValue: 5 + - key: schedule + defaultValue: 50 + - key: shm-size + defaultValue: 1024 + microservice: + application: "{{application}}" + schedule: "{{schedule}}" + agent: + name: "{{agent-name}}" + natsConfig: + natsAccess: "{{nats-access}}" + natsRule: "{{nats-rule}}" + images: + registry: "{{registry-id}}" + arm64: "{{arm64-image}}" + amd64: "{{amd64-image}}" + models: + bindPath: "{{bind-path}}" + permissions: "{{permissions}}" + items: + - name: "{{model1}}" + container: + hostNetworkMode: false + shmSize: "{{shm-size}}" + commands: [] +` + const result = await YamlParserService.parseMicroserviceTemplateFile(yaml) + expect(result.variables).to.eql([ + { key: 'application', defaultValue: 'test-app' }, + { key: 'agent-name', defaultValue: 'lima' }, + { key: 'nats-access', defaultValue: true }, + { key: 'registry-id', defaultValue: 5 }, + { key: 'schedule', defaultValue: 50 }, + { key: 'shm-size', defaultValue: 1024 } + ]) + expect(result.microservice.application).to.equal('{{application}}') + expect(result.microservice.agentName).to.equal('{{agent-name}}') + expect(result.microservice.registryId).to.equal('{{registry-id}}') + expect(result.microservice.schedule).to.equal('{{schedule}}') + expect(result.microservice.shmSize).to.equal('{{shm-size}}') + expect(result.microservice.natsConfig).to.eql({ + natsAccess: '{{nats-access}}', + natsRule: '{{nats-rule}}' + }) + expect(result.microservice.models.items).to.eql([{ name: '{{model1}}' }]) + expect(result.microservice).to.not.have.property('pidMode') + expect(result.microservice).to.not.have.property('cmd') + }) + + it('accepts iofog.org/v3 apiVersion', async () => { + const yaml = ` +apiVersion: iofog.org/v3 +kind: MicroserviceTemplate +metadata: + name: nginx-edge +spec: + microservice: + images: + x86: nginx:alpine + container: + env: [] +` + const result = await YamlParserService.parseMicroserviceTemplateFile(yaml) + expect(result.name).to.equal('nginx-edge') + }) + + it('omits name on update and rejects a path mismatch', async () => { + const yaml = ` +kind: MicroserviceTemplate +metadata: + name: nginx-edge +spec: + microservice: + images: + x86: nginx:alpine + container: + env: [] +` + const updated = await YamlParserService.parseMicroserviceTemplateFile(yaml, { + isUpdate: true, + templateName: 'nginx-edge' + }) + expect(updated).to.not.have.property('name') + expect(updated.microservice.images[0].containerImage).to.equal('nginx:alpine') + + await expect(YamlParserService.parseMicroserviceTemplateFile(yaml, { + isUpdate: true, + templateName: 'other-template' + })).to.be.rejectedWith(/doesn't match endpoint path/) + }) + }) }) diff --git a/test/support/first-fog-sqlite-harness.js b/test/support/first-fog-sqlite-harness.js index 52c8df449..71e7f6b5a 100644 --- a/test/support/first-fog-sqlite-harness.js +++ b/test/support/first-fog-sqlite-harness.js @@ -1,6 +1,7 @@ 'use strict' const fs = require('fs') +const os = require('os') const path = require('path') const express = require('express') @@ -15,7 +16,8 @@ const ENV_KEYS = [ 'OIDC_BOOTSTRAP_ADMIN_USERNAME', 'OIDC_BOOTSTRAP_ADMIN_PASSWORD', 'CONTROL_PLANE', - 'NODE_ENV' + 'NODE_ENV', + 'SQLITE_STORAGE_DIR' ] function snapshotEnv (keys) { @@ -38,8 +40,13 @@ function applyEnv (values) { } } +function sqliteStorageDir () { + return process.env.SQLITE_STORAGE_DIR || + path.resolve(__dirname, '../../src/data/sqlite_files') +} + function sqliteStoragePath (dbName) { - return path.resolve(__dirname, '../../src/data/sqlite_files', dbName) + return path.join(sqliteStorageDir(), dbName) } function cleanupSqliteFiles (dbName) { @@ -98,10 +105,13 @@ async function driveReconcileUntilReady (fogUuid, { async function createFirstFogSqliteHarness () { const envSnapshot = snapshotEnv(ENV_KEYS) const dbName = `first-fog-int-${Date.now()}-${Math.random().toString(36).slice(2)}.sqlite` + const storageDir = path.join(os.tmpdir(), 'controller-integration-sqlite') + fs.mkdirSync(storageDir, { recursive: true }) applyEnv({ DB_PROVIDER: 'sqlite', DB_NAME: dbName, + SQLITE_STORAGE_DIR: storageDir, AUTH_MODE: 'embedded', CONTROLLER_PUBLIC_URL: 'http://controller.test', AUTH_INSECURE_ALLOW_HTTP: 'true',