diff --git a/.circleci/config.yml b/.circleci/config.yml index 955b6c6b..206de606 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ jobs: - checkout - run: name: install dependencies - command: pip3 install --user .[dev] + command: pip3 install --user ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt .[dev] - run: name: confirm black version command: black --version @@ -33,7 +33,7 @@ jobs: - checkout - run: name: install dependencies - command: pip3 install --user '.[dev]' + command: pip3 install --user ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt '.[dev]' - run: name: run tests command: python -m pytest @@ -60,7 +60,7 @@ jobs: - run: name: install dependencies command: > - pip3 install --user .[dev] + pip3 install --user ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt .[dev] - run: name: check version command: | @@ -92,7 +92,7 @@ jobs: at: ~/openaev-client - run: name: install dependencies - command: pip3 install --user . + command: pip3 install --user ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt . - run: name: install twine command: pip3 install twine diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d2944d28..a54f4e9f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | - uv venv && uv pip install -e '.[dev]' coverage + uv venv && uv pip install -e ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt -e '.[dev]' coverage - name: Run tests with coverage run: uv run coverage run -m pytest --junitxml="junit.xml" @@ -34,7 +34,7 @@ jobs: - name: Generate coverage XML run: uv run coverage xml -o coverage.xml - + - name: Upload coverage to Codecov if: ${{ !cancelled() && hashFiles('coverage.xml') != '' }} uses: codecov/codecov-action@v7 diff --git a/libs/sdk-requirements.txt b/libs/sdk-requirements.txt new file mode 100644 index 00000000..aefde822 --- /dev/null +++ b/libs/sdk-requirements.txt @@ -0,0 +1,7 @@ +# SDKs hosted in sibling repositories, required by pyoaev while it keeps the +# deprecation shims of its former daemon helpers. Install together with the +# local xtm-oaev-sdk, e.g.: +# pip install ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt '.[dev]' +# Switch to version pins once the SDKs are published on PyPI. +collectors-sdk @ git+https://github.com/OpenAEV-Platform/collectors.git@feat/303-sdk-init#subdirectory=libs/collectors-sdk +injectors-sdk @ git+https://github.com/OpenAEV-Platform/injectors.git@feat/303-sdk-init#subdirectory=libs/injectors-sdk diff --git a/libs/xtm-oaev-sdk/CHANGELOG.md b/libs/xtm-oaev-sdk/CHANGELOG.md new file mode 100644 index 00000000..abf7aa5d --- /dev/null +++ b/libs/xtm-oaev-sdk/CHANGELOG.md @@ -0,0 +1,38 @@ +# Changelog + +Every notable change to xtm-oaev-sdk lands here. Every seam change too, with the new +version of its seam +([seam-changes-are-versioned-contract-changes](docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md)). + +## [0.1.0] - Unreleased + +First version. The shared, client-free code of `pyoaev` 3.261014.1 moves out and into +this SDK. The former `pyoaev` paths keep working, each one firing a +`DeprecationWarning` ([migration guide](docs/migration_guide.md)). + +### Added + +- `common`: the `OpenAEVError` hierarchy, `AppLogger`, logging setup, JSON encoding, + SSL helpers, asset and security-domain enums, and the `BaseClient`, + `DaemonProtocol` and `AppLoggerProtocol` protocols. Seam `1.0.0`. +- `configuration`: `Configuration`, `ConfigurationHint`, `SettingsLoader`, the settings + models, sources and schema generator. Seam `1.0.0`. +- `contracts`: `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, + the contract elements and enums, and the credential references (`CredentialType`, + `ContractReferencedCredential`, `build_single_referenced_credential_element`, from + `pyoaev.credential`). Seam `1.0.0`. +- `signatures`: `SignatureManager`, `build_network_configs`, `build_marker`, the + matching types and the inbound I/O models. Seam `1.0.0`. +- `port_io`: the wire models sent to the platform. `PORT_IO_CONTRACT_VERSION` `1.0.0`. +- `daemons`: `BaseDaemon` and `PingAlive`. Seam `1.0.0`. +- `deprecation` (new, not from `pyoaev`): `make_deprecated_getattr`, the PEP 562 + deprecation primitive. Seam `1.0.0`. + +### Changed (compared with pyoaev) + +- `ExpectationType` splits in two: `SignatureExpectationType` and + `ContractExpectationType`. +- `BaseDaemon` and `SignatureManager` stop building the API client. Pass + `api_client=` / `transport=` yourself. +- `requires-python` moves from `>=3.11` to `>=3.12`, the version `typing.override` + actually needs. Released `pyoaev` already failed to import on 3.11. diff --git a/libs/xtm-oaev-sdk/CLAUDE.md b/libs/xtm-oaev-sdk/CLAUDE.md new file mode 100644 index 00000000..c5444944 --- /dev/null +++ b/libs/xtm-oaev-sdk/CLAUDE.md @@ -0,0 +1,89 @@ +# xtm-oaev-sdk: agent instructions + +Applies to work under `libs/xtm-oaev-sdk/`, and to the `pyoaev/` deprecation shims when +code moves out of `pyoaev`. Paths below are relative to the repository root. + +xtm-oaev-sdk is the shared, client-free runtime of the OpenAEV connectors +(configuration, contracts, signatures, daemons, errors). `pyoaev` is the HTTP client +and serves the former names through deprecation shims. collectors-sdk and injectors-sdk +build on this SDK from their own repositories. + +## Hard rules + +1. NEVER break legacy `pyoaev` code. Every public name of the latest released `pyoaev` + (recorded in `test/fixtures/legacy_surface.json`) MUST keep resolving from its legacy path, returning the SDK object and emitting a + `DeprecationWarning` that names the new import. Why: connectors pin `pyoaev`, import + its old paths and patch them in their tests. +2. Respect the layering of every capability: + - `seam//` imports neither `public/` nor `_core/`; + - `_core//` NEVER imports `public/` (type-only imports go under `TYPE_CHECKING`); + - `public/.py` imports `_core/` ONLY to assign module-level defaults typed by + their port (`_resolver: ConfigurationResolverPort = ConfigurationResolver(...)`); + - every exported name is an adapter defined in `public/` or a contract defined in + `seam/`, NEVER a `_core` object. + + Why: the core can then be rewritten freely while the public API keeps working. +3. Every piece of behaviour goes through a seam `Protocol` port, one `_core` + implementation and a `public` adapter, one-liners included. No exceptions. +4. A public adapter MUST keep the signatures, docstrings, attributes, overridable hooks + and framework base class of the code it replaces. When the old code called a method on + `self`, pass that bound method to the core as a callable, so overrides and test + patches keep applying. +5. NEVER import `pyoaev`, `pika`, `thefuzz` or `requests` in the SDK. The API client is + injected (`BaseClient`, `SignatureTransportProtocol`). +6. Users import from the package root (`from xtm_oaev_sdk import X`). A new public name + goes in its `public/.py` `__all__` and in `public/__init__.py`. +7. A seam change (field, type, enum member, error, port signature) is a contract change: + bump that capability's `SEAM_VERSION` (`PORT_IO_CONTRACT_VERSION` for `seam/port_io`) + and add an entry to `libs/xtm-oaev-sdk/CHANGELOG.md`. Then regenerate the seam golden + in the same change: `python libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py`. + NEVER regenerate it to make a failing `test/seam/test_seam_golden.py` pass without a + deliberate seam change; follow `libs/xtm-oaev-sdk/CONTRIBUTING.md`, "Seam golden". +8. NEVER regenerate `test/fixtures/legacy_surface.json` from the working tree: it records + the released `pyoaev`. NEVER rewrite the legacy tests moved under + `libs/xtm-oaev-sdk/test//shim/`. + +## Scope + +Only make the changes directly requested. Do not add capabilities, abstractions, files, +dependencies or refactors beyond the task. + +## Stop and ask before + +- changing a seam contract or a `SEAM_VERSION`; +- renaming or removing a public name, or removing a `pyoaev` shim; +- adding or changing a dependency in any `pyproject.toml` or in `libs/sdk-requirements.txt`; +- editing `test/fixtures/legacy_surface.json`, the seam golden + `libs/xtm-oaev-sdk/test/seam/fixtures/seam_golden.json`, or CI configuration; +- deleting any file. + +## Done when + +Run from the repository root, and report the real output: + +```bash +isort --check-only . && black --check . && flake8 --ignore=E,W --extend-exclude=.venv . +python -m pytest libs/xtm-oaev-sdk/test test/test_legacy_surface.py test/test_sdk_split_shims.py +python -m pytest +``` + +All three pass. If a check fails, say which one and why. NEVER report a check as passing +without running it. + +## Conventions + +- Tests live in `libs/xtm-oaev-sdk/test//`, exercise behaviour through the + public API, and are written as `_given_*` / `_when_*` / `_then_*` helpers with a + one-line `"""It ..."""` docstring. Copy the style of the neighbouring tests. +- Adding a capability: follow the checklist at the end of `libs/xtm-oaev-sdk/docs/seam_strategy.md`. +- Recording a decision: add `libs/xtm-oaev-sdk/docs/tdr/.tdr.md` with the header of + an existing record and `status: draft`, plus a row in `docs/tdr/TDR_ToC.md`. +- Commit messages, when asked for: gitmoji + Conventional Commits with the `sdk` scope, + e.g. `:recycle: refactor(sdk): enhancing port-adapter public boundaries for configuration`. + +## Read when relevant + +- `libs/xtm-oaev-sdk/docs/seam_strategy.md`: layering patterns and capability map. +- `libs/xtm-oaev-sdk/docs/shim_strategy.md`: moving code out of `pyoaev` without breaking it. +- `libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md`: the decision behind each rule above. +- `libs/xtm-oaev-sdk/CONTRIBUTING.md`: setup and repository conventions. diff --git a/libs/xtm-oaev-sdk/CONTRIBUTING.md b/libs/xtm-oaev-sdk/CONTRIBUTING.md new file mode 100644 index 00000000..c270bf9d --- /dev/null +++ b/libs/xtm-oaev-sdk/CONTRIBUTING.md @@ -0,0 +1,172 @@ +# Contributing to xtm-oaev-sdk + +This guide is about what's specific to the SDKs. Everything repository-wide +(Conventional Commits titles with an issue reference, signed commits, labels, coverage) +lives in the repository's [CONTRIBUTING.md](../../CONTRIBUTING.md). `collectors-sdk` +and `injectors-sdk` run on this guide too, plus a short one of their own. + +## Set up + +From the repository root, on Python 3.12 or later, since the code uses `typing.override`: + +```bash +python -m venv .venv && . .venv/bin/activate +pip install -e ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt -e '.[dev]' +``` + +`libs/sdk-requirements.txt` pulls `collectors-sdk` and `injectors-sdk` from git, +because `pyoaev` depends on them. + +## Run the checks + +```bash +isort --check-only . +black --check . +flake8 --ignore=E,W --extend-exclude=.venv . +python -m pytest # pyoaev and xtm-oaev-sdk +python -m pytest libs/xtm-oaev-sdk/test # the SDK alone +``` + +CI runs the exact same four tools. Locally, `pre-commit run --files ` takes +care of isort, black and the file hygiene hooks. + +## Where code goes + +Every capability is cut in three: `public/` (adapters), `seam/` (contracts), `_core/` +(implementations). [docs/seam_strategy.md](docs/seam_strategy.md) lays out the layering +and the patterns that hold it together. The rules, each one nailed down as a TDR in +[docs/tdr](docs/tdr/TDR_ToC.md): + +- the seam imports neither `public` nor `_core`, `_core` never imports `public`, and + `public` touches `_core` only to wire default implementations + ([public-seam-core-dependency-rule](docs/tdr/public-seam-core-dependency-rule.tdr.md)); +- every piece of behaviour goes behind a port, one-liners included + ([every-behaviour-behind-a-port](docs/tdr/every-behaviour-behind-a-port.tdr.md)); +- adapters keep the signatures, attributes, hooks and base classes of what they replace + ([adapters-keep-the-user-facing-shape](docs/tdr/adapters-keep-the-user-facing-shape.tdr.md)); +- users import from the package root, nowhere else + ([import-from-the-package-root](docs/tdr/import-from-the-package-root.tdr.md)); +- the SDK never imports `pyoaev`, `pika`, `thefuzz` or `requests` + ([sdks-never-import-pyoaev](docs/tdr/sdks-never-import-pyoaev.tdr.md), + [xtm-oaev-sdk-stays-connector-agnostic](docs/tdr/xtm-oaev-sdk-stays-connector-agnostic.tdr.md)). + +`test/test_boundaries.py` catches the forbidden imports. + +Adding a capability? The checklist sits at the end of +[docs/seam_strategy.md](docs/seam_strategy.md). Follow it. + +## Changing a seam + +A seam change is a contract change. Treat it like one +([seam-changes-are-versioned-contract-changes](docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md)): + +1. bump the capability's `SEAM_VERSION` (`PORT_IO_CONTRACT_VERSION` for `seam/port_io`); +2. regenerate the seam golden in the same change, not in a follow-up (see + [Seam golden](#seam-golden-testseam)); +3. add an entry to [CHANGELOG.md](CHANGELOG.md); +4. if a public name changes or disappears, the old one keeps working through a + deprecation shim (`make_deprecated_getattr`) first. + +## Seam golden (`test/seam`) + +The seam is the contract between the public API and the core, and part of it rides the +wire: payloads sent to the platform, data coming back, the connector manifest. Every +change to it must be deliberate. `libs/xtm-oaev-sdk/test/seam/` fingerprints every +public object of every seam module, so any change fails a test until someone signs off +on it +([every-seam-object-is-fingerprinted](docs/tdr/every-seam-object-is-fingerprinted.tdr.md)). + +### What is in there + +| File | Role | +|---|---| +| `generate_seam_golden.py` | Finds the seam on its own: every `*.py` file under `xtm_oaev_sdk/seam/`, every public top-level definition (class, function, constant, type alias). It fingerprints each one, then rewrites the golden: changed fingerprints updated, new objects added, deleted objects removed. It prints what moved, and warns when a capability changed but its version did not. | +| `fixtures/seam_golden.json` | The golden: one entry per object, `".": {"kind", "sha256"}`. | +| `test_seam_golden.py` | Recomputes the fingerprints and fails on any object added, removed or changed. It also checks that the package root re-exports the seam's own objects. | +| `test_seam_closed.py` | Checks that `port_io` imports only the signatures seam, and that the signatures seam imports nothing from the SDK. | + +No list to maintain. The next run picks up any new seam object. + +collectors-sdk and injectors-sdk carry the same golden under their own `test/seam/`. There, +run the generator from `libs/`: `python test/seam/generate_seam_golden.py`. + +### What a fingerprint covers + +The sha256 of the definition's normalized AST. Formatting, comments and docstrings do +not count. So it moves with fields, defaults, enum members, method signatures, +validators and constant values. Pydantic models add their validation and serialization +JSON schemas on top, model and field descriptions included. Those reach the wire, so +rewording a pydantic model's docstring is a change. + +### Updating it on purpose + +From the repository root: + +1. make the seam change; +2. bump the capability's `SEAM_VERSION` (`PORT_IO_CONTRACT_VERSION` for `seam/port_io`); +3. regenerate the golden: + + ```bash + python libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py + ``` + +4. run `python -m pytest libs/xtm-oaev-sdk/test/seam`; +5. commit the seam change, the version bump and the golden in one go, with a + [CHANGELOG.md](CHANGELOG.md) entry. + +### Seeing what changed + +A failure names every object that moved, as `.`. `git diff` that +definition: the code diff is the change. A pydantic model can move with no code change +at all (a pydantic upgrade, a referenced type). Then compare its schema on both commits: + +```bash +python -c "import json; from xtm_oaev_sdk.seam.port_io import ExecutionDetails as M; print(json.dumps(M.model_json_schema(), indent=2, sort_keys=True))" +``` + +NEVER regenerate the golden just to turn a red test green. NEVER edit a hash by hand. If +a fingerprint moved with no seam change, find out why before you decide whether the +contract really changed. + +## Moving code out of pyoaev + +Legacy code keeps working. Non-negotiable +([no-breaking-change-for-legacy-pyoaev-code](docs/tdr/no-breaking-change-for-legacy-pyoaev-code.tdr.md)): + +- serve the old path with a shim, by the rules of [docs/shim_strategy.md](docs/shim_strategy.md); +- move the old tests verbatim into `libs/xtm-oaev-sdk/test//shim/`; +- `test/test_legacy_surface.py` stays green. Never regenerate its fixture from the + working tree: it records the released `pyoaev`, and that is the whole point. + +## Tests + +Tests live in `libs/xtm-oaev-sdk/test//` and go through the public API to +hit behaviour. Write them as `_given_` / `_when_` / `_then_` helpers, topped with a +one-line "It ..." docstring: + +```python +def test_configuration_resolves_with_override_priority(tmp_path, monkeypatch) -> None: + """It prioritizes override over env, file, and default.""" + module = _given_public_module() + ... + _when_set_configuration_value(configuration, "target", "override-value") +``` + +## Commits + +Repository conventions first. On top of them, the SDK history prefixes a gitmoji and +uses the `sdk` scope, one capability or concern per commit: + +``` +:recycle: refactor(sdk): enhancing port-adapter public boundaries for configuration +``` + +## Documentation + +- [README.md](README.md): the intent, and a quickstart; +- [docs/shim_strategy.md](docs/shim_strategy.md) and + [docs/seam_strategy.md](docs/seam_strategy.md): the how and the why; +- [docs/migration_guide.md](docs/migration_guide.md): getting connectors onto the SDKs; +- [docs/tdr](docs/tdr/TDR_ToC.md): the technical decision records. A new decision gets + its own `.tdr.md`, same header as the others, `draft` status, and a line in + `TDR_ToC.md`. diff --git a/libs/xtm-oaev-sdk/README.md b/libs/xtm-oaev-sdk/README.md new file mode 100644 index 00000000..964131f0 --- /dev/null +++ b/libs/xtm-oaev-sdk/README.md @@ -0,0 +1,104 @@ +# xtm-oaev-sdk + +The **common SDK** of the OpenAEV connector ecosystem. Everything collectors and +injectors share lives here, and none of it cares which HTTP client you run. + +`pyoaev` used to be a monolith. One package carried the OpenAEV API client *and* the +configuration system, the contract builder, the signature pipeline, the daemon +runtime, the error hierarchy and a drawer of assorted helpers. This SDK cuts the +shared, client-free half out of it. The stack now reads: + +``` +xtm-oaev-sdk shared vocabulary and runtime (this package) +├── collectors-sdk collector runtime (OpenAEV-Platform/collectors, libs/) +├── injectors-sdk injector runtime (OpenAEV-Platform/injectors, libs/) +└── pyoaev the OpenAEV HTTP client, plus deprecation shims of its former code +``` + +The SDK never depends on `pyoaev`. When it has to talk to the platform, the API +client gets **injected** (`BaseClient` protocol); `pyoaev.OpenAEV` is just one +implementation of it. + +## What's inside + +| Capability | Public API | +|---|---| +| common | `OpenAEVError` and its subclasses, `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, SSL helpers, asset and security-domain enums, `BaseClient` / `DaemonProtocol` / `AppLoggerProtocol` | +| configuration | `Configuration`, `ConfigurationHint`, `SettingsLoader`, `BaseConfigModel`, `ConfigLoaderOAEV`, `ConfigLoaderCollector`, `ConnectorConfigSchemaGenerator`, sources, serializer types | +| contracts | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, every contract element (`ContractText`, `ContractSelect`, …) and enum, credential references (`CredentialType`, `ContractReferencedCredential`, `build_single_referenced_credential_element`) | +| signatures | `SignatureManager`, `build_network_configs`, `build_marker`, matching types, the inbound I/O models and the port I/O wire models (`PORT_IO_CONTRACT_VERSION`) | +| daemons | `BaseDaemon`, `PingAlive` | +| deprecation | `make_deprecated_getattr` (PEP 562 deprecation primitive) | + +Import from the package root. Always. That is the stable API: + +```python +from xtm_oaev_sdk import Configuration, ContractBuilder, SignatureManager +``` + +## Quickstart + +```python +from xtm_oaev_sdk import ( + Configuration, + ContractBuilder, + ContractCheckbox, + ContractText, + SignatureManager, + build_network_configs, +) + +# Hint-based configuration: override, then env var, then config file, then default +config = Configuration({"log_level": {"env": "LOG_LEVEL", "default": "info"}}) +config.get("log_level") # 'info' + +# Contract fields, fluent builder +fields = ( + ContractBuilder() + .mandatory(ContractText(key="target", label="Target")) + .optional(ContractCheckbox(key="verbose", label="Verbose")) + .build_fields() +) + +# Signatures: the manager ships them through an injected transport +# (pyoaev's `OpenAEV().signature` is one) +manager = SignatureManager(transport=api.signature) +signatures = manager.build_execution_signatures(build_network_configs(["10.0.0.1"])) +``` + +## Architecture + +Every capability is cut into three layers, the same way across all three SDKs: + +``` +public/.py adapters: what users import; speak seam types, delegate to _core +seam// contracts: data models, enums, errors, Protocol ports (closed, versioned) +_core// implementations of the seam ports +``` + +`public` → `seam` ← `_core`. The seam imports neither of the other two, the core +never touches `public`, and `public` never hands a core object to users. Keep the +core answering the seam and the public API keeps working. Touch the seam and you +have changed a contract: `SEAM_VERSION` bump, changelog entry. The full wiring is in +[docs/seam_strategy.md](docs/seam_strategy.md). + +## Migrating from pyoaev + +Every former `pyoaev` path still works, and each one fires a `DeprecationWarning` +pointing at the new import. How that holds together: +[docs/shim_strategy.md](docs/shim_strategy.md). + +```python +from pyoaev.configuration import Configuration # DeprecationWarning, still works +from xtm_oaev_sdk import Configuration # new home +``` + +## Development + +```bash +pip install -e libs/xtm-oaev-sdk +python -m pytest libs/xtm-oaev-sdk/test +``` + +Python: 3.12 or later. The code uses `typing.override`, exactly as pyoaev already did. +Runtime dependencies: pydantic, pydantic-settings, PyYAML, python-json-logger. diff --git a/libs/xtm-oaev-sdk/docs/migration_guide.md b/libs/xtm-oaev-sdk/docs/migration_guide.md new file mode 100644 index 00000000..9c84d83f --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/migration_guide.md @@ -0,0 +1,323 @@ +# Migrating from pyoaev to the SDKs + +This guide is for you if your collector or injector still pulls shared code out of +`pyoaev`: configuration, contracts, signatures, daemons, helpers, errors. That code has +moved into three SDKs, and `pyoaev` is back to its real job: the OpenAEV API client. + +## Nothing breaks today + +Every former `pyoaev` path still works. Touch one and it fires a `DeprecationWarning` +naming the new import, then hands you the same object as before. The classes that used +to build the API client are the exception: they come back as a thin subclass that still builds the API client for you. +Migrate at your own pace, one import at a time. The legacy paths only disappear in a +future `pyoaev` major release. The machinery is laid out in +[shim_strategy.md](shim_strategy.md). + +## 1. Pick your dependencies + +| You write | Depend on | +|---|---| +| a collector | `collectors-sdk` and `pyoaev` | +| an injector | `injectors-sdk` and `pyoaev` | +| shared code only (configuration, contracts, signatures) | `xtm-oaev-sdk` | + +`collectors-sdk` and `injectors-sdk` drag `xtm-oaev-sdk` in on their own. Keep `pyoaev` +around for one thing: the API client (`OpenAEV`). + +```toml +dependencies = [ + "pyoaev (>=…)", + "collectors-sdk (>=0.1.0,<0.2.0)", # or injectors-sdk +] +``` + +Until the SDKs land on PyPI, pull them straight from their repositories, exactly like +`pyoaev`'s CI does in `libs/sdk-requirements.txt`: + +``` +xtm-oaev-sdk @ git+https://github.com/OpenAEV-Platform/client-python.git@#subdirectory=libs/xtm-oaev-sdk +collectors-sdk @ git+https://github.com/OpenAEV-Platform/collectors.git@#subdirectory=libs/collectors-sdk +injectors-sdk @ git+https://github.com/OpenAEV-Platform/injectors.git@#subdirectory=libs/injectors-sdk +``` + +## 2. Find the legacy imports + +Make the deprecation warnings fatal. Every one of them names its own replacement: + +```bash +python -m pytest -W error::DeprecationWarning +python -W error::DeprecationWarning -c "from pyoaev.helpers import OpenAEVInjectorHelper" +# DeprecationWarning: Use 'from injectors_sdk import OpenAEVInjectorHelper' and pass api_client=OpenAEV(...) instead. +``` + +Importing `pyoaev` itself stays silent. Only the moved names speak up. + +## 3. Change the imports + +Import from the package root (`xtm_oaev_sdk`, `collectors_sdk`, `injectors_sdk`). +Always. Every name of `pyoaev` that moved is in the tables below, next to +every legacy module that exposed it. + +### To xtm-oaev-sdk + +| Legacy name | Legacy modules | New import | +|---|---|---| +| `AppLogger` | `pyoaev.utils` | `from xtm_oaev_sdk import AppLogger` | +| `AssetCategory` | `pyoaev`, `pyoaev.asset_types` | `from xtm_oaev_sdk import AssetCategory` | +| `AssetCriticality` | `pyoaev`, `pyoaev.asset_types` | `from xtm_oaev_sdk import AssetCriticality` | +| `AssetSubCategory` | `pyoaev`, `pyoaev.asset_types` | `from xtm_oaev_sdk import AssetSubCategory` | +| `BaseConfigModel` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import BaseConfigModel` | +| `BaseDaemon` | `pyoaev.daemons`, `pyoaev.daemons.base_daemon`, `pyoaev.daemons.collector_daemon` | `from xtm_oaev_sdk import BaseDaemon` | +| `build_marker` | `pyoaev.signatures.ai_marker` | `from xtm_oaev_sdk import build_marker` | +| `build_network_configs` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models` | `from xtm_oaev_sdk import build_network_configs` | +| `build_single_referenced_credential_element` | `pyoaev.credential`, `pyoaev.credential.utils` | `from xtm_oaev_sdk import build_single_referenced_credential_element` | +| `CloudInjectorConfig` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import CloudInjectorConfig` | +| `CloudProvider` | `pyoaev`, `pyoaev.asset_types` | `from xtm_oaev_sdk import CloudProvider` | +| `ConfigLoaderCollector` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import ConfigLoaderCollector` | +| `ConfigLoaderOAEV` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import ConfigLoaderOAEV` | +| `Configuration` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.configuration`, `pyoaev.daemons.base_daemon`, `pyoaev.daemons.collector_daemon`, `pyoaev.helpers` | `from xtm_oaev_sdk import Configuration` | +| `ConfigurationError` | `pyoaev`, `pyoaev.exceptions`, `pyoaev.helpers` | `from xtm_oaev_sdk import ConfigurationError` | +| `ConfigurationHint` | `pyoaev.configuration.configuration` | `from xtm_oaev_sdk import ConfigurationHint` | +| `ConnectorConfigSchemaGenerator` | `pyoaev.configuration.configuration`, `pyoaev.configuration.connector_config_schema_generator` | `from xtm_oaev_sdk import ConnectorConfigSchemaGenerator` | +| `Contract` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import Contract` | +| `ContractAiTarget` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractAiTarget` | +| `ContractAsset` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractAsset` | +| `ContractAssetGroup` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractAssetGroup` | +| `ContractAttachment` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractAttachment` | +| `ContractBuilder` | `pyoaev`, `pyoaev.contracts`, `pyoaev.contracts.contract_builder` | `from xtm_oaev_sdk import ContractBuilder` | +| `ContractCardinality` | `pyoaev.contracts.contract_config`, `pyoaev.contracts.contract_utils`, `pyoaev.contracts.variable_helper` | `from xtm_oaev_sdk import ContractCardinality` | +| `ContractCardinalityElement` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractCardinalityElement` | +| `ContractCheckbox` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractCheckbox` | +| `ContractConfig` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractConfig` | +| `ContractElement` | `pyoaev.contracts.contract_builder`, `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractElement` | +| `ContractExpectations` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractExpectations` | +| `ContractFieldKey` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractFieldKey` | +| `ContractFieldType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractFieldType` | +| `ContractOutputElement` | `pyoaev.contracts.contract_builder`, `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractOutputElement` | +| `ContractOutputType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractOutputType` | +| `ContractPayload` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractPayload` | +| `ContractReferencedCredential` | `pyoaev.contracts.contract_config`, `pyoaev.credential.utils` | `from xtm_oaev_sdk import ContractReferencedCredential` | +| `ContractSelect` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractSelect` | +| `ContractTeam` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractTeam` | +| `ContractText` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractText` | +| `ContractTextArea` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractTextArea` | +| `ContractTuple` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractTuple` | +| `ContractVariable` | `pyoaev.contracts.contract_config`, `pyoaev.contracts.contract_utils`, `pyoaev.contracts.variable_helper` | `from xtm_oaev_sdk import ContractVariable` | +| `CredentialType` | `pyoaev.contracts.contract_config`, `pyoaev.credential`, `pyoaev.credential.types`, `pyoaev.credential.utils` | `from xtm_oaev_sdk import CredentialType` | +| `CustomJsonFormatter` | `pyoaev.utils` | `from xtm_oaev_sdk import CustomJsonFormatter` | +| `data_to_temp_file` | `pyoaev.helpers` | `from xtm_oaev_sdk import data_to_temp_file` | +| `DictionarySource` | `pyoaev.configuration.configuration`, `pyoaev.configuration.sources` | `from xtm_oaev_sdk import DictionarySource` | +| `Domain` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import Domain` | +| `EnhancedJSONEncoder` | `pyoaev.utils` | `from xtm_oaev_sdk import EnhancedJSONEncoder` | +| `EnvironmentSource` | `pyoaev.configuration.configuration`, `pyoaev.configuration.sources` | `from xtm_oaev_sdk import EnvironmentSource` | +| `ExecutionDetails` | `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ExecutionDetails` | +| `ExecutionSignature` | `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ExecutionSignature` | +| `ExecutionStatus` | `pyoaev.signatures.models`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import ExecutionStatus` | +| `Expectation` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import Expectation` | +| `ExpectationSignatureGroup` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ExpectationSignatureGroup` | +| `ExpectationType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractExpectationType` | +| `ExpectationType` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import SignatureExpectationType` | +| `ExtraSignatureData` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ExtraSignatureData` | +| `FALSY` | `pyoaev.helpers` | `from xtm_oaev_sdk import is_falsy` | +| `InjectExecutionActions` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import InjectExecutionActions` | +| `InjectorConfig` | `pyoaev.signatures`, `pyoaev.signatures.models` | `from xtm_oaev_sdk import InjectorConfig` | +| `is_falsy` | `pyoaev.configuration.configuration` | `from xtm_oaev_sdk import is_falsy` | +| `is_memory_certificate` | `pyoaev.helpers` | `from xtm_oaev_sdk import is_memory_certificate` | +| `is_truthy` | `pyoaev.configuration.configuration` | `from xtm_oaev_sdk import is_truthy` | +| `LinkedFieldModel` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import LinkedFieldModel` | +| `logger` | `pyoaev.daemons.base_daemon`, `pyoaev.utils` | `from xtm_oaev_sdk import logger` | +| `MatchTypes` | `pyoaev`, `pyoaev.apis.inject_expectation.model.expectation`, `pyoaev.signatures`, `pyoaev.signatures.signature_match`, `pyoaev.signatures.signature_type`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import MatchTypes` | +| `NetworkInjectorConfig` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import NetworkInjectorConfig` | +| `OpenAEVAuthenticationError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVAuthenticationError` | +| `OpenAEVCreateError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVCreateError` | +| `OpenAEVDeleteError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVDeleteError` | +| `OpenAEVError` | `pyoaev.daemons.base_daemon`, `pyoaev.exceptions`, `pyoaev.signatures.signature_manager`, `pyoaev.signatures.signature_match` | `from xtm_oaev_sdk import OpenAEVError` | +| `OpenAEVGetError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVGetError` | +| `OpenAEVHeadError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVHeadError` | +| `OpenAEVHttpError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVHttpError` | +| `OpenAEVListError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVListError` | +| `OpenAEVParsingError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVParsingError` | +| `OpenAEVUpdateError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVUpdateError` | +| `PingAlive` | `pyoaev.daemons.collector_daemon`, `pyoaev.helpers`, `pyoaev.utils` | `from xtm_oaev_sdk import PingAlive` | +| `prepare_contracts` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import prepare_contracts` | +| `PrimitiveType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import PrimitiveType` | +| `RedirectError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import RedirectError` | +| `SecurityDomains` | `pyoaev.security_domain.types` | `from xtm_oaev_sdk import SecurityDomains` | +| `SecurityPlatformType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import SecurityPlatformType` | +| `SettingsLoader` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import SettingsLoader` | +| `setup_logging_config` | `pyoaev.utils` | `from xtm_oaev_sdk import setup_logging_config` | +| `SignatureCallbackPayload` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models` | `from xtm_oaev_sdk import SignatureCallbackPayload` | +| `SignatureManager` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import SignatureManager` | +| `SignatureMatch` | `pyoaev.signatures.signature_match`, `pyoaev.signatures.signature_type` | `from xtm_oaev_sdk import SignatureMatch` | +| `SignatureOutputStructure` | `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import SignatureOutputStructure` | +| `SignaturePayload` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import SignaturePayload` | +| `SignatureTarget` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import SignatureTarget` | +| `SignatureTransmissionError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import SignatureTransmissionError` | +| `SignatureType` | `pyoaev.apis.inject_expectation.model.expectation`, `pyoaev.signatures.signature_type` | `from xtm_oaev_sdk import SignatureType` | +| `SignatureTypes` | `pyoaev`, `pyoaev.apis.inject_expectation.model.expectation`, `pyoaev.signatures`, `pyoaev.signatures.signature_type`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import SignatureTypes` | +| `SignatureValue` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import SignatureValue` | +| `ssl_cert_chain` | `pyoaev.helpers` | `from xtm_oaev_sdk import ssl_cert_chain` | +| `ssl_verify_locations` | `pyoaev.helpers` | `from xtm_oaev_sdk import ssl_verify_locations` | +| `SupportedLanguage` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import SupportedLanguage` | +| `TargetSignatures` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import TargetSignatures` | +| `ToolErrorInfo` | `pyoaev.signatures.models` | `from xtm_oaev_sdk import ToolErrorInfo` | +| `ToolOutput` | `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ToolOutput` | +| `ToolTimeoutInfo` | `pyoaev.signatures.models` | `from xtm_oaev_sdk import ToolTimeoutInfo` | +| `TRUTHY` | `pyoaev.helpers` | `from xtm_oaev_sdk import is_truthy` | +| `VariableHelper` | `pyoaev.contracts.contract_config`, `pyoaev.contracts.variable_helper` | `from xtm_oaev_sdk import VariableHelper` | +| `VariableType` | `pyoaev.contracts.contract_utils`, `pyoaev.contracts.variable_helper` | `from xtm_oaev_sdk import VariableType` | + +### To collectors-sdk + +| Legacy name | Legacy modules | New import | +|---|---|---| +| `CollectorDaemon` | `pyoaev.daemons`, `pyoaev.daemons.collector_daemon`, `pyoaev.helpers` | `from collectors_sdk import CollectorDaemon` | +| `DEFAULT_PERIOD_SECONDS` | `pyoaev.daemons.collector_daemon` | `from collectors_sdk import DEFAULT_PERIOD_SECONDS` | +| `DEFAULT_PLATFORM_TAG_COLOR` | `pyoaev.daemons.collector_daemon` | `from collectors_sdk import DEFAULT_PLATFORM_TAG_COLOR` | +| `DetectionExpectation` | `pyoaev.apis`, `pyoaev.apis.inject_expectation`, `pyoaev.apis.inject_expectation.inject_expectation`, `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import DetectionExpectation` | +| `Expectation` | `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import Expectation` | +| `ExpectationSignature` | `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import ExpectationSignature` | +| `ExpectationTypeEnum` | `pyoaev.apis`, `pyoaev.apis.inject_expectation`, `pyoaev.apis.inject_expectation.inject_expectation`, `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import ExpectationTypeEnum` | +| `OpenAEVCollectorHelper` | `pyoaev.helpers` | `from collectors_sdk import CollectorDaemon` | +| `OpenAEVDetectionHelper` | `pyoaev.helpers` | `from collectors_sdk import OpenAEVDetectionHelper` | +| `PreventionExpectation` | `pyoaev.apis`, `pyoaev.apis.inject_expectation`, `pyoaev.apis.inject_expectation.inject_expectation`, `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import PreventionExpectation` | + +### To injectors-sdk + +| Legacy name | Legacy modules | New import | +|---|---|---| +| `create_mq_ssl_context` | `pyoaev.helpers` | `from injectors_sdk import create_mq_ssl_context` | +| `ListenQueue` | `pyoaev.helpers` | `from injectors_sdk import ListenQueue` | +| `OpenAEVConfigHelper` | `pyoaev.helpers` | `from injectors_sdk import OpenAEVConfigHelper` | +| `OpenAEVInjectorHelper` | `pyoaev.helpers` | `from injectors_sdk import OpenAEVInjectorHelper` | + +### Without replacement + +| Legacy name | Legacy modules | New import | +|---|---|---| +| `CHALLENGES_URI` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `COMCHECK` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `EXERCISE` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `LESSONS_URI` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `PLAYER_URI` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `SCOREBOARD_URI` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `TEAMS` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `USER` | `pyoaev.contracts.variable_helper` | no replacement (see below) | + +## 4. Inject the API client + +The SDKs never build the API client. You build it, and you hand it over. Four legacy +classes used to build it themselves from the configuration (`openaev_url`, +`openaev_token`, `openaev_tenant_id`), and their legacy paths still do. + +Collector: + +```python +# before +from pyoaev.daemons import CollectorDaemon + +CollectorDaemon(configuration=configuration, callback=collect, collector_type="openaev_my_collector") + +# after +from pyoaev import OpenAEV +from collectors_sdk import CollectorDaemon + +api = OpenAEV( + url=configuration.get("openaev_url"), + token=configuration.get("openaev_token"), + tenant_id=configuration.get("openaev_tenant_id"), +) +CollectorDaemon( + configuration=configuration, + callback=collect, + api_client=api, + collector_type="openaev_my_collector", +) +``` + +`BaseDaemon` (`pyoaev.daemons` → `xtm_oaev_sdk`) plays by the same rule: same +`api_client=`. + +Injector: + +```python +# before +from pyoaev.helpers import OpenAEVInjectorHelper + +helper = OpenAEVInjectorHelper(config, icon) + +# after +from pyoaev import OpenAEV +from injectors_sdk import OpenAEVInjectorHelper + +api = OpenAEV( + url=config.get_conf("openaev_url"), + token=config.get_conf("openaev_token"), + tenant_id=config.get_conf("openaev_tenant_id"), +) +helper = OpenAEVInjectorHelper(config, icon, api_client=api) +``` + +Signature manager: + +```python +# before +from pyoaev.signatures import SignatureManager + +manager = SignatureManager(client, logger) + +# after +from xtm_oaev_sdk import SignatureManager + +manager = SignatureManager(transport=client.signature, logger=logger) +``` + +Collector expectations: + +```python +# before +for expectation in api.inject_expectation.expectations_models_for_source(source_id=source_id): + ... + +# after +from collectors_sdk import expectations_models_for_source + +for expectation in expectations_models_for_source(api.inject_expectation, source_id=source_id): + ... +``` + +The client fetches the expectation data. collectors-sdk turns it into +`DetectionExpectation` / `PreventionExpectation` models. The former method still works, +firing a `DeprecationWarning`. + +## Renamed names + +| Legacy | New | +|---|---| +| `pyoaev.signatures.ExpectationType` | `xtm_oaev_sdk.SignatureExpectationType` | +| `pyoaev.contracts.contract_config.ExpectationType` | `xtm_oaev_sdk.ContractExpectationType` | +| `pyoaev.helpers.TRUTHY` / `FALSY` | `xtm_oaev_sdk.is_truthy` / `is_falsy` | +| `pyoaev.helpers.OpenAEVCollectorHelper` (already deprecated) | `collectors_sdk.CollectorDaemon` | + +The constants of `pyoaev.contracts.variable_helper` (`USER`, `EXERCISE`, `TEAMS`, +`COMCHECK` and the `*_URI` ones) get no public replacement. Reach for the built-in +variables of `xtm_oaev_sdk.VariableHelper` instead. + +## What stays in pyoaev + +The API client, and everything wired around it: `OpenAEV`, `pyoaev.apis` (including +`inputs.search`), `pyoaev.backends`, `pyoaev.base`, `pyoaev.client`, `pyoaev.mixins`, and +the REST client helpers `pyoaev.utils.get_content_type` / `response_content` / +`EncodedId` / `RequiredOptional` / `copy_dict` / `remove_none_from_dict`, plus +`pyoaev.exceptions.on_http_error`. Nothing to migrate there. + +## Things you may notice + +- In tests, the legacy patch targets (`pyoaev.daemons.base_daemon.OpenAEV`, + `pyoaev.helpers.OpenAEV`, `pyoaev.utils.PingAlive.start`…) keep biting as long as + your code runs through the legacy paths. Once it imports from the SDKs, patch the + names it actually looks up now. +- `from import X` on a legacy package (for example `from pyoaev.signatures + import SignatureManager`) fires the warning twice. Plain modules fire it once. +- An `AppLogger` built without a name, and a `SignatureManager` built without a logger, + now log under the SDK module name, not the former `pyoaev` one. diff --git a/libs/xtm-oaev-sdk/docs/seam_strategy.md b/libs/xtm-oaev-sdk/docs/seam_strategy.md new file mode 100644 index 00000000..97925511 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/seam_strategy.md @@ -0,0 +1,237 @@ +# Seam strategy: port/adapter layering of the SDKs + +`xtm-oaev-sdk`, `collectors-sdk` and `injectors-sdk` share one build. Every +capability is cut into a **public** adapter layer, a **seam** of contracts, and a +**core** implementation. This document covers the intent, the rules, the patterns +that applied them without touching behaviour, and how to add a capability. + +## Intent + +| Layer | Role | +|---|---| +| `public/.py` | What users import: ready-to-use adapters (classes, functions) plus the seam contracts users need. Adapters speak seam types and hand the work to the core. | +| `seam//` | The **contract** between public and core: data models, enums, errors, value objects, and `Protocol` ports, plus `SEAM_VERSION`. | +| `_core//` | The code behind the public API: one implementation per seam port. | + +As long as the core keeps answering the seam, the public layer keeps working, and +so does every consumer downstream of it. The core can be rewritten at will. A change +*to the seam* is another matter: that is a contract change. It bumps `SEAM_VERSION`, +earns a changelog entry, and needs a deprecation once it reaches the public surface. + +## Rules + +``` +public ──► seam ◄── _core + └───────────────────────► (wiring only: default implementations) +``` + +1. **The seam is closed.** It imports only the standard library, third-party + libraries (pydantic…), other packages' *public* API (e.g. `collectors_sdk` seams + import `xtm_oaev_sdk`) and sibling seam modules. Never `_core`. Never `public`. +2. **The core implements the seam.** It imports seams, other core modules and + third-party libraries. Never `public`, not even lazily; type-only imports go under + `TYPE_CHECKING`. +3. **Public never hands out a core object.** Every exported name is either an adapter + defined in `public/` or a contract defined in `seam/`. Public touches `_core` for + one reason only, picking the default implementation of each port: + + ```python + _truthiness: TruthinessPort = Truthiness() + _sources: ConfigurationSourcesPort = ConfigurationSources() + _resolver: ConfigurationResolverPort = ConfigurationResolver(_sources, _truthiness) + ``` + +4. **Users import from the package root** (`from xtm_oaev_sdk import X`). The layers + are internal wiring. + +## What goes where + +### In the seam: data contracts + +Pydantic models, dataclasses, enums, exceptions, type aliases and constants. In +short, whatever crosses the boundary. They may carry **their own invariants**, logic +that only reads or derives from their own data, because that logic is part of what +the data *is*: + +- validators and derived fields (`ConfigurationHint`, `ExpectationData`'s null + signatures coerced to `[]`); +- `__post_init__` invariants of the contract elements (derived `type`, forced + cardinality, predefined expectations); +- value objects: `SignatureMatch` (score invariant), `SignatureType.make_struct_for_matching`; +- exceptions, including `OpenAEVError`'s extraction of a readable message from its + own response body: the core raises them and users catch them, so they have to live + where both sides can reach them. + +Anything that does I/O, orchestrates, or leans on another capability is +**behaviour**, and it goes behind a port. + +### Behind a port: everything else, however small + +Every piece of behaviour gets a `Protocol` port in the seam, an implementation in +the core and an adapter in public, **even a one-liner**. `is_truthy` gets its +`TruthinessPort`, `Truthiness` and public `is_truthy`, same treatment as +`SignatureManager`. Uniformity is the whole point: no "too small to need a seam" +exception left to argue about. + +A seam holds two kinds of protocols: + +- **Consumer protocols**: what users type against or implement: `BaseClient`, + `DaemonProtocol`, `AppLoggerProtocol`, `ConfigurationProtocol`, + `ContractBuilderProtocol`, `SignatureManagerProtocol`, and the driven port + `SignatureTransportProtocol` (implemented by `pyoaev`'s signature API). +- **Internal ports**: what a public adapter needs from the core: + `ConfigurationResolverPort`, `SignatureEnginePort`, `CollectorRegistrationPort`… + +## Patterns used to adapt without changing behaviour + +The layering went onto existing code with behaviour held identical: public surface, +wire payloads and connector test suites, compared before and after. Pulling that off +took a handful of recurring patterns. + +### Data + behaviour classes: split, the public class extends the seam data + +When a user-facing class mixes data and behaviour (`Contract`, `Expectation`), the +seam keeps the data (`ContractData`, `ExpectationData`) and the public class extends +it, bolting the behaviour on through ports. Users keep constructing the public class; +the core only ever sees the seam type. + +```python +@dataclass +class Contract(ContractData): + variables: list[ContractVariable] = field( + default_factory=lambda: _variables.default_variables() + ) + + def to_contract_add_input(self, source_id: str) -> dict[str, Any]: + return _serializer.to_add_input(self, source_id) +``` + +### Framework subclasses stay framework subclasses + +Adapters that users hand to a framework keep their base class, override only the +hook, and delegate the logic: `EnhancedJSONEncoder` (`json.JSONEncoder.default`), +`CustomJsonFormatter` (pythonjsonlogger `add_fields`), `ConnectorConfigSchemaGenerator` +(pydantic `GenerateJsonSchema`), `SettingsLoader` (pydantic-settings +`settings_customise_sources`), `PingAlive` / `ListenQueue` (`threading.Thread`). + +### Stateful adapters keep their attributes and hooks + +`BaseDaemon`, `CollectorDaemon`, `SignatureManager`, `AppLogger`, +`OpenAEVInjectorHelper` keep their attributes (`_configuration`, `api`, `logger`, +`local_logger`, IP caches…) and their overridable hooks (`_setup`, `_start_loop`, +`_try_callback`, `_create_default_api_client`, `_utcnow`…). Subclasses and tests +lean on them. The adapter holds the state; the core holds the logic. + +### Pass callables to keep dispatch and patch points + +Where the original code called back into its own object, the adapter passes that +callback down to the core, so overrides and patches still land: + +```python +def _source_context(self, config): + return _engine.source_context( + config, self.resolve_container_ip, lambda: self._cached_ipv6 + ) +``` + +`SignatureManager.post_execution_updates` passes `self._utcnow` (read only after the +tool output validates, as before); `Expectation.match_alert` passes +`self.match_fuzzy` / `self.match_simple`; `OpenAEVDetectionHelper` passes +`self.match_alert_element_fuzzy` / `self._decode_value`; `CollectorDaemon` passes +`self._try_callback` to the scheduler. A dependency the core cannot import travels +the same way: `Configuration.schema()` hands the public +`ConnectorConfigSchemaGenerator` to the core resolver. + +### A state protocol when the core drives a stateful loop + +`ListenQueue`'s RabbitMQ loop rewrites public attributes (`pika_connection`, +`channel`, `thread`…). The seam declares a `QueueListenerProtocol` describing that +state; the core runs the loop (connect, consume, recover, process, stop) against it, +and the adapter's methods shrink to one-line delegations: + +```python +def run(self) -> None: + _consumer.run(self) +``` + +## Versioning + +- Every capability seam exposes `SEAM_VERSION`. `port_io`, a pure + wire schema, carries `PORT_IO_CONTRACT_VERSION` instead. +- Every public seam object is fingerprinted in a golden: any seam change fails a test + until someone regenerates the golden on purpose, version bump included. See `test/seam/` and + [every-seam-object-is-fingerprinted](tdr/every-seam-object-is-fingerprinted.tdr.md). +- Policy: a change to a seam (field, type, enum member, error, port signature) bumps + its version and lands in the changelog. If it reaches the public surface, the old + name goes through a deprecation shim first (see [shim_strategy.md](shim_strategy.md)). + +## Walkthrough: the configuration capability + +```python +# seam/configuration/ports.py — the contract (closed) +@runtime_checkable +class TruthinessPort(Protocol): + def is_truthy(self, value: str) -> bool: ... + def is_falsy(self, value: str) -> bool: ... + +# _core/configuration/truthiness.py — the implementation (imports the seam only) +class Truthiness: + def is_truthy(self, value: str) -> bool: + return value.lower() in ["yes", "true"] + +# public/configuration.py — the adapter (what users import) +_truthiness: TruthinessPort = Truthiness() + +def is_truthy(value: str) -> bool: + return _truthiness.is_truthy(value) +``` + +The same capability carries the heavier cases too. `Configuration` keeps its +constructor, `get`, `set` and `schema`, and delegates resolution to +`ConfigurationResolverPort`. `SettingsLoader` stays a `BaseSettings` and delegates +source selection to `SettingsSourcesPort`. The settings models (`BaseConfigModel`, +`ConfigLoaderOAEV`…) are data contracts in the seam. + +## Capability map + +### xtm-oaev-sdk + +| Capability | Seam: contracts | Seam: ports | Core | Public adapters | +|---|---|---|---|---| +| common | the error hierarchy, the asset and security-domain enums, `BaseClient`, `DaemonProtocol`, `AppLoggerProtocol` | `JsonEncodingPort`, `LoggingPort`, `CertificatesPort` | `JsonEncoding`, `LoggingSetup`, `Certificates` | `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, the SSL helpers | +| configuration | `ConfigurationHint`, settings models, serializer types, `CONFIGURATION_TYPES`, `ConfigurationProtocol` | `ConfigurationResolverPort`, `ConfigurationSourcesPort`, `ConnectorSchemaPort`, `SettingsSourcesPort`, `TruthinessPort` | `ConfigurationResolver`, `ConfigurationSources`, `ConnectorSchemaTransformer`, `SettingsSourceSelector`, `Truthiness` | `Configuration`, `EnvironmentSource`, `DictionarySource`, `ConnectorConfigSchemaGenerator`, `SettingsLoader`, `is_truthy`, `is_falsy` | +| contracts | `ContractData`, contract elements and enums, `ContractVariable`, `Expectation`, `CredentialType`, `ContractBuilderProtocol` | `ContractAssemblyPort`, `ContractVariablesPort`, `ContractSerializerPort`, `ContractCredentialsPort` | `ContractAssembly`, `BuiltinContractVariables`, `ContractSerializer`, `CredentialReferences` | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, `build_single_referenced_credential_element` | +| signatures | inbound I/O models, matching types and policies, `ExtraSignatureData`, `SignatureTransportProtocol`, `SignatureManagerProtocol` | `SignatureEnginePort`, `ContainerAddressPort`, `NetworkTargetsPort`, `MarkerPort` | `SignatureEngine`, `ContainerAddressResolver`, `NetworkTargets`, `MarkerBuilder` | `SignatureManager`, `build_network_configs`, `build_marker` | +| port_io | wire models and enums, `PORT_IO_CONTRACT_VERSION` | (none: pure wire schema) | | re-exported as is | +| daemons | (consumer protocols live in common) | `DaemonRuntimePort`, `HeartbeatPort` | `DaemonRuntime`, `Heartbeat` | `BaseDaemon`, `PingAlive` | +| deprecation | `DeprecatedNames` | `DeprecationPort` | `DeprecationShims` | `make_deprecated_getattr` | + +### collectors-sdk + +| Capability | Seam: contracts | Seam: ports | Core | Public adapters | +|---|---|---|---|---| +| daemons | `DEFAULT_PERIOD_SECONDS`, `DEFAULT_PLATFORM_TAG_COLOR` | `CollectorRegistrationPort`, `CollectorSchedulePort` | `CollectorRegistration`, `CollectorSchedule` | `CollectorDaemon` | +| detection | | `AlertMatchingPort` | `AlertMatcher` | `OpenAEVDetectionHelper` | +| expectations | `ExpectationData`, `ExpectationSignature`, `ExpectationTypeEnum`, `ExpectationApiProtocol` | `ExpectationLoadingPort`, `ExpectationMatchingPort`, `ExpectationUpdatePort` | `ExpectationLoader`, `ExpectationMatcher`, `ExpectationUpdates` | `Expectation`, `DetectionExpectation`, `PreventionExpectation`, `expectations_models_for_source` | + +### injectors-sdk + +| Capability | Seam: contracts | Seam: ports | Core | Public adapters | +|---|---|---|---|---| +| helpers | `QueueListenerProtocol` (listener state) | `ConfigLookupPort`, `InjectorRegistrationPort`, `QueueConsumerPort`, `MqSslPort` | `ConfigLookup`, `InjectorRegistration`, `QueueConsumer`, `MqSsl` | `OpenAEVConfigHelper`, `OpenAEVInjectorHelper`, `ListenQueue`, `create_mq_ssl_context` | + +## Adding a capability + +1. **Seam** (`seam//`): the data contracts in `models.py`, one `Protocol` + per piece of behaviour the public API needs in `ports.py`, `SEAM_VERSION = "1.0.0"` + in `__init__.py`. Import nothing from `_core` or `public`. +2. **Core** (`_core//`): one class per port, implementing it with the + exact method signatures. Import the seam, never `public`. +3. **Public** (`public/.py`): module-level default implementations typed + by their port, adapters carrying the user-facing signatures and delegating to them, + and the seam contracts users need. Set `__all__`. +4. **Root**: import the module in `public/__init__.py` and extend `__all__`. +5. **Tests**: behaviour through the public API; core implementations through their + port when useful. +6. If the capability replaces a `pyoaev` name, add its deprecation shim in `pyoaev` + (see [shim_strategy.md](shim_strategy.md)). diff --git a/libs/xtm-oaev-sdk/docs/shim_strategy.md b/libs/xtm-oaev-sdk/docs/shim_strategy.md new file mode 100644 index 00000000..300c87f7 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/shim_strategy.md @@ -0,0 +1,191 @@ +# Shim strategy: splitting pyoaev without breaking anyone + +`pyoaev` used to carry everything: the OpenAEV API client, and on top of it +configuration, contracts, signatures, the daemon runtime, errors, helpers. That +shared code now lives in three SDKs (`xtm-oaev-sdk`, `collectors-sdk`, +`injectors-sdk`), and `pyoaev` is back to its actual job: being the API client. + +One rule governs the split: **no breaking change for existing code.** Every +collector and injector written against `pyoaev` keeps running untouched until the +shims are deliberately pulled. This document lays out how that holds, and why it is +built this way. + +## The contract + +For every name a released `pyoaev` exposed: + +1. **The legacy path still resolves**: `from pyoaev.configuration import Configuration`, + `pyoaev.helpers.OpenAEVInjectorHelper`, `from pyoaev import AssetCategory`… +2. **It resolves to the SDK object**, and to the *identical* object whenever possible, + so `is`, `isinstance`, subclassing, pickling-by-reference and introspection behave + exactly as before. +3. **Access fires a `DeprecationWarning`** naming the new import, pinned to the + **consumer's line**, so the default warning filters actually show it. +4. **Importing `pyoaev` stays silent**: only touching a moved name warns. +5. **Patch targets keep landing**: what connectors' tests patch (`pyoaev.utils.PingAlive.start`, + `pyoaev.daemons.collector_daemon.CollectorDaemon._setup`, + `pyoaev.daemons.base_daemon.OpenAEV`, `pyoaev.signatures.signature_manager.socket.gethostbyname`…) + still hits the code that actually runs. + +## Dependency direction + +``` +xtm-oaev-sdk ← collectors-sdk + ↑ ← injectors-sdk + └──────────────┴────────── pyoaev (API client + shims) +``` + +- The SDKs **never** import `pyoaev`. When they need the platform, the API client + gets injected (`xtm_oaev_sdk.BaseClient` protocol; `pyoaev.OpenAEV` satisfies it). +- `pyoaev` **hard-depends** on the three SDKs for the whole deprecation window. That + is what keeps a connector that only lists `pyoaev` alive. An optional extra + (`pyoaev[collectors]`) would have broken exactly those connectors. +- The direct dependencies `pyoaev` no longer imports (`pika`, `thefuzz`, `PyYAML`, + `pydantic-settings`, `python-json-logger`) are gone from its `pyproject.toml`. + They still arrive transitively through the SDKs, so a `pyoaev` install pulls the + same third-party packages as before. + +## The mechanism: PEP 562 module `__getattr__` + +Every shim module serves its moved names lazily, through a module-level +`__getattr__` built by the SDK primitive `make_deprecated_getattr`: + +```python +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`.""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "DictionarySource", + "EnvironmentSource", +] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) +``` + +`make_deprecated_getattr(mapping, module_name=...)` validates the +`{old_name: (message, replacement)}` mapping once, then hands back a `__getattr__` +that warns (`DeprecationWarning`, `stacklevel=2`) and returns the replacement **as +is**. Unknown names still raise `AttributeError`. Typos never get swallowed. + +### Why not the alternatives + +- **`typing_extensions.deprecated(...)(cls)`** does not wrap the + class. It **patches the original class in place** (`__new__` / + `__init_subclass__`). The moment anything imported `pyoaev`, users of + `xtm_oaev_sdk.Configuration` *also* caught "use `xtm_oaev_sdk`" warnings. Rejected. +- **A wrapper class or proxy** breaks identity: `isinstance` against the SDK class, + metaclasses and signature introspection all shift. Rejected. +- **A plain re-export** (`from xtm_oaev_sdk import X`) keeps identity but never + warns, so nobody would ever migrate. Rejected. +- **Optional dependencies or lazy imports of the SDKs** would make legacy imports + blow up for connectors that only list `pyoaev`. Rejected (see dependency direction). + +## The kinds of shims + +| Kind | Example | How it is served | +|---|---|---| +| **Moved, identical** (the vast majority) | `pyoaev.configuration.Configuration` → `xtm_oaev_sdk.Configuration` | the SDK object itself | +| **Renamed** | `pyoaev.signatures.ExpectationType` → `xtm_oaev_sdk.SignatureExpectationType`; `pyoaev.contracts.contract_config.ExpectationType` → `xtm_oaev_sdk.ContractExpectationType` | the SDK object; the message names the new name | +| **Client-building adapter** | `pyoaev.daemons.BaseDaemon`, `pyoaev.daemons.CollectorDaemon`, `pyoaev.signatures.SignatureManager`, `pyoaev.helpers.OpenAEVInjectorHelper` | a thin subclass of the SDK class (see below) | +| **Value without public replacement** | `pyoaev.contracts.variable_helper.USER`…, `pyoaev.helpers.TRUTHY` / `FALSY` | the literal value, with a message pointing at the right API | +| **Private legacy helper** | `pyoaev.helpers._is_base64_encoded` | the collectors-sdk implementation, "private and will be removed" | +| **Legacy-only glue** | `pyoaev.helpers.OpenAEVCollectorHelper` (already deprecated) | kept in `pyoaev` on top of the collectors-sdk daemon, warns on access | +| **Re-exported names** | `pyoaev.daemons.base_daemon.Configuration`, `pyoaev.apis.inject_expectation.model.expectation.SignatureTypes`… | the SDK object (names a module merely imported are part of its surface too) | + +### Client-building adapters + +The SDKs take the API client as an argument (`api_client=` / `transport=`). The +legacy constructors built `OpenAEV(...)` themselves. So those legacy names resolve to +a thin subclass of the SDK class whose only job is to restore that behaviour: + +| Legacy name | Subclass of | What it adds | +|---|---|---| +| `pyoaev.daemons.BaseDaemon` | `xtm_oaev_sdk.BaseDaemon` | `_create_default_api_client()` builds `OpenAEV` from `openaev_url` / `openaev_token` / `openaev_tenant_id` | +| `pyoaev.daemons.CollectorDaemon` | `collectors_sdk.CollectorDaemon` + the class above | same hook; still an instance of the legacy `BaseDaemon` | +| `pyoaev.signatures.SignatureManager` | `xtm_oaev_sdk.SignatureManager` | `SignatureManager(client, logger)` uses `client.signature` as transport | +| `pyoaev.helpers.OpenAEVInjectorHelper` | `injectors_sdk.OpenAEVInjectorHelper` | `_create_default_api_client(config)` builds `OpenAEV` from the config helper | + +`isinstance(legacy_object, )` holds. Each adapter is **one object**, +whatever path reaches it (`pyoaev.daemons.CollectorDaemon is +pyoaev.daemons.collector_daemon.CollectorDaemon is pyoaev.helpers.CollectorDaemon`), +so patching a method on any of those paths hits the class connectors instantiate. + +## Rules learned the hard way + +- **Assign `__getattr__` directly.** A `def __getattr__(name): return + _deprecated_getattr(name)` wrapper slips in an extra frame. The warning then gets + pinned to the shim file, and Python's default filters (which only show + `DeprecationWarning` for `__main__`) hide it from scripts. +- **Never star-import a shim module.** `from pyoaev.signatures import *` walks + `__all__`, trips every lazy name and makes `import pyoaev` warn. Instead, the + `pyoaev` package root binds the shim submodules and serves their names through its + own `__getattr__` (`pyoaev.__all__` is unchanged, so `from pyoaev import *` keeps + exporting the same names, and warns for the moved ones). +- **pyoaev's own code imports from the SDKs**, never from its shims. Importing any + `pyoaev` module, `pyoaev.apis` included, stays silent. +- **Keep patch surfaces.** `pyoaev.signatures.signature_manager` still imports + `socket` and `subprocess` (tests patch `...signature_manager.socket.gethostbyname`, + which reaches the shared stdlib module the SDK uses). The client-building adapters + call `OpenAEV` through their module global, so `patch("pyoaev.daemons.base_daemon.OpenAEV")` + and `patch("pyoaev.helpers.OpenAEV")` still bite. +- **`__all__` lists lazily served names**: flake8 flags F822 outside `__init__.py`, + so the line carries `# noqa: F822` with the reason. +- **Known quirk**: `from import X` on a shim *package* warns twice + (importlib's `_handle_fromlist` probes with `hasattr` before the real lookup). + Plain modules warn once. Harmless, and baked into PEP 562. + +## Where the shims are + +| pyoaev module | Serves | +|---|---| +| `pyoaev/__init__.py` | the moved names it exported (asset enums, exceptions, configuration, contracts, signatures) | +| `pyoaev/configuration/*` | xtm-oaev-sdk configuration | +| `pyoaev/contracts/*` | xtm-oaev-sdk contracts (`ExpectationType` renamed, variable constants) | +| `pyoaev/credential/*` | xtm-oaev-sdk contracts (credential references) | +| `pyoaev/signatures/*` | xtm-oaev-sdk signatures (`ExpectationType` renamed, `SignatureManager` adapter) | +| `pyoaev/exceptions.py`, `asset_types.py`, `security_domain/types.py` | xtm-oaev-sdk common; `exceptions.py` keeps the REST client decorator `on_http_error` | +| `pyoaev/utils.py` | xtm-oaev-sdk common and `PingAlive`; keeps the REST client helpers `get_content_type`, `response_content`, `EncodedId`, `RequiredOptional`, `copy_dict`, `remove_none_from_dict` | +| `pyoaev/daemons/*` | `BaseDaemon` / `CollectorDaemon` adapters | +| `pyoaev/helpers.py` | collectors-sdk detection, injectors-sdk helpers (`OpenAEVInjectorHelper` adapter), xtm SSL helpers, `OpenAEVCollectorHelper` | +| `pyoaev/apis/__init__.py`, `apis/inject_expectation/*` | collectors-sdk expectations; `InjectExpectationManager.expectations_models_for_source` delegates to `collectors_sdk.expectations_models_for_source`, with a warning | + +## How it is tested + +| Test | Guards | +|---|---| +| `libs/xtm-oaev-sdk/test//shim/test_shim_contract.py` | identity, warning text, pinning to the consumer's line, silent imports, unknown names, adapters (client construction, `isinstance`) | +| `libs/xtm-oaev-sdk/test//shim/test_*.py` | the original pyoaev tests, moved **verbatim**: they still drive the legacy paths | +| `test/test_sdk_split_shims.py` | the collectors-sdk / injectors-sdk shims, adapters and connector patch targets | +| `test/test_legacy_surface.py` | every public name of the **released** pyoaev, recorded in `test/fixtures/legacy_surface.json` by `test/generate_legacy_surface.py` from the release, not the working tree) still resolves | + +Beyond the suites, the split went through the real collectors|injectors own test suites +run once against the released `pyoaev` and once +against the new stack. Identical results. + +## Observable differences (not API breaks) + +- Legacy imports warn (that is the whole point of the shims). +- The client-building adapters are subclasses, not the identical SDK classes. +- An `AppLogger` built without a name, and a `SignatureManager` built without a + logger, log under the SDK module name instead of the former pyoaev one. + +## Removing the shims + +In a major release of `pyoaev`: + +1. delete the shim modules and the `__getattr__` of `pyoaev/__init__.py`; +2. delete the client-building adapters, `OpenAEVCollectorHelper` and + `InjectExpectationManager.expectations_models_for_source`; +3. drop the `xtm-oaev-sdk` / `collectors-sdk` / `injectors-sdk` dependencies only the + shims need (keep `xtm-oaev-sdk` if the client still leans on its errors and + transport models); +4. delete `test/test_legacy_surface.py` and the `shim/` test folders; +5. announce it: every removed name has been warning since the split, new import included. diff --git a/libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md b/libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md new file mode 100644 index 00000000..dae0cb36 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md @@ -0,0 +1,52 @@ +# Technical Decision Records + +The technical decision records of the SDK split. Each one is a rule that future work on +xtm-oaev-sdk, collectors-sdk, injectors-sdk and the `pyoaev` shims is expected to respect. + +## Compatibility with pyoaev + +| TDR | Status | Rule | +|---|---|---| +| [Legacy pyoaev code keeps working until a pyoaev major release](no-breaking-change-for-legacy-pyoaev-code.tdr.md) | draft | Every public name of the latest released pyoaev must keep resolving from its legacy path, with a DeprecationWarning naming its new import, until a pyoaev major release removes the shims. | +| [The SDKs never depend on pyoaev; the API client is injected](sdks-never-import-pyoaev.tdr.md) | draft | xtm-oaev-sdk, collectors-sdk and injectors-sdk must never import pyoaev; anything that talks to the platform receives its API client or transport as an argument typed by a seam protocol. | +| [pyoaev hard-depends on the SDKs that host its former code](pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md) | draft | While pyoaev ships deprecation shims, it must declare a hard dependency on every SDK those shims serve, never an optional extra. | +| [Legacy names are served by a directly assigned PEP 562 __getattr__](legacy-names-served-by-pep562-module-getattr.tdr.md) | draft | A moved name must be served from its legacy module by a module-level __getattr__ built with make_deprecated_getattr and assigned directly, never wrapped in a def. | +| [Shims return the identical SDK object; adapters only where constructors differ](shims-return-the-identical-sdk-object.tdr.md) | draft | A shim must return the SDK object itself; a legacy name may resolve to a thin subclass only to restore legacy constructor behaviour, and that subclass must be one single object whatever legacy path reaches it. | +| [Importing pyoaev never warns; only touching a moved name does](importing-pyoaev-stays-silent.tdr.md) | draft | Importing any pyoaev module must not emit a DeprecationWarning; only accessing a moved name may warn. | +| [Legacy patch targets are part of the compatibility surface](legacy-patch-targets-are-part-of-the-surface.tdr.md) | draft | Module attributes that connectors patch must stay reachable at their legacy path and still reach the code that actually runs. | +| [The legacy surface is guarded against the released pyoaev](legacy-surface-guarded-from-the-release.tdr.md) | draft | The legacy-surface guard must be generated from the released pyoaev package, and the pre-split tests of moved code must be kept verbatim against their legacy paths. | + +## Layering + +| TDR | Status | Rule | +|---|---|---| +| [public and _core depend on the seam; the seam depends on neither](public-seam-core-dependency-rule.tdr.md) | draft | Within each SDK capability, the seam imports neither public nor _core, _core never imports public, and public imports _core only to wire the default implementation of each port. | +| [Every piece of behaviour goes behind a port, even a one-liner](every-behaviour-behind-a-port.tdr.md) | draft | Every piece of behaviour exposed by an SDK goes through a seam Protocol port, one _core implementation and a public adapter, whatever its size. | +| [The seam holds data contracts and only their intrinsic invariants](seam-holds-data-contracts-and-their-own-invariants.tdr.md) | draft | A seam may contain data contracts and logic that only reads or derives from their own data; anything doing I/O, orchestrating, or needing another capability goes behind a port. | +| [Data-and-behaviour classes split: the public class extends the seam data](public-class-extends-seam-data.tdr.md) | draft | When a user-facing class mixes data and behaviour, the seam holds the data class and the public class extends it, adding behaviour through ports; the core only knows the seam type. | +| [Public adapters keep the shape of the code they replace](adapters-keep-the-user-facing-shape.tdr.md) | draft | A public adapter must keep the signatures, docstrings, attributes, overridable hooks and framework base class of the code it replaces; only the logic moves to the core. | +| [Adapters pass their own methods to the core to keep dispatch and patch points](pass-bound-callables-to-keep-dispatch.tdr.md) | draft | When the original code dispatched through its own object, the adapter must pass the bound method or a callable to the core instead of letting the core reimplement the call. | + +## Contracts and versioning + +| TDR | Status | Rule | +|---|---|---| +| [A seam change is a versioned contract change](seam-changes-are-versioned-contract-changes.tdr.md) | draft | Any change to a seam contract (field, type, enum member, error, port signature) must bump that seam's version and be recorded in the changelog, and a change reaching the public surface must go through a deprecation shim first. | +| [Every seam object is fingerprinted](every-seam-object-is-fingerprinted.tdr.md) | draft | Every public top-level object of every seam module must be fingerprinted in one golden, discovered dynamically, and the golden is regenerated only together with a deliberate seam change and its version bump. | +| [Seams crossing a process boundary pin their JSON schemas](cross-process-seams-pin-their-json-schemas.tdr.md) | superseded by [every-seam-object-is-fingerprinted](every-seam-object-is-fingerprinted.tdr.md) | A seam whose models cross a process boundary must pin their JSON schemas with golden hashes, regenerated only together with a version bump. | + +## Public API, tests and checks + +| TDR | Status | Rule | +|---|---|---| +| [Consumers import from the package root only](import-from-the-package-root.tdr.md) | draft | Consumers must import SDK names from the package root (xtm_oaev_sdk, collectors_sdk, injectors_sdk); public, seam and _core module paths are internal and may move. | +| [Colliding moved names are renamed with their capability prefix](colliding-names-get-a-capability-prefix.tdr.md) | draft | A moved name that collides with another capability's name in the flat root namespace must be renamed with its capability as prefix, and its legacy shim must name the new name. | +| [xtm-oaev-sdk stays agnostic of collectors, injectors and transports](xtm-oaev-sdk-stays-connector-agnostic.tdr.md) | draft | xtm-oaev-sdk must not contain collector- or injector-specific runtime, nor import pika, thefuzz or requests; those belong to collectors-sdk, injectors-sdk or pyoaev. | +| [Every SDK module imports on its own in a fresh interpreter](every-sdk-module-imports-alone.tdr.md) | draft | Every SDK module must import successfully as the first import of a fresh interpreter running with -W error. | + +## Adding a TDR + +Create `.tdr.md`, the slug naming the rule. Same header as the other records, +`status: draft`, then a row in this table. A TDR moves to `accepted` once validated. The +day it stops governing, it goes `superseded` (with `superseded_by` / `supersedes` set on +both records) or `revoked`. diff --git a/libs/xtm-oaev-sdk/docs/tdr/adapters-keep-the-user-facing-shape.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/adapters-keep-the-user-facing-shape.tdr.md new file mode 100644 index 00000000..57f517e1 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/adapters-keep-the-user-facing-shape.tdr.md @@ -0,0 +1,71 @@ +# Public adapters keep the shape of the code they replace +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +The classes you use from the SDK look and behave exactly like before: same names, same +arguments, same methods to override. Only the work they do moved, to a place behind +them. Think of swapping a car's engine without touching the steering wheel or the +pedals. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The layering went onto existing code with behaviour held identical. Users subclass SDK +> classes, hand them to frameworks, read their attributes and override their hooks. A +> facade would snap inheritance. + +--- + +## **The TDR** +| **A public adapter must keep the signatures, docstrings, attributes, overridable hooks and framework base class of the code it replaces; only the logic moves to the core.** + + + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- Framework subclasses stay subclasses and override only the hook: `EnhancedJSONEncoder` (`json.JSONEncoder.default`), `CustomJsonFormatter` (`add_fields`), `ConnectorConfigSchemaGenerator` (`GenerateJsonSchema`), `SettingsLoader` (`settings_customise_sources`), `PingAlive` / `ListenQueue` (`threading.Thread`). +- Stateful adapters keep attributes (`_configuration`, `api`, `logger`, `local_logger`…) and hooks (`_setup`, `_start_loop`, `_try_callback`, `_create_default_api_client`, `_utcnow`…). +- The public surface was compared before and after the refactors. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| A facade in front of classes users subclass (`BaseDaemon`, `CollectorDaemon`, `BaseConfigModel`) | Would break inheritance | The adapter is the class; the logic behind it moves | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- an SDK major version is allowed to change its public shape; +- a hook is formally deprecated. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- Adapters hold the state. The core holds the logic. + +--- + +## **Resources** +- [Seam strategy, patterns](../seam_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/colliding-names-get-a-capability-prefix.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/colliding-names-get-a-capability-prefix.tdr.md new file mode 100644 index 00000000..4420c5cf --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/colliding-names-get-a-capability-prefix.tdr.md @@ -0,0 +1,68 @@ +# Colliding moved names are renamed with their capability prefix +--- +scope: [xtm-oaev-sdk, public-api, shims] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +You import everything in the SDK from one single place, so two things can't share a +name. pyoaev had two different things both called `ExpectationType`. They became +`SignatureExpectationType` and `ContractExpectationType`. The old names still work, and +they tell you the new one. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The SDK root namespace is flat (TDR [import-from-the-package-root](import-from-the-package-root.tdr.md)). `pyoaev` had two different +> `ExpectationType`, one in `signatures` and one in `contracts.contract_config`. Two +> classes, one name, one flat root: they could not both keep it. + +--- + +## **The TDR** +| **A moved name that collides with another capability's name in the flat root namespace must be renamed with its capability as prefix, and its legacy shim must name the new name.** + +> The rule generalises the two existing cases. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `pyoaev.signatures.ExpectationType` → `xtm_oaev_sdk.SignatureExpectationType` +- `pyoaev.contracts.contract_config.ExpectationType` → `xtm_oaev_sdk.ContractExpectationType` + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- public sub-namespaces are introduced; +- a collision appears where a capability prefix does not disambiguate. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- No material lesson beyond the TDR itself. + +--- + +## **Resources** +- [Shim strategy, kinds of shims](../shim_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md new file mode 100644 index 00000000..7a5e4ec8 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md @@ -0,0 +1,74 @@ +# Seams crossing a process boundary pin their JSON schemas +--- +scope: [xtm-oaev-sdk, signatures, port_io, testing, versioning] +status: superseded +superseded_by: every-seam-object-is-fingerprinted +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/15 08:36:35 +--- + +## ELI5 + +Some of our data gets read by another program, like the platform. Change the shape of +that data by accident and the other program breaks, while our own tests notice nothing. +So we keep a fingerprint of each shape. Change the shape, a test fails, and you have to +choose, on purpose, to bump the version and update the fingerprint. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The inbound signature I/O (injector configs, execution signatures, tool output) and the +> port I/O wire payloads leave the process. Let a schema drift silently and the other side +> breaks while every local test stays green. + +--- + +## **The TDR** +| **A seam whose models cross a process boundary must pin their JSON schemas with golden hashes, regenerated only together with a version bump.** + +> Seams whose models never leave the process carry a `SEAM_VERSION` and no pinned schema. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `test/seam/test_signatures_seam_schema_pin.py` and `test/seam/test_port_io_schema_pin.py` hash each model's `model_json_schema()`, serialized with sorted keys, and hold it against the golden hashes in `test/seam/fixtures/`. +- The same tests also assert the seam version, make sure the seam module stays closed, and (signatures) that the package root re-exports the seam's own objects. +- To change the contract: change the model, bump the version and its expected value in the version test, run `test/seam/generate_*_golden.py`, and commit all of it together, in one go. The step-by-step procedure lives in `CONTRIBUTING.md`, "Pinned seam schemas". + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the schemas are published and checked by the platform side; +- a CI contract tool replaces the in-repository pins; +- dependency upgrades keep moving the hashes while the wire format stays exactly the same. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- These two pins predate the call to keep architecture checks out of the SDK tests. +- The golden files store hashes, not schemas. A failure tells you *that* a schema moved, never *how*: dumping `model_json_schema()` on both commits and diffing is the only way to see it. +- The hashed schema includes the docstrings of models and enums. Reword one and the pin trips, as surely as if you had retyped a field. +- Each pin test and its generator keep their own copy of the model list. Pin a new model and you edit both. + +--- + +## **Resources** +- [Schema pin tests](../../test/seam/) +- [Contributing: seam golden, which replaced these pins](../../CONTRIBUTING.md#seam-golden-testseam) diff --git a/libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md new file mode 100644 index 00000000..7e9c3713 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md @@ -0,0 +1,70 @@ +# Every piece of behaviour goes behind a port, even a one-liner +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, architecture] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Every piece of logic, even a one-line function, follows the same three-part pattern: an +agreement (the port), the code that does the job, and the public entry point people +call. No exception for small things. That way nobody wastes time arguing about which +rule applies. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Give a layering rule size-based exceptions and every new helper turns into a debate +> about whether it deserves a seam. The call was uniformity: "we want unicity everywhere +> and not some exception". + +--- + +## **The TDR** +| **Every piece of behaviour exposed by an SDK goes through a seam Protocol port, one _core implementation and a public adapter, whatever its size.** + +> Data contracts are not behaviour; they live in the seam directly (TDR [seam-holds-data-contracts-and-their-own-invariants](seam-holds-data-contracts-and-their-own-invariants.tdr.md)). + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +`is_truthy` gets `TruthinessPort`, `Truthiness` and the public `is_truthy`: exactly the +treatment `SignatureManager` gets. `build_marker` goes through `MarkerPort` / `MarkerBuilder`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Lesson | +| --- | --- | +| Reserve ports for real I/O boundaries; leave pure helpers such as `copy_dict` without a seam | Uniformity beats per-case judgement | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the boilerplate measurably slows contributions or reviews; +- a generator or base class can provide the port/adapter pair, changing what "one implementation per port" costs. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- The price is more files for trivial helpers. + +--- + +## **Resources** +- [Seam strategy, behind a port](../seam_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/every-sdk-module-imports-alone.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/every-sdk-module-imports-alone.tdr.md new file mode 100644 index 00000000..9e16227c --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/every-sdk-module-imports-alone.tdr.md @@ -0,0 +1,65 @@ +# Every SDK module imports on its own in a fresh interpreter +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, testing] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Every file of the SDK has to work as the very first thing you import, in a fresh Python, +with warnings turned into errors. That flushes out hidden loops between files, the kind +that only work because the tests happen to import things in a lucky order. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Three layers per capability, plus module-level default wiring: an import cycle can hide +> behind a lucky import order in the test session. + +--- + +## **The TDR** +| **Every SDK module must import successfully as the first import of a fresh interpreter running with -W error.** + + + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +`test_every_sdk_module_imports_first_in_a_fresh_interpreter` (in each SDK's +`test/test_boundaries.py`) runs `python -W error -c "import "` for every module. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- a CI tool checks import cycles statically and the subprocess cost becomes a problem. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- No material lesson beyond the TDR itself. + +--- + +## **Resources** +- [Boundaries test](../../test/test_boundaries.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md new file mode 100644 index 00000000..9d420154 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md @@ -0,0 +1,79 @@ +# Every seam object is fingerprinted +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, seam, testing, versioning] +status: draft +superseded_by: +supersedes: cross-process-seams-pin-their-json-schemas +Aliases: +Creation Date: 2026/10/15 08:36:35 +Modification Date: 2026/10/15 08:49:12 +--- + +## ELI5 + +The seam is the promise between the parts of the SDK, and part of that promise travels +to the platform. Every piece of it gets a fingerprint. Change a piece, even by accident, +and its fingerprint changes: a test goes red. Then you choose, on purpose, to bump the +version and save the new fingerprint. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Pinning the JSON schemas of two seam modules left the rest of the seam wide open: +> ports, enums living elsewhere, constants, functions. And each pin kept its own model +> list, which nobody remembered to extend. A seam change could walk right through with +> every test green. + +--- + +## **The TDR** +| **Every public top-level object of every seam module must be fingerprinted in one golden, discovered dynamically, and the golden is regenerated only together with a deliberate seam change and its version bump.** + +> No hand-kept list. The next run picks up any new seam module or object. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `test/seam/generate_seam_golden.py` parses every `*.py` file under `xtm_oaev_sdk/seam/` and fingerprints each public class, function, constant and type alias: the sha256 of its normalized AST, minus formatting, comments and docstrings. Pydantic models get their validation and serialization JSON schemas folded in too. +- It rewrites `test/seam/fixtures/seam_golden.json`: changed fingerprints updated, new objects added, deleted ones removed. It prints what moved, and warns when a capability changed but its version did not. +- `test/seam/test_seam_golden.py` recomputes the fingerprints and fails on anything added, removed or changed, and names every object. It also checks that the package root re-exports the seam's own objects. +- `test/seam/test_seam_closed.py` keeps the two closed-import checks: `port_io` imports only the signatures seam, and the signatures seam imports nothing from the SDK. +- collectors-sdk and injectors-sdk carry the same generator and golden test under their own `test/seam/`, run from their `libs/` directory. +- The step-by-step procedure lives in `CONTRIBUTING.md`, "Seam golden". + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +- **Keep the JSON schema pins, add more models to their lists.** Rejected: they only see pydantic models, and someone still has to keep the lists by hand. +- **Fingerprint `inspect.getsource()`.** Rejected: a comment or a reformat would trip it. That is noise, not a contract change. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the seam contracts are published and checked by the platform side; +- a CI contract tool replaces the in-repository golden; +- Python or pydantic upgrades keep moving fingerprints while no contract changed. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- The AST drops formatting and comments for free. Docstrings do not go quietly: strip them by hand. Skip the AST's empty fields too, or the text shifts between Python versions. +- A pydantic model's JSON schema carries its docstring and field descriptions. Those reach the wire, so rewording one counts as a change. +- The golden stores hashes, not definitions. A failure tells you which object moved; `git diff` on it tells you how. + +--- + +## **Resources** +- [Seam golden tests](../../test/seam/) +- [Contributing: seam golden](../../CONTRIBUTING.md#seam-golden-testseam) +- [Superseded: seams crossing a process boundary pin their JSON schemas](cross-process-seams-pin-their-json-schemas.tdr.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/import-from-the-package-root.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/import-from-the-package-root.tdr.md new file mode 100644 index 00000000..ee866067 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/import-from-the-package-root.tdr.md @@ -0,0 +1,70 @@ +# Consumers import from the package root only +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, public-api] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Always import from the top of the SDK: `from xtm_oaev_sdk import X`. The folders inside +(`public`, `seam`, `_core`) are the SDK's internal plumbing. What's in there can move at +any time, without asking you. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The layers are internal wiring. Refactors moved names between `public`, `seam` and +> `_core` modules (out of `seam/configuration/__init__.py`, for one) and the package root +> never flinched. + +--- + +## **The TDR** +| **Consumers must import SDK names from the package root (xtm_oaev_sdk, collectors_sdk, injectors_sdk); public, seam and _core module paths are internal and may move.** + +> The package root's `__all__` is the stable API. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +from xtm_oaev_sdk import Configuration, ContractBuilder, SignatureManager +``` + +Every legacy shim message points to the root import: `Use 'from xtm_oaev_sdk import X' instead.` + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the root namespace grows too large to stay flat and sub-namespaces become public; +- a consumer needs a seam type that the root does not export. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- A surface tool that treats every module as public reports internal moves as removals. + +--- + +## **Resources** +- [README](../../README.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/importing-pyoaev-stays-silent.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/importing-pyoaev-stays-silent.tdr.md new file mode 100644 index 00000000..6fcbdce5 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/importing-pyoaev-stays-silent.tdr.md @@ -0,0 +1,71 @@ +# Importing pyoaev never warns; only touching a moved name does +--- +scope: [pyoaev, shims, deprecation] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Writing `import pyoaev` must never show a deprecation warning. A warning only shows up +when you actually use a name that moved to the SDKs. Otherwise it would warn people +about code they never touched, and crash programs that treat warnings as errors. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> A warning on a bare `import pyoaev` hits code that never touched a moved name, and it +> crashes anything running under `-W error`. + +--- + +## **The TDR** +| **Importing any pyoaev module must not emit a DeprecationWarning; only accessing a moved name may warn.** + +> `pyoaev`'s own code imports from the SDKs, never from its shims. And no module +> star-imports a shim module. Ever. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- The package root binds the shim submodules and serves their names through its own `__getattr__`; `pyoaev.__all__` is unchanged. +- `test_importing_pyoaev_modules_is_silent` imports every `pyoaev` module under `-W error`. +- `test_sdk_objects_stay_silent_once_pyoaev_is_loaded` checks that SDK users get no warning. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| Star-import shim modules from `pyoaev/__init__` | `import *` walks `__all__` and trips every lazy name: `import pyoaev` warned, and crashed under `-W error` | Serve package-level names through the root `__getattr__` | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the shims are removed; +- a moved name has to be eagerly bound at import time for a technical reason. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- `from pyoaev import *` still warns for the moved names in `pyoaev.__all__`. That is the signal doing its job, not a leak. + +--- + +## **Resources** +- [Shim contract tests](../../test/common/shim/test_shim_contract.py) +- [collectors/injectors shim tests](../../../../test/test_sdk_split_shims.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/legacy-names-served-by-pep562-module-getattr.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/legacy-names-served-by-pep562-module-getattr.tdr.md new file mode 100644 index 00000000..2f4135db --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/legacy-names-served-by-pep562-module-getattr.tdr.md @@ -0,0 +1,82 @@ +# Legacy names are served by a directly assigned PEP 562 __getattr__ +--- +scope: [pyoaev, deprecation, shims] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +When old code asks pyoaev for a name that moved, a small lookup function in the old +module catches the request. It shows a warning pointing to the new import, then hands +back the real SDK object. It's plugged in directly, so the warning points at your line +of code, not at pyoaev's. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Legacy access has three jobs: warn, so people migrate; return the SDK object, so nothing +> else moves; and leave SDK users alone. + +--- + +## **The TDR** +| **A moved name must be served from its legacy module by a module-level __getattr__ built with make_deprecated_getattr and assigned directly, never wrapped in a def.** + +> The returned `__getattr__` warns with `DeprecationWarning` and `stacklevel=2`, which pins +> the warning on the consumer's line. Unknown names still raise `AttributeError`. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) +``` + +`__all__` lists the lazily served names, flagged with `# noqa: F822`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| `typing_extensions.deprecated(...)(cls)` | Patches the original class in place: the moment anything imported `pyoaev`, `xtm_oaev_sdk.Configuration()` warned too | Never mutate the SDK object to deprecate its legacy name | +| `def __getattr__(name): return _deprecated_getattr(name)` | The extra frame pins the warning on the shim file, and the default filters hide it from scripts | Assign the built `__getattr__` directly | +| Wrapper class or proxy | Breaks identity: `isinstance`, metaclasses, signature introspection | See TDR [shims-return-the-identical-sdk-object](shims-return-the-identical-sdk-object.tdr.md) | +| Plain re-export | Keeps identity, never warns. Nobody ever migrates | The warning is the migration signal | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the shims are removed. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- `from import X` on a shim package warns twice (importlib's `_handle_fromlist` probes with `hasattr`); plain modules warn once. Accepted: that one is baked into PEP 562. + +--- + +## **Resources** +- [Shim strategy, mechanism](../shim_strategy.md) +- [Deprecation adapter](../../xtm_oaev_sdk/public/deprecation.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/legacy-patch-targets-are-part-of-the-surface.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/legacy-patch-targets-are-part-of-the-surface.tdr.md new file mode 100644 index 00000000..c1dd05e8 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/legacy-patch-targets-are-part-of-the-surface.tdr.md @@ -0,0 +1,70 @@ +# Legacy patch targets are part of the compatibility surface +--- +scope: [pyoaev, shims, testing, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +The connectors' tests swap some pyoaev pieces for fakes: they "patch" them. Those pieces +have to stay where the tests expect them, and swapping them has to keep affecting the +code that really runs. Otherwise those tests break, with no import error to tell you +why. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Connector test suites patch `pyoaev` module attributes. Before a single line moved, +> every `pyoaev` patch target used by the connectors' tests was listed. Move the code +> without keeping them and those suites break, without a single import error. + +--- + +## **The TDR** +| **Module attributes that connectors patch must stay reachable at their legacy path and still reach the code that actually runs.** + + + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `pyoaev.signatures.signature_manager` still imports `socket` and `subprocess`: patching `...signature_manager.socket.gethostbyname` reaches the shared stdlib module the SDK uses. +- The client-building adapters call `OpenAEV` through their module global, so `patch("pyoaev.daemons.base_daemon.OpenAEV")` and `patch("pyoaev.helpers.OpenAEV")` apply. +- `pyoaev.utils.PingAlive.start` and `pyoaev.daemons.collector_daemon.CollectorDaemon._setup` land on the class the connectors instantiate (TDR [shims-return-the-identical-sdk-object](shims-return-the-identical-sdk-object.tdr.md)). + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored: the patch targets counted as surface from day one. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the connectors stop patching `pyoaev` paths; +- the shims are removed. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- A patch target can be a stdlib module re-bound in a legacy module. Keep the import line and it keeps working; that is all it takes. + +--- + +## **Resources** +- [collectors/injectors shim tests](../../../../test/test_sdk_split_shims.py) +- [Shim strategy, rules learned the hard way](../shim_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/legacy-surface-guarded-from-the-release.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/legacy-surface-guarded-from-the-release.tdr.md new file mode 100644 index 00000000..5b2dc520 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/legacy-surface-guarded-from-the-release.tdr.md @@ -0,0 +1,68 @@ +# The legacy surface is guarded against the released pyoaev +--- +scope: [pyoaev, testing, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +We keep a list of every public name in the pyoaev people actually installed, and a test +checks they all still work. That list comes from the real release, not from our work in +progress. Otherwise it would happily accept a name we lost by mistake. The old tests of +the moved code stay exactly as they were. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Generate the guard from the working tree and it blesses whatever the split produced, +> lost names included. The reference has to be what users actually installed. + +--- + +## **The TDR** +| **The legacy-surface guard must be generated from the released pyoaev package, and the pre-split tests of moved code must be kept verbatim against their legacy paths.** + +> New behaviour tests target the SDKs. The moved `pyoaev` tests live under +> `libs/xtm-oaev-sdk/test//shim/`, still hammering the legacy imports. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `test/generate_legacy_surface.py` records `test/fixtures/legacy_surface.json` from `pyoaev`. +- `test/test_legacy_surface.py` checks every one still resolves. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the shims are removed. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- Moved verbatim, the old tests become a second guard: same assertions, still driving the legacy paths. + +--- + +## **Resources** +- [Legacy surface guard](../../../../test/test_legacy_surface.py) +- [Legacy surface generator](../../../../test/generate_legacy_surface.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/no-breaking-change-for-legacy-pyoaev-code.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/no-breaking-change-for-legacy-pyoaev-code.tdr.md new file mode 100644 index 00000000..7aa51332 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/no-breaking-change-for-legacy-pyoaev-code.tdr.md @@ -0,0 +1,79 @@ +# Legacy pyoaev code keeps working until a pyoaev major release +--- +scope: [pyoaev, xtm-oaev-sdk, collectors-sdk, injectors-sdk, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +pyoaev used to hold everything, and its shared code moved into the SDKs. Old code that +imports from pyoaev must keep working, with nothing more than a warning telling where +each thing went. That holds until a future major pyoaev release removes the old paths. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> `pyoaev` was a monolith: the OpenAEV API client, with configuration, contracts, +> signatures, the daemon runtime, errors and helpers piled on top. That shared code now +> lives in `xtm-oaev-sdk`, `collectors-sdk` and `injectors-sdk`. +> +> The collectors and injectors in the connector repositories import the former `pyoaev` +> paths, and their test suites patch them. The split had to land without dragging every +> connector into a migration on the same day: "no breaking change for old code". + +--- + +## **The TDR** +| **Every public name of the latest released pyoaev must keep resolving from its legacy path, with a DeprecationWarning naming its new import, until a pyoaev major release removes the shims.** + +> That covers names, the behaviour of legacy constructors (see TDR [shims-return-the-identical-sdk-object](shims-return-the-identical-sdk-object.tdr.md)) and the patch +> targets connectors lean on (see TDR [legacy-patch-targets-are-part-of-the-surface](legacy-patch-targets-are-part-of-the-surface.tdr.md)). What moves into the SDKs is the released +> `pyoaev` behaviour. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `from pyoaev.configuration import Configuration` warns and returns `xtm_oaev_sdk.Configuration`. +- `test/test_legacy_surface.py` checks that the names in modules of `pyoaev` still resolve (TDR [legacy-surface-guarded-from-the-release](legacy-surface-guarded-from-the-release.tdr.md)). +- The real collectors|injectors own suites ran against the released `pyoaev`, then against the new stack. Identical results. +- The removal steps for the next major release are already written down in `docs/shim_strategy.md`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- a `pyoaev` major release is planned: the shims are removed and this TDR is revoked or superseded; +- every collector|injector imports from the SDKs and nobody consumes the legacy paths anymore; +- a deprecation policy fixes a time- or release-bound window instead of "until the next major". + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- The legacy surface runs wider than `__all__`: collectors also use names a module merely imported (`pyoaev.daemons.base_daemon.Configuration`), and patch targets. +- A few observable differences are not API breaks. They were accepted, then documented: warnings on legacy access, SDK module names for unnamed loggers, thin subclasses for the client-building adapters. + +--- + +## **Resources** +- [Shim strategy](../shim_strategy.md) +- [Legacy surface guard](../../../../test/test_legacy_surface.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/pass-bound-callables-to-keep-dispatch.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/pass-bound-callables-to-keep-dispatch.tdr.md new file mode 100644 index 00000000..87767310 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/pass-bound-callables-to-keep-dispatch.tdr.md @@ -0,0 +1,77 @@ +# Adapters pass their own methods to the core to keep dispatch and patch points +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Some SDK classes used to call their own methods, and that logic moved to the core. So +now the class hands its own method to the core, instead of letting the core redo the +call. Override that method or fake it, and it still counts. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Logic moves to the core. If the core then reimplemented the calls that used to go +> through `self`, subclass overrides and test patches would be bypassed without a sound. + +--- + +## **The TDR** +| **When the original code dispatched through its own object, the adapter must pass the bound method or a callable to the core instead of letting the core reimplement the call.** + +> Same deal for a dependency the core may not import: the adapter passes it down. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +def _source_context(self, config): + return _engine.source_context( + config, self.resolve_container_ip, lambda: self._cached_ipv6 + ) +``` + +- `SignatureManager.post_execution_updates` passes `self._utcnow`. +- `Expectation.match_alert` passes `self.match_fuzzy` / `self.match_simple`. +- `OpenAEVDetectionHelper` passes `self.match_alert_element_fuzzy` / `self._decode_value`. +- `CollectorDaemon` passes `self._try_callback` to the scheduler. +- `Configuration.schema()` passes the public `ConnectorConfigSchemaGenerator` to the core resolver. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the hooks are formally deprecated and no longer need to stay overridable; +- the core gets an explicit strategy object replacing the callables. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- Call order is part of the behaviour: `_utcnow` is still read only after the tool output validates. + +--- + +## **Resources** +- [Seam strategy, pass callables](../seam_strategy.md) +- [Signatures adapter](../../xtm_oaev_sdk/public/signatures.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/public-class-extends-seam-data.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/public-class-extends-seam-data.tdr.md new file mode 100644 index 00000000..0c10066b --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/public-class-extends-seam-data.tdr.md @@ -0,0 +1,79 @@ +# Data-and-behaviour classes split: the public class extends the seam data +--- +scope: [xtm-oaev-sdk, collectors-sdk, architecture] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Some classes, like `Contract`, hold both data and actions. We cut them in two: the plain +data lives in the seam (the agreement layer), and the public class adds the actions on +top. You still create `Contract(...)` exactly as before. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> `Contract` and `Expectation` are what users construct, and both mix data with behaviour +> (default variables, serialization, matching, updates). The seam cannot depend on +> behaviour. Users must keep constructing the same class. + +--- + +## **The TDR** +| **When a user-facing class mixes data and behaviour, the seam holds the data class and the public class extends it, adding behaviour through ports; the core only knows the seam type.** + + + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +@dataclass +class Contract(ContractData): + variables: list[ContractVariable] = field( + default_factory=lambda: _variables.default_variables() + ) + + def to_contract_add_input(self, source_id: str) -> dict[str, Any]: + return _serializer.to_add_input(self, source_id) +``` + +collectors-sdk plays the same move with `ExpectationData` / `Expectation`. After the +contracts split, the JSON of `to_contract_add_input()`, `to_contract_update_input()` and +`prepare_contracts()` came out byte-identical to the previous commit. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the data and behaviour can be separated in the user-facing API itself (a major version); +- the core needs behaviour from the public subclass. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: + +--- + +## **Resources** +- [Seam strategy, patterns](../seam_strategy.md) +- [Contracts adapter](../../xtm_oaev_sdk/public/contracts.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/public-seam-core-dependency-rule.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/public-seam-core-dependency-rule.tdr.md new file mode 100644 index 00000000..f3e735b3 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/public-seam-core-dependency-rule.tdr.md @@ -0,0 +1,73 @@ +# public and _core depend on the seam; the seam depends on neither +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, architecture] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Each part of an SDK has three layers: `public` (what users touch), `_core` (the engine) +and the seam (the agreement between the two). Both sides lean on the agreement, never on +each other. The one exception: `public` chooses which engine to plug in. So the engine +can be rebuilt freely, as long as it keeps the agreement. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The SDKs were designed in three layers: `public` is what users get, `_core` is the code +> behind it, and the seam is the contract between the two, so that "as long as `_core` +> answers it, `public` stays usable with no breaking change". + +--- + +## **The TDR** +| **Within each SDK capability, the seam imports neither public nor _core, _core never imports public, and public imports _core only to wire the default implementation of each port.** + +> Public never hands out a core object. Every exported name is either an adapter defined +> in `public/` or a contract defined in `seam/`. The seam may import the standard library, +> third-party libraries, other packages' public API and sibling seam modules, and that is +> the whole list. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +_truthiness: TruthinessPort = Truthiness() +_sources: ConfigurationSourcesPort = ConfigurationSources() +_resolver: ConfigurationResolverPort = ConfigurationResolver(_sources, _truthiness) +``` + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- users need to swap implementations at runtime beyond the module-level defaults; + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: + +--- + +## **Resources** +- [Seam strategy, rules](../seam_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md new file mode 100644 index 00000000..a47af668 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md @@ -0,0 +1,72 @@ +# pyoaev hard-depends on the SDKs that host its former code +--- +scope: [pyoaev, packaging, dependencies, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Old connectors only install pyoaev, but what they use lives in the SDKs now. So +installing pyoaev always installs the SDKs too, not only when you ask for an option. +That's what keeps old connectors running while the old paths still exist. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> A connector that lists only `pyoaev` still imports `CollectorDaemon`, +> `OpenAEVInjectorHelper` and friends from their legacy paths. Those names live in the +> SDKs now. For TDR [no-breaking-change-for-legacy-pyoaev-code](no-breaking-change-for-legacy-pyoaev-code.tdr.md) to hold, `pyoaev` has to carry them in. + +--- + +## **The TDR** +| **While pyoaev ships deprecation shims, it must declare a hard dependency on every SDK those shims serve, never an optional extra.** + +> Third-party packages `pyoaev` no longer imports itself (`pika`, `thefuzz`, `PyYAML`, +> `pydantic-settings`, `python-json-logger`) are gone from its own declaration. They still +> ride in transitively through the SDKs, so an install pulls the same packages as before. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `pyproject.toml` requires `xtm-oaev-sdk`, `collectors-sdk` and `injectors-sdk` (`>=0.1.0,<0.2.0`). +- Until the SDKs are published, CI pulls collectors-sdk and injectors-sdk straight from git through `libs/sdk-requirements.txt`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| Optional extras (`pyoaev[collectors]`, `pyoaev[injectors]`) with lazily loaded SDKs | Proposed to spare every `pyoaev` install both SDKs; legacy imports would then blow up for connectors listing only `pyoaev` | Rejected: no breaking change wins | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the shims are removed: `collectors-sdk` and `injectors-sdk` can be dropped (keep `xtm-oaev-sdk` if the client still uses its errors and transport models); +- an SDK gains heavy dependencies that `pyoaev` users should not pay for. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- The install footprint did not grow. What moved out of `pyoaev` comes straight back in through the SDKs. + +--- + +## **Resources** +- [pyproject.toml](../../../../pyproject.toml) +- [SDK requirements for CI](../../../../libs/sdk-requirements.txt) diff --git a/libs/xtm-oaev-sdk/docs/tdr/sdks-never-import-pyoaev.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/sdks-never-import-pyoaev.tdr.md new file mode 100644 index 00000000..64137eeb --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/sdks-never-import-pyoaev.tdr.md @@ -0,0 +1,79 @@ +# The SDKs never depend on pyoaev; the API client is injected +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, dependencies] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +The SDKs never use pyoaev. When they need to talk to the OpenAEV platform, you hand them +the client (`api_client=` or `transport=`). pyoaev already depends on the SDKs: the +other way around would make a loop, and chain the SDKs to one specific client. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> `pyoaev` depends on the SDKs (TDR [pyoaev-hard-depends-on-the-sdks-during-deprecation](pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md)) and is meant to end up as the OpenAEV HTTP +> client, and only that. The SDKs carry the shared, client-free runtime. One import back +> into `pyoaev` and you get a circular package dependency, with that runtime welded to a +> single client. + +--- + +## **The TDR** +| **xtm-oaev-sdk, collectors-sdk and injectors-sdk must never import pyoaev; anything that talks to the platform receives its API client or transport as an argument typed by a seam protocol.** + +> The injected types are `BaseClient` (seam `common`) and `SignatureTransportProtocol` +> (seam `signatures`). `pyoaev.OpenAEV` and its signature API satisfy them. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +SignatureManager(transport=api.signature) +CollectorDaemon(configuration=configuration, callback=collect, api_client=api, collector_type="...") +OpenAEVInjectorHelper(config, icon, api_client=api) +``` + +- No injected client? `BaseDaemon` and `OpenAEVInjectorHelper` fall back on the overridable `_create_default_api_client` hook, and the SDK default raises rather than build one. +- `pyoaev` brings the legacy self-building constructors back by overriding that hook (TDR [shims-return-the-identical-sdk-object](shims-return-the-identical-sdk-object.tdr.md)). +- `test/test_boundaries.py` in each SDK fails the moment a source imports `pyoaev` (or `requests`). + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| Keep the legacy constructors that built `OpenAEV(...)` themselves | That drags `pyoaev` into the SDK | Client construction lives in `pyoaev`'s client-building adapters, and only there | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the API client itself moves into an SDK; +- an SDK needs a client capability that cannot be expressed as a protocol without importing `pyoaev` types. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- Pulling client construction out took a hook (`_create_default_api_client`): that is how `pyoaev` restores the old constructors without the SDK ever knowing. + +--- + +## **Resources** +- [xtm-oaev-sdk boundaries test](../../test/test_boundaries.py) +- [Shim strategy, dependency direction](../shim_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md new file mode 100644 index 00000000..29c37ee5 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md @@ -0,0 +1,70 @@ +# A seam change is a versioned contract change +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, versioning] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +The seam is the agreement between the public part and the engine. Changing the engine is +free. Changing the agreement changes things for users, so it gets a new version number +and a changelog line. And if users can see the change, the old way keeps working, with a +warning, first. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The whole point of the seam: the public API survives any core rewrite. Touch the seam +> itself and you have changed the contract, and unlike a core change, that has to be +> announced. + +--- + +## **The TDR** +| **Any change to a seam contract (field, type, enum member, error, port signature) must bump that seam's version and be recorded in the changelog, and a change reaching the public surface must go through a deprecation shim first.** + +> Every capability seam exposes `SEAM_VERSION`; `port_io` is versioned by +> `PORT_IO_CONTRACT_VERSION`. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- every capability seam of the three SDKs defines `SEAM_VERSION`. +- `seam/port_io/models.py` defines `PORT_IO_CONTRACT_VERSION`. +- Deprecations use `make_deprecated_getattr` (TDR [legacy-names-served-by-pep562-module-getattr](legacy-names-served-by-pep562-module-getattr.tdr.md)). + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- seam versions get derived automatically by tooling; +- the release process ties seam versions to package versions. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- No changelog file exists in the SDKs yet. The first seam change gets to create it. + +--- + +## **Resources** +- [Seam strategy, versioning](../seam_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md new file mode 100644 index 00000000..e38d026d --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md @@ -0,0 +1,69 @@ +# The seam holds data contracts and only their intrinsic invariants +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, architecture] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +The seam only holds data: models, lists of allowed values, errors, and small rules that +check or compute something from that data alone. Anything that talks to the outside +world, runs the show, or needs another part of the SDK goes behind a port instead. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The seam must stay closed (TDR [public-seam-core-dependency-rule](public-seam-core-dependency-rule.tdr.md)), yet the data crossing it carries rules of its +> own: validators, derived fields, errors that extract their own message. Those rules are +> part of what the data is. I/O and orchestration are not. + +--- + +## **The TDR** +| **A seam may contain data contracts and logic that only reads or derives from their own data; anything doing I/O, orchestrating, or needing another capability goes behind a port.** + +> Data contracts: pydantic models, dataclasses, enums, exceptions, type aliases, constants. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- validators and derived fields: `ConfigurationHint`, `ExpectationData`'s null signatures coerced to `[]`; +- `__post_init__` invariants of the contract elements; +- value objects: `SignatureMatch`, `SignatureType.make_struct_for_matching`; +- exceptions, including `OpenAEVError`'s message extraction: the core raises them and users catch them, so both sides must reach them. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- an invariant needs external state or another capability: it must move behind a port; +- the seams are generated from an external schema that cannot carry logic. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- Exceptions belong in the seam: both sides of the boundary need them. + +--- + +## **Resources** +- [Seam strategy, what goes where](../seam_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/shims-return-the-identical-sdk-object.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/shims-return-the-identical-sdk-object.tdr.md new file mode 100644 index 00000000..d844f7ce --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/shims-return-the-identical-sdk-object.tdr.md @@ -0,0 +1,80 @@ +# Shims return the identical SDK object; adapters only where constructors differ +--- +scope: [pyoaev, shims, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +When old code asks pyoaev for a moved name, it gets the very same object the SDK +provides. Not a copy, not a look-alike, so checks like `isinstance` and `is` keep +working. The only exceptions are the classes that used to build the API client +themselves: they come back as a thin subclass that still does it. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Legacy code compares with `is`, checks `isinstance`, subclasses, pickles by reference +> and patches. Hand it anything but the SDK object itself and one of those can snap. Some +> legacy constructors, though, built the `OpenAEV` client themselves, and the SDKs no +> longer do that (TDR [sdks-never-import-pyoaev](sdks-never-import-pyoaev.tdr.md)). + +--- + +## **The TDR** +| **A shim must return the SDK object itself; a legacy name may resolve to a thin subclass only to restore legacy constructor behaviour, and that subclass must be one single object whatever legacy path reaches it.** + +> A legacy value with no public replacement (`variable_helper.USER`…, `helpers.TRUTHY` / +> `FALSY`) is served as the literal value, with a message pointing at the right API. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +| Legacy name | Subclass of | Adds | +|---|---|---| +| `pyoaev.daemons.BaseDaemon` | `xtm_oaev_sdk.BaseDaemon` | `_create_default_api_client()` builds `OpenAEV` | +| `pyoaev.daemons.CollectorDaemon` | `collectors_sdk.CollectorDaemon` + the legacy `BaseDaemon` | same hook | +| `pyoaev.signatures.SignatureManager` | `xtm_oaev_sdk.SignatureManager` | `SignatureManager(client, logger)` sends through `client.signature` | +| `pyoaev.helpers.OpenAEVInjectorHelper` | `injectors_sdk.OpenAEVInjectorHelper` | `_create_default_api_client(config)` builds `OpenAEV` | + +`pyoaev.daemons.CollectorDaemon is pyoaev.daemons.collector_daemon.CollectorDaemon is pyoaev.helpers.CollectorDaemon`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| Wrapper class or proxy for every legacy name | Breaks identity, `isinstance` against the SDK class, metaclasses and introspection | Return the object; subclass only where behaviour must be restored | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the SDK classes gain a client factory that makes the adapters unnecessary; +- the shims are removed. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- The legacy `CollectorDaemon` has to inherit from the legacy `BaseDaemon` adapter too. Skip that and `isinstance(daemon, pyoaev.daemons.BaseDaemon)` fails. + +--- + +## **Resources** +- [Shim strategy, client-building adapters](../shim_strategy.md) +- [Shim contract tests](../../test/daemons/shim/test_shim_contract.py) +- [collectors/injectors shim tests](../../../../test/test_sdk_split_shims.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/xtm-oaev-sdk-stays-connector-agnostic.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/xtm-oaev-sdk-stays-connector-agnostic.tdr.md new file mode 100644 index 00000000..3429f778 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/xtm-oaev-sdk-stays-connector-agnostic.tdr.md @@ -0,0 +1,67 @@ +# xtm-oaev-sdk stays agnostic of collectors, injectors and transports +--- +scope: [xtm-oaev-sdk, dependencies, architecture] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +xtm-oaev-sdk is the shared base of both collectors and injectors. Nothing in it may +serve only one of them, and it stays away from libraries that belong elsewhere: RabbitMQ +and fuzzy matching go with injectors-sdk and collectors-sdk, HTTP requests with pyoaev. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> xtm-oaev-sdk is "the common sdk" under collectors-sdk and injectors-sdk. Put collector- +> or injector-specific code in it and you force it on the other connector type. + +--- + +## **The TDR** +| **xtm-oaev-sdk must not contain collector- or injector-specific runtime, nor import pika, thefuzz or requests; those belong to collectors-sdk, injectors-sdk or pyoaev.** + + + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +`test/test_boundaries.py` fails if an SDK source imports `pyoaev`, `requests`, `pika` or +`thefuzz`, or defines `ListenQueue`, `OpenAEVConfigHelper`, `OpenAEVCollectorHelper`, +`OpenAEVInjectorHelper`, `OpenAEVDetectionHelper` or `create_mq_ssl_context`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- collectors and injectors converge on a shared transport; +- a third connector type needs part of a type-specific runtime. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- No material lesson beyond the TDR itself. + +--- + +## **Resources** +- [Boundaries test](../../test/test_boundaries.py) diff --git a/libs/xtm-oaev-sdk/pyproject.toml b/libs/xtm-oaev-sdk/pyproject.toml new file mode 100644 index 00000000..1731914c --- /dev/null +++ b/libs/xtm-oaev-sdk/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "xtm-oaev-sdk" +version = "0.1.0" +description = "OpenAEV SDK core (DDD + Light Hex Architecture): configuration, contracts, signatures, and shared utilities." +authors = [ + {name = "Filigran", email = "contact@filigran.io"} +] +license = { text = "Apache-2.0" } +requires-python = ">=3.12" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Libraries :: Python Modules" +] +dependencies = [ + "pydantic (>=2.13.3,<2.14.0)", + "pydantic-settings (>=2.15.0,<2.16.0)", + "PyYAML (>=6.0,<6.1)", + "python_json_logger (>=3.3.0,<3.4.0)", +] + +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["xtm_oaev_sdk*"] + +[tool.setuptools.package-data] +xtm_oaev_sdk = ["py.typed"] diff --git a/test/configuration/__init__.py b/libs/xtm-oaev-sdk/test/common/__init__.py similarity index 100% rename from test/configuration/__init__.py rename to libs/xtm-oaev-sdk/test/common/__init__.py diff --git a/test/contracts/__init__.py b/libs/xtm-oaev-sdk/test/common/shim/__init__.py similarity index 100% rename from test/contracts/__init__.py rename to libs/xtm-oaev-sdk/test/common/shim/__init__.py diff --git a/test/test_exceptions.py b/libs/xtm-oaev-sdk/test/common/shim/test_exceptions.py similarity index 100% rename from test/test_exceptions.py rename to libs/xtm-oaev-sdk/test/common/shim/test_exceptions.py diff --git a/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py new file mode 100644 index 00000000..116a8644 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py @@ -0,0 +1,144 @@ +"""Contract of the pyoaev shims for the shared (common) capability. + +Legacy imports must keep working, return the *identical* xtm_oaev_sdk object, +and warn; importing pyoaev's own modules and using the SDK directly must stay +silent. +""" + +import subprocess +import sys +import warnings + +import pytest +import xtm_oaev_sdk as sdk + +_ERRORS = [ + "ConfigurationError", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SignatureTransmissionError", +] +_UTILS = [ + "AppLogger", + "CustomJsonFormatter", + "EnhancedJSONEncoder", + "logger", + "setup_logging_config", +] +_SSL = [ + "data_to_temp_file", + "is_memory_certificate", + "ssl_cert_chain", + "ssl_verify_locations", +] +_ASSET_TYPES = [ + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "CloudProvider", +] + +LEGACY_NAMES = ( + [("pyoaev.exceptions", name) for name in _ERRORS] + + [("pyoaev.utils", name) for name in _UTILS] + + [("pyoaev.helpers", name) for name in _SSL] + + [("pyoaev.asset_types", name) for name in _ASSET_TYPES] + + [("pyoaev.security_domain.types", "SecurityDomains")] + + [("pyoaev", name) for name in _ASSET_TYPES] + + [("pyoaev", name) for name in _ERRORS if name != "OpenAEVError"] +) + + +_CONSUMER = "legacy_consumer.py" + + +def _given_fresh_interpreter_script(script: str) -> list[str]: + return [sys.executable, "-W", "error::DeprecationWarning", "-c", script] + + +def _when_run(command: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(command, capture_output=True, text=True) + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec( + compile(f"from {module} import {name} as _legacy", _CONSUMER, "exec"), + namespace, + ) + return namespace["_legacy"], caught + + +def _then_succeeds(result: subprocess.CompletedProcess) -> None: + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize(("module", "name"), LEGACY_NAMES) +def test_legacy_import_warns_and_returns_the_sdk_object(module, name) -> None: + """It warns towards xtm_oaev_sdk and returns the identical object.""" + legacy, caught = _when_import_legacy_name(module, name) + assert legacy is getattr(sdk, name) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert deprecations + assert f"from xtm_oaev_sdk import {name}" in str(deprecations[0].message) + # attributed to the consumer's line, so default warning filters show it + assert any(w.filename == _CONSUMER for w in deprecations) + + +def test_pyoaev_all_still_lists_legacy_exceptions() -> None: + """It keeps the legacy star-export surface of pyoaev.""" + import pyoaev + + for name in _ERRORS: + if name != "OpenAEVError": + assert name in pyoaev.__all__ + for name in _ASSET_TYPES: + assert name in pyoaev.__all__ + + +def test_importing_pyoaev_modules_is_silent() -> None: + """It imports pyoaev and its own modules without any DeprecationWarning.""" + script = ( + "import pyoaev, pyoaev.apis, pyoaev.base, pyoaev.client, pyoaev.mixins\n" + "import pyoaev.helpers, pyoaev.daemons, pyoaev.contracts, pyoaev.signatures\n" + "import pyoaev.utils, pyoaev.exceptions, pyoaev.asset_types\n" + "import pyoaev.security_domain.types\n" + ) + result = _when_run(_given_fresh_interpreter_script(script)) + _then_succeeds(result) + + +def test_sdk_objects_stay_silent_once_pyoaev_is_loaded() -> None: + """It never alters SDK objects: native use stays silent next to pyoaev.""" + script = ( + "import pyoaev, pyoaev.exceptions, pyoaev.utils, pyoaev.helpers\n" + "import xtm_oaev_sdk as sdk\n" + "sdk.EnhancedJSONEncoder().encode({'a': 1})\n" + "sdk.AssetCategory('WEB_APPLICATION')\n" + "try:\n" + " raise sdk.OpenAEVHttpError('boom', 500)\n" + "except sdk.OpenAEVError:\n" + " pass\n" + ) + result = _when_run(_given_fresh_interpreter_script(script)) + _then_succeeds(result) + + +@pytest.mark.parametrize( + "module", ["pyoaev.exceptions", "pyoaev.utils", "pyoaev.helpers"] +) +def test_unknown_legacy_name_still_raises(module: str) -> None: + """It does not swallow genuine misses.""" + with pytest.raises(ImportError): + exec(f"from {module} import DoesNotExist", {}) diff --git a/libs/xtm-oaev-sdk/test/common/shim/test_utils.py b/libs/xtm-oaev-sdk/test/common/shim/test_utils.py new file mode 100644 index 00000000..7c00bfe1 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/common/shim/test_utils.py @@ -0,0 +1,94 @@ +import dataclasses +import json +import logging +import unittest +from unittest.mock import MagicMock, patch + +from pythonjsonlogger.json import JsonFormatter + +import pyoaev.utils as module + + +@dataclasses.dataclass +class _SampleData: + value: int + + +class TestUtils(unittest.TestCase): + def test_custom_json_formatter_inherits_non_deprecated_formatter(self): + self.assertTrue(issubclass(module.CustomJsonFormatter, JsonFormatter)) + + def test_enhanced_json_encoder_serializes_not_dataclasses(self): + data = {"test": "this is a test", "other": 3} + classic_json_dump = json.dumps(data) + enhanced_json_dump = json.dumps(data, cls=module.EnhancedJSONEncoder) + self.assertFalse(dataclasses.is_dataclass(data)) + self.assertEqual( + classic_json_dump, + enhanced_json_dump, + ) + + def test_enhanced_json_encoder_serializes_dataclasses(self): + self.assertEqual( + json.dumps(_SampleData(value=3), cls=module.EnhancedJSONEncoder), + '{"value": 3}', + ) + + def test_custom_json_formatter_add_fields_sets_timestamp_and_level(self): + formatter = module.CustomJsonFormatter("%(message)s") + record = logging.LogRecord( + name="test", + level=logging.INFO, + pathname=__file__, + lineno=1, + msg="hello", + args=(), + exc_info=None, + ) + log_record = {} + + formatter.add_fields(log_record, record, {}) + + self.assertIn("timestamp", log_record) + self.assertEqual(log_record["level"], "INFO") + + def test_setup_logging_config_json_logging_true_uses_custom_formatter(self): + with patch("logging.basicConfig") as mock_basic_config: + module.setup_logging_config(logging.INFO, json_logging=True) + + kwargs = mock_basic_config.call_args.kwargs + self.assertEqual(kwargs["level"], logging.INFO) + self.assertIn("handlers", kwargs) + self.assertIsInstance( + kwargs["handlers"][0].formatter, module.CustomJsonFormatter + ) + + def test_setup_logging_config_json_logging_false_calls_basic_config(self): + with patch("logging.basicConfig") as mock_basic_config: + module.setup_logging_config(logging.WARNING, json_logging=False) + + mock_basic_config.assert_called_once_with(level=logging.WARNING) + + def test_app_logger_methods_delegate_to_local_logger(self): + with patch("xtm_oaev_sdk.public.common.setup_logging_config"): + app_logger = module.AppLogger(logging.INFO) + app_logger.local_logger = MagicMock() + + app_logger.debug("d", {"x": 1}) + app_logger.info("i") + app_logger.warning("w") + app_logger.error("e") + + self.assertTrue(app_logger.local_logger.debug.called) + self.assertTrue(app_logger.local_logger.info.called) + self.assertTrue(app_logger.local_logger.warning.called) + self.assertTrue(app_logger.local_logger.error.called) + + def test_logger_helper_returns_app_logger(self): + with patch("xtm_oaev_sdk.public.common.setup_logging_config"): + helper = module.logger(logging.INFO) + self.assertIsInstance(helper, module.AppLogger) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/common/test_exceptions.py b/libs/xtm-oaev-sdk/test/common/test_exceptions.py new file mode 100644 index 00000000..a98afda0 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/common/test_exceptions.py @@ -0,0 +1,62 @@ +import unittest +from unittest.mock import MagicMock + +import xtm_oaev_sdk as module + + +class TestExceptions(unittest.TestCase): + def test_openaeverror_init(self): + error_message = MagicMock() + response_code = MagicMock() + response_body = MagicMock() + + exception = module.OpenAEVError(error_message, response_code, response_body) + + self.assertEqual(exception.response_code, response_code) + self.assertEqual(exception.response_body, response_body) + self.assertEqual(exception.error_message, error_message.decode.return_value) + + def test_openaeverror_init_auto_decode(self): + response_code = MagicMock() + response_body = MagicMock() + + exception = module.OpenAEVError(b"test", response_code, response_body) + + self.assertEqual(exception.error_message, "test") + + exception = module.OpenAEVError("test", response_code, response_body) + + self.assertEqual(exception.error_message, "test") + + def test_openaeverror_strcast_no_message(self): + error_message = None + response_code = 418 + response_body = MagicMock() + + exception = module.OpenAEVError(error_message, response_code, response_body) + + strdata = str(exception) + + self.assertEqual(strdata, "418: Unknown error") + + def test_openaeverror_strcast_no_message_no_code(self): + error_message = None + response_code = None + response_body = MagicMock() + + exception = module.OpenAEVError(error_message, response_code, response_body) + + strdata = str(exception) + + self.assertEqual(strdata, "Unknown error") + + def test_openaeverror_strcast_with_body(self): + error_message = None + response_code = 418 + response_body = b'{"message": "I am a teapot"}' + + exception = module.OpenAEVError(error_message, response_code, response_body) + + strdata = str(exception) + + self.assertEqual(strdata, "418: I am a teapot") diff --git a/libs/xtm-oaev-sdk/test/common/test_ssl_utils.py b/libs/xtm-oaev-sdk/test/common/test_ssl_utils.py new file mode 100644 index 00000000..7df6755d --- /dev/null +++ b/libs/xtm-oaev-sdk/test/common/test_ssl_utils.py @@ -0,0 +1,92 @@ +"""Tests for the SSL helpers.""" + +import os +from typing import Any +from unittest.mock import MagicMock + +import xtm_oaev_sdk as sdk + +PEM_CERT = "-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----\n" +PEM_KEY = "-----BEGIN PRIVATE KEY-----\nBBBB\n-----END PRIVATE KEY-----\n" + + +def _given_ssl_context() -> MagicMock: + return MagicMock() + + +def _given_recording_context(recorded: dict[str, Any]) -> MagicMock: + def _record(cert: str, key: str | None, passphrase: str | None) -> None: + recorded["cert"] = cert + recorded["key"] = key + recorded["passphrase"] = passphrase + with open(cert) as handle: + recorded["cert_content"] = handle.read() + + context = MagicMock() + context.load_cert_chain.side_effect = _record + return context + + +def test_data_to_temp_file_writes_content_owner_only() -> None: + """It writes the data to a new file readable only by its owner.""" + path = sdk.data_to_temp_file("secret") + try: + with open(path) as handle: + assert handle.read() == "secret" + assert os.stat(path).st_mode & 0o077 == 0 + finally: + os.unlink(path) + + +def test_is_memory_certificate_detects_pem_data() -> None: + """It distinguishes in-memory PEM data from a file path.""" + assert sdk.is_memory_certificate(PEM_CERT) is True + assert sdk.is_memory_certificate("/etc/ssl/cert.pem") is False + + +def test_ssl_verify_locations_ignores_none() -> None: + """It leaves the context untouched without CA data.""" + context = _given_ssl_context() + sdk.ssl_verify_locations(context, None) + context.load_verify_locations.assert_not_called() + + +def test_ssl_verify_locations_loads_memory_ca_as_cadata() -> None: + """It loads in-memory CA data through cadata.""" + context = _given_ssl_context() + sdk.ssl_verify_locations(context, PEM_CERT) + context.load_verify_locations.assert_called_once_with(cadata=PEM_CERT) + + +def test_ssl_verify_locations_loads_ca_path_as_cafile() -> None: + """It loads a CA path through cafile.""" + context = _given_ssl_context() + sdk.ssl_verify_locations(context, "/etc/ssl/ca.pem") + context.load_verify_locations.assert_called_once_with(cafile="/etc/ssl/ca.pem") + + +def test_ssl_cert_chain_ignores_missing_certificate() -> None: + """It does nothing without certificate data.""" + context = _given_ssl_context() + sdk.ssl_cert_chain(context, None, PEM_KEY, None) + context.load_cert_chain.assert_not_called() + + +def test_ssl_cert_chain_passes_paths_through() -> None: + """It loads certificate and key paths as given.""" + context = _given_ssl_context() + sdk.ssl_cert_chain(context, "/tmp/cert.pem", "/tmp/key.pem", "pass") + context.load_cert_chain.assert_called_once_with( + "/tmp/cert.pem", "/tmp/key.pem", "pass" + ) + + +def test_ssl_cert_chain_bridges_memory_pem_through_removed_temp_files() -> None: + """It writes in-memory PEM data to temp files, loads them, then removes them.""" + recorded: dict[str, Any] = {} + context = _given_recording_context(recorded) + sdk.ssl_cert_chain(context, PEM_CERT, PEM_KEY, "pass") + assert recorded["cert_content"] == PEM_CERT + assert recorded["passphrase"] == "pass" + assert not os.path.exists(recorded["cert"]) + assert not os.path.exists(recorded["key"]) diff --git a/libs/xtm-oaev-sdk/test/common/test_utils.py b/libs/xtm-oaev-sdk/test/common/test_utils.py new file mode 100644 index 00000000..df2c7909 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/common/test_utils.py @@ -0,0 +1,237 @@ +"""Tests for xtm_oaev_sdk utils public surface.""" + +import dataclasses +import logging +import subprocess +import sys +import warnings +from typing import Any + +import pytest +import xtm_oaev_sdk as sdk + + +@dataclasses.dataclass +class _SampleDataClass: + name: str + value: int + + +class _UnsupportedType: + pass + + +class _RecordingLogger: + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + def debug(self, message: str, **kwargs: Any) -> None: + self.calls.append({"level": "debug", "message": message, "kwargs": kwargs}) + + def info(self, message: str, **kwargs: Any) -> None: + self.calls.append({"level": "info", "message": message, "kwargs": kwargs}) + + def warning(self, message: str, **kwargs: Any) -> None: + self.calls.append({"level": "warning", "message": message, "kwargs": kwargs}) + + def error(self, message: str, **kwargs: Any) -> None: + self.calls.append({"level": "error", "message": message, "kwargs": kwargs}) + + +def _given_public_symbol(name: str) -> Any: + return getattr(sdk, name) + + +def _given_app_logger() -> Any: + app_logger_cls = _given_public_symbol("AppLogger") + return app_logger_cls(logging.INFO, json_logging=False, name="initial") + + +def _given_app_logger_with_recording_logger() -> tuple[Any, _RecordingLogger]: + app_logger = _given_app_logger() + recorder = _RecordingLogger() + app_logger.local_logger = recorder + return app_logger, recorder + + +def _given_json_encoder() -> Any: + encoder_cls = _given_public_symbol("EnhancedJSONEncoder") + return encoder_cls() + + +def _given_clean_root_logging_state() -> logging.Logger: + root_logger = logging.getLogger() + root_logger.handlers.clear() + root_logger.setLevel(logging.NOTSET) + return root_logger + + +def _when_call_logger_with_name(app_logger: Any, name: str) -> Any: + return app_logger(name) + + +def _when_call_app_logger_method( + app_logger: Any, method_name: str, message: str, meta: dict[str, Any] +) -> None: + getattr(app_logger, method_name)(message, meta) + + +def _when_set_third_party_logger_level(lib: str, level: int) -> None: + app_logger_cls = _given_public_symbol("AppLogger") + app_logger_cls.setup_logger_level(lib, level) + + +def _when_encode_with_default(encoder: Any, value: Any) -> Any: + return encoder.default(value) + + +def _when_encode_unsupported_type_raises(encoder: Any, value: Any) -> Exception: + with pytest.raises(TypeError) as exc_info: + encoder.default(value) + return exc_info.value + + +def _when_call_logger(level: int, json_logging: bool) -> Any: + logger_fn = _given_public_symbol("logger") + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + return logger_fn(level, json_logging) + + +def _when_setup_logging(level: int, json_logging: bool) -> None: + setup_fn = _given_public_symbol("setup_logging_config") + setup_fn(level, json_logging=json_logging) + + +def _then_logger_name_switched_and_self_returned( + original: Any, switched: Any, expected_name: str +) -> None: + assert switched is original + assert original.local_logger.name == expected_name + + +def _then_recording_logger_has_call( + recorder: _RecordingLogger, + expected_level: str, + expected_message: str, + expected_meta: dict[str, Any], +) -> None: + assert len(recorder.calls) == 1 + call = recorder.calls[0] + assert call["level"] == expected_level + assert call["message"] == expected_message + assert call["kwargs"]["extra"] == {"attributes": expected_meta} + if expected_level == "error": + assert call["kwargs"]["exc_info"] == 1 + + +def _then_third_party_logger_level_is(lib: str, expected_level: int) -> None: + assert logging.getLogger(lib).level == expected_level + + +def _then_runtime_checkable_protocol_satisfied(instance: Any) -> None: + protocol_cls = _given_public_symbol("AppLoggerProtocol") + assert isinstance(instance, protocol_cls) + + +def _then_encoded_object_equals(encoded: Any, expected: Any) -> None: + assert encoded == expected + + +def _then_type_error_for_unsupported_type(exc: Exception) -> None: + assert "not json serializable" in str(exc).lower() + + +def _then_is_app_logger_instance(instance: Any) -> None: + app_logger_cls = _given_public_symbol("AppLogger") + assert isinstance(instance, app_logger_cls) + + +def _then_root_logger_has_json_formatter(expected_level: int) -> None: + custom_formatter_cls = _given_public_symbol("CustomJsonFormatter") + root_logger = logging.getLogger() + assert root_logger.handlers + assert isinstance(root_logger.handlers[0].formatter, custom_formatter_cls) + assert root_logger.level == expected_level + + +def _then_root_logger_uses_basic_config_level(expected_level: int) -> None: + root_logger = logging.getLogger() + assert root_logger.level == expected_level + + +def test_app_logger_call_switches_name_and_returns_self() -> None: + """Verify AppLogger.__call__ switches logger name and returns self.""" + app_logger = _given_app_logger() + switched = _when_call_logger_with_name(app_logger, "switched") + _then_logger_name_switched_and_self_returned(app_logger, switched, "switched") + + +@pytest.mark.parametrize("level_name", ["debug", "info", "warning", "error"]) +def test_app_logger_logs_with_meta_dict(level_name: str) -> None: + """Verify AppLogger log methods accept metadata and call the right level.""" + app_logger, recorder = _given_app_logger_with_recording_logger() + _when_call_app_logger_method(app_logger, level_name, "message", {"k": "v"}) + _then_recording_logger_has_call(recorder, level_name, "message", {"k": "v"}) + + +def test_app_logger_setup_logger_level_adjusts_third_party_logger() -> None: + """Verify AppLogger.setup_logger_level updates external logger level.""" + _when_set_third_party_logger_level("third.party.lib", logging.ERROR) + _then_third_party_logger_level_is("third.party.lib", logging.ERROR) + + +def test_app_logger_protocol_runtime_checkable() -> None: + """Verify AppLogger satisfies AppLoggerProtocol runtime isinstance check.""" + app_logger = _given_app_logger() + _then_runtime_checkable_protocol_satisfied(app_logger) + + +def test_enhanced_json_encoder_serializes_dataclass() -> None: + """Verify EnhancedJSONEncoder serializes dataclass instances via asdict.""" + encoder = _given_json_encoder() + encoded = _when_encode_with_default(encoder, _SampleDataClass(name="n", value=1)) + _then_encoded_object_equals(encoded, {"name": "n", "value": 1}) + + +def test_enhanced_json_encoder_raises_type_error_for_unsupported_types() -> None: + """Verify EnhancedJSONEncoder raises TypeError on unsupported objects.""" + encoder = _given_json_encoder() + exc = _when_encode_unsupported_type_raises(encoder, _UnsupportedType()) + _then_type_error_for_unsupported_type(exc) + + +def test_logger_function_returns_app_logger() -> None: + """Verify logger() returns an AppLogger without warning.""" + logger_instance = _when_call_logger(logging.INFO, json_logging=False) + _then_is_app_logger_instance(logger_instance) + + +def test_setup_logging_config_json_logging_true_sets_json_handler() -> None: + """Verify setup_logging_config configures JSON formatter when enabled.""" + _given_clean_root_logging_state() + _when_setup_logging(logging.INFO, json_logging=True) + _then_root_logger_has_json_formatter(logging.INFO) + + +def test_setup_logging_config_json_logging_false_uses_basic_config() -> None: + """Verify setup_logging_config uses basicConfig when JSON logging is disabled.""" + _given_clean_root_logging_state() + _when_setup_logging(logging.DEBUG, json_logging=False) + _then_root_logger_uses_basic_config_level(logging.DEBUG) + + +def test_importing_utils_does_not_raise_jsonlogger_deprecation_warning() -> None: + """Verify the JSON formatter uses the non-deprecated pythonjsonlogger path.""" + result = subprocess.run( + [ + sys.executable, + "-W", + "error::DeprecationWarning", + "-c", + "import xtm_oaev_sdk", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/test/credential/__init__.py b/libs/xtm-oaev-sdk/test/configuration/__init__.py similarity index 100% rename from test/credential/__init__.py rename to libs/xtm-oaev-sdk/test/configuration/__init__.py diff --git a/libs/xtm-oaev-sdk/test/configuration/shim/__init__.py b/libs/xtm-oaev-sdk/test/configuration/shim/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/configuration/test_configuration.py b/libs/xtm-oaev-sdk/test/configuration/shim/test_configuration.py similarity index 100% rename from test/configuration/test_configuration.py rename to libs/xtm-oaev-sdk/test/configuration/shim/test_configuration.py diff --git a/test/configuration/test_connector_config_schema_generator.py b/libs/xtm-oaev-sdk/test/configuration/shim/test_connector_config_schema_generator.py similarity index 100% rename from test/configuration/test_connector_config_schema_generator.py rename to libs/xtm-oaev-sdk/test/configuration/shim/test_connector_config_schema_generator.py diff --git a/libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py new file mode 100644 index 00000000..7bd77c42 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py @@ -0,0 +1,113 @@ +"""Contract of the pyoaev configuration shims. + +Legacy imports must keep working, return the *identical* xtm_oaev_sdk object, +and warn; importing pyoaev itself and using the SDK directly must stay silent. +""" + +import subprocess +import sys +import warnings + +import pytest +import xtm_oaev_sdk as sdk + +LEGACY_NAMES = [ + ("pyoaev", "BaseConfigModel"), + ("pyoaev", "ConfigLoaderCollector"), + ("pyoaev", "ConfigLoaderOAEV"), + ("pyoaev", "Configuration"), + ("pyoaev", "SettingsLoader"), + ("pyoaev.configuration", "BaseConfigModel"), + ("pyoaev.configuration", "ConfigLoaderCollector"), + ("pyoaev.configuration", "ConfigLoaderOAEV"), + ("pyoaev.configuration", "Configuration"), + ("pyoaev.configuration", "SettingsLoader"), + ("pyoaev.configuration.configuration", "CONFIGURATION_TYPES"), + ("pyoaev.configuration.configuration", "Configuration"), + ("pyoaev.configuration.configuration", "ConfigurationHint"), + ("pyoaev.configuration.configuration", "ConnectorConfigSchemaGenerator"), + ("pyoaev.configuration.configuration", "DictionarySource"), + ("pyoaev.configuration.configuration", "EnvironmentSource"), + ("pyoaev.configuration.configuration", "is_falsy"), + ("pyoaev.configuration.configuration", "is_truthy"), + ("pyoaev.configuration.settings_loader", "BaseConfigModel"), + ("pyoaev.configuration.settings_loader", "ConfigLoaderCollector"), + ("pyoaev.configuration.settings_loader", "ConfigLoaderOAEV"), + ("pyoaev.configuration.settings_loader", "HttpUrlToString"), + ("pyoaev.configuration.settings_loader", "LogLevelToLower"), + ("pyoaev.configuration.settings_loader", "SettingsLoader"), + ("pyoaev.configuration.settings_loader", "TimedeltaInSeconds"), + ("pyoaev.configuration.sources", "DictionarySource"), + ("pyoaev.configuration.sources", "EnvironmentSource"), + ( + "pyoaev.configuration.connector_config_schema_generator", + "ConnectorConfigSchemaGenerator", + ), +] + + +_CONSUMER = "legacy_consumer.py" + + +def _given_fresh_interpreter_script(script: str) -> list[str]: + return [sys.executable, "-W", "error::DeprecationWarning", "-c", script] + + +def _when_run(command: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(command, capture_output=True, text=True) + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec( + compile(f"from {module} import {name} as _legacy", _CONSUMER, "exec"), + namespace, + ) + return namespace["_legacy"], caught + + +def _then_succeeds(result: subprocess.CompletedProcess) -> None: + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize(("module", "name"), LEGACY_NAMES) +def test_legacy_import_warns_and_returns_the_sdk_object(module, name) -> None: + """It warns towards xtm_oaev_sdk and returns the identical object.""" + legacy, caught = _when_import_legacy_name(module, name) + assert legacy is getattr(sdk, name) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert deprecations + assert f"from xtm_oaev_sdk import {name}" in str(deprecations[0].message) + # attributed to the consumer's line, so default warning filters show it + assert any(w.filename == _CONSUMER for w in deprecations) + + +def test_importing_pyoaev_is_silent() -> None: + """It imports pyoaev without any DeprecationWarning.""" + result = _when_run(_given_fresh_interpreter_script("import pyoaev")) + _then_succeeds(result) + + +def test_sdk_classes_stay_silent_once_pyoaev_is_loaded() -> None: + """It never alters SDK objects: native use stays silent next to pyoaev.""" + script = ( + "import pyoaev, pyoaev.configuration.configuration\n" + "import xtm_oaev_sdk as sdk\n" + "sdk.Configuration(config_hints={})\n" + "sdk.EnvironmentSource()\n" + "class _C(sdk.BaseConfigModel):\n" + " pass\n" + "_C()\n" + ) + result = _when_run(_given_fresh_interpreter_script(script)) + _then_succeeds(result) + + +def test_unknown_legacy_name_still_raises() -> None: + """It does not swallow genuine misses.""" + import pyoaev.configuration + + with pytest.raises(AttributeError): + pyoaev.configuration.DoesNotExist diff --git a/test/configuration/test_sources.py b/libs/xtm-oaev-sdk/test/configuration/shim/test_sources.py similarity index 100% rename from test/configuration/test_sources.py rename to libs/xtm-oaev-sdk/test/configuration/shim/test_sources.py diff --git a/libs/xtm-oaev-sdk/test/configuration/test_configuration.py b/libs/xtm-oaev-sdk/test/configuration/test_configuration.py new file mode 100644 index 00000000..34c96a00 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/configuration/test_configuration.py @@ -0,0 +1,487 @@ +"""Tests for configuration public surface.""" + +from pathlib import Path +from typing import Any + +import pytest +import xtm_oaev_sdk as sdk +from pydantic import ValidationError + + +def _given_public_module() -> Any: + return sdk + + +def _given_required_symbol(module: Any, symbol_name: str) -> Any: + assert hasattr(module, symbol_name), f"Missing public symbol: {symbol_name}" + return getattr(module, symbol_name) + + +def _given_missing_config_file_path(tmp_path: Path) -> str: + return str(tmp_path / "missing-config.yml") + + +def _given_configuration( + module: Any, + config_hints: dict[str, dict[str, Any] | str], + config_values: dict[str, Any] | None, + config_file_path: str, +) -> Any: + configuration = _given_required_symbol(module, "Configuration") + return configuration( + config_hints=config_hints, + config_values=config_values, + config_file_path=config_file_path, + ) + + +def _given_config_hints_for_priority() -> dict[str, dict[str, Any]]: + return { + "target": { + "env": "XTM_OAEV_TARGET", + "file_path": ["section", "target"], + "default": "default-value", + } + } + + +def _given_config_values_for_priority() -> dict[str, dict[str, str]]: + return {"section": {"target": "file-value"}} + + +def _given_set_env( + monkeypatch: pytest.MonkeyPatch, key: str, value: str | None +) -> None: + if value is None: + monkeypatch.delenv(key, raising=False) + else: + monkeypatch.setenv(key, value) + + +def _when_set_configuration_value(configuration: Any, key: str, value: Any) -> None: + configuration.set(key, value) + + +def _when_get_configuration_value(configuration: Any, key: str) -> Any: + return configuration.get(key) + + +def _when_get_environment_value(environment_source: Any, env_key: str | None) -> Any: + return environment_source.get(env_key) + + +def _when_get_dictionary_value( + dictionary_source: Any, path: list[str] | None, source_dict: dict[str, Any] +) -> Any: + return dictionary_source.get(path, source_dict) + + +def _when_assign_attribute(target: Any, attribute: str, value: Any) -> None: + setattr(target, attribute, value) + + +def _when_create_base_model_with_extra(base_config_model: Any) -> Any: + class _Model(base_config_model): + required: str + + return _Model(required="ok", extra_value="extra") + + +def _when_create_base_model_for_frozen_check(base_config_model: Any) -> Any: + class _Model(base_config_model): + value: int + + return _Model(value=1) + + +def _when_get_private_hint(configuration: Any, key: str) -> Any: + return configuration._Configuration__config_hints[key] # noqa: SLF001 + + +def _then_value_equals(actual: Any, expected: Any) -> None: + assert actual == expected + + +def _then_value_is_none(actual: Any) -> None: + assert actual is None + + +def _then_public_symbols_exist(module: Any, symbols: list[str]) -> None: + for symbol in symbols: + assert hasattr(module, symbol), f"Missing public symbol: {symbol}" + + +def _then_protocol_runtime_checkable(protocol: Any) -> None: + assert getattr(protocol, "_is_runtime_protocol", False) is True + + +def _then_instance_satisfies_protocol(instance: Any, protocol: Any) -> None: + assert isinstance(instance, protocol) + + +def _then_raises_validation_error(callable_obj: Any, *args: Any, **kwargs: Any) -> None: + with pytest.raises(ValidationError): + callable_obj(*args, **kwargs) + + +def _then_raises_assertion_error(callable_obj: Any, *args: Any, **kwargs: Any) -> None: + with pytest.raises(AssertionError): + callable_obj(*args, **kwargs) + + +def _then_hint_fields_match(hint: Any, expected: dict[str, Any]) -> None: + assert hint.data == expected["data"] + assert hint.env == expected["env"] + assert hint.file_path == expected["file_path"] + assert hint.is_number == expected["is_number"] + assert hint.default == expected["default"] + + +def _then_hint_is_instance_of(hint: Any, hint_class: Any) -> None: + assert isinstance(hint, hint_class) + + +def _then_model_allows_extra_fields( + model_instance: Any, key: str, expected: Any +) -> None: + assert model_instance.model_dump()[key] == expected + + +def _then_model_fields_include(model_class: Any, expected_fields: set[str]) -> None: + assert expected_fields.issubset(set(model_class.model_fields.keys())) + + +def test_public_surface_exports_configuration_symbols() -> None: + """It exposes all expected configuration-related public symbols.""" + module = _given_public_module() + _then_public_symbols_exist( + module, + [ + "ConfigurationProtocol", + "Configuration", + "ConfigurationHint", + "EnvironmentSource", + "DictionarySource", + "BaseConfigModel", + "ConfigLoaderOAEV", + "ConfigLoaderCollector", + "CONFIGURATION_TYPES", + ], + ) + + +def test_configuration_resolves_with_override_priority( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It prioritizes override over env, file, and default.""" + module = _given_public_module() + hints = _given_config_hints_for_priority() + config_values = _given_config_values_for_priority() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, hints, config_values, missing_path) + _given_set_env(monkeypatch, "XTM_OAEV_TARGET", "env-value") + _when_set_configuration_value(configuration, "target", "override-value") + result = _when_get_configuration_value(configuration, "target") + _then_value_equals(result, "override-value") + + +def test_configuration_get_prefers_env_when_no_override( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It reads environment value when no override is set.""" + module = _given_public_module() + hints = _given_config_hints_for_priority() + config_values = _given_config_values_for_priority() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, hints, config_values, missing_path) + _given_set_env(monkeypatch, "XTM_OAEV_TARGET", "env-value") + result = _when_get_configuration_value(configuration, "target") + _then_value_equals(result, "env-value") + + +def test_configuration_get_prefers_file_when_env_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It reads file-backed value when env is absent.""" + module = _given_public_module() + hints = _given_config_hints_for_priority() + config_values = _given_config_values_for_priority() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, hints, config_values, missing_path) + _given_set_env(monkeypatch, "XTM_OAEV_TARGET", None) + result = _when_get_configuration_value(configuration, "target") + _then_value_equals(result, "file-value") + + +def test_configuration_get_falls_back_to_default( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It uses default when override, env, and file are unavailable.""" + module = _given_public_module() + hints = _given_config_hints_for_priority() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, hints, {}, missing_path) + _given_set_env(monkeypatch, "XTM_OAEV_TARGET", None) + result = _when_get_configuration_value(configuration, "target") + _then_value_equals(result, "default-value") + + +def test_configuration_get_returns_none_for_unknown_key(tmp_path: Path) -> None: + """It returns None for an unknown configuration key.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, {"known": {"default": "x"}}, {}, missing_path + ) + result = _when_get_configuration_value(configuration, "unknown") + _then_value_is_none(result) + + +def test_configuration_set_creates_new_hint_for_new_key(tmp_path: Path) -> None: + """It creates a new ConfigurationHint when setting a new key.""" + module = _given_public_module() + hint_class = _given_required_symbol(module, "ConfigurationHint") + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, {}, {}, missing_path) + _when_set_configuration_value(configuration, "new_key", "new-value") + hint = _when_get_private_hint(configuration, "new_key") + _then_hint_is_instance_of(hint, hint_class) + _then_value_equals(hint.data, "new-value") + + +def test_configuration_set_overrides_existing_hint_and_keeps_its_sources( + tmp_path: Path, +) -> None: + """It overrides an existing key while keeping the hint's other fields.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, + {"target": {"env": "XTM_OAEV_TARGET", "default": "old"}}, + {}, + missing_path, + ) + _when_set_configuration_value(configuration, "target", "new") + result = _when_get_configuration_value(configuration, "target") + hint = _when_get_private_hint(configuration, "target") + _then_value_equals(result, "new") + _then_value_equals(hint.env, "XTM_OAEV_TARGET") + _then_value_equals(hint.default, "old") + + +def test_configuration_coerces_number_when_is_number_true(tmp_path: Path) -> None: + """It coerces values to int when is_number hint is true.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, + {"count": {"default": "7", "is_number": True}}, + {}, + missing_path, + ) + result = _when_get_configuration_value(configuration, "count") + _then_value_equals(result, 7) + + +def test_configuration_coerces_truthy_strings_to_true(tmp_path: Path) -> None: + """It coerces yes/true strings to True case-insensitively.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, + {"enabled": {"default": "YeS"}}, + {}, + missing_path, + ) + result = _when_get_configuration_value(configuration, "enabled") + _then_value_equals(result, True) + + +def test_configuration_coerces_falsy_strings_to_false(tmp_path: Path) -> None: + """It coerces no/false strings to False case-insensitively.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, + {"enabled": {"default": "FaLsE"}}, + {}, + missing_path, + ) + result = _when_get_configuration_value(configuration, "enabled") + _then_value_equals(result, False) + + +def test_configuration_coerces_empty_string_to_none(tmp_path: Path) -> None: + """It coerces empty strings to None.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, + {"value": {"default": ""}}, + {}, + missing_path, + ) + result = _when_get_configuration_value(configuration, "value") + _then_value_is_none(result) + + +def test_configuration_hint_is_mutable() -> None: + """It accepts direct field assignment: ``Configuration.set()`` mutates + the hint in place, so the hint is a plain mutable model.""" + module = _given_public_module() + configuration_hint = _given_required_symbol(module, "ConfigurationHint") + hint = configuration_hint(data="a") + _when_assign_attribute(hint, "data", "b") + assert hint.data == "b" + + +def test_configuration_hint_stores_all_fields() -> None: + """It stores data, env, file_path, is_number, and default fields.""" + module = _given_public_module() + configuration_hint = _given_required_symbol(module, "ConfigurationHint") + hint = configuration_hint( + data="x", + env="XTM_OAEV_ENV", + file_path=["section", "key"], + is_number=True, + default="fallback", + ) + _then_hint_fields_match( + hint, + { + "data": "x", + "env": "XTM_OAEV_ENV", + "file_path": ["section", "key"], + "is_number": True, + "default": "fallback", + }, + ) + + +def test_configuration_protocol_is_runtime_checkable_and_implemented( + tmp_path: Path, +) -> None: + """It is runtime-checkable and Configuration satisfies it.""" + module = _given_public_module() + protocol = _given_required_symbol(module, "ConfigurationProtocol") + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, {}, {}, missing_path) + _then_protocol_runtime_checkable(protocol) + _then_instance_satisfies_protocol(configuration, protocol) + + +def test_environment_source_get_returns_none_for_none_key() -> None: + """It returns None when env var key is None.""" + module = _given_public_module() + environment_source = _given_required_symbol(module, "EnvironmentSource") + result = _when_get_environment_value(environment_source, None) + _then_value_is_none(result) + + +def test_environment_source_get_returns_none_for_empty_key() -> None: + """It returns None when env var key is empty.""" + module = _given_public_module() + environment_source = _given_required_symbol(module, "EnvironmentSource") + result = _when_get_environment_value(environment_source, "") + _then_value_is_none(result) + + +def test_environment_source_get_returns_existing_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """It returns environment value when variable exists.""" + module = _given_public_module() + environment_source = _given_required_symbol(module, "EnvironmentSource") + _given_set_env(monkeypatch, "EXISTING_VAR", "value-from-env") + result = _when_get_environment_value(environment_source, "EXISTING_VAR") + _then_value_equals(result, "value-from-env") + + +def test_environment_source_get_returns_none_for_missing_var( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """It returns None when environment variable does not exist.""" + module = _given_public_module() + environment_source = _given_required_symbol(module, "EnvironmentSource") + _given_set_env(monkeypatch, "NONEXISTENT_VAR", None) + result = _when_get_environment_value(environment_source, "NONEXISTENT_VAR") + _then_value_is_none(result) + + +def test_dictionary_source_get_returns_none_for_none_path() -> None: + """It returns None when dictionary path is None.""" + module = _given_public_module() + dictionary_source = _given_required_symbol(module, "DictionarySource") + result = _when_get_dictionary_value( + dictionary_source, None, {"section": {"key": "value"}} + ) + _then_value_is_none(result) + + +def test_dictionary_source_get_returns_nested_value() -> None: + """It returns value for a valid two-level path.""" + module = _given_public_module() + dictionary_source = _given_required_symbol(module, "DictionarySource") + result = _when_get_dictionary_value( + dictionary_source, + ["section", "key"], + {"section": {"key": "val"}}, + ) + _then_value_equals(result, "val") + + +def test_dictionary_source_get_returns_none_for_missing_nested_key() -> None: + """It returns None when nested key is missing.""" + module = _given_public_module() + dictionary_source = _given_required_symbol(module, "DictionarySource") + result = _when_get_dictionary_value( + dictionary_source, + ["section", "missing"], + {"section": {"key": "val"}}, + ) + _then_value_is_none(result) + + +def test_dictionary_source_get_raises_for_invalid_path_length() -> None: + """It raises AssertionError when path is not exactly two elements.""" + module = _given_public_module() + dictionary_source = _given_required_symbol(module, "DictionarySource") + _then_raises_assertion_error( + _when_get_dictionary_value, + dictionary_source, + ["section"], + {"section": {"key": "val"}}, + ) + + +def test_base_config_model_is_frozen() -> None: + """It prevents assignment on BaseConfigModel subclasses.""" + module = _given_public_module() + base_config_model = _given_required_symbol(module, "BaseConfigModel") + instance = _when_create_base_model_for_frozen_check(base_config_model) + _then_raises_validation_error(_when_assign_attribute, instance, "value", 2) + + +def test_base_config_model_allows_extra_fields() -> None: + """It accepts extra fields on BaseConfigModel subclasses.""" + module = _given_public_module() + base_config_model = _given_required_symbol(module, "BaseConfigModel") + instance = _when_create_base_model_with_extra(base_config_model) + _then_model_allows_extra_fields(instance, "extra_value", "extra") + + +def test_config_loader_oaev_has_expected_fields() -> None: + """It defines url, token, and tenant_id fields.""" + module = _given_public_module() + config_loader_oaev = _given_required_symbol(module, "ConfigLoaderOAEV") + _then_model_fields_include(config_loader_oaev, {"url", "token", "tenant_id"}) + + +def test_config_loader_collector_has_expected_fields() -> None: + """It defines id, name, log_level, period, and icon_filepath fields.""" + module = _given_public_module() + config_loader_collector = _given_required_symbol(module, "ConfigLoaderCollector") + _then_model_fields_include( + config_loader_collector, + {"id", "name", "log_level", "period", "icon_filepath"}, + ) diff --git a/libs/xtm-oaev-sdk/test/contracts/__init__.py b/libs/xtm-oaev-sdk/test/contracts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/test/contracts/shim/__init__.py b/libs/xtm-oaev-sdk/test/contracts/shim/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/contracts/test_contract_element_argument_type.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_contract_element_argument_type.py similarity index 100% rename from test/contracts/test_contract_element_argument_type.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_contract_element_argument_type.py diff --git a/test/contracts/test_contract_expectations.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_contract_expectations.py similarity index 100% rename from test/contracts/test_contract_expectations.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_contract_expectations.py diff --git a/test/contracts/test_contract_output_types.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_contract_output_types.py similarity index 100% rename from test/contracts/test_contract_output_types.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_contract_output_types.py diff --git a/test/contracts/test_contract_referenced_credential.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_contract_referenced_credential.py similarity index 100% rename from test/contracts/test_contract_referenced_credential.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_contract_referenced_credential.py diff --git a/test/credential/test_credential_types.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_credential_types.py similarity index 100% rename from test/credential/test_credential_types.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_credential_types.py diff --git a/test/credential/test_credential_utils.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_credential_utils.py similarity index 100% rename from test/credential/test_credential_utils.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_credential_utils.py diff --git a/test/contracts/test_primitive_type.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_primitive_type.py similarity index 100% rename from test/contracts/test_primitive_type.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_primitive_type.py diff --git a/libs/xtm-oaev-sdk/test/contracts/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_shim_contract.py new file mode 100644 index 00000000..ca56f1b2 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/shim/test_shim_contract.py @@ -0,0 +1,153 @@ +"""Contract of the pyoaev contracts shims. + +Legacy imports must keep working, return the *identical* xtm_oaev_sdk object +(``ExpectationType`` was renamed ``ContractExpectationType``), warn, and be +attributed to the consumer's line. +""" + +import subprocess +import sys +import warnings + +import pytest +import xtm_oaev_sdk as sdk + +_CONSUMER = "legacy_consumer.py" +_RENAMED = {"ExpectationType": "ContractExpectationType"} + +_CONTRACT_CONFIG = [ + "SupportedLanguage", + "ContractFieldType", + "ContractFieldKey", + "ContractOutputType", + "PrimitiveType", + "ExpectationType", + "SecurityPlatformType", + "Expectation", + "LinkedFieldModel", + "ContractElement", + "ContractCardinalityElement", + "ContractOutputElement", + "ContractConfig", + "Domain", + "Contract", + "ContractTeam", + "ContractText", + "prepare_contracts", + "ContractTuple", + "ContractTextArea", + "ContractCheckbox", + "ContractAttachment", + "ContractExpectations", + "ContractSelect", + "ContractAsset", + "ContractAssetGroup", + "ContractAiTarget", + "ContractPayload", + "ContractReferencedCredential", + "CredentialType", + "EnhancedJSONEncoder", + "ContractCardinality", + "ContractVariable", + "VariableHelper", +] + +LEGACY_NAMES = ( + [("pyoaev", "ContractBuilder"), ("pyoaev.contracts", "ContractBuilder")] + + [ + ("pyoaev.contracts.contract_builder", name) + for name in ("ContractBuilder", "ContractElement", "ContractOutputElement") + ] + + [("pyoaev.contracts.contract_config", name) for name in _CONTRACT_CONFIG] + + [ + ("pyoaev.contracts.contract_utils", name) + for name in ("ContractCardinality", "VariableType", "ContractVariable") + ] + + [ + ("pyoaev.contracts.variable_helper", name) + for name in ( + "VariableHelper", + "ContractCardinality", + "ContractVariable", + "VariableType", + ) + ] + + [ + ("pyoaev.credential", name) + for name in ("CredentialType", "build_single_referenced_credential_element") + ] + + [("pyoaev.credential.types", "CredentialType")] + + [ + ("pyoaev.credential.utils", name) + for name in ( + "ContractReferencedCredential", + "CredentialType", + "build_single_referenced_credential_element", + ) + ] +) + +LEGACY_CONSTANTS = { + "USER": "user", + "EXERCISE": "exercise", + "TEAMS": "teams", + "COMCHECK": "comcheck", + "PLAYER_URI": "player_uri", + "CHALLENGES_URI": "challenges_uri", + "SCOREBOARD_URI": "scoreboard_uri", + "LESSONS_URI": "lessons_uri", +} + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + source = f"from {module} import {name} as _legacy" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(compile(source, _CONSUMER, "exec"), namespace) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + return namespace["_legacy"], deprecations + + +@pytest.mark.parametrize(("module", "name"), LEGACY_NAMES) +def test_legacy_import_warns_and_returns_the_sdk_object(module, name) -> None: + """It warns towards xtm_oaev_sdk and returns the identical object.""" + sdk_name = _RENAMED.get(name, name) + legacy, deprecations = _when_import_legacy_name(module, name) + assert legacy is getattr(sdk, sdk_name) + assert deprecations + assert f"from xtm_oaev_sdk import {sdk_name}" in str(deprecations[0].message) + # attributed to the consumer's line, so default warning filters show it + assert any(w.filename == _CONSUMER for w in deprecations) + + +@pytest.mark.parametrize(("name", "value"), LEGACY_CONSTANTS.items()) +def test_legacy_variable_constants_keep_their_value_and_warn(name, value) -> None: + """It keeps the unexported module constants working, with a warning.""" + legacy, deprecations = _when_import_legacy_name( + "pyoaev.contracts.variable_helper", name + ) + assert legacy == value + assert deprecations + + +def test_importing_pyoaev_contracts_is_silent() -> None: + """It imports pyoaev and its contracts modules without any warning.""" + script = ( + "import pyoaev, pyoaev.contracts, pyoaev.contracts.contract_builder\n" + "import pyoaev.contracts.contract_config, pyoaev.contracts.contract_utils\n" + "import pyoaev.contracts.variable_helper\n" + "import pyoaev.credential, pyoaev.credential.types, pyoaev.credential.utils\n" + ) + result = subprocess.run( + [sys.executable, "-W", "error::DeprecationWarning", "-c", script], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + +def test_unknown_legacy_name_still_raises() -> None: + """It does not swallow genuine misses.""" + with pytest.raises(ImportError): + exec("from pyoaev.contracts.contract_config import DoesNotExist", {}) diff --git a/libs/xtm-oaev-sdk/test/contracts/test_contract_element_argument_type.py b/libs/xtm-oaev-sdk/test/contracts/test_contract_element_argument_type.py new file mode 100644 index 00000000..4cad01c4 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_contract_element_argument_type.py @@ -0,0 +1,41 @@ +import json +import unittest + +import xtm_oaev_sdk as utils +from xtm_oaev_sdk import ContractText, PrimitiveType + + +def _serialize(field): + return json.loads(json.dumps(field, cls=utils.EnhancedJSONEncoder)) + + +class ContractElementArgumentTypeTest(unittest.TestCase): + def test_defaults_to_none_when_not_declared(self): + """Existing contracts that never pass argumentType keep working + unchanged — the platform normalizes a null value to PrimitiveType.Text + on its own, so this is not a breaking change for anything already + deployed.""" + untyped = ContractText(key="uri", label="URL") + self.assertIsNone(untyped.argumentType) + self.assertIsNone(_serialize(untyped)["argumentType"]) + + def test_explicit_argument_type_round_trips(self): + typed = ContractText( + key="basicUser", label="Username", argumentType=PrimitiveType.Username + ) + self.assertEqual(typed.argumentType, PrimitiveType.Username) + self.assertEqual(_serialize(typed)["argumentType"], "username") + + def test_argument_type_is_independent_of_widget_type(self): + """A field's rendering (ContractFieldType, on `type`) and its chaining + semantics (PrimitiveType, on `argumentType`) are separate axes.""" + typed = ContractText( + key="basicUser", label="Username", argumentType=PrimitiveType.Username + ) + serialized = _serialize(typed) + self.assertEqual(serialized["type"], "text") + self.assertEqual(serialized["argumentType"], "username") + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_contract_expectations.py b/libs/xtm-oaev-sdk/test/contracts/test_contract_expectations.py new file mode 100644 index 00000000..90b9be8b --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_contract_expectations.py @@ -0,0 +1,165 @@ +import json +import unittest + +import xtm_oaev_sdk as utils +from xtm_oaev_sdk import ContractCardinality, ContractExpectations +from xtm_oaev_sdk import ContractExpectationType as ExpectationType +from xtm_oaev_sdk import Expectation, SecurityPlatformType + + +def _expectation(expectation_type, name, is_predefined, expected_platforms=None): + # Only pass the field when declared so the default (None) path exercises + # the dataclass default_factory instead of an explicit empty list. + kwargs = {} + if expected_platforms is not None: + kwargs["expectation_expected_security_platform_types"] = expected_platforms + return Expectation( + expectation_type=expectation_type, + expectation_name=name, + expectation_description="", + expectation_score=100, + expectation_expectation_group=False, + expectation_is_predefined=is_predefined, + **kwargs, + ) + + +def _serialize(field): + return json.loads(json.dumps(field, cls=utils.EnhancedJSONEncoder)) + + +class TestContractExpectations(unittest.TestCase): + def test_predefined_derived_from_flag(self): + """ + The platform pre-fills expectations from the field-level + ``predefinedExpectations`` array, so it must be populated from the + expectations flagged ``expectation_is_predefined=True``. + """ + field = ContractExpectations( + key="expectations", + label="Expectations", + cardinality=ContractCardinality.Multiple, + availableExpectations=[ + _expectation(ExpectationType.detection, "Detection", True), + _expectation(ExpectationType.prevention, "Prevention", True), + _expectation(ExpectationType.vulnerability, "Not vulnerable", False), + ], + ) + + data = _serialize(field) + + self.assertEqual("expectation", data["type"]) + self.assertEqual( + ["DETECTION", "PREVENTION", "VULNERABILITY"], + [e["expectation_type"] for e in data["availableExpectations"]], + ) + # Only the flagged ones are pre-filled. + self.assertEqual( + ["DETECTION", "PREVENTION"], + [e["expectation_type"] for e in data["predefinedExpectations"]], + ) + + def test_explicit_predefined_is_kept(self): + detection = _expectation(ExpectationType.detection, "Detection", False) + field = ContractExpectations( + key="expectations", + label="Expectations", + availableExpectations=[detection], + predefinedExpectations=[detection], + ) + + data = _serialize(field) + + self.assertEqual( + ["DETECTION"], + [e["expectation_type"] for e in data["predefinedExpectations"]], + ) + + def test_explicit_empty_list_overrides_flags(self): + """ + An explicit empty ``predefinedExpectations`` list must suppress the + derivation even when some available expectations carry the predefined + flag, while still serializing as an (empty) array. + """ + field = ContractExpectations( + key="expectations", + label="Expectations", + availableExpectations=[ + _expectation(ExpectationType.detection, "Detection", True), + ], + predefinedExpectations=[], + ) + + data = _serialize(field) + + self.assertEqual([], data["predefinedExpectations"]) + + def test_no_flag_yields_no_predefined(self): + field = ContractExpectations( + key="expectations", + label="Expectations", + availableExpectations=[ + _expectation(ExpectationType.vulnerability, "Not vulnerable", False), + ], + ) + + data = _serialize(field) + + self.assertEqual([], data["predefinedExpectations"]) + + def test_expected_security_platform_types_default_empty(self): + """An expectation without declared platforms serializes an empty list.""" + field = ContractExpectations( + key="expectations", + label="Expectations", + availableExpectations=[ + _expectation(ExpectationType.detection, "Detection", True), + ], + ) + + data = _serialize(field) + + self.assertEqual( + [], + data["availableExpectations"][0][ + "expectation_expected_security_platform_types" + ], + ) + + def test_expected_security_platform_types_serialize(self): + """Declared platform types serialize as their string values on both + the available and predefined arrays.""" + field = ContractExpectations( + key="expectations", + label="Expectations", + availableExpectations=[ + _expectation( + ExpectationType.detection, + "Detection", + True, + expected_platforms=[ + SecurityPlatformType.EDR, + SecurityPlatformType.XDR, + ], + ), + ], + ) + + data = _serialize(field) + + self.assertEqual( + ["EDR", "XDR"], + data["availableExpectations"][0][ + "expectation_expected_security_platform_types" + ], + ) + self.assertEqual( + ["EDR", "XDR"], + data["predefinedExpectations"][0][ + "expectation_expected_security_platform_types" + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_contract_output_types.py b/libs/xtm-oaev-sdk/test/contracts/test_contract_output_types.py new file mode 100644 index 00000000..5e49c7ce --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_contract_output_types.py @@ -0,0 +1,22 @@ +import unittest + +from xtm_oaev_sdk import ContractOutputType + + +class ContractOutputTypeTest(unittest.TestCase): + def test_file_wire_label(self): + # The wire label is a public contract shared with the platform enum and every + # injector that declares a `file` output; it must stay exactly "file". + self.assertEqual(ContractOutputType.File.value, "file") + self.assertEqual(ContractOutputType.File, "file") + + def test_action_output_wire_label(self): + # The wire label is a public contract shared with the platform enum + # (io.openaev.database.model.ContractOutputType.ActionOutput); it must stay + # exactly "action_output". + self.assertEqual(ContractOutputType.ActionOutput.value, "action_output") + self.assertEqual(ContractOutputType.ActionOutput, "action_output") + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_contract_referenced_credential.py b/libs/xtm-oaev-sdk/test/contracts/test_contract_referenced_credential.py new file mode 100644 index 00000000..302abb2d --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_contract_referenced_credential.py @@ -0,0 +1,53 @@ +import json +import unittest + +from xtm_oaev_sdk import ( + ContractFieldKey, + ContractFieldType, + ContractReferencedCredential, + CredentialType, + EnhancedJSONEncoder, +) + + +def _serialize(field): + return json.loads(json.dumps(field, cls=EnhancedJSONEncoder)) + + +class ContractReferencedCredentialTest(unittest.TestCase): + def test_defaults_match_platform_contract_expectations(self): + field = ContractReferencedCredential() + serialized = _serialize(field) + + self.assertEqual(field.key, ContractFieldKey.CredentialReference.value) + self.assertEqual(field.label, "Select a credential reference") + self.assertFalse(field.mandatory) + self.assertTrue(field.multiple) + self.assertIsNone(field.credential_reference_type) + self.assertEqual(field.type, ContractFieldType.CredentialReference.value) + self.assertEqual(serialized["key"], "credential_reference") + self.assertEqual(serialized["type"], "credential-reference") + self.assertEqual(serialized["label"], "Select a credential reference") + self.assertFalse(serialized["mandatory"]) + self.assertTrue(serialized["multiple"]) + self.assertIsNone(serialized["credential_reference_type"]) + + def test_explicit_credential_type_serializes_to_platform_label(self): + field = ContractReferencedCredential( + credential_reference_type=CredentialType.CLOUD_AZURE + ) + + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AZURE) + self.assertEqual(_serialize(field)["credential_reference_type"], "CLOUD_AZURE") + + def test_identity_credential_type_serializes_to_platform_label(self): + field = ContractReferencedCredential( + credential_reference_type=CredentialType.IDENTITY + ) + + self.assertEqual(field.credential_reference_type, CredentialType.IDENTITY) + self.assertEqual(_serialize(field)["credential_reference_type"], "IDENTITY") + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_contracts.py b/libs/xtm-oaev-sdk/test/contracts/test_contracts.py new file mode 100644 index 00000000..ada58710 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_contracts.py @@ -0,0 +1,538 @@ +"""Tests for the contracts public surface exposed by xtm_oaev_sdk.""" + +import json + +import pytest +import xtm_oaev_sdk as sdk + + +def _given_public_symbol(name: str): + return getattr(sdk, name, None) + + +def _then_public_symbol_exists(symbol, name: str) -> None: + assert ( + symbol is not None + ), f"Expected xtm_oaev_sdk.{name} to exist on public surface" + + +def _given_existing_public_symbol(name: str): + symbol = _given_public_symbol(name) + _then_public_symbol_exists(symbol, name) + return symbol + + +def _given_contract_text(key: str = "text_key", label: str = "Text label"): + contract_text = _given_existing_public_symbol("ContractText") + return contract_text(key=key, label=label) + + +def _given_contract_checkbox(key: str = "checkbox_key", label: str = "Checkbox label"): + contract_checkbox = _given_existing_public_symbol("ContractCheckbox") + return contract_checkbox(key=key, label=label) + + +def _given_contract_tuple(key: str = "tuple_key", label: str = "Tuple label"): + contract_tuple = _given_existing_public_symbol("ContractTuple") + return contract_tuple(key=key, label=label) + + +def _given_contract_select(key: str = "select_key", label: str = "Select label"): + contract_select = _given_existing_public_symbol("ContractSelect") + return contract_select(key=key, label=label) + + +def _given_contract_attachment( + key: str = "attachment_key", label: str = "Attachment label" +): + contract_attachment = _given_existing_public_symbol("ContractAttachment") + return contract_attachment(key=key, label=label) + + +def _given_contract_textarea(key: str = "textarea_key", label: str = "Textarea label"): + contract_textarea = _given_existing_public_symbol("ContractTextArea") + return contract_textarea(key=key, label=label) + + +def _given_contract_team(key: str = "team_key", label: str = "Team label"): + contract_team = _given_existing_public_symbol("ContractTeam") + return contract_team(key=key, label=label) + + +def _given_contract_expectations( + key: str = "expectations_key", label: str = "Expectations label" +): + contract_expectations = _given_existing_public_symbol("ContractExpectations") + return contract_expectations(key=key, label=label) + + +def _given_contract_payload(key: str = "payload_key", label: str = "Payload label"): + contract_payload = _given_existing_public_symbol("ContractPayload") + return contract_payload(key=key, label=label) + + +def _given_contract_asset(label: str = "Asset label"): + contract_asset = _given_existing_public_symbol("ContractAsset") + return contract_asset(label=label) + + +def _given_contract_asset_group(label: str = "Asset Group label"): + contract_asset_group = _given_existing_public_symbol("ContractAssetGroup") + return contract_asset_group(label=label) + + +def _given_contract_output_element(): + output_element = _given_existing_public_symbol("ContractOutputElement") + contract_output_type = _given_existing_public_symbol("ContractOutputType") + return output_element( + type=contract_output_type.Text.value, + field="output_field", + labels=["Output field"], + isFindingCompatible=True, + isMultiple=False, + ) + + +def _given_contract_config(): + contract_config = _given_existing_public_symbol("ContractConfig") + supported_language = _given_existing_public_symbol("SupportedLanguage") + return contract_config( + type="injector", + expose=True, + label={supported_language.en: "Injector", supported_language.fr: "Injecteur"}, + color_dark="#111111", + color_light="#eeeeee", + ) + + +def _given_contract_instance(): + contract = _given_existing_public_symbol("Contract") + supported_language = _given_existing_public_symbol("SupportedLanguage") + return contract( + contract_id="contract-1", + label={supported_language.en: "Contract", supported_language.fr: "Contrat"}, + fields=[_given_contract_text()], + outputs=[_given_contract_output_element()], + config=_given_contract_config(), + manual=False, + ) + + +def _given_contract_builder_instance(): + contract_builder = _given_existing_public_symbol("ContractBuilder") + return contract_builder() + + +def _when_building_fluent_contract_builder_sequence(builder): + first = builder.mandatory(_given_contract_text("f1", "F1")) + second = builder.optional(_given_contract_checkbox("f2", "F2")) + third = builder.mandatory_group( + [_given_contract_text("f3", "F3"), _given_contract_text("f4", "F4")] + ) + fourth = builder.add_fields([_given_contract_select("f5", "F5")]) + fifth = builder.add_outputs([_given_contract_output_element()]) + built_fields = builder.build_fields() + built_outputs = builder.build_outputs() + return first, second, third, fourth, fifth, built_fields, built_outputs + + +def _then_builder_fluent_sequence_is_chainable(builder, call_results) -> None: + for result in call_results[:5]: + assert result is builder + + +def _then_builder_accumulates_expected_fields_and_outputs( + built_fields, built_outputs +) -> None: + assert [field.key for field in built_fields] == ["f1", "f2", "f3", "f4", "f5"] + assert len(built_outputs) == 1 + + +def _when_checking_builder_protocol(builder): + contract_builder_protocol = _given_existing_public_symbol("ContractBuilderProtocol") + return isinstance(builder, contract_builder_protocol) + + +def _then_value_is_true(value: bool) -> None: + assert value is True + + +def _when_reading_element_type(element): + return element.type + + +def _then_element_type_equals(actual: str, expected: str) -> None: + assert actual == expected + + +def _when_reading_tuple_cardinality(contract_tuple): + return contract_tuple.cardinality + + +def _then_cardinality_is_multiple(cardinality) -> None: + contract_cardinality = _given_existing_public_symbol("ContractCardinality") + assert cardinality == contract_cardinality.Multiple + + +def _when_reading_key(element): + return element.key + + +def _then_key_equals(actual: str, expected: str) -> None: + assert actual == expected + + +def _when_serializing_contract_add_input(contract_instance): + return contract_instance.to_contract_add_input("injector-123") + + +def _when_serializing_contract_update_input(contract_instance): + return contract_instance.to_contract_update_input() + + +def _then_add_input_contains_injector_and_contract_content(serialized: dict) -> None: + assert serialized["injector_id"] == "injector-123" + assert "contract_content" in serialized + assert isinstance(serialized["contract_content"], str) + assert json.loads(serialized["contract_content"])["contract_id"] == "contract-1" + + +def _then_update_input_omits_injector_id(serialized: dict) -> None: + assert "injector_id" not in serialized + assert "contract_content" in serialized + + +def _when_adding_attack_pattern(contract_instance): + contract_instance.add_attack_pattern("T1234") + return contract_instance.contract_attack_patterns_external_ids + + +def _then_list_contains_single_value(values: list[str], expected: str) -> None: + assert values == [expected] + + +def _when_adding_vulnerability(contract_instance): + contract_instance.add_vulnerability("CVE-2024-0001") + return contract_instance.contract_vulnerability_external_ids + + +def _when_adding_variable(contract_instance): + contract_variable = _given_existing_public_symbol("ContractVariable") + variable_type = _given_existing_public_symbol("VariableType") + contract_cardinality = _given_existing_public_symbol("ContractCardinality") + new_variable = contract_variable( + key="custom", + label="Custom variable", + type=variable_type.String, + cardinality=contract_cardinality.One, + ) + contract_instance.add_variable(new_variable) + return contract_instance.variables + + +def _then_variables_include_custom_key(variables) -> None: + assert any(variable.key == "custom" for variable in variables) + + +def _when_reading_default_contract_variables(contract_instance): + return contract_instance.variables + + +def _then_default_variables_include_expected_keys(variables) -> None: + keys = [variable.key for variable in variables] + assert len(keys) == 7 + assert "user" in keys + assert "exercise" in keys + assert "teams" in keys + assert "player_uri" in keys + assert "challenges_uri" in keys + assert "scoreboard_uri" in keys + assert "lessons_uri" in keys + + +def _when_building_user_variable(): + variable_helper = _given_existing_public_symbol("VariableHelper") + return variable_helper.user_variable() + + +def _when_building_exercise_variable(): + variable_helper = _given_existing_public_symbol("VariableHelper") + return variable_helper.exercise_variable() + + +def _when_building_team_variable(): + variable_helper = _given_existing_public_symbol("VariableHelper") + return variable_helper.team_variable() + + +def _when_building_uri_variables(): + variable_helper = _given_existing_public_symbol("VariableHelper") + return variable_helper.uri_variables() + + +def _then_variable_has_children_count(variable, expected: int) -> None: + assert len(variable.children) == expected + + +def _then_variable_cardinality_is_multiple(variable) -> None: + contract_cardinality = _given_existing_public_symbol("ContractCardinality") + assert variable.cardinality == contract_cardinality.Multiple + + +def _then_uri_variables_have_expected_length(uri_variables) -> None: + assert len(uri_variables) == 4 + + +def _when_preparing_contracts(contracts): + prepare_contracts = _given_existing_public_symbol("prepare_contracts") + return prepare_contracts(contracts) + + +def _then_prepared_contracts_have_expected_shape(serialized_contracts) -> None: + assert len(serialized_contracts) == 1 + assert serialized_contracts[0]["contract_id"] == "contract-1" + assert "contract_content" in serialized_contracts[0] + assert isinstance(serialized_contracts[0]["contract_content"], str) + + +def _when_reading_enum_values(): + contract_cardinality = _given_existing_public_symbol("ContractCardinality") + variable_type = _given_existing_public_symbol("VariableType") + return contract_cardinality, variable_type + + +def _then_enum_values_match(contract_cardinality, variable_type) -> None: + assert contract_cardinality.One.value == "1" + assert contract_cardinality.Multiple.value == "n" + assert variable_type.String.value == "String" + assert variable_type.Object.value == "Object" + + +def _when_reading_contract_expectation_members(): + contract_expectation_type = _given_existing_public_symbol("ContractExpectationType") + return list(contract_expectation_type) + + +def _when_reading_supported_language_members(): + supported_language = _given_existing_public_symbol("SupportedLanguage") + return list(supported_language) + + +def _then_collection_has_length(collection, expected: int) -> None: + assert len(collection) == expected + + +def _when_building_mandatory_group(builder): + first = _given_contract_text("group_1", "Group 1") + second = _given_contract_text("group_2", "Group 2") + builder.mandatory_group([first, second]) + return first, second + + +def _then_elements_have_shared_mandatory_groups(elements) -> None: + keys = [element.key for element in elements] + for element in elements: + assert element.mandatory is True + assert element.mandatoryGroups == keys + + +@pytest.mark.parametrize( + ("factory_name", "expected_type"), + [ + ("_given_contract_text", "text"), + ("_given_contract_checkbox", "checkbox"), + ("_given_contract_tuple", "tuple"), + ("_given_contract_asset", "asset"), + ("_given_contract_asset_group", "asset-group"), + ("_given_contract_attachment", "attachment"), + ("_given_contract_select", "select"), + ("_given_contract_textarea", "textarea"), + ("_given_contract_team", "team"), + ("_given_contract_expectations", "expectation"), + ("_given_contract_payload", "payload"), + ], +) +def test_contract_element_subclasses_set_type_from_get_type( + factory_name: str, expected_type: str +) -> None: + """Verifies each ContractElement subclass sets its type via __post_init__.""" + element = globals()[factory_name]() + actual_type = _when_reading_element_type(element) + _then_element_type_equals(actual_type, expected_type) + + +@pytest.mark.parametrize( + "symbol_name", + [ + "ContractBuilder", + "ContractBuilderProtocol", + "ContractCardinality", + "ContractVariable", + "VariableType", + "VariableHelper", + "Contract", + "ContractConfig", + "ContractElement", + "ContractText", + "ContractCheckbox", + "ContractTuple", + "ContractAsset", + "ContractAssetGroup", + "ContractExpectationType", + "ContractExpectations", + "ContractPayload", + "ContractSelect", + "ContractAttachment", + "ContractTextArea", + "ContractTeam", + "Expectation", + "Domain", + "LinkedFieldModel", + "SupportedLanguage", + "prepare_contracts", + "ContractOutputElement", + "ContractOutputType", + "ContractFieldType", + "ContractFieldKey", + "ContractCardinalityElement", + ], +) +def test_contract_public_surface_exposes_expected_symbols(symbol_name: str) -> None: + """Verifies the xtm_oaev_sdk public surface exposes required contract symbols.""" + symbol = _given_public_symbol(symbol_name) + _then_public_symbol_exists(symbol, symbol_name) + + +def test_contract_builder_fluent_api_is_chainable_and_accumulates_fields() -> None: + """Verifies ContractBuilder fluent methods return self and collect fields/outputs.""" + builder = _given_contract_builder_instance() + call_results = _when_building_fluent_contract_builder_sequence(builder) + _then_builder_fluent_sequence_is_chainable(builder, call_results) + _then_builder_accumulates_expected_fields_and_outputs( + call_results[5], call_results[6] + ) + + +def test_contract_builder_satisfies_protocol() -> None: + """Verifies ContractBuilder satisfies ContractBuilderProtocol runtime checks.""" + builder = _given_contract_builder_instance() + result = _when_checking_builder_protocol(builder) + _then_value_is_true(result) + + +def test_contract_tuple_forces_multiple_cardinality() -> None: + """Verifies ContractTuple forces cardinality to ContractCardinality.Multiple.""" + contract_tuple = _given_contract_tuple() + cardinality = _when_reading_tuple_cardinality(contract_tuple) + _then_cardinality_is_multiple(cardinality) + + +def test_contract_asset_key_is_auto_set_to_assets() -> None: + """Verifies ContractAsset key is automatically set to assets.""" + contract_asset = _given_contract_asset() + key = _when_reading_key(contract_asset) + _then_key_equals(key, "assets") + + +def test_contract_asset_group_key_is_auto_set_to_asset_groups() -> None: + """Verifies ContractAssetGroup key is automatically set to asset_groups.""" + contract_asset_group = _given_contract_asset_group() + key = _when_reading_key(contract_asset_group) + _then_key_equals(key, "asset_groups") + + +def test_contract_to_contract_add_input_includes_injector_id_and_content() -> None: + """Verifies contract add payload includes injector_id and serialized content.""" + contract_instance = _given_contract_instance() + serialized = _when_serializing_contract_add_input(contract_instance) + _then_add_input_contains_injector_and_contract_content(serialized) + + +def test_contract_to_contract_update_input_omits_injector_id() -> None: + """Verifies contract update payload omits injector_id.""" + contract_instance = _given_contract_instance() + serialized = _when_serializing_contract_update_input(contract_instance) + _then_update_input_omits_injector_id(serialized) + + +def test_contract_add_attack_pattern_appends_value() -> None: + """Verifies add_attack_pattern appends values to the attack pattern list.""" + contract_instance = _given_contract_instance() + values = _when_adding_attack_pattern(contract_instance) + _then_list_contains_single_value(values, "T1234") + + +def test_contract_add_vulnerability_appends_value() -> None: + """Verifies add_vulnerability appends values to the vulnerability list.""" + contract_instance = _given_contract_instance() + values = _when_adding_vulnerability(contract_instance) + _then_list_contains_single_value(values, "CVE-2024-0001") + + +def test_contract_add_variable_appends_to_variables_list() -> None: + """Verifies add_variable appends a custom variable to contract variables.""" + contract_instance = _given_contract_instance() + variables = _when_adding_variable(contract_instance) + _then_variables_include_custom_key(variables) + + +def test_contract_default_variables_include_expected_core_and_uri_variables() -> None: + """Verifies default contract variables include user, exercise, teams, and 4 URIs.""" + contract_instance = _given_contract_instance() + variables = _when_reading_default_contract_variables(contract_instance) + _then_default_variables_include_expected_keys(variables) + + +def test_variable_helper_user_variable_contains_five_children() -> None: + """Verifies VariableHelper.user_variable returns a variable with five children.""" + user_variable = _when_building_user_variable() + _then_variable_has_children_count(user_variable, 5) + + +def test_variable_helper_exercise_variable_contains_three_children() -> None: + """Verifies VariableHelper.exercise_variable returns a variable with three children.""" + exercise_variable = _when_building_exercise_variable() + _then_variable_has_children_count(exercise_variable, 3) + + +def test_variable_helper_team_variable_has_multiple_cardinality() -> None: + """Verifies VariableHelper.team_variable returns a variable with multiple cardinality.""" + team_variable = _when_building_team_variable() + _then_variable_cardinality_is_multiple(team_variable) + + +def test_variable_helper_uri_variables_returns_four_items() -> None: + """Verifies VariableHelper.uri_variables returns four URI contract variables.""" + uri_variables = _when_building_uri_variables() + _then_uri_variables_have_expected_length(uri_variables) + + +def test_prepare_contracts_serializes_contracts_to_dict_list() -> None: + """Verifies prepare_contracts serializes Contract instances to dict payloads.""" + serialized_contracts = _when_preparing_contracts([_given_contract_instance()]) + _then_prepared_contracts_have_expected_shape(serialized_contracts) + + +def test_contract_enums_expose_expected_values() -> None: + """Verifies ContractCardinality and VariableType expose expected values.""" + contract_cardinality, variable_type = _when_reading_enum_values() + _then_enum_values_match(contract_cardinality, variable_type) + + +def test_contract_expectation_type_exposes_eight_members() -> None: + """Verifies ContractExpectationType contains exactly eight members.""" + members = _when_reading_contract_expectation_members() + _then_collection_has_length(members, 8) + + +def test_supported_language_exposes_fr_and_en_members() -> None: + """Verifies SupportedLanguage contains exactly fr and en members.""" + members = _when_reading_supported_language_members() + _then_collection_has_length(members, 2) + + +def test_contract_builder_mandatory_group_sets_all_group_keys() -> None: + """Verifies mandatory_group sets shared mandatoryGroups across all grouped elements.""" + builder = _given_contract_builder_instance() + elements = _when_building_mandatory_group(builder) + _then_elements_have_shared_mandatory_groups(elements) diff --git a/libs/xtm-oaev-sdk/test/contracts/test_credential_types.py b/libs/xtm-oaev-sdk/test/contracts/test_credential_types.py new file mode 100644 index 00000000..76f02bc3 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_credential_types.py @@ -0,0 +1,27 @@ +import unittest + +from xtm_oaev_sdk import CredentialType + + +class CredentialTypeTest(unittest.TestCase): + def test_every_value_matches_the_supported_platform_labels(self): + expected_labels = { + "IDENTITY", + "CLOUD_AWS", + "CLOUD_AZURE", + "CLOUD_GCP", + } + actual_labels = {member.value for member in CredentialType} + self.assertEqual(actual_labels, expected_labels) + + def test_identity_wire_label(self): + self.assertEqual(CredentialType.IDENTITY.value, "IDENTITY") + self.assertEqual(CredentialType.IDENTITY, "IDENTITY") + + def test_aws_wire_label(self): + self.assertEqual(CredentialType.CLOUD_AWS.value, "CLOUD_AWS") + self.assertEqual(CredentialType.CLOUD_AWS, "CLOUD_AWS") + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_credential_utils.py b/libs/xtm-oaev-sdk/test/contracts/test_credential_utils.py new file mode 100644 index 00000000..295db924 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_credential_utils.py @@ -0,0 +1,59 @@ +import json +import unittest + +from xtm_oaev_sdk import ( + ContractReferencedCredential, + CredentialType, + EnhancedJSONEncoder, + build_single_referenced_credential_element, +) + + +def _serialize(field): + return json.loads(json.dumps(field, cls=EnhancedJSONEncoder)) + + +class CredentialUtilsTest(unittest.TestCase): + def test_aws_provider_maps_to_aws_credential_reference(self): + field = build_single_referenced_credential_element("aws") + + self.assertIsInstance(field, ContractReferencedCredential) + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AWS) + + def test_eks_provider_maps_to_aws_credential_reference(self): + field = build_single_referenced_credential_element("eks") + + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AWS) + + def test_azure_provider_maps_to_azure_credential_reference(self): + field = build_single_referenced_credential_element("azure") + + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AZURE) + + def test_gcp_provider_maps_to_gcp_credential_reference(self): + field = build_single_referenced_credential_element("gcp") + + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_GCP) + + def test_provider_mapping_is_case_insensitive(self): + field = build_single_referenced_credential_element("AWS") + + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AWS) + + def test_package_public_helper_builds_the_same_field(self): + field = build_single_referenced_credential_element("azure") + + self.assertIsInstance(field, ContractReferencedCredential) + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AZURE) + self.assertEqual(_serialize(field)["credential_reference_type"], "CLOUD_AZURE") + self.assertFalse(field.multiple) + + def test_unknown_provider_leaves_credential_type_empty(self): + field = build_single_referenced_credential_element("openstack") + + self.assertIsNone(field.credential_reference_type) + self.assertIsNone(_serialize(field)["credential_reference_type"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_primitive_type.py b/libs/xtm-oaev-sdk/test/contracts/test_primitive_type.py new file mode 100644 index 00000000..76863d7f --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_primitive_type.py @@ -0,0 +1,77 @@ +import unittest + +from xtm_oaev_sdk import PrimitiveType + + +class PrimitiveTypeTest(unittest.TestCase): + def test_username_wire_label(self): + # The wire label is a public contract shared with the platform enum + # (io.openaev.database.model.PrimitiveType.Username); it must stay + # exactly "username". + self.assertEqual(PrimitiveType.Username.value, "username") + self.assertEqual(PrimitiveType.Username, "username") + + def test_host_wire_label(self): + self.assertEqual(PrimitiveType.Host.value, "host") + self.assertEqual(PrimitiveType.Host, "host") + + def test_text_wire_label(self): + # The platform's own normalizer defaults a missing argumentType to + # exactly this value — it must stay "text". + self.assertEqual(PrimitiveType.Text.value, "text") + self.assertEqual(PrimitiveType.Text, "text") + + def test_action_output_wire_label(self): + # Shared with ContractOutputType.ActionOutput; both enums describe the + # same platform-side value from two different angles (an output + # producing it vs. an input typed to receive it) and must not drift + # apart. + self.assertEqual(PrimitiveType.ActionOutput.value, "action_output") + + def test_every_value_matches_the_openaev_platform_enum(self): + # io.openaev.database.model.PrimitiveType, transcribed label-for-label. + # Keep this set in sync with that file, not just the four spot-checked + # above. + expected_labels = { + "account_with_password_not_required", + "action_output", + "admin_username", + "asreproastable_account", + "asset_group_id", + "asset_id", + "computer_name", + "cve", + "delegation_account", + "document", + "domain", + "file_name", + "file_path", + "group_name", + "hash", + "host", + "ipv4", + "ipv6", + "ip_subnet", + "kerberoastable_account", + "key", + "number", + "password", + "permissions", + "port", + "service", + "severity", + "share_name", + "sid", + "targeted-asset", + "text", + "username", + "value", + "vulnerability_name", + "vulnerability_status", + } + actual_labels = {member.value for member in PrimitiveType} + self.assertEqual(actual_labels, expected_labels) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/daemons/__init__.py b/libs/xtm-oaev-sdk/test/daemons/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/test/daemons/shim/__init__.py b/libs/xtm-oaev-sdk/test/daemons/shim/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/daemons/test_base_daemon.py b/libs/xtm-oaev-sdk/test/daemons/shim/test_base_daemon.py similarity index 100% rename from test/daemons/test_base_daemon.py rename to libs/xtm-oaev-sdk/test/daemons/shim/test_base_daemon.py diff --git a/libs/xtm-oaev-sdk/test/daemons/shim/test_ping_alive.py b/libs/xtm-oaev-sdk/test/daemons/shim/test_ping_alive.py new file mode 100644 index 00000000..ecffa164 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/daemons/shim/test_ping_alive.py @@ -0,0 +1,57 @@ +import unittest +from unittest.mock import MagicMock + +import pyoaev.utils as module + + +class TestPingAlive(unittest.TestCase): + def test_pingalive_ping_uses_injector_branch(self): + api = MagicMock() + logger = MagicMock() + ping_alive = module.PingAlive( + api=api, config={"id": 1}, logger=logger, ping_type="injector" + ) + ping_alive.exit_event.is_set = MagicMock(side_effect=[False, True]) + ping_alive.exit_event.wait = MagicMock() + + ping_alive.ping() + + api.injector.create.assert_called_once_with({"id": 1}, False) + ping_alive.exit_event.wait.assert_called_once_with(40) + + def test_pingalive_ping_uses_collector_branch_and_logs_errors(self): + api = MagicMock() + api.collector.create.side_effect = Exception("boom") + logger = MagicMock() + ping_alive = module.PingAlive( + api=api, config={}, logger=logger, ping_type="collector" + ) + ping_alive.exit_event.is_set = MagicMock(side_effect=[False, True]) + ping_alive.exit_event.wait = MagicMock() + + ping_alive.ping() + + logger.error.assert_called_once() + ping_alive.exit_event.wait.assert_called_once_with(40) + + def test_pingalive_run_and_stop(self): + ping_alive = module.PingAlive( + api=MagicMock(), config={}, logger=MagicMock(), ping_type="collector" + ) + ping_alive.ping = MagicMock() + + ping_alive.run() + ping_alive.stop() + + ping_alive.logger.info.assert_any_call( + "Starting PingAlive thread", {"tenant_id": str(ping_alive.tenant_id)} + ) + ping_alive.logger.info.assert_any_call( + "Preparing PingAlive for clean shutdown", + {"tenant_id": str(ping_alive.tenant_id)}, + ) + self.assertTrue(ping_alive.exit_event.is_set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/daemons/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/daemons/shim/test_shim_contract.py new file mode 100644 index 00000000..b898a31a --- /dev/null +++ b/libs/xtm-oaev-sdk/test/daemons/shim/test_shim_contract.py @@ -0,0 +1,99 @@ +"""Contract of the pyoaev daemon runtime shims. + +``PingAlive`` resolves to the *identical* xtm_oaev_sdk object. ``BaseDaemon`` +resolves to a subclass that keeps building the OpenAEV client when none is +injected. Both warn, attributed to the consumer's line. +""" + +import subprocess +import sys +import warnings +from unittest.mock import MagicMock, patch + +import pytest +import xtm_oaev_sdk as sdk + +_CONSUMER = "legacy_consumer.py" + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + source = f"from {module} import {name} as _legacy" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(compile(source, _CONSUMER, "exec"), namespace) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + return namespace["_legacy"], deprecations + + +def _then_warns_at_consumer(deprecations, sdk_name: str) -> None: + assert deprecations + assert f"from xtm_oaev_sdk import {sdk_name}" in str(deprecations[0].message) + assert any(w.filename == _CONSUMER for w in deprecations) + + +@pytest.mark.parametrize("module", ["pyoaev.utils"]) +def test_legacy_ping_alive_is_the_sdk_object(module: str) -> None: + """It warns and returns the identical PingAlive.""" + legacy, deprecations = _when_import_legacy_name(module, "PingAlive") + assert legacy is sdk.PingAlive + _then_warns_at_consumer(deprecations, "PingAlive") + + +@pytest.mark.parametrize("module", ["pyoaev.daemons", "pyoaev.daemons.base_daemon"]) +def test_legacy_base_daemon_builds_the_openaev_client(module: str) -> None: + """It warns and still builds the OpenAEV client from configuration.""" + legacy, deprecations = _when_import_legacy_name(module, "BaseDaemon") + _then_warns_at_consumer(deprecations, "BaseDaemon") + assert issubclass(legacy, sdk.BaseDaemon) + + class _Daemon(legacy): + def _setup(self): + pass + + def _start_loop(self): + pass + + configuration = sdk.Configuration( + config_hints={ + "openaev_url": {"data": "http://example.com"}, + "openaev_token": {"data": "token"}, + "openaev_tenant_id": {"data": "tenant"}, + } + ) + with patch("pyoaev.daemons.base_daemon.OpenAEV") as client_class: + daemon = _Daemon(configuration=configuration, logger=MagicMock()) + + client_class.assert_called_once_with( + url="http://example.com", token="token", tenant_id="tenant" + ) + assert daemon.api is client_class.return_value + + +def test_legacy_collector_daemon_is_still_a_legacy_base_daemon() -> None: + """It keeps the legacy class hierarchy.""" + from pyoaev.daemons import CollectorDaemon + + legacy_base, _ = _when_import_legacy_name("pyoaev.daemons", "BaseDaemon") + assert issubclass(CollectorDaemon, legacy_base) + + +def test_openaev_client_satisfies_base_client() -> None: + """It accepts the real pyoaev client as an injected API client.""" + from pyoaev import OpenAEV + + assert isinstance(OpenAEV(url="http://example.com", token="t"), sdk.BaseClient) + + +def test_importing_pyoaev_daemon_modules_is_silent() -> None: + """It imports the daemon runtime modules without any warning.""" + script = ( + "import pyoaev, pyoaev.daemons, pyoaev.daemons.base_daemon\n" + "import pyoaev.daemons.collector_daemon, pyoaev.helpers, pyoaev.utils\n" + ) + result = subprocess.run( + [sys.executable, "-W", "error::DeprecationWarning", "-c", script], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/libs/xtm-oaev-sdk/test/daemons/test_base_daemon.py b/libs/xtm-oaev-sdk/test/daemons/test_base_daemon.py new file mode 100644 index 00000000..8077df55 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/daemons/test_base_daemon.py @@ -0,0 +1,160 @@ +"""Tests for the SDK BaseDaemon (API client injected).""" + +import unittest +import unittest.mock + +import xtm_oaev_sdk as sdk + +TEST_DAEMON_CONFIGURATION = sdk.Configuration( + config_hints={ + "log_level": {"data": "info"}, + "name": {"data": "my test daemon"}, + "collector_id": {"data": "collector-id"}, + } +) + + +class DaemonForTest(sdk.BaseDaemon): + def _setup(self): + pass + + def _start_loop(self): + pass + + +def create_mock_daemon(callback: callable = None): + mock_setup = unittest.mock.MagicMock() + mock_start_loop = unittest.mock.MagicMock() + inner_mock_func = unittest.mock.MagicMock() + + class MockDaemon(sdk.BaseDaemon): + def _setup(self): + mock_setup() + + def _start_loop(self): + mock_start_loop() + + def bound_method(self): + inner_mock_func() + + return ( + MockDaemon( + configuration=TEST_DAEMON_CONFIGURATION, + callback=callback, + logger=unittest.mock.MagicMock(), + api_client=unittest.mock.MagicMock(), + ), + mock_setup, + mock_start_loop, + inner_mock_func, + ) + + +@unittest.mock.patch("argparse.ArgumentParser") +class TestBaseDaemon(unittest.TestCase): + def test_injected_api_client_is_used(self, _): + client = unittest.mock.MagicMock() + daemon = DaemonForTest( + configuration=TEST_DAEMON_CONFIGURATION, api_client=client + ) + + self.assertIs(daemon.api, client) + + def test_without_api_client_ctor_raises(self, _): + with self.assertRaises(sdk.OpenAEVError): + DaemonForTest(configuration=TEST_DAEMON_CONFIGURATION) + + def test_without_api_client_subclass_hook_builds_it(self, _): + client = unittest.mock.MagicMock() + + class HookedDaemon(DaemonForTest): + def _create_default_api_client(self): + return client + + daemon = HookedDaemon(configuration=TEST_DAEMON_CONFIGURATION) + + self.assertIs(daemon.api, client) + + def test_default_logger_is_named_from_configuration(self, _): + daemon = DaemonForTest( + configuration=TEST_DAEMON_CONFIGURATION, + api_client=unittest.mock.MagicMock(), + ) + + self.assertIsInstance(daemon.logger, sdk.AppLogger) + + def test_get_id_reads_legacy_keys(self, _): + daemon, *_ = create_mock_daemon() + + self.assertEqual(daemon.get_id(), "collector-id") + + def test_when_no_callback_daemon_cant_start(self, m_argparser): + m_argparser.return_value.parse_args.return_value = unittest.mock.MagicMock( + dump_config_schema=False + ) + daemon, mock_setup, mock_start_loop, _ = create_mock_daemon() + + with self.assertRaises(sdk.OpenAEVError): + daemon.start() + + mock_setup.assert_not_called() + mock_start_loop.assert_not_called() + + def test_when_callback_daemon_can_start(self, m_argparser): + m_argparser.return_value.parse_args.return_value = unittest.mock.MagicMock( + dump_config_schema=False + ) + daemon, mock_setup, mock_start_loop, _ = create_mock_daemon(lambda: None) + + daemon.start() + + mock_setup.assert_called_once() + mock_start_loop.assert_called_once() + + def test_when_dump_config_schema_daemon_prints_schema_and_stops(self, m_argparser): + m_argparser.return_value.parse_args.return_value = unittest.mock.MagicMock( + dump_config_schema=True + ) + daemon, mock_setup, mock_start_loop, _ = create_mock_daemon(lambda: None) + + with unittest.mock.patch.object( + TEST_DAEMON_CONFIGURATION, "schema", return_value={} + ), unittest.mock.patch("builtins.print") as mock_print: + daemon.start() + + mock_print.assert_called_once_with({}) + mock_setup.assert_not_called() + mock_start_loop.assert_not_called() + + def test_when_callback_is_bound_method_daemon_can_call(self, _): + daemon, _, _, inner_mock_func = create_mock_daemon() + daemon.set_callback(daemon.bound_method) + + daemon._try_callback() + + inner_mock_func.assert_called_once() + + def test_when_callback_is_func_with_collector_parameter_daemon_can_call(self, _): + daemon, *_ = create_mock_daemon() + inner_mock_func = unittest.mock.MagicMock() + daemon.set_callback(lambda collector: inner_mock_func(collector)) + + daemon._try_callback() + + inner_mock_func.assert_called_once_with(daemon) + + def test_when_callback_raises_daemon_logs_and_survives(self, _): + daemon, *_ = create_mock_daemon() + + def failing(): + raise RuntimeError("boom") + + daemon.set_callback(failing) + + daemon._try_callback() + + daemon.logger.error.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/daemons/test_ping_alive.py b/libs/xtm-oaev-sdk/test/daemons/test_ping_alive.py new file mode 100644 index 00000000..67501a31 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/daemons/test_ping_alive.py @@ -0,0 +1,59 @@ +"""Tests for the PingAlive heartbeat thread.""" + +import unittest +from unittest.mock import MagicMock + +import xtm_oaev_sdk as sdk + + +class TestPingAlive(unittest.TestCase): + def _given_ping(self, ping_type: str, api=None, config=None) -> sdk.PingAlive: + ping = sdk.PingAlive( + api=api or MagicMock(), + config=config if config is not None else {}, + logger=MagicMock(), + ping_type=ping_type, + ) + ping.exit_event.is_set = MagicMock(side_effect=[False, True]) + ping.exit_event.wait = MagicMock() + return ping + + def test_injector_ping_registers_the_injector_every_40_seconds(self): + api = MagicMock() + ping = self._given_ping("injector", api=api, config={"id": 1}) + + ping.ping() + + api.injector.create.assert_called_once_with({"id": 1}, False) + ping.exit_event.wait.assert_called_once_with(40) + + def test_collector_ping_logs_api_errors_and_keeps_going(self): + api = MagicMock() + api.collector.create.side_effect = Exception("boom") + ping = self._given_ping("collector", api=api) + + ping.ping() + + ping.logger.error.assert_called_once() + ping.exit_event.wait.assert_called_once_with(40) + + def test_tenant_id_comes_from_the_client(self): + api = MagicMock() + api.tenant_id = "tenant-1" + + ping = sdk.PingAlive(api=api, config={}, logger=MagicMock(), ping_type="x") + + self.assertEqual(ping.tenant_id, "tenant-1") + + def test_stop_sets_the_exit_event(self): + ping = sdk.PingAlive( + api=MagicMock(), config={}, logger=MagicMock(), ping_type="collector" + ) + + ping.stop() + + self.assertTrue(ping.exit_event.is_set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/daemons/test_protocols.py b/libs/xtm-oaev-sdk/test/daemons/test_protocols.py new file mode 100644 index 00000000..23d14cdc --- /dev/null +++ b/libs/xtm-oaev-sdk/test/daemons/test_protocols.py @@ -0,0 +1,109 @@ +"""Tests for the BaseClient and DaemonProtocol runtime protocols.""" + +from typing import Any +from unittest.mock import MagicMock + +import pytest +import xtm_oaev_sdk as sdk + +_CLIENT_METHODS = [ + "http_delete", + "http_get", + "http_head", + "http_list", + "http_patch", + "http_post", + "http_put", +] +_CLIENT_PROPERTIES = [ + "backend", + "headers", + "order_by", + "pagination", + "per_page", + "session", + "ssl_verify", + "tenant_id", + "timeout", + "url", +] +_DAEMON_METHODS = ["get_id", "set_callback", "start"] + + +def _given_object_with(members: list[str]) -> Any: + instance = MagicMock(spec=members) + for name in members: + setattr(instance, name, MagicMock()) + return instance + + +def _when_reading_members(protocol: type, *, properties: bool) -> list[str]: + return sorted( + name + for name, value in protocol.__dict__.items() + if not name.startswith("_") + and isinstance(value, property) is properties + and (properties or callable(value)) + ) + + +@pytest.mark.parametrize("protocol", [sdk.BaseClient, sdk.DaemonProtocol]) +def test_protocols_are_runtime_checkable(protocol: type) -> None: + """It allows isinstance checks against the protocol.""" + assert getattr(protocol, "_is_runtime_protocol", False) is True + + +def test_base_client_declares_the_http_surface() -> None: + """It declares exactly the HTTP methods and client properties.""" + assert _when_reading_members(sdk.BaseClient, properties=False) == _CLIENT_METHODS + assert _when_reading_members(sdk.BaseClient, properties=True) == _CLIENT_PROPERTIES + + +def test_daemon_protocol_declares_the_lifecycle_surface() -> None: + """It declares exactly the public daemon lifecycle methods.""" + assert _when_reading_members(sdk.DaemonProtocol, properties=False) == ( + _DAEMON_METHODS + ) + + +@pytest.mark.parametrize( + ("protocol", "members"), + [ + (sdk.BaseClient, _CLIENT_METHODS + _CLIENT_PROPERTIES), + (sdk.DaemonProtocol, _DAEMON_METHODS), + ], +) +def test_protocols_accept_full_implementations(protocol: type, members) -> None: + """It accepts an object exposing every member.""" + assert isinstance(_given_object_with(members), protocol) + + +@pytest.mark.parametrize( + ("protocol", "members", "missing"), + [(sdk.BaseClient, _CLIENT_METHODS + _CLIENT_PROPERTIES, m) for m in _CLIENT_METHODS] + + [(sdk.DaemonProtocol, _DAEMON_METHODS, m) for m in _DAEMON_METHODS], +) +def test_protocols_reject_incomplete_implementations( + protocol: type, members, missing: str +) -> None: + """It rejects an object missing a member.""" + incomplete = [m for m in members if m != missing] + assert not isinstance(_given_object_with(incomplete), protocol) + + +def test_base_daemon_satisfies_daemon_protocol() -> None: + """It implements the daemon lifecycle protocol.""" + + class _Daemon(sdk.BaseDaemon): + def _setup(self): + pass + + def _start_loop(self): + pass + + daemon = _Daemon( + configuration=sdk.Configuration(config_hints={}), + api_client=MagicMock(), + logger=MagicMock(), + ) + assert isinstance(daemon, sdk.DaemonProtocol) diff --git a/libs/xtm-oaev-sdk/test/seam/__init__.py b/libs/xtm-oaev-sdk/test/seam/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/test/seam/fixtures/seam_golden.json b/libs/xtm-oaev-sdk/test/seam/fixtures/seam_golden.json new file mode 100644 index 00000000..8ff3ea3e --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/fixtures/seam_golden.json @@ -0,0 +1,462 @@ +{ + "xtm_oaev_sdk.seam.common.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.common.asset_types.AssetCategory": { + "kind": "class", + "sha256": "cb67f3d2a86d93fbb49632b69aac7774a1e31e3db05caf133dd91b5690d0c038" + }, + "xtm_oaev_sdk.seam.common.asset_types.AssetCriticality": { + "kind": "class", + "sha256": "d3dea966125525447d49a427bc2d367bdcc5aec55fc56b991e587796f5fe4da5" + }, + "xtm_oaev_sdk.seam.common.asset_types.AssetSubCategory": { + "kind": "class", + "sha256": "231d692d0185a13b417635bc42a83a1b7ee5a9cbf459e9fedc34327c400db920" + }, + "xtm_oaev_sdk.seam.common.asset_types.CloudProvider": { + "kind": "class", + "sha256": "c2a4fbf4b7164d34f0b8a2bf7504d1dcbc802e1e0313e9759d758b068fe5e6d6" + }, + "xtm_oaev_sdk.seam.common.errors.ConfigurationError": { + "kind": "class", + "sha256": "125c52aa906fe42d596f504d38ada75f770e485fffaac96d30acbca1b048737f" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVAuthenticationError": { + "kind": "class", + "sha256": "b4d32e039bd713e2e07514a31e70985e8be126b15f95f5d354fd2c2747c10942" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVCreateError": { + "kind": "class", + "sha256": "dd175bda1f009adb5aa6f34dc89e7cf93deac5ded93bf8602bedb8ad85c779e1" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVDeleteError": { + "kind": "class", + "sha256": "8a47c256d3a07a25812a7fe3779593ae805be299fbebd46dfe598d060953039e" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVError": { + "kind": "class", + "sha256": "f3c868a0b346592a84a66b71427732e5ff010efd7d9b2ef11f777da0914aded5" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVGetError": { + "kind": "class", + "sha256": "7464137d3c19d05ab07fc37335458d5921bbb1aad37f509ddcb1b11b7386ede7" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVHeadError": { + "kind": "class", + "sha256": "6c8d650e59878e26c39db48bbb81d62aba25dcfa7ddd6220bd2313dbcad61894" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVHttpError": { + "kind": "class", + "sha256": "5e161b4cacac55feef064b5f83697684906ca2b374d7e7444c1b1513eeabef50" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVListError": { + "kind": "class", + "sha256": "76f81f2f44676b5fc8b8e2f09a51cd0101ed422ef0fa282694e974446bd5c293" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVParsingError": { + "kind": "class", + "sha256": "3ed6ea9145afa9ff953356e88a51fe31e7ec07ec3f3e0a04101ccac3892c7f5c" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVUpdateError": { + "kind": "class", + "sha256": "87d0e9ab82f0ddabf054815332454a3e58e09e431b78f4f13efaf1e70d6e0c0c" + }, + "xtm_oaev_sdk.seam.common.errors.RedirectError": { + "kind": "class", + "sha256": "2d2529e12ab4453fd29a4600c66583dc3490715f61039b6bb89b9a3455407d58" + }, + "xtm_oaev_sdk.seam.common.errors.SignatureTransmissionError": { + "kind": "class", + "sha256": "9543fea4fe7a04030a363ec7539fc05b63327b5608af454aa55f4ee722e1be6c" + }, + "xtm_oaev_sdk.seam.common.ports.CertificatesPort": { + "kind": "class", + "sha256": "99ac2da214d3b0882f5d47d9d4212edf5965efc5006fd4b1758c3a8242686885" + }, + "xtm_oaev_sdk.seam.common.ports.JsonEncodingPort": { + "kind": "class", + "sha256": "c5592e4eb2c586c13e1a8ae633b1d18557c895bc2e0067e5d1957200e7ee0f35" + }, + "xtm_oaev_sdk.seam.common.ports.LoggingPort": { + "kind": "class", + "sha256": "31d0542014237114be994ae7c2b9023e2dd65a2c9c3758fc7fb0c638e23b19a2" + }, + "xtm_oaev_sdk.seam.common.protocols.AppLoggerProtocol": { + "kind": "class", + "sha256": "98410e8645499cef7ecf8d51fbf81613e1cdb9bb2ac19e345512c38d22eef273" + }, + "xtm_oaev_sdk.seam.common.protocols.BaseClient": { + "kind": "class", + "sha256": "9727fa9446d2be988fa26c983aaf23f65128fd86a6b8fd742ee2fd4c03b99120" + }, + "xtm_oaev_sdk.seam.common.protocols.DaemonProtocol": { + "kind": "class", + "sha256": "6e04f94e2ba40fa1b883e3cc8350d55f35c201fdfa5daf7a3ea8c17c8e66cab9" + }, + "xtm_oaev_sdk.seam.common.security_domains.SecurityDomains": { + "kind": "class", + "sha256": "8a27be9c4fd908e1cdab182e559ef8d4689410a405d11b424d3fb0f23995db30" + }, + "xtm_oaev_sdk.seam.configuration.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.configuration.models.BaseConfigModel": { + "kind": "pydantic model", + "sha256": "2744c173dd64c929eba8ba559e138e56cc76a5638c3d74705f3924ef156d651f" + }, + "xtm_oaev_sdk.seam.configuration.models.CONFIGURATION_TYPES": { + "kind": "assignment", + "sha256": "3a094919b672c6be37a56a9d698b39f75c372815e9e6703c32ee799aec21175c" + }, + "xtm_oaev_sdk.seam.configuration.models.ConfigLoaderCollector": { + "kind": "pydantic model", + "sha256": "b791f4b65fc00537c117e0664a894867e8f34b4a8abec4f8eabea3a16e062431" + }, + "xtm_oaev_sdk.seam.configuration.models.ConfigLoaderOAEV": { + "kind": "pydantic model", + "sha256": "066ed8c10d20abfe63042ec2be39c0cae23f9a15088681dc48d9a8a003925a62" + }, + "xtm_oaev_sdk.seam.configuration.models.ConfigurationHint": { + "kind": "pydantic model", + "sha256": "59e06102d55627f0c3e41c04116cbaa5f9a0269df750be9a56c9ed1121660c1d" + }, + "xtm_oaev_sdk.seam.configuration.models.HttpUrlToString": { + "kind": "assignment", + "sha256": "7ce246816b036a5b93ecc244dfddec85adf28f3d53562c92052225071323dc96" + }, + "xtm_oaev_sdk.seam.configuration.models.LogLevelToLower": { + "kind": "assignment", + "sha256": "969e62cd77ce3f1aa7df05a3d2b8dd5c4a9a74dcd6bb5b31bcac5ff4386348d7" + }, + "xtm_oaev_sdk.seam.configuration.models.TimedeltaInSeconds": { + "kind": "assignment", + "sha256": "fdc6122dbc50a989af4baed5b798f25143eeb2c9d4c7e0d6b5189079d19e5ef4" + }, + "xtm_oaev_sdk.seam.configuration.ports.ConfigurationProtocol": { + "kind": "class", + "sha256": "43a6d1c58de96eeb8a268e62983d1c0d04c3ec5ca60bc7db3cf6440f1b5e2ef5" + }, + "xtm_oaev_sdk.seam.configuration.ports.ConfigurationResolverPort": { + "kind": "class", + "sha256": "c36adb803d996956a4a88fefcd3468f9a2e67dec149eb3170cff4d6e5532c8f9" + }, + "xtm_oaev_sdk.seam.configuration.ports.ConfigurationSourcesPort": { + "kind": "class", + "sha256": "7951e9dfa860e4c8d2553c9ca55cf46d59ce41435aa64aa8c0773245251ca161" + }, + "xtm_oaev_sdk.seam.configuration.ports.ConnectorSchemaPort": { + "kind": "class", + "sha256": "9ba35717945700beeaa45407c628283098b2ed983858af8a9ffefa5ad78cce8e" + }, + "xtm_oaev_sdk.seam.configuration.ports.SettingsSourcesPort": { + "kind": "class", + "sha256": "67681b5861788fdcf12585371293b814ef5a32418dbee64f28934ef919c24ecf" + }, + "xtm_oaev_sdk.seam.configuration.ports.TruthinessPort": { + "kind": "class", + "sha256": "3c5238b25e5c46bbde80ec4843a624776521e65e323ec018eb41531a8ef63535" + }, + "xtm_oaev_sdk.seam.contracts.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractAiTarget": { + "kind": "class", + "sha256": "bec76a7237ac27571146a586dfc1867275978925020cb495e92c487bfb3dbc82" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractAsset": { + "kind": "class", + "sha256": "4118fa0f218b9fa9ad13cb46f54cd42708826b9b8597c681f1f423846781aae0" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractAssetGroup": { + "kind": "class", + "sha256": "e68cdc9fd005c8d71e3399b8dd0cdfb82f4d06e32f6a6dbcb46971266942bbb7" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractAttachment": { + "kind": "class", + "sha256": "cb454a9aef9ac29dffc69eca75a577fce9160222f273a21ced23431e80ed62e9" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractCardinality": { + "kind": "class", + "sha256": "2c81f89d9e843a1a0171f3a4bd3646595d4ce398e5bd9a9361aa4d9bf94e17e7" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractCardinalityElement": { + "kind": "class", + "sha256": "1c6592f1e5664ad6ead85276e6987dd081896f59ae86ced5e6ab8ce1220987ee" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractCheckbox": { + "kind": "class", + "sha256": "9dd31fc0a96c97e790fface784a338b58682e02d8f9cbba9a32a44f33fc1f4d2" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractConfig": { + "kind": "class", + "sha256": "0186912bf55cc89902df1a42041c2b57e3b033cdfe79f7bf7ce4e043b7f35281" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractData": { + "kind": "class", + "sha256": "8abb76a14b43660f67ecad3a47ab6026b3785bfa832d8b6c5969ad209d024112" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractElement": { + "kind": "class", + "sha256": "43fa42de3e2c120636e4f4ae800952f11d0a75b2bf7f9db4b3afb129609031b6" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractExpectationType": { + "kind": "class", + "sha256": "cdbf56a38b1b4cff63229ad6fdb5939b07ffbc7e292112fbcb77206f173b3dc3" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractExpectations": { + "kind": "class", + "sha256": "81f1e6aa56166766600d43854360c6b565ba88c7b38c1c7c9aba1c55a440a768" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractFieldKey": { + "kind": "class", + "sha256": "3aa5cdebc64b0e09fa18950aa7c1bff2f1fad4cbfd4ac106a200cf01a453c7ae" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractFieldType": { + "kind": "class", + "sha256": "3efc706f2a68aa7fb006df594a5708ee018e9d615bbd572d96f7a64f65c0fbb7" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractOutputElement": { + "kind": "class", + "sha256": "903e0d46c2f0ccfbe550f0cd067d212f67905f208940fc06f8638ee4854d5a97" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractOutputType": { + "kind": "class", + "sha256": "52604db256eb23b5d7dc0a564a5e41f7e4b3e764ad89af0d97162471ca8be3d2" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractPayload": { + "kind": "class", + "sha256": "a424a20ef31a74cdce9c3b3b585b0d46a408c1dabdc4d70a53e88a5a830dab63" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractReferencedCredential": { + "kind": "class", + "sha256": "cfadf2858d353ad60b2ab963ed8dbc98054dec687862cbaf70efe3acf3f00ea2" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractSelect": { + "kind": "class", + "sha256": "1ff539b29b70b4090f175b11e7cf42f5476643a32937531301c27b8af2eba3ee" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractTeam": { + "kind": "class", + "sha256": "47b63fd6ca940482b964b0ecedb182de60a249a5fbf9310642745f7b1148b214" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractText": { + "kind": "class", + "sha256": "9e8170048f9340f1e3178ffe60e428c8574bca27148357856dbe5303fc89df15" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractTextArea": { + "kind": "class", + "sha256": "dc9d182716bf85bad565b2b7d0a9025e2869701f6e7dfb14b83009685771967e" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractTuple": { + "kind": "class", + "sha256": "a89fdb4ccf80bbbf87b19e3d330978bb995657d82e02c72fa814f57bfaa7eed6" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractVariable": { + "kind": "class", + "sha256": "46522f5a0c908d7376b8ffa389d87113a972848467711a46cdc4dc5d6262d49e" + }, + "xtm_oaev_sdk.seam.contracts.models.CredentialType": { + "kind": "class", + "sha256": "7e29de5fbd972fab40d3525275d4c93fbb54347869c1a5efe24b6737f2719890" + }, + "xtm_oaev_sdk.seam.contracts.models.Domain": { + "kind": "class", + "sha256": "26fd50e1160ff864f4ac1404a08e1594a99c5df39d3f482c0c6492de3a08f223" + }, + "xtm_oaev_sdk.seam.contracts.models.Expectation": { + "kind": "class", + "sha256": "8de1930e714539c664d53e0fee6e37ee8c2af6fe72502181db9c0cc6bd75046d" + }, + "xtm_oaev_sdk.seam.contracts.models.LinkedFieldModel": { + "kind": "class", + "sha256": "73c3eec7ba6d4182212b9d9e09cee3b11cc437b3a8d8997b84c54fad8731d234" + }, + "xtm_oaev_sdk.seam.contracts.models.PrimitiveType": { + "kind": "class", + "sha256": "741b00f6883baea168f1ea19c705eba1e1e9f001e02a95a3570c471eafe2c020" + }, + "xtm_oaev_sdk.seam.contracts.models.SecurityPlatformType": { + "kind": "class", + "sha256": "8d9bdf6573fa6c32cae8784b83bef9cd19532db2238ef45cc65b6ad9078d5988" + }, + "xtm_oaev_sdk.seam.contracts.models.SupportedLanguage": { + "kind": "class", + "sha256": "a367cd2ecbe05dc4c118940ab719dabf0128dfacd8fbf18ec162bcd3b4677592" + }, + "xtm_oaev_sdk.seam.contracts.models.VariableType": { + "kind": "class", + "sha256": "22dc85d597bba70a70742f392b5d621514da4d23662f116b00f33808c8262593" + }, + "xtm_oaev_sdk.seam.contracts.ports.ContractAssemblyPort": { + "kind": "class", + "sha256": "0bd7a273151c8726d905138f48a45239d9d86e776cc82686465607a6a0d6fe5e" + }, + "xtm_oaev_sdk.seam.contracts.ports.ContractBuilderProtocol": { + "kind": "class", + "sha256": "c64343786d3ce2d454cb8966b36ec7a3afecc359ef954191fa9ca5ec22af76ff" + }, + "xtm_oaev_sdk.seam.contracts.ports.ContractCredentialsPort": { + "kind": "class", + "sha256": "d2320f0e0d1a601d423d7478a7106758061a83a99644bebfa91edd7f197e1687" + }, + "xtm_oaev_sdk.seam.contracts.ports.ContractSerializerPort": { + "kind": "class", + "sha256": "0366d67e6b581aac3fdc5f6dfeb4e7eba65a21da98a1fc89a0add1c192cb3327" + }, + "xtm_oaev_sdk.seam.contracts.ports.ContractVariablesPort": { + "kind": "class", + "sha256": "a3845ad0c166456174e25d299cff4a16cae2117ce13bd10174fb1c843f183886" + }, + "xtm_oaev_sdk.seam.daemons.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.daemons.ports.DaemonRuntimePort": { + "kind": "class", + "sha256": "057f096c283a09b2cb560b9d39de886df4f237c20c438ce02acab991523c2e2f" + }, + "xtm_oaev_sdk.seam.daemons.ports.HeartbeatPort": { + "kind": "class", + "sha256": "1e42dda89a11a88eead8163c170c27d5136996820d4d1fc3464bd6dfed7fc242" + }, + "xtm_oaev_sdk.seam.deprecation.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.deprecation.ports.DeprecatedNames": { + "kind": "assignment", + "sha256": "0fa67de53de219b64454162f3fea7f07bed0504929cc8520bba25a5a7794d0b0" + }, + "xtm_oaev_sdk.seam.deprecation.ports.DeprecationPort": { + "kind": "class", + "sha256": "c0631fd4c1bfa509bce632c1c815865d504293e100b8798981df36567bad5c3b" + }, + "xtm_oaev_sdk.seam.port_io.models.ExecutionDetails": { + "kind": "pydantic model", + "sha256": "4d9546d2033c3d68bc31a688481bc580fb876fa1f984495bd8d1ac4bb8c6ce15" + }, + "xtm_oaev_sdk.seam.port_io.models.ExecutionStatus": { + "kind": "class", + "sha256": "1ac92fe06592d1ef55703df64941baf70b75c646d138e4f7698abe02bba50af0" + }, + "xtm_oaev_sdk.seam.port_io.models.ExpectationSignatureGroup": { + "kind": "pydantic model", + "sha256": "1349c78c33892854ef77030364dd310f1dd98ebf23688b97267a046bd5926a8c" + }, + "xtm_oaev_sdk.seam.port_io.models.InjectExecutionActions": { + "kind": "class", + "sha256": "45eaf82367b11a87617ec5563c1109b2ec9050fd5c58f70a7c5875c4352f9eb1" + }, + "xtm_oaev_sdk.seam.port_io.models.PORT_IO_CONTRACT_VERSION": { + "kind": "assignment", + "sha256": "4b3aa3a5b079ff1222ce2bb723750e882555bc2c857097f589066dcae2398d0a" + }, + "xtm_oaev_sdk.seam.port_io.models.SignatureCallbackPayload": { + "kind": "pydantic model", + "sha256": "c9bc7d84b064b595fd1cb7000d43d47246cb1026b89b2607e4870443e17dde61" + }, + "xtm_oaev_sdk.seam.port_io.models.SignatureExpectationType": { + "kind": "class", + "sha256": "d66c56b5404fbb512cb3d29ff52dcef0cb0a984fdfab1492f59185854c8117e3" + }, + "xtm_oaev_sdk.seam.port_io.models.SignatureOutputStructure": { + "kind": "pydantic model", + "sha256": "81f15a75c0b933604f13462c1721c2698e46acfdd170e3d629516f7f68946f5d" + }, + "xtm_oaev_sdk.seam.port_io.models.SignaturePayload": { + "kind": "pydantic model", + "sha256": "88f1171c10351454a081797da8dd3331d4fcda745b1b88654d7a2fbb50c41eab" + }, + "xtm_oaev_sdk.seam.port_io.models.SignatureTarget": { + "kind": "pydantic model", + "sha256": "f1d7d41f96daaaf60fe0a4fe6ae31e702c69f22080e096e867a632299ce033a5" + }, + "xtm_oaev_sdk.seam.port_io.models.SignatureValue": { + "kind": "pydantic model", + "sha256": "84f20eb4416d5279034c1c56108f5c28d52d2a9842e8d14cbde288f084f73cc1" + }, + "xtm_oaev_sdk.seam.port_io.models.TargetSignatures": { + "kind": "pydantic model", + "sha256": "59c1618a317ff3f4390d3c0531e815168dce910204987062c9e044c6466b908d" + }, + "xtm_oaev_sdk.seam.signatures.models.ExtraSignatureData": { + "kind": "pydantic model", + "sha256": "b69dd6b7476302eff126622e5d662fc297ca964c0484e6f948589cd1450ae4c5" + }, + "xtm_oaev_sdk.seam.signatures.models.MatchTypes": { + "kind": "class", + "sha256": "99eb73ffc33a3163857a2fa3d5ca00d8c56184ea98984fedf7a173fc197eb306" + }, + "xtm_oaev_sdk.seam.signatures.models.SignatureMatch": { + "kind": "class", + "sha256": "99908bfec243c4fb233ffeca62108abb0929ad8f0fbac1ce8eb434779fa0fc16" + }, + "xtm_oaev_sdk.seam.signatures.models.SignatureType": { + "kind": "class", + "sha256": "1eb1fba1a1470c6a2dd8d453ce6b318e2f9db20627c1ba4f6d36ef3c4266605a" + }, + "xtm_oaev_sdk.seam.signatures.models.SignatureTypes": { + "kind": "class", + "sha256": "46b9692d876bb8344229523d0fa65e46f4c59d521ebc24c47994a39a950756ae" + }, + "xtm_oaev_sdk.seam.signatures.ports.ContainerAddressPort": { + "kind": "class", + "sha256": "7fc3b66bf6cbf6aa3ee01c2d6273acabd35a5e1fe75d542709a91c22fb6f8f07" + }, + "xtm_oaev_sdk.seam.signatures.ports.MarkerPort": { + "kind": "class", + "sha256": "a6b4deafa934f4cf37987788f84b17df4dd69cffc8a9e7cf356682d735188ef1" + }, + "xtm_oaev_sdk.seam.signatures.ports.NetworkTargetsPort": { + "kind": "class", + "sha256": "9b60ef67d3ecd1370ad20d52a31a0b500bada8ed098e9d863249e421f3f77f74" + }, + "xtm_oaev_sdk.seam.signatures.ports.SignatureEnginePort": { + "kind": "class", + "sha256": "fdd1998986921f3c8d4b40d1a08f4897c10cee4b39b0d7186e0044b4841ea166" + }, + "xtm_oaev_sdk.seam.signatures.ports.SignatureManagerProtocol": { + "kind": "class", + "sha256": "7254b834dd9f952a5c248eb3039f7ccc9f88ec92e8279b211bf5968b14bfe513" + }, + "xtm_oaev_sdk.seam.signatures.ports.SignatureTransportProtocol": { + "kind": "class", + "sha256": "0bb363c3e37df931c589bbeb5b5febdb40910e35768c39b2d5e8a039829e8bb8" + }, + "xtm_oaev_sdk.seam.signatures.seam.CloudInjectorConfig": { + "kind": "pydantic model", + "sha256": "6dfe3660acd0b99ec29b1898a88501ba75fc83d11e6c967ac130a77d6950a980" + }, + "xtm_oaev_sdk.seam.signatures.seam.ExecutionSignature": { + "kind": "pydantic model", + "sha256": "75fcceb51cf8dd3c0c3e2b7eb65f8b0644b08ec6e7b8d012d5624250600553ed" + }, + "xtm_oaev_sdk.seam.signatures.seam.InjectorConfig": { + "kind": "assignment", + "sha256": "dd77ee2c6e42ebacd37e4177bc5d10df3b0321f279ef57b3dfd7a31fd934f100" + }, + "xtm_oaev_sdk.seam.signatures.seam.NetworkInjectorConfig": { + "kind": "pydantic model", + "sha256": "ceafda27b94d3b0043d34965dc5a927f8b8a1b833d0b2269140a04dbac797d89" + }, + "xtm_oaev_sdk.seam.signatures.seam.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.signatures.seam.ToolErrorInfo": { + "kind": "pydantic model", + "sha256": "27a3d760a76e74e0a556293741de45032384c3df107c1b33504c2b0003018f53" + }, + "xtm_oaev_sdk.seam.signatures.seam.ToolOutput": { + "kind": "pydantic model", + "sha256": "25d4fc1fa9f7e6bfc358a1104fd7e0fb620a232d70a1c3d8fcc65e6a79c88c9d" + }, + "xtm_oaev_sdk.seam.signatures.seam.ToolTimeoutInfo": { + "kind": "pydantic model", + "sha256": "57f73a9a8003e3907ef8e81120a31c2d2a7efb99d85c0effee699b133b05a44a" + } +} diff --git a/libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py b/libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py new file mode 100644 index 00000000..9c330bf1 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py @@ -0,0 +1,187 @@ +"""Fingerprint every public object of the xtm-oaev-sdk seam and update the golden. + +Discovery is dynamic: every ``*.py`` file under ``xtm_oaev_sdk/seam`` is parsed, and +every public top-level definition (class, function, assignment) gets a sha256 of: + +- its normalized AST: formatting, comments and docstrings are ignored, so the hash + moves only when the definition itself changes (fields, defaults, enum members, + signatures, validators, constant values...); +- for pydantic models, their validation and serialization JSON schemas too, which + also catch what reaches the wire through the types they reference. + +Run from the repository root after a deliberate seam change, together with the +``SEAM_VERSION`` bump of the capability (``PORT_IO_CONTRACT_VERSION`` for port_io):: + + python libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py + +It rewrites ``fixtures/seam_golden.json``: changed fingerprints are updated, new +objects added, and objects that no longer exist removed. ``test_seam_golden.py`` +fails on any difference between the golden and the code. +""" + +import ast +import copy +import hashlib +import importlib +import json +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import xtm_oaev_sdk.seam + +SEAM_ROOT = Path(xtm_oaev_sdk.seam.__file__).parent +PACKAGE_ROOT = SEAM_ROOT.parent.parent +GOLDEN_PATH = Path(__file__).parent / "fixtures" / "seam_golden.json" + +_IGNORED_AST_FIELDS = {"ctx", "type_comment"} +_DOCSTRING_OWNERS = (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef) + + +def _module_name(path: Path) -> str: + parts = path.relative_to(PACKAGE_ROOT).with_suffix("").parts + return ".".join(parts[:-1] if parts[-1] == "__init__" else parts) + + +def _defined_names(node: ast.stmt) -> list[str]: + if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + return [node.name] + if isinstance(node, ast.Assign): + return [t.id for t in node.targets if isinstance(t, ast.Name)] + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + return [node.target.id] + if isinstance(node, ast.TypeAlias): + return [node.name.id] + return [] + + +def _without_docstrings(node: ast.stmt) -> ast.stmt: + node = copy.deepcopy(node) + for child in ast.walk(node): + if isinstance(child, _DOCSTRING_OWNERS) and child.body: + first = child.body[0] + if ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + ): + child.body = child.body[1:] + return node + + +def _canonical_ast(node: Any) -> str: + """Python-version-stable text of an AST: no positions, no empty fields.""" + if isinstance(node, ast.AST): + fields = [] + for field in node._fields: + if field in _IGNORED_AST_FIELDS: + continue + value = getattr(node, field, None) + if value is None or value == []: + continue + fields.append(f"{field}={_canonical_ast(value)}") + return f"{type(node).__name__}({', '.join(fields)})" + if isinstance(node, list): + return "[" + ", ".join(_canonical_ast(item) for item in node) + "]" + return repr(node) + + +def _pydantic_schemas(obj: Any) -> str | None: + from pydantic import BaseModel + + if not (isinstance(obj, type) and issubclass(obj, BaseModel)): + return None + schemas = { + mode: obj.model_json_schema(mode=mode) + for mode in ("validation", "serialization") + } + return json.dumps(schemas, sort_keys=True) + + +def iter_seam_definitions() -> Iterator[tuple[str, str, ast.stmt, Any]]: + """Every public top-level definition of the seam: (module, name, node, object).""" + for path in sorted(SEAM_ROOT.rglob("*.py")): + module_name = _module_name(path) + module = importlib.import_module(module_name) + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + for name in _defined_names(node): + if not name.startswith("_"): + yield module_name, name, node, getattr(module, name) + + +def _kind(node: ast.stmt, obj: Any) -> str: + if _pydantic_schemas(obj) is not None: + return "pydantic model" + if isinstance(node, ast.ClassDef): + return "class" + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return "function" + return "assignment" + + +def compute_seam_fingerprints() -> dict[str, dict[str, str]]: + """``{".": {"kind": ..., "sha256": ...}}`` for the whole seam.""" + fingerprints = {} + for module_name, name, node, obj in iter_seam_definitions(): + text = "ast:" + _canonical_ast(_without_docstrings(node)) + schemas = _pydantic_schemas(obj) + if schemas is not None: + text += "\nschemas:" + schemas + fingerprints[f"{module_name}.{name}"] = { + "kind": _kind(node, obj), + "sha256": hashlib.sha256(text.encode()).hexdigest(), + } + return dict(sorted(fingerprints.items())) + + +def diff_fingerprints( + golden: dict[str, dict[str, str]], current: dict[str, dict[str, str]] +) -> dict[str, list[str]]: + """Seam objects added, removed and changed since the golden.""" + return { + "added": sorted(current.keys() - golden.keys()), + "removed": sorted(golden.keys() - current.keys()), + "changed": sorted( + key + for key in current.keys() & golden.keys() + if current[key]["sha256"] != golden[key]["sha256"] + ), + } + + +def _capability(key: str) -> str: + return key.split(".")[2] + + +def _unversioned_capabilities(diff: dict[str, list[str]]) -> list[str]: + touched = {_capability(key) for keys in diff.values() for key in keys} + bumped = { + _capability(key) + for keys in diff.values() + for key in keys + if key.rsplit(".", 1)[1].endswith("VERSION") + } + return sorted(touched - bumped) + + +def main() -> int: + golden = json.loads(GOLDEN_PATH.read_text()) if GOLDEN_PATH.exists() else {} + current = compute_seam_fingerprints() + diff = diff_fingerprints(golden, current) + for change, keys in diff.items(): + for key in keys: + print(f"{change:8} {key}") + for capability in _unversioned_capabilities(diff): + print( + f"WARNING the {capability} seam changed but its version did not: " + "bump it if this is a contract change." + ) + GOLDEN_PATH.write_text(json.dumps(current, indent=2) + "\n") + print(f"wrote {len(current)} fingerprints to {GOLDEN_PATH}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/libs/xtm-oaev-sdk/test/seam/test_seam_closed.py b/libs/xtm-oaev-sdk/test/seam/test_seam_closed.py new file mode 100644 index 00000000..29f5e502 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/test_seam_closed.py @@ -0,0 +1,36 @@ +"""The two wire-facing seam modules stay closed to the SDK internals.""" + +import ast +from pathlib import Path + +from xtm_oaev_sdk.seam.port_io import models as port_io_models +from xtm_oaev_sdk.seam.signatures import seam as seam_module + + +def test_port_io_models_import_only_the_signatures_seam(): + """The wire contract is closed: no import of the capability internals.""" + source = Path(port_io_models.__file__).read_text() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.ImportFrom) and (node.module or "").startswith( + "xtm_oaev_sdk" + ): + assert node.module == "xtm_oaev_sdk.seam.signatures.seam", node.module + elif isinstance(node, ast.Import): + for alias in node.names: + assert not alias.name.startswith("xtm_oaev_sdk"), alias.name + + +def test_seam_imports_nothing_from_capability(): + """The seam is pure data: no imports from xtm_oaev_sdk itself.""" + source = Path(seam_module.__file__).read_text() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + for alias in node.names: + assert not alias.name.startswith( + "xtm_oaev_sdk" + ), f"seam imports capability module {alias.name!r} — pure-data violation" + elif isinstance(node, ast.ImportFrom): + module = node.module or "" + assert not module.startswith( + "xtm_oaev_sdk" + ), f"seam imports from capability module {module!r} — pure-data violation" diff --git a/libs/xtm-oaev-sdk/test/seam/test_seam_golden.py b/libs/xtm-oaev-sdk/test/seam/test_seam_golden.py new file mode 100644 index 00000000..f1ab096a --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/test_seam_golden.py @@ -0,0 +1,66 @@ +"""The seam golden: every public seam object is fingerprinted, and any drift fails. + +The fingerprints come from ``generate_seam_golden.py``, which discovers the seam +dynamically (every ``*.py`` file, every public top-level definition). On a failure, +``git diff`` the objects listed: they are exactly what changed. If the change is +deliberate, bump the capability's ``SEAM_VERSION`` (``PORT_IO_CONTRACT_VERSION`` for +port_io), then run, from the repository root:: + + python libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py +""" + +import importlib.util +import inspect +import json +from pathlib import Path +from types import ModuleType + +import xtm_oaev_sdk + +_GENERATOR = Path(__file__).parent / "generate_seam_golden.py" + + +def _given_generator() -> ModuleType: + spec = importlib.util.spec_from_file_location("generate_seam_golden", _GENERATOR) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _given_golden(generator: ModuleType) -> dict[str, dict[str, str]]: + return json.loads(generator.GOLDEN_PATH.read_text()) + + +def _when_fingerprinting_the_seam(generator: ModuleType) -> dict[str, dict[str, str]]: + return generator.compute_seam_fingerprints() + + +def _then_no_drift(diff: dict[str, list[str]]) -> None: + drift = [f"{change}: {key}" for change, keys in diff.items() for key in keys] + assert not drift, ( + "The seam drifted from its golden:\n " + + "\n ".join(drift) + + "\nIf deliberate: bump the capability's version, then run " + "`python libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py`." + ) + + +def test_seam_matches_its_golden() -> None: + """It fails on any seam object added, removed or changed since the golden.""" + generator = _given_generator() + golden = _given_golden(generator) + current = _when_fingerprinting_the_seam(generator) + _then_no_drift(generator.diff_fingerprints(golden, current)) + + +def test_root_reexports_are_the_seam_objects() -> None: + """It hands out, from the package root, the very objects the seam defines.""" + generator = _given_generator() + broken = [ + f"{module}.{name}" + for module, name, _, obj in generator.iter_seam_definitions() + if (inspect.isclass(obj) or inspect.isfunction(obj)) + and hasattr(xtm_oaev_sdk, name) + and getattr(xtm_oaev_sdk, name) is not obj + ] + assert not broken, f"Root re-exports are not the seam objects: {broken}" diff --git a/libs/xtm-oaev-sdk/test/signatures/__init__.py b/libs/xtm-oaev-sdk/test/signatures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_ip_resolution_constraints.feature b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_ip_resolution_constraints.feature new file mode 100644 index 00000000..30ab2524 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_ip_resolution_constraints.feature @@ -0,0 +1,8 @@ +Feature: SignatureManager IP resolution constraints + + Scenario: resolve_container_ip returns unknown and emits exactly one warning when all strategies fail + Given all IP resolution strategies are mocked to fail + When I call resolve_container_ip + Then the returned value is the string "unknown" + And exactly 1 WARNING log message is emitted + And no exception propagates from resolve_container_ip diff --git a/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_post_execution_constraints.feature b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_post_execution_constraints.feature new file mode 100644 index 00000000..ec7b4dc0 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_post_execution_constraints.feature @@ -0,0 +1,39 @@ +Feature: SignatureManager post-execution constraints + As an injector using the OpenAEV client + I want post-execution compilation to handle failure and timeout edge cases + So that incomplete executions still produce valid signature records + + Background: + Given a SignatureManager initialised with constructor SignatureManager(transport, logger) + And a execution_signatures object containing: + | key | value | + | source_ipv4 | 172.17.0.2 | + | target_ipv4 | 10.0.0.1 | + | target_hostname | host-a.internal | + | start_time | 2024-06-26T06:00:00Z | + And a execution_details object containing: + | key | value | + | start_time | 2024-06-26T06:00:00Z | + + Scenario: Tool crash sets execution_status to failed and uses crash timestamp as end_time + Given a tool_output containing error_info with exit_code=1 and crash_timestamp="2024-06-26T06:05:00Z" + When I call post_execution_updates with the execution_details, execution_signatures and tool_output + Then execution_status equals "ERROR" + And end_time equals "2024-06-26T06:05:00Z" + And the execution signature model contains every previous parameter unchanged + And the execution details model contain every previous parameter pair unchanged + + Scenario: Timeout sets execution_status to timeout and includes available partial results + Given a tool_output containing timeout_info with partial_results=["result-A", "result-B"] + When I call post_execution_updates with the execution_details, execution_signatures and tool_output + Then execution_status equals "TIMEOUT" + And the returned dict contains the partial results ["result-A", "result-B"] from timeout_info + And the execution signature model contains every previous parameter unchanged + And the execution details model contain every previous parameter pair unchanged + + Scenario: Timeout with no partial results still sets execution_status to timeout + Given a tool_output containing timeout_info with no partial results available + When I call post_execution_updates with the execution_details, execution_signatures and tool_output + Then execution_status equals "TIMEOUT" + And the execution signature model contains every previous parameter unchanged + And the execution details model contain every previous parameter pair unchanged diff --git a/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_pre_execution_constraints.feature b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_pre_execution_constraints.feature new file mode 100644 index 00000000..ddff8521 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_pre_execution_constraints.feature @@ -0,0 +1,15 @@ +Feature: SignatureManager pre-execution constraints + As an injector using the OpenAEV client + I want pre-execution compilation to handle timing edge cases correctly + So that signatures always reflect the actual moment of execution + + Background: + Given a SignatureManager initialised with constructor SignatureManager(transport, logger) + + Scenario: start_time is generated at method-call time not at class instantiation time + Given a SignatureManager that was instantiated at timestamp T0 + And 5 seconds elapse after instantiation + And a NetworkInjectorConfig with target_ipv4="192.168.1.10" + When I call build_execution_signatures with the config at timestamp T1 + Then the start_time in the returned dict equals T1 within 1 second tolerance + And start_time does not equal T0 diff --git a/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_ip_resolution.feature b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_ip_resolution.feature new file mode 100644 index 00000000..0751541b --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_ip_resolution.feature @@ -0,0 +1,12 @@ +Feature: SignatureManager container IP resolution + + Scenario Outline: resolve_container_ip returns a valid IPv4 in each supported execution environment + Given a SignatureManager running in a "" environment + When I call resolve_container_ip + Then the returned value is a non-empty valid IPv4 address string + + Examples: + | environment | + | Docker | + | Kubernetes | + | bare-metal | diff --git a/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_post_execution.feature b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_post_execution.feature new file mode 100644 index 00000000..746c5a81 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_post_execution.feature @@ -0,0 +1,28 @@ +Feature: SignatureManager post-execution execution elements update + As an injector using the OpenAEV client + I want to update both execution signatures and execution details with execution results + So that each inject has a complete execution record including outcome and timing + + Background: + Given a SignatureManager initialised with constructor SignatureManager(transport, logger) + And a execution_signatures object containing: + | key | value | + | source_ipv4 | 172.17.0.2 | + | target_ipv4 | 10.0.0.1 | + | target_hostname | host-a.internal | + | start_time | 2024-06-26T06:00:00Z | + And a execution_details object containing: + | key | value | + | start_time | 2024-06-26T06:00:00Z | + + Scenario: Successful execution updates end_time and execution_status in execution signatures and execution details + Given a tool_output indicating successful completion with no errors and no timeout + When I call post_execution_updates with the execution_details, execution_signatures and tool_output + Then the execution signature model contains every previous parameter unchanged + And the end_time parameter in the execution signature model is a UTC ISO 8601 string + And this end_time is chronologically greater than or equal to start_time "2024-06-26T06:00:00Z" + And the execution details model contain every previous parameter pair unchanged + And the end_time parameter in the execution details model is a datetime object + And this end_time is chronologically greater than or equal to start_time "2024-06-26T06:00:00Z" + And the execution_status parameter in the execution details model is equal to "EXECUTED" + And the execution_action parameter in the execution details model is equal to "data" diff --git a/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_pre_execution.feature b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_pre_execution.feature new file mode 100644 index 00000000..0f1edf49 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_pre_execution.feature @@ -0,0 +1,83 @@ +Feature: SignatureManager pre-execution signature compilation + As an injector using the OpenAEV client + I want to compile category-specific pre-execution signatures + So that each inject has a correct, typed signature payload before execution begins + + Background: + Given a SignatureManager initialised with constructor SignatureManager(transport, logger) + + Scenario: Network category returns required IP and timing fields and no cloud or query fields + Given a NetworkInjectorConfig with target_ipv4="192.168.1.10" + And the running container has a resolvable IPv4 address + When I call build_execution_signatures with the config + Then the returned dict contains source_ipv4 as a non-empty valid IPv4 address string + And the returned dict contains start_time as a UTC ISO 8601 string + And the returned dict contains target_ipv4 equal to "192.168.1.10" + But the returned dict does not contain cloud_provider + And the returned dict does not contain cloud_account_id + And the returned dict does not contain cloud_region + And the returned dict does not contain target_service + And the returned dict does not contain query + + Scenario: Network hostname target returns hostname and no target_ipv4 + Given a NetworkInjectorConfig with target_hostname="target.example.com" + And the running container has a resolvable IPv4 address + When I call build_execution_signatures with the config + Then the returned dict contains target_hostname equal to "target.example.com" + And the returned dict contains source_ipv4 as a non-empty valid IPv4 address string + But the returned dict does not contain target_ipv4 + + Scenario: Cloud category returns required cloud identity fields and no IP fields + Given a CloudInjectorConfig with cloud_provider="aws", cloud_account_id="123456789012", cloud_region="eu-west-1", and target_service="ec2" + When I call build_execution_signatures with the config + Then the returned dict contains cloud_provider equal to "aws" + And the returned dict contains cloud_account_id equal to "123456789012" + And the returned dict contains cloud_region equal to "eu-west-1" + And the returned dict contains target_service equal to "ec2" + And the returned dict contains start_time as a UTC ISO 8601 string + But the returned dict does not contain source_ipv4 + And the returned dict does not contain source_ipv6 + And the returned dict does not contain target_ipv4 + And the returned dict does not contain target_ipv6 + + Scenario Outline: Network multi-target returns one dict per target with a shared source IP + Given a list of 3 NetworkInjectorConfig with target_ipv4 "10.0.0.1", "10.0.0.2", "10.0.0.3" + And the running container has a resolvable IPv4 address "172.17.0.2" + When I call build_execution_signatures with the config list + Then the return value is a list of exactly 3 dicts + And the dict at position contains target_ipv4 equal to "" + And the dict at position contains source_ipv4 equal to "172.17.0.2" + + Examples: + | index | target_ip | + | 0 | 10.0.0.1 | + | 1 | 10.0.0.2 | + | 2 | 10.0.0.3 | + + Scenario: All network multi-target dicts share the same source_ipv4 + Given a list of 3 NetworkInjectorConfig built from default IPv4 targets + And the running container has a resolvable IPv4 address + When I call build_execution_signatures with the config list + Then the return value is a list of 3 dicts + And all 3 dicts contain the same source_ipv4 value + + Scenario Outline: Cloud multi-region returns one dict per region with a shared account ID + Given a list of 3 CloudInjectorConfig with cloud_account_id="123456789012" and regions "us-east-1", "eu-west-1", "ap-southeast-1" + When I call build_execution_signatures with the config list + Then the return value is a list of exactly 3 dicts + And the dict at position contains cloud_region equal to "" + And the dict at position contains cloud_account_id equal to "123456789012" + + Examples: + | index | region | + | 0 | us-east-1 | + | 1 | eu-west-1 | + | 2 | ap-southeast-1 | + + Scenario: Builder classifies a mixed list of targets into typed configs + Given a raw mixed target list "10.0.0.1", "2001:db8::1", "target.example.com" + When I build network configs from the raw list + Then the builder returns 3 NetworkInjectorConfig + And the config at position 0 has target_ipv4 equal to "10.0.0.1" + And the config at position 1 has target_ipv6 equal to "2001:db8::1" + And the config at position 2 has target_hostname equal to "target.example.com" diff --git a/libs/xtm-oaev-sdk/test/signatures/shim/__init__.py b/libs/xtm-oaev-sdk/test/signatures/shim/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/signatures/constraints/signature_manager_post_execution_constraints.feature b/libs/xtm-oaev-sdk/test/signatures/shim/constraints/signature_manager_post_execution_constraints.feature similarity index 100% rename from test/signatures/constraints/signature_manager_post_execution_constraints.feature rename to libs/xtm-oaev-sdk/test/signatures/shim/constraints/signature_manager_post_execution_constraints.feature diff --git a/test/signatures/constraints/signature_manager_pre_execution_constraints.feature b/libs/xtm-oaev-sdk/test/signatures/shim/constraints/signature_manager_pre_execution_constraints.feature similarity index 100% rename from test/signatures/constraints/signature_manager_pre_execution_constraints.feature rename to libs/xtm-oaev-sdk/test/signatures/shim/constraints/signature_manager_pre_execution_constraints.feature diff --git a/test/signatures/features/signature_manager_post_execution.feature b/libs/xtm-oaev-sdk/test/signatures/shim/features/signature_manager_post_execution.feature similarity index 100% rename from test/signatures/features/signature_manager_post_execution.feature rename to libs/xtm-oaev-sdk/test/signatures/shim/features/signature_manager_post_execution.feature diff --git a/test/signatures/features/signature_manager_pre_execution.feature b/libs/xtm-oaev-sdk/test/signatures/shim/features/signature_manager_pre_execution.feature similarity index 100% rename from test/signatures/features/signature_manager_pre_execution.feature rename to libs/xtm-oaev-sdk/test/signatures/shim/features/signature_manager_pre_execution.feature diff --git a/test/signatures/test_ai_marker.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_ai_marker.py similarity index 100% rename from test/signatures/test_ai_marker.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_ai_marker.py diff --git a/test/signatures/test_ai_signature_types.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_ai_signature_types.py similarity index 100% rename from test/signatures/test_ai_signature_types.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_ai_signature_types.py diff --git a/libs/xtm-oaev-sdk/test/signatures/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_shim_contract.py new file mode 100644 index 00000000..fe4e380e --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/shim/test_shim_contract.py @@ -0,0 +1,149 @@ +"""Contract of the pyoaev signatures shims. + +Legacy imports must keep working and warn. They return the *identical* +xtm_oaev_sdk object, except for the renamed ``ExpectationType`` (now +``SignatureExpectationType``) and ``SignatureManager``, which resolves to a +subclass keeping the client-based constructor. +""" + +import subprocess +import sys +import warnings +from unittest.mock import MagicMock + +import pytest +import xtm_oaev_sdk as sdk + +_CONSUMER = "legacy_consumer.py" +_RENAMED = {"ExpectationType": "SignatureExpectationType"} + +_PACKAGE = [ + "CloudInjectorConfig", + "ExpectationSignatureGroup", + "ExpectationType", + "ExtraSignatureData", + "InjectorConfig", + "InjectExecutionActions", + "MatchTypes", + "NetworkInjectorConfig", + "SignatureCallbackPayload", + "SignaturePayload", + "SignatureTarget", + "SignatureTypes", + "SignatureValue", + "TargetSignatures", + "build_network_configs", +] +_MODELS = [ + "SignatureValue", + "ExpectationSignatureGroup", + "SignatureTarget", + "TargetSignatures", + "SignaturePayload", + "SignatureCallbackPayload", + "ExecutionSignature", + "ToolErrorInfo", + "ToolTimeoutInfo", + "ToolOutput", + "NetworkInjectorConfig", + "CloudInjectorConfig", + "InjectorConfig", + "build_network_configs", + "ExecutionDetails", + "ExtraSignatureData", + "SignatureOutputStructure", + "ExecutionStatus", + "ExpectationType", + "InjectExecutionActions", +] +_TYPES = [ + "ExpectationType", + "ExecutionStatus", + "InjectExecutionActions", + "MatchTypes", + "SignatureTypes", +] + +LEGACY_NAMES = ( + [("pyoaev", name) for name in _PACKAGE] + + [("pyoaev.signatures", name) for name in _PACKAGE] + + [("pyoaev.signatures.models", name) for name in _MODELS] + + [("pyoaev.signatures.types", name) for name in _TYPES] + + [("pyoaev.signatures.signature_match", "SignatureMatch")] + + [("pyoaev.signatures.signature_type", "SignatureType")] + + [("pyoaev.signatures.ai_marker", "build_marker")] + + [("pyoaev.signatures.signature_manager", "ExecutionDetails")] + + [("pyoaev.signatures.signature_manager", "OpenAEVError")] +) +LEGACY_MANAGERS = [ + "pyoaev", + "pyoaev.signatures", + "pyoaev.signatures.signature_manager", +] + + +def _given_fresh_interpreter_script(script: str) -> list[str]: + return [sys.executable, "-W", "error::DeprecationWarning", "-c", script] + + +def _when_run(command: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(command, capture_output=True, text=True) + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + source = f"from {module} import {name} as _legacy" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(compile(source, _CONSUMER, "exec"), namespace) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + return namespace["_legacy"], deprecations + + +def _then_succeeds(result: subprocess.CompletedProcess) -> None: + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize(("module", "name"), LEGACY_NAMES) +def test_legacy_import_warns_and_returns_the_sdk_object(module, name) -> None: + """It warns towards xtm_oaev_sdk and returns the identical object.""" + sdk_name = _RENAMED.get(name, name) + legacy, deprecations = _when_import_legacy_name(module, name) + assert legacy is getattr(sdk, sdk_name) + assert deprecations + assert f"from xtm_oaev_sdk import {sdk_name}" in str(deprecations[0].message) + # attributed to the consumer's line, so default warning filters show it + assert any(w.filename == _CONSUMER for w in deprecations) + + +@pytest.mark.parametrize("module", LEGACY_MANAGERS) +def test_legacy_signature_manager_keeps_the_client_constructor(module) -> None: + """It warns and sends through ``client.signature`` like before.""" + legacy_manager, deprecations = _when_import_legacy_name(module, "SignatureManager") + assert deprecations + assert issubclass(legacy_manager, sdk.SignatureManager) + + client = MagicMock() + manager = legacy_manager(client, logger=None) + + assert manager.client is client + assert manager.transport is client.signature + + +def test_importing_pyoaev_modules_is_silent() -> None: + """It imports pyoaev and its signatures modules without any warning.""" + script = ( + "import pyoaev, pyoaev.apis, pyoaev.helpers, pyoaev.daemons\n" + "import pyoaev.signatures, pyoaev.signatures.models\n" + "import pyoaev.signatures.signature_manager, pyoaev.signatures.types\n" + "import pyoaev.signatures.signature_match\n" + "import pyoaev.signatures.signature_type, pyoaev.signatures.ai_marker\n" + ) + result = _when_run(_given_fresh_interpreter_script(script)) + _then_succeeds(result) + + +def test_unknown_legacy_name_still_raises() -> None: + """It does not swallow genuine misses.""" + with pytest.raises(ImportError): + exec("from pyoaev.signatures import DoesNotExist", {}) diff --git a/test/signatures/test_signature_manager_post_execution.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_signature_manager_post_execution.py similarity index 100% rename from test/signatures/test_signature_manager_post_execution.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_signature_manager_post_execution.py diff --git a/test/signatures/test_signature_manager_pre_execution.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_signature_manager_pre_execution.py similarity index 100% rename from test/signatures/test_signature_manager_pre_execution.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_signature_manager_pre_execution.py diff --git a/test/signatures/test_signature_match.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_signature_match.py similarity index 100% rename from test/signatures/test_signature_match.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_signature_match.py diff --git a/test/signatures/test_signature_type.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_signature_type.py similarity index 100% rename from test/signatures/test_signature_type.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_signature_type.py diff --git a/libs/xtm-oaev-sdk/test/signatures/test_ai_signature_types.py b/libs/xtm-oaev-sdk/test/signatures/test_ai_signature_types.py new file mode 100644 index 00000000..c64a9298 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_ai_signature_types.py @@ -0,0 +1,28 @@ +import unittest + +from xtm_oaev_sdk import MatchTypes, SignatureType, SignatureTypes + + +class TestAiSignatureTypes(unittest.TestCase): + def test_ai_signature_type_values(self): + self.assertEqual( + SignatureTypes.SIG_TYPE_AI_REQUEST_MARKER.value, "ai_request_marker" + ) + self.assertEqual( + SignatureTypes.SIG_TYPE_AI_TARGET_ENDPOINT.value, "ai_target_endpoint" + ) + + def test_ai_request_marker_usable_in_signature_type(self): + signature_type = SignatureType( + label=SignatureTypes.SIG_TYPE_AI_REQUEST_MARKER, + match_type=MatchTypes.MATCH_TYPE_SIMPLE, + ) + + struct = signature_type.make_struct_for_matching(data="oaevdeadbeef") + + self.assertEqual(struct.get("type"), MatchTypes.MATCH_TYPE_SIMPLE.value) + self.assertEqual(struct.get("data"), "oaevdeadbeef") + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py new file mode 100644 index 00000000..e1e7d42e --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py @@ -0,0 +1,110 @@ +import ipaddress +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from pytest_bdd import given, parsers, scenario, then, when +from xtm_oaev_sdk import SignatureManager + + +@scenario( + "features/signature_manager_ip_resolution.feature", + "resolve_container_ip returns a valid IPv4 in each supported execution environment", +) +def test_resolve_container_ip_returns_valid_ipv4(): + pass + + +@scenario( + "constraints/signature_manager_ip_resolution_constraints.feature", + "resolve_container_ip returns unknown and emits exactly one warning when all strategies fail", +) +def test_resolve_container_ip_returns_unknown_when_all_strategies_fail(): + pass + + +@pytest.fixture +def context(): + logger = MagicMock() + return { + "logger": logger, + "signature_manager": SignatureManager(transport=MagicMock(), logger=logger), + } + + +@given(parsers.parse('a SignatureManager running in a "{environment}" environment')) +def signature_manager_environment(context, monkeypatch, environment): + if environment == "Docker": + monkeypatch.setattr( + "socket.gethostbyname", + lambda _: "172.17.0.2", + ) + monkeypatch.setattr( + "subprocess.run", + lambda *args, **kwargs: SimpleNamespace( + returncode=0, + stdout="172.17.0.2\n", + ), + ) + elif environment == "Kubernetes": + monkeypatch.setattr( + "socket.gethostbyname", + lambda _: (_ for _ in ()).throw(OSError("socket fail")), + ) + monkeypatch.setattr( + "subprocess.run", + lambda *args, **kwargs: SimpleNamespace( + returncode=0, + stdout="10.244.0.8\n", + ), + ) + else: + monkeypatch.setattr( + "socket.gethostbyname", + lambda _: "192.0.2.20", + ) + + +@given("all IP resolution strategies are mocked to fail") +def all_ip_resolution_strategies_fail(context, monkeypatch): + monkeypatch.setattr( + "socket.gethostbyname", + lambda _: (_ for _ in ()).throw(OSError("socket fail")), + ) + monkeypatch.setattr( + "subprocess.run", + lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("command fail")), + ) + + +@when("I call resolve_container_ip") +def call_resolve_container_ip(context): + context["resolve_exception"] = None + try: + context["resolved_ip"] = context["signature_manager"].resolve_container_ip() + except Exception as exc: + context["resolve_exception"] = exc + + +@then("the returned value is a non-empty valid IPv4 address string") +def assert_returned_value_valid_ipv4(context): + assert context["resolve_exception"] is None + resolved_ip = context["resolved_ip"] + assert isinstance(resolved_ip, str) + assert resolved_ip.strip() != "" + assert ipaddress.ip_address(resolved_ip).version == 4 + + +@then(parsers.parse('the returned value is the string "{expected_value}"')) +def assert_returned_value_matches(context, expected_value): + assert context["resolved_ip"] == expected_value + + +@then(parsers.parse("exactly {count:d} WARNING log message is emitted")) +def assert_warning_count(context, count): + assert context["logger"].warning.call_count == count + + +@then("no exception propagates from resolve_container_ip") +def assert_no_exception_from_resolve_container_ip(context): + assert context["resolve_exception"] is None diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_post_execution.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_post_execution.py new file mode 100644 index 00000000..32c1f905 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_post_execution.py @@ -0,0 +1,223 @@ +from datetime import datetime +from unittest.mock import MagicMock + +import pytest +from pytest_bdd import given, parsers, scenario, then, when +from xtm_oaev_sdk import ( + ExecutionDetails, + ExecutionSignature, + ExecutionStatus, + SignatureManager, +) + + +@scenario( + "features/signature_manager_post_execution.feature", + "Successful execution updates end_time and execution_status in execution signatures and execution details", +) +def test_successful_execution_merges_post_execution_fields(): + pass + + +@scenario( + "constraints/signature_manager_post_execution_constraints.feature", + "Tool crash sets execution_status to failed and uses crash timestamp as end_time", +) +def test_tool_crash_sets_failed_status_and_crash_timestamp_end_time(): + pass + + +@scenario( + "constraints/signature_manager_post_execution_constraints.feature", + "Timeout sets execution_status to timeout and includes available partial results", +) +def test_timeout_sets_timeout_status_and_includes_partial_results(): + pass + + +@scenario( + "constraints/signature_manager_post_execution_constraints.feature", + "Timeout with no partial results still sets execution_status to timeout", +) +def test_timeout_without_partial_results_still_sets_timeout_status(): + pass + + +@pytest.fixture +def context(): + return {} + + +def _parse_iso8601_utc(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +@given( + "a SignatureManager initialised with constructor SignatureManager(transport, logger)" +) +def signature_manager(context): + context["signature_manager"] = SignatureManager( + transport=MagicMock(), logger=MagicMock() + ) + + +@given( + "a execution_signatures object containing:", + target_fixture="execution_signatures", +) +def execution_signatures(): + return ExecutionSignature( + source_ipv4="172.17.0.2", + target_ipv4="10.0.0.1", + target_hostname="host-a.internal", + start_time="2024-06-26T06:00:00Z", + ) + + +@given( + "a execution_details object containing:", + target_fixture="execution_details", +) +def execution_details(): + return ExecutionDetails( + execution_status=ExecutionStatus.INFO, + start_time=datetime.strptime("2024-06-26T06:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + + +@given( + "a tool_output indicating successful completion with no errors and no timeout", + target_fixture="tool_output", +) +def successful_tool_output(): + return {"status": "EXECUTED"} + + +@given( + 'a tool_output containing error_info with exit_code=1 and crash_timestamp="2024-06-26T06:05:00Z"', + target_fixture="tool_output", +) +def crashed_tool_output(): + return { + "error_info": { + "exit_code": 1, + "crash_timestamp": "2024-06-26T06:05:00Z", + } + } + + +@given( + 'a tool_output containing timeout_info with partial_results=["result-A", "result-B"]', + target_fixture="tool_output", +) +def timeout_tool_output_with_partial_results(): + return {"timeout_info": {"partial_results": ["result-A", "result-B"]}} + + +@given( + "a tool_output containing timeout_info with no partial results available", + target_fixture="tool_output", +) +def timeout_tool_output_with_no_partial_results(): + return {"timeout_info": {"partial_results": []}} + + +@when( + "I call post_execution_updates with the execution_details, execution_signatures and tool_output" +) +def post_execution_update( + context, execution_details, execution_signatures, tool_output +): + context["signature_manager"].post_execution_updates( + execution_details, execution_signatures, tool_output + ) + context["execution_details_result"] = execution_details + context["execution_signatures_result"] = execution_signatures + + +@then("the execution signature model contains every previous parameter unchanged") +def execution_signatures_unchanged(context, execution_signatures): + basic_exec_sig = execution_signatures + exec_sig_result = context["execution_signatures_result"] + + assert exec_sig_result.source_ipv4 == basic_exec_sig.source_ipv4 + assert exec_sig_result.source_ipv6 == basic_exec_sig.source_ipv6 + assert exec_sig_result.target_ipv4 == basic_exec_sig.target_ipv4 + assert exec_sig_result.target_ipv6 == basic_exec_sig.target_ipv6 + assert exec_sig_result.target_hostname == basic_exec_sig.target_hostname + assert exec_sig_result.cloud_provider == basic_exec_sig.cloud_provider + assert exec_sig_result.cloud_account_id == basic_exec_sig.cloud_account_id + assert exec_sig_result.cloud_region == basic_exec_sig.cloud_region + assert exec_sig_result.target_service == basic_exec_sig.target_service + + +@then( + "the end_time parameter in the execution signature model is a UTC ISO 8601 string" +) +def result_contains_iso8601_end_time(context): + end_time = context["execution_signatures_result"].end_time + assert isinstance(end_time, str) + _parse_iso8601_utc(end_time) + + +@then( + parsers.parse( + 'this end_time is chronologically greater than or equal to start_time "{start_time}"' + ) +) +def end_time_at_or_after_start_time(context, start_time): + end_time_dt = _parse_iso8601_utc(context["execution_signatures_result"].end_time) + start_time_dt = _parse_iso8601_utc(start_time) + assert end_time_dt >= start_time_dt + + +@then(parsers.parse('end_time equals "{expected_end_time}"')) +def end_time_equals(context, expected_end_time): + assert context["execution_signatures_result"].end_time == expected_end_time + + +@then( + 'the returned dict contains the partial results ["result-A", "result-B"] from timeout_info' +) +def contains_timeout_partial_results(context): + assert context["execution_signatures_result"].partial_results == [ + "result-A", + "result-B", + ] + + +@then("the execution details model contain every previous parameter pair unchanged") +def execution_details_unchanged(context, execution_details): + basic_exec_details = execution_details + exec_details_result = context["execution_details_result"] + + assert exec_details_result.start_time == basic_exec_details.start_time + assert exec_details_result.execution_message == basic_exec_details.execution_message + + +@then("the end_time parameter in the execution details model is a datetime object") +def result_contains_datetime_end_time(context): + end_time = context["execution_details_result"].end_time + assert isinstance(end_time, datetime) + + +@then( + parsers.parse( + 'the execution_status parameter in the execution details model is equal to "{status}"' + ) +) +@then(parsers.parse('execution_status equals "{status}"')) +def execution_status_equals(context, status): + assert context["execution_details_result"].execution_status == ExecutionStatus( + status + ) + + +@then( + parsers.parse( + 'the execution_action parameter in the execution details model is equal to "{action}"' + ) +) +@then(parsers.parse('execution_action equals "{action}"')) +def execution_action_equals(context, action): + assert context["execution_details_result"].execution_action == action diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py new file mode 100644 index 00000000..20743c33 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py @@ -0,0 +1,466 @@ +import ipaddress +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock, patch + +import pytest +from pytest_bdd import given, parsers, scenario, then, when +from xtm_oaev_sdk import ( + CloudInjectorConfig, + ExecutionSignature, + NetworkInjectorConfig, + SignatureManager, + build_network_configs, +) + +# -------------------------------------------------- +# SCENARIOS +# -------------------------------------------------- + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Network category returns required IP and timing fields and no cloud or query fields", +) +def test_network_category_required_fields(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Network hostname target returns hostname and no target_ipv4", +) +def test_network_hostname_target(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Cloud category returns required cloud identity fields and no IP fields", +) +def test_cloud_category_required_fields(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Network multi-target returns one dict per target with a shared source IP", +) +def test_network_multi_target(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "All network multi-target dicts share the same source_ipv4", +) +def test_network_multi_target_shared_source(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Cloud multi-region returns one dict per region with a shared account ID", +) +def test_cloud_multi_region(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Builder classifies a mixed list of targets into typed configs", +) +def test_builder_classifies_mixed_targets(): + pass + + +@scenario( + "constraints/signature_manager_pre_execution_constraints.feature", + "start_time is generated at method-call time not at class instantiation time", +) +def test_start_time_generated_at_call_time(): + pass + + +# -------------------------------------------------- +# FIXTURE CONTEXT +# -------------------------------------------------- + + +@pytest.fixture +def context(): + return {} + + +# -------------------------------------------------- +# HELPERS +# -------------------------------------------------- + + +def parse_utc_iso8601(value): + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed.astimezone(timezone.utc) + + +# -------------------------------------------------- +# GIVEN +# -------------------------------------------------- + + +@given( + "a SignatureManager initialised with constructor SignatureManager(transport, logger)", + target_fixture="signature_manager", +) +def signature_manager(): + return SignatureManager(transport=MagicMock(), logger=None) + + +@given( + parsers.parse('a NetworkInjectorConfig with target_ipv4="{target_ipv4}"'), + target_fixture="config", +) +def network_config_ipv4(target_ipv4): + return NetworkInjectorConfig(target_ipv4=target_ipv4) + + +@given( + parsers.parse('a NetworkInjectorConfig with target_hostname="{target_hostname}"'), + target_fixture="config", +) +def network_config_hostname(target_hostname): + return NetworkInjectorConfig(target_hostname=target_hostname) + + +@given( + parsers.parse( + 'a CloudInjectorConfig with cloud_provider="{cloud_provider}", ' + 'cloud_account_id="{cloud_account_id}", cloud_region="{cloud_region}", ' + 'and target_service="{target_service}"' + ), + target_fixture="config", +) +def cloud_config_single( + cloud_provider, + cloud_account_id, + cloud_region, + target_service, +): + return CloudInjectorConfig( + cloud_provider=cloud_provider, + cloud_account_id=cloud_account_id, + cloud_region=cloud_region, + target_service=target_service, + ) + + +@given( + parsers.parse( + "a list of 3 NetworkInjectorConfig with target_ipv4 " + '"{ip_1}", "{ip_2}", "{ip_3}"' + ), + target_fixture="config", +) +def network_config_list(ip_1, ip_2, ip_3): + return [ + NetworkInjectorConfig(target_ipv4=ip_1), + NetworkInjectorConfig(target_ipv4=ip_2), + NetworkInjectorConfig(target_ipv4=ip_3), + ] + + +@given( + "a list of 3 NetworkInjectorConfig built from default IPv4 targets", + target_fixture="config", +) +def network_config_list_default(): + return [ + NetworkInjectorConfig(target_ipv4="10.0.0.1"), + NetworkInjectorConfig(target_ipv4="10.0.0.2"), + NetworkInjectorConfig(target_ipv4="10.0.0.3"), + ] + + +@given( + parsers.parse( + 'a list of 3 CloudInjectorConfig with cloud_account_id="{cloud_account_id}" ' + 'and regions "{region_1}", "{region_2}", "{region_3}"' + ), + target_fixture="config", +) +def cloud_config_list(cloud_account_id, region_1, region_2, region_3): + return [ + CloudInjectorConfig( + cloud_provider="aws", + cloud_account_id=cloud_account_id, + cloud_region=region, + ) + for region in (region_1, region_2, region_3) + ] + + +@given( + parsers.parse('a raw mixed target list "{value_1}", "{value_2}", "{value_3}"'), + target_fixture="raw_targets", +) +def raw_mixed_target_list(value_1, value_2, value_3): + return [value_1, value_2, value_3] + + +@given( + "the running container has a resolvable IPv4 address", + target_fixture="source_ipv4", +) +def resolvable_container_ipv4(request): + patcher = patch( + "xtm_oaev_sdk.SignatureManager.resolve_container_ip", + return_value="172.17.0.2", + ) + patcher.start() + request.addfinalizer(patcher.stop) + return "172.17.0.2" + + +@given( + parsers.parse( + 'the running container has a resolvable IPv4 address "{source_ipv4}"' + ), + target_fixture="source_ipv4", +) +def resolvable_container_ipv4_explicit(request, source_ipv4): + patcher = patch( + "xtm_oaev_sdk.SignatureManager.resolve_container_ip", + return_value=source_ipv4, + ) + patcher.start() + request.addfinalizer(patcher.stop) + return source_ipv4 + + +@given( + "a SignatureManager that was instantiated at timestamp T0", + target_fixture="signature_manager", +) +def signature_manager_at_t0(context): + t0 = datetime(2024, 6, 26, 6, 6, 1, tzinfo=timezone.utc) + context["t0"] = t0 + manager = SignatureManager(transport=MagicMock(), logger=None) + manager._test_t0 = t0 + return manager + + +@given( + "5 seconds elapse after instantiation", + target_fixture="t1", +) +def elapsed_5_seconds(context): + t1 = context["t0"] + timedelta(seconds=5) + context["t1"] = t1 + return t1 + + +# -------------------------------------------------- +# WHEN +# -------------------------------------------------- + + +@when( + "I call build_execution_signatures with the config", + target_fixture="result", +) +def call_build_with_config(signature_manager, config): + return signature_manager.build_execution_signatures(config=config) + + +@when( + "I call build_execution_signatures with the config list", + target_fixture="result", +) +def call_build_with_config_list(signature_manager, config): + return signature_manager.build_execution_signatures(config=config) + + +@when( + "I call build_execution_signatures with the config at timestamp T1", + target_fixture="result", +) +def call_build_at_t1(signature_manager, config, t1): + with patch.object(signature_manager, "_utcnow", return_value=t1): + return signature_manager.build_execution_signatures(config=config) + + +@when( + "I build network configs from the raw list", + target_fixture="built_configs", +) +def build_configs_from_raw(raw_targets): + return build_network_configs(raw_targets) + + +# -------------------------------------------------- +# THEN +# -------------------------------------------------- + + +@then("the returned dict contains source_ipv4 as a non-empty valid IPv4 address string") +def source_ipv4_is_valid(result): + source_ipv4 = result.source_ipv4 + assert source_ipv4 + ipaddress.IPv4Address(source_ipv4) + + +@then("the returned dict contains start_time as a UTC ISO 8601 string") +def start_time_is_utc_iso8601(result): + start_time = result.start_time + parsed = parse_utc_iso8601(start_time) + assert parsed.tzinfo is not None + + +@then(parsers.parse('the returned dict contains target_ipv4 equal to "{value}"')) +def returned_dict_target_ipv4(result, value): + assert result.target_ipv4 == value + + +@then(parsers.parse('the returned dict contains target_hostname equal to "{value}"')) +def returned_dict_target_hostname(result, value): + assert result.target_hostname == value + + +@then(parsers.parse('the returned dict contains cloud_provider equal to "{value}"')) +def returned_dict_cloud_provider(result, value): + assert result.cloud_provider == value + + +@then(parsers.parse('the returned dict contains cloud_account_id equal to "{value}"')) +def returned_dict_cloud_account_id(result, value): + assert result.cloud_account_id == value + + +@then(parsers.parse('the returned dict contains cloud_region equal to "{value}"')) +def returned_dict_cloud_region(result, value): + assert result.cloud_region == value + + +@then(parsers.parse('the returned dict contains target_service equal to "{value}"')) +def returned_dict_target_service(result, value): + assert result.target_service == value + + +@then(parsers.parse('the returned dict contains query equal to "{value}"')) +def returned_dict_query(result, value): + assert result.query == value + + +@then(parsers.parse("the returned dict does not contain {field}")) +def returned_dict_does_not_contain_field(result, field): + assert field not in result + + +@then("the return value is a list of exactly 3 dicts") +def return_value_is_list_of_three_dicts(result): + assert isinstance(result, list) + assert len(result) == 3 + assert all(isinstance(item, ExecutionSignature) for item in result) + + +@then(parsers.parse("the return value is a list of {count:d} dicts")) +def return_value_is_list_of_n_dicts(result, count): + assert isinstance(result, list) + assert len(result) == count + assert all(isinstance(item, ExecutionSignature) for item in result) + + +@then( + parsers.parse( + 'the dict at position {index:d} contains target_ipv4 equal to "{target_ip}"' + ) +) +def list_dict_contains_target_ipv4_at_position(result, index, target_ip): + assert result[index].target_ipv4 == target_ip + + +@then( + parsers.parse( + 'the dict at position {index:d} contains source_ipv4 equal to "{source_ipv4}"' + ) +) +def list_dict_contains_source_ipv4_at_position( + result, + index, + source_ipv4, +): + assert result[index].source_ipv4 == source_ipv4 + + +@then( + parsers.parse( + 'the dict at position {index:d} contains cloud_region equal to "{region}"' + ) +) +def list_dict_contains_cloud_region_at_position(result, index, region): + assert result[index].cloud_region == region + + +@then( + parsers.parse( + 'the dict at position {index:d} contains cloud_account_id equal to "{account_id}"' + ) +) +def list_dict_contains_cloud_account_id_at_position(result, index, account_id): + assert result[index].cloud_account_id == account_id + + +@then("all 3 dicts contain the same source_ipv4 value") +def all_dicts_share_same_source_ipv4(result): + assert isinstance(result, list) + assert len(result) == 3 + source_values = {item.source_ipv4 for item in result} + assert len(source_values) == 1 + ipaddress.IPv4Address(next(iter(source_values))) + + +@then("the start_time in the returned dict equals T1 within 1 second tolerance") +def start_time_equals_t1_with_tolerance(result, t1): + start_time = parse_utc_iso8601(result.start_time) + delta_seconds = abs((start_time - t1).total_seconds()) + assert delta_seconds <= 1 + + +@then("start_time does not equal T0") +def start_time_not_equal_t0(result, signature_manager): + start_time = parse_utc_iso8601(result.start_time) + assert start_time != signature_manager._test_t0 + + +@then(parsers.parse("the builder returns {count:d} NetworkInjectorConfig")) +def builder_returns_n_configs(built_configs, count): + assert isinstance(built_configs, list) + assert len(built_configs) == count + assert all(isinstance(c, NetworkInjectorConfig) for c in built_configs) + + +@then( + parsers.parse('the config at position {index:d} has target_ipv4 equal to "{value}"') +) +def config_has_target_ipv4(built_configs, index, value): + assert built_configs[index].target_ipv4 == value + assert built_configs[index].target_ipv6 is None + assert built_configs[index].target_hostname is None + + +@then( + parsers.parse('the config at position {index:d} has target_ipv6 equal to "{value}"') +) +def config_has_target_ipv6(built_configs, index, value): + assert built_configs[index].target_ipv6 == value + assert built_configs[index].target_ipv4 is None + assert built_configs[index].target_hostname is None + + +@then( + parsers.parse( + 'the config at position {index:d} has target_hostname equal to "{value}"' + ) +) +def config_has_target_hostname(built_configs, index, value): + assert built_configs[index].target_hostname == value + assert built_configs[index].target_ipv4 is None + assert built_configs[index].target_ipv6 is None diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_match.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_match.py new file mode 100644 index 00000000..ce5f2234 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_match.py @@ -0,0 +1,23 @@ +import unittest + +from xtm_oaev_sdk import MatchTypes, OpenAEVError, SignatureMatch + + +class TestSignatureMatch(unittest.TestCase): + def test_non_simple_match_with_non_null_score_throws(self): + score = None + self.assertRaises( + OpenAEVError, SignatureMatch, MatchTypes.MATCH_TYPE_FUZZY, score + ) + + def test_simple_match_with_null_score_does_not_throw(self): + score = None + SignatureMatch(match_type=MatchTypes.MATCH_TYPE_SIMPLE, match_score=score) + + def test_fuzzy_match_with_0_score_does_not_throw(self): + score = 0 + SignatureMatch(match_type=MatchTypes.MATCH_TYPE_FUZZY, match_score=score) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_type.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_type.py new file mode 100644 index 00000000..1747b813 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_type.py @@ -0,0 +1,56 @@ +import unittest + +from xtm_oaev_sdk import MatchTypes, SignatureType, SignatureTypes + + +class TestSignatureType(unittest.TestCase): + def test_make_struct_create_expected_struct_for_simple_sig_type(self): + simple_signature_type_label = SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS + simple_signature_type = SignatureType( + label=simple_signature_type_label, match_type=MatchTypes.MATCH_TYPE_SIMPLE + ) + + data = "just a simple string" + simple_struct = simple_signature_type.make_struct_for_matching(data=data) + + self.assertEqual(simple_struct.get("type"), MatchTypes.MATCH_TYPE_SIMPLE.value) + self.assertEqual(simple_struct.get("data"), data) + self.assertFalse("score" in simple_struct.keys()) + + def test_make_struct_create_expected_struct_for_fuzzy_sig_type(self): + fuzzy_signature_type_label = SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS + fuzzy_signature_type_score = 50 + fuzzy_signature_type = SignatureType( + label=fuzzy_signature_type_label, + match_type=MatchTypes.MATCH_TYPE_FUZZY, + match_score=fuzzy_signature_type_score, + ) + + data = "just another simple string" + simple_struct = fuzzy_signature_type.make_struct_for_matching(data=data) + + self.assertEqual(simple_struct.get("type"), MatchTypes.MATCH_TYPE_FUZZY.value) + self.assertEqual(simple_struct.get("data"), data) + self.assertEqual(simple_struct.get("score"), fuzzy_signature_type_score) + + def test_make_struct_create_expected_struct_for_fuzzy_sig_type_when_score_is_0( + self, + ): + fuzzy_signature_type_label = SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS + fuzzy_signature_type_score = 0 + fuzzy_signature_type = SignatureType( + label=fuzzy_signature_type_label, + match_type=MatchTypes.MATCH_TYPE_FUZZY, + match_score=fuzzy_signature_type_score, + ) + + data = "just another simple string" + simple_struct = fuzzy_signature_type.make_struct_for_matching(data=data) + + self.assertEqual(simple_struct.get("type"), MatchTypes.MATCH_TYPE_FUZZY.value) + self.assertEqual(simple_struct.get("data"), data) + self.assertEqual(simple_struct.get("score"), fuzzy_signature_type_score) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/test_boundaries.py b/libs/xtm-oaev-sdk/test/test_boundaries.py new file mode 100644 index 00000000..b77990f0 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/test_boundaries.py @@ -0,0 +1,64 @@ +"""SDK dependency and source-boundary assertions.""" + +import pathlib +import re +import subprocess +import sys + +import pytest + +_SDK_ROOT = pathlib.Path(__file__).resolve().parents[1] / "xtm_oaev_sdk" + + +def _when_searching_sdk_sources(pattern: str) -> list[tuple[str, str]]: + regex = re.compile(pattern) + return [ + (str(path.relative_to(_SDK_ROOT)), line) + for path in sorted(_SDK_ROOT.rglob("*.py")) + for line in path.read_text(encoding="utf-8").splitlines() + if regex.search(line) + ] + + +@pytest.mark.parametrize("package", ["pyoaev", "requests", "pika", "thefuzz"]) +def test_sdk_sources_do_not_import(package: str) -> None: + """It never imports the legacy client nor its transport dependencies.""" + pattern = rf"^\s*(?:from|import)\s+{package}(?:\.|\b)" + assert _when_searching_sdk_sources(pattern) == [] + + +@pytest.mark.parametrize( + "artifact", + [ + "class ListenQueue", + "class OpenAEVConfigHelper", + "class OpenAEVCollectorHelper", + "class OpenAEVInjectorHelper", + "class OpenAEVDetectionHelper", + "def create_mq_ssl_context", + ], +) +def test_sdk_sources_do_not_define_client_helpers(artifact: str) -> None: + """It leaves transport and daemon helpers to the client.""" + assert _when_searching_sdk_sources(rf"^\s*{artifact}\b") == [] + + +def _sdk_modules() -> list[str]: + package_root = _SDK_ROOT.parent + return sorted( + ".".join(path.relative_to(package_root).with_suffix("").parts).removesuffix( + ".__init__" + ) + for path in _SDK_ROOT.rglob("*.py") + ) + + +@pytest.mark.parametrize("module", _sdk_modules()) +def test_every_sdk_module_imports_first_in_a_fresh_interpreter(module: str) -> None: + """It has no import cycle hidden by a lucky import order.""" + result = subprocess.run( + [sys.executable, "-W", "error", "-c", f"import {module}"], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/libs/xtm-oaev-sdk/test/test_deprecation.py b/libs/xtm-oaev-sdk/test/test_deprecation.py new file mode 100644 index 00000000..52da97ca --- /dev/null +++ b/libs/xtm-oaev-sdk/test/test_deprecation.py @@ -0,0 +1,150 @@ +"""Tests for the reusable PEP 562 module-deprecation factory. + +Covers the exact pattern the legacy modules hand-roll: a ``{old: (msg, target)}`` +mapping driving a module ``__getattr__`` that warns and returns the *same* +object, preserves identity (no wrapper), and still raises on genuine misses. +""" + +import sys +import types +import warnings + +import pytest +from xtm_oaev_sdk import make_deprecated_getattr + + +# --- sentinel: a real object the deprecated name should alias, exactly --- +class _Sentinel: + pass + + +_TARGET = _Sentinel() + + +def _install_module(name, mapping, *, module_name=None): + """Create a bare module that uses the factory as its PEP 562 ``__getattr__``.""" + mod = types.ModuleType(name) + mod.__getattr__ = make_deprecated_getattr(mapping, module_name=module_name or name) + sys.modules[name] = mod + return mod + + +def _capture(fn): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = fn() + return result, caught + + +# --- 1. the core contract: warn once per access + return the exact object --- +def test_deprecated_access_warns_and_preserves_identity() -> None: + mod = _install_module("tdep_ident", {"Old": ("Use 'new' instead.", _TARGET)}) + try: + result, caught = _capture(lambda: mod.Old) + finally: + sys.modules.pop("tdep_ident", None) + + assert result is _TARGET, "the shim must return the exact object, not a wrapper" + assert len(caught) == 1 + assert isinstance(caught[0].message, DeprecationWarning) + assert str(caught[0].message) == "Use 'new' instead." + + +# --- 2. repeated access keeps returning the same object --- +def test_repeated_access_is_stable() -> None: + mod = _install_module("tdep_repeat", {"Old": ("msg", _TARGET)}) + try: + first = mod.Old + second = mod.Old + finally: + sys.modules.pop("tdep_repeat", None) + assert first is _TARGET + assert second is _TARGET + + +# --- 3. a real `from import ` triggers the shim --- +def test_from_import_triggers_deprecation() -> None: + _install_module("tdep_fromimp", {"Old": ("moved", _TARGET)}) + try: + namespace: dict = {} + result, caught = _capture( + lambda: exec("from tdep_fromimp import Old as _got", namespace) + ) + assert namespace["_got"] is _TARGET + assert any(isinstance(w.message, DeprecationWarning) for w in caught) + finally: + sys.modules.pop("tdep_fromimp", None) + + +# --- 4. a name that is NOT deprecated must still raise AttributeError --- +def test_unknown_name_raises_attribution_error() -> None: + mod = _install_module( + "tdep_miss", {"Old": ("moved", _TARGET)}, module_name="tdep_miss" + ) + try: + with pytest.raises(AttributeError) as exc: + _capture(lambda: mod.DoesNotExist) + assert "tdep_miss" in str(exc.value) + assert "DoesNotExist" in str(exc.value) + finally: + sys.modules.pop("tdep_miss", None) + + +# --- 5. a `from` import of an unknown name surfaces as ImportError --- +def test_from_import_of_missing_name_raises_import_error() -> None: + _install_module("tdep_miss2", {"Old": ("moved", _TARGET)}) + try: + with pytest.raises(ImportError): + _capture(lambda: exec("from tdep_miss2 import Nope", {})) + finally: + sys.modules.pop("tdep_miss2", None) + + +# --- 6. the factory validates its input up front (import-time failure) --- +def test_malformed_mapping_raises_value_error() -> None: + with pytest.raises(ValueError): + make_deprecated_getattr({123: ("msg", _TARGET)}) # non-string key + with pytest.raises(ValueError): + make_deprecated_getattr({"Old": "just-a-string"}) # not a 2-tuple + with pytest.raises(ValueError): + make_deprecated_getattr({"Old": ("only-a-message",)}) # 1-tuple + + +# --- 7. input mapping is copied; later mutation does not change the shim --- +def test_mapping_is_copied_not_referenced() -> None: + mapping = {"Old": ("msg", _TARGET)} + mod = _install_module("tdep_copy", mapping) + try: + mapping["Old"] = ("CHANGED", object()) # mutate the caller's mapping + result, _ = _capture(lambda: mod.Old) + assert result is _TARGET + assert str(_[0].message) == "msg" + finally: + sys.modules.pop("tdep_copy", None) + + +# --- 8. the primitive is a public, documented export --- +def test_public_export_and_all() -> None: + import xtm_oaev_sdk as sdk + + assert sdk.make_deprecated_getattr is make_deprecated_getattr + assert "make_deprecated_getattr" in sdk.__all__ + + +# --- 9. one factory per module; modules do not interfere with each other --- +def test_two_modules_independent() -> None: + a = _install_module("tdep_a", {"X": ("a-msg", "A")}) + b = _install_module("tdep_b", {"Y": ("b-msg", "B")}) + try: + result, _ = _capture(lambda: (a.X, b.Y)) + assert result == ("A", "B") + # cross-module name does not exist on the other + with pytest.raises(AttributeError): + a.Y + finally: + sys.modules.pop("tdep_a", None) + sys.modules.pop("tdep_b", None) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py new file mode 100644 index 00000000..3c399d68 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -0,0 +1,41 @@ +"""xtm-oaev-sdk — OpenAEV SDK core (DDD + Light Hex Architecture). + +Architecture: + _core/common/ Common implementations (HTTP errors, logging, JSON + encoding, dict tools, certificates) + _core/configuration/ Configuration feature implementation + _core/contracts/ OpenAEV scenarios feature implementation + _core/daemons/ Daemon runtime (base daemon, heartbeat) + _core/deprecation/ Deprecation shims (PEP 562 __getattr__) + _core/signatures/ Signature feature implementation + seam/common/ Shared errors, enums, value objects, protocols, ports + seam/configuration/ Configuration contracts and ports (closed, versioned) + seam/contracts/ Scenario contracts and ports (closed, versioned) + seam/daemons/ Daemon runtime ports (closed, versioned) + seam/signatures/ Signature contracts, inbound I/O models and ports (closed) + seam/port_io/ Port I/O wire schema (versioned, seam-owned) + seam/deprecation/ Deprecation port (closed, versioned) + public/common Common adapters + public/configuration Configuration adapters + public/contracts Scenario contract adapters + public/daemons Daemon runtime adapters (API client injected) + public/signatures Signature manager and helpers adapters + public/deprecation PEP 562 deprecation primitive adapter + public/ User-facing re-exports + +Port/adapter layering, capability by capability (see ``seam``): + +- ``seam`` holds the contracts — data models, enums, errors and the ``Protocol`` + ports — and imports nothing from ``_core`` nor ``public``; +- ``_core`` implements the seam ports and never imports ``public``; +- ``public`` exposes adapters that speak seam types and delegate to ``_core``, + plus the seam contracts themselves — never a core object. + +""" + +__version__ = "0.1.0" + +from xtm_oaev_sdk.public import * # noqa: F401, F403 +from xtm_oaev_sdk.public import __all__ as _public_all + +__all__ = list(_public_all) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/__init__.py new file mode 100644 index 00000000..8cff0516 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/__init__.py @@ -0,0 +1,6 @@ +"""xtm_oaev_sdk._core: private implementation boundary. + +Do NOT import from this module directly. Use xtm_oaev_sdk public surface. +""" + +__all__: list[str] = [] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py new file mode 100644 index 00000000..fcc3faa4 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py @@ -0,0 +1,11 @@ +"""Common core: implementations of the common seam ports.""" + +from .certificates import Certificates +from .encoding import JsonEncoding +from .logging_setup import LoggingSetup + +__all__ = [ + "Certificates", + "JsonEncoding", + "LoggingSetup", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/certificates.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/certificates.py new file mode 100644 index 00000000..73b7d8fb --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/certificates.py @@ -0,0 +1,73 @@ +"""Certificates: load files or in-memory PEM data into SSL contexts. + +Thanks to https://bugs.python.org/issue16487 it is not possible to load +in-memory PEM data into an SSL context: it is written to a temporary file first. +""" + +import os +import ssl +import tempfile + +__all__ = ["Certificates"] + + +class Certificates: + """Implements ``CertificatesPort``.""" + + # As cert must be written in files to be loaded in ssl context + # Creates a temporary file in the most secure manner possible + def data_to_temp_file(self, data: str) -> str: + # The file is readable and writable only by the creating user ID. + # If the operating system uses permission bits to indicate whether a + # file is executable, the file is executable by no one. The file + # descriptor is not inherited by children of this process. + file_descriptor, file_path = tempfile.mkstemp() + with os.fdopen(file_descriptor, "w") as open_file: + open_file.write(data) + open_file.close() + return file_path + + def is_memory_certificate(self, certificate: str) -> bool: + return certificate.startswith("-----BEGIN") + + def ssl_cert_chain( + self, + ssl_context: ssl.SSLContext, + cert_data: str | None, + key_data: str | None, + passphrase: str | None, + ) -> None: + if cert_data is None: + return + + cert_file_path = None + key_file_path = None + + # Cert loading + if self.is_memory_certificate(cert_data): + cert_file_path = self.data_to_temp_file(cert_data) + cert = cert_file_path if cert_file_path is not None else cert_data + + # Key loading + if key_data is not None and self.is_memory_certificate(key_data): + key_file_path = self.data_to_temp_file(key_data) + key = key_file_path if key_file_path is not None else key_data + + # Load cert + ssl_context.load_cert_chain(cert, key, passphrase) + # Remove temp files + if cert_file_path is not None: + os.unlink(cert_file_path) + if key_file_path is not None: + os.unlink(key_file_path) + + def ssl_verify_locations( + self, ssl_context: ssl.SSLContext, certdata: str | None + ) -> None: + if certdata is None: + return + + if self.is_memory_certificate(certdata): + ssl_context.load_verify_locations(cadata=certdata) + else: + ssl_context.load_verify_locations(cafile=certdata) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/encoding.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/encoding.py new file mode 100644 index 00000000..b49315dd --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/encoding.py @@ -0,0 +1,18 @@ +"""JSON encoding of the objects the standard encoder does not support.""" + +import dataclasses +from typing import Any + +__all__ = ["JsonEncoding"] + + +class JsonEncoding: + """Implements ``JsonEncodingPort``.""" + + def encode_default(self, o: Any) -> Any: + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + # same error as json.JSONEncoder.default + raise TypeError( + f"Object of type {o.__class__.__name__} is not JSON serializable" + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/logging_setup.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/logging_setup.py new file mode 100644 index 00000000..cbe3298c --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/logging_setup.py @@ -0,0 +1,49 @@ +"""Logging setup: root logger configuration and structured record fields.""" + +import datetime +import logging +from typing import Any + +__all__ = ["LoggingSetup"] + + +class LoggingSetup: + """Implements ``LoggingPort``.""" + + def setup_logging_config( + self, + level: int, + json_logging: bool, + formatter_class: type[logging.Formatter], + ) -> None: + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("pika").setLevel(logging.ERROR) + if json_logging: + log_handler = logging.StreamHandler() + log_handler.setLevel(level) + formatter = formatter_class("%(timestamp)s %(level)s %(name)s %(message)s") + log_handler.setFormatter(formatter) + logging.basicConfig(handlers=[log_handler], level=level, force=True) + else: + logging.basicConfig(level=level) + + def get_logger(self, name: str) -> logging.Logger: + return logging.getLogger(name) + + def prepare_meta(self, meta: Any) -> dict[str, Any] | None: + return None if meta is None else {"attributes": meta} + + def set_library_level(self, lib: str, log_level: int) -> None: + logging.getLogger(lib).setLevel(log_level) + + def complete_record( + self, log_record: dict[str, Any], record: logging.LogRecord + ) -> None: + if not log_record.get("timestamp"): + # This doesn't use record.created, so it is slightly off + now = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + log_record["timestamp"] = now + if log_record.get("level"): + log_record["level"] = log_record["level"].upper() + else: + log_record["level"] = record.levelname diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py new file mode 100644 index 00000000..e892e1a8 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py @@ -0,0 +1,15 @@ +"""Configuration core: implementations of the configuration seam ports.""" + +from .resolver import ConfigurationResolver +from .schema import ConnectorSchemaTransformer +from .settings_sources import SettingsSourceSelector +from .sources import ConfigurationSources +from .truthiness import Truthiness + +__all__ = [ + "ConfigurationResolver", + "ConfigurationSources", + "ConnectorSchemaTransformer", + "SettingsSourceSelector", + "Truthiness", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/resolver.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/resolver.py new file mode 100644 index 00000000..26fa5aa5 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/resolver.py @@ -0,0 +1,103 @@ +"""Configuration resolution: override, then env var, then file, then default.""" + +import os.path +from typing import Any + +import yaml +from pydantic_settings import BaseSettings +from xtm_oaev_sdk._core.configuration.sources import ConfigurationSources +from xtm_oaev_sdk._core.configuration.truthiness import Truthiness +from xtm_oaev_sdk.seam.configuration import ( + CONFIGURATION_TYPES, + ConfigurationHint, + ConfigurationSourcesPort, + TruthinessPort, +) + +__all__ = ["ConfigurationResolver"] + + +class ConfigurationResolver: + """Implements ``ConfigurationResolverPort``.""" + + def __init__( + self, + sources: ConfigurationSourcesPort | None = None, + truthiness: TruthinessPort | None = None, + ) -> None: + self._sources = sources or ConfigurationSources() + self._truthiness = truthiness or Truthiness() + + def build_hints( + self, config_hints: dict[str, dict[str, Any] | str] + ) -> dict[str, ConfigurationHint]: + return { + key: ( + ConfigurationHint(**value) + if isinstance(value, dict) + else ConfigurationHint(default=value) + ) + for key, value in config_hints.items() + } + + def load_values( + self, config_values: dict[str, Any] | None, config_file_path: str + ) -> dict[str, Any]: + file_contents = ( + yaml.load(open(config_file_path), Loader=yaml.FullLoader) + if os.path.isfile(config_file_path) + else {} + ) + return (config_values or {}) | file_contents + + def resolve( + self, hint: ConfigurationHint, values: dict[str, Any] + ) -> CONFIGURATION_TYPES: + return self._process_value_to_type( + (self._dig_sources(hint, values) if hint.data is None else hint.data), + hint.is_number, + ) + + def override( + self, + hints: dict[str, ConfigurationHint], + config_key: str, + value: CONFIGURATION_TYPES, + ) -> None: + if config_key not in hints: + hints[config_key] = ConfigurationHint(data=value) + else: + hints[config_key].data = value + + def generate_schema( + self, base_model: BaseSettings | None, schema_generator: type + ) -> dict[str, Any]: + return base_model.model_json_schema( # type: ignore[union-attr] + by_alias=False, + schema_generator=schema_generator, + mode="validation", + ) + + def _process_value_to_type( + self, value: CONFIGURATION_TYPES, is_number_hint: bool | None + ) -> CONFIGURATION_TYPES: + if value is None: + return value + if isinstance(value, int) or is_number_hint: + return int(value) + if isinstance(value, str): + if self._truthiness.is_truthy(value): + return True + if self._truthiness.is_falsy(value): + return False + if len(value) == 0: + return None + return value + + def _dig_sources( + self, hint: ConfigurationHint, values: dict[str, Any] + ) -> CONFIGURATION_TYPES: + result = self._sources.from_environment( + hint.env + ) or self._sources.from_dictionary(hint.file_path, values) + return result or hint.default diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/schema.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/schema.py new file mode 100644 index 00000000..f9754739 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/schema.py @@ -0,0 +1,108 @@ +"""Connector manifest schema: flat, dereferenced, filtered JSON schema. + +Adapted from the OpenCTI connectors configuration schema generator: +https://github.com/OpenCTI-Platform/connectors/blob/5c8cf1235f62f5651c9c08d0b67f1bd182662c8a/shared/tools/composer/generate_connectors_config_schemas/generate_connector_config_json_schema.py.sample +""" + +from copy import deepcopy +from typing import Any + +__all__ = ["ConnectorSchemaTransformer"] + +# attributes filtered from the connector configuration before generating the manifest +_FILTERED_ATTRIBUTES = [ + # connector id is generated + "CONNECTOR_ID", +] + + +class ConnectorSchemaTransformer: + """Implements ``ConnectorSchemaPort``.""" + + def dereference_schema(self, schema_with_refs: dict[str, Any]) -> dict[str, Any]: + def _resolve(schema: Any, root: dict[str, Any]) -> Any: + if isinstance(schema, dict): + if "$ref" in schema: + ref_path = schema["$ref"] + if ref_path.startswith("#/$defs/"): + def_name = ref_path.split("/")[-1] + # Deep copy to avoid mutating $defs + resolved = deepcopy(root["$defs"][def_name]) + return _resolve(resolved, root) + else: + raise ValueError(f"Unsupported ref format: {ref_path}") + else: + return { + schema_key: _resolve(schema_value, root) + for schema_key, schema_value in schema.items() + } + elif isinstance(schema, list): + return [_resolve(item, root) for item in schema] + else: + return schema + + return _resolve(deepcopy(schema_with_refs), schema_with_refs) + + def flatten_config_loader_schema( + self, root_schema: dict[str, Any] + ) -> dict[str, Any]: + flat_json_schema: dict[str, Any] = { + "$schema": root_schema["$schema"], + "$id": root_schema["$id"], + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": root_schema.get("additionalProperties", True), + } + + for ( + config_loader_namespace_name, + config_loader_namespace_schema, + ) in root_schema["properties"].items(): + config_schema = config_loader_namespace_schema.get("properties", {}) + required_config_vars = config_loader_namespace_schema.get("required", []) + + for config_var_name, config_var_schema in config_schema.items(): + property_name = ( + f"{config_loader_namespace_name.upper()}_{config_var_name.upper()}" + ) + + config_var_schema.pop("title", None) + + flat_json_schema["properties"][property_name] = config_var_schema + + if config_var_name in required_config_vars: + flat_json_schema["required"].append(property_name) + + return flat_json_schema + + def filter_schema(self, schema: dict[str, Any]) -> dict[str, Any]: + for filtered_attribute in _FILTERED_ATTRIBUTES: + if filtered_attribute in schema["properties"]: + del schema["properties"][filtered_attribute] + schema.update( + { + "required": [ + item + for item in schema["required"] + if item != filtered_attribute + ] + } + ) + + return schema + + def finalize(self, json_schema: dict[str, Any], dialect: str) -> dict[str, Any]: + json_schema["$schema"] = dialect + json_schema["$id"] = "config.schema.json" + dereferenced_schema = self.dereference_schema(json_schema) + flattened_schema = self.flatten_config_loader_schema(dereferenced_schema) + return self.filter_schema(flattened_schema) + + def collapse_nullable(self, inner_json_schema: dict[str, Any]) -> dict[str, Any]: + null_schema = {"type": "null"} + + if inner_json_schema == null_schema: + return null_schema + else: + return inner_json_schema diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_sources.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_sources.py new file mode 100644 index 00000000..57a994c2 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_sources.py @@ -0,0 +1,44 @@ +"""Settings sources: environment variables first, then the YAML or .env file.""" + +import os +from pathlib import Path + +from pydantic_settings import ( + BaseSettings, + DotEnvSettingsSource, + PydanticBaseSettingsSource, + YamlConfigSettingsSource, +) + +__all__ = ["SettingsSourceSelector"] + + +class SettingsSourceSelector: + """Implements ``SettingsSourcesPort``.""" + + def select_sources( + self, + settings_cls: type[BaseSettings], + env_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + _main_path = os.curdir + + settings_cls.model_config["env_file"] = f"{_main_path}/../.env" + + if not settings_cls.model_config["yaml_file"]: + if Path(f"{_main_path}/config.yml").is_file(): + settings_cls.model_config["yaml_file"] = f"{_main_path}/config.yml" + if Path(f"{_main_path}/../config.yml").is_file(): + settings_cls.model_config["yaml_file"] = f"{_main_path}/../config.yml" + + if Path(settings_cls.model_config["yaml_file"] or "").is_file(): # type: ignore + return ( + env_settings, + YamlConfigSettingsSource(settings_cls), + ) + if Path(settings_cls.model_config["env_file"] or "").is_file(): # type: ignore + return ( + env_settings, + DotEnvSettingsSource(settings_cls), + ) + return (env_settings,) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py new file mode 100644 index 00000000..9f48aac4 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py @@ -0,0 +1,31 @@ +"""Configuration sources: environment variables and nested dictionaries.""" + +import os +from typing import Any + +__all__ = ["ConfigurationSources"] + + +class ConfigurationSources: + """Implements ``ConfigurationSourcesPort``.""" + + def from_environment(self, env_var: str | None) -> str | None: + return os.getenv(env_var) if env_var else None + + # this is quite hacky + # it only strictly handles two levels of keys in a dict + def from_dictionary( + self, config_key_path: list[str] | None, source_dict: dict[str, Any] + ) -> str | None: + if config_key_path is None: + return None + + assert ( + isinstance(config_key_path, list) + and len(config_key_path) == 2 + and all([len(path_part) > 0 for path_part in config_key_path]) + ) + + return source_dict.get(config_key_path[0], {config_key_path[1]: None}).get( + config_key_path[1] + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/truthiness.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/truthiness.py new file mode 100644 index 00000000..8eeb16b2 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/truthiness.py @@ -0,0 +1,13 @@ +"""Truthiness: the strings signalling a boolean configuration value.""" + +__all__ = ["Truthiness"] + + +class Truthiness: + """Implements ``TruthinessPort``.""" + + def is_truthy(self, value: str) -> bool: + return value.lower() in ["yes", "true"] + + def is_falsy(self, value: str) -> bool: + return value.lower() in ["no", "false"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py new file mode 100644 index 00000000..72299378 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py @@ -0,0 +1,13 @@ +"""Contracts core: implementations of the contracts seam ports.""" + +from .assembly import ContractAssembly +from .credentials import CredentialReferences +from .serializer import ContractSerializer +from .variables import BuiltinContractVariables + +__all__ = [ + "BuiltinContractVariables", + "ContractAssembly", + "ContractSerializer", + "CredentialReferences", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/assembly.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/assembly.py new file mode 100644 index 00000000..ef0092b7 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/assembly.py @@ -0,0 +1,27 @@ +"""Contract assembly: what a builder does to the fields it collects.""" + +from typing import Any + +from xtm_oaev_sdk.seam.contracts import ContractElement + +__all__ = ["ContractAssembly"] + + +class ContractAssembly: + """Implements ``ContractAssemblyPort``.""" + + def concat(self, existing: list[Any], extra: list[Any]) -> list[Any]: + return existing + extra + + def set_mandatory( + self, element: ContractElement, mandatory: bool + ) -> ContractElement: + element.mandatory = mandatory + return element + + def require_group(self, elements: list[ContractElement]) -> list[ContractElement]: + keys: list[str] = list(map(lambda iterable: iterable.key, elements)) + for element in elements: + element.mandatory = True + element.mandatoryGroups = keys + return elements diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py new file mode 100644 index 00000000..5ec25383 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py @@ -0,0 +1,37 @@ +"""Credential-reference fields: provider-to-credential-type mapping. + +Injectors call ``build_single_referenced_credential_element`` instead of +re-implementing the provider-to-credential-type mapping in each project. This +keeps contract generation consistent across the Python ecosystem and aligns the +serialized contract payload with the OpenAEV platform's expected values. +""" + +from typing import Optional + +from xtm_oaev_sdk.seam.contracts import ContractReferencedCredential, CredentialType + +__all__ = ["CredentialReferences"] + +_PROVIDER_TO_CREDENTIAL_TYPE = { + "aws": CredentialType.CLOUD_AWS, + "eks": CredentialType.CLOUD_AWS, + "azure": CredentialType.CLOUD_AZURE, + "gcp": CredentialType.CLOUD_GCP, +} + + +class CredentialReferences: + """Implements ``ContractCredentialsPort``.""" + + def build_single_referenced_credential_element( + self, provider_name: str + ) -> ContractReferencedCredential: + return ContractReferencedCredential( + credential_reference_type=self._resolve_credential_type(provider_name), + multiple=False, + ) + + @staticmethod + def _resolve_credential_type(provider_name: str) -> Optional[CredentialType]: + normalized_provider = provider_name.casefold() + return _PROVIDER_TO_CREDENTIAL_TYPE.get(normalized_provider) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py new file mode 100644 index 00000000..99a583a9 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py @@ -0,0 +1,85 @@ +"""Contract serialization into platform API payloads.""" + +import json +from typing import Any + +from xtm_oaev_sdk._core.common import JsonEncoding +from xtm_oaev_sdk.seam.contracts import ContractData + +__all__ = ["ContractSerializer"] + + +class ContractSerializer: + """Implements ``ContractSerializerPort``.""" + + def __init__(self) -> None: + self._encoding = JsonEncoding() + + def to_add_input(self, contract: ContractData, source_id: str) -> dict[str, Any]: + """Serialize a contract for the platform creation API. + + Args: + source_id: The injector ID that owns this contract. + + Returns: + Dict payload for the contract creation endpoint. + """ + return { + "contract_id": contract.contract_id, + "external_contract_id": contract.external_id, + "injector_id": source_id, + "contract_manual": contract.manual, + "contract_labels": contract.label, + "contract_attack_patterns_external_ids": contract.contract_attack_patterns_external_ids, + "contract_vulnerability_external_ids": contract.contract_vulnerability_external_ids, + "contract_content": json.dumps( + contract, default=self._encoding.encode_default + ), + "is_atomic_testing": contract.is_atomic_testing, + "contract_platforms": contract.platforms, + "contract_domains": contract.domains, + } + + def to_update_input(self, contract: ContractData) -> dict[str, Any]: + """Serialize a contract for the platform update API. + + Returns: + Dict payload for the contract update endpoint. + """ + return { + "contract_manual": contract.manual, + "contract_labels": contract.label, + "contract_attack_patterns_external_ids": contract.contract_attack_patterns_external_ids, + "contract_vulnerability_external_ids": contract.contract_vulnerability_external_ids, + "contract_content": json.dumps( + contract, default=self._encoding.encode_default + ), + "is_atomic_testing": contract.is_atomic_testing, + "contract_platforms": contract.platforms, + "contract_domains": contract.domains, + } + + def prepare(self, contracts: list[ContractData]) -> list[dict[str, Any]]: + """Serialize a list of contracts for bulk platform registration. + + Args: + contracts: List of Contract instances to serialize. + + Returns: + List of dicts suitable for the bulk contract registration API. + """ + return list( + map( + lambda c: { + "contract_id": c.contract_id, + "contract_labels": c.label, + "contract_attack_patterns_external_ids": c.contract_attack_patterns_external_ids, + "contract_content": json.dumps( + c, default=self._encoding.encode_default + ), + "contract_platforms": c.platforms, + "contract_domains": c.domains, + }, + contracts, + ) + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variables.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variables.py new file mode 100644 index 00000000..bd859cf1 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variables.py @@ -0,0 +1,138 @@ +"""Built-in contract template variables (user, exercise, team, platform URIs).""" + +from xtm_oaev_sdk.seam.contracts import ( + ContractCardinality, + ContractVariable, + VariableType, +) + +__all__ = ["BuiltinContractVariables"] + +USER = "user" +EXERCISE = "exercise" +TEAMS = "teams" +COMCHECK = "comcheck" +PLAYER_URI = "player_uri" +CHALLENGES_URI = "challenges_uri" +SCOREBOARD_URI = "scoreboard_uri" +LESSONS_URI = "lessons_uri" + + +class BuiltinContractVariables: + """Implements ``ContractVariablesPort``.""" + + def user_variable(self) -> ContractVariable: + """Create the user template variable with standard children.""" + return ContractVariable( + key=USER, + label="User that will receive the injection", + type=VariableType.String, + cardinality=ContractCardinality.One, + children=[ + ContractVariable( + key=USER + ".id", + label="Id of the user in the platform", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=USER + ".email", + label="Email of the user", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=USER + ".firstname", + label="Firstname of the user", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=USER + ".lastname", + label="Lastname of the user", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=USER + ".lang", + label="Lang of the user", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ], + ) + + def exercise_variable(self) -> ContractVariable: + """Create the exercise template variable with standard children.""" + return ContractVariable( + key=EXERCISE, + label="Exercise of the current injection", + type=VariableType.Object, + cardinality=ContractCardinality.One, + children=[ + ContractVariable( + key=EXERCISE + ".id", + label="Id of the exercise in the platform", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=EXERCISE + ".name", + label="Name of the exercise", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=EXERCISE + ".description", + label="Description of the exercise", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ], + ) + + def team_variable(self) -> ContractVariable: + """Create the teams template variable.""" + return ContractVariable( + key=TEAMS, + label="List of team name for the injection", + type=VariableType.String, + cardinality=ContractCardinality.Multiple, + ) + + def uri_variables(self) -> list[ContractVariable]: + """Create platform URI template variables.""" + return [ + ContractVariable( + key=PLAYER_URI, + label="Player interface platform link", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=CHALLENGES_URI, + label="Challenges interface platform link", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=SCOREBOARD_URI, + label="Scoreboard interface platform link", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=LESSONS_URI, + label="Lessons learned interface platform link", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ] + + def default_variables(self) -> list[ContractVariable]: + """Create the variables every contract starts with.""" + return [ + self.user_variable(), + self.exercise_variable(), + self.team_variable(), + ] + self.uri_variables() diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py new file mode 100644 index 00000000..6a5b1fc7 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py @@ -0,0 +1,6 @@ +"""Daemons core: implementations of the daemons seam ports.""" + +from .heartbeat import Heartbeat +from .runtime import DaemonRuntime + +__all__ = ["DaemonRuntime", "Heartbeat"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/heartbeat.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/heartbeat.py new file mode 100644 index 00000000..5e8a57c6 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/heartbeat.py @@ -0,0 +1,34 @@ +"""Heartbeat: keep a daemon registered on the platform.""" + +import threading +from typing import Any + +__all__ = ["Heartbeat"] + + +class Heartbeat: + """Implements ``HeartbeatPort``.""" + + def tenant_of(self, api: Any) -> Any: + return getattr(api, "tenant_id", None) + + def tenant_meta(self, tenant_id: Any) -> dict[str, Any]: + return {"tenant_id": str(tenant_id) if tenant_id else None} + + def ping( + self, + api: Any, + config: Any, + ping_type: str, + logger: Any, + exit_event: threading.Event, + ) -> None: + while not exit_event.is_set(): + try: + if ping_type == "injector": + api.injector.create(config, False) + else: + api.collector.create(config, False) + except Exception as err: # pylint: disable=broad-except + logger.error("Error pinging the API: " + str(err)) + exit_event.wait(40) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/runtime.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/runtime.py new file mode 100644 index 00000000..80968cb0 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/runtime.py @@ -0,0 +1,80 @@ +"""Daemon runtime: lifecycle shared by collector and injector daemons.""" + +import argparse +from inspect import signature +from types import FunctionType +from typing import Any, Callable + +from xtm_oaev_sdk.seam.common import BaseClient, OpenAEVError +from xtm_oaev_sdk.seam.configuration import ConfigurationProtocol + +__all__ = ["DaemonRuntime"] + + +class DaemonRuntime: + """Implements ``DaemonRuntimePort``.""" + + # compatibility layer: in order for older configs to still work, search for legacy names + def resolve_log_level(self, configuration: ConfigurationProtocol) -> str: + return ( + configuration.get("log_level") + or configuration.get("collector_log_level") + or configuration.get("injector_log_level") + or "info" + ) + + def resolve_name(self, configuration: ConfigurationProtocol) -> str: + return ( + configuration.get("name") + or configuration.get("collector_name") + or configuration.get("injector_name") + or "daemon" + ) + + def resolve_id(self, configuration: ConfigurationProtocol) -> Any: + return ( + configuration.get("id") + or configuration.get("collector_id") + or configuration.get("injector_id") + ) + + def invoke_callback( + self, callback: Callable[..., Any] | None, daemon: Any, logger: Any + ) -> None: + try: + # this is some black magic to allow injecting the collector daemon instance + # into an arbitrary callback that has a specific argument name + # this allow for avoiding subclassing the CollectorDaemon class just to provide the callback + # Example: + # + # def standalone_func(collector): + # collector.api.call_openaev() + # + # CollectorDaemon(config=, standalone_func).start() + if ( + isinstance(callback, FunctionType) + and "collector" in signature(callback).parameters + ): + callback(collector=daemon) + else: + callback() + except Exception as err: # pylint: disable=broad-except + logger.error(f"Error calling: {err}") + + def dump_config_schema_requested(self) -> bool: + parser = argparse.ArgumentParser(description="parse daemon options") + parser.add_argument("--dump-config-schema", action="store_true") + args = parser.parse_args() + return bool(args.dump_config_schema) + + def dump_config_schema(self, configuration: ConfigurationProtocol) -> None: + print(configuration.schema()) + + def ensure_callback(self, callback: Callable[..., Any] | None) -> None: + if callback is None: + raise OpenAEVError("This daemon has no configured callback.") + + def default_api_client(self) -> BaseClient: + raise OpenAEVError( + "No api_client was provided to the daemon; inject an OpenAEV API client." + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/__init__.py new file mode 100644 index 00000000..8d899be1 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/__init__.py @@ -0,0 +1,5 @@ +"""Deprecation core: implementation of the deprecation seam port.""" + +from .shims import DeprecationShims + +__all__ = ["DeprecationShims"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/shims.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/shims.py new file mode 100644 index 00000000..39eaaa6d --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/shims.py @@ -0,0 +1,41 @@ +"""Deprecation shims: PEP 562 ``__getattr__`` serving retired names.""" + +import warnings +from collections.abc import Mapping +from typing import Any, Callable + +__all__ = ["DeprecationShims"] + + +class DeprecationShims: + """Implements ``DeprecationPort``.""" + + def resolve_mapping( + self, mapping: Mapping[str, tuple[str, Any]] + ) -> dict[str, tuple[str, Any]]: + resolved: dict[str, tuple[str, Any]] = {} + for key, value in mapping.items(): + if not isinstance(key, str) or not key: + raise ValueError( + f"deprecated-name mapping keys must be non-empty strings, got {key!r}" + ) + if not isinstance(value, tuple) or len(value) != 2: + raise ValueError( + f"mapping entry for {key!r} must be (message, target), got {value!r}" + ) + resolved[key] = (value[0], value[1]) + return resolved + + def build_getattr( + self, resolved: dict[str, tuple[str, Any]], module_name: str | None + ) -> Callable[[str], Any]: + def __getattr__(name: str) -> Any: + entry = resolved.get(name) + if entry is None: + where = module_name if module_name is not None else "this module" + raise AttributeError(f"{where} has no attribute {name!r}") from None + message, target = entry + warnings.warn(message, DeprecationWarning, stacklevel=2) + return target + + return __getattr__ diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py new file mode 100644 index 00000000..86b58811 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py @@ -0,0 +1,13 @@ +"""Signatures core: implementations of the signatures seam ports.""" + +from .addresses import ContainerAddressResolver +from .engine import SignatureEngine +from .marker import MarkerBuilder +from .targets import NetworkTargets + +__all__ = [ + "ContainerAddressResolver", + "MarkerBuilder", + "NetworkTargets", + "SignatureEngine", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/addresses.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/addresses.py new file mode 100644 index 00000000..bc8d8b80 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/addresses.py @@ -0,0 +1,60 @@ +"""Container addresses: the source IPs of network execution signatures.""" + +import logging +import os +import socket +import subprocess + +__all__ = ["ContainerAddressResolver"] + + +class ContainerAddressResolver: + """Implements ``ContainerAddressPort``.""" + + def resolve_ipv4(self, logger: logging.Logger) -> str | None: + env_ip = os.environ.get("CONTAINER_IP") + if env_ip: + return env_ip + + try: + ip = socket.gethostbyname(socket.gethostname()) + if ip and ip != "127.0.0.1": + return ip + except (socket.gaierror, OSError): + pass + + try: + result = subprocess.run( + ["hostname", "-i"], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + if result.returncode == 0: + ip = result.stdout.strip().split()[0] + if ip and ip != "127.0.0.1": + return ip + except (OSError, RuntimeError, subprocess.TimeoutExpired): + pass + + logger.warning("Could not resolve container IP; returning 'unknown'") + return None + + def resolve_ipv6(self) -> str | None: + try: + infos = socket.getaddrinfo( + socket.gethostname(), None, socket.AF_INET6, socket.SOCK_STREAM + ) + for info in infos: + addr = info[4][0] + if ( + isinstance(addr, str) + and addr + and not addr.startswith("::1") + and not addr.startswith("fe80") + ): + return addr + except (socket.gaierror, OSError): + pass + return None diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/engine.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/engine.py new file mode 100644 index 00000000..5c3dca3b --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/engine.py @@ -0,0 +1,132 @@ +"""Signature engine: compile pre-execution signatures, merge tool output, +package the wire payload.""" + +from datetime import datetime, timezone +from typing import Any, Callable + +from pydantic import ValidationError +from xtm_oaev_sdk.seam.common import OpenAEVError +from xtm_oaev_sdk.seam.port_io import ( + ExecutionDetails, + ExpectationSignatureGroup, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, +) +from xtm_oaev_sdk.seam.signatures import ( + CloudInjectorConfig, + ExecutionSignature, + ExtraSignatureData, + InjectorConfig, + NetworkInjectorConfig, + ToolOutput, +) + +__all__ = ["SignatureEngine"] + + +class SignatureEngine: + """Implements ``SignatureEnginePort``.""" + + def now(self) -> datetime: + return datetime.now(timezone.utc) + + def validate_configs(self, configs: list[InjectorConfig]) -> None: + if not configs: + raise ValueError("build_execution_signatures requires at least one config") + first_type = type(configs[0]) + for c in configs: + if not isinstance(c, first_type): + raise ValueError( + "build_execution_signatures does not mix injector config types; " + f"got {sorted({type(c).__name__ for c in configs})}" + ) + + def format_start_time(self, now: datetime) -> str: + return now.strftime("%Y-%m-%dT%H:%M:%SZ") + + def source_context( + self, + config: InjectorConfig, + source_ipv4: Callable[[], str], + source_ipv6: Callable[[], str | None], + ) -> dict[str, Any]: + if isinstance(config, NetworkInjectorConfig): + return {"source_ipv4": source_ipv4(), "source_ipv6": source_ipv6()} + if isinstance(config, CloudInjectorConfig): + return {} + raise TypeError(f"unsupported injector config type: {type(config).__name__}") + + def build_signature( + self, config: InjectorConfig, start_time: str, source_context: dict[str, Any] + ) -> ExecutionSignature: + base: dict[str, Any] = {"start_time": start_time} + base.update(source_context) + base.update(config.model_dump(exclude_none=True)) + return ExecutionSignature(**base) + + def apply_tool_output( + self, + execution_details: ExecutionDetails, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + tool_output: dict[str, Any], + now: Callable[[], datetime], + ) -> None: + try: + tool_output = ToolOutput.model_validate(tool_output or {}) + except ValidationError as exc: + raise OpenAEVError(error_message=f"Invalid tool_output: {exc}") from exc + + now = now() + execution_details.post_execution_update(tool_output, now) + if isinstance(execution_signatures, list): + for exec_sig in execution_signatures: + exec_sig.post_execution_update(tool_output, now) + else: + execution_signatures.post_execution_update(tool_output, now) + + def build_payload( + self, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + targets_meta: Any | list[Any], + expectation_types: list[str], + extra_signatures: ExtraSignatureData, + ) -> dict[str, Any]: + if not isinstance(execution_signatures, list): + execution_signatures = [execution_signatures] + if isinstance(targets_meta, dict): + targets_meta = [targets_meta] * len(execution_signatures) + + targets = [] + for signature, target in zip(execution_signatures, targets_meta): + signature_values = [] + for expectation_type in expectation_types: + signature_data = signature.model_dump(exclude_none=True) + signature_data.update(extra_signatures.get_extra(expectation_type)) + values = [ + SignatureValue(signature_type=key, signature_value=value) + for key, value in signature_data.items() + ] + signature_values.append( + ExpectationSignatureGroup( + expectation_type=expectation_type, values=values + ) + ) + targets.append( + TargetSignatures( + signature_target=SignatureTarget( + agent=target.agent_id, + asset=target.asset_id, + asset_group=target.asset_group_id, + ), + signature_values=signature_values, + ) + ) + return SignaturePayload(targets=targets).model_dump() + + def output_structure(self, signatures: dict[str, Any]) -> SignatureOutputStructure: + return SignatureOutputStructure( + expectation_signatures=SignaturePayload(**signatures) + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/marker.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/marker.py new file mode 100644 index 00000000..f68ed684 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/marker.py @@ -0,0 +1,14 @@ +"""AI marker: deterministic per-inject canary shared by the AI red-team injector +and the AI defense collectors.""" + +import hashlib + +__all__ = ["MarkerBuilder"] + + +class MarkerBuilder: + """Implements ``MarkerPort``.""" + + def build_marker(self, inject_id: str, agent_id: str = "") -> str: + seed = f"{inject_id}:{agent_id}".encode("utf-8") + return "oaev" + hashlib.sha256(seed).hexdigest()[:16] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/targets.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/targets.py new file mode 100644 index 00000000..e46f2850 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/targets.py @@ -0,0 +1,40 @@ +"""Network targets: raw injector targets into network injector configs.""" + +import ipaddress +from typing import Any + +from xtm_oaev_sdk.seam.signatures import NetworkInjectorConfig + +__all__ = ["NetworkTargets"] + + +class NetworkTargets: + """Implements ``NetworkTargetsPort``.""" + + def build_network_configs( + self, + targets: list[str | dict[str, Any] | NetworkInjectorConfig], + ) -> list[NetworkInjectorConfig]: + configs: list[NetworkInjectorConfig] = [] + for target in targets: + if isinstance(target, NetworkInjectorConfig): + configs.append(target) + elif isinstance(target, dict): + configs.append(NetworkInjectorConfig(**target)) + elif isinstance(target, str): + configs.append(self._classify_network_target(target)) + else: + raise TypeError( + f"unsupported network target type: {type(target).__name__}" + ) + return configs + + def _classify_network_target(self, value: str) -> NetworkInjectorConfig: + """Decide whether ``value`` is an IPv4, IPv6, or hostname and wrap it.""" + try: + addr = ipaddress.ip_address(value) + except ValueError: + return NetworkInjectorConfig(target_hostname=value) + if isinstance(addr, ipaddress.IPv4Address): + return NetworkInjectorConfig(target_ipv4=value) + return NetworkInjectorConfig(target_ipv6=value) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py new file mode 100644 index 00000000..416842aa --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -0,0 +1,270 @@ +"""Public API surface for xtm-oaev-sdk.""" + +__version__ = "0.1.0" + +# --- Common (shared) --- +from xtm_oaev_sdk.public.common import ( + AppLogger, + AppLoggerProtocol, + AssetCategory, + AssetCriticality, + AssetSubCategory, + BaseClient, + CloudProvider, + ConfigurationError, + CustomJsonFormatter, + DaemonProtocol, + EnhancedJSONEncoder, + OpenAEVAuthenticationError, + OpenAEVCreateError, + OpenAEVDeleteError, + OpenAEVError, + OpenAEVGetError, + OpenAEVHeadError, + OpenAEVHttpError, + OpenAEVListError, + OpenAEVParsingError, + OpenAEVUpdateError, + RedirectError, + SecurityDomains, + SignatureTransmissionError, + data_to_temp_file, + is_memory_certificate, + logger, + setup_logging_config, + ssl_cert_chain, + ssl_verify_locations, +) + +# --- Configuration --- +from xtm_oaev_sdk.public.configuration import ( + CONFIGURATION_TYPES, + BaseConfigModel, + ConfigLoaderCollector, + ConfigLoaderOAEV, + Configuration, + ConfigurationHint, + ConfigurationProtocol, + ConnectorConfigSchemaGenerator, + DictionarySource, + EnvironmentSource, + HttpUrlToString, + LogLevelToLower, + SettingsLoader, + TimedeltaInSeconds, + is_falsy, + is_truthy, +) + +# --- Contracts (scenarios) --- +from xtm_oaev_sdk.public.contracts import ( + Contract, + ContractAiTarget, + ContractAsset, + ContractAssetGroup, + ContractAttachment, + ContractBuilder, + ContractBuilderProtocol, + ContractCardinality, + ContractCardinalityElement, + ContractCheckbox, + ContractConfig, + ContractElement, + ContractExpectations, + ContractExpectationType, + ContractFieldKey, + ContractFieldType, + ContractOutputElement, + ContractOutputType, + ContractPayload, + ContractReferencedCredential, + ContractSelect, + ContractTeam, + ContractText, + ContractTextArea, + ContractTuple, + ContractVariable, + CredentialType, + Domain, + Expectation, + LinkedFieldModel, + PrimitiveType, + SecurityPlatformType, + SupportedLanguage, + VariableHelper, + VariableType, + build_single_referenced_credential_element, + prepare_contracts, +) + +# --- Daemons --- +from xtm_oaev_sdk.public.daemons import BaseDaemon, PingAlive + +# --- Seam / compatibility --- +from xtm_oaev_sdk.public.deprecation import make_deprecated_getattr + +# --- Signatures --- +from xtm_oaev_sdk.public.signatures import ( + CloudInjectorConfig, + ExecutionSignature, + ExtraSignatureData, + InjectorConfig, + MatchTypes, + NetworkInjectorConfig, + SignatureManager, + SignatureManagerProtocol, + SignatureMatch, + SignatureTransportProtocol, + SignatureType, + SignatureTypes, + ToolErrorInfo, + ToolOutput, + ToolTimeoutInfo, + build_marker, + build_network_configs, +) + +# --- Port I/O contract (versioned wire schema) --- +from xtm_oaev_sdk.seam.port_io import ( + PORT_IO_CONTRACT_VERSION, + ExecutionDetails, + ExecutionStatus, + ExpectationSignatureGroup, + InjectExecutionActions, + SignatureCallbackPayload, + SignatureExpectationType, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, +) + +__all__ = [ + "__version__", + # Common: errors + "ConfigurationError", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SignatureTransmissionError", + # Common: utils + "AppLogger", + "AppLoggerProtocol", + "CustomJsonFormatter", + "EnhancedJSONEncoder", + "logger", + "setup_logging_config", + # Common: protocols + "BaseClient", + "DaemonProtocol", + # Common: ssl + "data_to_temp_file", + "is_memory_certificate", + "ssl_cert_chain", + "ssl_verify_locations", + # Common: enums + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "CloudProvider", + "SecurityDomains", + # Configuration + "BaseConfigModel", + "CONFIGURATION_TYPES", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", + "ConfigurationHint", + "ConfigurationProtocol", + "ConnectorConfigSchemaGenerator", + "DictionarySource", + "EnvironmentSource", + "HttpUrlToString", + "LogLevelToLower", + "SettingsLoader", + "TimedeltaInSeconds", + "is_falsy", + "is_truthy", + # Contracts (scenarios) + "Contract", + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractBuilder", + "ContractBuilderProtocol", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractElement", + "ContractExpectationType", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "CredentialType", + "Domain", + "Expectation", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableHelper", + "VariableType", + "build_single_referenced_credential_element", + "prepare_contracts", + # Daemons + "BaseDaemon", + "PingAlive", + # Signatures + "CloudInjectorConfig", + "ExecutionDetails", + "ExecutionSignature", + "ExecutionStatus", + "ExpectationSignatureGroup", + "ExtraSignatureData", + "InjectExecutionActions", + "InjectorConfig", + "MatchTypes", + "NetworkInjectorConfig", + "SignatureCallbackPayload", + "SignatureExpectationType", + "SignatureManager", + "SignatureManagerProtocol", + "SignatureMatch", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureTransportProtocol", + "SignatureType", + "SignatureTypes", + "SignatureValue", + "TargetSignatures", + "ToolErrorInfo", + "ToolOutput", + "ToolTimeoutInfo", + "build_marker", + "build_network_configs", + # Seam / compatibility + "PORT_IO_CONTRACT_VERSION", + "make_deprecated_getattr", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py new file mode 100644 index 00000000..7d2df976 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py @@ -0,0 +1,281 @@ +"""Common adapters: HTTP error translation, logging, JSON encoding, dict and +certificate helpers, plus the shared errors, enums, value objects and +protocols of the common seam. + +Each adapter speaks the common seam types and delegates to the core +implementation of its port. +""" + +import json +import logging +import ssl +from typing import Any + +from pythonjsonlogger.json import JsonFormatter +from xtm_oaev_sdk._core.common import Certificates, JsonEncoding, LoggingSetup +from xtm_oaev_sdk.seam.common import ( + AppLoggerProtocol, + AssetCategory, + AssetCriticality, + AssetSubCategory, + BaseClient, + CertificatesPort, + CloudProvider, + ConfigurationError, + DaemonProtocol, + JsonEncodingPort, + LoggingPort, + OpenAEVAuthenticationError, + OpenAEVCreateError, + OpenAEVDeleteError, + OpenAEVError, + OpenAEVGetError, + OpenAEVHeadError, + OpenAEVHttpError, + OpenAEVListError, + OpenAEVParsingError, + OpenAEVUpdateError, + RedirectError, + SecurityDomains, + SignatureTransmissionError, +) + +__all__ = [ + "AppLogger", + "AppLoggerProtocol", + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "BaseClient", + "CloudProvider", + "ConfigurationError", + "CustomJsonFormatter", + "DaemonProtocol", + "EnhancedJSONEncoder", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SecurityDomains", + "SignatureTransmissionError", + "data_to_temp_file", + "is_memory_certificate", + "logger", + "setup_logging_config", + "ssl_cert_chain", + "ssl_verify_locations", +] + +_encoding: JsonEncodingPort = JsonEncoding() +_logging: LoggingPort = LoggingSetup() +_certificates: CertificatesPort = Certificates() + + +class EnhancedJSONEncoder(json.JSONEncoder): + """JSON encoder that serializes ``@dataclass`` instances via + ``dataclasses.asdict()``. + + Example: + >>> import json + >>> from dataclasses import dataclass + >>> @dataclass + ... class Point: + ... x: int + ... y: int + >>> json.dumps(Point(1, 2), cls=EnhancedJSONEncoder) + '{"x": 1, "y": 2}' + """ + + def default(self, o: Any) -> Any: + """Serialize dataclasses to dicts. + + Args: + o: Object to serialize. + + Returns: + A JSON-serializable representation. + + Raises: + TypeError: If the object is not serializable. + """ + return _encoding.encode_default(o) + + +class CustomJsonFormatter(JsonFormatter): + """JSON log formatter with ISO timestamp and uppercase level. + + Extends pythonjsonlogger's JsonFormatter to add: + - UTC ISO 8601 timestamp if not already present. + - Uppercase level field normalization. + """ + + def add_fields( + self, + log_record: dict[str, Any], + record: logging.LogRecord, + message_dict: dict[str, Any], + ) -> None: + """Add timestamp and level fields to the log record. + + Args: + log_record: The output dict being built. + record: The stdlib LogRecord. + message_dict: Extra message fields. + """ + super().add_fields(log_record, record, message_dict) + _logging.complete_record(log_record, record) + + +def setup_logging_config(level: int, json_logging: bool = True) -> None: + """Configure root logger with JSON or plain text output. + + Silences noisy third-party loggers (urllib3, pika) and sets up + the root logger with either structured JSON or basic text formatting. + + Args: + level: Logging level (e.g., logging.DEBUG, logging.INFO). + json_logging: If True, use CustomJsonFormatter for structured output. + If False, use basicConfig plain text. + """ + _logging.setup_logging_config(level, json_logging, CustomJsonFormatter) + + +class AppLogger: + """Structured application logger with JSON support. + + Provides a simple logging interface with optional JSON formatting + and metadata attachment. Supports name-switching via __call__ for + per-module logger instances. + + Args: + level: Logging level (e.g., logging.DEBUG). + json_logging: Whether to use JSON-formatted output. + name: Logger name (defaults to module name). + + Example: + >>> log = AppLogger(logging.INFO) + >>> log.info("Server started", {"port": 8080}) + >>> module_log = log("my_module") + >>> module_log.debug("Processing request") + """ + + def __init__(self, level: int, json_logging: bool = True, name: str = __name__): + self.log_level = level + self.json_logging = json_logging + setup_logging_config(self.log_level, self.json_logging) + self.local_logger = _logging.get_logger(name) + + def __call__(self, name: str) -> "AppLogger": + """Switch to a named logger instance. + + Args: + name: New logger name. + + Returns: + Self with updated logger name. + """ + self.local_logger = _logging.get_logger(name) + return self + + @staticmethod + def prepare_meta(meta: Any = None) -> dict[str, Any] | None: + """Wrap metadata for structured logging extra field. + + Args: + meta: Arbitrary metadata to attach to log entry. + + Returns: + Dict with 'attributes' key, or None if no metadata. + """ + return _logging.prepare_meta(meta) + + @staticmethod + def setup_logger_level(lib: str, log_level: int) -> None: + """Override log level for a third-party library. + + Args: + lib: Library logger name (e.g., "urllib3"). + log_level: Desired logging level. + """ + _logging.set_library_level(lib, log_level) + + def debug(self, message: str, meta: Any = None) -> None: + """Log at DEBUG level. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.debug(message, extra=AppLogger.prepare_meta(meta)) + + def info(self, message: str, meta: Any = None) -> None: + """Log at INFO level. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.info(message, extra=AppLogger.prepare_meta(meta)) + + def warning(self, message: str, meta: Any = None) -> None: + """Log at WARNING level. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.warning(message, extra=AppLogger.prepare_meta(meta)) + + def error(self, message: str, meta: Any = None) -> None: + """Log at ERROR level with traceback. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.error(message, exc_info=1, extra=AppLogger.prepare_meta(meta)) + + +def logger(level: int, json_logging: bool = True) -> AppLogger: + """Create an AppLogger instance. + + Args: + level: Logging level. + json_logging: Whether to use JSON output. + + Returns: + An AppLogger instance. + """ + return AppLogger(level, json_logging) + + +def data_to_temp_file(data: str) -> str: + """Write data to a secure temporary file and return its path.""" + return _certificates.data_to_temp_file(data) + + +def is_memory_certificate(certificate: str) -> bool: + """Return True if the certificate is PEM-encoded in memory (not a file path).""" + return _certificates.is_memory_certificate(certificate) + + +def ssl_cert_chain( + ssl_context: ssl.SSLContext, + cert_data: str | None, + key_data: str | None, + passphrase: str | None, +) -> None: + """Load a certificate chain into an SSL context, handling in-memory PEM data.""" + _certificates.ssl_cert_chain(ssl_context, cert_data, key_data, passphrase) + + +def ssl_verify_locations(ssl_context: ssl.SSLContext, certdata: str | None) -> None: + """Load CA verification locations into an SSL context.""" + _certificates.ssl_verify_locations(ssl_context, certdata) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/configuration.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/configuration.py new file mode 100644 index 00000000..15447541 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/configuration.py @@ -0,0 +1,380 @@ +"""Configuration adapters: hint-based value resolution and settings loading. + +Each adapter speaks the configuration seam contracts and delegates to the core +implementation of its port. + +Example: + >>> config = Configuration({ + ... "log_level": {"env": "LOG_LEVEL", "default": "info"}, + ... "api_url": {"env": "API_URL", "file_path": ["openaev", "url"]}, + ... }) + >>> config.get("log_level") + 'info' + >>> config.set("log_level", "debug") + >>> config.get("log_level") + 'debug' +""" + +import os +from typing import Any, override + +from pydantic.json_schema import GenerateJsonSchema +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, +) +from xtm_oaev_sdk._core.configuration import ( + ConfigurationResolver, + ConfigurationSources, + ConnectorSchemaTransformer, + SettingsSourceSelector, + Truthiness, +) +from xtm_oaev_sdk.seam.configuration import ( + CONFIGURATION_TYPES, + BaseConfigModel, + ConfigLoaderCollector, + ConfigLoaderOAEV, + ConfigurationHint, + ConfigurationProtocol, + ConfigurationResolverPort, + ConfigurationSourcesPort, + ConnectorSchemaPort, + HttpUrlToString, + LogLevelToLower, + SettingsSourcesPort, + TimedeltaInSeconds, + TruthinessPort, +) + +__all__ = [ + "BaseConfigModel", + "CONFIGURATION_TYPES", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", + "ConfigurationHint", + "ConfigurationProtocol", + "ConnectorConfigSchemaGenerator", + "DictionarySource", + "EnvironmentSource", + "HttpUrlToString", + "LogLevelToLower", + "SettingsLoader", + "TimedeltaInSeconds", + "is_falsy", + "is_truthy", +] + +_truthiness: TruthinessPort = Truthiness() +_sources: ConfigurationSourcesPort = ConfigurationSources() +_resolver: ConfigurationResolverPort = ConfigurationResolver(_sources, _truthiness) +_schema: ConnectorSchemaPort = ConnectorSchemaTransformer() +_settings_sources: SettingsSourcesPort = SettingsSourceSelector() + + +def is_truthy(value: str) -> bool: + """Asserts whether a given string signals a "True" value. + + :param value: value to test + :type value: str + + :return: whether the string represents True or not. + :rtype: bool + """ + return _truthiness.is_truthy(value) + + +def is_falsy(value: str) -> bool: + """Asserts whether a given string signals a "False" value. + + :param value: value to test + :type value: str + + :return: whether the string represents False or not. + :rtype: bool + """ + return _truthiness.is_falsy(value) + + +class EnvironmentSource: + """Resolves configuration values from environment variables. + + Example: + >>> EnvironmentSource.get("HOME") + '/home/user' + >>> EnvironmentSource.get(None) is None + True + """ + + @classmethod + def get(cls, env_var: str | None) -> str | None: + """Get the value of an environment variable. + + Args: + env_var: Name of the environment variable to read. + Returns None if env_var is None or empty. + + Returns: + The environment variable value, or None if not set. + """ + return _sources.from_environment(env_var) + + +class DictionarySource: + """Resolves configuration values from nested dictionary structures. + + Handles two-level key paths into a JSON-like configuration dict, + typically loaded from a YAML or JSON config file. + + Example: + >>> source = {"openaev": {"url": "http://localhost:8080"}} + >>> DictionarySource.get(["openaev", "url"], source) + 'http://localhost:8080' + """ + + @classmethod + def get( + cls, config_key_path: list[str] | None, source_dict: dict[str, Any] + ) -> str | None: + """Get a value from a nested dict using a two-level key path. + + Args: + config_key_path: A list of exactly 2 non-empty strings representing + the path ``[section, key]`` into the source dict. + source_dict: The nested dictionary to search. + + Returns: + The value at the specified path, or None if not found. + + Raises: + AssertionError: If config_key_path is not a 2-element list of + non-empty strings. + """ + return _sources.from_dictionary(config_key_path, source_dict) + + +class ConnectorConfigSchemaGenerator(GenerateJsonSchema): + """Custom JSON schema generator for connector configurations. + + Produces a flat schema where all config variables are at root level, + with ``$ref`` entries fully resolved and filtered attributes removed. + + This is used by ``Configuration.schema()`` to produce manifests + compatible with the OpenAEV platform connector registration API. + """ + + @staticmethod + def dereference_schema(schema_with_refs: dict[str, Any]) -> dict[str, Any]: + """Resolve all internal ``$ref`` entries in a JSON schema. + + Args: + schema_with_refs: A JSON schema dict potentially containing + ``$ref`` pointers to ``$defs``. + + Returns: + A new schema with all references inlined. + + Raises: + ValueError: If a ``$ref`` uses an unsupported format. + """ + return _schema.dereference_schema(schema_with_refs) + + @staticmethod + def flatten_config_loader_schema(root_schema: dict[str, Any]) -> dict[str, Any]: + """Flatten a hierarchical config schema to root-level properties. + + Transforms nested namespace properties into flat + ``NAMESPACE_VARIABLE`` format suitable for environment variable + mapping. + + Args: + root_schema: The dereferenced schema to flatten. + + Returns: + A flat schema with all properties at root level. + """ + return _schema.flatten_config_loader_schema(root_schema) + + @staticmethod + def filter_schema(schema: dict[str, Any]) -> dict[str, Any]: + """Remove filtered attributes from the schema. + + Args: + schema: The flat schema to filter. + + Returns: + The schema with filtered attributes removed. + """ + return _schema.filter_schema(schema) + + @override + def generate(self, schema: Any, mode: str = "validation") -> dict[str, Any]: + """Generate a flat, dereferenced, filtered JSON schema. + + Args: + schema: The Pydantic core schema to process. + mode: Schema generation mode. + + Returns: + The final connector configuration schema. + """ + json_schema = super().generate(schema, mode=mode) + return _schema.finalize(json_schema, self.schema_dialect) + + @override + def nullable_schema(self, schema: Any) -> dict[str, Any]: + """Generate schema for nullable fields without ``anyOf``. + + Overrides the default behavior to produce cleaner schemas + that don't use the ``anyOf`` keyword for optional fields. + + Args: + schema: The core schema for the nullable type. + + Returns: + The generated JSON schema for the nullable type. + """ + return _schema.collapse_nullable(self.generate_inner(schema["schema"])) + + +class Configuration: + """Hint-based configuration resolver with multi-source lookup. + + Resolves configuration values by checking (in priority order): + 1. Explicit override via ``.set()`` + 2. Environment variable (if ``env`` hint provided) + 3. Config file value (if ``file_path`` hint provided) + 4. Default value (if ``default`` hint provided) + + Args: + config_hints: Mapping of config keys to hint definitions. + Values can be dicts (parsed as ConfigurationHint) or strings + (treated as default values). + config_values: Pre-loaded configuration dict (e.g., from YAML). + config_file_path: Path to YAML config file. Defaults to ./config.yml. + config_base_model: Optional Pydantic BaseSettings for schema generation. + + Example: + >>> config = Configuration({ + ... "url": {"env": "API_URL", "default": "http://localhost"}, + ... "timeout": {"env": "TIMEOUT", "is_number": True, "default": "30"}, + ... }) + >>> config.get("url") + 'http://localhost' + >>> config.get("timeout") + 30 + """ + + def __init__( + self, + config_hints: dict[str, dict[str, Any] | str], + config_values: dict[str, Any] | None = None, + config_file_path: str = os.path.join(os.curdir, "config.yml"), + config_base_model: BaseSettings | None = None, + ): + self.__config_hints: dict[str, ConfigurationHint] = _resolver.build_hints( + config_hints + ) + self.__config_values = _resolver.load_values(config_values, config_file_path) + self.__base_model = config_base_model + + def get(self, config_key: str) -> CONFIGURATION_TYPES: + """Get the resolved value for a configuration key. + + Resolution order: override → env var → file → default. + + Args: + config_key: The configuration key to look up. + + Returns: + The resolved value, or None if the key is unknown or + no source provides a value. + """ + config = self.__config_hints.get(config_key) + if config is None: + return None + + return _resolver.resolve(config, self.__config_values) + + def set(self, config_key: str, value: CONFIGURATION_TYPES) -> None: + """Set a runtime override for a configuration key. + + After calling this, ``get(config_key)`` will return the + overridden value regardless of environment or file sources. + + Args: + config_key: The configuration key to override. + value: The new value to return for this key. + """ + _resolver.override(self.__config_hints, config_key, value) + + def schema(self) -> dict[str, Any]: + """Generate the connector configuration JSON schema. + + Uses the configured base model's Pydantic schema generation + with the ConnectorConfigSchemaGenerator for flattened output. + + Returns: + A JSON schema dict describing all configuration properties. + + Raises: + AttributeError: If no config_base_model was provided. + """ + return _resolver.generate_schema( + self.__base_model, schema_generator=ConnectorConfigSchemaGenerator + ) + + +class SettingsLoader(BaseSettings): + """Base settings class with prioritized multi-source loading. + + Automatically discovers and loads configuration from environment + variables, YAML files, and .env files with a defined priority order. + + Subclass this to define your connector's configuration schema. + + Example: + >>> class MySettings(SettingsLoader): + ... model_config = SettingsConfigDict(yaml_file="config.yml") + ... api_url: str + ... timeout: int = 30 + """ + + model_config = SettingsConfigDict( + frozen=True, + extra="allow", + env_nested_delimiter="_", + env_nested_max_split=1, + enable_decoding=False, + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Configure source resolution order. + + Resolution priority: + 1. Environment variables (always checked first) + 2. YAML file (if config.yml exists) + 3. .env file (if .env exists and no YAML found) + 4. Default values (implicit via Pydantic) + + Args: + settings_cls: The settings class being configured. + init_settings: Init-based settings source. + env_settings: Environment variable source. + dotenv_settings: .env file source. + file_secret_settings: Secrets file source. + + Returns: + Tuple of settings sources in priority order. + """ + return _settings_sources.select_sources(settings_cls, env_settings) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/contracts.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/contracts.py new file mode 100644 index 00000000..8cfdd063 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/contracts.py @@ -0,0 +1,308 @@ +"""Contract adapters: contract definitions, field builder, template variables. + +Each adapter speaks the contracts seam types and delegates to the core +implementation of its port. + +Example: + >>> builder = ContractBuilder() + >>> builder.mandatory(ContractText(key="name", label="Name")) + >>> builder.optional(ContractCheckbox(key="verbose", label="Verbose")) + >>> fields = builder.build_fields() +""" + +from dataclasses import dataclass, field +from typing import Any + +from xtm_oaev_sdk._core.contracts import ( + BuiltinContractVariables, + ContractAssembly, + ContractSerializer, + CredentialReferences, +) +from xtm_oaev_sdk.seam.contracts import ( + ContractAiTarget, + ContractAssemblyPort, + ContractAsset, + ContractAssetGroup, + ContractAttachment, + ContractBuilderProtocol, + ContractCardinality, + ContractCardinalityElement, + ContractCheckbox, + ContractConfig, + ContractCredentialsPort, + ContractData, + ContractElement, + ContractExpectations, + ContractExpectationType, + ContractFieldKey, + ContractFieldType, + ContractOutputElement, + ContractOutputType, + ContractPayload, + ContractReferencedCredential, + ContractSelect, + ContractSerializerPort, + ContractTeam, + ContractText, + ContractTextArea, + ContractTuple, + ContractVariable, + ContractVariablesPort, + CredentialType, + Domain, + Expectation, + LinkedFieldModel, + PrimitiveType, + SecurityPlatformType, + SupportedLanguage, + VariableType, +) + +__all__ = [ + "Contract", + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractBuilder", + "ContractBuilderProtocol", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractElement", + "ContractExpectationType", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "CredentialType", + "Domain", + "Expectation", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableHelper", + "VariableType", + "build_single_referenced_credential_element", + "prepare_contracts", +] + +_assembly: ContractAssemblyPort = ContractAssembly() +_variables: ContractVariablesPort = BuiltinContractVariables() +_serializer: ContractSerializerPort = ContractSerializer() +_credentials: ContractCredentialsPort = CredentialReferences() + + +class VariableHelper: + """Factory for standard contract template variables. + + Provides pre-built variable definitions for common injection + targets (users, exercises, teams, platform URIs). + + Example: + >>> variables = [ + ... VariableHelper.user_variable(), + ... VariableHelper.exercise_variable(), + ... VariableHelper.team_variable(), + ... ] + VariableHelper.uri_variables() + """ + + @staticmethod + def user_variable() -> ContractVariable: + """Create the user template variable with standard children.""" + return _variables.user_variable() + + @staticmethod + def exercise_variable() -> ContractVariable: + """Create the exercise template variable with standard children.""" + return _variables.exercise_variable() + + @staticmethod + def team_variable() -> ContractVariable: + """Create the teams template variable.""" + return _variables.team_variable() + + @staticmethod + def uri_variables() -> list[ContractVariable]: + """Create platform URI template variables.""" + return _variables.uri_variables() + + +@dataclass +class Contract(ContractData): + """A complete contract definition for an injector or collector. + + Contains fields (inputs), outputs, configuration, and metadata + for registering the contract with the OpenAEV platform. + + Args: + contract_id: Unique contract identifier. + label: Localized contract labels. + fields: Input field definitions. + outputs: Output field declarations. + config: Display configuration. + manual: Whether this is a manual-execution contract. + variables: Template variables available for injection. + """ + + variables: list[ContractVariable] = field( + default_factory=lambda: _variables.default_variables() + ) + + def to_contract_add_input(self, source_id: str) -> dict[str, Any]: + """Serialize this contract for the platform creation API. + + Args: + source_id: The injector ID that owns this contract. + + Returns: + Dict payload for the contract creation endpoint. + """ + return _serializer.to_add_input(self, source_id) + + def to_contract_update_input(self) -> dict[str, Any]: + """Serialize this contract for the platform update API. + + Returns: + Dict payload for the contract update endpoint. + """ + return _serializer.to_update_input(self) + + +def prepare_contracts(contracts: list[Contract]) -> list[dict[str, Any]]: + """Serialize a list of contracts for bulk platform registration. + + Args: + contracts: List of Contract instances to serialize. + + Returns: + List of dicts suitable for the bulk contract registration API. + """ + return _serializer.prepare(contracts) + + +def build_single_referenced_credential_element( + provider_name: str, +) -> ContractReferencedCredential: + """Build a credential-reference field for a provider-specific contract + with ``multiple`` value at ``False``. + + This is the centralized entry point injectors should use when they need the + OpenAEV inject form to ask for one referenced credential. The helper keeps + provider-to-``CredentialType`` mapping consistent across injector projects. + """ + return _credentials.build_single_referenced_credential_element(provider_name) + + +class ContractBuilder: + """Fluent builder for contract field and output declarations. + + Provides a chainable API to construct contract definitions + with mandatory, optional, and grouped fields. + + Example: + >>> b = ContractBuilder() + >>> b.mandatory(ContractText(key="target", label="Target URL")) + >>> b.optional(ContractCheckbox(key="verify_ssl", label="Verify SSL")) + >>> fields = b.build_fields() + >>> len(fields) + 2 + """ + + fields: list[ContractElement] + outputs: list[ContractOutputElement] + + def __init__(self) -> None: + self.fields = [] + self.outputs = [] + + def add_fields(self, fields: list[ContractElement]) -> "ContractBuilder": + """Add multiple fields at once. + + Args: + fields: List of contract elements to add. + + Returns: + Self for chaining. + """ + self.fields = _assembly.concat(self.fields, fields) + return self + + def add_outputs(self, outputs: list[ContractOutputElement]) -> "ContractBuilder": + """Add multiple output declarations at once. + + Args: + outputs: List of output elements to add. + + Returns: + Self for chaining. + """ + self.outputs = _assembly.concat(self.outputs, outputs) + return self + + def mandatory(self, element: ContractElement) -> "ContractBuilder": + """Add a mandatory field. + + Args: + element: The contract element to mark as mandatory. + + Returns: + Self for chaining. + """ + self.fields.append(_assembly.set_mandatory(element, True)) + return self + + def optional(self, element: ContractElement) -> "ContractBuilder": + """Add an optional field. + + Args: + element: The contract element to mark as optional. + + Returns: + Self for chaining. + """ + self.fields.append(_assembly.set_mandatory(element, False)) + return self + + def mandatory_group(self, elements: list[ContractElement]) -> "ContractBuilder": + """Add a group of mutually-required fields. + + All fields in the group reference each other's keys and are + marked mandatory. + + Args: + elements: List of contract elements forming the group. + + Returns: + Self for chaining. + """ + self.fields.extend(_assembly.require_group(elements)) + return self + + def build_fields(self) -> list[ContractElement]: + """Return all accumulated contract fields. + + Returns: + List of ContractElement instances added via this builder. + """ + return self.fields + + def build_outputs(self) -> list[ContractOutputElement]: + """Return all accumulated output declarations. + + Returns: + List of ContractOutputElement instances added via this builder. + """ + return self.outputs diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/daemons.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/daemons.py new file mode 100644 index 00000000..2a292756 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/daemons.py @@ -0,0 +1,149 @@ +"""Daemon adapters: the base daemon runtime and its heartbeat. + +Each adapter speaks the daemons seam types and delegates to the core +implementation of its port. The API client is injected (``BaseClient``). +""" + +import threading +from abc import ABC, abstractmethod + +from xtm_oaev_sdk._core.daemons import DaemonRuntime, Heartbeat +from xtm_oaev_sdk.public.common import logger +from xtm_oaev_sdk.public.configuration import Configuration +from xtm_oaev_sdk.seam.common import BaseClient +from xtm_oaev_sdk.seam.daemons import DaemonRuntimePort, HeartbeatPort + +__all__ = ["BaseDaemon", "PingAlive"] + +_runtime: DaemonRuntimePort = DaemonRuntime() +_heartbeat: HeartbeatPort = Heartbeat() + + +class BaseDaemon(ABC): + """A base class for implementing a kind of daemon that periodically polls + a given callback. + + :param configuration: configuration to provide the daemon + (allowing for looking up values within the callback for example) + :type configuration: Configuration + :param callback: a method or function to periodically call, defaults to None. + :type callback: callable, optional + :param logger: a logger object, to log events. if not supplied, a default logger + will be spawned to provide this functionality. + :type logger: Any + :param api_client: an API client that will provide connectivity with other systems, + conforming to the ``BaseClient`` protocol. When omitted, the client is built by + ``_create_default_api_client()``, which the SDK cannot do on its own: it never + depends on a concrete client. Inject one, or override the hook. + :type api_client: BaseClient, optional + """ + + def __init__( + self, + configuration: Configuration, + callback: callable = None, + logger=None, + api_client: BaseClient | None = None, + ): + self._configuration = configuration + self._callback = callback + self.api = api_client or self._create_default_api_client() + + # logging + # compatibility layer: in order for older configs to still work, search for legacy names + actual_log_level = _runtime.resolve_log_level(self._configuration) + actual_log_name = _runtime.resolve_name(self._configuration) + self.logger = logger or BaseDaemon.__get_default_logger( + actual_log_level.upper(), + actual_log_name, + ) + + @abstractmethod + def _setup(self): + """A run-once method that inheritors must implement. This serves to instantiate + all useful objects and functionality for the implementor to run. + """ + pass + + @abstractmethod + def _start_loop(self): + """Starts the daemon's main execution loop. Implementors should implement + the main execution logic in here. + """ + pass + + def _try_callback(self): + """Tries to call the configured callback. Note that if any error is thrown, + it is immediately swallowed (but still logged) allowing the collector to keep + running. This is useful for any transient issue (e.g. API endpoint down...). + """ + _runtime.invoke_callback(self._callback, self, self.logger) + + def start(self): + """Start the daemon. This will run the implementor's run-once setup method and + follow-up with the main execution loop. Note that at this point, if there is no + configured callback, the method will abort and kill the daemon. + """ + if _runtime.dump_config_schema_requested(): + _runtime.dump_config_schema(self._configuration) + return + + _runtime.ensure_callback(self._callback) + self._setup() + self._start_loop() + + def set_callback(self, callback: callable): + """Configures a callback to call in the main execution loop. If the callback + was not provided in the daemon's ctor, this should be set before calling start(). + """ + self._callback = callback + + def get_id(self): + """Returns the daemon instance's ID contained in configuration. Configuration + must define any of these keys: `id`, `collector_id`, `injector_id`. + """ + return _runtime.resolve_id(self._configuration) + + def _create_default_api_client(self) -> BaseClient: + """Builds the API client used when none was injected. The SDK has no + concrete client to build: client libraries override this hook (e.g. to + construct one from the ``openaev_url``, ``openaev_token`` and + ``openaev_tenant_id`` configuration keys). + """ + return _runtime.default_api_client() + + @classmethod + def __get_default_logger(cls, log_level, name): + return logger(log_level)(name) + + +class PingAlive(threading.Thread): + """Heartbeat thread keeping a daemon registered on the platform.""" + + def __init__(self, api, config, logger, ping_type) -> None: + threading.Thread.__init__(self) + self.ping_type = ping_type + self.api = api + self.tenant_id = _heartbeat.tenant_of(self.api) + self.config = config + self.logger = logger + self.in_error = False + self.exit_event = threading.Event() + + def ping(self) -> None: + _heartbeat.ping( + self.api, self.config, self.ping_type, self.logger, self.exit_event + ) + + def run(self) -> None: + self.logger.info( + "Starting PingAlive thread", _heartbeat.tenant_meta(self.tenant_id) + ) + self.ping() + + def stop(self) -> None: + self.logger.info( + "Preparing PingAlive for clean shutdown", + _heartbeat.tenant_meta(self.tenant_id), + ) + self.exit_event.set() diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/deprecation.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/deprecation.py new file mode 100644 index 00000000..36f3150e --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/deprecation.py @@ -0,0 +1,85 @@ +"""PEP 562 module deprecation — a reusable compatibility primitive. + +This is the mechanism behind *"this name moved / was renamed / is going +away, here's the new home."* + +A module that wants to retire one or more names without breaking existing +consumers declares a mapping from each old name to +``(deprecation_message, replacement)`` and installs a module-level +``__getattr__`` built by :func:`make_deprecated_getattr`:: + + from xtm_oaev_sdk import make_deprecated_getattr + + __getattr__ = make_deprecated_getattr( # PEP 562 + {"OldName": ("Use 'from xtm_oaev_sdk import NewName' instead.", NewName)}, + module_name=__name__, + ) + +Assign the factory's result directly: a ``def __getattr__`` wrapper adds a +frame, and the warning would then point at the shim instead of the caller. + +Why a factory, and why it returns the object directly: + +- **One primitive, many modules.** The ``__getattr__`` body is otherwise + copy-pasted across a dozen legacy modules. A factory turns each copy into + a data-driven mapping, so the warn / return / raise logic lives in a single + tested place and every module only supplies the names it retires. +- **Identity is preserved.** The factory returns *exactly* the object the new + name refers to — the same object, not a wrapper. No ``__new__`` / ``__call__`` + interception, no PEP 702 class. Wrappers deliberately mutate ``__new__`` and + add indirection that breaks ``isinstance`` checks, metaclass handling, and + signature introspection, so they are *not* used here. +- **Real misses still fail.** A name that is not in the mapping raises + ``AttributeError`` (naming the module), so a typo or a not-yet-retired name + is never silently swallowed by the deprecation layer. + +This adapter speaks the deprecation seam types and delegates to the core +implementation of its port. +""" + +from collections.abc import Mapping as _Mapping +from typing import Any + +from xtm_oaev_sdk._core.deprecation import DeprecationShims +from xtm_oaev_sdk.seam.deprecation import DeprecationPort + +__all__ = ["make_deprecated_getattr"] + +_deprecation: DeprecationPort = DeprecationShims() + + +def make_deprecated_getattr( + mapping: _Mapping[str, tuple[str, Any]], + *, + module_name: str | None = None, +): + """Build a PEP 562 module-level ``__getattr__`` for a set of deprecated names. + + Parameters + ---------- + mapping: + ``{old_name: (deprecation_message, replacement)}``. On first access to + ``old_name`` the ``deprecation_message`` is emitted as a + ``DeprecationWarning`` (``stacklevel=2``) and the ``replacement`` object + is returned. + module_name: + Optional dotted name used in the ``AttributeError`` for a name that is + *not* in ``mapping`` (e.g. ``"pyoaev.helpers"``). Defaults to + ``"this module"``. + + Returns + ------- + A zero-argument-per-call ``__getattr__(name)`` callable, suitable for + assignment as a module's ``__getattr__``. + + Notes + ----- + The input mapping is validated once and copied, so later mutation of the + caller's mapping does not affect the returned callable. A malformed entry + (non-string key, or a value that is not a two-item ``(message, target)``) + raises ``ValueError`` at call time — i.e. at import time of the module that + installs the shim — rather than at first access. + """ + return _deprecation.build_getattr( + _deprecation.resolve_mapping(mapping), module_name + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/signatures.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/signatures.py new file mode 100644 index 00000000..2028e7fc --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/signatures.py @@ -0,0 +1,255 @@ +"""Signature adapters: the signature manager and the target/marker helpers. + +Each adapter speaks the signatures seam types and delegates to the core +implementation of its port; signatures are shipped through an injected +``SignatureTransportProtocol``. +""" + +import logging +from datetime import datetime +from typing import Any + +from xtm_oaev_sdk._core.signatures import ( + ContainerAddressResolver, + MarkerBuilder, + NetworkTargets, + SignatureEngine, +) +from xtm_oaev_sdk.seam.port_io import ExecutionDetails +from xtm_oaev_sdk.seam.signatures import ( + CloudInjectorConfig, + ContainerAddressPort, + ExecutionSignature, + ExtraSignatureData, + InjectorConfig, + MarkerPort, + MatchTypes, + NetworkInjectorConfig, + NetworkTargetsPort, + SignatureEnginePort, + SignatureManagerProtocol, + SignatureMatch, + SignatureTransportProtocol, + SignatureType, + SignatureTypes, + ToolErrorInfo, + ToolOutput, + ToolTimeoutInfo, +) + +__all__ = [ + "CloudInjectorConfig", + "ExecutionSignature", + "ExtraSignatureData", + "InjectorConfig", + "MatchTypes", + "NetworkInjectorConfig", + "SignatureManager", + "SignatureManagerProtocol", + "SignatureMatch", + "SignatureTransportProtocol", + "SignatureType", + "SignatureTypes", + "ToolErrorInfo", + "ToolOutput", + "ToolTimeoutInfo", + "build_marker", + "build_network_configs", +] + +_engine: SignatureEnginePort = SignatureEngine() +_addresses: ContainerAddressPort = ContainerAddressResolver() +_targets: NetworkTargetsPort = NetworkTargets() +_marker: MarkerPort = MarkerBuilder() + + +class SignatureManager: + """End-to-end signature pipeline: compile, merge, transmit. One class, three jobs.""" + + DEFAULT_MAX_PAYLOAD_SIZE = 5_242_880 # 5 MiB + + def __init__( + self, + transport: SignatureTransportProtocol, + logger: logging.Logger | None = None, + max_payload_size: int = DEFAULT_MAX_PAYLOAD_SIZE, + ) -> None: + """Initialize manager with a transport implementation for signature delivery.""" + self.transport = transport + self.logger = logger or logging.getLogger(__name__) + self.max_payload_size = max_payload_size + self._cached_ipv4: str | None = None + self._cached_ipv6: str | None = None + + def _utcnow(self) -> datetime: + """Current UTC time. Carved out so tests can pin the clock.""" + return _engine.now() + + def build_execution_signatures( + self, + config: InjectorConfig | list[InjectorConfig], + ) -> ExecutionSignature | list[ExecutionSignature]: + """Build pre-execution signature dicts from one or more typed injector configs. + + The category is carried by the config type itself + (:class:`NetworkInjectorConfig`, :class:`CloudInjectorConfig`), + so no separate ``category`` flag is needed. + + Args: + config: A single injector config or a homogeneous list of them. + Multi-target injects must be expressed as a list. + + Returns: + One ExecutionSignature object when a single config is given, + otherwise a list of ExecutionSignature in input order. + + Raises: + ValueError: Empty list, or mixed config types in a single call. + TypeError: Unknown injector config type. + """ + configs = list(config) if isinstance(config, list) else [config] + _engine.validate_configs(configs) + start_time = _engine.format_start_time(self._utcnow()) + results = [self._build_one(cfg, start_time) for cfg in configs] + return results[0] if len(results) == 1 else results + + def _build_one(self, config: InjectorConfig, start_time: str) -> ExecutionSignature: + """Project a single injector config into a flat pre-execution signature dict. + + Common pipeline for every category: + 1. Seed the base dict with ``start_time`` and category-specific context + (network gets resolved source IPs; cloud add nothing). + 2. Layer the config's own fields on top. + 3. Run it through :class:`ExecutionSignature` for validation + and emit JSON-ready output stripped of ``None``\\s. + """ + return _engine.build_signature(config, start_time, self._source_context(config)) + + def _source_context(self, config: InjectorConfig) -> dict[str, Any]: + """Return the source identity bits injected for the config's category. + + Only network signatures need the running container's source IPs; + cloud rows have no source identity to carry. + """ + return _engine.source_context( + config, self.resolve_container_ip, lambda: self._cached_ipv6 + ) + + def post_execution_updates( + self, + execution_details: ExecutionDetails, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + tool_output: dict[str, Any], + ) -> None: + """ + Update both execution details and execution signatures according to tool output + """ + _engine.apply_tool_output( + execution_details, execution_signatures, tool_output, self._utcnow + ) + + @staticmethod + def build_payload( + execution_signatures: ExecutionSignature | list[ExecutionSignature], + targets_meta: Any | list[Any], + expectation_types: list[str], + extra_signatures: ExtraSignatureData = ExtraSignatureData(), + ) -> dict[str, Any]: + """Build the nested wire payload from flat post-execution signatures. + + Bridges the gap between compile_post_execution_signatures output (flat dicts) + and send_signatures input (nested wire format). + + Args: + execution_signatures: A single post-execution ExecutionSignature or a list (multi-targets). + targets_meta: Target metadata dict(s) with keys like agent, asset, asset_group. + expectation_types: The 1+ expectation type labels (e.g. ['DETECTION', 'PREVENTION']). + extra_signatures: Optional mapping of expectation types to additional signature fields that will be merged + into the base post_signatures. + + Returns: + A payload dict ready for send_signatures. + """ + return _engine.build_payload( + execution_signatures, targets_meta, expectation_types, extra_signatures + ) + + def send_signatures( + self, + inject_id: str, + execution_details: ExecutionDetails, + signatures: dict[str, Any], + ) -> None: + """Ship signatures to the callback endpoint via the configured transport. + + Constructs typed ``SignatureOutputStructure`` and ``ExecutionDetails`` + models, then delegates transport (retry, envelope splitting, validation) + to the configured signature transport. + + Args: + inject_id: Inject UUID. + execution_details: execution-related metadata as an ExecutionDetails object + signatures: Full signatures dict with a ``targets`` list. + + Raises: + SignatureTransmissionError: Validation failed, 4xx hit, or retries exhausted. + """ + sig_output = _engine.output_structure(signatures) + self.transport.send_signatures( + inject_id, + sig_output, + execution_details, + max_payload_size=self.max_payload_size, + logger=self.logger, + ) + + def resolve_container_ip(self) -> str: + """Sniff the container's primary IPv4. Env var, hostname, then ``hostname -i``. + + Returns: + The IPv4 string, or ``'unknown'`` with a warning when all strategies fail. + """ + if self._cached_ipv4 and self._cached_ipv4 != "unknown": + return self._cached_ipv4 + + ip = _addresses.resolve_ipv4(self.logger) + if ip is None: + self._cached_ipv4 = "unknown" + return "unknown" + self._cached_ipv4 = ip + self._resolve_ipv6() + return ip + + def _resolve_ipv6(self) -> None: + """Best-effort IPv6 sniff. Silent on failure, no exceptions escape.""" + addr = _addresses.resolve_ipv6() + if addr: + self._cached_ipv6 = addr + + +def build_network_configs( + targets: list[str | dict[str, Any] | NetworkInjectorConfig], +) -> list[NetworkInjectorConfig]: + """Forge a list of `NetworkInjectorConfig` from a heterogeneous target list. + + Each item is one distinct asset. Accepted shapes: + + - `NetworkInjectorConfig`: passed through unchanged. + - `dict`: validated against :class:`NetworkInjectorConfig`. + - `str`: auto-classified into IPv4 / IPv6 / hostname. + + Args: + targets: Raw target list straight out of the injector. + + Returns: + One `NetworkInjectorConfig` per input target, order preserved. + + Raises: + TypeError: An item is not one of the accepted shapes. + ValidationError: A dict item fails the one-identity invariant. + """ + return _targets.build_network_configs(targets) + + +def build_marker(inject_id: str, agent_id: str = "") -> str: + return _marker.build_marker(inject_id, agent_id) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/py.typed b/libs/xtm-oaev-sdk/xtm_oaev_sdk/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py new file mode 100644 index 00000000..93083f68 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -0,0 +1,37 @@ +"""The architectural **seam** — the versioned compatibility boundary. + +Everything in this layer pins a stable interface so that the two sides of it +can evolve independently without leaking either: + +- **``common``** — the shared vocabulary (errors, enums, value objects, + consumer protocols) and the common ports; closed and versioned + (``SEAM_VERSION``). +- **``configuration``** — the configuration contracts (hints, settings + models, serialization types) and the ports its public adapters delegate + to; closed and versioned (``SEAM_VERSION``). +- **``contracts``** — the OpenAEV scenario data contracts (contract + definitions, field elements, enums, template variables), the builder + protocol and the ports its public adapters delegate to; closed and + versioned (``SEAM_VERSION``). +- **``daemons``** — the ports of the daemon runtime shared by collectors and + injectors (lifecycle, heartbeat); closed and versioned (``SEAM_VERSION``). +- **``signatures``** — the matching vocabulary and policies, the signature + ports, and the inbound I/O models (``seam/signatures/seam.py``, versioned + ``SEAM_VERSION``). +- **``port_io``** — the port I/O wire schema crossing the SDK↔platform + boundary; the seam **owns** the models (``seam/port_io/models.py``), + versioned ``PORT_IO_CONTRACT_VERSION``. +- **``deprecation``** — the port behind the PEP 562 deprecation primitive + (``make_deprecated_getattr``) that retires public names without breaking + importers; closed and versioned (``SEAM_VERSION``). + +A seam is **closed**: it imports only the standard library, third-party data +libraries and sibling seam modules — never ``_core`` nor ``public``. The core +implements its ports; the public adapters speak its types and delegate to the +core. As long as the core keeps answering the seam, the public surface keeps +working; a seam change is a contract change (``SEAM_VERSION`` bump, changelog). + +Every seam package is closed. +""" + +__all__: list[str] = [] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py new file mode 100644 index 00000000..91ddf9d3 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py @@ -0,0 +1,76 @@ +"""Common seam: the shared vocabulary and the common ports. + +Errors, enums, value objects and consumer protocols every capability speaks, +plus the ports the common public adapters delegate to. Closed: it imports +nothing from ``xtm_oaev_sdk._core`` nor ``xtm_oaev_sdk.public``. Any change to +these contracts must bump :data:`SEAM_VERSION`. +""" + +from xtm_oaev_sdk.seam.common.asset_types import ( + AssetCategory, + AssetCriticality, + AssetSubCategory, + CloudProvider, +) +from xtm_oaev_sdk.seam.common.errors import ( + ConfigurationError, + OpenAEVAuthenticationError, + OpenAEVCreateError, + OpenAEVDeleteError, + OpenAEVError, + OpenAEVGetError, + OpenAEVHeadError, + OpenAEVHttpError, + OpenAEVListError, + OpenAEVParsingError, + OpenAEVUpdateError, + RedirectError, + SignatureTransmissionError, +) +from xtm_oaev_sdk.seam.common.ports import ( + CertificatesPort, + JsonEncodingPort, + LoggingPort, +) +from xtm_oaev_sdk.seam.common.protocols import ( + AppLoggerProtocol, + BaseClient, + DaemonProtocol, +) +from xtm_oaev_sdk.seam.common.security_domains import SecurityDomains + +#: Version of the common seam. Bump on any contract change. +SEAM_VERSION = "1.0.0" + +__all__ = [ + "SEAM_VERSION", + # Errors + "ConfigurationError", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SignatureTransmissionError", + # Enums + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "CloudProvider", + # Enums (security domains) + "SecurityDomains", + # Consumer protocols + "AppLoggerProtocol", + "BaseClient", + "DaemonProtocol", + # Ports + "CertificatesPort", + "JsonEncodingPort", + "LoggingPort", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/asset_types.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/asset_types.py new file mode 100644 index 00000000..769cdb3e --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/asset_types.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +"""Canonical asset taxonomy for OpenAEV (mirrors the backend enums). + +These give producers (collectors, custom integrations) a single, typed vocabulary for the +two-level asset categorization (category + subcategory), criticality and cloud provider, instead +of free-form strings. They are plain ``str`` enums so they can be dropped straight into the dict +payloads sent to the API. +""" + +from enum import Enum + +__all__ = [ + "AssetCategory", + "AssetSubCategory", + "CloudProvider", + "AssetCriticality", +] + + +class AssetCategory(str, Enum): + HOST = "HOST" + CONTAINER_WORKLOAD = "CONTAINER_WORKLOAD" + CLOUD_RESOURCE = "CLOUD_RESOURCE" + WEB_APPLICATION = "WEB_APPLICATION" + NETWORK_DEVICE = "NETWORK_DEVICE" + MOBILE_DEVICE = "MOBILE_DEVICE" + IOT_OT_DEVICE = "IOT_OT_DEVICE" + IDENTITY = "IDENTITY" + SAAS_APPLICATION = "SAAS_APPLICATION" + AI_TARGET = "AI_TARGET" + SECURITY_PLATFORM = "SECURITY_PLATFORM" + GENERIC_ASSET = "GENERIC_ASSET" + + +class AssetSubCategory(str, Enum): + # HOST + SERVER = "SERVER" + WORKSTATION = "WORKSTATION" + LAPTOP = "LAPTOP" + VIRTUAL_MACHINE = "VIRTUAL_MACHINE" + HYPERVISOR = "HYPERVISOR" + MAINFRAME = "MAINFRAME" + THIN_CLIENT = "THIN_CLIENT" + # CONTAINER_WORKLOAD + CONTAINER = "CONTAINER" + CONTAINER_IMAGE = "CONTAINER_IMAGE" + KUBERNETES_POD = "KUBERNETES_POD" + KUBERNETES_CLUSTER = "KUBERNETES_CLUSTER" + KUBERNETES_NODE = "KUBERNETES_NODE" + SERVERLESS_FUNCTION = "SERVERLESS_FUNCTION" + # CLOUD_RESOURCE + COMPUTE = "COMPUTE" + STORAGE = "STORAGE" + DATABASE = "DATABASE" + NETWORKING = "NETWORKING" + SERVERLESS = "SERVERLESS" + CONTAINER_REGISTRY = "CONTAINER_REGISTRY" + KUBERNETES = "KUBERNETES" + IAM_PRINCIPAL = "IAM_PRINCIPAL" + SECRETS_KEY_MGMT = "SECRETS_KEY_MGMT" + MESSAGING_QUEUE = "MESSAGING_QUEUE" + ANALYTICS_DATA = "ANALYTICS_DATA" + AI_ML_SERVICE = "AI_ML_SERVICE" + IAC_TEMPLATE = "IAC_TEMPLATE" + CLOUD_OTHER = "CLOUD_OTHER" + # WEB_APPLICATION + WEBSITE = "WEBSITE" + WEB_API = "WEB_API" + SINGLE_PAGE_APP = "SINGLE_PAGE_APP" + GRAPHQL_API = "GRAPHQL_API" + WEB_SERVICE = "WEB_SERVICE" + MICROSERVICE = "MICROSERVICE" + # NETWORK_DEVICE + ROUTER = "ROUTER" + SWITCH = "SWITCH" + FIREWALL = "FIREWALL" + LOAD_BALANCER = "LOAD_BALANCER" + VPN_GATEWAY = "VPN_GATEWAY" + WIRELESS_AP = "WIRELESS_AP" + PROXY = "PROXY" + DNS_SERVER = "DNS_SERVER" + DHCP_SERVER = "DHCP_SERVER" + SAN_NAS = "SAN_NAS" + NETWORK_OTHER = "NETWORK_OTHER" + # MOBILE_DEVICE + SMARTPHONE = "SMARTPHONE" + TABLET = "TABLET" + # IOT_OT_DEVICE + IOT_SENSOR = "IOT_SENSOR" + IP_CAMERA = "IP_CAMERA" + GATEWAY = "GATEWAY" + POINT_OF_SALE = "POINT_OF_SALE" + MEDIA_DEVICE = "MEDIA_DEVICE" + PLC = "PLC" + RTU = "RTU" + HMI = "HMI" + SCADA_HISTORIAN = "SCADA_HISTORIAN" + MEDICAL_DEVICE = "MEDICAL_DEVICE" + PRINTER_PERIPHERAL = "PRINTER_PERIPHERAL" + BUILDING_MGMT = "BUILDING_MGMT" + # IDENTITY + USER_ACCOUNT = "USER_ACCOUNT" + SERVICE_ACCOUNT = "SERVICE_ACCOUNT" + GROUP = "GROUP" + ROLE = "ROLE" + SHARED_MAILBOX = "SHARED_MAILBOX" + NON_HUMAN_IDENTITY = "NON_HUMAN_IDENTITY" + # SAAS_APPLICATION + SAAS_APP = "SAAS_APP" + SAAS_TENANT = "SAAS_TENANT" + # AI_TARGET + LLM_MODEL = "LLM_MODEL" + AI_AGENT = "AI_AGENT" + MCP_SERVER = "MCP_SERVER" + RAG_PIPELINE = "RAG_PIPELINE" + # SECURITY_PLATFORM + EDR = "EDR" + XDR = "XDR" + SIEM = "SIEM" + SOAR = "SOAR" + NDR = "NDR" + ISPM = "ISPM" + LLM_FIREWALL = "LLM_FIREWALL" + AI_GATEWAY = "AI_GATEWAY" + + +class CloudProvider(str, Enum): + AWS = "AWS" + AZURE = "AZURE" + GCP = "GCP" + OCI = "OCI" + ALIBABA = "ALIBABA" + KUBERNETES = "KUBERNETES" + OTHER = "OTHER" + + +class AssetCriticality(str, Enum): + VERY_HIGH = "VERY_HIGH" + HIGH = "HIGH" + MEDIUM = "MEDIUM" + LOW = "LOW" + UNKNOWN = "UNKNOWN" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/errors.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/errors.py new file mode 100644 index 00000000..13d9f49e --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/errors.py @@ -0,0 +1,277 @@ +"""xtm-oaev-sdk error hierarchy. + +All SDK exceptions derive from OpenAEVError. Consumers catch specific +subclasses for granular handling or catch the base class as a catch-all. + +Exception tree:: + + OpenAEVError (base) + ├── OpenAEVAuthenticationError — credential/token failures + ├── OpenAEVHttpError — generic HTTP transport errors + │ └── (wrapped by the HTTP client, e.g. pyoaev's on_http_error, into caller-specific types) + ├── OpenAEVParsingError — response deserialization failures + ├── RedirectError — unexpected HTTP redirects + ├── OpenAEVHeadError — HEAD request failures + ├── OpenAEVGetError — GET request failures + ├── OpenAEVUpdateError — PUT/PATCH request failures + ├── OpenAEVListError — list/pagination failures + ├── OpenAEVCreateError — POST creation failures + ├── SignatureTransmissionError — signature delivery failures + └── ConfigurationError — configuration loading/validation failures + +Example: + >>> from xtm_oaev_sdk import OpenAEVError, OpenAEVHttpError + >>> try: + ... api.call() + ... except OpenAEVHttpError as e: + ... print(f"HTTP {e.response_code}: {e.error_message}") + ... except OpenAEVError as e: + ... print(f"SDK error: {e}") +""" + +from typing import TYPE_CHECKING, Optional, Union + +__all__ = [ + "OpenAEVError", + "OpenAEVAuthenticationError", + "OpenAEVHttpError", + "OpenAEVParsingError", + "RedirectError", + "OpenAEVHeadError", + "OpenAEVGetError", + "OpenAEVUpdateError", + "OpenAEVListError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "SignatureTransmissionError", + "ConfigurationError", +] + + +class OpenAEVError(Exception): + """Base exception for all xtm-oaev-sdk errors. + + Provides structured error context including HTTP status codes and + response bodies when available. Automatically extracts meaningful + error messages from JSON response payloads. + + Args: + error_message: Human-readable error description. Accepts str or bytes. + response_code: HTTP status code if the error originated from an API call. + response_body: Raw HTTP response body for deeper error extraction. + """ + + def __init__( + self, + error_message: Union[str, bytes] = "", + response_code: Optional[int] = None, + response_body: Optional[bytes] = None, + ) -> None: + Exception.__init__(self, error_message) + self.response_code = response_code + self.response_body = response_body + try: + if TYPE_CHECKING: + assert isinstance(error_message, bytes) + self.error_message = error_message.decode() + except Exception: + if TYPE_CHECKING: + assert isinstance(error_message, str) + self.error_message = error_message + + def __str__(self) -> str: + message = self.error_message + + generic_messages = ( + "Internal Server Error", + "Bad Request", + "Not Found", + "Unauthorized", + "Forbidden", + "Service Unavailable", + "Gateway Timeout", + "Unknown error", + "Validation Failed", + ) + + if ( + not message or (message in generic_messages and len(message) < 30) + ) and self.response_body is not None: + try: + import json + + body = self.response_body.decode(errors="ignore") + data = json.loads(body) + extracted_msg = None + + if isinstance(data, dict): + if "error" in data: + err = data.get("error") + if isinstance(err, dict) and "message" in err: + extracted_msg = err.get("message") + elif isinstance(err, str): + extracted_msg = err + elif "message" in data: + extracted_msg = data.get("message") + elif "execution_message" in data: + extracted_msg = data.get("execution_message") + elif "detail" in data: + extracted_msg = data.get("detail") + elif "errors" in data: + errs = data.get("errors") + if isinstance(errs, list) and errs: + parts = [] + for item in errs: + if isinstance(item, dict) and "message" in item: + parts.append(str(item.get("message"))) + else: + parts.append(str(item)) + extracted_msg = "; ".join(parts) + elif isinstance(errs, dict): + if "children" in errs: + validation_errors = [] + children = errs.get("children", {}) + for field, field_errors in children.items(): + if ( + isinstance(field_errors, dict) + and "errors" in field_errors + ): + field_error_list = field_errors.get( + "errors", [] + ) + if field_error_list: + for err_msg in field_error_list: + validation_errors.append( + f"{field}: {err_msg}" + ) + if validation_errors: + base_msg = data.get("message", "Validation Failed") + extracted_msg = ( + f"{base_msg}: {'; '.join(validation_errors)}" + ) + else: + parts = [] + for key, value in errs.items(): + if value: + parts.append(f"{key}: {value}") + if parts: + extracted_msg = "; ".join(parts) + elif isinstance(errs, str): + extracted_msg = errs + + if extracted_msg and extracted_msg not in generic_messages: + message = str(extracted_msg) + elif not message: + message = body[:500] + + except json.JSONDecodeError: + if not message or message in generic_messages: + try: + decoded = self.response_body.decode(errors="ignore")[:500] + if decoded and decoded not in generic_messages: + message = decoded + except Exception: + pass + except Exception: + pass + + if not message: + message = "Unknown error" + + message = " ".join(message.split()) + + if self.response_code is not None: + return f"{self.response_code}: {message}" + return f"{message}" + + +class OpenAEVAuthenticationError(OpenAEVError): + """Authentication or authorization failure. + + Raised when credentials are invalid, tokens are expired, or the + authenticated user lacks permission for the requested operation. + """ + + pass + + +class OpenAEVHttpError(OpenAEVError): + """Generic HTTP transport error. + + Raised on unexpected HTTP responses. HTTP clients (e.g. pyoaev's + ``on_http_error`` decorator) wrap these into operation-specific types. + """ + + pass + + +class OpenAEVParsingError(OpenAEVError): + """Response deserialization failure. + + Raised when an API response cannot be parsed into the expected format + (malformed JSON, unexpected schema, encoding issues). + """ + + pass + + +class RedirectError(OpenAEVError): + """Unexpected HTTP redirect encountered.""" + + pass + + +class OpenAEVHeadError(OpenAEVError): + """HEAD request failure.""" + + pass + + +class OpenAEVGetError(OpenAEVError): + """GET request failure.""" + + pass + + +class OpenAEVUpdateError(OpenAEVError): + """PUT/PATCH update request failure.""" + + pass + + +class OpenAEVListError(OpenAEVError): + """List/pagination request failure.""" + + pass + + +class OpenAEVCreateError(OpenAEVError): + """POST creation request failure.""" + + pass + + +class OpenAEVDeleteError(OpenAEVError): + """DELETE request failure.""" + + pass + + +class SignatureTransmissionError(OpenAEVError): + """Signature delivery failure. + + Raised when signatures cannot be transmitted to the platform. + Validation rejected them, 4xx slammed the door, or retries ran dry. + """ + + pass + + +class ConfigurationError(OpenAEVError): + """Configuration loading or validation failure. + + Raised when required configuration keys are missing, values fail + validation, or configuration sources are unreachable. + """ + + pass diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py new file mode 100644 index 00000000..5ff10fe9 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py @@ -0,0 +1,71 @@ +"""Common ports: the behaviour the public adapters expect from the core.""" + +import logging +import ssl +from typing import Any, Protocol, runtime_checkable + +__all__ = [ + "CertificatesPort", + "JsonEncodingPort", + "LoggingPort", +] + + +@runtime_checkable +class JsonEncodingPort(Protocol): + """Encodes the objects the standard JSON encoder does not support.""" + + def encode_default(self, o: Any) -> Any: + """JSON-serializable form of ``o``; TypeError when unsupported.""" + + +@runtime_checkable +class LoggingPort(Protocol): + """Configures and feeds the standard logging machinery.""" + + def setup_logging_config( + self, + level: int, + json_logging: bool, + formatter_class: type[logging.Formatter], + ) -> None: + """Configure the root logger, as JSON with ``formatter_class`` or text.""" + + def get_logger(self, name: str) -> logging.Logger: + """The named standard logger.""" + + def prepare_meta(self, meta: Any) -> dict[str, Any] | None: + """The ``extra`` mapping carrying the structured metadata.""" + + def set_library_level(self, lib: str, log_level: int) -> None: + """Override the log level of a third-party library.""" + + def complete_record( + self, log_record: dict[str, Any], record: logging.LogRecord + ) -> None: + """Add the timestamp and uppercase level fields of a JSON record.""" + + +@runtime_checkable +class CertificatesPort(Protocol): + """Loads certificates, files or in-memory PEM data, into SSL contexts.""" + + def data_to_temp_file(self, data: str) -> str: + """Path of a new owner-only temporary file holding ``data``.""" + + def is_memory_certificate(self, certificate: str) -> bool: + """Whether ``certificate`` is PEM data rather than a file path.""" + + def ssl_cert_chain( + self, + ssl_context: ssl.SSLContext, + cert_data: str | None, + key_data: str | None, + passphrase: str | None, + ) -> None: + """Load a certificate chain into ``ssl_context``.""" + + def ssl_verify_locations( + self, ssl_context: ssl.SSLContext, certdata: str | None + ) -> None: + """Load CA verification locations into ``ssl_context``.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/protocols.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/protocols.py new file mode 100644 index 00000000..97964ebb --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/protocols.py @@ -0,0 +1,152 @@ +"""Consumer-facing protocols: the API client, daemons and loggers. + +Extension SDKs and consumers type-hint against these protocols instead of a +concrete implementation. +""" + +from typing import Any, Callable, Protocol, runtime_checkable + +__all__ = ["AppLoggerProtocol", "BaseClient", "DaemonProtocol"] + + +@runtime_checkable +class AppLoggerProtocol(Protocol): + """Behavioral contract for SDK logging services. + + Any logger implementation used by SDK consumers must satisfy this + protocol. The built-in AppLogger class implements it. + + The __call__ method allows switching the logger name (factory pattern): + >>> log = AppLogger(logging.DEBUG) + >>> module_log = log("my_module") # returns self with new name + """ + + def __call__(self, name: str) -> "AppLoggerProtocol": + """Switch logger name and return self.""" + ... + + def debug(self, message: str, meta: Any = None) -> None: + """Log at DEBUG level.""" + ... + + def info(self, message: str, meta: Any = None) -> None: + """Log at INFO level.""" + ... + + def warning(self, message: str, meta: Any = None) -> None: + """Log at WARNING level.""" + ... + + def error(self, message: str, meta: Any = None) -> None: + """Log at ERROR level with traceback.""" + ... + + @staticmethod + def setup_logger_level(lib: str, log_level: int) -> None: + """Override log level for a third-party library.""" + ... + + +@runtime_checkable +class BaseClient(Protocol): + """Protocol defining the behavioral contract for an OpenAEV HTTP client. + + Any class implementing all methods and properties defined here + will pass isinstance(obj, BaseClient) checks at runtime. + + IMPORTANT: Properties are used (not bare attributes) to ensure + isinstance checks work on Python 3.11 where @runtime_checkable + only inspects methods, not data attributes. + """ + + @property + def per_page(self) -> int: + """Number of items per page requested from the API.""" + + @property + def pagination(self) -> str: + """Pagination mode requested from the API.""" + + @property + def order_by(self) -> str: + """Sort key requested from the API.""" + + @property + def url(self) -> str: + """Base URL of the OpenAEV API.""" + + @property + def headers(self) -> dict[str, str]: + """HTTP headers sent with every request.""" + + @property + def ssl_verify(self) -> bool | str: + """TLS verification flag or CA bundle path.""" + + @property + def timeout(self) -> int | float | None: + """Request timeout in seconds.""" + + @property + def tenant_id(self) -> str | None: + """Tenant scoping the API requests, if any.""" + + @property + def session(self) -> Any: + """The underlying HTTP session (e.g. requests.Session).""" + ... + + @property + def backend(self) -> Any: + """The HTTP backend implementation handling transport.""" + ... + + def http_head(self, path: str, **kwargs: Any) -> Any: + """Send a HEAD request.""" + + def http_get(self, path: str, **kwargs: Any) -> Any: + """Send a GET request.""" + + def http_post(self, path: str, **kwargs: Any) -> Any: + """Send a POST request.""" + + def http_patch(self, path: str, **kwargs: Any) -> Any: + """Send a PATCH request.""" + + def http_put(self, path: str, **kwargs: Any) -> Any: + """Send a PUT request.""" + + def http_delete(self, path: str, **kwargs: Any) -> Any: + """Send a DELETE request.""" + + def http_list(self, path: str, **kwargs: Any) -> Any: + """Send a paginated GET request.""" + + +@runtime_checkable +class DaemonProtocol(Protocol): + """Protocol defining the behavioral contract for a daemon runtime. + + Any class implementing all methods defined here will pass + isinstance(obj, DaemonProtocol) checks at runtime. + + The three methods capture the public lifecycle surface shared by + CollectorDaemon and injector daemon runtimes: + + - ``start()`` — run setup then enter the main loop. + - ``set_callback(callback)`` — configure the periodic callback. + - ``get_id()`` — return the daemon instance identifier. + + IMPORTANT: Only public methods are part of this contract. + Implementation details (_setup, _start_loop, _try_callback) + are excluded — they belong to the concrete daemon class. + """ + + def start(self) -> None: + """Run the daemon setup, then its main loop.""" + + def set_callback(self, callback: Callable[..., Any]) -> None: + """Configure the callback run by the main loop.""" + + def get_id(self) -> str | None: + """Return the daemon instance identifier.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/security_domains.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/security_domains.py new file mode 100644 index 00000000..eb201bd2 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/security_domains.py @@ -0,0 +1,45 @@ +"""Security domain classification types. + +Defines the taxonomy of security domains used to categorize signatures, +injections, and detection scenarios across the OpenAEV platform. + +Each domain carries a display name and color for UI rendering. + +Example: + >>> from xtm_oaev_sdk import SecurityDomains + >>> domain = SecurityDomains.ENDPOINT + >>> domain.value["domain_name"] + 'Endpoint' + >>> domain.value["domain_color"] + '#389CFF' +""" + +from enum import Enum + +__all__ = ["SecurityDomains"] + + +class SecurityDomains(Enum): + """Security domain classification for signatures and injections. + + Each member maps to a dict with: + - ``domain_name``: Human-readable display name. + - ``domain_color``: Hex color code for UI rendering. + """ + + ENDPOINT = {"domain_name": "Endpoint", "domain_color": "#389CFF"} + NETWORK = {"domain_name": "Network", "domain_color": "#009933"} + WEB_APP = {"domain_name": "Web App", "domain_color": "#FF9933"} + EMAIL_INFILTRATION = { + "domain_name": "E-mail Infiltration", + "domain_color": "#FF6666", + } + DATA_EXFILTRATION = {"domain_name": "Data Exfiltration", "domain_color": "#9933CC"} + URL_FILTERING = {"domain_name": "URL Filtering", "domain_color": "#66CCFF"} + CLOUD = {"domain_name": "Cloud", "domain_color": "#9999CC"} + TABLE_TOP = {"domain_name": "Tabletop", "domain_color": "#FFCC33"} + ARTIFICIAL_INTELLIGENCE = { + "domain_name": "Artificial Intelligence", + "domain_color": "#7C4DFF", + } + TOCLASSIFY = {"domain_name": "To classify", "domain_color": "#FFFFFF"} diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py new file mode 100644 index 00000000..0e41fca0 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py @@ -0,0 +1,48 @@ +"""Configuration seam: the contract between the public adapters and the core. + +Closed: it imports nothing from ``xtm_oaev_sdk._core`` nor +``xtm_oaev_sdk.public``. Any change to these contracts is a breaking change for +one side of the boundary and must bump :data:`SEAM_VERSION`. +""" + +from xtm_oaev_sdk.seam.configuration.models import ( + CONFIGURATION_TYPES, + BaseConfigModel, + ConfigLoaderCollector, + ConfigLoaderOAEV, + ConfigurationHint, + HttpUrlToString, + LogLevelToLower, + TimedeltaInSeconds, +) +from xtm_oaev_sdk.seam.configuration.ports import ( + ConfigurationProtocol, + ConfigurationResolverPort, + ConfigurationSourcesPort, + ConnectorSchemaPort, + SettingsSourcesPort, + TruthinessPort, +) + +#: Version of the configuration seam. Bump on any contract change. +SEAM_VERSION = "1.0.0" + +__all__ = [ + "SEAM_VERSION", + # Data contracts + "BaseConfigModel", + "CONFIGURATION_TYPES", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "ConfigurationHint", + "HttpUrlToString", + "LogLevelToLower", + "TimedeltaInSeconds", + # Ports + "ConfigurationProtocol", + "ConfigurationResolverPort", + "ConfigurationSourcesPort", + "ConnectorSchemaPort", + "SettingsSourcesPort", + "TruthinessPort", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/models.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/models.py new file mode 100644 index 00000000..6a25398e --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/models.py @@ -0,0 +1,157 @@ +"""Configuration data contracts: hints, base models and serialization types. + +Settings models are frozen, allow extra fields for forward compatibility and +validate their defaults. Source priority when loading settings (highest to +lowest): environment variables, YAML config file, .env file, defaults. +""" + +from abc import ABC +from datetime import timedelta +from typing import Annotated, Any, Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, PlainSerializer + +__all__ = [ + "BaseConfigModel", + "CONFIGURATION_TYPES", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "ConfigurationHint", + "HttpUrlToString", + "LogLevelToLower", + "TimedeltaInSeconds", +] + +CONFIGURATION_TYPES = str | int | bool | Any | None +"""Union type for configuration values.""" + + +class ConfigurationHint(BaseModel): + """An individual configuration hint defining where a value can be found. + + Hints specify the resolution strategy for a configuration key: + which environment variable to check, which file path to follow, + and what default to use if no source provides a value. + + The model is mutable: ``Configuration.set()`` assigns ``data`` in place. + + Args: + data: Override value. When set, this value is returned directly. + env: Environment variable name to read. + file_path: JSON path (nested keys) in the config file. + is_number: Whether to interpret the resolved value as an integer. + default: Fallback value when no source provides one. + + Example: + >>> hint = ConfigurationHint(env="LOG_LEVEL", default="info") + >>> hint.data = "debug" # override + """ + + data: CONFIGURATION_TYPES = Field(default=None) + env: str | None = Field(default=None) + file_path: list[str] | None = Field(default=None) + is_number: bool | None = Field(default=False) + default: CONFIGURATION_TYPES = Field(default=None) + + +class BaseConfigModel(BaseModel, ABC): + """Abstract base for frozen configuration models. + + All configuration models inherit from this class to ensure + immutability after initialization and allow extra fields + for forward compatibility. + + Example: + >>> class MyConfig(BaseConfigModel): + ... url: str + ... timeout: int = 30 + >>> cfg = MyConfig(url="http://localhost") + >>> cfg.url = "other" # raises ValidationError (frozen) + """ + + model_config = ConfigDict(extra="allow", frozen=True, validate_default=True) + + +# Annotated type aliases for common configuration field patterns + +LogLevelToLower = Annotated[ + Literal["debug", "info", "warn", "error"], + PlainSerializer(lambda v: "".join(v), return_type=str), +] +"""Log level literal type that serializes to lowercase string.""" + +HttpUrlToString = Annotated[HttpUrl, PlainSerializer(str, return_type=str)] +"""HTTP URL type that serializes to plain string.""" + +TimedeltaInSeconds = Annotated[ + timedelta, PlainSerializer(lambda v: int(v.total_seconds()), return_type=int) +] +"""Timedelta type that serializes to integer seconds.""" + + +class ConfigLoaderOAEV(BaseConfigModel): + """OpenAEV platform connection configuration. + + Args: + url: The OpenAEV platform URL. + token: Authentication token for the platform API. + tenant_id: Optional tenant identifier for multi-tenant deployments. + """ + + url: HttpUrlToString = Field( + description="The OpenAEV platform URL.", + ) + token: str = Field( + description="The token for the OpenAEV platform.", + ) + tenant_id: UUID | None = Field( + default=None, + description="Identifier of the tenant within the OpenAEV platform. " + "Used in multi-tenant environments to scope API requests and ensure data isolation between different tenants.", + ) + + +class ConfigLoaderCollector(BaseConfigModel): + """Base collector configuration settings. + + Common settings shared by all collectors including identification, + logging level, scheduling period, and UI customization. + + Args: + id: Unique identifier of the collector instance. + name: Human-readable collector name. + log_level: Logging verbosity (debug/info/warn/error). + period: Duration between scheduled runs (ISO 8601). + icon_filepath: Path to the collector's icon file. + """ + + id: str = Field(description="ID of the collector.") + name: str = Field(description="Name of the collector") + log_level: LogLevelToLower | None = Field( + default="error", + description="Determines the verbosity of the logs.", + ) + period: timedelta | None = Field( + default=timedelta(minutes=1), + description="Duration between two scheduled runs of the collector (ISO 8601 format).", + ) + icon_filepath: str | None = Field( + description="Path to the icon file of the collector.", + ) + author: str | None = Field( + default=None, + description="Optional author override for the connector's payloads and contracts. " + "When absent, the platform attributes them to the connector's name.", + ) + platform_description: str | None = Field( + default=None, + description="Description applied to the security platform auto-created by the " + "collector, so its read-only card is not empty in the platform UI.", + ) + platform_tags: list[str] | str | None = Field( + default=None, + description="Tag names applied to the security platform auto-created by the " + "collector. Accepts a list of names or a single comma-separated string " + "(the usual shape when configured through an environment variable).", + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/ports.py new file mode 100644 index 00000000..d5360bec --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/ports.py @@ -0,0 +1,140 @@ +"""Configuration ports: the behaviour the public layer expects from the core. + +``ConfigurationProtocol`` is the consumer-facing contract every configuration +service satisfies. The other protocols are the ports the public adapters +delegate to; the core implements them. +""" + +from typing import Any, Protocol, runtime_checkable + +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource +from xtm_oaev_sdk.seam.configuration.models import ( + CONFIGURATION_TYPES, + ConfigurationHint, +) + +__all__ = [ + "ConfigurationProtocol", + "ConfigurationResolverPort", + "ConfigurationSourcesPort", + "ConnectorSchemaPort", + "SettingsSourcesPort", + "TruthinessPort", +] + + +@runtime_checkable +class ConfigurationProtocol(Protocol): + """Behavioral contract for configuration services. + + Any configuration implementation used by SDK consumers must satisfy + this protocol. The built-in Configuration class implements it. + + Methods: + get: Retrieve a typed configuration value by key. + set: Override a configuration value at runtime. + schema: Generate the connector configuration JSON schema. + """ + + def get(self, config_key: str) -> CONFIGURATION_TYPES: + """Get configuration value by key.""" + ... + + def set(self, config_key: str, value: CONFIGURATION_TYPES) -> None: + """Set a runtime override for a configuration key.""" + ... + + def schema(self) -> dict[str, Any]: + """Generate the configuration JSON schema.""" + ... + + +@runtime_checkable +class TruthinessPort(Protocol): + """Interprets the strings signalling a boolean value.""" + + def is_truthy(self, value: str) -> bool: + """Whether ``value`` signals True.""" + + def is_falsy(self, value: str) -> bool: + """Whether ``value`` signals False.""" + + +@runtime_checkable +class ConfigurationSourcesPort(Protocol): + """Reads raw configuration values from their backing stores.""" + + def from_environment(self, env_var: str | None) -> str | None: + """Value of an environment variable, None when unset or unnamed.""" + + def from_dictionary( + self, config_key_path: list[str] | None, source_dict: dict[str, Any] + ) -> str | None: + """Value at a two-level ``[section, key]`` path of a nested dict.""" + + +@runtime_checkable +class ConfigurationResolverPort(Protocol): + """Resolves configuration values from hints and loaded values.""" + + def build_hints( + self, config_hints: dict[str, dict[str, Any] | str] + ) -> dict[str, ConfigurationHint]: + """Hints from their definitions (a dict, or a bare default value).""" + + def load_values( + self, config_values: dict[str, Any] | None, config_file_path: str + ) -> dict[str, Any]: + """Given values merged with the YAML file contents, when it exists.""" + + def resolve( + self, hint: ConfigurationHint, values: dict[str, Any] + ) -> CONFIGURATION_TYPES: + """Typed value of a hint: override, then env var, file, default.""" + + def override( + self, + hints: dict[str, ConfigurationHint], + config_key: str, + value: CONFIGURATION_TYPES, + ) -> None: + """Set a runtime override, creating the hint when unknown.""" + + def generate_schema( + self, base_model: BaseSettings | None, schema_generator: type + ) -> dict[str, Any]: + """Connector configuration JSON schema of ``base_model``.""" + + +@runtime_checkable +class ConnectorSchemaPort(Protocol): + """Transforms a pydantic JSON schema into a connector manifest schema.""" + + def dereference_schema(self, schema_with_refs: dict[str, Any]) -> dict[str, Any]: + """Schema with every internal ``$ref`` inlined.""" + + def flatten_config_loader_schema( + self, root_schema: dict[str, Any] + ) -> dict[str, Any]: + """Schema with every config variable at root level.""" + + def filter_schema(self, schema: dict[str, Any]) -> dict[str, Any]: + """Schema without the attributes generated by the platform.""" + + def finalize(self, json_schema: dict[str, Any], dialect: str) -> dict[str, Any]: + """Flat, dereferenced, filtered manifest schema.""" + + def collapse_nullable(self, inner_json_schema: dict[str, Any]) -> dict[str, Any]: + """Schema of a nullable field, without ``anyOf``.""" + + +@runtime_checkable +class SettingsSourcesPort(Protocol): + """Chooses where settings are loaded from, in priority order.""" + + def select_sources( + self, + settings_cls: type[BaseSettings], + env_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Environment variables first, then the YAML or .env file found.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py new file mode 100644 index 00000000..8b1b31da --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py @@ -0,0 +1,94 @@ +"""Contracts seam: the contract between the public adapters and the core. + +Closed: it imports nothing from ``xtm_oaev_sdk._core`` nor +``xtm_oaev_sdk.public``. Any change to these contracts is a breaking change for +one side of the boundary and must bump :data:`SEAM_VERSION`. +""" + +from xtm_oaev_sdk.seam.contracts.models import ( + ContractAiTarget, + ContractAsset, + ContractAssetGroup, + ContractAttachment, + ContractCardinality, + ContractCardinalityElement, + ContractCheckbox, + ContractConfig, + ContractData, + ContractElement, + ContractExpectations, + ContractExpectationType, + ContractFieldKey, + ContractFieldType, + ContractOutputElement, + ContractOutputType, + ContractPayload, + ContractReferencedCredential, + ContractSelect, + ContractTeam, + ContractText, + ContractTextArea, + ContractTuple, + ContractVariable, + CredentialType, + Domain, + Expectation, + LinkedFieldModel, + PrimitiveType, + SecurityPlatformType, + SupportedLanguage, + VariableType, +) +from xtm_oaev_sdk.seam.contracts.ports import ( + ContractAssemblyPort, + ContractBuilderProtocol, + ContractCredentialsPort, + ContractSerializerPort, + ContractVariablesPort, +) + +#: Version of the contracts seam. Bump on any contract change. +SEAM_VERSION = "1.0.0" + +__all__ = [ + "SEAM_VERSION", + # Data contracts + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractData", + "ContractElement", + "ContractExpectationType", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "CredentialType", + "Domain", + "Expectation", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableType", + # Ports + "ContractAssemblyPort", + "ContractBuilderProtocol", + "ContractCredentialsPort", + "ContractSerializerPort", + "ContractVariablesPort", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/models.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/models.py new file mode 100644 index 00000000..2a3b39aa --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/models.py @@ -0,0 +1,599 @@ +"""Contract data contracts: field types, elements and contract definitions. + +The contract model hierarchy injectors and collectors use to declare their +input fields, output types, expectations and template variables. Elements own +their invariants (derived ``type``, cardinality, predefined expectations). +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Optional + +__all__ = [ + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractData", + "ContractElement", + "ContractExpectationType", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "CredentialType", + "Domain", + "Expectation", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableType", +] + + +class ContractCardinality(str, Enum): + """Field cardinality in a contract definition. + + Attributes: + One: Single value expected. + Multiple: Multiple values (list) expected. + """ + + One: str = "1" + Multiple: str = "n" + + +class VariableType(str, Enum): + """Variable type classification for contract template variables. + + Attributes: + String: Simple string value. + Object: Nested object with children. + """ + + String: str = "String" + Object: str = "Object" + + +@dataclass +class ContractVariable: + """A template variable available for injection into contract fields. + + Args: + key: Variable identifier (dot-notation for nested, e.g. "user.email"). + label: Human-readable description of the variable. + type: Variable type (String or Object). + cardinality: Whether this variable holds one or many values. + children: Nested child variables (for Object type). + """ + + key: str + label: str + type: VariableType + cardinality: ContractCardinality + children: list["ContractVariable"] = field(default_factory=list) + + +class CredentialType(str, Enum): + """Credential reference types shared with the OpenAEV platform. + + The Python mirror of the OpenAEV backend credential-type enum used by + credential secret references. Values must stay label-for-label in sync with + the platform so contracts can declare credential references without any + translation layer. + """ + + IDENTITY = "IDENTITY" + CLOUD_AWS = "CLOUD_AWS" + CLOUD_AZURE = "CLOUD_AZURE" + CLOUD_GCP = "CLOUD_GCP" + + +class SupportedLanguage(str, Enum): + """Supported languages for contract label localization.""" + + fr: str = "fr" + en: str = "en" + + +class ContractFieldType(str, Enum): + """Available field types for contract input definitions.""" + + Text: str = "text" + Number: str = "number" + Tuple: str = "tuple" + Checkbox: str = "checkbox" + Textarea: str = "textarea" + Select: str = "select" + Article: str = "article" + Challenge: str = "challenge" + DependencySelect: str = "dependency-select" + Attachment: str = "attachment" + Team: str = "team" + Expectation: str = "expectation" + Asset: str = "asset" + AssetGroup: str = "asset-group" + AiTarget: str = "ai-target" + Payload: str = "payload" + CredentialReference: str = "credential-reference" + + +class ContractFieldKey(str, Enum): + """Predefined field keys for special contract fields.""" + + Asset: str = "assets" + AssetGroup: str = "asset_groups" + CredentialReference: str = "credential_reference" + + +class ContractOutputType(str, Enum): + """Available output types for contract result declarations.""" + + Text: str = "text" + ActionOutput: str = "action_output" + Number: str = "number" + Port: str = "port" + PortsScan: str = "portscan" + IPv4: str = "ipv4" + IPv6: str = "ipv6" + CVE: str = "cve" + Asset: str = "asset" + Credentials: str = "credentials" + Username: str = "username" + Share: str = "share" + File: str = "file" + AdminUsername: str = "admin_username" + Group: str = "group" + Computer: str = "computer" + PasswordPolicy: str = "password_policy" + Delegation: str = "delegation" + Sid: str = "sid" + Vulnerability: str = "vulnerability" + AccountWithPasswordNotRequired: str = "account_with_password_not_required" + ExpectationSignature: str = "expectation_signature" + AsreproastableAccount: str = "asreproastable_account" + KerberoastableAccount: str = "kerberoastable_account" + + +class PrimitiveType(str, Enum): + """Semantic type of a contract argument's value, independent of how the field + renders in the UI (see ``ContractFieldType``). Mirrors the openaev platform's + own ``PrimitiveType`` label-for-label so a contract pushed from here needs no + translation on the other end. + """ + + AccountWithPasswordNotRequired: str = "account_with_password_not_required" + ActionOutput: str = "action_output" + AdminUsername: str = "admin_username" + AsreproastableAccount: str = "asreproastable_account" + AssetGroupId: str = "asset_group_id" + AssetId: str = "asset_id" + ComputerName: str = "computer_name" + CVE: str = "cve" + DelegationAccount: str = "delegation_account" + Document: str = "document" + Domain: str = "domain" + FileName: str = "file_name" + FilePath: str = "file_path" + GroupName: str = "group_name" + Hash: str = "hash" + Host: str = "host" + IPv4: str = "ipv4" + IPv6: str = "ipv6" + IpSubnet: str = "ip_subnet" + KerberoastableAccount: str = "kerberoastable_account" + Key: str = "key" + Number: str = "number" + Password: str = "password" + Permissions: str = "permissions" + Port: str = "port" + Service: str = "service" + Severity: str = "severity" + ShareName: str = "share_name" + SID: str = "sid" + TargetedAsset: str = "targeted-asset" + Text: str = "text" + Username: str = "username" + Value: str = "value" + VulnerabilityName: str = "vulnerability_name" + VulnerabilityStatus: str = "vulnerability_status" + + +class ContractExpectationType(str, Enum): + """Expectation type classification for contract fields. + + Defines the 8 expectation categories available in contract definitions. + Renamed from ExpectationType to avoid collision with signatures module. + """ + + text: str = "TEXT" + document: str = "DOCUMENT" + article: str = "ARTICLE" + challenge: str = "CHALLENGE" + manual: str = "MANUAL" + prevention: str = "PREVENTION" + detection: str = "DETECTION" + vulnerability: str = "VULNERABILITY" + + +class SecurityPlatformType(str, Enum): + """Categories of security platform expected to fulfil a technical expectation. + + When an expectation declares one or more of these, the platform focuses the + technical (DETECTION / PREVENTION / VULNERABILITY) result on collectors of + those types only (instead of every connected security platform). An empty + list means "any security platform". + """ + + EDR: str = "EDR" + XDR: str = "XDR" + SIEM: str = "SIEM" + SOAR: str = "SOAR" + NDR: str = "NDR" + ISPM: str = "ISPM" + LLM_FIREWALL: str = "LLM_FIREWALL" + AI_GATEWAY: str = "AI_GATEWAY" + + +@dataclass +class Expectation: + """An expectation entry within a contract expectations field. + + Args: + expectation_type: The category of expectation. + expectation_name: Human-readable name. + expectation_description: Detailed description. + expectation_score: Score weight for this expectation. + expectation_expectation_group: Whether this belongs to a group. + """ + + expectation_type: ContractExpectationType + expectation_name: str + expectation_description: str + expectation_score: int + expectation_expectation_group: bool + expectation_is_predefined: bool = False + expectation_is_multi_selectable: bool = False + # Security platform types expected to fulfil this expectation. Empty = any + # platform (unchanged behaviour). Typically set for technical expectations + # (DETECTION / PREVENTION / VULNERABILITY), left empty for MANUAL ones. + expectation_expected_security_platform_types: list[SecurityPlatformType] = field( + default_factory=list + ) + + +@dataclass +class LinkedFieldModel: + """A linked field reference within a contract element.""" + + key: str + type: ContractFieldType + + +@dataclass +class ContractElement(ABC): + """Abstract base for all contract field elements. + + Subclass this to define new field types. Each subclass must + implement the ``get_type`` property returning the field type string. + + Args: + key: Unique field identifier. + label: Human-readable field label. + mandatory: Whether this field is required. + readOnly: Whether this field is read-only in the UI. + """ + + key: str + label: str + type: str = field(default="", init=False) + mandatoryGroups: list[str] = field(default_factory=list) + mandatoryConditionFields: list[str] = field(default_factory=list) + mandatoryConditionValues: dict[str, Any] = field(default_factory=list) + visibleConditionFields: list[str] = field(default_factory=list) + visibleConditionValues: dict[str, Any] = field(default_factory=list) + linkedFields: list[str] = field(default_factory=list) + mandatory: bool = False + readOnly: bool = False + # The argument's chaining/semantic type (``PrimitiveType``) — matches the + # platform's own ``argumentType`` label-for-label. ``None`` for fields with + # no established type; the platform normalizes a missing value to ``Text``. + argumentType: PrimitiveType | None = None + + @property + @abstractmethod + def get_type(self) -> str: + """Return the contract field type string.""" + pass + + def __post_init__(self) -> None: + self.type = self.get_type + + +@dataclass +class ContractCardinalityElement(ContractElement, ABC): + """Contract element with cardinality (single vs multiple values).""" + + cardinality: str = ContractCardinality.One + defaultValue: list[str] = field(default_factory=list) + + +@dataclass +class ContractOutputElement(ABC): + """An output field declaration in a contract. + + Args: + type: Output type (from ContractOutputType). + field: Field identifier. + labels: Localized labels for the output. + isFindingCompatible: Whether output maps to a finding. + isMultiple: Whether multiple values are produced. + """ + + type: str + field: str + labels: list[str] + isFindingCompatible: bool + isMultiple: bool + + +@dataclass +class ContractConfig: + """Contract display configuration. + + Args: + type: Contract type identifier. + expose: Whether this contract is visible in the UI. + label: Localized labels. + color_dark: Hex color for dark theme. + color_light: Hex color for light theme. + """ + + type: str + expose: bool + label: dict[SupportedLanguage, str] + color_dark: str + color_light: str + + +@dataclass +class Domain: + """A domain classification for a contract.""" + + domain_id: str + domain_name: str + domain_color: str + domain_created_at: str + domain_updated_at: str + + +@dataclass +class ContractData: + """Data of a complete contract definition for an injector or collector. + + Contains fields (inputs), outputs, configuration, and metadata + for registering the contract with the OpenAEV platform. + + Args: + contract_id: Unique contract identifier. + label: Localized contract labels. + fields: Input field definitions. + outputs: Output field declarations. + config: Display configuration. + manual: Whether this is a manual-execution contract. + variables: Template variables available for injection. + """ + + contract_id: str + label: dict[SupportedLanguage, str] + fields: list[ContractElement] + outputs: list[ContractOutputElement] + config: ContractConfig + manual: bool + variables: list[ContractVariable] = field(default_factory=list) + contract_attack_patterns_external_ids: list[str] = field(default_factory=list) + contract_vulnerability_external_ids: list[str] = field(default_factory=list) + is_atomic_testing: bool = True + platforms: list[str] = field(default_factory=list) + external_id: str | None = None + domains: list[Domain] | None = None + + def add_attack_pattern(self, var: str) -> None: + """Append an attack pattern external ID to this contract.""" + self.contract_attack_patterns_external_ids.append(var) + + def add_vulnerability(self, var: str) -> None: + """Append a vulnerability external ID to this contract.""" + self.contract_vulnerability_external_ids.append(var) + + def add_variable(self, var: ContractVariable) -> None: + """Add a custom template variable to this contract.""" + self.variables.append(var) + + +@dataclass +class ContractTeam(ContractCardinalityElement): + """Team selection field.""" + + @property + def get_type(self) -> str: + return ContractFieldType.Team.value + + +@dataclass +class ContractText(ContractCardinalityElement): + """Text input field with optional default value.""" + + defaultValue: str = "" + + @property + def get_type(self) -> str: + return ContractFieldType.Text.value + + +@dataclass +class ContractTuple(ContractCardinalityElement): + """Tuple (multi-value) input field with optional file attachment.""" + + def __post_init__(self) -> None: + super().__post_init__() + self.cardinality = ContractCardinality.Multiple + + attachmentKey: str | None = None + contractAttachment: bool = False + tupleFilePrefix: str = "file :: " + + @property + def get_type(self) -> str: + return ContractFieldType.Tuple.value + + +@dataclass +class ContractTextArea(ContractCardinalityElement): + """Textarea input field with optional rich text support.""" + + defaultValue: str = "" + richText: bool = False + + @property + def get_type(self) -> str: + return ContractFieldType.Textarea.value + + +@dataclass +class ContractCheckbox(ContractElement): + """Checkbox (boolean) input field.""" + + defaultValue: bool = False + + @property + def get_type(self) -> str: + return ContractFieldType.Checkbox.value + + +@dataclass +class ContractAttachment(ContractCardinalityElement): + """File attachment field.""" + + @property + def get_type(self) -> str: + return ContractFieldType.Attachment.value + + +@dataclass +class ContractReferencedCredential(ContractElement): + """Credential reference field, optionally restricted to one credential type.""" + + key: str = field(default=ContractFieldKey.CredentialReference.value, init=False) + label: str = "Select a credential reference" + mandatory: bool = False + credential_reference_type: Optional[CredentialType] = None + multiple: bool = True + + @property + def get_type(self) -> str: + return ContractFieldType.CredentialReference.value + + +@dataclass +class ContractExpectations(ContractCardinalityElement): + """Expectation selection field with predefined options. + + ``availableExpectations`` is the full set the user may pick from; + ``predefinedExpectations`` is the subset pre-filled by default. When + ``predefinedExpectations`` is not provided explicitly it is derived from + the available expectations flagged ``expectation_is_predefined=True``. + """ + + cardinality: str = ContractCardinality.Multiple + # Full set the user may choose from in the "add expectation" picker. + availableExpectations: list[Expectation] = field(default_factory=list) + # Subset pre-filled by default. ``None`` means "not provided": the list is + # then derived from the available expectations flagged predefined. + predefinedExpectations: list[Expectation] | None = None + + def __post_init__(self) -> None: + super().__post_init__() + if self.predefinedExpectations is None: + self.predefinedExpectations = [ + expectation + for expectation in self.availableExpectations + if expectation.expectation_is_predefined + ] + + @property + def get_type(self) -> str: + return ContractFieldType.Expectation.value + + +@dataclass +class ContractSelect(ContractCardinalityElement): + """Dropdown select field with predefined choices.""" + + choices: dict[str, str] | None = None + + @property + def get_type(self) -> str: + return ContractFieldType.Select.value + + +@dataclass +class ContractAsset(ContractCardinalityElement): + """Asset selection field (key auto-set to 'assets').""" + + key: str = field(default=ContractFieldKey.Asset.value, init=False) + + @property + def get_type(self) -> str: + return ContractFieldType.Asset.value + + +@dataclass +class ContractAssetGroup(ContractCardinalityElement): + """Asset group selection field (key auto-set to 'asset_groups').""" + + key: str = field(default=ContractFieldKey.AssetGroup.value, init=False) + + @property + def get_type(self) -> str: + return ContractFieldType.AssetGroup.value + + +@dataclass +class ContractAiTarget(ContractCardinalityElement): + """Selector for a pre-configured AiTarget asset (LLM endpoint / AI agent under test). + + Renders in the UI as an entity autocomplete (backed by the platform ``/ai_targets`` + options endpoint), analogous to the asset / asset group pickers. The selected value is + the AI target's ``asset_id`` (AI targets are AI_TARGET-category assets), resolved by the + executing injector at run time. Unlike ``ContractAsset`` the key is free-form so a + contract can name the field as it wishes. + """ + + @property + def get_type(self) -> str: + return ContractFieldType.AiTarget.value + + +@dataclass +class ContractPayload(ContractCardinalityElement): + """Payload selection field.""" + + @property + def get_type(self) -> str: + return ContractFieldType.Payload.value diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/ports.py new file mode 100644 index 00000000..29cf6721 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/ports.py @@ -0,0 +1,125 @@ +"""Contract ports: the behaviour the public layer expects from the core. + +``ContractBuilderProtocol`` is the consumer-facing contract of field builders. +The other protocols are the ports the public adapters delegate to; the core +implements them. +""" + +from typing import Any, Protocol, runtime_checkable + +from xtm_oaev_sdk.seam.contracts.models import ( + ContractData, + ContractElement, + ContractOutputElement, + ContractReferencedCredential, + ContractVariable, +) + +__all__ = [ + "ContractAssemblyPort", + "ContractBuilderProtocol", + "ContractCredentialsPort", + "ContractSerializerPort", + "ContractVariablesPort", +] + + +@runtime_checkable +class ContractBuilderProtocol(Protocol): + """Behavioral contract for contract field builders. + + Defines the chainable API for constructing contract field sets. + Consumers can code against this protocol for testability. + """ + + def add_fields(self, fields: list[ContractElement]) -> "ContractBuilderProtocol": + """Add multiple fields at once.""" + ... + + def add_outputs( + self, outputs: list[ContractOutputElement] + ) -> "ContractBuilderProtocol": + """Add multiple output declarations at once.""" + ... + + def mandatory(self, element: ContractElement) -> "ContractBuilderProtocol": + """Add a mandatory field.""" + ... + + def optional(self, element: ContractElement) -> "ContractBuilderProtocol": + """Add an optional field.""" + ... + + def mandatory_group( + self, elements: list[ContractElement] + ) -> "ContractBuilderProtocol": + """Add a group of mutually-required fields.""" + ... + + def build_fields(self) -> list[ContractElement]: + """Return the accumulated fields.""" + ... + + def build_outputs(self) -> list[ContractOutputElement]: + """Return the accumulated outputs.""" + ... + + +@runtime_checkable +class ContractAssemblyPort(Protocol): + """Assembles the field and output lists of a contract.""" + + def concat(self, existing: list[Any], extra: list[Any]) -> list[Any]: + """New list made of ``existing`` followed by ``extra``.""" + + def set_mandatory( + self, element: ContractElement, mandatory: bool + ) -> ContractElement: + """The element, marked mandatory or optional.""" + + def require_group(self, elements: list[ContractElement]) -> list[ContractElement]: + """The elements, mandatory and referencing each other's keys.""" + + +@runtime_checkable +class ContractVariablesPort(Protocol): + """Provides the built-in contract template variables.""" + + def user_variable(self) -> ContractVariable: + """The user template variable with its standard children.""" + + def exercise_variable(self) -> ContractVariable: + """The exercise template variable with its standard children.""" + + def team_variable(self) -> ContractVariable: + """The teams template variable.""" + + def uri_variables(self) -> list[ContractVariable]: + """The platform URI template variables.""" + + def default_variables(self) -> list[ContractVariable]: + """The variables every contract starts with.""" + + +@runtime_checkable +class ContractSerializerPort(Protocol): + """Serializes contracts into platform API payloads.""" + + def to_add_input(self, contract: ContractData, source_id: str) -> dict[str, Any]: + """Payload of the contract creation endpoint.""" + + def to_update_input(self, contract: ContractData) -> dict[str, Any]: + """Payload of the contract update endpoint.""" + + def prepare(self, contracts: list[ContractData]) -> list[dict[str, Any]]: + """Payloads of the bulk contract registration endpoint.""" + + +@runtime_checkable +class ContractCredentialsPort(Protocol): + """Builds the credential-reference fields of a contract.""" + + def build_single_referenced_credential_element( + self, provider_name: str + ) -> ContractReferencedCredential: + """Credential-reference field asking for one credential of the provider's type.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py new file mode 100644 index 00000000..4f2e43c0 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py @@ -0,0 +1,14 @@ +"""Daemons seam: the contract between the public daemon runtime and the core. + +The daemon consumer protocols (``DaemonProtocol``, ``BaseClient``) belong to +the common seam. Closed: it imports nothing from ``xtm_oaev_sdk._core`` nor +``xtm_oaev_sdk.public``. Any change to these contracts must bump +:data:`SEAM_VERSION`. +""" + +from xtm_oaev_sdk.seam.daemons.ports import DaemonRuntimePort, HeartbeatPort + +#: Version of the daemons seam. Bump on any contract change. +SEAM_VERSION = "1.0.0" + +__all__ = ["SEAM_VERSION", "DaemonRuntimePort", "HeartbeatPort"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/ports.py new file mode 100644 index 00000000..fa79cb96 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/ports.py @@ -0,0 +1,62 @@ +"""Daemon ports: the behaviour the public daemon runtime expects from the core.""" + +import threading +from typing import Any, Callable, Protocol, runtime_checkable + +from xtm_oaev_sdk.seam.common import BaseClient +from xtm_oaev_sdk.seam.configuration import ConfigurationProtocol + +__all__ = ["DaemonRuntimePort", "HeartbeatPort"] + + +@runtime_checkable +class DaemonRuntimePort(Protocol): + """Runs the shared lifecycle of collector and injector daemons.""" + + def resolve_log_level(self, configuration: ConfigurationProtocol) -> str: + """Log level, from the generic or the legacy collector/injector keys.""" + + def resolve_name(self, configuration: ConfigurationProtocol) -> str: + """Daemon name, from the generic or the legacy collector/injector keys.""" + + def resolve_id(self, configuration: ConfigurationProtocol) -> Any: + """Daemon id, from ``id``, ``collector_id`` or ``injector_id``.""" + + def invoke_callback( + self, callback: Callable[..., Any] | None, daemon: Any, logger: Any + ) -> None: + """Call the callback (injecting the daemon as ``collector`` when asked + for), logging and swallowing any error.""" + + def dump_config_schema_requested(self) -> bool: + """Whether the daemon was started with ``--dump-config-schema``.""" + + def dump_config_schema(self, configuration: ConfigurationProtocol) -> None: + """Print the configuration JSON schema.""" + + def ensure_callback(self, callback: Callable[..., Any] | None) -> None: + """Raise when the daemon has no callback to run.""" + + def default_api_client(self) -> BaseClient: + """The API client used when none is injected: none, the SDK refuses.""" + + +@runtime_checkable +class HeartbeatPort(Protocol): + """Keeps a daemon registered on the platform.""" + + def tenant_of(self, api: Any) -> Any: + """Tenant the API client is scoped to, if any.""" + + def tenant_meta(self, tenant_id: Any) -> dict[str, Any]: + """Log metadata identifying the tenant.""" + + def ping( + self, + api: Any, + config: Any, + ping_type: str, + logger: Any, + exit_event: threading.Event, + ) -> None: + """Register the daemon every 40 seconds until ``exit_event`` is set.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/__init__.py new file mode 100644 index 00000000..e25c9c67 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/__init__.py @@ -0,0 +1,14 @@ +"""Deprecation seam: the contract between the public deprecation adapter and +the core. + +Closed: it imports nothing from ``xtm_oaev_sdk._core`` nor +``xtm_oaev_sdk.public``. Any change to these contracts must bump +:data:`SEAM_VERSION`. +""" + +from xtm_oaev_sdk.seam.deprecation.ports import DeprecatedNames, DeprecationPort + +#: Version of the deprecation seam. Bump on any contract change. +SEAM_VERSION = "1.0.0" + +__all__ = ["SEAM_VERSION", "DeprecatedNames", "DeprecationPort"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/ports.py new file mode 100644 index 00000000..2e149d67 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/ports.py @@ -0,0 +1,23 @@ +"""Deprecation port: serve retired public names without breaking importers.""" + +from collections.abc import Mapping +from typing import Any, Callable, Protocol, runtime_checkable + +__all__ = ["DeprecatedNames", "DeprecationPort"] + +DeprecatedNames = Mapping[str, tuple[str, Any]] +"""``{old_name: (deprecation_message, replacement)}``.""" + + +@runtime_checkable +class DeprecationPort(Protocol): + """Builds PEP 562 module ``__getattr__`` functions for retired names.""" + + def resolve_mapping(self, mapping: DeprecatedNames) -> dict[str, tuple[str, Any]]: + """Validated copy of the mapping; ValueError on a malformed entry.""" + + def build_getattr( + self, resolved: dict[str, tuple[str, Any]], module_name: str | None + ) -> Callable[[str], Any]: + """``__getattr__`` warning (at the caller's line) and returning the + replacement of a retired name, raising AttributeError otherwise.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py new file mode 100644 index 00000000..7bbfdb9b --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py @@ -0,0 +1,50 @@ +"""Port I/O data contract: the versioned wire schema crossing the SDK↔platform seam. + +These models — and the three wire-vocabulary enums they serialise — define +the exact JSON shapes transmitted between the SDK's transport port +(``SignatureTransportProtocol``) and the platform's callback endpoint. This package is the re-export surface; the seam **owns** the +models (``xtm_oaev_sdk.seam.port_io.models``), and the capability's port +(``_core/signatures``) re-exports them back for its historical surface. + +They form a pinnable, anti-breaking boundary: + +- contract unchanged + port (internal) changes → no breaking change +- contract unchanged + adapter (public) changes → retrocompat holds +- contract changes → **breaking change** + +Any modification to the JSON schema of a model in this module is a breaking +change for consumers and must bump :data:`PORT_IO_CONTRACT_VERSION`. + +The seam golden (``test/seam/test_seam_golden.py``) fingerprints every public +object of the seam, this module included, and fails on drift. +""" + +from xtm_oaev_sdk.seam.port_io.models import ( + PORT_IO_CONTRACT_VERSION, + ExecutionDetails, + ExecutionStatus, + ExpectationSignatureGroup, + InjectExecutionActions, + SignatureCallbackPayload, + SignatureExpectationType, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, +) + +__all__ = [ + "PORT_IO_CONTRACT_VERSION", + "ExecutionDetails", + "ExpectationSignatureGroup", + "ExecutionStatus", + "InjectExecutionActions", + "SignatureCallbackPayload", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureValue", + "SignatureExpectationType", + "TargetSignatures", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py new file mode 100644 index 00000000..2bf0b685 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py @@ -0,0 +1,299 @@ +"""The port I/O wire schema — the seam-owned models crossing the SDK↔platform boundary. + +These pydantic models — together with the three wire-vocabulary enums +they serialise (``SignatureExpectationType``, ``ExecutionStatus``, +``InjectExecutionActions``) — are the exact JSON shapes transmitted between +the SDK's transport port (``SignatureTransportProtocol``) and the +platform's callback endpoint. The seam is their **owner**: the capability's +port (``_core/signatures``) imports and re-exports from here, never the +reverse, so the two sides of the boundary can evolve independently. + +- contract unchanged + port (internal) changes → no breaking change +- contract unchanged + adapter (public) changes → retrocompat holds +- contract changes → **breaking change** + +Any modification to the JSON schema of a model in this module is a breaking +change for consumers and must bump :data:`PORT_IO_CONTRACT_VERSION`. + +The seam golden (``test/seam/test_seam_golden.py``) fingerprints every public +object of the seam, this module included, and fails on drift. + +The module is **closed**: at runtime it imports only the standard library and +pydantic. ``ToolOutput`` (from the signatures inbound seam, +``seam/signatures/seam.py``) is referenced for type checking only, by +``ExecutionDetails.post_execution_update``: importing it at runtime would run +``seam/signatures/__init__`` and, through it, ``_core`` — an import cycle back +into this module. The wire contract therefore has no hidden dependency on +internal layout. +""" + +import math +from collections import defaultdict +from datetime import datetime, timezone +from enum import Enum +from typing import TYPE_CHECKING, Any + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + JsonValue, + TypeAdapter, + computed_field, + field_validator, +) + +if TYPE_CHECKING: + from xtm_oaev_sdk.seam.signatures.seam import ToolOutput + +#: Version of the port I/O wire schema. Bump on any breaking contract change. +PORT_IO_CONTRACT_VERSION = "1.0.0" + +# --- Wire vocabulary ----------------------------------------------------------- +# The three enums that appear in the JSON contract below. They live here, in +# the seam, because they are part of the wire schema; the internal +# implementation imports them from here (``_core/signatures/types`` +# re-exports them for its historical import paths). + + +class SignatureExpectationType(str, Enum): + """Expectation categories for a signature.""" + + DETECTION = "DETECTION" + PREVENTION = "PREVENTION" + VULNERABILITY = "VULNERABILITY" + + +class ExecutionStatus(str, Enum): + """Outcome status of a single inject tool execution. + + A ``str`` enum so it serialises to its string form in callback payloads. + Mirrors the platform's execution-status taxonomy label-for-label. + """ + + EXECUTED = "EXECUTED" + EXECUTED_WITH_CLEANUP_FAILURE = "EXECUTED_WITH_CLEANUP_FAILURE" + WARNING = "WARNING" + ACCESS_DENIED = "ACCESS_DENIED" + ERROR = "ERROR" + COMMAND_NOT_FOUND = "COMMAND_NOT_FOUND" + COMMAND_CANNOT_BE_EXECUTED = "COMMAND_CANNOT_BE_EXECUTED" + PREREQUISITE_FAILED = "PREREQUISITE_FAILED" + INVALID_USAGE = "INVALID_USAGE" + TIMEOUT = "TIMEOUT" + INTERRUPTED = "INTERRUPTED" + ASSET_AGENTLESS = "ASSET_AGENTLESS" + AGENT_INACTIVE = "AGENT_INACTIVE" + AGENT_OVERLOADED = "AGENT_OVERLOADED" + INFO = "INFO" + PARTIAL = "PARTIAL" + MAYBE_PREVENTED = "MAYBE_PREVENTED" + MAYBE_PARTIAL_PREVENTED = "MAYBE_PARTIAL_PREVENTED" + + +class InjectExecutionActions(str, Enum): + """Execution action phases used by injection signatures.""" + + PREREQUISITE_CHECK = "prerequisite_check" + PREREQUISITE_EXECUTION = "prerequisite_execution" + CLEANUP_EXECUTION = "cleanup_execution" + COMMAND_EXECUTION = "command_execution" + DNS_RESOLUTION = "dns_resolution" + FILE_EXECUTION = "file_execution" + FILE_DROP = "file_drop" + DATA = "data" + COMPLETE = "complete" + + +class SignatureValue(BaseModel): + """One signature observation: a type and the value it carries.""" + + model_config = ConfigDict(extra="allow") + + signature_type: str + signature_value: JsonValue + + +class ExpectationSignatureGroup(BaseModel): + """Values bound to a single expectation type (DETECTION, PREVENTION, ...).""" + + model_config = ConfigDict(extra="allow") + + expectation_type: str + values: list[SignatureValue] + + @field_validator("expectation_type", mode="after") + @classmethod + def is_expectation_type(cls, value: str) -> str: + SignatureExpectationType(value.upper()) + return value + + +class SignatureTarget(BaseModel): + """Target identity on the wire.""" + + model_config = ConfigDict(extra="forbid") + + agent: str | None = None + asset: str | None = None + asset_group: str | None = None + + +class TargetSignatures(BaseModel): + """A target plus everything observed about it, grouped by expectation.""" + + model_config = ConfigDict(extra="allow") + + signature_target: SignatureTarget + signature_values: list[ExpectationSignatureGroup] + + +class SignaturePayload(BaseModel): + """Inner ``signatures`` body: a list of targets, nothing else.""" + + model_config = ConfigDict(extra="allow") + + targets: list[TargetSignatures] + + +class SignatureOutputStructure(BaseModel): + """Structured output to be serialized as a str in the callback payload yet data has to follow model.""" + + model_config = ConfigDict(populate_by_name=True, extra="forbid") + + expectation_signatures: SignaturePayload + + def normalize_signature_payload(self) -> None: + """ + Regroup signature_values by expectation_type within each target. + """ + normalized_targets: list[TargetSignatures] = [] + + for target in self.expectation_signatures.targets: + if not target.signature_values: + normalized_targets.append(target) + continue + + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + order: list[str] = [] + + for entry in target.signature_values: + if entry.expectation_type not in order: + order.append(entry.expectation_type) + grouped[entry.expectation_type].extend(entry.values) + + normalized_target = TargetSignatures( + signature_target=target.signature_target, + signature_values=[ + ExpectationSignatureGroup( + expectation_type=expectation_type, + values=grouped[expectation_type], + ) + for expectation_type in order + ], + ) + + normalized_targets.append(normalized_target) + + self.expectation_signatures.targets = normalized_targets + + +class ExecutionDetails(BaseModel): + """Helper to wrap the execution-related details for the callback payload.""" + + model_config = ConfigDict(extra="forbid") + + start_time: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + end_time: datetime | None = None + + execution_status: ExecutionStatus | None = None + execution_action: InjectExecutionActions | None = None + + @computed_field + @property + def execution_message(self) -> str: + action = self.execution_action.value if self.execution_action else "unknown" + status = self.execution_action.value if self.execution_action else "unknown" + return f"Current action: {action} - Current status: {status}" + + @computed_field + @property + def execution_duration(self) -> float: + try: + return (self.end_time - self.start_time).total_seconds() + except: + return 0.0 + + def post_execution_update(self, tool_output: "ToolOutput", now: datetime) -> None: + """ + Update execution-related metadata according to tool output and now timestamp. + """ + self.end_time = now + + if tool_output.error_info and tool_output.error_info.exit_code != 0: + self.execution_status = ExecutionStatus.ERROR + if tool_output.error_info.crash_timestamp: + self.end_time = datetime.strptime( + tool_output.error_info.crash_timestamp, "%Y-%m-%dT%H:%M:%SZ" + ) + elif tool_output.timeout_info: + self.execution_status = ExecutionStatus.TIMEOUT + elif tool_output.status == "partial": + self.execution_status = ExecutionStatus.ERROR + else: + self.execution_status = ExecutionStatus.EXECUTED + + self.execution_action = InjectExecutionActions("data") + + +class SignatureCallbackPayload(BaseModel): + """Outer POST envelope validated by ``SignatureApiManager`` before wire transmission.""" + + model_config = ConfigDict(populate_by_name=True, extra="forbid") + + execution_message: str + execution_output_structured: str | None = None + execution_status: str + execution_duration: int | None = None + execution_action: InjectExecutionActions | None = None + + @field_validator("execution_output_structured", mode="after") + @classmethod + def is_proper_signature_output_structure(cls, value: str) -> str | None: + if value is None: + return None + TypeAdapter(SignatureOutputStructure).validate_json(value) + return value + + @classmethod + def build_from_models( + cls, signatures: SignatureOutputStructure, execution_details: ExecutionDetails + ): + """Producing a SignatureCallbackPayload from the data of a SignatureOutputStructure and of a ExecutionDetails.""" + return cls( + execution_message=execution_details.execution_message, + execution_output_structured=signatures.model_dump_json(exclude_none=True), + execution_status=execution_details.execution_status, + execution_duration=( + math.ceil(execution_details.execution_duration) + if execution_details.execution_duration is not None + else None + ), + execution_action=execution_details.execution_action, + ) + + +__all__ = [ + "PORT_IO_CONTRACT_VERSION", + "SignatureExpectationType", + "ExecutionStatus", + "InjectExecutionActions", + "SignatureValue", + "ExpectationSignatureGroup", + "SignatureTarget", + "TargetSignatures", + "SignaturePayload", + "SignatureOutputStructure", + "ExecutionDetails", + "SignatureCallbackPayload", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py new file mode 100644 index 00000000..cafa5e92 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py @@ -0,0 +1,58 @@ +"""Signatures seam: the contract between the public adapters and the core. + +The inbound I/O models (``seam.py``, versioned :data:`SEAM_VERSION` and +schema-pinned), the matching vocabulary and policies, and the signature ports. +The wire models live in :mod:`xtm_oaev_sdk.seam.port_io`. Closed: it imports +nothing from ``xtm_oaev_sdk._core`` nor ``xtm_oaev_sdk.public``. +""" + +from xtm_oaev_sdk.seam.signatures.models import ( + ExtraSignatureData, + MatchTypes, + SignatureMatch, + SignatureType, + SignatureTypes, +) +from xtm_oaev_sdk.seam.signatures.ports import ( + ContainerAddressPort, + MarkerPort, + NetworkTargetsPort, + SignatureEnginePort, + SignatureManagerProtocol, + SignatureTransportProtocol, +) +from xtm_oaev_sdk.seam.signatures.seam import ( + SEAM_VERSION, + CloudInjectorConfig, + ExecutionSignature, + InjectorConfig, + NetworkInjectorConfig, + ToolErrorInfo, + ToolOutput, + ToolTimeoutInfo, +) + +__all__ = [ + "SEAM_VERSION", + # Inbound I/O models (seam.py) + "CloudInjectorConfig", + "ExecutionSignature", + "InjectorConfig", + "NetworkInjectorConfig", + "ToolErrorInfo", + "ToolOutput", + "ToolTimeoutInfo", + # Data contracts + "ExtraSignatureData", + "MatchTypes", + "SignatureMatch", + "SignatureType", + "SignatureTypes", + # Ports + "ContainerAddressPort", + "MarkerPort", + "NetworkTargetsPort", + "SignatureEnginePort", + "SignatureManagerProtocol", + "SignatureTransportProtocol", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/models.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/models.py new file mode 100644 index 00000000..03a62b2d --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/models.py @@ -0,0 +1,142 @@ +"""Signature data contracts: matching vocabulary, matching policies and the +extra signatures merged into the default ones. + +The inbound I/O models live in :mod:`xtm_oaev_sdk.seam.signatures.seam` and the +wire models in :mod:`xtm_oaev_sdk.seam.port_io`. +""" + +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field, JsonValue +from xtm_oaev_sdk.seam.common import OpenAEVError + +__all__ = [ + "ExtraSignatureData", + "MatchTypes", + "SignatureMatch", + "SignatureType", + "SignatureTypes", +] + + +class MatchTypes(str, Enum): + """Supported match algorithm types for signature evaluation.""" + + MATCH_TYPE_FUZZY = "fuzzy" + MATCH_TYPE_SIMPLE = "simple" + + +class SignatureTypes(str, Enum): + """Supported signature field identifiers.""" + + SIG_TYPE_PROCESS_NAME = "process_name" + SIG_TYPE_PARENT_PROCESS_NAME = "parent_process_name" + SIG_TYPE_COMMAND_LINE = "command_line" + SIG_TYPE_IPV4_ADDRESS = "ipv4_address" + SIG_TYPE_IPV6_ADDRESS = "ipv6_address" + SIG_TYPE_SOURCE_IPV4_ADDRESS = "source_ipv4_address" + SIG_TYPE_SOURCE_IPV6_ADDRESS = "source_ipv6_address" + SIG_TYPE_TARGET_IPV4_ADDRESS = "target_ipv4_address" + SIG_TYPE_TARGET_IPV6_ADDRESS = "target_ipv6_address" + SIG_TYPE_HOSTNAME = "hostname" + SIG_TYPE_TARGET_HOSTNAME_ADDRESS = "target_hostname_address" + SIG_TYPE_START_DATE = "start_date" + SIG_TYPE_END_DATE = "end_date" + SIG_TYPE_CLOUD_PROVIDER = "cloud_provider" + SIG_TYPE_CLOUD_ACCOUNT_ID = "cloud_account_id" + SIG_TYPE_CLOUD_REGION = "cloud_region" + SIG_TYPE_TARGET_SERVICE = "target_service" + SIG_TYPE_QUERY = "query" + SIG_TYPE_URL_HASH = "url_hash" + SIG_TYPE_FILE_NAME = "file_name" + SIG_TYPE_FILE_HASH = "file_hash" + SIG_TYPE_SOURCE_EMAIL = "source_email" + SIG_TYPE_TARGET_EMAIL = "target_email" + SIG_TYPE_EMAIL_CUSTOM_HEADER = "email_custom_header" + # AI adversarial validation: correlate AI defense (LLM firewall / guardrail) events back to a + # specific AI inject execution. + SIG_TYPE_AI_REQUEST_MARKER = "ai_request_marker" + SIG_TYPE_AI_TARGET_ENDPOINT = "ai_target_endpoint" + + +class SignatureMatch: + """Represents a signature matching outcome. + + Args: + match_type: The matching strategy/result type. + match_score: Optional score produced by matching. + + Raises: + OpenAEVError: If `match_score` is not provided for non-simple match + types. + """ + + def __init__(self, match_type: MatchTypes, match_score: int | None): + if match_score is None and match_type != MatchTypes.MATCH_TYPE_SIMPLE: + raise OpenAEVError( + f"Match type {match_type} requires score to be set, found score = {match_score}" + ) + self.match_type = match_type + self.match_score = match_score + + +class SignatureType: + """Describe a signature type and its matching policy. + + Args: + label: Type specifier. + match_type: Matching policy used when trying to match this signature + type, for example fuzzy or simple. + match_score: Threshold score used when ``match_type`` is fuzzy. + """ + + def __init__( + self, + label: SignatureTypes, + match_type: MatchTypes = MatchTypes.MATCH_TYPE_SIMPLE, + match_score: int | None = None, + ) -> None: + self.label = label + self.match_policy = SignatureMatch(match_type, match_score) + + def make_struct_for_matching(self, data: Any) -> dict[str, Any]: + """Format matching metadata for helper-based signature matching. + + Args: + data: Arbitrary data, most often a string or numeric primitive. + + Returns: + A dictionary with matching specifiers containing: + - ``type``: Matching type string. + - ``data``: Provided data payload. + - ``score``: Optional score threshold when configured. + """ + struct: dict[str, Any] = { + "type": self.match_policy.match_type.value, + "data": data, + } + + if self.match_policy.match_score is not None: + struct["score"] = self.match_policy.match_score + + return struct + + +class ExtraSignatureData(BaseModel): + """Format for extra signatures added to the default signatures.""" + + detection: dict[str, JsonValue] = Field(default_factory=dict) + prevention: dict[str, JsonValue] = Field(default_factory=dict) + vulnerability: dict[str, JsonValue] = Field(default_factory=dict) + + def get_extra(self, expectation_type: str) -> dict[str, JsonValue]: + if expectation_type.lower() == "detection": + return self.detection + if expectation_type.lower() == "prevention": + return self.prevention + if expectation_type.lower() == "vulnerability": + return self.vulnerability + raise ValueError( + f"Expectation type should be one of the available parameters: {list(self.model_fields.keys())}" + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/ports.py new file mode 100644 index 00000000..f4966707 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/ports.py @@ -0,0 +1,164 @@ +"""Signature ports. + +``SignatureTransportProtocol`` is the driven port the signature manager ships +signatures through (implemented by API clients); ``SignatureManagerProtocol`` +is the consumer-facing contract of signature managers. The other protocols +are the ports the public adapters delegate to; the core implements them. +""" + +import logging +from datetime import datetime +from typing import Any, Callable, Protocol, runtime_checkable + +from xtm_oaev_sdk.seam.port_io import ExecutionDetails, SignatureOutputStructure +from xtm_oaev_sdk.seam.signatures.models import ExtraSignatureData +from xtm_oaev_sdk.seam.signatures.seam import ( + ExecutionSignature, + InjectorConfig, + NetworkInjectorConfig, +) + +__all__ = [ + "ContainerAddressPort", + "MarkerPort", + "NetworkTargetsPort", + "SignatureEnginePort", + "SignatureManagerProtocol", + "SignatureTransportProtocol", +] + + +@runtime_checkable +class SignatureTransportProtocol(Protocol): + """Transport contract for signature delivery. + + pyoaev implements this via its Signature API manager. + The SDK only knows this interface, never the concrete client. + """ + + def send_signatures( + self, + inject_id: str, + sig_output: SignatureOutputStructure, + execution_details: ExecutionDetails, + *, + max_payload_size: int, + logger: logging.Logger, + ) -> None: + """Transmit signatures to the platform callback endpoint.""" + ... + + +@runtime_checkable +class SignatureManagerProtocol(Protocol): + """Behavioral contract for the signature lifecycle manager. + + Covers the full compile → merge → transmit pipeline. + """ + + def build_execution_signatures( + self, + config: InjectorConfig | list[InjectorConfig], + ) -> ExecutionSignature | list[ExecutionSignature]: + """Build pre-execution signatures from injector configs.""" + ... + + def post_execution_updates( + self, + execution_details: ExecutionDetails, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + tool_output: dict[str, Any], + ) -> None: + """Update execution metadata after tool completion.""" + ... + + def send_signatures( + self, + inject_id: str, + execution_details: ExecutionDetails, + signatures: dict[str, Any], + ) -> None: + """Ship signatures to the callback endpoint.""" + ... + + def resolve_container_ip(self) -> str: + """Resolve the container's primary IPv4 address.""" + ... + + +@runtime_checkable +class SignatureEnginePort(Protocol): + """Compiles, merges and packages execution signatures.""" + + def now(self) -> datetime: + """Current UTC time.""" + + def validate_configs(self, configs: list[InjectorConfig]) -> None: + """Raise unless there is at least one config, all of the same type.""" + + def format_start_time(self, now: datetime) -> str: + """Start time of the execution signatures.""" + + def source_context( + self, + config: InjectorConfig, + source_ipv4: Callable[[], str], + source_ipv6: Callable[[], str | None], + ) -> dict[str, Any]: + """Source identity fields of a config's signature.""" + + def build_signature( + self, config: InjectorConfig, start_time: str, source_context: dict[str, Any] + ) -> ExecutionSignature: + """Pre-execution signature of one config.""" + + def apply_tool_output( + self, + execution_details: ExecutionDetails, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + tool_output: dict[str, Any], + now: Callable[[], datetime], + ) -> None: + """Merge the validated tool output into details and signatures, at the + time ``now`` returns once the output is valid.""" + + def build_payload( + self, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + targets_meta: Any | list[Any], + expectation_types: list[str], + extra_signatures: ExtraSignatureData, + ) -> dict[str, Any]: + """Wire payload of the signatures, per target and expectation type.""" + + def output_structure(self, signatures: dict[str, Any]) -> SignatureOutputStructure: + """Structure shipped through the transport.""" + + +@runtime_checkable +class ContainerAddressPort(Protocol): + """Resolves the addresses of the container running the injector.""" + + def resolve_ipv4(self, logger: logging.Logger) -> str | None: + """Container IPv4 (env, socket, ``hostname -i``), None when unresolved.""" + + def resolve_ipv6(self) -> str | None: + """Container global IPv6, None when there is none.""" + + +@runtime_checkable +class NetworkTargetsPort(Protocol): + """Turns raw injector targets into network injector configs.""" + + def build_network_configs( + self, targets: list[str | dict[str, Any] | NetworkInjectorConfig] + ) -> list[NetworkInjectorConfig]: + """One config per target, order preserved.""" + + +@runtime_checkable +class MarkerPort(Protocol): + """Derives the per-inject canary marker of AI injects.""" + + def build_marker(self, inject_id: str, agent_id: str = "") -> str: + """Deterministic marker of an inject (and agent).""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py new file mode 100644 index 00000000..6571d876 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py @@ -0,0 +1,145 @@ +"""The signatures capability's inbound seam: the versioned I/O models. + +The frozen boundary both sides of the signatures capability share: + +- **input face** — what is handed to the injector to execute: the + ``InjectorConfig`` family (one network target, or one cloud target row) + and the ``ExecutionSignature`` (the execution instruction and its + outcome fields). +- **tool result face** — what the injected tool reports back: + ``ToolOutput`` and its ``ToolErrorInfo`` / ``ToolTimeoutInfo``. + +Pure data: this module imports nothing from the capability. The +dependency rule holds — the port imports the seam, never the other way, +and this module is the only thing both sides may import. + +Any modification to the JSON schema of a model in this module is a +breaking change for consumers and must bump :data:`SEAM_VERSION`. + +The seam golden (``test/seam/test_seam_golden.py``) fingerprints every public +object of the seam, this module included, and fails on drift. +""" + +from datetime import datetime, timezone + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +#: Version of the signatures inbound seam. Bump on any breaking contract change. +SEAM_VERSION = "1.0.0" + + +class ToolErrorInfo(BaseModel): + """Crash report. Non-zero exit code and a timestamp if the tool left one behind.""" + + model_config = ConfigDict(extra="allow") + + exit_code: int = 0 + crash_timestamp: str | None = None + + +class ToolTimeoutInfo(BaseModel): + """Timeout report. Whatever partial loot was rescued before the kill signal.""" + + model_config = ConfigDict(extra="allow") + + partial_results: list[str] = Field(default_factory=list) + + +class ToolOutput(BaseModel): + """Whatever the tool spat out: status, error info, timeout info, or injector extras.""" + + model_config = ConfigDict(extra="allow") + + status: str | None = None + error_info: ToolErrorInfo | None = None + timeout_info: ToolTimeoutInfo | None = None + + +class ExecutionSignature(BaseModel): + """ + Execution signature data. Field set varies by category: network, cloud. Plus outcome, end_time, and any partial results. + """ + + model_config = ConfigDict(extra="allow") + + # Timing always emitted at call time. + start_time: str = Field( + default_factory=lambda: datetime.now(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + ) + end_time: str | None = None + partial_results: list[str] | None = None + + # Network identity + source_ipv4: str | None = None + source_ipv6: str | None = None + target_ipv4: str | None = None + target_ipv6: str | None = None + target_hostname: str | None = None + + # Cloud identity + cloud_provider: str | None = None + cloud_account_id: str | None = None + cloud_region: str | None = None + target_service: str | None = None + + def post_execution_update(self, tool_output: ToolOutput, now: datetime) -> None: + """Update timing and partial results from tool output.""" + self.end_time = now.strftime("%Y-%m-%dT%H:%M:%SZ") + + if tool_output.error_info and tool_output.error_info.crash_timestamp: + self.end_time = tool_output.error_info.crash_timestamp + + if tool_output.timeout_info and tool_output.timeout_info.partial_results: + self.partial_results = tool_output.timeout_info.partial_results + + +class NetworkInjectorConfig(BaseModel): + """A single network target. Exactly one of ``target_ipv4``, ``target_ipv6``, or ``target_hostname``.""" + + model_config = ConfigDict(extra="forbid") + + target_ipv4: str | None = None + target_ipv6: str | None = None + target_hostname: str | None = None + + @model_validator(mode="before") + @classmethod + def check_one(cls, data): + """Ensure exactly one target identity field is provided.""" + assert ( + sum( + value is not None + for key, value in data.items() + if key in ["target_ipv4", "target_ipv6", "target_hostname"] + ) + == 1 + ) + return data + + +class CloudInjectorConfig(BaseModel): + """A single cloud target row. One config per region; fan out by passing a list.""" + + model_config = ConfigDict(extra="forbid") + + cloud_provider: str + cloud_account_id: str + cloud_region: str + target_service: str | None = None + + +InjectorConfig = NetworkInjectorConfig | CloudInjectorConfig + + +__all__ = [ + "SEAM_VERSION", + "ToolErrorInfo", + "ToolTimeoutInfo", + "ToolOutput", + "ExecutionSignature", + "NetworkInjectorConfig", + "CloudInjectorConfig", + "InjectorConfig", +] diff --git a/pyoaev/__init__.py b/pyoaev/__init__.py index 049ebd03..62415734 100644 --- a/pyoaev/__init__.py +++ b/pyoaev/__init__.py @@ -1,6 +1,16 @@ # -*- coding: utf-8 -*- __version__ = "3.261014.1" +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +from pyoaev import ( # noqa: F401 + asset_types, + configuration, + contracts, + exceptions, + signatures, +) from pyoaev._version import ( # noqa: F401 __author__, __copyright__, @@ -8,19 +18,9 @@ __license__, __title__, ) -from pyoaev.asset_types import ( # noqa: F401 - AssetCategory, - AssetCriticality, - AssetSubCategory, - CloudProvider, -) from pyoaev.client import OpenAEV # noqa: F401 -from pyoaev.configuration import * # noqa: F401,F403,F405 -from pyoaev.contracts import * # noqa: F401,F403,F405 -from pyoaev.exceptions import * # noqa: F401,F403,F405 -from pyoaev.signatures import * # noqa: F401,F403,F405 -__all__ = [ +__all__ = [ # moved names are served by the PEP 562 __getattr__ below "__author__", "__copyright__", "__email__", @@ -33,4 +33,25 @@ "CloudProvider", "AssetCriticality", ] -__all__.extend(exceptions.__all__) # noqa: F405 +__all__.extend(exceptions.__all__) + +# Names moved to xtm_oaev_sdk: served lazily (PEP 562) so that `import pyoaev` +# stays silent and only legacy access to a moved name warns. +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in ( + *asset_types.__all__, + *configuration.__all__, + *contracts.__all__, + *exceptions.__all__, + ) + }, + **signatures._DEPRECATED, + }, + module_name=__name__, +) diff --git a/pyoaev/apis/__init__.py b/pyoaev/apis/__init__.py index 67bea1d6..d8cd3342 100644 --- a/pyoaev/apis/__init__.py +++ b/pyoaev/apis/__init__.py @@ -1,3 +1,6 @@ +import collectors_sdk as _collectors_sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + from .ai_target import * # noqa: F401,F403 from .attack_pattern import * # noqa: F401,F403 from .collector import * # noqa: F401,F403 @@ -20,4 +23,22 @@ from .user import * # noqa: F401,F403 from .vulnerability import * # noqa: F401,F403 -__all__ = [name for name in dir() if not name.startswith("_")] +# The expectation models moved to collectors_sdk; their legacy names stay in +# ``__all__`` and are served by the PEP 562 __getattr__ below, with a warning. +_MOVED_MODELS = ["DetectionExpectation", "ExpectationTypeEnum", "PreventionExpectation"] + +__all__ = [ # noqa: F822 (moved names are served by the PEP 562 __getattr__ below) + *[name for name in dir() if not name.startswith("_")], + *_MOVED_MODELS, +] + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in _MOVED_MODELS + }, + module_name=__name__, +) diff --git a/pyoaev/apis/ai_target.py b/pyoaev/apis/ai_target.py index c1dfdf91..7ec68eb9 100644 --- a/pyoaev/apis/ai_target.py +++ b/pyoaev/apis/ai_target.py @@ -1,30 +1,30 @@ -from pyoaev.base import RESTManager, RESTObject -from pyoaev.mixins import CreateMixin, DeleteMixin, GetMixin, ListMixin, UpdateMixin -from pyoaev.utils import RequiredOptional - - -class AiTarget(RESTObject): - _id_attr = "asset_id" - - -class AiTargetManager( - GetMixin, ListMixin, CreateMixin, UpdateMixin, DeleteMixin, RESTManager -): - """Manage AI Target assets (LLM endpoints / AI agents under adversarial test).""" - - _path = "/ai_targets" - _obj_cls = AiTarget - _create_attrs = RequiredOptional( - required=("asset_name", "ai_target_provider"), - optional=( - "asset_description", - "asset_tags", - "asset_external_reference", - "ai_target_endpoint", - "ai_target_model", - "ai_target_modality", - "ai_target_system_prompt", - "ai_target_token", - "ai_target_configuration", - ), - ) +from pyoaev.base import RESTManager, RESTObject +from pyoaev.mixins import CreateMixin, DeleteMixin, GetMixin, ListMixin, UpdateMixin +from pyoaev.utils import RequiredOptional + + +class AiTarget(RESTObject): + _id_attr = "asset_id" + + +class AiTargetManager( + GetMixin, ListMixin, CreateMixin, UpdateMixin, DeleteMixin, RESTManager +): + """Manage AI Target assets (LLM endpoints / AI agents under adversarial test).""" + + _path = "/ai_targets" + _obj_cls = AiTarget + _create_attrs = RequiredOptional( + required=("asset_name", "ai_target_provider"), + optional=( + "asset_description", + "asset_tags", + "asset_external_reference", + "ai_target_endpoint", + "ai_target_model", + "ai_target_modality", + "ai_target_system_prompt", + "ai_target_token", + "ai_target_configuration", + ), + ) diff --git a/pyoaev/apis/attack_pattern.py b/pyoaev/apis/attack_pattern.py index 89b800b9..bf74d1c4 100644 --- a/pyoaev/apis/attack_pattern.py +++ b/pyoaev/apis/attack_pattern.py @@ -1,7 +1,9 @@ from typing import Any, Dict, List -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class AttackPattern(RESTObject): @@ -12,7 +14,7 @@ class AttackPatternManager(RESTManager): _path = "/attack_patterns" _obj_cls = AttackPattern - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert( self, attack_patterns: List[Dict[str, Any]], diff --git a/pyoaev/apis/collector.py b/pyoaev/apis/collector.py index 69ff43fb..8673f380 100644 --- a/pyoaev/apis/collector.py +++ b/pyoaev/apis/collector.py @@ -1,7 +1,9 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin, GetMixin, ListMixin, UpdateMixin from pyoaev.utils import RequiredOptional @@ -28,7 +30,7 @@ class CollectorManager(GetMixin, ListMixin, CreateMixin, UpdateMixin, RESTManage ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def get(self, collector_id: str, **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/" + collector_id result = self.openaev.http_get(path, **kwargs) diff --git a/pyoaev/apis/cve.py b/pyoaev/apis/cve.py index 45145f2b..1604d40e 100644 --- a/pyoaev/apis/cve.py +++ b/pyoaev/apis/cve.py @@ -1,7 +1,9 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Cve(RESTObject): @@ -11,7 +13,7 @@ class Cve(RESTObject): class CveManager(RESTManager): _path = "/cves" - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/bulk" result = self.openaev.http_post(path, post_data=data, **kwargs) diff --git a/pyoaev/apis/document.py b/pyoaev/apis/document.py index 38f3ab8a..c1b70d93 100644 --- a/pyoaev/apis/document.py +++ b/pyoaev/apis/document.py @@ -1,7 +1,9 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Document(RESTObject): @@ -12,13 +14,13 @@ class DocumentManager(RESTManager): _path = "/documents" _obj_cls = Document - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def download(self, document_id: str, **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/" + document_id + "/file" result = self.openaev.http_get(path, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert( self, document: Dict[str, Any], file: tuple, **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/endpoint.py b/pyoaev/apis/endpoint.py index 543ccb14..f4354384 100644 --- a/pyoaev/apis/endpoint.py +++ b/pyoaev/apis/endpoint.py @@ -1,8 +1,10 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.apis.inputs.search import SearchPaginationInput from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.utils import RequiredOptional @@ -40,19 +42,19 @@ class EndpointManager(RESTManager): ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def get(self, asset_id: str, **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/" + asset_id result = self.openaev.http_get(path, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, endpoint: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/agentless/upsert" result = self.openaev.http_post(path, post_data=endpoint, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def searchTargets( self, input: SearchPaginationInput, **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/inject.py b/pyoaev/apis/inject.py index 7fb6add1..2a6c7b3b 100644 --- a/pyoaev/apis/inject.py +++ b/pyoaev/apis/inject.py @@ -1,7 +1,9 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Inject(RESTObject): @@ -12,7 +14,7 @@ class InjectManager(RESTManager): _path = "/injects" _obj_cls = Inject - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def execution_callback( self, inject_id: str, data: Dict[str, Any], **kwargs: Any ) -> Dict[str, Any]: @@ -20,7 +22,7 @@ def execution_callback( result = self.openaev.http_post(path, post_data=data, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def execution_reception( self, inject_id: str, data: Dict[str, Any], **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/inject_expectation/__init__.py b/pyoaev/apis/inject_expectation/__init__.py index 183bc0e2..8f08ea7e 100644 --- a/pyoaev/apis/inject_expectation/__init__.py +++ b/pyoaev/apis/inject_expectation/__init__.py @@ -1 +1,22 @@ +import collectors_sdk as _collectors_sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +from . import model # noqa: F401 (legacy: exposed as pyoaev.apis.model) from .inject_expectation import * # noqa: F401,F403 + +# The expectation models moved to collectors_sdk; their legacy names here +# resolve to the identical objects and emit a DeprecationWarning on access. +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in ( + "DetectionExpectation", + "ExpectationTypeEnum", + "PreventionExpectation", + ) + }, + module_name=__name__, +) diff --git a/pyoaev/apis/inject_expectation/inject_expectation.py b/pyoaev/apis/inject_expectation/inject_expectation.py index be1c453a..4193fa13 100644 --- a/pyoaev/apis/inject_expectation/inject_expectation.py +++ b/pyoaev/apis/inject_expectation/inject_expectation.py @@ -1,12 +1,12 @@ +import warnings as _warnings from typing import Any, Dict, List -from pyoaev import exceptions as exc -from pyoaev.apis.inject_expectation.model import ( - DetectionExpectation, - ExpectationTypeEnum, - PreventionExpectation, -) +import collectors_sdk as _collectors_sdk +import xtm_oaev_sdk as exc +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import ListMixin, UpdateMixin from pyoaev.utils import RequiredOptional @@ -20,7 +20,7 @@ class InjectExpectationManager(ListMixin, UpdateMixin, RESTManager): _obj_cls = InjectExpectation _update_attrs = RequiredOptional(required=("collector_id", "result", "is_success")) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def expectations_assets_for_source( self, source_id: str, expiration_time: int = None, **kwargs: Any ) -> Dict[str, Any]: @@ -40,7 +40,7 @@ def expectations_assets_for_source( ) return result - @exc.on_http_error(exc.OpenAEVListError) + @on_http_error(exc.OpenAEVListError) def ai_expectations_for_source( self, source_id: str, **kwargs: Any ) -> List[Dict[str, Any]]: @@ -86,35 +86,22 @@ def expectations_models_for_source(self, source_id: str, **kwargs: Any): :return: a list of expectation objects :rtype: list[DetectionExpectation|PreventionExpectation] + + .. deprecated:: + Building the expectation models is the collector runtime's job: use + ``collectors_sdk.expectations_models_for_source(api.inject_expectation, + source_id)`` instead. """ - # TODO: we should implement a more clever mechanism to obtain - # specialised Expectation instances rather than just if/elseing - # through this list of possibilities. - expectations = [] - for expectation_dict in self.expectations_assets_for_source( - source_id=source_id, **kwargs - ): - if ( - expectation_dict["inject_expectation_type"] - == ExpectationTypeEnum.Detection.value - ): - expectations.append( - DetectionExpectation(**expectation_dict, api_client=self) - ) - elif ( - expectation_dict["inject_expectation_type"] - == ExpectationTypeEnum.Prevention.value - ): - expectations.append( - PreventionExpectation(**expectation_dict, api_client=self) - ) - else: - expectations.append( - PreventionExpectation(**expectation_dict, api_client=self) - ) - return expectations + _warnings.warn( + "expectations_models_for_source is deprecated. Use 'from collectors_sdk " + "import expectations_models_for_source' and call " + "expectations_models_for_source(api.inject_expectation, source_id) instead.", + DeprecationWarning, + stacklevel=2, + ) + return _collectors_sdk.expectations_models_for_source(self, source_id, **kwargs) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def prevention_expectations_for_source( self, source_id: str, **kwargs: Any ) -> Dict[str, Any]: @@ -122,7 +109,7 @@ def prevention_expectations_for_source( result = self.openaev.http_get(path, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def detection_expectations_for_source( self, source_id: str, expiration_time: int = None, **kwargs: Any ) -> Dict[str, Any]: @@ -136,7 +123,7 @@ def detection_expectations_for_source( ) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def update( self, inject_expectation_id: str, @@ -147,7 +134,7 @@ def update( result = self.openaev.http_put(path, post_data=inject_expectation, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def bulk_update( self, inject_expectation_input_by_id: Dict[str, Dict[str, Any]], @@ -157,3 +144,19 @@ def bulk_update( self.openaev.http_put( path, post_data={"inputs": inject_expectation_input_by_id}, **kwargs ) + + +# The expectation models moved to collectors_sdk; their legacy names here +# resolve to the identical objects and emit a DeprecationWarning on access. +_MOVED_MODELS = ["DetectionExpectation", "ExpectationTypeEnum", "PreventionExpectation"] + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in _MOVED_MODELS + }, + module_name=__name__, +) diff --git a/pyoaev/apis/inject_expectation/model/__init__.py b/pyoaev/apis/inject_expectation/model/__init__.py index 8cedb2d1..7b605b57 100644 --- a/pyoaev/apis/inject_expectation/model/__init__.py +++ b/pyoaev/apis/inject_expectation/model/__init__.py @@ -1,7 +1,25 @@ -from .expectation import ( - DetectionExpectation, - ExpectationTypeEnum, - PreventionExpectation, -) +"""DEPRECATED: moved to :mod:`collectors_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +collectors_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import collectors_sdk as _collectors_sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr -__all__ = ["DetectionExpectation", "ExpectationTypeEnum", "PreventionExpectation"] +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "DetectionExpectation", + "ExpectationTypeEnum", + "PreventionExpectation", +] + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in (*__all__, "Expectation", "ExpectationSignature") + }, + module_name=__name__, +) diff --git a/pyoaev/apis/inject_expectation/model/expectation.py b/pyoaev/apis/inject_expectation/model/expectation.py index 3e64ce0d..94080ba3 100644 --- a/pyoaev/apis/inject_expectation/model/expectation.py +++ b/pyoaev/apis/inject_expectation/model/expectation.py @@ -1,183 +1,37 @@ -from enum import Enum -from typing import Any, List -from uuid import UUID - -from pydantic import BaseModel, Field, field_validator -from thefuzz import fuzz - -from pyoaev.signatures.signature_type import SignatureType -from pyoaev.signatures.types import MatchTypes, SignatureTypes - - -class ExpectationTypeEnum(str, Enum): - """Types of Expectations""" - - Detection = "DETECTION" - Prevention = "PREVENTION" - Vulnerability = "VULNERABILITY" - Other = "other" - - @classmethod - def _missing_(cls, value): - return cls.Other - - -class ExpectationSignature(BaseModel): - """An expectation signature describes a known marker potentially - found in alerting data in security software. For example, an - expectation signature can be a process image name, a command - line, or any other relevant piece of data. - """ - - type: SignatureTypes - value: str - - -class Expectation(BaseModel): - """An expectation represents an expected outcome of a BAS run. - For example, in the case of running an attack command line, the - expectation may be that security software has _detected_ it, while - another expectation may be that the attack was _prevented_. - """ - - inject_expectation_id: UUID - inject_expectation_signatures: List[ExpectationSignature] = Field( - default_factory=list - ) - - success_label: str = "Success" - failure_label: str = "Failure" - - @field_validator("inject_expectation_signatures", mode="before") - @classmethod - def _coerce_none_signatures_to_empty(cls, value: Any) -> Any: - # The platform serializes this field as null when an expectation has no - # signatures yet. Without this, a single such expectation raises a - # ValidationError that aborts the whole batch fetch (collectors#490). - return [] if value is None else value - - def __init__(self, *a, **kw): - super().__init__(*a, **kw) - self.__api_client = kw["api_client"] - - def update(self, success, sender_id, metadata): - """Update the expectation object in OpenAEV with the supplied outcome. - - :param success: whether the expectation was fulfilled (true) or not (false) - :type success: bool - :param sender_id: identifier of the collector that is updating the expectation - :type sender_id: string - :param metadata: arbitrary dictionary of additional data relevant to updating the expectation - :type metadata: dict[string,string] - """ - self.__api_client.update( - self.inject_expectation_id, - inject_expectation={ - "collector_id": sender_id, - "result": (self.success_label if success else self.failure_label), - "is_success": success, - "metadata": metadata, - }, - ) - - def match_alert(self, relevant_signature_types: list[SignatureType], alert_data): - """Matches an alert's data against the current expectation signatures - to see if the alert is relevant to the current expectation's inject, - i.e. this alert was triggered by the execution of the inject to which - belongs the expectation. - - :param relevant_signature_types: filter of signature types that we want to consider. - Only the signature types listed in this collection may be checked for matching. - :type relevant_signature_types: list[SignatureType] - :param alert_data: list of possibly relevant markers found in an alert. - :type alert_data: dict[SignatureTypes, dict] - - :return: whether the alert matches the expectation signatures or not. - :rtype: bool - """ - relevant_expectation_signatures = [ - signature - for signature in self.inject_expectation_signatures - if signature.type in [type.label for type in relevant_signature_types] - ] - if not any(relevant_expectation_signatures): - return False - - for relevant_expectation_signature in relevant_expectation_signatures: - if not ( - alert_signature_for_type := alert_data.get( - relevant_expectation_signature.type.value - ) - ): - return False - - if alert_signature_for_type[ - "type" - ] == MatchTypes.MATCH_TYPE_FUZZY and not self.match_fuzzy( - alert_signature_for_type["data"], - relevant_expectation_signature.value, - alert_signature_for_type["score"], - ): - return False - if alert_signature_for_type[ - "type" - ] == MatchTypes.MATCH_TYPE_SIMPLE and not self.match_simple( - alert_signature_for_type["data"], relevant_expectation_signature.value - ): - return False - - return True - - @staticmethod - def match_fuzzy(tested: list[str], reference: str, threshold: int): - """Applies a fuzzy match against a known reference to a list of candidates - - :param tested: list of strings candidate for fuzzy matching - :type tested: list[str] - :param reference: the reference against which to try to fuzzy match - :type reference: str - :param threshold: string overlap percentage threshold above which to declare a match - :type threshold: int - - :return: whether any of the candidate is a match against the reference - :rtype: bool - """ - actual_tested = [tested] if isinstance(tested, str) else tested - for value in actual_tested: - ratio = fuzz.ratio(value, reference) - if ratio >= threshold: - return True - return False - - @staticmethod - def match_simple(tested: list[str], reference: str): - """A simple strict, case-sensitive string matching between a list of - candidates and a reference. - - :param tested: list of strings candidate for fuzzy matching - :type tested: list[str] - :param reference: the reference against which to try to fuzzy match - :type reference: str - - :return: whether any of the candidate is a match against the reference - :rtype: bool - """ - return Expectation.match_fuzzy(tested, reference, threshold=100) - - -class DetectionExpectation(Expectation): - """An expectation that is specific to Detection, i.e. that is used - by OpenAEV to assert that an inject's execution was detected. - """ - - success_label: str = "Detected" - failure_label: str = "Not Detected" - - -class PreventionExpectation(Expectation): - """An expectation that is specific to Prevention, i.e. that is used - by OpenAEV to assert that an inject's execution was prevented. - """ - - success_label: str = "Prevented" - failure_label: str = "Not Prevented" +"""DEPRECATED: moved to :mod:`collectors_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +collectors_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import collectors_sdk as _collectors_sdk +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "DetectionExpectation", + "Expectation", + "ExpectationSignature", + "ExpectationTypeEnum", + "PreventionExpectation", +] + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in __all__ + }, + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in ("MatchTypes", "SignatureType", "SignatureTypes") + }, + }, + module_name=__name__, +) diff --git a/pyoaev/apis/inject_expectation_trace.py b/pyoaev/apis/inject_expectation_trace.py index ac9e154d..7474e619 100644 --- a/pyoaev/apis/inject_expectation_trace.py +++ b/pyoaev/apis/inject_expectation_trace.py @@ -1,7 +1,9 @@ from typing import Any, Dict, List -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin from pyoaev.utils import RequiredOptional @@ -23,7 +25,7 @@ class InjectExpectationTraceManager(CreateMixin, RESTManager): ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def bulk_create( self, payload: Dict[str, List[Dict[str, str]]], **kwargs: Any ) -> dict[str, Any]: diff --git a/pyoaev/apis/injector_contract.py b/pyoaev/apis/injector_contract.py index 8be83c61..d4e42db4 100644 --- a/pyoaev/apis/injector_contract.py +++ b/pyoaev/apis/injector_contract.py @@ -1,8 +1,10 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.apis.inputs.search import InjectorContractSearchPaginationInput from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin, DeleteMixin, UpdateMixin from pyoaev.utils import RequiredOptional @@ -46,7 +48,7 @@ class InjectorContractManager(CreateMixin, UpdateMixin, DeleteMixin, RESTManager ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def search( self, input: InjectorContractSearchPaginationInput, **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/kill_chain_phase.py b/pyoaev/apis/kill_chain_phase.py index a40ebb22..8ba373f4 100644 --- a/pyoaev/apis/kill_chain_phase.py +++ b/pyoaev/apis/kill_chain_phase.py @@ -1,7 +1,9 @@ from typing import Any, Dict, List -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class KillChainPhase(RESTObject): @@ -12,7 +14,7 @@ class KillChainPhaseManager(RESTManager): _path = "/kill_chain_phases" _obj_cls = KillChainPhase - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert( self, kill_chain_phases: List[Dict[str, Any]], **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/payload.py b/pyoaev/apis/payload.py index 3de5b0f3..0f3bf57a 100644 --- a/pyoaev/apis/payload.py +++ b/pyoaev/apis/payload.py @@ -1,7 +1,9 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Payload(RESTObject): @@ -12,13 +14,13 @@ class PayloadManager(RESTManager): _path = "/payloads" _obj_cls = Payload - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, payload: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/upsert" result = self.openaev.http_post(path, post_data=payload, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def deprecate( self, payloads_processed: Dict[str, Any], **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/security_platform.py b/pyoaev/apis/security_platform.py index 184474d8..c7ad83dc 100644 --- a/pyoaev/apis/security_platform.py +++ b/pyoaev/apis/security_platform.py @@ -1,7 +1,9 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin, GetMixin, ListMixin, UpdateMixin from pyoaev.utils import RequiredOptional @@ -24,7 +26,7 @@ class SecurityPlatformManager( ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert( self, security_platform: Dict[str, Any], **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/signature.py b/pyoaev/apis/signature.py index 0fe19719..fdd89c36 100644 --- a/pyoaev/apis/signature.py +++ b/pyoaev/apis/signature.py @@ -5,17 +5,18 @@ import time from typing import Any +import xtm_oaev_sdk as exc from pydantic import ValidationError - -from pyoaev import exceptions as exc -from pyoaev.base import RESTManager, RESTObject -from pyoaev.exceptions import SignatureTransmissionError -from pyoaev.signatures.models import ( +from xtm_oaev_sdk import ( ExecutionDetails, SignatureCallbackPayload, SignatureOutputStructure, + SignatureTransmissionError, ) +from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error + class Signature(RESTObject): """REST object placeholder for signature callback responses.""" @@ -207,7 +208,7 @@ def _build_envelope( SignatureCallbackPayload.model_validate(envelope) return envelope - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def callback( self, inject_id: str, data: dict[str, Any], **kwargs: Any ) -> dict[str, Any]: @@ -243,7 +244,7 @@ def _send_with_retry( Raises: SignatureTransmissionError: 4xx error or all retries exhausted. """ - from pyoaev.exceptions import OpenAEVError + from xtm_oaev_sdk import OpenAEVError effective_logger = logger if logger is not None else self._logger last_error: Exception | None = None diff --git a/pyoaev/apis/tag.py b/pyoaev/apis/tag.py index e112b0c6..72941f15 100644 --- a/pyoaev/apis/tag.py +++ b/pyoaev/apis/tag.py @@ -1,7 +1,9 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Tag(RESTObject): @@ -12,7 +14,7 @@ class TagManager(RESTManager): _path = "/tags" _obj_cls = Tag - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/upsert" result = self.openaev.http_post(path, post_data=data, **kwargs) diff --git a/pyoaev/apis/team.py b/pyoaev/apis/team.py index a4c68a19..947be9d7 100644 --- a/pyoaev/apis/team.py +++ b/pyoaev/apis/team.py @@ -1,7 +1,9 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin, ListMixin, UpdateMixin from pyoaev.utils import RequiredOptional @@ -18,7 +20,7 @@ class TeamManager(CreateMixin, ListMixin, UpdateMixin, RESTManager): optional=("team_description", "team_organization", "team_tags"), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, team: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/upsert" result = self.openaev.http_post(path, post_data=team, **kwargs) diff --git a/pyoaev/apis/user.py b/pyoaev/apis/user.py index 64ac0db0..29055913 100644 --- a/pyoaev/apis/user.py +++ b/pyoaev/apis/user.py @@ -1,7 +1,9 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin, ListMixin, UpdateMixin from pyoaev.utils import RequiredOptional @@ -24,7 +26,7 @@ class UserManager(CreateMixin, ListMixin, UpdateMixin, RESTManager): ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, user: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/upsert" result = self.openaev.http_post(path, post_data=user, **kwargs) diff --git a/pyoaev/apis/vulnerability.py b/pyoaev/apis/vulnerability.py index 5418afce..73d6bd48 100644 --- a/pyoaev/apis/vulnerability.py +++ b/pyoaev/apis/vulnerability.py @@ -1,7 +1,9 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +import xtm_oaev_sdk as exc + from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Vulnerability(RESTObject): @@ -11,7 +13,7 @@ class Vulnerability(RESTObject): class VulnerabilityManager(RESTManager): _path = "/vulnerabilities" - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/bulk" result = self.openaev.http_post(path, post_data=data, **kwargs) diff --git a/pyoaev/asset_types.py b/pyoaev/asset_types.py index b8e0b418..616815fc 100644 --- a/pyoaev/asset_types.py +++ b/pyoaev/asset_types.py @@ -1,135 +1,25 @@ -# -*- coding: utf-8 -*- -"""Canonical asset taxonomy for OpenAEV (mirrors the backend enums). - -These give producers (collectors, custom integrations) a single, typed vocabulary for the -two-level asset categorization (category + subcategory), criticality and cloud provider, instead -of free-form strings. They are plain ``str`` enums so they can be dropped straight into the dict -payloads sent to the API. -""" - -from enum import Enum - - -class AssetCategory(str, Enum): - HOST = "HOST" - CONTAINER_WORKLOAD = "CONTAINER_WORKLOAD" - CLOUD_RESOURCE = "CLOUD_RESOURCE" - WEB_APPLICATION = "WEB_APPLICATION" - NETWORK_DEVICE = "NETWORK_DEVICE" - MOBILE_DEVICE = "MOBILE_DEVICE" - IOT_OT_DEVICE = "IOT_OT_DEVICE" - IDENTITY = "IDENTITY" - SAAS_APPLICATION = "SAAS_APPLICATION" - AI_TARGET = "AI_TARGET" - SECURITY_PLATFORM = "SECURITY_PLATFORM" - GENERIC_ASSET = "GENERIC_ASSET" - - -class AssetSubCategory(str, Enum): - # HOST - SERVER = "SERVER" - WORKSTATION = "WORKSTATION" - LAPTOP = "LAPTOP" - VIRTUAL_MACHINE = "VIRTUAL_MACHINE" - HYPERVISOR = "HYPERVISOR" - MAINFRAME = "MAINFRAME" - THIN_CLIENT = "THIN_CLIENT" - # CONTAINER_WORKLOAD - CONTAINER = "CONTAINER" - CONTAINER_IMAGE = "CONTAINER_IMAGE" - KUBERNETES_POD = "KUBERNETES_POD" - KUBERNETES_CLUSTER = "KUBERNETES_CLUSTER" - KUBERNETES_NODE = "KUBERNETES_NODE" - SERVERLESS_FUNCTION = "SERVERLESS_FUNCTION" - # CLOUD_RESOURCE - COMPUTE = "COMPUTE" - STORAGE = "STORAGE" - DATABASE = "DATABASE" - NETWORKING = "NETWORKING" - SERVERLESS = "SERVERLESS" - CONTAINER_REGISTRY = "CONTAINER_REGISTRY" - KUBERNETES = "KUBERNETES" - IAM_PRINCIPAL = "IAM_PRINCIPAL" - SECRETS_KEY_MGMT = "SECRETS_KEY_MGMT" - MESSAGING_QUEUE = "MESSAGING_QUEUE" - ANALYTICS_DATA = "ANALYTICS_DATA" - AI_ML_SERVICE = "AI_ML_SERVICE" - IAC_TEMPLATE = "IAC_TEMPLATE" - CLOUD_OTHER = "CLOUD_OTHER" - # WEB_APPLICATION - WEBSITE = "WEBSITE" - WEB_API = "WEB_API" - SINGLE_PAGE_APP = "SINGLE_PAGE_APP" - GRAPHQL_API = "GRAPHQL_API" - WEB_SERVICE = "WEB_SERVICE" - MICROSERVICE = "MICROSERVICE" - # NETWORK_DEVICE - ROUTER = "ROUTER" - SWITCH = "SWITCH" - FIREWALL = "FIREWALL" - LOAD_BALANCER = "LOAD_BALANCER" - VPN_GATEWAY = "VPN_GATEWAY" - WIRELESS_AP = "WIRELESS_AP" - PROXY = "PROXY" - DNS_SERVER = "DNS_SERVER" - DHCP_SERVER = "DHCP_SERVER" - SAN_NAS = "SAN_NAS" - NETWORK_OTHER = "NETWORK_OTHER" - # MOBILE_DEVICE - SMARTPHONE = "SMARTPHONE" - TABLET = "TABLET" - # IOT_OT_DEVICE - IOT_SENSOR = "IOT_SENSOR" - IP_CAMERA = "IP_CAMERA" - GATEWAY = "GATEWAY" - POINT_OF_SALE = "POINT_OF_SALE" - MEDIA_DEVICE = "MEDIA_DEVICE" - PLC = "PLC" - RTU = "RTU" - HMI = "HMI" - SCADA_HISTORIAN = "SCADA_HISTORIAN" - MEDICAL_DEVICE = "MEDICAL_DEVICE" - PRINTER_PERIPHERAL = "PRINTER_PERIPHERAL" - BUILDING_MGMT = "BUILDING_MGMT" - # IDENTITY - USER_ACCOUNT = "USER_ACCOUNT" - SERVICE_ACCOUNT = "SERVICE_ACCOUNT" - GROUP = "GROUP" - ROLE = "ROLE" - SHARED_MAILBOX = "SHARED_MAILBOX" - NON_HUMAN_IDENTITY = "NON_HUMAN_IDENTITY" - # SAAS_APPLICATION - SAAS_APP = "SAAS_APP" - SAAS_TENANT = "SAAS_TENANT" - # AI_TARGET - LLM_MODEL = "LLM_MODEL" - AI_AGENT = "AI_AGENT" - MCP_SERVER = "MCP_SERVER" - RAG_PIPELINE = "RAG_PIPELINE" - # SECURITY_PLATFORM - EDR = "EDR" - XDR = "XDR" - SIEM = "SIEM" - SOAR = "SOAR" - NDR = "NDR" - ISPM = "ISPM" - LLM_FIREWALL = "LLM_FIREWALL" - AI_GATEWAY = "AI_GATEWAY" - - -class CloudProvider(str, Enum): - AWS = "AWS" - AZURE = "AZURE" - GCP = "GCP" - OCI = "OCI" - ALIBABA = "ALIBABA" - KUBERNETES = "KUBERNETES" - OTHER = "OTHER" - - -class AssetCriticality(str, Enum): - VERY_HIGH = "VERY_HIGH" - HIGH = "HIGH" - MEDIUM = "MEDIUM" - LOW = "LOW" - UNKNOWN = "UNKNOWN" +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "AssetCategory", + "AssetSubCategory", + "CloudProvider", + "AssetCriticality", +] + +_MOVED = __all__ + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in _MOVED + }, + module_name=__name__, +) diff --git a/pyoaev/base.py b/pyoaev/base.py index 10fd2f20..3cb970fa 100644 --- a/pyoaev/base.py +++ b/pyoaev/base.py @@ -15,9 +15,10 @@ get_type_hints, ) -from pyoaev.exceptions import OpenAEVParsingError +from xtm_oaev_sdk import OpenAEVParsingError + +from pyoaev.utils import EncodedId -from . import utils from .client import OpenAEV, OpenAEVList __all__ = [ @@ -215,7 +216,7 @@ def encoded_id(self) -> Optional[Union[int, str]]: path""" obj_id = self.get_id() if isinstance(obj_id, str): - obj_id = utils.EncodedId(obj_id) + obj_id = EncodedId(obj_id) return obj_id @@ -316,12 +317,12 @@ def _compute_path(self, path: Optional[str] = None) -> Optional[str]: if self._parent is None or not self._from_parent_attrs: return path - data: Dict[str, Optional[utils.EncodedId]] = {} + data: Dict[str, Optional[EncodedId]] = {} for self_attr, parent_attr in self._from_parent_attrs.items(): if not hasattr(self._parent, parent_attr): data[self_attr] = None continue - data[self_attr] = utils.EncodedId(getattr(self._parent, parent_attr)) + data[self_attr] = EncodedId(getattr(self._parent, parent_attr)) self._parent_attrs = data return path.format(**data) diff --git a/pyoaev/client.py b/pyoaev/client.py index f0f295f7..5c1b3dd2 100644 --- a/pyoaev/client.py +++ b/pyoaev/client.py @@ -3,9 +3,11 @@ from uuid import UUID import requests +import xtm_oaev_sdk as exceptions -from pyoaev import exceptions, utils +from pyoaev import utils from pyoaev._version import __version__ # noqa: F401 +from pyoaev.utils import copy_dict REDIRECT_MSG = ( "pyoaev detected a {status_code} ({reason!r}) redirection. You must update " @@ -170,17 +172,17 @@ def http_request( # parse user-provided URL params to ensure we don't add our own duplicates parsed = parse.urlparse(raw_url) params = parse.parse_qs(parsed.query) - utils.copy_dict(src=query_data, dest=params) + copy_dict(src=query_data, dest=params) url = parse.urlunparse(parsed._replace(query="")) if "query_parameters" in kwargs: - utils.copy_dict(src=kwargs["query_parameters"], dest=params) + copy_dict(src=kwargs["query_parameters"], dest=params) for arg in ("per_page", "page"): if arg in kwargs: params[arg] = kwargs[arg] else: - utils.copy_dict(src=kwargs, dest=params) + copy_dict(src=kwargs, dest=params) opts = self._get_session_opts() diff --git a/pyoaev/configuration/__init__.py b/pyoaev/configuration/__init__.py index a4741f92..b28d0462 100644 --- a/pyoaev/configuration/__init__.py +++ b/pyoaev/configuration/__init__.py @@ -1,15 +1,24 @@ -from .configuration import Configuration -from .settings_loader import ( - BaseConfigModel, - ConfigLoaderCollector, - ConfigLoaderOAEV, - SettingsLoader, -) +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr __all__ = [ - "Configuration", - "ConfigLoaderOAEV", + "BaseConfigModel", "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", "SettingsLoader", - "BaseConfigModel", ] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/configuration/configuration.py b/pyoaev/configuration/configuration.py index ae7af778..a5348569 100644 --- a/pyoaev/configuration/configuration.py +++ b/pyoaev/configuration/configuration.py @@ -1,213 +1,27 @@ -import os -import os.path -from typing import Any, Dict, Optional - -import yaml -from pydantic import BaseModel, Field -from pydantic_settings import BaseSettings - -from pyoaev.configuration.connector_config_schema_generator import ( - ConnectorConfigSchemaGenerator, +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "CONFIGURATION_TYPES", + "Configuration", + "ConfigurationHint", + "ConnectorConfigSchemaGenerator", + "DictionarySource", + "EnvironmentSource", + "is_falsy", + "is_truthy", +] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, ) -from pyoaev.configuration.sources import DictionarySource, EnvironmentSource - -CONFIGURATION_TYPES = str | int | bool | Any | None - - -def is_truthy(value: str) -> bool: - """Asserts whether a given string signals a "True" value - - :param value: value to test - :type value: str - - :return: whether the string represents True or not. - :rtype: bool - """ - return value.lower() in ["yes", "true"] - - -def is_falsy(value: str) -> bool: - """Asserts whether a given string signals a "False" value - - :param value: value to test - :type value: str - - :return: whether the string represents False or not. - :rtype: bool - """ - return value.lower() in ["no", "false"] - - -class ConfigurationHint(BaseModel): - """An individual configuration hint. This allows for specifying - where any given configuration key can be found, in env vars, - config files. Additionally, it may define a default value or - a discrete override value. - """ - - data: Optional[CONFIGURATION_TYPES] = Field(default=None) - """Override value; when set, getting the configuration value for - the key described in this instance returns this value. - """ - env: Optional[str] = Field(default=None) - """Defines which env var should be read for getting the value - """ - file_path: Optional[list[str]] = Field(default=None) - """Defines a JSON path (nested keys) to follow in the provided - config file for reaching the value. - - Example: ["toplevel", "subkey"] will hint for searching for - the config key at { "toplevel": { "subkey": { "config_key"}} - """ - is_number: Optional[bool] = Field(default=False) - """Hints at whteher the configuration value should be - interpreted as a number. - """ - default: Optional[CONFIGURATION_TYPES] = Field(default=None) - """When defined, provides a default value for whenever none of the - hinted locations or the data field have a value. - """ - - -class Configuration: - """A configuration object providing ways to an interface for getting - configuration values. It should be provided with a collection of hints - to enable its behaviour. - - :param config_hints: a dictionary of hints, for which the key is the - desired configuration key (e.g. "log_level") and the value is either - a dictionary of hints (see ConfigurationHint) or a standalone string. - In the latter case, the string will be interpreted as a default value. - - Example: - .. code-block:: python - { - "my_config_key": { - "env" : "MY_CONFIG_VALUE_ENV_VAR", - "file_path": ["first_level", "second_level"] - }, - "my_other_config_key: "discrete value" - } - :type config_hints: Dict[str, dict | str] - :param config_values: dictionary of config values to preemptively load into the - Configuration object. The format of this dictionary should follow the patterns - chosen in the file_path property of ConfigurationHint object passed - as config_hints, defaults to None - - Example: - .. code-block:: python - { - "first_level": { - "second_level": { - "my_config_key": "some value" - } - } - } - :type config_values: dict (json), optional - :param config_file_path: path to the configuration file. The file should - contain a json structure that matches the format of the config_values param, - defaults to './config.yml' (relative path). - :type config_file_path: str - """ - - def __init__( - self, - config_hints: Dict[str, dict | str], - config_values: dict = None, - config_file_path: str = os.path.join(os.curdir, "config.yml"), - config_base_model: BaseSettings = None, - ): - self.__config_hints = { - key: ( - ConfigurationHint(**value) - if isinstance(value, dict) - else ConfigurationHint(**{"default": value}) - ) - for key, value in config_hints.items() - } - - file_contents = ( - yaml.load(open(config_file_path), Loader=yaml.FullLoader) - if os.path.isfile(config_file_path) - else {} - ) - - self.__config_values = (config_values or {}) | file_contents - - self.__base_model = config_base_model - - def get(self, config_key: str) -> CONFIGURATION_TYPES: - """Gets the value pointed to by the configuration key. If the key is defined - with actual hints (as opposed to a discrete value), it will use those hints to - potentially find a value. If the key was not defined as part of the supplied - config_hints, this will always return None. - - :param config_key: the configuration key to search a value for. - :type config_key: str - - :return: the value pointed to by the configuration key, or None if not found - :rtype: CONFIGURATION_TYPES - """ - config = self.__config_hints.get(config_key) - if config is None: - return None - - return self.__process_value_to_type( - ( - self.__dig_config_sources_for_key(config) - if config.data is None - else config.data - ), - config.is_number, - ) - - def set(self, config_key: str, value: CONFIGURATION_TYPES): - """Sets an arbitrary value in the Configuration object, for - the supplied configuration key, after which any request for the value - of that key will return this new value. - - :param config_key: the configuration key to set a value for. - :type config_key: str - :param value: the new value to set for the configuration key. - :type value: CONFIGURATION_TYPES - """ - if config_key not in self.__config_hints: - self.__config_hints[config_key] = ConfigurationHint(**{"data": value}) - else: - self.__config_hints[config_key].data = value - - def schema(self): - """ - Generates the complete connector schema using a custom schema generator compatible with Pydantic. - Isolate custom class generator, Pydantic expects a class, not an instance - Always subclass GenerateJsonSchema and pass the class to Pydantic, not an instance - :return: The generated connector schema as a dictionary. - """ - return self.__base_model.model_json_schema( - by_alias=False, - schema_generator=ConnectorConfigSchemaGenerator, - mode="validation", - ) - - @staticmethod - def __process_value_to_type(value: CONFIGURATION_TYPES, is_number_hint: bool): - if value is None: - return value - if isinstance(value, int) or is_number_hint: - return int(value) - if isinstance(value, str): - if is_truthy(value): - return True - if is_falsy(value): - return False - if len(value) == 0: - return None - return value - - def __dig_config_sources_for_key( - self, config: ConfigurationHint - ) -> CONFIGURATION_TYPES: - result = EnvironmentSource.get(config.env) or DictionarySource.get( - config.file_path, self.__config_values - ) - return result or config.default diff --git a/pyoaev/configuration/connector_config_schema_generator.py b/pyoaev/configuration/connector_config_schema_generator.py index 3c8ac436..14b6bf3c 100644 --- a/pyoaev/configuration/connector_config_schema_generator.py +++ b/pyoaev/configuration/connector_config_schema_generator.py @@ -1,127 +1,20 @@ -## ADAPTED FROM https://github.com/OpenCTI-Platform/connectors/blob/5c8cf1235f62f5651c9c08d0b67f1bd182662c8a/shared/tools/composer/generate_connectors_config_schemas/generate_connector_config_json_schema.py.sample +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -from copy import deepcopy -from typing import override +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -from pydantic.json_schema import GenerateJsonSchema +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr -# attributes filtered from the connector configuration before generating the manifest -__FILTERED_ATTRIBUTES__ = [ - # connector id is generated - "CONNECTOR_ID", +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ConnectorConfigSchemaGenerator", ] - -class ConnectorConfigSchemaGenerator(GenerateJsonSchema): - @staticmethod - def dereference_schema(schema_with_refs): - """Return a new schema with all internal $ref resolved.""" - - def _resolve(schema, root): - if isinstance(schema, dict): - if "$ref" in schema: - ref_path = schema["$ref"] - if ref_path.startswith("#/$defs/"): - def_name = ref_path.split("/")[-1] - # Deep copy to avoid mutating $defs - resolved = deepcopy(root["$defs"][def_name]) - return _resolve(resolved, root) - else: - raise ValueError(f"Unsupported ref format: {ref_path}") - else: - return { - schema_key: _resolve(schema_value, root) - for schema_key, schema_value in schema.items() - } - elif isinstance(schema, list): - return [_resolve(item, root) for item in schema] - else: - return schema - - return _resolve(deepcopy(schema_with_refs), schema_with_refs) - - @staticmethod - def flatten_config_loader_schema(root_schema: dict): - """ - Flatten config loader schema so all config vars are described at root level. - - :param root_schema: Original schema. - :return: Flatten schema. - """ - flat_json_schema = { - "$schema": root_schema["$schema"], - "$id": root_schema["$id"], - "type": "object", - "properties": {}, - "required": [], - "additionalProperties": root_schema.get("additionalProperties", True), - } - - for ( - config_loader_namespace_name, - config_loader_namespace_schema, - ) in root_schema["properties"].items(): - config_schema = config_loader_namespace_schema.get("properties", {}) - required_config_vars = config_loader_namespace_schema.get("required", []) - - for config_var_name, config_var_schema in config_schema.items(): - property_name = ( - f"{config_loader_namespace_name.upper()}_{config_var_name.upper()}" - ) - - config_var_schema.pop("title", None) - - flat_json_schema["properties"][property_name] = config_var_schema - - if config_var_name in required_config_vars: - flat_json_schema["required"].append(property_name) - - return flat_json_schema - - @staticmethod - def filter_schema(schema): - for filtered_attribute in __FILTERED_ATTRIBUTES__: - if filtered_attribute in schema["properties"]: - del schema["properties"][filtered_attribute] - schema.update( - { - "required": [ - item - for item in schema["required"] - if item != filtered_attribute - ] - } - ) - - return schema - - @override - def generate(self, schema, mode="validation"): - json_schema = super().generate(schema, mode=mode) - - json_schema["$schema"] = self.schema_dialect - json_schema["$id"] = "config.schema.json" - dereferenced_schema = self.dereference_schema(json_schema) - flattened_schema = self.flatten_config_loader_schema(dereferenced_schema) - return self.filter_schema(flattened_schema) - - @override - def nullable_schema(self, schema): - """Generates a JSON schema that matches a schema that allows null values. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - - Notes: - This method overrides `GenerateJsonSchema.nullable_schema` to generate schemas without `anyOf` keyword. - """ - null_schema = {"type": "null"} - inner_json_schema = self.generate_inner(schema["schema"]) - - if inner_json_schema == null_schema: - return null_schema - else: - return inner_json_schema +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/configuration/settings_loader.py b/pyoaev/configuration/settings_loader.py index 908dfbfd..39d52a2c 100644 --- a/pyoaev/configuration/settings_loader.py +++ b/pyoaev/configuration/settings_loader.py @@ -1,147 +1,26 @@ -import os -from abc import ABC -from datetime import timedelta -from pathlib import Path -from typing import Annotated, Literal -from uuid import UUID - -from pydantic import BaseModel, ConfigDict, Field, HttpUrl, PlainSerializer -from pydantic_settings import ( - BaseSettings, - DotEnvSettingsSource, - PydanticBaseSettingsSource, - SettingsConfigDict, - YamlConfigSettingsSource, -) - - -class BaseConfigModel(BaseModel, ABC): - """Base class for global config models - To prevent attributes from being modified after initialization. - """ - - model_config = ConfigDict(extra="allow", frozen=True, validate_default=True) - - -class SettingsLoader(BaseSettings): - model_config = SettingsConfigDict( - frozen=True, - extra="allow", - env_nested_delimiter="_", - env_nested_max_split=1, - enable_decoding=False, - ) - - @classmethod - def settings_customise_sources( - cls, - settings_cls: type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> tuple[PydanticBaseSettingsSource, ...]: - """Customise the sources of settings for the connector. - - This method is called by the Pydantic BaseSettings class to determine the order of sources. - The configuration come in this order either from: - 1. Environment variables - 2. YAML file - 3. .env file - 4. Default values - - The variables loading order will remain the same as in `pycti.get_config_variable()`: - 1. If a config.yml file is found, the order will be: `ENV VAR` → config.yml → default value - 2. If a .env file is found, the order will be: `ENV VAR` → .env → default value - """ - _main_path = os.curdir - - settings_cls.model_config["env_file"] = f"{_main_path}/../.env" - - if not settings_cls.model_config["yaml_file"]: - if Path(f"{_main_path}/config.yml").is_file(): - settings_cls.model_config["yaml_file"] = f"{_main_path}/config.yml" - if Path(f"{_main_path}/../config.yml").is_file(): - settings_cls.model_config["yaml_file"] = f"{_main_path}/../config.yml" - - if Path(settings_cls.model_config["yaml_file"] or "").is_file(): # type: ignore - return ( - env_settings, - YamlConfigSettingsSource(settings_cls), - ) - if Path(settings_cls.model_config["env_file"] or "").is_file(): # type: ignore - return ( - env_settings, - DotEnvSettingsSource(settings_cls), - ) - return (env_settings,) - - -LogLevelToLower = Annotated[ - Literal["debug", "info", "warn", "error"], - PlainSerializer(lambda v: "".join(v), return_type=str), -] - -HttpUrlToString = Annotated[HttpUrl, PlainSerializer(str, return_type=str)] -TimedeltaInSeconds = Annotated[ - timedelta, PlainSerializer(lambda v: int(v.total_seconds()), return_type=int) +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "BaseConfigModel", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "HttpUrlToString", + "LogLevelToLower", + "SettingsLoader", + "TimedeltaInSeconds", ] - -class ConfigLoaderOAEV(BaseConfigModel): - """OpenAEV/OpenAEV platform configuration settings. - - Contains URL and authentication token for connecting to the OpenAEV platform. - """ - - url: HttpUrlToString = Field( - description="The OpenAEV platform URL.", - ) - token: str = Field( - description="The token for the OpenAEV platform.", - ) - tenant_id: UUID | None = Field( - default=None, - description="Identifier of the tenant within the OpenAEV platform. Used in multi-tenant environments to scope " - "API requests and ensure data isolation between different tenants.", - ) - - -class ConfigLoaderCollector(BaseConfigModel): - """Base collector configuration settings. - - Contains common collector settings including identification, logging, - scheduling, and platform information. - """ - - id: str = Field(description="ID of the collector.") - - name: str = Field(description="Name of the collector") - - log_level: LogLevelToLower | None = Field( - default="error", - description="Determines the verbosity of the logs.", - ) - period: timedelta | None = Field( - default=timedelta(minutes=1), - description="Duration between two scheduled runs of the collector (ISO 8601 format).", - ) - icon_filepath: str | None = Field( - description="Path to the icon file of the collector.", - ) - author: str | None = Field( - default=None, - description="Optional author override for the connector's payloads and contracts. " - "When absent, the platform attributes them to the connector's name.", - ) - platform_description: str | None = Field( - default=None, - description="Description applied to the security platform auto-created by the " - "collector, so its read-only card is not empty in the platform UI.", - ) - platform_tags: list[str] | str | None = Field( - default=None, - description="Tag names applied to the security platform auto-created by the " - "collector. Accepts a list of names or a single comma-separated string " - "(the usual shape when configured through an environment variable).", - ) +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/configuration/sources.py b/pyoaev/configuration/sources.py index aeb53850..9481637b 100644 --- a/pyoaev/configuration/sources.py +++ b/pyoaev/configuration/sources.py @@ -1,47 +1,21 @@ -import os - - -class EnvironmentSource: - """A utility for fecthing a value in the env vars.""" - - @classmethod - def get(cls, env_var: str) -> str | None: - """Gets the value for the specified env var - - :param env_var: the name of the env var to query - :type env_var: str - - :return: value of the env var, or None if not found - :rtype: str | None - """ - return os.getenv(env_var) if env_var else None - - -class DictionarySource: - """A utility for fetching a value from within a JSON-like (nested dict) structure""" - - # this is quite hacky - # it only strictly handles two levels of keys in a dict - @classmethod - def get(cls, config_key_path: list[str] | None, source_dict: dict) -> str | None: - """Gets the value for the specified env var - - :param config_key_path: the two-level dictionary path to the config key - :type config_key_path: list[str] - :param source_dict: JSON-like (nested dict) structure containing config values. - :type source_dict: dict - - :return: value for the config key at specified path, or None if not found - :rtype: str | None - """ - if config_key_path is None: - return None - - assert ( - isinstance(config_key_path, list) - and len(config_key_path) == 2 - and all([len(path_part) > 0 for path_part in config_key_path]) - ) - return source_dict.get(config_key_path[0], {config_key_path[1]: None}).get( - config_key_path[1] - ) +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "DictionarySource", + "EnvironmentSource", +] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/contracts/__init__.py b/pyoaev/contracts/__init__.py index a9b7be78..730c4484 100644 --- a/pyoaev/contracts/__init__.py +++ b/pyoaev/contracts/__init__.py @@ -1,5 +1,25 @@ -from .contract_builder import ContractBuilder # noqa: F401,F403 +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -__all__ = [ +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) "ContractBuilder", ] + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/contracts/contract_builder.py b/pyoaev/contracts/contract_builder.py index d7b583ea..0e1b380c 100644 --- a/pyoaev/contracts/contract_builder.py +++ b/pyoaev/contracts/contract_builder.py @@ -1,44 +1,27 @@ -from typing import List - -from pyoaev.contracts.contract_config import ContractElement, ContractOutputElement - - -class ContractBuilder: - fields: List[ContractElement] - outputs: List[ContractOutputElement] - - def __init__(self): - self.fields = [] - self.outputs = [] - - def add_fields(self, fields: List[ContractElement]): - self.fields = self.fields + fields - return self - - def add_outputs(self, outputs: List[ContractOutputElement]): - self.outputs = self.outputs + outputs - return self - - def mandatory(self, element: ContractElement): - element.mandatory = True - self.fields.append(element) - return self - - def optional(self, element: ContractElement): - element.mandatory = False - self.fields.append(element) - return self - - def mandatory_group(self, elements: List[ContractElement]): - keys: List[str] = list(map(lambda iterable: iterable.key, elements)) - for element in elements: - element.mandatory = True - element.mandatoryGroups = keys - self.fields.append(element) - return self - - def build_fields(self) -> List[ContractElement]: - return self.fields - - def build_outputs(self) -> List[ContractOutputElement]: - return self.outputs +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ContractBuilder", + "ContractElement", + "ContractOutputElement", +] + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/contracts/contract_config.py b/pyoaev/contracts/contract_config.py index 7d5e95e6..18623977 100644 --- a/pyoaev/contracts/contract_config.py +++ b/pyoaev/contracts/contract_config.py @@ -1,462 +1,61 @@ -import json -from abc import ABC, abstractmethod -from dataclasses import dataclass, field -from enum import Enum -from typing import Dict, List, Optional - -from pyoaev import utils -from pyoaev.contracts.contract_utils import ContractCardinality, ContractVariable -from pyoaev.contracts.variable_helper import VariableHelper -from pyoaev.credential.types import CredentialType - - -class SupportedLanguage(str, Enum): - fr: str = "fr" - en: str = "en" - - -class ContractFieldType(str, Enum): - Text: str = "text" - Number: str = "number" - Tuple: str = "tuple" - Checkbox: str = "checkbox" - Textarea: str = "textarea" - Select: str = "select" - Article: str = "article" - Challenge: str = "challenge" - DependencySelect: str = "dependency-select" - Attachment: str = "attachment" - Team: str = "team" - Expectation: str = "expectation" - Asset: str = "asset" - AssetGroup: str = "asset-group" - AiTarget: str = "ai-target" - Payload: str = "payload" - CredentialReference: str = "credential-reference" - - -class ContractFieldKey(str, Enum): - Asset: str = "assets" - AssetGroup: str = "asset_groups" - CredentialReference: str = "credential_reference" - - -class ContractOutputType(str, Enum): - Text: str = "text" - ActionOutput: str = "action_output" - Number: str = "number" - Port: str = "port" - PortsScan: str = "portscan" - IPv4: str = "ipv4" - IPv6: str = "ipv6" - CVE: str = "cve" - Asset: str = "asset" - Credentials: str = "credentials" - Username: str = "username" - Share: str = "share" - File: str = "file" - AdminUsername: str = "admin_username" - Group: str = "group" - Computer: str = "computer" - PasswordPolicy: str = "password_policy" - Delegation: str = "delegation" - Sid: str = "sid" - Vulnerability: str = "vulnerability" - AccountWithPasswordNotRequired: str = "account_with_password_not_required" - ExpectationSignature: str = "expectation_signature" - AsreproastableAccount: str = "asreproastable_account" - KerberoastableAccount: str = "kerberoastable_account" - - -class PrimitiveType(str, Enum): - """The semantic type of a contract argument's value, independent of how the - field renders in the UI (see ``ContractFieldType``) and independent of - chaining specifically — an argument can carry a ``PrimitiveType`` and be - filled by hand, exactly as it can be auto-linked from a prior step's - matching output. Mirrors ``io.openaev.database.model.PrimitiveType`` - (the openaev platform's own enum) label-for-label; every value here must - stay in sync with it. - """ - - AccountWithPasswordNotRequired: str = "account_with_password_not_required" - ActionOutput: str = "action_output" - AdminUsername: str = "admin_username" - AsreproastableAccount: str = "asreproastable_account" - AssetGroupId: str = "asset_group_id" - AssetId: str = "asset_id" - ComputerName: str = "computer_name" - CVE: str = "cve" - DelegationAccount: str = "delegation_account" - Document: str = "document" - Domain: str = "domain" - FileName: str = "file_name" - FilePath: str = "file_path" - GroupName: str = "group_name" - Hash: str = "hash" - Host: str = "host" - IPv4: str = "ipv4" - IPv6: str = "ipv6" - IpSubnet: str = "ip_subnet" - KerberoastableAccount: str = "kerberoastable_account" - Key: str = "key" - Number: str = "number" - Password: str = "password" - Permissions: str = "permissions" - Port: str = "port" - Service: str = "service" - Severity: str = "severity" - ShareName: str = "share_name" - SID: str = "sid" - TargetedAsset: str = "targeted-asset" - Text: str = "text" - Username: str = "username" - Value: str = "value" - VulnerabilityName: str = "vulnerability_name" - VulnerabilityStatus: str = "vulnerability_status" - - -class ExpectationType(str, Enum): - text: str = "TEXT" - document: str = "DOCUMENT" - article: str = "ARTICLE" - challenge: str = "CHALLENGE" - manual: str = "MANUAL" - prevention: str = "PREVENTION" - detection: str = "DETECTION" - vulnerability: str = "VULNERABILITY" - - -class SecurityPlatformType(str, Enum): - """Categories of security platform expected to fulfil a technical expectation. - - When an expectation declares one or more of these, the platform focuses the - technical (DETECTION / PREVENTION / VULNERABILITY) result on collectors of - those types only (instead of every connected security platform). An empty - list means "any security platform". - """ - - EDR: str = "EDR" - XDR: str = "XDR" - SIEM: str = "SIEM" - SOAR: str = "SOAR" - NDR: str = "NDR" - ISPM: str = "ISPM" - LLM_FIREWALL: str = "LLM_FIREWALL" - AI_GATEWAY: str = "AI_GATEWAY" - - -@dataclass -class Expectation: - expectation_type: ExpectationType - expectation_name: str - expectation_description: str - expectation_score: int - expectation_expectation_group: bool - expectation_is_predefined: bool = False - expectation_is_multi_selectable: bool = False - # Security platform types expected to fulfil this expectation. Empty = any - # platform (unchanged behaviour). Typically set for technical expectations - # (DETECTION / PREVENTION / VULNERABILITY), left empty for MANUAL ones. - expectation_expected_security_platform_types: List[SecurityPlatformType] = field( - default_factory=list - ) - - -@dataclass -class LinkedFieldModel: - key: str - type: ContractFieldType - - -@dataclass -class ContractElement(ABC): - key: str - label: str - type: str = field(default="", init=False) - mandatoryGroups: List[str] = field(default_factory=list) - mandatoryConditionFields: List[str] = field(default_factory=list) - mandatoryConditionValues: Dict[str, any] = field(default_factory=list) - visibleConditionFields: List[str] = field(default_factory=list) - visibleConditionValues: Dict[str, any] = field(default_factory=list) - linkedFields: List[str] = field(default_factory=list) - mandatory: bool = False - readOnly: bool = False - # The argument's chaining/semantic type (``PrimitiveType``), e.g. "username" - # or "host" — matches the platform's own ``argumentType`` field - # (``io.openaev.database.model.ContractElement``) label-for-label so a - # contract pushed from here needs no translation on the other end. Left - # unset (``None``) for fields with no established type: the platform - # normalizes a missing/null/empty value to ``PrimitiveType.Text`` on its - # own, so omitting this is always safe and never a breaking change for - # existing contracts. - argumentType: Optional[PrimitiveType] = None - - @property - @abstractmethod - def get_type(self) -> str: - pass - - def __post_init__(self): - self.type = self.get_type - - -@dataclass -class ContractCardinalityElement(ContractElement, ABC): - cardinality: str = ContractCardinality.One - defaultValue: List[str] = field(default_factory=list) - - -@dataclass -class ContractOutputElement(ABC): - type: str - field: str - labels: List[str] - isFindingCompatible: bool - isMultiple: bool - - -@dataclass -class ContractConfig: - type: str - expose: bool - label: dict[SupportedLanguage, str] - color_dark: str - color_light: str - - -@dataclass -class Domain: - domain_id: str - domain_name: str - domain_color: str - domain_created_at: str - domain_updated_at: str - - -@dataclass -class Contract: - contract_id: str - label: dict[SupportedLanguage, str] - fields: List[ContractElement] - outputs: List[ContractOutputElement] - config: ContractConfig - manual: bool - variables: List[ContractVariable] = field( - default_factory=lambda: ( - [ - VariableHelper.user_variable(), - VariableHelper.exercise_variable(), - VariableHelper.team_variable(), - ] - + VariableHelper.uri_variables() - ) - ) - contract_attack_patterns_external_ids: List[str] = field(default_factory=list) - contract_vulnerability_external_ids: List[str] = field(default_factory=list) - is_atomic_testing: bool = True - platforms: List[str] = field(default_factory=list) - external_id: str = None - domains: List[Domain] = None - - def add_attack_pattern(self, var: str): - self.contract_attack_patterns_external_ids.append(var) - - def add_vulnerability(self, var: str): - self.contract_vulnerability_external_ids.append(var) - - def add_variable(self, var: ContractVariable): - self.variables.append(var) - - def to_contract_add_input(self, source_id: str): - return { - "contract_id": self.contract_id, - "external_contract_id": self.external_id, - "injector_id": source_id, - "contract_manual": self.manual, - "contract_labels": self.label, - "contract_attack_patterns_external_ids": self.contract_attack_patterns_external_ids, - "contract_vulnerability_external_ids": self.contract_vulnerability_external_ids, - "contract_content": json.dumps(self, cls=utils.EnhancedJSONEncoder), - "is_atomic_testing": self.is_atomic_testing, - "contract_platforms": self.platforms, - "contract_domains": self.domains, - } - - def to_contract_update_input(self): - return { - "contract_manual": self.manual, - "contract_labels": self.label, - "contract_attack_patterns_external_ids": self.contract_attack_patterns_external_ids, - "contract_vulnerability_external_ids": self.contract_vulnerability_external_ids, - "contract_content": json.dumps(self, cls=utils.EnhancedJSONEncoder), - "is_atomic_testing": self.is_atomic_testing, - "contract_platforms": self.platforms, - "contract_domains": self.domains, - } - - -@dataclass -class ContractTeam(ContractCardinalityElement): - @property - def get_type(self) -> str: - return ContractFieldType.Team.value - - -@dataclass -class ContractText(ContractCardinalityElement): - defaultValue: str = "" - - @property - def get_type(self) -> str: - return ContractFieldType.Text.value - - -def prepare_contracts(contracts): - return list( - map( - lambda c: { - "contract_id": c.contract_id, - "contract_labels": c.label, - "contract_attack_patterns_external_ids": c.contract_attack_patterns_external_ids, - "contract_content": json.dumps(c, cls=utils.EnhancedJSONEncoder), - "contract_platforms": c.platforms, - "contract_domains": c.domains, - }, - contracts, - ) - ) - - -@dataclass -class ContractTuple(ContractCardinalityElement): - def __post_init__(self): - super().__post_init__() - self.cardinality = ContractCardinality.Multiple - - attachmentKey: str = None - contractAttachment: bool = attachmentKey is not None - tupleFilePrefix: str = "file :: " - - @property - def get_type(self) -> str: - return ContractFieldType.Tuple.value - - -@dataclass -class ContractTextArea(ContractCardinalityElement): - defaultValue: str = "" - richText: bool = False - - @property - def get_type(self) -> str: - return ContractFieldType.Textarea.value - - -@dataclass -class ContractCheckbox(ContractElement): - defaultValue: bool = False - - @property - def get_type(self) -> str: - return ContractFieldType.Checkbox.value - - -@dataclass -class ContractAttachment(ContractCardinalityElement): - @property - def get_type(self) -> str: - return ContractFieldType.Attachment.value - - -@dataclass -class ContractReferencedCredential(ContractElement): - key: str = field(default=ContractFieldKey.CredentialReference.value, init=False) - label: str = "Select a credential reference" - mandatory: bool = False - credential_reference_type: Optional[CredentialType] = None - multiple: bool = True - - @property - def get_type(self) -> str: - return ContractFieldType.CredentialReference.value - - -@dataclass -class ContractExpectations(ContractCardinalityElement): - cardinality: str = ContractCardinality.Multiple - # Full set the user may choose from in the "add expectation" picker. - availableExpectations: List[Expectation] = field(default_factory=list) - # Subset pre-filled by default when the inject / atomic testing is created. - # The OpenAEV platform reads this field-level array (not the per-expectation - # flag) to pre-populate expectations, so it must be emitted in the contract. - # None means "not provided": the list is then derived from the expectations - # flagged expectation_is_predefined=True. Pass an explicit list (possibly - # empty) to override the derivation. - predefinedExpectations: Optional[List[Expectation]] = None - - def __post_init__(self): - super().__post_init__() - # When the caller did not pass an explicit predefined list, derive it from - # the expectations flagged predefined in the available set. This keeps the - # ergonomic single-list API (flag each Expectation with - # expectation_is_predefined=True) working end-to-end on the platform. - if self.predefinedExpectations is None: - self.predefinedExpectations = [ - expectation - for expectation in self.availableExpectations - if expectation.expectation_is_predefined - ] - - @property - def get_type(self) -> str: - return ContractFieldType.Expectation.value - - -@dataclass -class ContractSelect(ContractCardinalityElement): - choices: dict[str, str] = None - - @property - def get_type(self) -> str: - return ContractFieldType.Select.value - - -@dataclass -class ContractAsset(ContractCardinalityElement): - key: str = field(default=ContractFieldKey.Asset.value, init=False) - - @property - def get_type(self) -> str: - return ContractFieldType.Asset.value - - -@dataclass -class ContractAssetGroup(ContractCardinalityElement): - key: str = field(default=ContractFieldKey.AssetGroup.value, init=False) - - @property - def get_type(self) -> str: - return ContractFieldType.AssetGroup.value - - -@dataclass -class ContractAiTarget(ContractCardinalityElement): - """Selector for a pre-configured AiTarget asset (LLM endpoint / AI agent under test). - - Renders in the UI as an entity autocomplete (backed by the platform ``/ai_targets`` - options endpoint), analogous to the asset / asset group pickers. The selected value is - the AI target's ``asset_id`` (AI targets are AI_TARGET-category assets, see - ``AiTarget._id_attr``), resolved by the executing injector at run time. Unlike - ``ContractAsset`` the key is free-form so a contract can name the field as it wishes. - """ - - @property - def get_type(self) -> str: - return ContractFieldType.AiTarget.value - - -@dataclass -class ContractPayload(ContractCardinalityElement): - @property - def get_type(self) -> str: - return ContractFieldType.Payload.value +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +``ExpectationType`` was renamed ``ContractExpectationType``. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "SupportedLanguage", + "ContractFieldType", + "ContractFieldKey", + "ContractOutputType", + "PrimitiveType", + "ExpectationType", + "SecurityPlatformType", + "Expectation", + "LinkedFieldModel", + "ContractElement", + "ContractCardinalityElement", + "ContractOutputElement", + "ContractConfig", + "Domain", + "Contract", + "ContractTeam", + "ContractText", + "prepare_contracts", + "ContractTuple", + "ContractTextArea", + "ContractCheckbox", + "ContractAttachment", + "ContractExpectations", + "ContractSelect", + "ContractAsset", + "ContractAssetGroup", + "ContractAiTarget", + "ContractPayload", + "ContractReferencedCredential", + "CredentialType", + "EnhancedJSONEncoder", + "ContractCardinality", + "ContractVariable", + "VariableHelper", +] + +_RENAMED = {"ExpectationType": "ContractExpectationType"} + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {_RENAMED.get(name, name)}' instead.", + getattr(_sdk, _RENAMED.get(name, name)), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/contracts/contract_utils.py b/pyoaev/contracts/contract_utils.py index 903f6c4c..d31c0220 100644 --- a/pyoaev/contracts/contract_utils.py +++ b/pyoaev/contracts/contract_utils.py @@ -1,22 +1,27 @@ -from dataclasses import dataclass, field -from enum import Enum -from typing import List +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -class ContractCardinality(str, Enum): - One: str = "1" - Multiple: str = "n" +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ContractCardinality", + "VariableType", + "ContractVariable", +] -class VariableType(str, Enum): - String: str = "String" - Object: str = "Object" - - -@dataclass -class ContractVariable: - key: str - label: str - type: VariableType - cardinality: ContractCardinality - children: List["ContractVariable"] = field(default_factory=list) +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/contracts/variable_helper.py b/pyoaev/contracts/variable_helper.py index 7649d84b..4018a182 100644 --- a/pyoaev/contracts/variable_helper.py +++ b/pyoaev/contracts/variable_helper.py @@ -1,124 +1,49 @@ -from pyoaev.contracts.contract_utils import ( - ContractCardinality, - ContractVariable, - VariableType, -) - -USER = "user" -EXERCISE = "exercise" -TEAMS = "teams" -COMCHECK = "comcheck" -PLAYER_URI = "player_uri" -CHALLENGES_URI = "challenges_uri" -SCOREBOARD_URI = "scoreboard_uri" -LESSONS_URI = "lessons_uri" +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +The module constants (``USER``, ``EXERCISE``, ...) are not part of the SDK +public surface; they keep their values here and warn. +""" -class VariableHelper: - @staticmethod - def user_variable(): - return ContractVariable( - key=USER, - label="User that will receive the injection", - type=VariableType.String, - cardinality=ContractCardinality.One, - children=[ - ContractVariable( - key=USER + ".id", - label="Id of the user in the platform", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=USER + ".email", - label="Email of the user", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=USER + ".firstname", - label="Firstname of the user", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=USER + ".lastname", - label="Lastname of the user", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=USER + ".lang", - label="Lang of the user", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ], - ) +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr - @staticmethod - def exercise_variable(): - return ContractVariable( - key=EXERCISE, - label="Exercise of the current injection", - type=VariableType.Object, - cardinality=ContractCardinality.One, - children=[ - ContractVariable( - key=EXERCISE + ".id", - label="Id of the exercise in the platform", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=EXERCISE + ".name", - label="Name of the exercise", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=EXERCISE + ".description", - label="Description of the exercise", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ], - ) +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "VariableHelper", + "ContractCardinality", + "ContractVariable", + "VariableType", +] - @staticmethod - def team_variable(): - return ContractVariable( - key=TEAMS, - label="List of team name for the injection", - type=VariableType.String, - cardinality=ContractCardinality.Multiple, - ) +_CONSTANTS = { + "USER": "user", + "EXERCISE": "exercise", + "TEAMS": "teams", + "COMCHECK": "comcheck", + "PLAYER_URI": "player_uri", + "CHALLENGES_URI": "challenges_uri", + "SCOREBOARD_URI": "scoreboard_uri", + "LESSONS_URI": "lessons_uri", +} - @staticmethod - def uri_variables(): - return [ - ContractVariable( - key=PLAYER_URI, - label="Player interface platform link", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=CHALLENGES_URI, - label="Challenges interface platform link", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=SCOREBOARD_URI, - label="Scoreboard interface platform link", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=LESSONS_URI, - label="Lessons learned interface platform link", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ] +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + **{ + name: ( + f"'{name}' is deprecated and has no public replacement in " + "xtm_oaev_sdk; use the VariableHelper built-in variables instead.", + value, + ) + for name, value in _CONSTANTS.items() + }, + }, + module_name=__name__, +) diff --git a/pyoaev/credential/__init__.py b/pyoaev/credential/__init__.py index 5086830e..abcc1536 100644 --- a/pyoaev/credential/__init__.py +++ b/pyoaev/credential/__init__.py @@ -1,15 +1,26 @@ -from .types import CredentialType +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -def build_single_referenced_credential_element(provider_name: str): - from .utils import ( - build_single_referenced_credential_element as _build_single_referenced_credential_element, - ) +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr - return _build_single_referenced_credential_element(provider_name) - - -__all__ = [ +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) "CredentialType", "build_single_referenced_credential_element", ] + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/credential/types.py b/pyoaev/credential/types.py index f2600a4d..a018bb10 100644 --- a/pyoaev/credential/types.py +++ b/pyoaev/credential/types.py @@ -1,16 +1,25 @@ -"""Credential reference types shared with the OpenAEV platform. +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -This enum is the Python mirror of the OpenAEV backend credential-type enum -used by credential secret references. Values must stay label-for-label in sync -with the platform so contracts can declare credential references without any -translation layer. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. """ -from enum import Enum +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "CredentialType", +] -class CredentialType(str, Enum): - IDENTITY = "IDENTITY" - CLOUD_AWS = "CLOUD_AWS" - CLOUD_AZURE = "CLOUD_AZURE" - CLOUD_GCP = "CLOUD_GCP" +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/credential/utils.py b/pyoaev/credential/utils.py index 3975e71f..bf7a91d9 100644 --- a/pyoaev/credential/utils.py +++ b/pyoaev/credential/utils.py @@ -1,41 +1,27 @@ -"""Helpers centralizing how injectors declare credential-reference fields. +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -Injectors can call ``build_single_referenced_credential_element`` instead of -re-implementing provider-to-credential-type mapping in each project. This keeps -contract generation consistent across the Python ecosystem and aligns the -serialized contract payload with the OpenAEV platform's expected values. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. """ -from typing import Optional - -from pyoaev.contracts.contract_config import ContractReferencedCredential -from pyoaev.credential.types import CredentialType - -_PROVIDER_TO_CREDENTIAL_TYPE = { - "aws": CredentialType.CLOUD_AWS, - "eks": CredentialType.CLOUD_AWS, - "azure": CredentialType.CLOUD_AZURE, - "gcp": CredentialType.CLOUD_GCP, -} - - -def _resolve_credential_type(provider_name: str) -> Optional[CredentialType]: - normalized_provider = provider_name.casefold() - return _PROVIDER_TO_CREDENTIAL_TYPE.get(normalized_provider) - - -def build_single_referenced_credential_element( - provider_name: str, -) -> ContractReferencedCredential: - """Build a credential-reference field for a provider-specific contract - with ``multiple`` value at ``False``. - - This is the centralized entry point injectors should use when they need the - OpenAEV inject form to ask for one referenced credential. The helper keeps - provider-to-``CredentialType`` mapping consistent across injector projects. - """ - - return ContractReferencedCredential( - credential_reference_type=_resolve_credential_type(provider_name), - multiple=False, - ) +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ContractReferencedCredential", + "CredentialType", + "build_single_referenced_credential_element", +] + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/daemons/__init__.py b/pyoaev/daemons/__init__.py index 4a0be2a9..2586f74d 100644 --- a/pyoaev/daemons/__init__.py +++ b/pyoaev/daemons/__init__.py @@ -1,4 +1,24 @@ -from .base_daemon import BaseDaemon -from .collector_daemon import CollectorDaemon +"""DEPRECATED: the daemon runtime moved to :mod:`xtm_oaev_sdk` and +:mod:`collectors_sdk`; the legacy names keep building the OpenAEV client.""" -__all__ = ["BaseDaemon", "CollectorDaemon"] +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +from .base_daemon import _ClientBaseDaemon +from .collector_daemon import _COLLECTOR_DAEMON_MESSAGE, _ClientCollectorDaemon + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "BaseDaemon", + "CollectorDaemon", +] + +__getattr__ = _make_deprecated_getattr( + { + "BaseDaemon": ( + "Use 'from xtm_oaev_sdk import BaseDaemon' and pass " + "api_client=OpenAEV(...) instead.", + _ClientBaseDaemon, + ), + "CollectorDaemon": (_COLLECTOR_DAEMON_MESSAGE, _ClientCollectorDaemon), + }, + module_name=__name__, +) diff --git a/pyoaev/daemons/base_daemon.py b/pyoaev/daemons/base_daemon.py index 9728a872..2e98d162 100644 --- a/pyoaev/daemons/base_daemon.py +++ b/pyoaev/daemons/base_daemon.py @@ -1,141 +1,44 @@ -import argparse -from abc import ABC, abstractmethod -from inspect import signature -from types import FunctionType -from uuid import UUID +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -from pyoaev.client import OpenAEV -from pyoaev.configuration import Configuration -from pyoaev.exceptions import OpenAEVError -from pyoaev.utils import logger +The SDK ``BaseDaemon`` takes an injected API client. The legacy name resolves +to a thin subclass that keeps building the ``OpenAEV`` client from the +``openaev_url``, ``openaev_token`` and ``openaev_tenant_id`` configuration +keys when none is passed, and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr +from pyoaev.client import OpenAEV -class BaseDaemon(ABC): - """A base class for implementing a kind of daemon that periodically polls - a given callback. - :param configuration: configuration to provide the daemon - (allowing for looking up values within the callback for example) - :type configuration: Configuration - :param callback: a method or function to periodically call, defaults to None. - :type callback: callable, optional - :param logger: a logger object, to log events. if not supplied, a default logger - will be spawned to provide this functionality. - :type logger: Any - :param api_client: an API client that will provide connectivity with other systems. - :type api_client: Any - """ +class _ClientBaseDaemon(_sdk.BaseDaemon): + """SDK BaseDaemon building an OpenAEV client when none is injected.""" - def __init__( - self, - configuration: Configuration, - callback: callable = None, - logger=None, - api_client=None, - ): - self._configuration = configuration - self._callback = callback - self.api = api_client or BaseDaemon.__get_default_api_client( + def _create_default_api_client(self) -> OpenAEV: + return OpenAEV( url=self._configuration.get("openaev_url"), token=self._configuration.get("openaev_token"), tenant_id=self._configuration.get("openaev_tenant_id"), ) - # logging - # compatibility layer: in order for older configs to still work, search for legacy names - actual_log_level = ( - self._configuration.get("log_level") - or self._configuration.get("collector_log_level") - or self._configuration.get("injector_log_level") - or "info" - ) - actual_log_name = ( - self._configuration.get("name") - or self._configuration.get("collector_name") - or self._configuration.get("injector_name") - or "daemon" - ) - self.logger = logger or BaseDaemon.__get_default_logger( - actual_log_level.upper(), - actual_log_name, - ) - - @abstractmethod - def _setup(self): - """A run-once method that inheritors must implement. This serves to instantiate - all useful objects and functionality for the implementor to run. - """ - pass - - @abstractmethod - def _start_loop(self): - """Starts the daemon's main execution loop. Implementors should implement - the main execution logic in here. - """ - pass - - def _try_callback(self): - """Tries to call the configured callback. Note that if any error is thrown, - it is immediately swallowed (but still logged) allowing the collector to keep - running. This is useful for any transient issue (e.g. API endpoint down...). - """ - try: - # this is some black magic to allow injecting the collector daemon instance - # into an arbitrary callback that has a specific argument name - # this allow for avoiding subclassing the CollectorDaemon class just to provide the callback - # Example: - # - # def standalone_func(collector): - # collector.api.call_openaev() - # - # CollectorDaemon(config=, standalone_func).start() - if ( - isinstance(self._callback, FunctionType) - and "collector" in signature(self._callback).parameters - ): - self._callback(collector=self) - else: - self._callback() - except Exception as err: # pylint: disable=broad-except - self.logger.error(f"Error calling: {err}") - - def start(self): - """Start the daemon. This will run the implementor's run-once setup method and - follow-up with the main execution loop. Note that at this point, if there is no - configured callback, the method will abort and kill the daemon. - """ - parser = argparse.ArgumentParser(description="parse daemon options") - parser.add_argument("--dump-config-schema", action="store_true") - args = parser.parse_args() - if args.dump_config_schema: - print(self._configuration.schema()) - return - - if self._callback is None: - raise OpenAEVError("This daemon has no configured callback.") - self._setup() - self._start_loop() - - def set_callback(self, callback: callable): - """Configures a callback to call in the main execution loop. If the callback - was not provided in the daemon's ctor, this should be set before calling start(). - """ - self._callback = callback - - def get_id(self): - """Returns the daemon instance's ID contained in configuration. Configuration - must define any of these keys: `id`, `collector_id`, `injector_id`. - """ - return ( - self._configuration.get("id") - or self._configuration.get("collector_id") - or self._configuration.get("injector_id") - ) - - @classmethod - def __get_default_api_client(cls, url, token, tenant_id: UUID | None): - return OpenAEV(url=url, token=token, tenant_id=tenant_id) - @classmethod - def __get_default_logger(cls, log_level, name): - return logger(log_level)(name) +__all__ = ["BaseDaemon"] # noqa: F822 (served by the PEP 562 __getattr__ below) + +__getattr__ = _make_deprecated_getattr( + { + "BaseDaemon": ( + "Use 'from xtm_oaev_sdk import BaseDaemon' and pass " + "api_client=OpenAEV(...) instead.", + _ClientBaseDaemon, + ), + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in ("Configuration", "OpenAEVError", "logger") + }, + }, + module_name=__name__, +) diff --git a/pyoaev/daemons/collector_daemon.py b/pyoaev/daemons/collector_daemon.py index 642f7441..9d4e26d1 100644 --- a/pyoaev/daemons/collector_daemon.py +++ b/pyoaev/daemons/collector_daemon.py @@ -1,144 +1,55 @@ -import sched -import time - -from pyoaev import Configuration -from pyoaev.daemons import BaseDaemon -from pyoaev.utils import PingAlive - -DEFAULT_PERIOD_SECONDS = 60 - -# Neutral grey used for tags auto-created alongside the security platform, -# consistent with the tag colors already used across collectors. -DEFAULT_PLATFORM_TAG_COLOR = "#6b7280" - - -class CollectorDaemon(BaseDaemon): - """Implementation of a daemon of Collector type. Note that it requires - specific configuration keys to run its setup. - `collector_icon_filepath`: relative path to an icon image (preferably PNG) - `collector_id`: unique identifier for the collector (UUIDv4) - `collector_period`: time to wait in seconds between each loop execution; note - that this time is added to the time the loop takes to run, so the actual total - time between each loop start is time_of_loop+period. - `collector_author` (optional): source-declared author override for the - collector's payloads and contracts; when absent, the platform attributes - them to the collector's name. - `collector_platform_description` (optional): description applied to the - security platform auto-created when `collector_platform` is set, so the - read-only card is not empty in the UI. - `collector_platform_tags` (optional): list of tag names (or a - comma-separated string) applied to the auto-created security platform. - """ - - def __init__( - self, - configuration: Configuration, - callback: callable = None, - logger=None, - api_client=None, - collector_type=None, - ): - super().__init__(configuration, callback, logger, api_client) - if collector_type is None: - raise ValueError("Must define a value for collector type") - self.collector_type = collector_type - - def _setup(self): - if self._configuration.get("collector_period") is None: - self._configuration.set("collector_period", DEFAULT_PERIOD_SECONDS) - icon_path = self._configuration.get("collector_icon_filepath") - icon_name = self._configuration.get("collector_id") + ".png" - with open(icon_path, "rb") as icon_file_handle: - collector_icon = (icon_name, icon_file_handle, "image/png") - document = self.api.document.upsert(document={}, file=collector_icon) - if self._configuration.get("collector_platform") is not None: - security_platform_input = { - "asset_name": self._configuration.get("collector_name"), - "asset_external_reference": self._configuration.get("collector_id"), - "security_platform_type": self._configuration.get( - "collector_platform" - ), - "security_platform_logo_light": document.get("document_id"), - "security_platform_logo_dark": document.get("document_id"), - } - description = self._configuration.get("collector_platform_description") - if description: - security_platform_input["asset_description"] = description - tag_ids = self.__upsert_platform_tags( - self._configuration.get("collector_platform_tags") - ) - if tag_ids: - security_platform_input["asset_tags"] = tag_ids - security_platform = self.api.security_platform.upsert( - security_platform_input - ) - else: - security_platform = {} - security_platform_id = security_platform.get("asset_id") - config = { - "collector_id": self._configuration.get("collector_id"), - "collector_name": self._configuration.get("collector_name"), - "collector_type": self.collector_type, - "collector_period": self._configuration.get("collector_period"), - "collector_security_platform": security_platform_id, - # Optional author override; None lets the platform fall back to - # the collector name. - "collector_author": self._configuration.get("collector_author"), - } - with open(icon_path, "rb") as icon_file_handle: - collector_icon = (icon_name, icon_file_handle, "image/png") - self.api.collector.create(config, collector_icon) - - PingAlive(self.api, config, self.logger, "collector").start() - - def __upsert_platform_tags(self, tag_names) -> list: - """Upserts the configured security platform tags and returns their ids. - - Accepts either a list of tag names or a comma-separated string; tag - upsert failures are logged and skipped so the collector setup never - fails because of a tag. - """ - if not tag_names: - return [] - if isinstance(tag_names, str): - tag_names = tag_names.split(",") - tag_ids = [] - for raw_name in tag_names: - # Strip whitespace for both string and list inputs (YAML list values - # can carry stray spaces too). - tag_name = raw_name.strip() if isinstance(raw_name, str) else raw_name - if not tag_name: - continue - try: - tag = self.api.tag.upsert( - {"tag_name": tag_name, "tag_color": DEFAULT_PLATFORM_TAG_COLOR} - ) - tag_id = tag.get("tag_id") - if tag_id: - tag_ids.append(tag_id) - except Exception as err: # noqa: BLE001 - self.logger.warning( - f"Could not upsert security platform tag {tag_name}: {err}" - ) - return tag_ids - - def _start_loop(self): - scheduler = sched.scheduler(time.time, time.sleep) - delay = self._configuration.get("collector_period") - self._try_callback() - scheduler.enter( - delay=delay, - priority=1, - action=self.__schedule, - argument=(scheduler, self._try_callback, delay), - ) - scheduler.run() - - def __schedule(self, scheduler, callback, delay): - callback() - scheduler.enter( - delay=delay, - priority=1, - action=self.__schedule, - argument=(scheduler, callback, delay), - ) +"""DEPRECATED: moved to :mod:`collectors_sdk`. + +The collectors-sdk ``CollectorDaemon`` takes an injected API client. The legacy +name resolves to a thin subclass that keeps building the ``OpenAEV`` client +from configuration when none is passed. Every other name resolves to the +*identical* SDK object. All names emit a :class:`DeprecationWarning` on access. +""" + +import collectors_sdk as _collectors_sdk +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +from pyoaev.daemons.base_daemon import _ClientBaseDaemon + + +class _ClientCollectorDaemon(_collectors_sdk.CollectorDaemon, _ClientBaseDaemon): + """collectors-sdk CollectorDaemon building an OpenAEV client when none is injected.""" + + +_COLLECTOR_DAEMON_MESSAGE = ( + "Use 'from collectors_sdk import CollectorDaemon' and pass " + "api_client=OpenAEV(...) instead." +) + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "CollectorDaemon", + "DEFAULT_PERIOD_SECONDS", + "DEFAULT_PLATFORM_TAG_COLOR", +] + +__getattr__ = _make_deprecated_getattr( + { + "CollectorDaemon": (_COLLECTOR_DAEMON_MESSAGE, _ClientCollectorDaemon), + "BaseDaemon": ( + "Use 'from xtm_oaev_sdk import BaseDaemon' and pass " + "api_client=OpenAEV(...) instead.", + _ClientBaseDaemon, + ), + **{ + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in ("DEFAULT_PERIOD_SECONDS", "DEFAULT_PLATFORM_TAG_COLOR") + }, + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in ("Configuration", "PingAlive") + }, + }, + module_name=__name__, +) diff --git a/pyoaev/exceptions.py b/pyoaev/exceptions.py index cca8e842..5f00376b 100644 --- a/pyoaev/exceptions.py +++ b/pyoaev/exceptions.py @@ -1,198 +1,36 @@ -import functools -from typing import TYPE_CHECKING, Any, Callable, Optional, Type, TypeVar, Union, cast - - -class OpenAEVError(Exception): - def __init__( - self, - error_message: Union[str, bytes] = "", - response_code: Optional[int] = None, - response_body: Optional[bytes] = None, - ) -> None: - Exception.__init__(self, error_message) - # Http status code - self.response_code = response_code - # Full http response - self.response_body = response_body - try: - # if we receive str/bytes we try to convert to unicode/str to have - # consistent message types (see #616) - if TYPE_CHECKING: - assert isinstance(error_message, bytes) - self.error_message = error_message.decode() - except Exception: - if TYPE_CHECKING: - assert isinstance(error_message, str) - self.error_message = error_message - - def __str__(self) -> str: - # Start with the provided error message - message = self.error_message - - # List of generic HTTP status messages that indicate we should look deeper - generic_messages = ( - "Internal Server Error", - "Bad Request", - "Not Found", - "Unauthorized", - "Forbidden", - "Service Unavailable", - "Gateway Timeout", - "Unknown error", - "Validation Failed", - ) - - # Only try to extract from response body if message is truly generic - # Don't override if we already have a specific error message - if ( - not message or (message in generic_messages and len(message) < 30) - ) and self.response_body is not None: - try: - import json - - body = self.response_body.decode(errors="ignore") - data = json.loads(body) - extracted_msg = None - - if isinstance(data, dict): - # Try various common error fields - if "error" in data: - err = data.get("error") - if isinstance(err, dict) and "message" in err: - extracted_msg = err.get("message") - elif isinstance(err, str): - extracted_msg = err - elif "message" in data: - extracted_msg = data.get("message") - elif "execution_message" in data: - extracted_msg = data.get("execution_message") - elif "detail" in data: - extracted_msg = data.get("detail") - elif "errors" in data: - errs = data.get("errors") - if isinstance(errs, list) and errs: - # Join any messages in the list - parts = [] - for item in errs: - if isinstance(item, dict) and "message" in item: - parts.append(str(item.get("message"))) - else: - parts.append(str(item)) - extracted_msg = "; ".join(parts) - elif isinstance(errs, dict): - # Handle nested validation errors structure - if "children" in errs: - validation_errors = [] - children = errs.get("children", {}) - for field, field_errors in children.items(): - if ( - isinstance(field_errors, dict) - and "errors" in field_errors - ): - field_error_list = field_errors.get( - "errors", [] - ) - if field_error_list: - for err_msg in field_error_list: - validation_errors.append( - f"{field}: {err_msg}" - ) - if validation_errors: - base_msg = data.get("message", "Validation Failed") - extracted_msg = ( - f"{base_msg}: {'; '.join(validation_errors)}" - ) - else: - # Try to get any string representation - parts = [] - for key, value in errs.items(): - if value: - parts.append(f"{key}: {value}") - if parts: - extracted_msg = "; ".join(parts) - elif isinstance(errs, str): - extracted_msg = errs - - # Use extracted message if it's better than what we have - if extracted_msg and extracted_msg not in generic_messages: - message = str(extracted_msg) - elif not message: - # Last resort: use the raw body - message = body[:500] - - except json.JSONDecodeError: - # Not JSON, use raw text if we don't have a good message - if not message or message in generic_messages: - try: - decoded = self.response_body.decode(errors="ignore")[:500] - if decoded and decoded not in generic_messages: - message = decoded - except Exception: - pass - except Exception: - pass - - # Final fallback - if not message: - message = "Unknown error" - - # Clean up the message - remove extra whitespace and newlines - message = " ".join(message.split()) - - if self.response_code is not None: - return f"{self.response_code}: {message}" - return f"{message}" - - -class OpenAEVAuthenticationError(OpenAEVError): - pass - - -class OpenAEVHttpError(OpenAEVError): - pass - - -class OpenAEVParsingError(OpenAEVError): - pass - - -class RedirectError(OpenAEVError): - pass - - -class OpenAEVHeadError(OpenAEVError): - pass - - -class OpenAEVGetError(OpenAEVError): - pass - - -class OpenAEVUpdateError(OpenAEVError): - pass - - -class OpenAEVListError(OpenAEVError): - pass - +"""DEPRECATED: the error types moved to :mod:`xtm_oaev_sdk`. -class OpenAEVCreateError(OpenAEVError): - pass +Backward-compatibility shim: every error name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +``on_http_error`` is a REST client helper and stays defined here. +""" +import functools +from typing import Any, Callable, Type, TypeVar, cast -class OpenAEVDeleteError(OpenAEVError): - pass - - -class SignatureTransmissionError(OpenAEVError): - """Signatures didn't make it. Validation rejected them, 4xx slammed the door, or retries ran dry.""" - - pass - +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import OpenAEVHttpError as _OpenAEVHttpError +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr -class ConfigurationError(OpenAEVError): - pass +# Unchanged public export list of the legacy module (also extended into +# ``pyoaev.__all__``); ``OpenAEVError`` and ``on_http_error`` were reachable +# but never exported. +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ConfigurationError", + "OpenAEVAuthenticationError", + "OpenAEVHttpError", + "OpenAEVParsingError", + "RedirectError", + "OpenAEVHeadError", + "OpenAEVListError", + "OpenAEVGetError", + "OpenAEVUpdateError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "SignatureTransmissionError", +] +_MOVED = [*__all__, "OpenAEVError"] # For an explanation of how these type-hints work see: # https://mypy.readthedocs.io/en/stable/generics.html#declaring-decorators @@ -207,7 +45,7 @@ def wrap(f: __F) -> __F: def wrapped_f(*args: Any, **kwargs: Any) -> Any: try: return f(*args, **kwargs) - except OpenAEVHttpError as e: + except _OpenAEVHttpError as e: raise error(e.error_message, e.response_code, e.response_body) from e return cast(__F, wrapped_f) @@ -215,18 +53,10 @@ def wrapped_f(*args: Any, **kwargs: Any) -> Any: return wrap -# Export manually to keep mypy happy -__all__ = [ - "ConfigurationError", - "OpenAEVAuthenticationError", - "OpenAEVHttpError", - "OpenAEVParsingError", - "RedirectError", - "OpenAEVHeadError", - "OpenAEVListError", - "OpenAEVGetError", - "OpenAEVUpdateError", - "OpenAEVCreateError", - "OpenAEVDeleteError", - "SignatureTransmissionError", -] +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in _MOVED + }, + module_name=__name__, +) diff --git a/pyoaev/helpers.py b/pyoaev/helpers.py index 6867ba32..5df41803 100644 --- a/pyoaev/helpers.py +++ b/pyoaev/helpers.py @@ -1,277 +1,43 @@ -import base64 -import json -import os -import os.path -import re -import sched -import ssl -import tempfile -import threading -import time -import traceback -from typing import Callable, Dict, List - -import pika -from thefuzz import fuzz - -from pyoaev import OpenAEV, utils -from pyoaev.configuration import Configuration -from pyoaev.daemons import CollectorDaemon -from pyoaev.exceptions import ConfigurationError - -TRUTHY: List[str] = ["yes", "true", "True"] -FALSY: List[str] = ["no", "false", "False"] - - -# As cert must be written in files to be loaded in ssl context -# Creates a temporary file in the most secure manner possible -def data_to_temp_file(data): - # The file is readable and writable only by the creating user ID. - # If the operating system uses permission bits to indicate whether a - # file is executable, the file is executable by no one. The file - # descriptor is not inherited by children of this process. - file_descriptor, file_path = tempfile.mkstemp() - with os.fdopen(file_descriptor, "w") as open_file: - open_file.write(data) - open_file.close() - return file_path - - -def is_memory_certificate(certificate): - return certificate.startswith("-----BEGIN") - - -def ssl_cert_chain(ssl_context, cert_data, key_data, passphrase): - if cert_data is None: - return - - cert_file_path = None - key_file_path = None - - # Cert loading - if cert_data is not None and is_memory_certificate(cert_data): - cert_file_path = data_to_temp_file(cert_data) - cert = cert_file_path if cert_file_path is not None else cert_data - - # Key loading - if key_data is not None and is_memory_certificate(key_data): - key_file_path = data_to_temp_file(key_data) - key = key_file_path if key_file_path is not None else key_data - - # Load cert - ssl_context.load_cert_chain(cert, key, passphrase) - # Remove temp files - if cert_file_path is not None: - os.unlink(cert_file_path) - if key_file_path is not None: - os.unlink(key_file_path) - - -def ssl_verify_locations(ssl_context, certdata): - if certdata is None: - return - - if is_memory_certificate(certdata): - ssl_context.load_verify_locations(cadata=certdata) - else: - ssl_context.load_verify_locations(cafile=certdata) - - -def create_mq_ssl_context(config) -> ssl.SSLContext: - config_obj = Configuration( - config_hints={ - "MQ_USE_SSL_CA": { - "env": "MQ_USE_SSL_CA", - "file_path": ["mq", "use_ssl_ca"], - }, - "MQ_USE_SSL_CERT": { - "env": "MQ_USE_SSL_CERT", - "file_path": ["mq", "use_ssl_cert"], - }, - "MQ_USE_SSL_KEY": { - "env": "MQ_USE_SSL_KEY", - "file_path": ["mq", "use_ssl_key"], - }, - "MQ_USE_SSL_REJECT_UNAUTHORIZED": { - "env": "MQ_USE_SSL_REJECT_UNAUTHORIZED", - "file_path": ["mq", "use_ssl_reject_unauthorized"], - "is_number": False, - "default": False, - }, - "MQ_USE_SSL_PASSPHRASE": { - "env": "MQ_USE_SSL_PASSPHRASE", - "file_path": ["mq", "use_ssl_passphrase"], - }, - }, - config_values=config, - ) - use_ssl_ca = config_obj.get("MQ_USE_SSL_CA") - use_ssl_cert = config_obj.get("MQ_USE_SSL_CERT") - use_ssl_key = config_obj.get("MQ_USE_SSL_KEY") - use_ssl_reject_unauthorized = config_obj.get("MQ_USE_SSL_REJECT_UNAUTHORIZED") - use_ssl_passphrase = config_obj.get("MQ_USE_SSL_PASSPHRASE") - ssl_context = ssl.create_default_context() - # If no rejection allowed, use private function to generate unverified context - if not use_ssl_reject_unauthorized: - # noinspection PyUnresolvedReferences,PyProtectedMember - ssl_context = ssl._create_unverified_context() - ssl_verify_locations(ssl_context, use_ssl_ca) - # Thanks to https://bugs.python.org/issue16487 is not possible today to easily use memory pem - # in SSL context. We need to write it to a temporary file before - ssl_cert_chain(ssl_context, use_ssl_cert, use_ssl_key, use_ssl_passphrase) - return ssl_context - - -class ListenQueue(threading.Thread): - def __init__( - self, - config: Dict, - injector_config, - logger, - callback, - ) -> None: - threading.Thread.__init__(self) - self.pika_credentials = None - self.pika_parameters = None - self.pika_connection = None - self.channel = None - - self.callback = callback - self.config = config - self.logger = logger - self.host = injector_config.connection["host"] - self.vhost = injector_config.connection["vhost"] - self.use_ssl = injector_config.connection["use_ssl"] - self.port = injector_config.connection["port"] - self.user = injector_config.connection["user"] - self.password = injector_config.connection["pass"] - self.queue_name = injector_config.listen - self.exit_event = threading.Event() - self.thread = None - - # noinspection PyUnusedLocal - def _process_message(self, channel, method, properties, body) -> None: - """process a message from the rabbit queue - - :param channel: channel instance - :type channel: callable - :param method: message methods - :type method: callable - :param properties: unused - :type properties: str - :param body: message body (data) - :type body: str or bytes or bytearray - """ - json_data = json.loads(body) - # Message should be ack before processing as we don't own the processing - # Not ACK the message here may lead to infinite re-deliver if the connector is broken - # Also ACK, will not have any impact on the blocking aspect of the following functions - channel.basic_ack(delivery_tag=method.delivery_tag) - self.thread = threading.Thread(target=self._data_handler, args=[json_data]) - self.thread.start() - - def _data_handler(self, json_data) -> None: - self.callback(json_data) - - def run(self) -> None: - self.logger.info("Starting ListenQueue thread") - while not self.exit_event.is_set(): - try: - self.logger.info("ListenQueue connecting to RabbitMQ.") - # Connect the broker - self.pika_credentials = pika.PlainCredentials(self.user, self.password) - self.pika_parameters = pika.ConnectionParameters( - host=self.host, - port=self.port, - virtual_host=self.vhost, - credentials=self.pika_credentials, - ssl_options=( - pika.SSLOptions(create_mq_ssl_context(self.config), self.host) - if self.use_ssl - else None - ), - ) - self.pika_connection = pika.BlockingConnection(self.pika_parameters) - self.channel = self.pika_connection.channel() - try: - # confirm_delivery is only for cluster mode rabbitMQ - # when not in cluster mode this line raise an exception - self.channel.confirm_delivery() - except Exception as err: # pylint: disable=broad-except - self.logger.error(str(err)) - self.channel.basic_qos(prefetch_count=1) - assert self.channel is not None - self.channel.basic_consume( - queue=self.queue_name, on_message_callback=self._process_message - ) - self.channel.start_consuming() - except Exception: # pylint: disable=broad-except - try: - self.pika_connection.close() - except Exception as errInException: - self.logger.error(str(errInException)) - traceback.print_exc() - # Wait some time and then retry ListenQueue again. - time.sleep(10) - - def stop(self): - self.logger.info("Preparing ListenQueue for clean shutdown") - self.exit_event.set() - self.pika_connection.close() - if self.thread: - self.thread.join() - - -class PingAlive(utils.PingAlive): - pass - - -### DEPRECATED -class OpenAEVConfigHelper: - def __init__(self, base_path, variables: Dict | None, config_obj: Configuration): - if config_obj is not None: - self.__config_obj = config_obj - else: - self.__config_obj = Configuration( - config_hints=variables, - config_file_path=os.path.join( - os.path.dirname(os.path.abspath(base_path)), "config.yml" - ), - ) - - @staticmethod - def from_configuration_object(config: Configuration): - return OpenAEVConfigHelper(None, None, config) - - def get_config_obj(self) -> Configuration: - return self.__config_obj - - def get_conf(self, variable, is_number=None, default=None, required=None): - result = None - try: - result = ( - self.__config_obj.get(variable) - if (self.__config_obj.get(variable) is not None) - else default - ) - except ConfigurationError: - result = default - finally: - if result is None and default is None and required: - raise ValueError( - f"Could not find required key {variable} with no available default." - ) - return result - - def to_configuration(self): - return self.__config_obj +"""DEPRECATED: the daemon helpers moved to :mod:`collectors_sdk`, +:mod:`injectors_sdk` and :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every legacy name resolves to the *identical* SDK +object and emits a :class:`DeprecationWarning` on access, with two adapters: +``OpenAEVInjectorHelper`` keeps building the ``OpenAEV`` client when none is +injected, and ``OpenAEVCollectorHelper`` (already deprecated) stays here as +legacy glue over the collectors-sdk ``CollectorDaemon``. +""" + +import collectors_sdk as _collectors_sdk +import injectors_sdk as _injectors_sdk +import xtm_oaev_sdk as _sdk +from collectors_sdk.public import detection as _detection +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +from pyoaev import OpenAEV +from pyoaev.daemons.collector_daemon import ( + _COLLECTOR_DAEMON_MESSAGE, + _ClientCollectorDaemon, +) + + +class _ClientInjectorHelper(_injectors_sdk.OpenAEVInjectorHelper): + """injectors-sdk OpenAEVInjectorHelper building an OpenAEV client when none + is injected.""" + + def _create_default_api_client(self, config) -> OpenAEV: + return OpenAEV( + url=config.get_conf("openaev_url"), + token=config.get_conf("openaev_token"), + tenant_id=config.get_conf("openaev_tenant_id"), + ) ### DEPRECATED -class OpenAEVCollectorHelper: +class _OpenAEVCollectorHelper: def __init__( self, - config: OpenAEVConfigHelper, + config: _injectors_sdk.OpenAEVConfigHelper, icon, collector_type=None, security_platform_type=None, @@ -284,17 +50,17 @@ def __init__( if security_platform_type is not None: config_obj.set("collector_platform", security_platform_type) - self.__daemon = CollectorDaemon( + self.__daemon = _ClientCollectorDaemon( configuration=config_obj, callback=None, collector_type=collector_type, ) self.__daemon.logger.warning( - f"DEPRECATED: this collector should be migrated to use {CollectorDaemon}." + f"DEPRECATED: this collector should be migrated to use {_ClientCollectorDaemon}." ) - self.logger_class = utils.logger( + self.logger_class = _sdk.logger( config.get_conf("collector_log_level", default="error").upper(), config.get_conf("collector_json_logging", default=True), ) @@ -317,148 +83,65 @@ def schedule(self, message_callback, delay): self.__daemon.start() -class OpenAEVInjectorHelper: - def __init__(self, config: OpenAEVConfigHelper, icon) -> None: - self.api = OpenAEV( - url=config.get_conf("openaev_url"), - token=config.get_conf("openaev_token"), - tenant_id=config.get_conf("openaev_tenant_id"), - ) - # Get the mq configuration from api - self.config = { - "injector_id": config.get_conf("injector_id"), - "injector_name": config.get_conf("injector_name"), - "injector_type": config.get_conf("injector_type"), - "injector_contracts": config.get_conf("injector_contracts"), - "injector_custom_contracts": config.get_conf( - "injector_custom_contracts", default=False - ), - "injector_category": config.get_conf("injector_category", default=None), - "injector_executor_commands": config.get_conf( - "injector_executor_commands", default=None - ), - "injector_executor_clear_commands": config.get_conf( - "injector_executor_clear_commands", default=None - ), - # Publisher/author of the injector's contracts (source-declared). - "injector_author": config.get_conf("injector_author", default=None), - } - - self.logger_class = utils.logger( - config.get_conf("injector_log_level", default="error").upper(), - config.get_conf("injector_json_logging", default=True), - ) - self.injector_logger = self.logger_class(config.get_conf("injector_name")) - - icon_name = config.get_conf("injector_type") + ".png" - injector_icon = (icon_name, icon, "image/png") - self.injector_config = self.api.injector.create(self.config, injector_icon) - self.connect_run_and_terminate = False - self.scheduler = sched.scheduler(time.time, time.sleep) - # Start ping thread - if not self.connect_run_and_terminate: - self.ping = PingAlive( - self.api, self.config, self.injector_logger, "injector" +_FROM_XTM = [ + "Configuration", + "ConfigurationError", + "PingAlive", + "data_to_temp_file", + "is_memory_certificate", + "logger", + "ssl_cert_chain", + "ssl_verify_locations", +] +_FROM_INJECTORS = ["ListenQueue", "OpenAEVConfigHelper", "create_mq_ssl_context"] +_TRUTHINESS = { + "TRUTHY": (["yes", "true", "True"], "is_truthy"), + "FALSY": (["no", "false", "False"], "is_falsy"), +} + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), ) - self.ping.start() - self.listen_queue = None - - def listen(self, message_callback: Callable[[Dict], None]) -> None: - self.listen_queue = ListenQueue( - self.config, self.injector_config, self.injector_logger, message_callback - ) - self.listen_queue.start() - - -class OpenAEVDetectionHelper: - def __init__(self, logger, relevant_signatures_types) -> None: - self.logger = logger - self.relevant_signatures_types = relevant_signatures_types - - def match_alert_element_fuzzy(self, signature_value, alert_values, fuzzy_scoring): - for alert_value in alert_values: - self.logger.info( - "Comparing alert value (" + alert_value + ", " + signature_value + ")" + for name in _FROM_XTM + }, + **{ + name: ( + f"Use 'from injectors_sdk import {name}' instead.", + getattr(_injectors_sdk, name), ) - ratio = fuzz.ratio(alert_value, signature_value) - if ratio > fuzzy_scoring: - self.logger.info("MATCHING! (score: " + str(ratio) + ")") - return True - return False - - def match_alert_elements(self, signatures, alert_data): - return self._match_alert_elements_original( - signatures, alert_data - ) or self._match_alert_elements_for_command_line(signatures, alert_data) - - def _match_alert_elements_original(self, signatures, alert_data): - # Example for alert_data - # {"process_name": {"list": ["xx", "yy"], "fuzzy": 90}} - relevant_signatures = [ - s for s in signatures if s["type"] in self.relevant_signatures_types - ] - - # Matching logics - signatures_number = len(relevant_signatures) - matching_number = 0 - for signature in relevant_signatures: - alert_data_for_signature = alert_data[signature["type"]] - signature_result = False - if alert_data_for_signature["type"] == "fuzzy": - signature_result = self.match_alert_element_fuzzy( - signature["value"], - alert_data_for_signature["data"], - alert_data_for_signature["score"], - ) - elif alert_data_for_signature["type"] == "simple": - signature_result = signature["value"] in str( - alert_data_for_signature["data"] - ) - - if signature_result: - matching_number = matching_number + 1 - - if signatures_number == matching_number: - return True - return False - - def _match_alert_elements_for_command_line(self, signatures, alert_data): - command_line_signatures = [ - signature - for signature in signatures - if signature.get("type") == "command_line" - ] - if len(command_line_signatures) == 0: - return False - key_types = ["command_line", "process_name", "file_name"] - alert_datas = [alert_data.get(key) for key in key_types if key in alert_data] - for signature in command_line_signatures: - signature_result = False - signature_value = self._decode_value(signature["value"]).strip().lower() - for alert_data in alert_datas: - trimmed_lowered_datas = [s.strip().lower() for s in alert_data["data"]] - signature_result = any( - data in signature_value for data in trimmed_lowered_datas - ) - if signature_result: - return True - return False - - def _decode_value(self, signature_value): - if _is_base64_encoded(signature_value): - try: - decoded_bytes = base64.b64decode(signature_value) - decoded_str = decoded_bytes.decode("utf-8") - return decoded_str - except Exception as e: - self.logger.error(str(e)) - else: - return signature_value - - -def _is_base64_encoded(str_maybe_base64): - # Check if the length is a multiple of 4 and matches the Base64 character set - base64_pattern = re.compile(r"^[A-Za-z0-9+/]*={0,2}$") - return len(str_maybe_base64) % 4 == 0 and bool( - base64_pattern.match(str_maybe_base64) - ) + for name in _FROM_INJECTORS + }, + "OpenAEVInjectorHelper": ( + "Use 'from injectors_sdk import OpenAEVInjectorHelper' and pass " + "api_client=OpenAEV(...) instead.", + _ClientInjectorHelper, + ), + "OpenAEVDetectionHelper": ( + "Use 'from collectors_sdk import OpenAEVDetectionHelper' instead.", + _collectors_sdk.OpenAEVDetectionHelper, + ), + "_is_base64_encoded": ( + "'_is_base64_encoded' is private and will be removed.", + _detection._is_base64_encoded, + ), + "CollectorDaemon": (_COLLECTOR_DAEMON_MESSAGE, _ClientCollectorDaemon), + "OpenAEVCollectorHelper": ( + "OpenAEVCollectorHelper is deprecated; use " + "'from collectors_sdk import CollectorDaemon' instead.", + _OpenAEVCollectorHelper, + ), + **{ + name: ( + f"'{name}' is deprecated; use 'from xtm_oaev_sdk import {helper}' " + "instead.", + value, + ) + for name, (value, helper) in _TRUTHINESS.items() + }, + }, + module_name=__name__, +) diff --git a/pyoaev/mixins.py b/pyoaev/mixins.py index 6309e311..dd98938a 100644 --- a/pyoaev/mixins.py +++ b/pyoaev/mixins.py @@ -12,11 +12,12 @@ ) import requests +import xtm_oaev_sdk as exc import pyoaev from pyoaev import base -from pyoaev import exceptions as exc -from pyoaev import utils +from pyoaev.exceptions import on_http_error +from pyoaev.utils import EncodedId __all__ = [ "GetMixin", @@ -33,7 +34,7 @@ class HeadMixin(_RestManagerBase): - @exc.on_http_error(exc.OpenAEVHeadError) + @on_http_error(exc.OpenAEVHeadError) def head( self, id: Optional[Union[str, int]] = None, **kwargs: Any ) -> "requests.structures.CaseInsensitiveDict[Any]": @@ -42,7 +43,7 @@ def head( path = self.path if id is not None: - path = f"{path}/{utils.EncodedId(id)}" + path = f"{path}/{EncodedId(id)}" return self.openaev.http_head(path, **kwargs) @@ -57,10 +58,10 @@ class GetMixin(HeadMixin, _RestManagerBase): _path: Optional[str] openaev: pyoaev.OpenAEV - @exc.on_http_error(exc.OpenAEVGetError) + @on_http_error(exc.OpenAEVGetError) def get(self, id: Union[str, int], **kwargs: Any) -> base.RESTObject: if isinstance(id, str): - id = utils.EncodedId(id) + id = EncodedId(id) path = f"{self.path}/{id}" if TYPE_CHECKING: assert self._obj_cls is not None @@ -80,7 +81,7 @@ class GetWithoutIdMixin(HeadMixin, _RestManagerBase): _path: Optional[str] openaev: pyoaev.OpenAEV - @exc.on_http_error(exc.OpenAEVGetError) + @on_http_error(exc.OpenAEVGetError) def get(self, **kwargs: Any) -> base.RESTObject: if TYPE_CHECKING: assert self.path is not None @@ -101,7 +102,7 @@ class ListMixin(HeadMixin, _RestManagerBase): _path: Optional[str] openaev: pyoaev.OpenAEV - @exc.on_http_error(exc.OpenAEVListError) + @on_http_error(exc.OpenAEVListError) def list(self, **kwargs: Any) -> Union[base.RESTObjectList, List[base.RESTObject]]: if self.openaev.per_page: @@ -159,7 +160,7 @@ def _get_update_method( http_method = self.openaev.http_put return http_method - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def update( self, id: Optional[Union[str, int]] = None, @@ -171,7 +172,7 @@ def update( if id is None: path = self.path else: - path = f"{self.path}/{utils.EncodedId(id)}" + path = f"{self.path}/{EncodedId(id)}" # excludes = [] # if self._obj_cls is not None and self._obj_cls._id_attr is not None: @@ -193,7 +194,7 @@ class CreateMixin(_RestManagerBase): _path: Optional[str] openaev: pyoaev.OpenAEV - @exc.on_http_error(exc.OpenAEVCreateError) + @on_http_error(exc.OpenAEVCreateError) def create( self, data: Optional[Dict[str, Any]] = None, icon: tuple = None, **kwargs: Any ) -> base.RESTObject: @@ -227,14 +228,14 @@ class DeleteMixin(_RestManagerBase): _path: Optional[str] openaev: pyoaev.OpenAEV - @exc.on_http_error(exc.OpenAEVDeleteError) + @on_http_error(exc.OpenAEVDeleteError) def delete( self, id: Optional[Union[str, int]] = None, **kwargs: Any ) -> requests.Response: if id is None: path = self.path else: - path = f"{self.path}/{utils.EncodedId(id)}" + path = f"{self.path}/{EncodedId(id)}" result = self.openaev.http_delete(path, **kwargs) if TYPE_CHECKING: diff --git a/pyoaev/security_domain/types.py b/pyoaev/security_domain/types.py index c48e48c1..157aa100 100644 --- a/pyoaev/security_domain/types.py +++ b/pyoaev/security_domain/types.py @@ -1,20 +1,22 @@ -from enum import Enum +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -class SecurityDomains(Enum): - ENDPOINT = {"domain_name": "Endpoint", "domain_color": "#389CFF"} - NETWORK = {"domain_name": "Network", "domain_color": "#009933"} - WEB_APP = {"domain_name": "Web App", "domain_color": "#FF9933"} - EMAIL_INFILTRATION = { - "domain_name": "E-mail Infiltration", - "domain_color": "#FF6666", - } - DATA_EXFILTRATION = {"domain_name": "Data Exfiltration", "domain_color": "#9933CC"} - URL_FILTERING = {"domain_name": "URL Filtering", "domain_color": "#66CCFF"} - CLOUD = {"domain_name": "Cloud", "domain_color": "#9999CC"} - TABLE_TOP = {"domain_name": "Tabletop", "domain_color": "#FFCC33"} - ARTIFICIAL_INTELLIGENCE = { - "domain_name": "Artificial Intelligence", - "domain_color": "#7C4DFF", - } - TOCLASSIFY = {"domain_name": "To classify", "domain_color": "#FFFFFF"} +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "SecurityDomains", +] + +_MOVED = __all__ + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in _MOVED + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/__init__.py b/pyoaev/signatures/__init__.py index dc2e0616..422f7b61 100644 --- a/pyoaev/signatures/__init__.py +++ b/pyoaev/signatures/__init__.py @@ -1,23 +1,16 @@ -from pyoaev.signatures.models import ( - CloudInjectorConfig, - ExpectationSignatureGroup, - ExtraSignatureData, - InjectorConfig, - NetworkInjectorConfig, - SignatureCallbackPayload, - SignaturePayload, - SignatureTarget, - SignatureValue, - TargetSignatures, - build_network_configs, -) -from pyoaev.signatures.signature_manager import SignatureManager -from pyoaev.signatures.types import ( - ExpectationType, - InjectExecutionActions, - MatchTypes, - SignatureTypes, -) +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access, with +two exceptions: ``ExpectationType`` was renamed ``SignatureExpectationType``, +and ``SignatureManager`` resolves to the client-based adapter described in +:mod:`pyoaev.signatures.signature_manager`. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +from pyoaev.signatures.signature_manager import _ClientSignatureManager __all__ = [ "CloudInjectorConfig", @@ -37,3 +30,24 @@ "TargetSignatures", "build_network_configs", ] + +_RENAMED = {"ExpectationType": "SignatureExpectationType"} + +# Also served from the pyoaev package root, which used to star-import it. +_DEPRECATED = { + "SignatureManager": ( + "Use 'from xtm_oaev_sdk import SignatureManager' with a " + "SignatureTransportProtocol transport (e.g. client.signature) instead.", + _ClientSignatureManager, + ), + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {_RENAMED.get(name, name)}' instead.", + getattr(_sdk, _RENAMED.get(name, name)), + ) + for name in __all__ + if name != "SignatureManager" + }, +} + +__getattr__ = _make_deprecated_getattr(_DEPRECATED, module_name=__name__) diff --git a/pyoaev/signatures/ai_marker.py b/pyoaev/signatures/ai_marker.py index 51165068..61d6e5fa 100644 --- a/pyoaev/signatures/ai_marker.py +++ b/pyoaev/signatures/ai_marker.py @@ -1,15 +1,20 @@ -"""Deterministic per-inject canary marker shared by the AI red-team injector and the AI defense -collectors. - -The marker is derived purely from the inject (and optional agent) id, so the injector that sends the -attack and the collector that validates an AI defense response compute the same value independently, -without the platform having to store it. It is emitted by the injector (request header + in-prompt -token) and matched by collectors against guardrail / firewall logs. -""" - -import hashlib - - -def build_marker(inject_id: str, agent_id: str = "") -> str: - seed = f"{inject_id}:{agent_id}".encode("utf-8") - return "oaev" + hashlib.sha256(seed).hexdigest()[:16] +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "build_marker", +] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/models.py b/pyoaev/signatures/models.py index c8e914f9..aee7504d 100644 --- a/pyoaev/signatures/models.py +++ b/pyoaev/signatures/models.py @@ -1,381 +1,13 @@ -"""Pydantic schemas pinning every shape SignatureManager touches.""" +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -import ipaddress -import math -from collections import defaultdict -from datetime import datetime, timezone -from typing import Any +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -from pydantic import ( - BaseModel, - ConfigDict, - Field, - JsonValue, - TypeAdapter, - computed_field, - field_validator, - model_validator, -) - -from pyoaev.signatures.types import ( - ExecutionStatus, - ExpectationType, - InjectExecutionActions, -) - - -class ToolErrorInfo(BaseModel): - """Crash report. Non-zero exit code and a timestamp if the tool left one behind.""" - - model_config = ConfigDict(extra="allow") - - exit_code: int = 0 - crash_timestamp: str | None = None - - -class ToolTimeoutInfo(BaseModel): - """Timeout report. Whatever partial loot was rescued before the kill signal.""" - - model_config = ConfigDict(extra="allow") - - partial_results: list[str] = Field(default_factory=list) - - -class ToolOutput(BaseModel): - """Whatever the tool spat out: status, error info, timeout info, or injector extras.""" - - model_config = ConfigDict(extra="allow") - - status: str | None = None - error_info: ToolErrorInfo | None = None - timeout_info: ToolTimeoutInfo | None = None - - -class SignatureValue(BaseModel): - """One signature observation: a type and the value it carries.""" - - model_config = ConfigDict(extra="allow") - - signature_type: str - signature_value: JsonValue - - -class ExpectationSignatureGroup(BaseModel): - """Values bound to a single expectation type (DETECTION, PREVENTION, ...).""" - - model_config = ConfigDict(extra="allow") - - expectation_type: str - values: list[SignatureValue] - - @field_validator("expectation_type", mode="after") - @classmethod - def is_expectation_type(cls, value: str) -> str: - ExpectationType(value.upper()) - return value - - -class ExtraSignatureData(BaseModel): - """Format for extra signatures added to the default signatures""" - - detection: dict[str, JsonValue] = Field(default_factory=dict) - prevention: dict[str, JsonValue] = Field(default_factory=dict) - vulnerability: dict[str, JsonValue] = Field(default_factory=dict) - - def get_extra(self, expectation_type: str) -> dict[str, JsonValue]: - if expectation_type.lower() == "detection": - return self.detection - if expectation_type.lower() == "prevention": - return self.prevention - if expectation_type.lower() == "vulnerability": - return self.vulnerability - raise ValueError( - f"Expectation type should be one of the available parameters: {list(self.model_fields.keys())}" - ) - - -class SignatureTarget(BaseModel): - """Target identity on the wire.""" - - model_config = ConfigDict(extra="forbid") - - agent: str | None = None - asset: str | None = None - asset_group: str | None = None - - -class TargetSignatures(BaseModel): - """A target plus everything observed about it, grouped by expectation.""" - - model_config = ConfigDict(extra="allow") - - signature_target: SignatureTarget - signature_values: list[ExpectationSignatureGroup] - - -class SignaturePayload(BaseModel): - """Inner ``signatures`` body: a list of targets, nothing else.""" - - model_config = ConfigDict(extra="allow") - - targets: list[TargetSignatures] - - -class SignatureOutputStructure(BaseModel): - """Structured output to be serialized as a str in the callback payload yet data has to follow model.""" - - model_config = ConfigDict(populate_by_name=True, extra="forbid") - - expectation_signatures: SignaturePayload - - def normalize_signature_payload(self) -> None: - """ - Regroup signature_values by expectation_type within each target. - """ - normalized_targets: list[TargetSignatures] = [] - - for target in self.expectation_signatures.targets: - if not target.signature_values: - normalized_targets.append(target) - continue - - grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) - order: list[str] = [] - - for entry in target.signature_values: - if entry.expectation_type not in order: - order.append(entry.expectation_type) - grouped[entry.expectation_type].extend(entry.values) - - normalized_target = TargetSignatures( - signature_target=target.signature_target, - signature_values=[ - ExpectationSignatureGroup( - expectation_type=expectation_type, - values=grouped[expectation_type], - ) - for expectation_type in order - ], - ) - - normalized_targets.append(normalized_target) - - self.expectation_signatures.targets = normalized_targets - - -class ExecutionDetails(BaseModel): - """Helper to wrap the execution-related details for the callback payload""" - - model_config = ConfigDict(extra="forbid") +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr - start_time: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) - end_time: datetime | None = None - - execution_status: ExecutionStatus | None = None - execution_action: InjectExecutionActions | None = None - - @computed_field - @property - def execution_message(self) -> str: - action = self.execution_action.value if self.execution_action else "unknown" - status = self.execution_action.value if self.execution_action else "unknown" - return f"Current action: {action} - Current status: {status}" - - @computed_field - @property - def execution_duration(self) -> float: - try: - return (self.end_time - self.start_time).total_seconds() - except: - return 0.0 - - def post_execution_update(self, tool_output: ToolOutput, now: datetime) -> None: - """ - Update execution-related metadata according to tool output and now timestamp - """ - self.end_time = now - - if tool_output.error_info and tool_output.error_info.exit_code != 0: - self.execution_status = ExecutionStatus.ERROR - if tool_output.error_info.crash_timestamp: - self.end_time = datetime.strptime( - tool_output.error_info.crash_timestamp, "%Y-%m-%dT%H:%M:%SZ" - ) - elif tool_output.timeout_info: - self.execution_status = ExecutionStatus.TIMEOUT - elif tool_output.status == "partial": - self.execution_status = ExecutionStatus.ERROR - else: - self.execution_status = ExecutionStatus.EXECUTED - - self.execution_action = InjectExecutionActions("data") - - -class SignatureCallbackPayload(BaseModel): - """Outer POST envelope validated by ``SignatureApiManager`` before wire transmission.""" - - model_config = ConfigDict(populate_by_name=True, extra="forbid") - - execution_message: str - execution_output_structured: str | None = None - execution_status: str - execution_duration: int | None = None - execution_action: InjectExecutionActions | None = None - - @field_validator("execution_output_structured", mode="after") - @classmethod - def is_proper_signature_output_structure(cls, value: str) -> str | None: - if value is None: - return None - TypeAdapter(SignatureOutputStructure).validate_json(value) - return value - - @classmethod - def build_from_models( - cls, signatures: SignatureOutputStructure, execution_details: ExecutionDetails - ): - """Producing a SignatureCallbackPayload from the data of a SignatureOutputStructure and of a ExecutionDetails.""" - return cls( - execution_message=execution_details.execution_message, - execution_output_structured=signatures.model_dump_json(exclude_none=True), - execution_status=execution_details.execution_status, - execution_duration=( - math.ceil(execution_details.execution_duration) - if execution_details.execution_duration is not None - else None - ), - execution_action=execution_details.execution_action, - ) - - -class ExecutionSignature(BaseModel): - """ - Execution signature data. Field set varies by category: network, cloud. Plus outcome, end_time, and any partial results. - """ - - model_config = ConfigDict(extra="allow") - - # Timing always emitted at call time. - start_time: str = Field( - default_factory=lambda: datetime.now(timezone.utc).strftime( - "%Y-%m-%dT%H:%M:%SZ" - ) - ) - end_time: str | None = None - partial_results: list[str] | None = None - - # Network identity - source_ipv4: str | None = None - source_ipv6: str | None = None - target_ipv4: str | None = None - target_ipv6: str | None = None - target_hostname: str | None = None - - # Cloud identity - cloud_provider: str | None = None - cloud_account_id: str | None = None - cloud_region: str | None = None - target_service: str | None = None - - def post_execution_update(self, tool_output: ToolOutput, now: datetime) -> None: - """ """ - self.end_time = now.strftime("%Y-%m-%dT%H:%M:%SZ") - - if tool_output.error_info and tool_output.error_info.crash_timestamp: - self.end_time = tool_output.error_info.crash_timestamp - - if tool_output.timeout_info and tool_output.timeout_info.partial_results: - self.partial_results = tool_output.timeout_info.partial_results - - -class NetworkInjectorConfig(BaseModel): - """A single network target. Exactly one of ``target_ipv4``, ``target_ipv6``, or ``target_hostname``.""" - - model_config = ConfigDict(extra="forbid") - - target_ipv4: str | None = None - target_ipv6: str | None = None - target_hostname: str | None = None - - @model_validator(mode="before") - @classmethod - def check_one(cls, data): - assert ( - sum( - value is not None - for key, value in data.items() - if key in ["target_ipv4", "target_ipv6", "target_hostname"] - ) - == 1 - ) - return data - - -class CloudInjectorConfig(BaseModel): - """A single cloud target row. One config per region; fan out by passing a list.""" - - model_config = ConfigDict(extra="forbid") - - cloud_provider: str - cloud_account_id: str - cloud_region: str - target_service: str | None = None - - -InjectorConfig = NetworkInjectorConfig | CloudInjectorConfig - - -# --------------------------------------------------------------------------- -# Builders. Cheap helpers to turn raw injector input into typed configs. -# --------------------------------------------------------------------------- - - -def _classify_network_target(value: str) -> NetworkInjectorConfig: - """Decide whether ``value`` is an IPv4, IPv6, or hostname and wrap it.""" - try: - addr = ipaddress.ip_address(value) - except ValueError: - return NetworkInjectorConfig(target_hostname=value) - if isinstance(addr, ipaddress.IPv4Address): - return NetworkInjectorConfig(target_ipv4=value) - return NetworkInjectorConfig(target_ipv6=value) - - -def build_network_configs( - targets: list[str | dict[str, Any] | NetworkInjectorConfig], -) -> list[NetworkInjectorConfig]: - """Forge a list of `NetworkInjectorConfig` from a heterogeneous target list. - - Each item is one distinct asset. Accepted shapes: - - - `NetworkInjectorConfig`: passed through unchanged. - - `dict`: validated against :class:`NetworkInjectorConfig`. - - `str`: auto-classified into IPv4 / IPv6 / hostname. - - Args: - targets: Raw target list straight out of the injector. - - Returns: - One `NetworkInjectorConfig` per input target, order preserved. - - Raises: - TypeError: An item is not one of the accepted shapes. - ValidationError: A dict item fails the one-identity invariant. - """ - configs: list[NetworkInjectorConfig] = [] - for target in targets: - if isinstance(target, NetworkInjectorConfig): - configs.append(target) - elif isinstance(target, dict): - configs.append(NetworkInjectorConfig(**target)) - elif isinstance(target, str): - configs.append(_classify_network_target(target)) - else: - raise TypeError(f"unsupported network target type: {type(target).__name__}") - return configs - - -__all__ = [ +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) "SignatureValue", "ExpectationSignatureGroup", "SignatureTarget", @@ -391,3 +23,26 @@ def build_network_configs( "InjectorConfig", "build_network_configs", ] + +_RENAMED = {"ExpectationType": "SignatureExpectationType"} + +_MOVED = [ + *__all__, + "ExecutionDetails", + "ExtraSignatureData", + "SignatureOutputStructure", + "ExecutionStatus", + "ExpectationType", + "InjectExecutionActions", +] + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from xtm_oaev_sdk import {_RENAMED.get(name, name)}' instead.", + getattr(_sdk, _RENAMED.get(name, name)), + ) + for name in _MOVED + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/signature_manager.py b/pyoaev/signatures/signature_manager.py index 28452342..8054eac1 100644 --- a/pyoaev/signatures/signature_manager.py +++ b/pyoaev/signatures/signature_manager.py @@ -1,298 +1,78 @@ -"""Signature lifecycle for OpenAEV injectors: compile pre, merge post, ship to backend.""" +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -import logging -import os -import socket -import subprocess -from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any +The SDK ``SignatureManager`` takes a ``SignatureTransportProtocol`` instead of +an ``OpenAEV`` client. The legacy name resolves to a thin subclass that keeps +the historical ``SignatureManager(client, logger=None, max_payload_size=...)`` +constructor by using ``client.signature`` as the transport. Every other name +resolves to the *identical* xtm_oaev_sdk object. All names emit a +:class:`DeprecationWarning` on access. -from pydantic import ValidationError +``socket`` and ``subprocess`` stay importable here so that legacy patch targets +such as ``pyoaev.signatures.signature_manager.socket.gethostbyname`` keep +reaching the shared standard-library modules the SDK uses. +""" -from pyoaev.exceptions import OpenAEVError -from pyoaev.signatures.models import ( - CloudInjectorConfig, - ExecutionDetails, - ExecutionSignature, - ExpectationSignatureGroup, - ExtraSignatureData, - InjectorConfig, - NetworkInjectorConfig, - SignatureOutputStructure, - SignaturePayload, - SignatureTarget, - SignatureValue, - TargetSignatures, - ToolOutput, -) +import socket # noqa: F401 +import subprocess # noqa: F401 +from typing import TYPE_CHECKING + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr if TYPE_CHECKING: from pyoaev.client import OpenAEV -class SignatureManager: - """End-to-end signature pipeline: compile, merge, transmit. One class, three jobs.""" - - DEFAULT_MAX_PAYLOAD_SIZE = 5_242_880 # 5 MiB +class _ClientSignatureManager(_sdk.SignatureManager): + """SDK SignatureManager driven by a pyoaev client (legacy constructor).""" def __init__( self, client: "OpenAEV", - logger: logging.Logger | None = None, - max_payload_size: int = DEFAULT_MAX_PAYLOAD_SIZE, - ) -> None: - self.client = client - self.logger = logger or logging.getLogger(__name__) - self.max_payload_size = max_payload_size - self._cached_ipv4: str | None = None - self._cached_ipv6: str | None = None - - def _utcnow(self) -> datetime: - """Current UTC time. Carved out so tests can pin the clock.""" - return datetime.now(timezone.utc) - - def build_execution_signatures( - self, - config: InjectorConfig | list[InjectorConfig], - ) -> ExecutionSignature | list[ExecutionSignature]: - """Build pre-execution signature dicts from one or more typed injector configs. - - The category is carried by the config type itself - (:class:`NetworkInjectorConfig`, :class:`CloudInjectorConfig`), - so no separate ``category`` flag is needed. - - Args: - config: A single injector config or a homogeneous list of them. - Multi-target injects must be expressed as a list. - - Returns: - One ExecutionSignature object when a single config is given, - otherwise a list of ExecutionSignature in input order. - - Raises: - ValueError: Empty list, or mixed config types in a single call. - TypeError: Unknown injector config type. - """ - configs = list(config) if isinstance(config, list) else [config] - if not configs: - raise ValueError("build_execution_signatures requires at least one config") - - first_type = type(configs[0]) - for c in configs: - if not isinstance(c, first_type): - raise ValueError( - "build_execution_signatures does not mix injector config types; " - f"got {sorted({type(c).__name__ for c in configs})}" - ) - - start_time = self._utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - results = [self._build_one(cfg, start_time) for cfg in configs] - return results[0] if len(results) == 1 else results - - def _build_one(self, config: InjectorConfig, start_time: str) -> ExecutionSignature: - """Project a single injector config into a flat pre-execution signature dict. - - Common pipeline for every category: - 1. Seed the base dict with ``start_time`` and category-specific context - (network gets resolved source IPs; cloud add nothing). - 2. Layer the config's own fields on top. - 3. Run it through :class:`ExecutionSignature` for validation - and emit JSON-ready output stripped of ``None``\\ s. - """ - base: dict[str, Any] = {"start_time": start_time} - base.update(self._source_context(config)) - base.update(config.model_dump(exclude_none=True)) - return ExecutionSignature(**base) - - def _source_context(self, config: InjectorConfig) -> dict[str, Any]: - """Return the source identity bits injected for the config's category. - - Only network signatures need the running container's source IPs; - cloud rows have no source identity to carry. - """ - if isinstance(config, NetworkInjectorConfig): - return { - "source_ipv4": self.resolve_container_ip(), - "source_ipv6": self._cached_ipv6, - } - if isinstance(config, CloudInjectorConfig): - return {} - raise TypeError(f"unsupported injector config type: {type(config).__name__}") - - def post_execution_updates( - self, - execution_details: ExecutionDetails, - execution_signatures: ExecutionSignature | list[ExecutionSignature], - tool_output: dict[str, Any], - ) -> None: - """ - Update both execution details and execution signatures according to tool output - """ - try: - tool_output = ToolOutput.model_validate(tool_output or {}) - except ValidationError as exc: - raise OpenAEVError( - error_message=f"Invalid tool_output: {exc}", - ) from exc - - now = self._utcnow() - - execution_details.post_execution_update(tool_output, now) - - if isinstance(execution_signatures, list): - for exec_sig in execution_signatures: - exec_sig.post_execution_update(tool_output, now) - else: - execution_signatures.post_execution_update(tool_output, now) - - @staticmethod - def build_payload( - execution_signatures: ExecutionSignature | list[ExecutionSignature], - targets_meta: Any | list[Any], - expectation_types: list[str], - extra_signatures: ExtraSignatureData = ExtraSignatureData(), - ) -> dict[str, Any]: - """Build the nested wire payload from flat post-execution signatures. - - Bridges the gap between compile_post_execution_signatures output (flat dicts) - and send_signatures input (nested wire format). - - Args: - execution_signatures: A single post-execution ExecutionSignature or a list (multi-targets). - targets_meta: Target metadata dict(s) with keys like agent, asset, asset_group. - expectation_types: The 1+ expectation type labels (e.g. ['DETECTION', 'PREVENTION']). - extra_signatures: Optional mapping of expectation types to additional signature fields that will be merged - into the base post_signatures. - - Returns: - A payload dict ready for send_signatures. - """ - if not isinstance(execution_signatures, list): - execution_signatures = [execution_signatures] - if isinstance(targets_meta, dict): - targets_meta = [targets_meta] * len(execution_signatures) - - targets = [] - for signature, target in zip(execution_signatures, targets_meta): - signature_values = [] - for expectation_type in expectation_types: - signature_data = signature.model_dump(exclude_none=True) - signature_data.update(extra_signatures.get_extra(expectation_type)) - values = [ - SignatureValue(signature_type=key, signature_value=value) - for key, value in signature_data.items() - ] - signature_values.append( - ExpectationSignatureGroup( - expectation_type=expectation_type, - values=values, - ) - ) - targets.append( - TargetSignatures( - signature_target=SignatureTarget( - agent=target.agent_id, - asset=target.asset_id, - asset_group=target.asset_group_id, - ), - signature_values=signature_values, - ) - ) - - return SignaturePayload(targets=targets).model_dump() - - def send_signatures( - self, - inject_id: str, - execution_details: ExecutionDetails, - signatures: dict[str, Any], + logger=None, + max_payload_size: int = _sdk.SignatureManager.DEFAULT_MAX_PAYLOAD_SIZE, ) -> None: - """Ship signatures to the callback endpoint via the Signature API manager. - - Constructs typed ``SignatureOutputStructure`` and ``ExecutionDetails`` - models, then delegates transport (retry, envelope splitting, validation) - to ``client.signature``. - - Args: - inject_id: Inject UUID. - execution_details: execution-related metadata as an ExecutionDetails object - signatures: Full signatures dict with a ``targets`` list. - - Raises: - SignatureTransmissionError: Validation failed, 4xx hit, or retries exhausted. - """ - sig_output = SignatureOutputStructure( - expectation_signatures=SignaturePayload(**signatures) - ) - - self.client.signature.send_signatures( - inject_id, - sig_output, - execution_details, - max_payload_size=self.max_payload_size, - logger=self.logger, + super().__init__( + transport=client.signature, + logger=logger, + max_payload_size=max_payload_size, ) + self.client = client - def resolve_container_ip(self) -> str: - """Sniff the container's primary IPv4. Env var, hostname, then ``hostname -i``. - - Returns: - The IPv4 string, or ``'unknown'`` with a warning when all strategies fail. - """ - if self._cached_ipv4 and self._cached_ipv4 != "unknown": - return self._cached_ipv4 - - env_ip = os.environ.get("CONTAINER_IP") - if env_ip: - self._cached_ipv4 = env_ip - self._resolve_ipv6() - return env_ip - - try: - ip = socket.gethostbyname(socket.gethostname()) - if ip and ip != "127.0.0.1": - self._cached_ipv4 = ip - self._resolve_ipv6() - return ip - except (socket.gaierror, OSError): - pass - - try: - result = subprocess.run( - ["hostname", "-i"], - capture_output=True, - text=True, - timeout=5, - check=False, - ) - if result.returncode == 0: - ip = result.stdout.strip().split()[0] - if ip and ip != "127.0.0.1": - self._cached_ipv4 = ip - self._resolve_ipv6() - return ip - except (OSError, RuntimeError, subprocess.TimeoutExpired): - pass - - self.logger.warning("Could not resolve container IP; returning 'unknown'") - self._cached_ipv4 = "unknown" - return "unknown" - def _resolve_ipv6(self) -> None: - """Best-effort IPv6 sniff. Silent on failure, no exceptions escape.""" - try: - infos = socket.getaddrinfo( - socket.gethostname(), None, socket.AF_INET6, socket.SOCK_STREAM +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "SignatureManager", + "OpenAEVError", + "CloudInjectorConfig", + "ExecutionDetails", + "ExecutionSignature", + "ExpectationSignatureGroup", + "ExtraSignatureData", + "InjectorConfig", + "NetworkInjectorConfig", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureValue", + "TargetSignatures", + "ToolOutput", +] + +__getattr__ = _make_deprecated_getattr( + { + "SignatureManager": ( + "Use 'from xtm_oaev_sdk import SignatureManager' with a " + "SignatureTransportProtocol transport (e.g. client.signature) instead.", + _ClientSignatureManager, + ), + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), ) - for info in infos: - addr = info[4][0] - if ( - isinstance(addr, str) - and addr - and not addr.startswith("::1") - and not addr.startswith("fe80") - ): - self._cached_ipv6 = addr - return - except (socket.gaierror, OSError): - pass + for name in __all__ + if name != "SignatureManager" + }, + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/signature_match.py b/pyoaev/signatures/signature_match.py index 75ce54d7..b52b9a72 100644 --- a/pyoaev/signatures/signature_match.py +++ b/pyoaev/signatures/signature_match.py @@ -1,12 +1,22 @@ -from pyoaev.exceptions import OpenAEVError -from pyoaev.signatures.types import MatchTypes - - -class SignatureMatch: - def __init__(self, match_type: MatchTypes, match_score: int | None): - if match_score is None and match_type != MatchTypes.MATCH_TYPE_SIMPLE: - raise OpenAEVError( - f"Match type {match_type} requires score to be set, found score = {match_score}" - ) - self.match_type = match_type - self.match_score = match_score +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "SignatureMatch", + "OpenAEVError", + "MatchTypes", +] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/signature_type.py b/pyoaev/signatures/signature_type.py index bab72dbd..c40f2f32 100644 --- a/pyoaev/signatures/signature_type.py +++ b/pyoaev/signatures/signature_type.py @@ -1,51 +1,23 @@ -from pyoaev.signatures.signature_match import SignatureMatch -from pyoaev.signatures.types import MatchTypes, SignatureTypes - - -class SignatureType: - """Describes a signature of some time and a matching policy - - :param label: Type specifier - :type label: SignatureTypes - :param match_type: the matching policy to use when trying - to match this signature type, e.g. fuzzy, simple... - :type match_type: MatchTypes - :param match_score: if the matching type is fuzzy, this is - the score to use as threshold, defaults to None - :type match_score: int, optional - """ - - def __init__( - self, - label: SignatureTypes, - match_type: MatchTypes = MatchTypes.MATCH_TYPE_SIMPLE, - match_score: int | None = None, - ): - self.label = label - self.match_policy = SignatureMatch(match_type, match_score) - - def make_struct_for_matching(self, data): - """Provided some `data`, formats a dictionary specifying the matching - policy to use by the helper to match expected signatures (from expectations) - with actual, alert signatures (from the security software) - - :param data: arbitrary data, but most often string or a number primitive - :type: Any - - :return: dictionary of matching specifiers:: - { - "type": str, - "data": any, - "score": (optional) int - } - :rtype: dict - """ - struct = { - "type": self.match_policy.match_type.value, - "data": data, - } - - if self.match_policy.match_score is not None: - struct["score"] = self.match_policy.match_score - - return struct +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "SignatureType", + "SignatureMatch", + "MatchTypes", + "SignatureTypes", +] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/types.py b/pyoaev/signatures/types.py index 5a5102ec..c76b946c 100644 --- a/pyoaev/signatures/types.py +++ b/pyoaev/signatures/types.py @@ -1,76 +1,29 @@ -from enum import Enum - - -class ExpectationType(str, Enum): - DETECTION = "DETECTION" - PREVENTION = "PREVENTION" - VULNERABILITY = "VULNERABILITY" - - -class ExecutionStatus(str, Enum): - EXECUTED = "EXECUTED" - EXECUTED_WITH_CLEANUP_FAILURE = "EXECUTED_WITH_CLEANUP_FAILURE" - WARNING = "WARNING" - ACCESS_DENIED = "ACCESS_DENIED" - ERROR = "ERROR" - COMMAND_NOT_FOUND = "COMMAND_NOT_FOUND" - COMMAND_CANNOT_BE_EXECUTED = "COMMAND_CANNOT_BE_EXECUTED" - PREREQUISITE_FAILED = "PREREQUISITE_FAILED" - INVALID_USAGE = "INVALID_USAGE" - TIMEOUT = "TIMEOUT" - INTERRUPTED = "INTERRUPTED" - ASSET_AGENTLESS = "ASSET_AGENTLESS" - AGENT_INACTIVE = "AGENT_INACTIVE" - AGENT_OVERLOADED = "AGENT_OVERLOADED" - INFO = "INFO" - PARTIAL = "PARTIAL" - MAYBE_PREVENTED = "MAYBE_PREVENTED" - MAYBE_PARTIAL_PREVENTED = "MAYBE_PARTIAL_PREVENTED" - - -class InjectExecutionActions(str, Enum): - PREREQUISITE_CHECK = "prerequisite_check" - PREREQUISITE_EXECUTION = "prerequisite_execution" - CLEANUP_EXECUTION = "cleanup_execution" - COMMAND_EXECUTION = "command_execution" - DNS_RESOLUTION = "dns_resolution" - FILE_EXECUTION = "file_execution" - FILE_DROP = "file_drop" - DATA = "data" - COMPLETE = "complete" - - -class MatchTypes(str, Enum): - MATCH_TYPE_FUZZY = "fuzzy" - MATCH_TYPE_SIMPLE = "simple" - - -class SignatureTypes(str, Enum): - SIG_TYPE_PROCESS_NAME = "process_name" - SIG_TYPE_PARENT_PROCESS_NAME = "parent_process_name" - SIG_TYPE_COMMAND_LINE = "command_line" - SIG_TYPE_IPV4_ADDRESS = "ipv4_address" - SIG_TYPE_IPV6_ADDRESS = "ipv6_address" - SIG_TYPE_SOURCE_IPV4_ADDRESS = "source_ipv4_address" - SIG_TYPE_SOURCE_IPV6_ADDRESS = "source_ipv6_address" - SIG_TYPE_TARGET_IPV4_ADDRESS = "target_ipv4_address" - SIG_TYPE_TARGET_IPV6_ADDRESS = "target_ipv6_address" - SIG_TYPE_HOSTNAME = "hostname" - SIG_TYPE_TARGET_HOSTNAME_ADDRESS = "target_hostname_address" - SIG_TYPE_START_DATE = "start_date" - SIG_TYPE_END_DATE = "end_date" - SIG_TYPE_CLOUD_PROVIDER = "cloud_provider" - SIG_TYPE_CLOUD_ACCOUNT_ID = "cloud_account_id" - SIG_TYPE_CLOUD_REGION = "cloud_region" - SIG_TYPE_TARGET_SERVICE = "target_service" - SIG_TYPE_QUERY = "query" - SIG_TYPE_URL_HASH = "url_hash" - SIG_TYPE_FILE_NAME = "file_name" - SIG_TYPE_FILE_HASH = "file_hash" - SIG_TYPE_SOURCE_EMAIL = "source_email" - SIG_TYPE_TARGET_EMAIL = "target_email" - SIG_TYPE_EMAIL_CUSTOM_HEADER = "email_custom_header" - # AI adversarial validation: correlate AI defense (LLM firewall / guardrail) events back to a - # specific AI inject execution. - SIG_TYPE_AI_REQUEST_MARKER = "ai_request_marker" - SIG_TYPE_AI_TARGET_ENDPOINT = "ai_target_endpoint" +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ExpectationType", + "ExecutionStatus", + "InjectExecutionActions", + "MatchTypes", + "SignatureTypes", +] + +_RENAMED = {"ExpectationType": "SignatureExpectationType"} + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from xtm_oaev_sdk import {_RENAMED.get(name, name)}' instead.", + getattr(_sdk, _RENAMED.get(name, name)), + ) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/utils.py b/pyoaev/utils.py index 21e7c7bd..a9177928 100644 --- a/pyoaev/utils.py +++ b/pyoaev/utils.py @@ -1,14 +1,38 @@ +"""Utilities specific to the pyoaev HTTP client. + +The shared utilities moved to :mod:`xtm_oaev_sdk`; their legacy names here +resolve to the *identical* SDK objects and emit a :class:`DeprecationWarning` +on access. ``EncodedId``, ``RequiredOptional``, ``copy_dict`` and +``remove_none_from_dict`` are REST client helpers and stay defined here. +""" + import dataclasses -import datetime import email.message -import json -import logging -import threading import urllib.parse from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union import requests -from pythonjsonlogger.json import JsonFormatter +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +_MOVED = [ + "AppLogger", + "CustomJsonFormatter", + "EnhancedJSONEncoder", + "logger", + "setup_logging_config", + "PingAlive", +] + +__all__ = [ # noqa: F822 (moved names are served by the PEP 562 __getattr__ below) + "EncodedId", + "RequiredOptional", + "copy_dict", + "get_content_type", + "remove_none_from_dict", + "response_content", + *_MOVED, +] class _StdoutStream: @@ -46,6 +70,18 @@ def response_content( return None +class EncodedId(str): + def __new__(cls, value: Union[str, int, "EncodedId"]) -> "EncodedId": + if isinstance(value, EncodedId): + return value + + if not isinstance(value, (int, str)): + raise TypeError(f"Unsupported type received: {type(value)}") + if isinstance(value, str): + value = urllib.parse.quote(value, safe="") + return super().__new__(cls, value) + + def copy_dict( *, src: Dict[str, Any], @@ -59,29 +95,10 @@ def copy_dict( dest[k] = v -class EncodedId(str): - def __new__(cls, value: Union[str, int, "EncodedId"]) -> "EncodedId": - if isinstance(value, EncodedId): - return value - - if not isinstance(value, (int, str)): - raise TypeError(f"Unsupported type received: {type(value)}") - if isinstance(value, str): - value = urllib.parse.quote(value, safe="") - return super().__new__(cls, value) - - def remove_none_from_dict(data: Dict[str, Any]) -> Dict[str, Any]: return {k: v for k, v in data.items() if v is not None} -class EnhancedJSONEncoder(json.JSONEncoder): - def default(self, o): - if dataclasses.is_dataclass(o): - return dataclasses.asdict(o) - return super().default(o) - - @dataclasses.dataclass(frozen=True) class RequiredOptional: required: Tuple[str, ...] = () @@ -116,103 +133,10 @@ def validate_attrs( ) -class CustomJsonFormatter(JsonFormatter): - def add_fields(self, log_record, record, message_dict): - super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict) - if not log_record.get("timestamp"): - # This doesn't use record.created, so it is slightly off - now = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") - log_record["timestamp"] = now - if log_record.get("level"): - log_record["level"] = log_record["level"].upper() - else: - log_record["level"] = record.levelname - - -def setup_logging_config(level, json_logging=True): - logging.getLogger("urllib3").setLevel(logging.WARNING) - logging.getLogger("pika").setLevel(logging.ERROR) - # Exceptions - if json_logging: - log_handler = logging.StreamHandler() - log_handler.setLevel(level) - formatter = CustomJsonFormatter("%(timestamp)s %(level)s %(name)s %(message)s") - log_handler.setFormatter(formatter) - logging.basicConfig(handlers=[log_handler], level=level, force=True) - else: - logging.basicConfig(level=level) - - -class AppLogger: - def __init__(self, level, json_logging=True, name: str = __name__): - self.log_level = level - self.json_logging = json_logging - setup_logging_config(self.log_level, self.json_logging) - self.local_logger = logging.getLogger(name) - - def __call__(self, name): - self.local_logger = logging.getLogger(name) - return self - - @staticmethod - def prepare_meta(meta=None): - return None if meta is None else {"attributes": meta} - - @staticmethod - def setup_logger_level(lib, log_level): - logging.getLogger(lib).setLevel(log_level) - - def debug(self, message, meta=None): - self.local_logger.debug(message, extra=AppLogger.prepare_meta(meta)) - - def info(self, message, meta=None): - self.local_logger.info(message, extra=AppLogger.prepare_meta(meta)) - - def warning(self, message, meta=None): - self.local_logger.warning(message, extra=AppLogger.prepare_meta(meta)) - - def error(self, message, meta=None): - # noinspection PyTypeChecker - self.local_logger.error(message, exc_info=1, extra=AppLogger.prepare_meta(meta)) - - -# DEPRECATED: compatibility -def logger(level, json_logging=True): - return AppLogger(level, json_logging) - - -class PingAlive(threading.Thread): - def __init__(self, api, config, logger, ping_type) -> None: - threading.Thread.__init__(self) - self.ping_type = ping_type - self.api = api - self.tenant_id = getattr(self.api, "tenant_id", None) - self.config = config - self.logger = logger - self.in_error = False - self.exit_event = threading.Event() - - def ping(self) -> None: - while not self.exit_event.is_set(): - try: - if self.ping_type == "injector": - self.api.injector.create(self.config, False) - else: - self.api.collector.create(self.config, False) - except Exception as err: # pylint: disable=broad-except - self.logger.error("Error pinging the API: " + str(err)) - self.exit_event.wait(40) - - def run(self) -> None: - self.logger.info( - "Starting PingAlive thread", - {"tenant_id": str(self.tenant_id) if self.tenant_id else None}, - ) - self.ping() - - def stop(self) -> None: - self.logger.info( - "Preparing PingAlive for clean shutdown", - {"tenant_id": str(self.tenant_id) if self.tenant_id else None}, - ) - self.exit_event.set() +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in _MOVED + }, + module_name=__name__, +) diff --git a/pyproject.toml b/pyproject.toml index 5a7bb308..12b5ab72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ maintainers = [ ] license = { text = "Apache-2.0" } readme = "README.md" -requires-python = ">=3.11" +requires-python = ">=3.12" classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -25,14 +25,14 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules" ] dependencies = [ - "pika (>=1.4.4,<1.5.0)", - "python_json_logger (>=3.3.0,<3.4.0)", - "PyYAML (>=6.0,<6.1)", "pydantic (>=2.13.3,<2.14.0)", - "pydantic-settings (>=2.15.0,<2.16.0)", "requests (>=2.33.0,<2.34.0)", "requests-toolbelt (>=1.0.0,<1.1.0)", - "thefuzz (>=0.22,<0.23)", + # OpenAEV SDKs (collectors-sdk and injectors-sdk host the former daemon + # helpers and stay required while pyoaev keeps their deprecation shims) + "xtm-oaev-sdk (>=0.1.0,<0.2.0)", + "collectors-sdk (>=0.1.0,<0.2.0)", + "injectors-sdk (>=0.1.0,<0.2.0)", ] [project.optional-dependencies] @@ -61,6 +61,7 @@ build-backend = "setuptools.build_meta" include-package-data = true [tool.setuptools.packages.find] +include = ["pyoaev*"] [tool.setuptools.dynamic] version = { attr = "pyoaev.__version__" } diff --git a/test/fixtures/legacy_surface.json b/test/fixtures/legacy_surface.json new file mode 100644 index 00000000..3a0a26dd --- /dev/null +++ b/test/fixtures/legacy_surface.json @@ -0,0 +1,657 @@ +{ + "modules": { + "pyoaev": [ + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "BaseConfigModel", + "CloudInjectorConfig", + "CloudProvider", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", + "ConfigurationError", + "ContractBuilder", + "ExpectationSignatureGroup", + "ExpectationType", + "ExtraSignatureData", + "InjectExecutionActions", + "MatchTypes", + "NetworkInjectorConfig", + "OpenAEV", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SettingsLoader", + "SignatureCallbackPayload", + "SignatureManager", + "SignaturePayload", + "SignatureTarget", + "SignatureTransmissionError", + "SignatureTypes", + "SignatureValue", + "TargetSignatures", + "__author__", + "__copyright__", + "__email__", + "__license__", + "__title__", + "__version__", + "build_network_configs" + ], + "pyoaev._version": [], + "pyoaev.apis": [ + "AiTarget", + "AiTargetManager", + "Any", + "AttackPattern", + "AttackPatternManager", + "Collector", + "CollectorManager", + "CreateMixin", + "Cve", + "CveManager", + "DeleteMixin", + "DetectionExpectation", + "Dict", + "Document", + "DocumentManager", + "Endpoint", + "EndpointManager", + "ExecutionDetails", + "ExpectationTypeEnum", + "GetMixin", + "GetWithoutIdMixin", + "Inject", + "InjectExpectation", + "InjectExpectationManager", + "InjectExpectationTrace", + "InjectExpectationTraceManager", + "InjectManager", + "Injector", + "InjectorContract", + "InjectorContractManager", + "InjectorContractSearchPaginationInput", + "InjectorManager", + "KillChainPhase", + "KillChainPhaseManager", + "List", + "ListMixin", + "Me", + "MeManager", + "Organization", + "OrganizationManager", + "Payload", + "PayloadManager", + "PreventionExpectation", + "RESTManager", + "RESTObject", + "RequiredOptional", + "SearchPaginationInput", + "SecurityPlatform", + "SecurityPlatformManager", + "Signature", + "SignatureApiManager", + "SignatureCallbackPayload", + "SignatureOutputStructure", + "SignatureTransmissionError", + "Tag", + "TagManager", + "Team", + "TeamManager", + "UpdateMixin", + "User", + "UserManager", + "ValidationError", + "Vulnerability", + "VulnerabilityManager", + "ai_target", + "attack_pattern", + "cast", + "collector", + "cve", + "document", + "endpoint", + "exc", + "inject", + "inject_expectation", + "inject_expectation_trace", + "injector", + "injector_contract", + "inputs", + "json", + "kill_chain_phase", + "logging", + "me", + "model", + "organization", + "payload", + "security_platform", + "signature", + "tag", + "team", + "time", + "user", + "vulnerability" + ], + "pyoaev.apis.ai_target": [ + "AiTarget", + "AiTargetManager", + "CreateMixin", + "DeleteMixin", + "GetMixin", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.attack_pattern": [ + "AttackPattern", + "AttackPatternManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.collector": [ + "Collector", + "CollectorManager", + "CreateMixin", + "GetMixin", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.cve": [ + "Cve", + "CveManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.document": [ + "Document", + "DocumentManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.endpoint": [ + "Endpoint", + "EndpointManager", + "RESTManager", + "RESTObject", + "RequiredOptional", + "SearchPaginationInput" + ], + "pyoaev.apis.inject": [ + "Inject", + "InjectManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.inject_expectation": [ + "DetectionExpectation", + "ExpectationTypeEnum", + "InjectExpectation", + "InjectExpectationManager", + "ListMixin", + "PreventionExpectation", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.inject_expectation.inject_expectation": [ + "DetectionExpectation", + "ExpectationTypeEnum", + "InjectExpectation", + "InjectExpectationManager", + "ListMixin", + "PreventionExpectation", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.inject_expectation.model": [ + "DetectionExpectation", + "ExpectationTypeEnum", + "PreventionExpectation" + ], + "pyoaev.apis.inject_expectation.model.expectation": [ + "DetectionExpectation", + "Expectation", + "ExpectationSignature", + "ExpectationTypeEnum", + "MatchTypes", + "PreventionExpectation", + "SignatureType", + "SignatureTypes" + ], + "pyoaev.apis.inject_expectation_trace": [ + "CreateMixin", + "InjectExpectationTrace", + "InjectExpectationTraceManager", + "RESTManager", + "RESTObject", + "RequiredOptional" + ], + "pyoaev.apis.injector": [ + "CreateMixin", + "GetMixin", + "Injector", + "InjectorManager", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.injector_contract": [ + "CreateMixin", + "DeleteMixin", + "InjectorContract", + "InjectorContractManager", + "InjectorContractSearchPaginationInput", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.inputs": [], + "pyoaev.apis.inputs.search": [ + "Filter", + "FilterGroup", + "InjectorContractSearchPaginationInput", + "SearchPaginationInput" + ], + "pyoaev.apis.kill_chain_phase": [ + "KillChainPhase", + "KillChainPhaseManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.me": [ + "GetWithoutIdMixin", + "Me", + "MeManager", + "RESTManager", + "RESTObject", + "UpdateMixin" + ], + "pyoaev.apis.organization": [ + "ListMixin", + "Organization", + "OrganizationManager", + "RESTManager", + "RESTObject", + "UpdateMixin" + ], + "pyoaev.apis.payload": [ + "Payload", + "PayloadManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.security_platform": [ + "CreateMixin", + "GetMixin", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "SecurityPlatform", + "SecurityPlatformManager", + "UpdateMixin" + ], + "pyoaev.apis.signature": [ + "ExecutionDetails", + "RESTManager", + "RESTObject", + "Signature", + "SignatureApiManager", + "SignatureCallbackPayload", + "SignatureOutputStructure", + "SignatureTransmissionError" + ], + "pyoaev.apis.tag": [ + "RESTManager", + "RESTObject", + "Tag", + "TagManager" + ], + "pyoaev.apis.team": [ + "CreateMixin", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "Team", + "TeamManager", + "UpdateMixin" + ], + "pyoaev.apis.user": [ + "CreateMixin", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin", + "User", + "UserManager" + ], + "pyoaev.apis.vulnerability": [ + "RESTManager", + "RESTObject", + "Vulnerability", + "VulnerabilityManager" + ], + "pyoaev.asset_types": [ + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "CloudProvider" + ], + "pyoaev.backends": [ + "DefaultBackend", + "DefaultResponse", + "RequestsBackend", + "RequestsResponse", + "TokenAuth" + ], + "pyoaev.backends.backend": [ + "Auth", + "RequestsBackend", + "RequestsResponse", + "SendData", + "TokenAuth" + ], + "pyoaev.backends.protocol": [ + "Backend", + "BackendResponse" + ], + "pyoaev.base": [ + "OpenAEV", + "OpenAEVList", + "OpenAEVParsingError", + "RESTManager", + "RESTObject", + "RESTObjectList" + ], + "pyoaev.client": [ + "OpenAEV", + "OpenAEVList", + "REDIRECT_MSG" + ], + "pyoaev.configuration": [ + "BaseConfigModel", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", + "SettingsLoader" + ], + "pyoaev.configuration.configuration": [ + "Configuration", + "ConfigurationHint", + "ConnectorConfigSchemaGenerator", + "DictionarySource", + "EnvironmentSource", + "is_falsy", + "is_truthy" + ], + "pyoaev.configuration.connector_config_schema_generator": [ + "ConnectorConfigSchemaGenerator" + ], + "pyoaev.configuration.settings_loader": [ + "BaseConfigModel", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "SettingsLoader" + ], + "pyoaev.configuration.sources": [ + "DictionarySource", + "EnvironmentSource" + ], + "pyoaev.contracts": [ + "ContractBuilder" + ], + "pyoaev.contracts.contract_builder": [ + "ContractBuilder", + "ContractElement", + "ContractOutputElement" + ], + "pyoaev.contracts.contract_config": [ + "Contract", + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractElement", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "CredentialType", + "Domain", + "Expectation", + "ExpectationType", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableHelper", + "prepare_contracts" + ], + "pyoaev.contracts.contract_utils": [ + "ContractCardinality", + "ContractVariable", + "VariableType" + ], + "pyoaev.contracts.variable_helper": [ + "CHALLENGES_URI", + "COMCHECK", + "ContractCardinality", + "ContractVariable", + "EXERCISE", + "LESSONS_URI", + "PLAYER_URI", + "SCOREBOARD_URI", + "TEAMS", + "USER", + "VariableHelper", + "VariableType" + ], + "pyoaev.credential": [ + "CredentialType", + "build_single_referenced_credential_element" + ], + "pyoaev.credential.types": [ + "CredentialType" + ], + "pyoaev.credential.utils": [ + "ContractReferencedCredential", + "CredentialType", + "build_single_referenced_credential_element" + ], + "pyoaev.daemons": [ + "BaseDaemon", + "CollectorDaemon" + ], + "pyoaev.daemons.base_daemon": [ + "BaseDaemon", + "Configuration", + "OpenAEV", + "OpenAEVError", + "logger" + ], + "pyoaev.daemons.collector_daemon": [ + "BaseDaemon", + "CollectorDaemon", + "Configuration", + "DEFAULT_PERIOD_SECONDS", + "DEFAULT_PLATFORM_TAG_COLOR", + "PingAlive" + ], + "pyoaev.exceptions": [ + "ConfigurationError", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SignatureTransmissionError", + "on_http_error" + ], + "pyoaev.helpers": [ + "CollectorDaemon", + "Configuration", + "ConfigurationError", + "FALSY", + "ListenQueue", + "OpenAEV", + "OpenAEVCollectorHelper", + "OpenAEVConfigHelper", + "OpenAEVDetectionHelper", + "OpenAEVInjectorHelper", + "PingAlive", + "TRUTHY", + "create_mq_ssl_context", + "data_to_temp_file", + "is_memory_certificate", + "ssl_cert_chain", + "ssl_verify_locations" + ], + "pyoaev.mixins": [ + "CreateMixin", + "DeleteMixin", + "GetMixin", + "GetWithoutIdMixin", + "HeadMixin", + "ListMixin", + "UpdateMethod", + "UpdateMixin" + ], + "pyoaev.security_domain": [], + "pyoaev.security_domain.types": [ + "SecurityDomains" + ], + "pyoaev.signatures": [ + "CloudInjectorConfig", + "ExpectationSignatureGroup", + "ExpectationType", + "ExtraSignatureData", + "InjectExecutionActions", + "InjectorConfig", + "MatchTypes", + "NetworkInjectorConfig", + "SignatureCallbackPayload", + "SignatureManager", + "SignaturePayload", + "SignatureTarget", + "SignatureTypes", + "SignatureValue", + "TargetSignatures", + "build_network_configs" + ], + "pyoaev.signatures.ai_marker": [ + "build_marker" + ], + "pyoaev.signatures.models": [ + "CloudInjectorConfig", + "ExecutionDetails", + "ExecutionSignature", + "ExecutionStatus", + "ExpectationSignatureGroup", + "ExpectationType", + "ExtraSignatureData", + "InjectExecutionActions", + "InjectorConfig", + "NetworkInjectorConfig", + "SignatureCallbackPayload", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureValue", + "TargetSignatures", + "ToolErrorInfo", + "ToolOutput", + "ToolTimeoutInfo", + "build_network_configs" + ], + "pyoaev.signatures.signature_manager": [ + "CloudInjectorConfig", + "ExecutionDetails", + "ExecutionSignature", + "ExpectationSignatureGroup", + "ExtraSignatureData", + "NetworkInjectorConfig", + "OpenAEVError", + "SignatureManager", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureValue", + "TargetSignatures", + "ToolOutput" + ], + "pyoaev.signatures.signature_match": [ + "MatchTypes", + "OpenAEVError", + "SignatureMatch" + ], + "pyoaev.signatures.signature_type": [ + "MatchTypes", + "SignatureMatch", + "SignatureType", + "SignatureTypes" + ], + "pyoaev.signatures.types": [ + "ExecutionStatus", + "ExpectationType", + "InjectExecutionActions", + "MatchTypes", + "SignatureTypes" + ], + "pyoaev.utils": [ + "AppLogger", + "CustomJsonFormatter", + "EncodedId", + "EnhancedJSONEncoder", + "PingAlive", + "RequiredOptional", + "copy_dict", + "get_content_type", + "logger", + "remove_none_from_dict", + "response_content", + "setup_logging_config" + ] + }, + "pyoaev_version": "3.261014.1" +} diff --git a/test/generate_legacy_surface.py b/test/generate_legacy_surface.py new file mode 100644 index 00000000..bc1500ea --- /dev/null +++ b/test/generate_legacy_surface.py @@ -0,0 +1,64 @@ +"""Record the public surface of an installed pyoaev release. + +The legacy surface guard (``test_legacy_surface.py``) asserts that every name a +released pyoaev exposed still resolves, so the SDK split never breaks existing +code. Regenerate the golden from the *released* package, never from the +working tree:: + + python -m venv /tmp/pyoaev-release + /tmp/pyoaev-release/bin/pip install "pyoaev==" + /tmp/pyoaev-release/bin/python test/generate_legacy_surface.py + +A public name is anything in a module's ``__all__``, any class or function +defined in pyoaev (including re-exports from other pyoaev modules), and any +upper-case data constant. +""" + +import importlib +import json +import pkgutil +import types +import warnings +from pathlib import Path + +import pyoaev + +GOLDEN_PATH = Path(__file__).parent / "fixtures" / "legacy_surface.json" + +_DATA_CONSTANT_TYPES = (str, int, float, list, tuple, dict) + + +def _is_public(name: str, obj: object, exported: set[str]) -> bool: + if name in exported: + return True + if name.startswith("_") or isinstance(obj, types.ModuleType): + return False + if (getattr(obj, "__module__", None) or "").startswith("pyoaev"): + return True + is_data = isinstance(obj, _DATA_CONSTANT_TYPES) and not isinstance(obj, bool) + return name.isupper() and is_data + + +def _public_names(module: types.ModuleType) -> list[str]: + exported = set(getattr(module, "__all__", ())) + names = [] + for name in dir(module): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + obj = getattr(module, name, None) + if _is_public(name, obj, exported): + names.append(name) + return sorted(names) + + +def main() -> None: + modules = {"pyoaev": _public_names(pyoaev)} + for info in pkgutil.walk_packages(pyoaev.__path__, "pyoaev."): + modules[info.name] = _public_names(importlib.import_module(info.name)) + surface = {"pyoaev_version": pyoaev.__version__, "modules": modules} + GOLDEN_PATH.write_text(json.dumps(surface, indent=2, sort_keys=True) + "\n") + print(f"wrote {GOLDEN_PATH}") + + +if __name__ == "__main__": + main() diff --git a/test/test_legacy_surface.py b/test/test_legacy_surface.py new file mode 100644 index 00000000..c5a135cb --- /dev/null +++ b/test/test_legacy_surface.py @@ -0,0 +1,67 @@ +"""Legacy surface guard: the SDK split must never break existing code. + +Every public name the released pyoaev exposed (recorded in +``fixtures/legacy_surface.json`` by ``generate_legacy_surface.py``) must still +resolve from the same module, possibly through a deprecation shim. +""" + +import importlib +import json +import warnings +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +GOLDEN_PATH = Path(__file__).parent / "fixtures" / "legacy_surface.json" + + +def _given_legacy_surface() -> dict[str, Any]: + return json.loads(GOLDEN_PATH.read_text(encoding="utf-8")) + + +def _given_legacy_names() -> list[tuple[str, str]]: + modules = _given_legacy_surface()["modules"] + return [(module, name) for module, names in modules.items() for name in names] + + +def _when_importing_module(module: str) -> ModuleType: + return importlib.import_module(module) + + +def _when_resolving_name(module: ModuleType, name: str) -> bool: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + try: + getattr(module, name) + except AttributeError: + return False + return True + + +def _then_name_resolves(resolved: bool, module: str, name: str) -> None: + assert resolved, ( + f"'{module}.{name}' no longer resolves: legacy code importing it breaks. " + "Serve it from the module's deprecation shim." + ) + + +def test_golden_records_a_released_pyoaev() -> None: + """It pins the surface of a released pyoaev version.""" + surface = _given_legacy_surface() + assert surface["pyoaev_version"] + assert surface["modules"]["pyoaev"] + + +@pytest.mark.parametrize("module", sorted(_given_legacy_surface()["modules"])) +def test_every_legacy_module_still_imports(module: str) -> None: + """It keeps every legacy module importable.""" + assert _when_importing_module(module).__name__ == module + + +@pytest.mark.parametrize(("module", "name"), _given_legacy_names()) +def test_every_legacy_name_still_resolves(module: str, name: str) -> None: + """It keeps every legacy name reachable from its original module.""" + resolved = _when_resolving_name(_when_importing_module(module), name) + _then_name_resolves(resolved, module, name) diff --git a/test/test_rest_client_helpers.py b/test/test_rest_client_helpers.py new file mode 100644 index 00000000..c9775674 --- /dev/null +++ b/test/test_rest_client_helpers.py @@ -0,0 +1,246 @@ +"""REST client helpers owned by pyoaev. + +RequiredOptional, EncodedId, copy_dict, remove_none_from_dict and on_http_error. +""" + +import dataclasses +import subprocess +import sys +from typing import Any + +import pytest +import xtm_oaev_sdk as sdk + +from pyoaev.exceptions import on_http_error +from pyoaev.utils import EncodedId, RequiredOptional, copy_dict, remove_none_from_dict + + +def _given_required_optional(**kwargs: Any) -> RequiredOptional: + return RequiredOptional(**kwargs) + + +def _given_exclusive_spec() -> RequiredOptional: + return _given_required_optional(exclusive=("id", "key")) + + +def _given_function_raising(error: Exception): + def function() -> None: + raise error + + return function + + +def _when_set_required_optional_field_raises( + instance: RequiredOptional, field_name: str, value: Any +) -> Exception: + with pytest.raises(dataclasses.FrozenInstanceError) as exc_info: + setattr(instance, field_name, value) + return exc_info.value + + +def _when_validate_attrs( + instance: RequiredOptional, data: dict[str, Any], excludes: list[str] | None = None +) -> None: + instance.validate_attrs(data=data, excludes=excludes) + + +def _when_validate_attrs_raises( + instance: RequiredOptional, data: dict[str, Any] +) -> Exception: + with pytest.raises(AttributeError) as exc_info: + _when_validate_attrs(instance, data) + return exc_info.value + + +def _when_encode_id(value: Any) -> EncodedId: + return EncodedId(value) + + +def _when_encode_invalid_id_raises(value: Any) -> Exception: + with pytest.raises(TypeError) as exc_info: + _when_encode_id(value) + return exc_info.value + + +def _when_copy_dict(src: dict[str, Any], dest: dict[str, Any]) -> None: + copy_dict(src=src, dest=dest) + + +def _when_remove_none_from_dict(data: dict[str, Any]) -> dict[str, Any]: + return remove_none_from_dict(data) + + +def _when_call_decorated_raises(function, error: type[Exception]) -> Exception: + with pytest.raises(error) as exc_info: + on_http_error(error)(function)() + return exc_info.value + + +def _when_run(script: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-W", "error::DeprecationWarning", "-c", script], + capture_output=True, + text=True, + ) + + +def _then_contains_text(exc: Exception, text: str) -> None: + assert text in str(exc) + + +def _then_error_is_frozen(exc: Exception) -> None: + assert isinstance(exc, dataclasses.FrozenInstanceError) + + +def _then_dict_equals(actual: dict[str, Any], expected: dict[str, Any]) -> None: + assert actual == expected + + +def _then_encoded_object_equals(encoded: EncodedId, expected: str) -> None: + assert isinstance(encoded, EncodedId) + assert encoded == expected + + +def test_required_optional_is_frozen() -> None: + """It rejects field assignment on RequiredOptional.""" + required_optional = _given_required_optional(required=("name",)) + exc = _when_set_required_optional_field_raises( + required_optional, "required", ("id",) + ) + _then_error_is_frozen(exc) + + +def test_required_optional_validate_attrs_missing_required_raises() -> None: + """It raises when a required attribute is missing.""" + required_optional = _given_required_optional(required=("name", "type")) + exc = _when_validate_attrs_raises(required_optional, {"name": "sdk"}) + _then_contains_text(exc, "Missing attributes") + + +@pytest.mark.parametrize( + ("data", "expected_text"), + [ + ({}, "Must provide one of these attributes"), + ({"id": "1", "key": "2"}, "Provide only one of these attributes"), + ], +) +def test_required_optional_validate_attrs_exclusive_violations_raise( + data: dict[str, Any], expected_text: str +) -> None: + """It raises when exclusive attributes are both missing or both present.""" + required_optional = _given_exclusive_spec() + exc = _when_validate_attrs_raises(required_optional, data) + _then_contains_text(exc, expected_text) + + +def test_required_optional_validate_attrs_excludes_skip_required_check() -> None: + """It skips the required check for excluded attributes.""" + required_optional = _given_required_optional(required=("name", "type")) + _when_validate_attrs(required_optional, {"name": "sdk"}, excludes=["type"]) + + +def test_required_optional_validate_attrs_valid_data_passes() -> None: + """It accepts data satisfying the required and exclusive rules.""" + required_optional = _given_required_optional( + required=("name",), exclusive=("id", "key") + ) + _when_validate_attrs(required_optional, {"name": "sdk", "id": "123"}) + + +def test_encoded_id_url_encodes_strings() -> None: + """It percent-encodes unsafe string characters.""" + encoded = _when_encode_id("hello world") + _then_encoded_object_equals(encoded, "hello%20world") + + +def test_encoded_id_is_idempotent_when_reencoded() -> None: + """It returns the same value when re-encoding an EncodedId.""" + encoded = _when_encode_id("hello world") + _then_encoded_object_equals(_when_encode_id(encoded), "hello%20world") + + +def test_encoded_id_rejects_non_str_non_int_values() -> None: + """It raises TypeError for unsupported value types.""" + exc = _when_encode_invalid_id_raises(3.14) + _then_contains_text(exc, "Unsupported type received") + + +def test_encoded_id_accepts_int_and_returns_string_representation() -> None: + """It passes integer values through as their string form.""" + encoded = _when_encode_id(42) + _then_encoded_object_equals(encoded, "42") + + +def test_copy_dict_flattens_nested_dict_keys() -> None: + """It flattens nested dict keys to bracket notation.""" + destination: dict[str, Any] = {} + _when_copy_dict(src={"filters": {"type": "malware"}}, dest=destination) + _then_dict_equals(destination, {"filters[type]": "malware"}) + + +def test_copy_dict_copies_non_dict_values_directly() -> None: + """It copies non-dict values without transformation.""" + destination: dict[str, Any] = {} + _when_copy_dict(src={"name": "sdk", "count": 2}, dest=destination) + _then_dict_equals(destination, {"name": "sdk", "count": 2}) + + +def test_remove_none_from_dict_strips_none_valued_keys() -> None: + """It removes the entries whose value is None.""" + result = _when_remove_none_from_dict({"a": 1, "b": None, "c": "x"}) + _then_dict_equals(result, {"a": 1, "c": "x"}) + + +def test_remove_none_from_dict_keeps_non_none_values() -> None: + """It keeps falsy values that are not None.""" + result = _when_remove_none_from_dict({"a": 0, "b": False, "c": ""}) + _then_dict_equals(result, {"a": 0, "b": False, "c": ""}) + + +def test_on_http_error_wraps_http_errors_into_the_given_type() -> None: + """It re-raises an HTTP error as the given error, keeping its details.""" + http_error = sdk.OpenAEVHttpError("boom", 404, b"missing") + exc = _when_call_decorated_raises( + _given_function_raising(http_error), sdk.OpenAEVGetError + ) + assert (exc.error_message, exc.response_code, exc.response_body) == ( + "boom", + 404, + b"missing", + ) + assert exc.__cause__ is http_error + + +def test_on_http_error_lets_other_errors_through() -> None: + """It does not touch errors that are not HTTP errors.""" + exc = _when_call_decorated_raises(_given_function_raising(KeyError("k")), KeyError) + _then_contains_text(exc, "k") + + +def test_on_http_error_returns_the_wrapped_result_and_metadata() -> None: + """It returns the decorated function's result and keeps its name.""" + + def answer() -> int: + return 42 + + decorated = on_http_error(sdk.OpenAEVGetError)(answer) + assert decorated() == 42 + assert decorated.__name__ == "answer" + + +def test_rest_client_helpers_are_pyoaev_own_and_silent() -> None: + """It serves the REST client helpers from pyoaev itself, without any warning.""" + result = _when_run( + "import pyoaev.utils, pyoaev.exceptions, xtm_oaev_sdk as sdk\n" + "assert pyoaev.utils.EncodedId.__module__ == 'pyoaev.utils'\n" + "assert pyoaev.utils.RequiredOptional.__module__ == 'pyoaev.utils'\n" + "assert pyoaev.utils.copy_dict.__module__ == 'pyoaev.utils'\n" + "assert pyoaev.utils.remove_none_from_dict.__module__ == 'pyoaev.utils'\n" + "assert pyoaev.exceptions.on_http_error.__module__ == 'pyoaev.exceptions'\n" + "assert not hasattr(sdk, 'EncodedId')\n" + "assert not hasattr(sdk, 'RequiredOptional')\n" + "assert not hasattr(sdk, 'on_http_error')\n" + "assert not hasattr(sdk, 'copy_dict')\n" + "assert not hasattr(sdk, 'remove_none_from_dict')\n" + ) + assert result.returncode == 0, result.stderr diff --git a/test/test_sdk_split_shims.py b/test/test_sdk_split_shims.py new file mode 100644 index 00000000..06747c8c --- /dev/null +++ b/test/test_sdk_split_shims.py @@ -0,0 +1,246 @@ +"""Contract of the pyoaev shims toward collectors-sdk and injectors-sdk. + +Legacy imports and patch targets of collectors and injectors must keep working: +names resolve to the *identical* SDK object, or to a thin subclass that keeps +building the OpenAEV client, and warn at the consumer's line. +""" + +import subprocess +import sys +import warnings +from unittest.mock import MagicMock, mock_open, patch + +import collectors_sdk +import injectors_sdk +import pytest +import xtm_oaev_sdk + +_CONSUMER = "legacy_consumer.py" + +IDENTICAL = [ + ("pyoaev.helpers", "OpenAEVDetectionHelper", collectors_sdk), + ("pyoaev.helpers", "ListenQueue", injectors_sdk), + ("pyoaev.helpers", "OpenAEVConfigHelper", injectors_sdk), + ("pyoaev.helpers", "create_mq_ssl_context", injectors_sdk), + ("pyoaev.helpers", "PingAlive", xtm_oaev_sdk), + ("pyoaev.helpers", "Configuration", xtm_oaev_sdk), + ("pyoaev.helpers", "ConfigurationError", xtm_oaev_sdk), + ("pyoaev.helpers", "logger", xtm_oaev_sdk), + ("pyoaev.daemons.collector_daemon", "DEFAULT_PERIOD_SECONDS", collectors_sdk), + ("pyoaev.daemons.collector_daemon", "DEFAULT_PLATFORM_TAG_COLOR", collectors_sdk), + ("pyoaev.apis.inject_expectation.model", "DetectionExpectation", collectors_sdk), + ("pyoaev.apis.inject_expectation.model", "PreventionExpectation", collectors_sdk), + ("pyoaev.apis.inject_expectation.model", "ExpectationTypeEnum", collectors_sdk), + ("pyoaev.apis.inject_expectation.model", "Expectation", collectors_sdk), + ("pyoaev.apis.inject_expectation.model.expectation", "Expectation", collectors_sdk), + ( + "pyoaev.apis.inject_expectation.model.expectation", + "ExpectationSignature", + collectors_sdk, + ), + *[ + (module, name, collectors_sdk) + for module in ( + "pyoaev.apis.inject_expectation", + "pyoaev.apis.inject_expectation.inject_expectation", + ) + for name in ( + "DetectionExpectation", + "ExpectationTypeEnum", + "PreventionExpectation", + ) + ], +] + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + source = f"from {module} import {name} as _legacy" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(compile(source, _CONSUMER, "exec"), namespace) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + return namespace["_legacy"], deprecations + + +def _then_warns_at_consumer(deprecations) -> None: + assert deprecations + assert any(w.filename == _CONSUMER for w in deprecations) + + +@pytest.mark.parametrize(("module", "name", "sdk"), IDENTICAL) +def test_legacy_name_is_the_sdk_object(module, name, sdk) -> None: + """It warns and returns the identical SDK object.""" + legacy, deprecations = _when_import_legacy_name(module, name) + assert legacy is getattr(sdk, name) + _then_warns_at_consumer(deprecations) + assert sdk.__name__ in str(deprecations[0].message) + + +@pytest.mark.parametrize( + "module", ["pyoaev.daemons", "pyoaev.daemons.collector_daemon", "pyoaev.helpers"] +) +def test_legacy_collector_daemon_is_one_client_building_subclass(module) -> None: + """It resolves every path to the same class, so class patches still apply.""" + from pyoaev.daemons.collector_daemon import _ClientCollectorDaemon + + legacy, deprecations = _when_import_legacy_name(module, "CollectorDaemon") + _then_warns_at_consumer(deprecations) + assert legacy is _ClientCollectorDaemon + assert issubclass(legacy, collectors_sdk.CollectorDaemon) + + +@patch("pyoaev.apis.DocumentManager.upsert", return_value={}) +@patch("pyoaev.apis.CollectorManager.create", return_value={}) +@patch("builtins.open", new_callable=mock_open, read_data="data") +@patch("pyoaev.utils.PingAlive.start") +def test_legacy_collector_daemon_builds_the_openaev_client(*_) -> None: + """It still works with only a configuration, as legacy collectors use it.""" + from pyoaev.client import OpenAEV + from pyoaev.configuration import Configuration + from pyoaev.daemons import CollectorDaemon + + config = Configuration( + config_hints={ + "openaev_url": {"data": "http://example.com"}, + "openaev_token": {"data": "token"}, + "collector_id": {"data": "collector-id"}, + } + ) + collector = CollectorDaemon(configuration=config, collector_type="test") + + assert isinstance(collector.api, OpenAEV) + collector._setup() + + +def test_legacy_collector_daemon_setup_patch_target_still_applies() -> None: + """It honours patches on pyoaev.daemons.collector_daemon.CollectorDaemon._setup.""" + from pyoaev.daemons import CollectorDaemon + + with patch("pyoaev.daemons.collector_daemon.CollectorDaemon._setup") as setup: + collector = CollectorDaemon( + configuration=xtm_oaev_sdk.Configuration(config_hints={}), + api_client=MagicMock(), + collector_type="test", + ) + collector._setup() + + setup.assert_called_once() + + +@patch("xtm_oaev_sdk.PingAlive.start") +def test_legacy_injector_helper_builds_the_openaev_client(_) -> None: + """It still builds the client from the config helper when none is injected.""" + from pyoaev.helpers import OpenAEVConfigHelper, OpenAEVInjectorHelper + + config = OpenAEVConfigHelper.from_configuration_object( + xtm_oaev_sdk.Configuration( + config_hints={ + "openaev_url": {"data": "http://example.com"}, + "openaev_token": {"data": "token"}, + "openaev_tenant_id": {"data": "tenant"}, + "injector_type": {"data": "openaev_test"}, + "injector_log_level": {"data": "info"}, + } + ) + ) + with patch("pyoaev.helpers.OpenAEV") as client_class: + helper = OpenAEVInjectorHelper(config, MagicMock()) + + client_class.assert_called_once_with( + url="http://example.com", token="token", tenant_id="tenant" + ) + assert helper.api is client_class.return_value + assert isinstance(helper, injectors_sdk.OpenAEVInjectorHelper) + + +@pytest.mark.parametrize( + ("name", "value"), + [("TRUTHY", ["yes", "true", "True"]), ("FALSY", ["no", "false", "False"])], +) +def test_legacy_truthiness_constants_keep_their_value(name, value) -> None: + """It keeps the unused legacy constants available, with a warning.""" + legacy, deprecations = _when_import_legacy_name("pyoaev.helpers", name) + assert legacy == value + _then_warns_at_consumer(deprecations) + + +def test_legacy_collector_helper_still_resolves() -> None: + """It keeps the already deprecated OpenAEVCollectorHelper available.""" + legacy, deprecations = _when_import_legacy_name( + "pyoaev.helpers", "OpenAEVCollectorHelper" + ) + _then_warns_at_consumer(deprecations) + assert callable(legacy) + + +def _given_expectation_manager(): + from pyoaev.apis import InjectExpectationManager + + manager = InjectExpectationManager(MagicMock()) + manager.expectations_assets_for_source = MagicMock( + return_value=[ + { + "inject_expectation_id": "4a1bb2c4-6a59-4b4b-a3c0-0d36a4f8e1c1", + "inject_expectation_type": "DETECTION", + "inject_expectation_signatures": [], + } + ] + ) + return manager + + +def _when_call_legacy_models_for_source(manager, **kwargs): + namespace: dict = {"manager": manager, "kwargs": kwargs} + source = "_result = manager.expectations_models_for_source(**kwargs)" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(compile(source, _CONSUMER, "exec"), namespace) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + return namespace["_result"], deprecations + + +def test_expectation_manager_builds_sdk_models() -> None: + """It still hands out collectors-sdk expectation models, with a warning.""" + manager = _given_expectation_manager() + + (model,), deprecations = _when_call_legacy_models_for_source( + manager, source_id="collector" + ) + + assert isinstance(model, collectors_sdk.DetectionExpectation) + _then_warns_at_consumer(deprecations) + assert "from collectors_sdk import expectations_models_for_source" in str( + deprecations[0].message + ) + + +def test_expectation_manager_delegates_to_collectors_sdk() -> None: + """It builds nothing itself: collectors-sdk builds the models from the manager.""" + manager = _given_expectation_manager() + sentinel = [object()] + + with patch.object( + collectors_sdk, "expectations_models_for_source", return_value=sentinel + ) as builder: + result, _ = _when_call_legacy_models_for_source( + manager, source_id="collector", expiration_time=5 + ) + + builder.assert_called_once_with(manager, "collector", expiration_time=5) + assert result is sentinel + + +def test_importing_pyoaev_modules_is_silent() -> None: + """It imports pyoaev and its shim modules without any warning.""" + script = ( + "import pyoaev, pyoaev.helpers, pyoaev.daemons, pyoaev.daemons.collector_daemon\n" + "import pyoaev.apis, pyoaev.apis.inject_expectation.model\n" + "import pyoaev.apis.inject_expectation.model.expectation\n" + ) + result = subprocess.run( + [sys.executable, "-W", "error::DeprecationWarning", "-c", script], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/test/test_utils.py b/test/test_utils.py index e5279b76..3f290703 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,111 +1,17 @@ -import dataclasses -import importlib -import json -import logging import unittest -import warnings from typing import Any, cast from unittest.mock import MagicMock, patch -from pythonjsonlogger.json import JsonFormatter - import pyoaev.utils as module -@dataclasses.dataclass -class _SampleData: - value: int - - class TestUtils(unittest.TestCase): - def test_custom_json_formatter_inherits_non_deprecated_formatter(self): - self.assertTrue(issubclass(module.CustomJsonFormatter, JsonFormatter)) - - def test_reloading_utils_does_not_raise_jsonlogger_deprecation_warning(self): - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - importlib.reload(module) - - deprecation_messages = [ - str(warning.message) - for warning in caught - if issubclass(warning.category, DeprecationWarning) - ] - self.assertFalse( - any( - "pythonjsonlogger.jsonlogger has been moved" in message - for message in deprecation_messages - ) - ) - def test_get_content_type(self): self.assertEqual( module.get_content_type("application/json; charset=utf-8"), "application/json", ) - def test_copy_dict_flattens_nested_dict(self): - destination = {} - module.copy_dict( - src={"a": 1, "meta": {"x": "y", "n": 2}}, - dest=destination, - ) - self.assertEqual(destination, {"a": 1, "meta[x]": "y", "meta[n]": 2}) - - def test_remove_none_from_dict(self): - self.assertEqual(module.remove_none_from_dict({"a": 1, "b": None}), {"a": 1}) - - def test_encoded_id_from_existing_instance_returns_same_object(self): - encoded_id = module.EncodedId("with space") - self.assertIs(module.EncodedId(encoded_id), encoded_id) - - def test_encoded_id_encodes_string_and_keeps_int(self): - self.assertEqual(module.EncodedId("a/b c"), "a%2Fb%20c") - self.assertEqual(module.EncodedId(42), "42") - - def test_encoded_id_rejects_unsupported_type(self): - with self.assertRaises(TypeError): - module.EncodedId(cast(Any, ["bad"])) - - def test_enhanced_json_encoder_serializes_not_dataclasses(self): - data = {"test": "this is a test", "other": 3} - classic_json_dump = json.dumps(data) - enhanced_json_dump = json.dumps(data, cls=module.EnhancedJSONEncoder) - self.assertFalse(module.dataclasses.is_dataclass(data)) - self.assertEqual( - classic_json_dump, - enhanced_json_dump, - ) - - def test_enhanced_json_encoder_serializes_dataclasses(self): - self.assertEqual( - json.dumps(_SampleData(value=3), cls=module.EnhancedJSONEncoder), - '{"value": 3}', - ) - - def test_required_optional_required_attribute_missing(self): - rules = module.RequiredOptional(required=("name",)) - with self.assertRaises(AttributeError): - rules.validate_attrs(data={}) - - def test_required_optional_excludes_required_attribute(self): - rules = module.RequiredOptional(required=("name",)) - rules.validate_attrs(data={}, excludes=["name"]) - - def test_required_optional_exclusive_allows_only_one_key(self): - rules = module.RequiredOptional(exclusive=("a", "b")) - with self.assertRaises(AttributeError): - rules.validate_attrs(data={"a": 1, "b": 2}) - - def test_required_optional_exclusive_requires_one_key(self): - rules = module.RequiredOptional(exclusive=("a", "b")) - with self.assertRaises(AttributeError): - rules.validate_attrs(data={}) - - def test_required_optional_exclusive_with_single_key_is_valid(self): - rules = module.RequiredOptional(exclusive=("a", "b")) - rules.validate_attrs(data={"a": 1}) - def test_response_content_returns_iterator_when_requested(self): response = MagicMock() response.iter_content.return_value = iter([b"a", b"b"]) @@ -167,107 +73,51 @@ def test_response_content_streamed_defaults_to_stdout_stream(self): mock_print.assert_called_once_with(b"visible") - def test_custom_json_formatter_add_fields_sets_timestamp_and_level(self): - formatter = module.CustomJsonFormatter("%(message)s") - record = logging.LogRecord( - name="test", - level=logging.INFO, - pathname=__file__, - lineno=1, - msg="hello", - args=(), - exc_info=None, - ) - log_record = {} - - formatter.add_fields(log_record, record, {}) - - self.assertIn("timestamp", log_record) - self.assertEqual(log_record["level"], "INFO") - - def test_setup_logging_config_json_logging_true_uses_custom_formatter(self): - with patch("pyoaev.utils.logging.basicConfig") as mock_basic_config: - module.setup_logging_config(logging.INFO, json_logging=True) - - kwargs = mock_basic_config.call_args.kwargs - self.assertEqual(kwargs["level"], logging.INFO) - self.assertIn("handlers", kwargs) - self.assertIsInstance( - kwargs["handlers"][0].formatter, module.CustomJsonFormatter - ) - - def test_setup_logging_config_json_logging_false_calls_basic_config(self): - with patch("pyoaev.utils.logging.basicConfig") as mock_basic_config: - module.setup_logging_config(logging.WARNING, json_logging=False) - - mock_basic_config.assert_called_once_with(level=logging.WARNING) - - def test_app_logger_methods_delegate_to_local_logger(self): - with patch("pyoaev.utils.setup_logging_config"): - app_logger = module.AppLogger(logging.INFO) - app_logger.local_logger = MagicMock() - - app_logger.debug("d", {"x": 1}) - app_logger.info("i") - app_logger.warning("w") - app_logger.error("e") - - self.assertTrue(app_logger.local_logger.debug.called) - self.assertTrue(app_logger.local_logger.info.called) - self.assertTrue(app_logger.local_logger.warning.called) - self.assertTrue(app_logger.local_logger.error.called) + def test_encoded_id_from_existing_instance_returns_same_object(self): + encoded_id = module.EncodedId("with space") + self.assertIs(module.EncodedId(encoded_id), encoded_id) - def test_logger_helper_returns_app_logger(self): - with patch("pyoaev.utils.setup_logging_config"): - helper = module.logger(logging.INFO) - self.assertIsInstance(helper, module.AppLogger) + def test_encoded_id_encodes_string_and_keeps_int(self): + self.assertEqual(module.EncodedId("a/b c"), "a%2Fb%20c") + self.assertEqual(module.EncodedId(42), "42") - def test_pingalive_ping_uses_injector_branch(self): - api = MagicMock() - logger = MagicMock() - ping_alive = module.PingAlive( - api=api, config={"id": 1}, logger=logger, ping_type="injector" - ) - ping_alive.exit_event.is_set = MagicMock(side_effect=[False, True]) - ping_alive.exit_event.wait = MagicMock() + def test_encoded_id_rejects_unsupported_type(self): + with self.assertRaises(TypeError): + module.EncodedId(cast(Any, ["bad"])) - ping_alive.ping() + def test_required_optional_required_attribute_missing(self): + rules = module.RequiredOptional(required=("name",)) + with self.assertRaises(AttributeError): + rules.validate_attrs(data={}) - api.injector.create.assert_called_once_with({"id": 1}, False) - ping_alive.exit_event.wait.assert_called_once_with(40) + def test_required_optional_excludes_required_attribute(self): + rules = module.RequiredOptional(required=("name",)) + rules.validate_attrs(data={}, excludes=["name"]) - def test_pingalive_ping_uses_collector_branch_and_logs_errors(self): - api = MagicMock() - api.collector.create.side_effect = Exception("boom") - logger = MagicMock() - ping_alive = module.PingAlive( - api=api, config={}, logger=logger, ping_type="collector" - ) - ping_alive.exit_event.is_set = MagicMock(side_effect=[False, True]) - ping_alive.exit_event.wait = MagicMock() + def test_required_optional_exclusive_allows_only_one_key(self): + rules = module.RequiredOptional(exclusive=("a", "b")) + with self.assertRaises(AttributeError): + rules.validate_attrs(data={"a": 1, "b": 2}) - ping_alive.ping() + def test_required_optional_exclusive_requires_one_key(self): + rules = module.RequiredOptional(exclusive=("a", "b")) + with self.assertRaises(AttributeError): + rules.validate_attrs(data={}) - logger.error.assert_called_once() - ping_alive.exit_event.wait.assert_called_once_with(40) + def test_required_optional_exclusive_with_single_key_is_valid(self): + rules = module.RequiredOptional(exclusive=("a", "b")) + rules.validate_attrs(data={"a": 1}) - def test_pingalive_run_and_stop(self): - ping_alive = module.PingAlive( - api=MagicMock(), config={}, logger=MagicMock(), ping_type="collector" + def test_copy_dict_flattens_nested_dict(self): + destination = {} + module.copy_dict( + src={"a": 1, "meta": {"x": "y", "n": 2}}, + dest=destination, ) - ping_alive.ping = MagicMock() - - ping_alive.run() - ping_alive.stop() + self.assertEqual(destination, {"a": 1, "meta[x]": "y", "meta[n]": 2}) - ping_alive.logger.info.assert_any_call( - "Starting PingAlive thread", {"tenant_id": str(ping_alive.tenant_id)} - ) - ping_alive.logger.info.assert_any_call( - "Preparing PingAlive for clean shutdown", - {"tenant_id": str(ping_alive.tenant_id)}, - ) - self.assertTrue(ping_alive.exit_event.is_set()) + def test_remove_none_from_dict(self): + self.assertEqual(module.remove_none_from_dict({"a": 1, "b": None}), {"a": 1}) if __name__ == "__main__":