From e99b4ac1da4c8b593a5af2909ea617b3870d8977 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 12:31:18 +0200 Subject: [PATCH 01/22] :tada: feat(sdk): migrate configuration to xtm-oaev-sdk with PEP 562 pyoaev shims --- .circleci/config.yml | 4 +- .github/workflows/coverage.yml | 4 +- .../xtm-oaev-sdk/README.md | 0 libs/xtm-oaev-sdk/pyproject.toml | 31 ++ .../test/configuration/shim/__init__.py | 0 .../configuration/shim}/test_configuration.py | 0 .../test_connector_config_schema_generator.py | 0 .../configuration/shim/test_shim_contract.py | 105 ++++ .../test/configuration/shim}/test_sources.py | 0 .../test/configuration/test_configuration.py | 487 ++++++++++++++++++ libs/xtm-oaev-sdk/test/test_deprecation.py | 152 ++++++ libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 19 + .../xtm_oaev_sdk/_core/__init__.py | 6 + .../_core/configuration/__init__.py | 44 ++ .../_core/configuration/configuration.py | 258 ++++++++++ .../connector_config_schema_generator.py | 176 +++++++ .../_core/configuration/settings_loader.py | 212 ++++++++ .../_core/configuration/sources.py | 77 +++ .../xtm_oaev_sdk/public/__init__.py | 49 ++ libs/xtm-oaev-sdk/xtm_oaev_sdk/py.typed | 0 .../xtm_oaev_sdk/seam/__init__.py | 21 + .../seam/configuration/__init__.py | 40 ++ .../xtm_oaev_sdk/seam/deprecation.py | 102 ++++ pyoaev/__init__.py | 19 +- pyoaev/configuration/__init__.py | 33 +- pyoaev/configuration/configuration.py | 238 +-------- .../connector_config_schema_generator.py | 137 +---- pyoaev/configuration/settings_loader.py | 169 +----- pyoaev/configuration/sources.py | 60 +-- pyoaev/daemons/base_daemon.py | 3 +- pyoaev/helpers.py | 2 +- pyproject.toml | 2 + 32 files changed, 1919 insertions(+), 531 deletions(-) rename test/configuration/__init__.py => libs/xtm-oaev-sdk/README.md (100%) create mode 100644 libs/xtm-oaev-sdk/pyproject.toml create mode 100644 libs/xtm-oaev-sdk/test/configuration/shim/__init__.py rename {test/configuration => libs/xtm-oaev-sdk/test/configuration/shim}/test_configuration.py (100%) rename {test/configuration => libs/xtm-oaev-sdk/test/configuration/shim}/test_connector_config_schema_generator.py (100%) create mode 100644 libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py rename {test/configuration => libs/xtm-oaev-sdk/test/configuration/shim}/test_sources.py (100%) create mode 100644 libs/xtm-oaev-sdk/test/configuration/test_configuration.py create mode 100644 libs/xtm-oaev-sdk/test/test_deprecation.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/configuration.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/connector_config_schema_generator.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_loader.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/py.typed create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 955b6c6b..aea511d9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ jobs: - checkout - run: name: install dependencies - command: pip3 install --user .[dev] + command: pip3 install --user ./libs/xtm-oaev-sdk .[dev] - run: name: confirm black version command: black --version @@ -33,7 +33,7 @@ jobs: - checkout - run: name: install dependencies - command: pip3 install --user '.[dev]' + command: pip3 install --user ./libs/xtm-oaev-sdk '.[dev]' - run: name: run tests command: python -m pytest diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index d2944d28..1cad7be8 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | - uv venv && uv pip install -e '.[dev]' coverage + uv venv && uv pip install -e ./libs/xtm-oaev-sdk -e '.[dev]' coverage - name: Run tests with coverage run: uv run coverage run -m pytest --junitxml="junit.xml" @@ -34,7 +34,7 @@ jobs: - name: Generate coverage XML run: uv run coverage xml -o coverage.xml - + - name: Upload coverage to Codecov if: ${{ !cancelled() && hashFiles('coverage.xml') != '' }} uses: codecov/codecov-action@v7 diff --git a/test/configuration/__init__.py b/libs/xtm-oaev-sdk/README.md similarity index 100% rename from test/configuration/__init__.py rename to libs/xtm-oaev-sdk/README.md diff --git a/libs/xtm-oaev-sdk/pyproject.toml b/libs/xtm-oaev-sdk/pyproject.toml new file mode 100644 index 00000000..0deaf771 --- /dev/null +++ b/libs/xtm-oaev-sdk/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "xtm-oaev-sdk" +version = "0.1.0" +description = "OpenAEV SDK core (DDD + Light Hex Architecture): configuration, contracts, signatures, and shared utilities." +authors = [ + {name = "Filigran", email = "contact@filigran.io"} +] +license = { text = "Apache-2.0" } +requires-python = ">=3.11" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Libraries :: Python Modules" +] +dependencies = [ + "pydantic (>=2.13.3,<2.14.0)", + "pydantic-settings (>=2.14.0,<2.15.0)", + "PyYAML (>=6.0,<6.1)", +] + +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +include = ["xtm_oaev_sdk*"] + +[tool.setuptools.package-data] +xtm_oaev_sdk = ["py.typed"] diff --git a/libs/xtm-oaev-sdk/test/configuration/shim/__init__.py b/libs/xtm-oaev-sdk/test/configuration/shim/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/configuration/test_configuration.py b/libs/xtm-oaev-sdk/test/configuration/shim/test_configuration.py similarity index 100% rename from test/configuration/test_configuration.py rename to libs/xtm-oaev-sdk/test/configuration/shim/test_configuration.py diff --git a/test/configuration/test_connector_config_schema_generator.py b/libs/xtm-oaev-sdk/test/configuration/shim/test_connector_config_schema_generator.py similarity index 100% rename from test/configuration/test_connector_config_schema_generator.py rename to libs/xtm-oaev-sdk/test/configuration/shim/test_connector_config_schema_generator.py diff --git a/libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py new file mode 100644 index 00000000..19d188e2 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py @@ -0,0 +1,105 @@ +"""Contract of the pyoaev configuration shims. + +Legacy imports must keep working, return the *identical* xtm_oaev_sdk object, +and warn; importing pyoaev itself and using the SDK directly must stay silent. +""" + +import subprocess +import sys +import warnings + +import pytest +import xtm_oaev_sdk as sdk + +LEGACY_NAMES = [ + ("pyoaev", "BaseConfigModel"), + ("pyoaev", "ConfigLoaderCollector"), + ("pyoaev", "ConfigLoaderOAEV"), + ("pyoaev", "Configuration"), + ("pyoaev", "SettingsLoader"), + ("pyoaev.configuration", "BaseConfigModel"), + ("pyoaev.configuration", "ConfigLoaderCollector"), + ("pyoaev.configuration", "ConfigLoaderOAEV"), + ("pyoaev.configuration", "Configuration"), + ("pyoaev.configuration", "SettingsLoader"), + ("pyoaev.configuration.configuration", "CONFIGURATION_TYPES"), + ("pyoaev.configuration.configuration", "Configuration"), + ("pyoaev.configuration.configuration", "ConfigurationHint"), + ("pyoaev.configuration.configuration", "ConnectorConfigSchemaGenerator"), + ("pyoaev.configuration.configuration", "DictionarySource"), + ("pyoaev.configuration.configuration", "EnvironmentSource"), + ("pyoaev.configuration.configuration", "is_falsy"), + ("pyoaev.configuration.configuration", "is_truthy"), + ("pyoaev.configuration.settings_loader", "BaseConfigModel"), + ("pyoaev.configuration.settings_loader", "ConfigLoaderCollector"), + ("pyoaev.configuration.settings_loader", "ConfigLoaderOAEV"), + ("pyoaev.configuration.settings_loader", "HttpUrlToString"), + ("pyoaev.configuration.settings_loader", "LogLevelToLower"), + ("pyoaev.configuration.settings_loader", "SettingsLoader"), + ("pyoaev.configuration.settings_loader", "TimedeltaInSeconds"), + ("pyoaev.configuration.sources", "DictionarySource"), + ("pyoaev.configuration.sources", "EnvironmentSource"), + ( + "pyoaev.configuration.connector_config_schema_generator", + "ConnectorConfigSchemaGenerator", + ), +] + + +def _given_fresh_interpreter_script(script: str) -> list[str]: + return [sys.executable, "-W", "error::DeprecationWarning", "-c", script] + + +def _when_run(command: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(command, capture_output=True, text=True) + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(f"from {module} import {name} as _legacy", namespace) + return namespace["_legacy"], caught + + +def _then_succeeds(result: subprocess.CompletedProcess) -> None: + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize(("module", "name"), LEGACY_NAMES) +def test_legacy_import_warns_and_returns_the_sdk_object(module, name) -> None: + """It warns towards xtm_oaev_sdk and returns the identical object.""" + legacy, caught = _when_import_legacy_name(module, name) + assert legacy is getattr(sdk, name) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert deprecations + assert f"from xtm_oaev_sdk import {name}" in str(deprecations[0].message) + + +def test_importing_pyoaev_is_silent() -> None: + """It imports pyoaev without any DeprecationWarning.""" + result = _when_run(_given_fresh_interpreter_script("import pyoaev")) + _then_succeeds(result) + + +def test_sdk_classes_stay_silent_once_pyoaev_is_loaded() -> None: + """It never alters SDK objects: native use stays silent next to pyoaev.""" + script = ( + "import pyoaev, pyoaev.configuration.configuration\n" + "import xtm_oaev_sdk as sdk\n" + "sdk.Configuration(config_hints={})\n" + "sdk.EnvironmentSource()\n" + "class _C(sdk.BaseConfigModel):\n" + " pass\n" + "_C()\n" + ) + result = _when_run(_given_fresh_interpreter_script(script)) + _then_succeeds(result) + + +def test_unknown_legacy_name_still_raises() -> None: + """It does not swallow genuine misses.""" + import pyoaev.configuration + + with pytest.raises(AttributeError): + pyoaev.configuration.DoesNotExist diff --git a/test/configuration/test_sources.py b/libs/xtm-oaev-sdk/test/configuration/shim/test_sources.py similarity index 100% rename from test/configuration/test_sources.py rename to libs/xtm-oaev-sdk/test/configuration/shim/test_sources.py diff --git a/libs/xtm-oaev-sdk/test/configuration/test_configuration.py b/libs/xtm-oaev-sdk/test/configuration/test_configuration.py new file mode 100644 index 00000000..34c96a00 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/configuration/test_configuration.py @@ -0,0 +1,487 @@ +"""Tests for configuration public surface.""" + +from pathlib import Path +from typing import Any + +import pytest +import xtm_oaev_sdk as sdk +from pydantic import ValidationError + + +def _given_public_module() -> Any: + return sdk + + +def _given_required_symbol(module: Any, symbol_name: str) -> Any: + assert hasattr(module, symbol_name), f"Missing public symbol: {symbol_name}" + return getattr(module, symbol_name) + + +def _given_missing_config_file_path(tmp_path: Path) -> str: + return str(tmp_path / "missing-config.yml") + + +def _given_configuration( + module: Any, + config_hints: dict[str, dict[str, Any] | str], + config_values: dict[str, Any] | None, + config_file_path: str, +) -> Any: + configuration = _given_required_symbol(module, "Configuration") + return configuration( + config_hints=config_hints, + config_values=config_values, + config_file_path=config_file_path, + ) + + +def _given_config_hints_for_priority() -> dict[str, dict[str, Any]]: + return { + "target": { + "env": "XTM_OAEV_TARGET", + "file_path": ["section", "target"], + "default": "default-value", + } + } + + +def _given_config_values_for_priority() -> dict[str, dict[str, str]]: + return {"section": {"target": "file-value"}} + + +def _given_set_env( + monkeypatch: pytest.MonkeyPatch, key: str, value: str | None +) -> None: + if value is None: + monkeypatch.delenv(key, raising=False) + else: + monkeypatch.setenv(key, value) + + +def _when_set_configuration_value(configuration: Any, key: str, value: Any) -> None: + configuration.set(key, value) + + +def _when_get_configuration_value(configuration: Any, key: str) -> Any: + return configuration.get(key) + + +def _when_get_environment_value(environment_source: Any, env_key: str | None) -> Any: + return environment_source.get(env_key) + + +def _when_get_dictionary_value( + dictionary_source: Any, path: list[str] | None, source_dict: dict[str, Any] +) -> Any: + return dictionary_source.get(path, source_dict) + + +def _when_assign_attribute(target: Any, attribute: str, value: Any) -> None: + setattr(target, attribute, value) + + +def _when_create_base_model_with_extra(base_config_model: Any) -> Any: + class _Model(base_config_model): + required: str + + return _Model(required="ok", extra_value="extra") + + +def _when_create_base_model_for_frozen_check(base_config_model: Any) -> Any: + class _Model(base_config_model): + value: int + + return _Model(value=1) + + +def _when_get_private_hint(configuration: Any, key: str) -> Any: + return configuration._Configuration__config_hints[key] # noqa: SLF001 + + +def _then_value_equals(actual: Any, expected: Any) -> None: + assert actual == expected + + +def _then_value_is_none(actual: Any) -> None: + assert actual is None + + +def _then_public_symbols_exist(module: Any, symbols: list[str]) -> None: + for symbol in symbols: + assert hasattr(module, symbol), f"Missing public symbol: {symbol}" + + +def _then_protocol_runtime_checkable(protocol: Any) -> None: + assert getattr(protocol, "_is_runtime_protocol", False) is True + + +def _then_instance_satisfies_protocol(instance: Any, protocol: Any) -> None: + assert isinstance(instance, protocol) + + +def _then_raises_validation_error(callable_obj: Any, *args: Any, **kwargs: Any) -> None: + with pytest.raises(ValidationError): + callable_obj(*args, **kwargs) + + +def _then_raises_assertion_error(callable_obj: Any, *args: Any, **kwargs: Any) -> None: + with pytest.raises(AssertionError): + callable_obj(*args, **kwargs) + + +def _then_hint_fields_match(hint: Any, expected: dict[str, Any]) -> None: + assert hint.data == expected["data"] + assert hint.env == expected["env"] + assert hint.file_path == expected["file_path"] + assert hint.is_number == expected["is_number"] + assert hint.default == expected["default"] + + +def _then_hint_is_instance_of(hint: Any, hint_class: Any) -> None: + assert isinstance(hint, hint_class) + + +def _then_model_allows_extra_fields( + model_instance: Any, key: str, expected: Any +) -> None: + assert model_instance.model_dump()[key] == expected + + +def _then_model_fields_include(model_class: Any, expected_fields: set[str]) -> None: + assert expected_fields.issubset(set(model_class.model_fields.keys())) + + +def test_public_surface_exports_configuration_symbols() -> None: + """It exposes all expected configuration-related public symbols.""" + module = _given_public_module() + _then_public_symbols_exist( + module, + [ + "ConfigurationProtocol", + "Configuration", + "ConfigurationHint", + "EnvironmentSource", + "DictionarySource", + "BaseConfigModel", + "ConfigLoaderOAEV", + "ConfigLoaderCollector", + "CONFIGURATION_TYPES", + ], + ) + + +def test_configuration_resolves_with_override_priority( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It prioritizes override over env, file, and default.""" + module = _given_public_module() + hints = _given_config_hints_for_priority() + config_values = _given_config_values_for_priority() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, hints, config_values, missing_path) + _given_set_env(monkeypatch, "XTM_OAEV_TARGET", "env-value") + _when_set_configuration_value(configuration, "target", "override-value") + result = _when_get_configuration_value(configuration, "target") + _then_value_equals(result, "override-value") + + +def test_configuration_get_prefers_env_when_no_override( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It reads environment value when no override is set.""" + module = _given_public_module() + hints = _given_config_hints_for_priority() + config_values = _given_config_values_for_priority() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, hints, config_values, missing_path) + _given_set_env(monkeypatch, "XTM_OAEV_TARGET", "env-value") + result = _when_get_configuration_value(configuration, "target") + _then_value_equals(result, "env-value") + + +def test_configuration_get_prefers_file_when_env_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It reads file-backed value when env is absent.""" + module = _given_public_module() + hints = _given_config_hints_for_priority() + config_values = _given_config_values_for_priority() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, hints, config_values, missing_path) + _given_set_env(monkeypatch, "XTM_OAEV_TARGET", None) + result = _when_get_configuration_value(configuration, "target") + _then_value_equals(result, "file-value") + + +def test_configuration_get_falls_back_to_default( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """It uses default when override, env, and file are unavailable.""" + module = _given_public_module() + hints = _given_config_hints_for_priority() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, hints, {}, missing_path) + _given_set_env(monkeypatch, "XTM_OAEV_TARGET", None) + result = _when_get_configuration_value(configuration, "target") + _then_value_equals(result, "default-value") + + +def test_configuration_get_returns_none_for_unknown_key(tmp_path: Path) -> None: + """It returns None for an unknown configuration key.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, {"known": {"default": "x"}}, {}, missing_path + ) + result = _when_get_configuration_value(configuration, "unknown") + _then_value_is_none(result) + + +def test_configuration_set_creates_new_hint_for_new_key(tmp_path: Path) -> None: + """It creates a new ConfigurationHint when setting a new key.""" + module = _given_public_module() + hint_class = _given_required_symbol(module, "ConfigurationHint") + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, {}, {}, missing_path) + _when_set_configuration_value(configuration, "new_key", "new-value") + hint = _when_get_private_hint(configuration, "new_key") + _then_hint_is_instance_of(hint, hint_class) + _then_value_equals(hint.data, "new-value") + + +def test_configuration_set_overrides_existing_hint_and_keeps_its_sources( + tmp_path: Path, +) -> None: + """It overrides an existing key while keeping the hint's other fields.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, + {"target": {"env": "XTM_OAEV_TARGET", "default": "old"}}, + {}, + missing_path, + ) + _when_set_configuration_value(configuration, "target", "new") + result = _when_get_configuration_value(configuration, "target") + hint = _when_get_private_hint(configuration, "target") + _then_value_equals(result, "new") + _then_value_equals(hint.env, "XTM_OAEV_TARGET") + _then_value_equals(hint.default, "old") + + +def test_configuration_coerces_number_when_is_number_true(tmp_path: Path) -> None: + """It coerces values to int when is_number hint is true.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, + {"count": {"default": "7", "is_number": True}}, + {}, + missing_path, + ) + result = _when_get_configuration_value(configuration, "count") + _then_value_equals(result, 7) + + +def test_configuration_coerces_truthy_strings_to_true(tmp_path: Path) -> None: + """It coerces yes/true strings to True case-insensitively.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, + {"enabled": {"default": "YeS"}}, + {}, + missing_path, + ) + result = _when_get_configuration_value(configuration, "enabled") + _then_value_equals(result, True) + + +def test_configuration_coerces_falsy_strings_to_false(tmp_path: Path) -> None: + """It coerces no/false strings to False case-insensitively.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, + {"enabled": {"default": "FaLsE"}}, + {}, + missing_path, + ) + result = _when_get_configuration_value(configuration, "enabled") + _then_value_equals(result, False) + + +def test_configuration_coerces_empty_string_to_none(tmp_path: Path) -> None: + """It coerces empty strings to None.""" + module = _given_public_module() + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration( + module, + {"value": {"default": ""}}, + {}, + missing_path, + ) + result = _when_get_configuration_value(configuration, "value") + _then_value_is_none(result) + + +def test_configuration_hint_is_mutable() -> None: + """It accepts direct field assignment: ``Configuration.set()`` mutates + the hint in place, so the hint is a plain mutable model.""" + module = _given_public_module() + configuration_hint = _given_required_symbol(module, "ConfigurationHint") + hint = configuration_hint(data="a") + _when_assign_attribute(hint, "data", "b") + assert hint.data == "b" + + +def test_configuration_hint_stores_all_fields() -> None: + """It stores data, env, file_path, is_number, and default fields.""" + module = _given_public_module() + configuration_hint = _given_required_symbol(module, "ConfigurationHint") + hint = configuration_hint( + data="x", + env="XTM_OAEV_ENV", + file_path=["section", "key"], + is_number=True, + default="fallback", + ) + _then_hint_fields_match( + hint, + { + "data": "x", + "env": "XTM_OAEV_ENV", + "file_path": ["section", "key"], + "is_number": True, + "default": "fallback", + }, + ) + + +def test_configuration_protocol_is_runtime_checkable_and_implemented( + tmp_path: Path, +) -> None: + """It is runtime-checkable and Configuration satisfies it.""" + module = _given_public_module() + protocol = _given_required_symbol(module, "ConfigurationProtocol") + missing_path = _given_missing_config_file_path(tmp_path) + configuration = _given_configuration(module, {}, {}, missing_path) + _then_protocol_runtime_checkable(protocol) + _then_instance_satisfies_protocol(configuration, protocol) + + +def test_environment_source_get_returns_none_for_none_key() -> None: + """It returns None when env var key is None.""" + module = _given_public_module() + environment_source = _given_required_symbol(module, "EnvironmentSource") + result = _when_get_environment_value(environment_source, None) + _then_value_is_none(result) + + +def test_environment_source_get_returns_none_for_empty_key() -> None: + """It returns None when env var key is empty.""" + module = _given_public_module() + environment_source = _given_required_symbol(module, "EnvironmentSource") + result = _when_get_environment_value(environment_source, "") + _then_value_is_none(result) + + +def test_environment_source_get_returns_existing_value( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """It returns environment value when variable exists.""" + module = _given_public_module() + environment_source = _given_required_symbol(module, "EnvironmentSource") + _given_set_env(monkeypatch, "EXISTING_VAR", "value-from-env") + result = _when_get_environment_value(environment_source, "EXISTING_VAR") + _then_value_equals(result, "value-from-env") + + +def test_environment_source_get_returns_none_for_missing_var( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """It returns None when environment variable does not exist.""" + module = _given_public_module() + environment_source = _given_required_symbol(module, "EnvironmentSource") + _given_set_env(monkeypatch, "NONEXISTENT_VAR", None) + result = _when_get_environment_value(environment_source, "NONEXISTENT_VAR") + _then_value_is_none(result) + + +def test_dictionary_source_get_returns_none_for_none_path() -> None: + """It returns None when dictionary path is None.""" + module = _given_public_module() + dictionary_source = _given_required_symbol(module, "DictionarySource") + result = _when_get_dictionary_value( + dictionary_source, None, {"section": {"key": "value"}} + ) + _then_value_is_none(result) + + +def test_dictionary_source_get_returns_nested_value() -> None: + """It returns value for a valid two-level path.""" + module = _given_public_module() + dictionary_source = _given_required_symbol(module, "DictionarySource") + result = _when_get_dictionary_value( + dictionary_source, + ["section", "key"], + {"section": {"key": "val"}}, + ) + _then_value_equals(result, "val") + + +def test_dictionary_source_get_returns_none_for_missing_nested_key() -> None: + """It returns None when nested key is missing.""" + module = _given_public_module() + dictionary_source = _given_required_symbol(module, "DictionarySource") + result = _when_get_dictionary_value( + dictionary_source, + ["section", "missing"], + {"section": {"key": "val"}}, + ) + _then_value_is_none(result) + + +def test_dictionary_source_get_raises_for_invalid_path_length() -> None: + """It raises AssertionError when path is not exactly two elements.""" + module = _given_public_module() + dictionary_source = _given_required_symbol(module, "DictionarySource") + _then_raises_assertion_error( + _when_get_dictionary_value, + dictionary_source, + ["section"], + {"section": {"key": "val"}}, + ) + + +def test_base_config_model_is_frozen() -> None: + """It prevents assignment on BaseConfigModel subclasses.""" + module = _given_public_module() + base_config_model = _given_required_symbol(module, "BaseConfigModel") + instance = _when_create_base_model_for_frozen_check(base_config_model) + _then_raises_validation_error(_when_assign_attribute, instance, "value", 2) + + +def test_base_config_model_allows_extra_fields() -> None: + """It accepts extra fields on BaseConfigModel subclasses.""" + module = _given_public_module() + base_config_model = _given_required_symbol(module, "BaseConfigModel") + instance = _when_create_base_model_with_extra(base_config_model) + _then_model_allows_extra_fields(instance, "extra_value", "extra") + + +def test_config_loader_oaev_has_expected_fields() -> None: + """It defines url, token, and tenant_id fields.""" + module = _given_public_module() + config_loader_oaev = _given_required_symbol(module, "ConfigLoaderOAEV") + _then_model_fields_include(config_loader_oaev, {"url", "token", "tenant_id"}) + + +def test_config_loader_collector_has_expected_fields() -> None: + """It defines id, name, log_level, period, and icon_filepath fields.""" + module = _given_public_module() + config_loader_collector = _given_required_symbol(module, "ConfigLoaderCollector") + _then_model_fields_include( + config_loader_collector, + {"id", "name", "log_level", "period", "icon_filepath"}, + ) diff --git a/libs/xtm-oaev-sdk/test/test_deprecation.py b/libs/xtm-oaev-sdk/test/test_deprecation.py new file mode 100644 index 00000000..4216acd3 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/test_deprecation.py @@ -0,0 +1,152 @@ +"""Tests for the reusable PEP 562 module-deprecation factory. + +Covers the exact pattern the legacy modules hand-roll: a ``{old: (msg, target)}`` +mapping driving a module ``__getattr__`` that warns and returns the *same* +object, preserves identity (no wrapper), and still raises on genuine misses. +""" + +import sys +import types +import warnings + +import pytest +from xtm_oaev_sdk import make_deprecated_getattr + + +# --- sentinel: a real object the deprecated name should alias, exactly --- +class _Sentinel: + pass + + +_TARGET = _Sentinel() + + +def _install_module(name, mapping, *, module_name=None): + """Create a bare module that uses the factory as its PEP 562 ``__getattr__``.""" + mod = types.ModuleType(name) + mod.__getattr__ = make_deprecated_getattr(mapping, module_name=module_name or name) + sys.modules[name] = mod + return mod + + +def _capture(fn): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = fn() + return result, caught + + +# --- 1. the core contract: warn once per access + return the exact object --- +def test_deprecated_access_warns_and_preserves_identity() -> None: + mod = _install_module("tdep_ident", {"Old": ("Use 'new' instead.", _TARGET)}) + try: + result, caught = _capture(lambda: mod.Old) + finally: + sys.modules.pop("tdep_ident", None) + + assert result is _TARGET, "the shim must return the exact object, not a wrapper" + assert len(caught) == 1 + assert isinstance(caught[0].message, DeprecationWarning) + assert str(caught[0].message) == "Use 'new' instead." + + +# --- 2. repeated access keeps returning the same object --- +def test_repeated_access_is_stable() -> None: + mod = _install_module("tdep_repeat", {"Old": ("msg", _TARGET)}) + try: + first = mod.Old + second = mod.Old + finally: + sys.modules.pop("tdep_repeat", None) + assert first is _TARGET + assert second is _TARGET + + +# --- 3. a real `from import ` triggers the shim --- +def test_from_import_triggers_deprecation() -> None: + _install_module("tdep_fromimp", {"Old": ("moved", _TARGET)}) + try: + namespace: dict = {} + result, caught = _capture( + lambda: exec("from tdep_fromimp import Old as _got", namespace) + ) + assert namespace["_got"] is _TARGET + assert any(isinstance(w.message, DeprecationWarning) for w in caught) + finally: + sys.modules.pop("tdep_fromimp", None) + + +# --- 4. a name that is NOT deprecated must still raise AttributeError --- +def test_unknown_name_raises_attribution_error() -> None: + mod = _install_module( + "tdep_miss", {"Old": ("moved", _TARGET)}, module_name="tdep_miss" + ) + try: + with pytest.raises(AttributeError) as exc: + _capture(lambda: mod.DoesNotExist) + assert "tdep_miss" in str(exc.value) + assert "DoesNotExist" in str(exc.value) + finally: + sys.modules.pop("tdep_miss", None) + + +# --- 5. a `from` import of an unknown name surfaces as ImportError --- +def test_from_import_of_missing_name_raises_import_error() -> None: + _install_module("tdep_miss2", {"Old": ("moved", _TARGET)}) + try: + with pytest.raises(ImportError): + _capture(lambda: exec("from tdep_miss2 import Nope", {})) + finally: + sys.modules.pop("tdep_miss2", None) + + +# --- 6. the factory validates its input up front (import-time failure) --- +def test_malformed_mapping_raises_value_error() -> None: + with pytest.raises(ValueError): + make_deprecated_getattr({123: ("msg", _TARGET)}) # non-string key + with pytest.raises(ValueError): + make_deprecated_getattr({"Old": "just-a-string"}) # not a 2-tuple + with pytest.raises(ValueError): + make_deprecated_getattr({"Old": ("only-a-message",)}) # 1-tuple + + +# --- 7. input mapping is copied; later mutation does not change the shim --- +def test_mapping_is_copied_not_referenced() -> None: + mapping = {"Old": ("msg", _TARGET)} + mod = _install_module("tdep_copy", mapping) + try: + mapping["Old"] = ("CHANGED", object()) # mutate the caller's mapping + result, _ = _capture(lambda: mod.Old) + assert result is _TARGET + assert str(_[0].message) == "msg" + finally: + sys.modules.pop("tdep_copy", None) + + +# --- 8. the primitive is a public, documented export --- +def test_public_export_and_all() -> None: + import xtm_oaev_sdk as sdk + from xtm_oaev_sdk.seam import make_deprecated_getattr as from_seam + + assert sdk.make_deprecated_getattr is make_deprecated_getattr + assert from_seam is make_deprecated_getattr + assert "make_deprecated_getattr" in sdk.__all__ + + +# --- 9. one factory per module; modules do not interfere with each other --- +def test_two_modules_independent() -> None: + a = _install_module("tdep_a", {"X": ("a-msg", "A")}) + b = _install_module("tdep_b", {"Y": ("b-msg", "B")}) + try: + result, _ = _capture(lambda: (a.X, b.Y)) + assert result == ("A", "B") + # cross-module name does not exist on the other + with pytest.raises(AttributeError): + a.Y + finally: + sys.modules.pop("tdep_a", None) + sys.modules.pop("tdep_b", None) + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(pytest.main([__file__, "-v"])) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py new file mode 100644 index 00000000..2c899e20 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -0,0 +1,19 @@ +"""xtm-oaev-sdk — OpenAEV SDK core (DDD + Light Hex Architecture). + +Architecture: + _core/configuration/ Configuration feature implementation + seam/configuration/ Configuration I/O surface + seam/deprecation.py PEP 562 deprecation primitive + public/ User-facing re-exports + +The seam is the compatibility boundary and the sole door between the public +surface and ``_core``: the public layer reaches ``_core`` exclusively through +the seam, and nothing in ``_core`` imports ``public``. +""" + +__version__ = "0.1.0" + +from xtm_oaev_sdk.public import * # noqa: F401, F403 +from xtm_oaev_sdk.public import __all__ as _public_all + +__all__ = list(_public_all) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/__init__.py new file mode 100644 index 00000000..8cff0516 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/__init__.py @@ -0,0 +1,6 @@ +"""xtm_oaev_sdk._core: private implementation boundary. + +Do NOT import from this module directly. Use xtm_oaev_sdk public surface. +""" + +__all__: list[str] = [] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py new file mode 100644 index 00000000..f5c2924d --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py @@ -0,0 +1,44 @@ +"""Configuration management: loading, hints, and schema generation. + +Re-exports the public configuration surface for internal engine use. +Public consumers import from xtm_oaev_sdk directly. +""" + +from .configuration import ( + CONFIGURATION_TYPES, + Configuration, + ConfigurationHint, + ConfigurationProtocol, + is_falsy, + is_truthy, +) +from .connector_config_schema_generator import ConnectorConfigSchemaGenerator +from .settings_loader import ( + BaseConfigModel, + ConfigLoaderCollector, + ConfigLoaderOAEV, + HttpUrlToString, + LogLevelToLower, + SettingsLoader, + TimedeltaInSeconds, +) +from .sources import DictionarySource, EnvironmentSource + +__all__ = [ + "BaseConfigModel", + "CONFIGURATION_TYPES", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", + "ConfigurationHint", + "ConfigurationProtocol", + "ConnectorConfigSchemaGenerator", + "DictionarySource", + "EnvironmentSource", + "HttpUrlToString", + "LogLevelToLower", + "SettingsLoader", + "TimedeltaInSeconds", + "is_falsy", + "is_truthy", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/configuration.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/configuration.py new file mode 100644 index 00000000..d29bc3da --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/configuration.py @@ -0,0 +1,258 @@ +"""Configuration management: hint-based value resolution. + +Provides the Configuration class that resolves values from multiple +sources (environment, files, overrides) using a hint-based lookup system. + +The Configuration class is the primary interface for SDK consumers to +access typed configuration values. + +Example: + >>> config = Configuration({ + ... "log_level": {"env": "LOG_LEVEL", "default": "info"}, + ... "api_url": {"env": "API_URL", "file_path": ["openaev", "url"]}, + ... }) + >>> config.get("log_level") + 'info' + >>> config.set("log_level", "debug") + >>> config.get("log_level") + 'debug' +""" + +import os +import os.path +from typing import Any, Protocol, runtime_checkable + +import yaml +from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings +from xtm_oaev_sdk._core.configuration.connector_config_schema_generator import ( + ConnectorConfigSchemaGenerator, +) +from xtm_oaev_sdk._core.configuration.sources import DictionarySource, EnvironmentSource + +__all__ = [ + "CONFIGURATION_TYPES", + "Configuration", + "ConfigurationHint", + "ConfigurationProtocol", + "is_falsy", + "is_truthy", +] + +CONFIGURATION_TYPES = str | int | bool | Any | None +"""Union type for configuration values.""" + + +def is_truthy(value: str) -> bool: + """Asserts whether a given string signals a "True" value. + + :param value: value to test + :type value: str + + :return: whether the string represents True or not. + :rtype: bool + """ + return value.lower() in ["yes", "true"] + + +def is_falsy(value: str) -> bool: + """Asserts whether a given string signals a "False" value. + + :param value: value to test + :type value: str + + :return: whether the string represents False or not. + :rtype: bool + """ + return value.lower() in ["no", "false"] + + +@runtime_checkable +class ConfigurationProtocol(Protocol): + """Behavioral contract for configuration services. + + Any configuration implementation used by SDK consumers must satisfy + this protocol. The built-in Configuration class implements it. + + Methods: + get: Retrieve a typed configuration value by key. + set: Override a configuration value at runtime. + schema: Generate the connector configuration JSON schema. + """ + + def get(self, config_key: str) -> CONFIGURATION_TYPES: + """Get configuration value by key.""" + ... + + def set(self, config_key: str, value: CONFIGURATION_TYPES) -> None: + """Set a runtime override for a configuration key.""" + ... + + def schema(self) -> dict[str, Any]: + """Generate the configuration JSON schema.""" + ... + + +class ConfigurationHint(BaseModel): + """An individual configuration hint defining where a value can be found. + + Hints specify the resolution strategy for a configuration key: + which environment variable to check, which file path to follow, + and what default to use if no source provides a value. + + The model is mutable: ``Configuration.set()`` assigns ``data`` in place. + + Args: + data: Override value. When set, this value is returned directly. + env: Environment variable name to read. + file_path: JSON path (nested keys) in the config file. + is_number: Whether to interpret the resolved value as an integer. + default: Fallback value when no source provides one. + + Example: + >>> hint = ConfigurationHint(env="LOG_LEVEL", default="info") + >>> hint.data = "debug" # override + """ + + data: CONFIGURATION_TYPES = Field(default=None) + env: str | None = Field(default=None) + file_path: list[str] | None = Field(default=None) + is_number: bool | None = Field(default=False) + default: CONFIGURATION_TYPES = Field(default=None) + + +class Configuration: + """Hint-based configuration resolver with multi-source lookup. + + Resolves configuration values by checking (in priority order): + 1. Explicit override via ``.set()`` + 2. Environment variable (if ``env`` hint provided) + 3. Config file value (if ``file_path`` hint provided) + 4. Default value (if ``default`` hint provided) + + Args: + config_hints: Mapping of config keys to hint definitions. + Values can be dicts (parsed as ConfigurationHint) or strings + (treated as default values). + config_values: Pre-loaded configuration dict (e.g., from YAML). + config_file_path: Path to YAML config file. Defaults to ./config.yml. + config_base_model: Optional Pydantic BaseSettings for schema generation. + + Example: + >>> config = Configuration({ + ... "url": {"env": "API_URL", "default": "http://localhost"}, + ... "timeout": {"env": "TIMEOUT", "is_number": True, "default": "30"}, + ... }) + >>> config.get("url") + 'http://localhost' + >>> config.get("timeout") + 30 + """ + + def __init__( + self, + config_hints: dict[str, dict[str, Any] | str], + config_values: dict[str, Any] | None = None, + config_file_path: str = os.path.join(os.curdir, "config.yml"), + config_base_model: BaseSettings | None = None, + ): + self.__config_hints: dict[str, ConfigurationHint] = { + key: ( + ConfigurationHint(**value) + if isinstance(value, dict) + else ConfigurationHint(default=value) + ) + for key, value in config_hints.items() + } + + file_contents = ( + yaml.load(open(config_file_path), Loader=yaml.FullLoader) + if os.path.isfile(config_file_path) + else {} + ) + + self.__config_values = (config_values or {}) | file_contents + self.__base_model = config_base_model + + def get(self, config_key: str) -> CONFIGURATION_TYPES: + """Get the resolved value for a configuration key. + + Resolution order: override → env var → file → default. + + Args: + config_key: The configuration key to look up. + + Returns: + The resolved value, or None if the key is unknown or + no source provides a value. + """ + config = self.__config_hints.get(config_key) + if config is None: + return None + + return self.__process_value_to_type( + ( + self.__dig_config_sources_for_key(config) + if config.data is None + else config.data + ), + config.is_number, + ) + + def set(self, config_key: str, value: CONFIGURATION_TYPES) -> None: + """Set a runtime override for a configuration key. + + After calling this, ``get(config_key)`` will return the + overridden value regardless of environment or file sources. + + Args: + config_key: The configuration key to override. + value: The new value to return for this key. + """ + if config_key not in self.__config_hints: + self.__config_hints[config_key] = ConfigurationHint(data=value) + else: + self.__config_hints[config_key].data = value + + def schema(self) -> dict[str, Any]: + """Generate the connector configuration JSON schema. + + Uses the configured base model's Pydantic schema generation + with the ConnectorConfigSchemaGenerator for flattened output. + + Returns: + A JSON schema dict describing all configuration properties. + + Raises: + AttributeError: If no config_base_model was provided. + """ + return self.__base_model.model_json_schema( # type: ignore[union-attr] + by_alias=False, + schema_generator=ConnectorConfigSchemaGenerator, + mode="validation", + ) + + @staticmethod + def __process_value_to_type( + value: CONFIGURATION_TYPES, is_number_hint: bool | None + ) -> CONFIGURATION_TYPES: + if value is None: + return value + if isinstance(value, int) or is_number_hint: + return int(value) + if isinstance(value, str): + if is_truthy(value): + return True + if is_falsy(value): + return False + if len(value) == 0: + return None + return value + + def __dig_config_sources_for_key( + self, config: ConfigurationHint + ) -> CONFIGURATION_TYPES: + result = EnvironmentSource.get(config.env) or DictionarySource.get( + config.file_path, self.__config_values + ) + return result or config.default diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/connector_config_schema_generator.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/connector_config_schema_generator.py new file mode 100644 index 00000000..fc250818 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/connector_config_schema_generator.py @@ -0,0 +1,176 @@ +"""Connector configuration JSON schema generator. + +Provides a custom Pydantic JSON schema generator that produces flattened, +dereferenced schemas suitable for connector configuration manifests. + +Adapted from the OpenCTI connectors configuration schema generator: +https://github.com/OpenCTI-Platform/connectors/blob/5c8cf1235f62f5651c9c08d0b67f1bd182662c8a/shared/tools/composer/generate_connectors_config_schemas/generate_connector_config_json_schema.py.sample +""" + +from copy import deepcopy +from typing import Any, override + +from pydantic.json_schema import GenerateJsonSchema + +__all__ = ["ConnectorConfigSchemaGenerator"] + +_FILTERED_ATTRIBUTES = [ + "CONNECTOR_ID", +] + + +class ConnectorConfigSchemaGenerator(GenerateJsonSchema): + """Custom JSON schema generator for connector configurations. + + Produces a flat schema where all config variables are at root level, + with ``$ref`` entries fully resolved and filtered attributes removed. + + This is used by ``Configuration.schema()`` to produce manifests + compatible with the OpenAEV platform connector registration API. + """ + + @staticmethod + def dereference_schema(schema_with_refs: dict[str, Any]) -> dict[str, Any]: + """Resolve all internal ``$ref`` entries in a JSON schema. + + Args: + schema_with_refs: A JSON schema dict potentially containing + ``$ref`` pointers to ``$defs``. + + Returns: + A new schema with all references inlined. + + Raises: + ValueError: If a ``$ref`` uses an unsupported format. + """ + + def _resolve(schema: Any, root: dict[str, Any]) -> Any: + if isinstance(schema, dict): + if "$ref" in schema: + ref_path = schema["$ref"] + if ref_path.startswith("#/$defs/"): + def_name = ref_path.split("/")[-1] + resolved = deepcopy(root["$defs"][def_name]) + return _resolve(resolved, root) + else: + raise ValueError(f"Unsupported ref format: {ref_path}") + else: + return { + schema_key: _resolve(schema_value, root) + for schema_key, schema_value in schema.items() + } + elif isinstance(schema, list): + return [_resolve(item, root) for item in schema] + else: + return schema + + return _resolve(deepcopy(schema_with_refs), schema_with_refs) + + @staticmethod + def flatten_config_loader_schema(root_schema: dict[str, Any]) -> dict[str, Any]: + """Flatten a hierarchical config schema to root-level properties. + + Transforms nested namespace properties into flat + ``NAMESPACE_VARIABLE`` format suitable for environment variable + mapping. + + Args: + root_schema: The dereferenced schema to flatten. + + Returns: + A flat schema with all properties at root level. + """ + flat_json_schema: dict[str, Any] = { + "$schema": root_schema["$schema"], + "$id": root_schema["$id"], + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": root_schema.get("additionalProperties", True), + } + + for ( + config_loader_namespace_name, + config_loader_namespace_schema, + ) in root_schema["properties"].items(): + config_schema = config_loader_namespace_schema.get("properties", {}) + required_config_vars = config_loader_namespace_schema.get("required", []) + + for config_var_name, config_var_schema in config_schema.items(): + property_name = ( + f"{config_loader_namespace_name.upper()}_{config_var_name.upper()}" + ) + + config_var_schema.pop("title", None) + + flat_json_schema["properties"][property_name] = config_var_schema + + if config_var_name in required_config_vars: + flat_json_schema["required"].append(property_name) + + return flat_json_schema + + @staticmethod + def filter_schema(schema: dict[str, Any]) -> dict[str, Any]: + """Remove filtered attributes from the schema. + + Args: + schema: The flat schema to filter. + + Returns: + The schema with filtered attributes removed. + """ + for filtered_attribute in _FILTERED_ATTRIBUTES: + if filtered_attribute in schema["properties"]: + del schema["properties"][filtered_attribute] + schema.update( + { + "required": [ + item + for item in schema["required"] + if item != filtered_attribute + ] + } + ) + + return schema + + @override + def generate(self, schema: Any, mode: str = "validation") -> dict[str, Any]: + """Generate a flat, dereferenced, filtered JSON schema. + + Args: + schema: The Pydantic core schema to process. + mode: Schema generation mode. + + Returns: + The final connector configuration schema. + """ + json_schema = super().generate(schema, mode=mode) + + json_schema["$schema"] = self.schema_dialect + json_schema["$id"] = "config.schema.json" + dereferenced_schema = self.dereference_schema(json_schema) + flattened_schema = self.flatten_config_loader_schema(dereferenced_schema) + return self.filter_schema(flattened_schema) + + @override + def nullable_schema(self, schema: Any) -> dict[str, Any]: + """Generate schema for nullable fields without ``anyOf``. + + Overrides the default behavior to produce cleaner schemas + that don't use the ``anyOf`` keyword for optional fields. + + Args: + schema: The core schema for the nullable type. + + Returns: + The generated JSON schema for the nullable type. + """ + null_schema = {"type": "null"} + inner_json_schema = self.generate_inner(schema["schema"]) + + if inner_json_schema == null_schema: + return null_schema + else: + return inner_json_schema diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_loader.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_loader.py new file mode 100644 index 00000000..31b260e8 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_loader.py @@ -0,0 +1,212 @@ +"""Settings loader: Pydantic-based configuration from env, YAML, and .env files. + +Provides base models and settings classes for structured, validated +configuration loading with prioritized source resolution. + +Source priority (highest to lowest): + 1. Environment variables + 2. YAML config file (config.yml) + 3. .env file + 4. Default values +""" + +import os +from abc import ABC +from datetime import timedelta +from pathlib import Path +from typing import Annotated, Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, PlainSerializer +from pydantic_settings import ( + BaseSettings, + DotEnvSettingsSource, + PydanticBaseSettingsSource, + SettingsConfigDict, + YamlConfigSettingsSource, +) + +__all__ = [ + "BaseConfigModel", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "HttpUrlToString", + "LogLevelToLower", + "SettingsLoader", + "TimedeltaInSeconds", +] + + +class BaseConfigModel(BaseModel, ABC): + """Abstract base for frozen configuration models. + + All configuration models inherit from this class to ensure + immutability after initialization and allow extra fields + for forward compatibility. + + Example: + >>> class MyConfig(BaseConfigModel): + ... url: str + ... timeout: int = 30 + >>> cfg = MyConfig(url="http://localhost") + >>> cfg.url = "other" # raises ValidationError (frozen) + """ + + model_config = ConfigDict(extra="allow", frozen=True, validate_default=True) + + +class SettingsLoader(BaseSettings): + """Base settings class with prioritized multi-source loading. + + Automatically discovers and loads configuration from environment + variables, YAML files, and .env files with a defined priority order. + + Subclass this to define your connector's configuration schema. + + Example: + >>> class MySettings(SettingsLoader): + ... model_config = SettingsConfigDict(yaml_file="config.yml") + ... api_url: str + ... timeout: int = 30 + """ + + model_config = SettingsConfigDict( + frozen=True, + extra="allow", + env_nested_delimiter="_", + env_nested_max_split=1, + enable_decoding=False, + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Configure source resolution order. + + Resolution priority: + 1. Environment variables (always checked first) + 2. YAML file (if config.yml exists) + 3. .env file (if .env exists and no YAML found) + 4. Default values (implicit via Pydantic) + + Args: + settings_cls: The settings class being configured. + init_settings: Init-based settings source. + env_settings: Environment variable source. + dotenv_settings: .env file source. + file_secret_settings: Secrets file source. + + Returns: + Tuple of settings sources in priority order. + """ + _main_path = os.curdir + + settings_cls.model_config["env_file"] = f"{_main_path}/../.env" + + if not settings_cls.model_config["yaml_file"]: + if Path(f"{_main_path}/config.yml").is_file(): + settings_cls.model_config["yaml_file"] = f"{_main_path}/config.yml" + if Path(f"{_main_path}/../config.yml").is_file(): + settings_cls.model_config["yaml_file"] = f"{_main_path}/../config.yml" + + if Path(settings_cls.model_config["yaml_file"] or "").is_file(): # type: ignore + return ( + env_settings, + YamlConfigSettingsSource(settings_cls), + ) + if Path(settings_cls.model_config["env_file"] or "").is_file(): # type: ignore + return ( + env_settings, + DotEnvSettingsSource(settings_cls), + ) + return (env_settings,) + + +# Annotated type aliases for common configuration field patterns + +LogLevelToLower = Annotated[ + Literal["debug", "info", "warn", "error"], + PlainSerializer(lambda v: "".join(v), return_type=str), +] +"""Log level literal type that serializes to lowercase string.""" + +HttpUrlToString = Annotated[HttpUrl, PlainSerializer(str, return_type=str)] +"""HTTP URL type that serializes to plain string.""" + +TimedeltaInSeconds = Annotated[ + timedelta, PlainSerializer(lambda v: int(v.total_seconds()), return_type=int) +] +"""Timedelta type that serializes to integer seconds.""" + + +class ConfigLoaderOAEV(BaseConfigModel): + """OpenAEV platform connection configuration. + + Args: + url: The OpenAEV platform URL. + token: Authentication token for the platform API. + tenant_id: Optional tenant identifier for multi-tenant deployments. + """ + + url: HttpUrlToString = Field( + description="The OpenAEV platform URL.", + ) + token: str = Field( + description="The token for the OpenAEV platform.", + ) + tenant_id: UUID | None = Field( + default=None, + description="Identifier of the tenant within the OpenAEV platform. " + "Used in multi-tenant environments to scope API requests and ensure data isolation between different tenants.", + ) + + +class ConfigLoaderCollector(BaseConfigModel): + """Base collector configuration settings. + + Common settings shared by all collectors including identification, + logging level, scheduling period, and UI customization. + + Args: + id: Unique identifier of the collector instance. + name: Human-readable collector name. + log_level: Logging verbosity (debug/info/warn/error). + period: Duration between scheduled runs (ISO 8601). + icon_filepath: Path to the collector's icon file. + """ + + id: str = Field(description="ID of the collector.") + name: str = Field(description="Name of the collector") + log_level: LogLevelToLower | None = Field( + default="error", + description="Determines the verbosity of the logs.", + ) + period: timedelta | None = Field( + default=timedelta(minutes=1), + description="Duration between two scheduled runs of the collector (ISO 8601 format).", + ) + icon_filepath: str | None = Field( + description="Path to the icon file of the collector.", + ) + author: str | None = Field( + default=None, + description="Optional author override for the connector's payloads and contracts. " + "When absent, the platform attributes them to the connector's name.", + ) + platform_description: str | None = Field( + default=None, + description="Description applied to the security platform auto-created by the " + "collector, so its read-only card is not empty in the platform UI.", + ) + platform_tags: list[str] | str | None = Field( + default=None, + description="Tag names applied to the security platform auto-created by the " + "collector. Accepts a list of names or a single comma-separated string " + "(the usual shape when configured through an environment variable).", + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py new file mode 100644 index 00000000..263215f4 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py @@ -0,0 +1,77 @@ +"""Configuration value sources: environment variables and dictionaries. + +Provides abstractions for resolving configuration values from different +backing stores without coupling the Configuration class to specific I/O. +""" + +import os +from typing import Any + +__all__ = ["DictionarySource", "EnvironmentSource"] + + +class EnvironmentSource: + """Resolves configuration values from environment variables. + + Example: + >>> EnvironmentSource.get("HOME") + '/home/user' + >>> EnvironmentSource.get(None) is None + True + """ + + @classmethod + def get(cls, env_var: str | None) -> str | None: + """Get the value of an environment variable. + + Args: + env_var: Name of the environment variable to read. + Returns None if env_var is None or empty. + + Returns: + The environment variable value, or None if not set. + """ + return os.getenv(env_var) if env_var else None + + +class DictionarySource: + """Resolves configuration values from nested dictionary structures. + + Handles two-level key paths into a JSON-like configuration dict, + typically loaded from a YAML or JSON config file. + + Example: + >>> source = {"openaev": {"url": "http://localhost:8080"}} + >>> DictionarySource.get(["openaev", "url"], source) + 'http://localhost:8080' + """ + + @classmethod + def get( + cls, config_key_path: list[str] | None, source_dict: dict[str, Any] + ) -> str | None: + """Get a value from a nested dict using a two-level key path. + + Args: + config_key_path: A list of exactly 2 non-empty strings representing + the path ``[section, key]`` into the source dict. + source_dict: The nested dictionary to search. + + Returns: + The value at the specified path, or None if not found. + + Raises: + AssertionError: If config_key_path is not a 2-element list of + non-empty strings. + """ + if config_key_path is None: + return None + + assert ( + isinstance(config_key_path, list) + and len(config_key_path) == 2 + and all([len(path_part) > 0 for path_part in config_key_path]) + ) + return source_dict.get(config_key_path[0], {config_key_path[1]: None}).get( + config_key_path[1] + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py new file mode 100644 index 00000000..ba499ef2 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -0,0 +1,49 @@ +"""Public API surface for xtm-oaev-sdk.""" + +__version__ = "0.1.0" + +# --- Configuration --- +from xtm_oaev_sdk.seam.configuration import ( + CONFIGURATION_TYPES, + BaseConfigModel, + ConfigLoaderCollector, + ConfigLoaderOAEV, + Configuration, + ConfigurationHint, + ConfigurationProtocol, + ConnectorConfigSchemaGenerator, + DictionarySource, + EnvironmentSource, + HttpUrlToString, + LogLevelToLower, + SettingsLoader, + TimedeltaInSeconds, + is_falsy, + is_truthy, +) + +# --- Seam / compatibility --- +from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr + +__all__ = [ + "__version__", + # Configuration + "BaseConfigModel", + "CONFIGURATION_TYPES", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", + "ConfigurationHint", + "ConfigurationProtocol", + "ConnectorConfigSchemaGenerator", + "DictionarySource", + "EnvironmentSource", + "HttpUrlToString", + "LogLevelToLower", + "SettingsLoader", + "TimedeltaInSeconds", + "is_falsy", + "is_truthy", + # Seam / compatibility + "make_deprecated_getattr", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/py.typed b/libs/xtm-oaev-sdk/xtm_oaev_sdk/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py new file mode 100644 index 00000000..9523baa1 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -0,0 +1,21 @@ +"""The architectural **seam** — the versioned compatibility boundary. + +Everything in this layer pins a stable interface so that the two sides of it +can evolve independently without leaking either: + +- **``configuration``** — the configuration capability's I/O models and + conversion primitives. +- **``deprecation``** — the PEP 562 deprecation primitive + (:func:`make_deprecated_getattr`) for retiring public names without + breaking importers. + +The package ``__init__`` surfaces present ``_core``'s stable vocabulary +through the boundary: they import ``_core`` so the public layer can. The seam +is the **sole door** between ``public/`` and ``_core/``: every public import +of a ``_core`` symbol crosses it. The seam never imports the public layer, +nor does ``_core``. +""" + +from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr + +__all__ = ["make_deprecated_getattr"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py new file mode 100644 index 00000000..f3934a9e --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py @@ -0,0 +1,40 @@ +"""Configuration feature contracts: the I/O surface plus the concrete +capability layer.""" + +from xtm_oaev_sdk._core.configuration import ( + CONFIGURATION_TYPES, + BaseConfigModel, + ConfigLoaderCollector, + ConfigLoaderOAEV, + Configuration, + ConfigurationHint, + ConfigurationProtocol, + ConnectorConfigSchemaGenerator, + DictionarySource, + EnvironmentSource, + HttpUrlToString, + LogLevelToLower, + SettingsLoader, + TimedeltaInSeconds, + is_falsy, + is_truthy, +) + +__all__ = [ + "BaseConfigModel", + "CONFIGURATION_TYPES", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", + "ConfigurationHint", + "ConfigurationProtocol", + "ConnectorConfigSchemaGenerator", + "DictionarySource", + "EnvironmentSource", + "HttpUrlToString", + "LogLevelToLower", + "SettingsLoader", + "TimedeltaInSeconds", + "is_falsy", + "is_truthy", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation.py new file mode 100644 index 00000000..625fd9f2 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation.py @@ -0,0 +1,102 @@ +"""PEP 562 module deprecation — a reusable compatibility primitive. + +This is the mechanism behind *"this name moved / was renamed / is going +away, here's the new home."* + +A module that wants to retire one or more names without breaking existing +consumers declares a mapping from each old name to +``(deprecation_message, replacement)`` and installs a module-level +``__getattr__`` built by :func:`make_deprecated_getattr`:: + + from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr + + _DEPRECATED: dict[str, tuple[str, object]] = { + "OldName": ("Use 'from xtm_oaev_sdk import NewName' instead.", NewName), + } + + def __getattr__(name: str): # PEP 562 + return make_deprecated_getattr(_DEPRECATED, module_name=__name__)(name) + +Why a factory, and why it returns the object directly: + +- **One primitive, many modules.** The ``__getattr__`` body is otherwise + copy-pasted across a dozen legacy modules. A factory turns each copy into + a data-driven mapping, so the warn / return / raise logic lives in a single + tested place and every module only supplies the names it retires. +- **Identity is preserved.** The factory returns *exactly* the object the new + name refers to — the same object, not a wrapper. No ``__new__`` / ``__call__`` + interception, no PEP 702 class. Wrappers deliberately mutate ``__new__`` and + add indirection that breaks ``isinstance`` checks, metaclass handling, and + signature introspection, so they are *not* used here. +- **Real misses still fail.** A name that is not in the mapping raises + ``AttributeError`` (naming the module), so a typo or a not-yet-retired name + is never silently swallowed by the deprecation layer. + +This module is a *pure compatibility primitive*: it imports nothing from the +SDK's capabilities, so the dependency direction of the seam holds — the seam +is imported by consumers, never the other way round. +""" + +from __future__ import annotations + +import warnings +from collections.abc import Mapping as _Mapping +from typing import Any + +__all__ = ["make_deprecated_getattr"] + + +def make_deprecated_getattr( + mapping: _Mapping[str, tuple[str, Any]], + *, + module_name: str | None = None, +): + """Build a PEP 562 module-level ``__getattr__`` for a set of deprecated names. + + Parameters + ---------- + mapping: + ``{old_name: (deprecation_message, replacement)}``. On first access to + ``old_name`` the ``deprecation_message`` is emitted as a + ``DeprecationWarning`` (``stacklevel=2``) and the ``replacement`` object + is returned. + module_name: + Optional dotted name used in the ``AttributeError`` for a name that is + *not* in ``mapping`` (e.g. ``"pyoaev.helpers"``). Defaults to + ``"this module"``. + + Returns + ------- + A zero-argument-per-call ``__getattr__(name)`` callable, suitable for + assignment as a module's ``__getattr__``. + + Notes + ----- + The input mapping is validated once and copied, so later mutation of the + caller's mapping does not affect the returned callable. A malformed entry + (non-string key, or a value that is not a two-item ``(message, target)``) + raises ``ValueError`` at call time — i.e. at import time of the module that + installs the shim — rather than at first access. + """ + resolved: dict[str, tuple[str, Any]] = {} + for key, value in mapping.items(): + if not isinstance(key, str) or not key: + raise ValueError( + f"deprecated-name mapping keys must be non-empty strings, got {key!r}" + ) + if not isinstance(value, tuple) or len(value) != 2: + raise ValueError( + f"mapping entry for {key!r} must be (message, target), got {value!r}" + ) + resolved[key] = (value[0], value[1]) + + def __getattr__(name: str) -> Any: + entry = resolved.get(name) + if entry is None: + where = module_name if module_name is not None else "this module" + raise AttributeError(f"{where} has no attribute {name!r}") from None + message, target = entry + warnings.warn(message, DeprecationWarning, stacklevel=2) + return target + + return __getattr__ diff --git a/pyoaev/__init__.py b/pyoaev/__init__.py index 049ebd03..14a7f808 100644 --- a/pyoaev/__init__.py +++ b/pyoaev/__init__.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- __version__ = "3.261014.1" +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +from pyoaev import configuration # noqa: F401 from pyoaev._version import ( # noqa: F401 __author__, __copyright__, @@ -15,7 +19,6 @@ CloudProvider, ) from pyoaev.client import OpenAEV # noqa: F401 -from pyoaev.configuration import * # noqa: F401,F403,F405 from pyoaev.contracts import * # noqa: F401,F403,F405 from pyoaev.exceptions import * # noqa: F401,F403,F405 from pyoaev.signatures import * # noqa: F401,F403,F405 @@ -34,3 +37,17 @@ "AssetCriticality", ] __all__.extend(exceptions.__all__) # noqa: F405 + +# Names moved to xtm_oaev_sdk: served lazily (PEP 562) so that `import pyoaev` +# stays silent and only legacy access to a moved name warns. +_deprecated_getattr = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in configuration.__all__ + }, + module_name=__name__, +) + + +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) diff --git a/pyoaev/configuration/__init__.py b/pyoaev/configuration/__init__.py index a4741f92..750e9c12 100644 --- a/pyoaev/configuration/__init__.py +++ b/pyoaev/configuration/__init__.py @@ -1,15 +1,28 @@ -from .configuration import Configuration -from .settings_loader import ( - BaseConfigModel, - ConfigLoaderCollector, - ConfigLoaderOAEV, - SettingsLoader, -) +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr __all__ = [ - "Configuration", - "ConfigLoaderOAEV", + "BaseConfigModel", "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", "SettingsLoader", - "BaseConfigModel", ] + +_deprecated_getattr = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) + + +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) diff --git a/pyoaev/configuration/configuration.py b/pyoaev/configuration/configuration.py index ae7af778..72d651bd 100644 --- a/pyoaev/configuration/configuration.py +++ b/pyoaev/configuration/configuration.py @@ -1,213 +1,31 @@ -import os -import os.path -from typing import Any, Dict, Optional - -import yaml -from pydantic import BaseModel, Field -from pydantic_settings import BaseSettings - -from pyoaev.configuration.connector_config_schema_generator import ( - ConnectorConfigSchemaGenerator, +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "CONFIGURATION_TYPES", + "Configuration", + "ConfigurationHint", + "ConnectorConfigSchemaGenerator", + "DictionarySource", + "EnvironmentSource", + "is_falsy", + "is_truthy", +] + +_deprecated_getattr = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, ) -from pyoaev.configuration.sources import DictionarySource, EnvironmentSource - -CONFIGURATION_TYPES = str | int | bool | Any | None - - -def is_truthy(value: str) -> bool: - """Asserts whether a given string signals a "True" value - - :param value: value to test - :type value: str - - :return: whether the string represents True or not. - :rtype: bool - """ - return value.lower() in ["yes", "true"] - - -def is_falsy(value: str) -> bool: - """Asserts whether a given string signals a "False" value - - :param value: value to test - :type value: str - - :return: whether the string represents False or not. - :rtype: bool - """ - return value.lower() in ["no", "false"] - - -class ConfigurationHint(BaseModel): - """An individual configuration hint. This allows for specifying - where any given configuration key can be found, in env vars, - config files. Additionally, it may define a default value or - a discrete override value. - """ - - data: Optional[CONFIGURATION_TYPES] = Field(default=None) - """Override value; when set, getting the configuration value for - the key described in this instance returns this value. - """ - env: Optional[str] = Field(default=None) - """Defines which env var should be read for getting the value - """ - file_path: Optional[list[str]] = Field(default=None) - """Defines a JSON path (nested keys) to follow in the provided - config file for reaching the value. - - Example: ["toplevel", "subkey"] will hint for searching for - the config key at { "toplevel": { "subkey": { "config_key"}} - """ - is_number: Optional[bool] = Field(default=False) - """Hints at whteher the configuration value should be - interpreted as a number. - """ - default: Optional[CONFIGURATION_TYPES] = Field(default=None) - """When defined, provides a default value for whenever none of the - hinted locations or the data field have a value. - """ - - -class Configuration: - """A configuration object providing ways to an interface for getting - configuration values. It should be provided with a collection of hints - to enable its behaviour. - - :param config_hints: a dictionary of hints, for which the key is the - desired configuration key (e.g. "log_level") and the value is either - a dictionary of hints (see ConfigurationHint) or a standalone string. - In the latter case, the string will be interpreted as a default value. - - Example: - .. code-block:: python - { - "my_config_key": { - "env" : "MY_CONFIG_VALUE_ENV_VAR", - "file_path": ["first_level", "second_level"] - }, - "my_other_config_key: "discrete value" - } - :type config_hints: Dict[str, dict | str] - :param config_values: dictionary of config values to preemptively load into the - Configuration object. The format of this dictionary should follow the patterns - chosen in the file_path property of ConfigurationHint object passed - as config_hints, defaults to None - - Example: - .. code-block:: python - { - "first_level": { - "second_level": { - "my_config_key": "some value" - } - } - } - :type config_values: dict (json), optional - :param config_file_path: path to the configuration file. The file should - contain a json structure that matches the format of the config_values param, - defaults to './config.yml' (relative path). - :type config_file_path: str - """ - - def __init__( - self, - config_hints: Dict[str, dict | str], - config_values: dict = None, - config_file_path: str = os.path.join(os.curdir, "config.yml"), - config_base_model: BaseSettings = None, - ): - self.__config_hints = { - key: ( - ConfigurationHint(**value) - if isinstance(value, dict) - else ConfigurationHint(**{"default": value}) - ) - for key, value in config_hints.items() - } - - file_contents = ( - yaml.load(open(config_file_path), Loader=yaml.FullLoader) - if os.path.isfile(config_file_path) - else {} - ) - - self.__config_values = (config_values or {}) | file_contents - - self.__base_model = config_base_model - - def get(self, config_key: str) -> CONFIGURATION_TYPES: - """Gets the value pointed to by the configuration key. If the key is defined - with actual hints (as opposed to a discrete value), it will use those hints to - potentially find a value. If the key was not defined as part of the supplied - config_hints, this will always return None. - - :param config_key: the configuration key to search a value for. - :type config_key: str - - :return: the value pointed to by the configuration key, or None if not found - :rtype: CONFIGURATION_TYPES - """ - config = self.__config_hints.get(config_key) - if config is None: - return None - - return self.__process_value_to_type( - ( - self.__dig_config_sources_for_key(config) - if config.data is None - else config.data - ), - config.is_number, - ) - - def set(self, config_key: str, value: CONFIGURATION_TYPES): - """Sets an arbitrary value in the Configuration object, for - the supplied configuration key, after which any request for the value - of that key will return this new value. - - :param config_key: the configuration key to set a value for. - :type config_key: str - :param value: the new value to set for the configuration key. - :type value: CONFIGURATION_TYPES - """ - if config_key not in self.__config_hints: - self.__config_hints[config_key] = ConfigurationHint(**{"data": value}) - else: - self.__config_hints[config_key].data = value - - def schema(self): - """ - Generates the complete connector schema using a custom schema generator compatible with Pydantic. - Isolate custom class generator, Pydantic expects a class, not an instance - Always subclass GenerateJsonSchema and pass the class to Pydantic, not an instance - :return: The generated connector schema as a dictionary. - """ - return self.__base_model.model_json_schema( - by_alias=False, - schema_generator=ConnectorConfigSchemaGenerator, - mode="validation", - ) - @staticmethod - def __process_value_to_type(value: CONFIGURATION_TYPES, is_number_hint: bool): - if value is None: - return value - if isinstance(value, int) or is_number_hint: - return int(value) - if isinstance(value, str): - if is_truthy(value): - return True - if is_falsy(value): - return False - if len(value) == 0: - return None - return value - def __dig_config_sources_for_key( - self, config: ConfigurationHint - ) -> CONFIGURATION_TYPES: - result = EnvironmentSource.get(config.env) or DictionarySource.get( - config.file_path, self.__config_values - ) - return result or config.default +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) diff --git a/pyoaev/configuration/connector_config_schema_generator.py b/pyoaev/configuration/connector_config_schema_generator.py index 3c8ac436..d6b045eb 100644 --- a/pyoaev/configuration/connector_config_schema_generator.py +++ b/pyoaev/configuration/connector_config_schema_generator.py @@ -1,127 +1,24 @@ -## ADAPTED FROM https://github.com/OpenCTI-Platform/connectors/blob/5c8cf1235f62f5651c9c08d0b67f1bd182662c8a/shared/tools/composer/generate_connectors_config_schemas/generate_connector_config_json_schema.py.sample +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -from copy import deepcopy -from typing import override +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -from pydantic.json_schema import GenerateJsonSchema +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr -# attributes filtered from the connector configuration before generating the manifest -__FILTERED_ATTRIBUTES__ = [ - # connector id is generated - "CONNECTOR_ID", +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ConnectorConfigSchemaGenerator", ] +_deprecated_getattr = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) -class ConnectorConfigSchemaGenerator(GenerateJsonSchema): - @staticmethod - def dereference_schema(schema_with_refs): - """Return a new schema with all internal $ref resolved.""" - def _resolve(schema, root): - if isinstance(schema, dict): - if "$ref" in schema: - ref_path = schema["$ref"] - if ref_path.startswith("#/$defs/"): - def_name = ref_path.split("/")[-1] - # Deep copy to avoid mutating $defs - resolved = deepcopy(root["$defs"][def_name]) - return _resolve(resolved, root) - else: - raise ValueError(f"Unsupported ref format: {ref_path}") - else: - return { - schema_key: _resolve(schema_value, root) - for schema_key, schema_value in schema.items() - } - elif isinstance(schema, list): - return [_resolve(item, root) for item in schema] - else: - return schema - - return _resolve(deepcopy(schema_with_refs), schema_with_refs) - - @staticmethod - def flatten_config_loader_schema(root_schema: dict): - """ - Flatten config loader schema so all config vars are described at root level. - - :param root_schema: Original schema. - :return: Flatten schema. - """ - flat_json_schema = { - "$schema": root_schema["$schema"], - "$id": root_schema["$id"], - "type": "object", - "properties": {}, - "required": [], - "additionalProperties": root_schema.get("additionalProperties", True), - } - - for ( - config_loader_namespace_name, - config_loader_namespace_schema, - ) in root_schema["properties"].items(): - config_schema = config_loader_namespace_schema.get("properties", {}) - required_config_vars = config_loader_namespace_schema.get("required", []) - - for config_var_name, config_var_schema in config_schema.items(): - property_name = ( - f"{config_loader_namespace_name.upper()}_{config_var_name.upper()}" - ) - - config_var_schema.pop("title", None) - - flat_json_schema["properties"][property_name] = config_var_schema - - if config_var_name in required_config_vars: - flat_json_schema["required"].append(property_name) - - return flat_json_schema - - @staticmethod - def filter_schema(schema): - for filtered_attribute in __FILTERED_ATTRIBUTES__: - if filtered_attribute in schema["properties"]: - del schema["properties"][filtered_attribute] - schema.update( - { - "required": [ - item - for item in schema["required"] - if item != filtered_attribute - ] - } - ) - - return schema - - @override - def generate(self, schema, mode="validation"): - json_schema = super().generate(schema, mode=mode) - - json_schema["$schema"] = self.schema_dialect - json_schema["$id"] = "config.schema.json" - dereferenced_schema = self.dereference_schema(json_schema) - flattened_schema = self.flatten_config_loader_schema(dereferenced_schema) - return self.filter_schema(flattened_schema) - - @override - def nullable_schema(self, schema): - """Generates a JSON schema that matches a schema that allows null values. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - - Notes: - This method overrides `GenerateJsonSchema.nullable_schema` to generate schemas without `anyOf` keyword. - """ - null_schema = {"type": "null"} - inner_json_schema = self.generate_inner(schema["schema"]) - - if inner_json_schema == null_schema: - return null_schema - else: - return inner_json_schema +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) diff --git a/pyoaev/configuration/settings_loader.py b/pyoaev/configuration/settings_loader.py index 908dfbfd..49ecea35 100644 --- a/pyoaev/configuration/settings_loader.py +++ b/pyoaev/configuration/settings_loader.py @@ -1,147 +1,30 @@ -import os -from abc import ABC -from datetime import timedelta -from pathlib import Path -from typing import Annotated, Literal -from uuid import UUID - -from pydantic import BaseModel, ConfigDict, Field, HttpUrl, PlainSerializer -from pydantic_settings import ( - BaseSettings, - DotEnvSettingsSource, - PydanticBaseSettingsSource, - SettingsConfigDict, - YamlConfigSettingsSource, -) - - -class BaseConfigModel(BaseModel, ABC): - """Base class for global config models - To prevent attributes from being modified after initialization. - """ - - model_config = ConfigDict(extra="allow", frozen=True, validate_default=True) - - -class SettingsLoader(BaseSettings): - model_config = SettingsConfigDict( - frozen=True, - extra="allow", - env_nested_delimiter="_", - env_nested_max_split=1, - enable_decoding=False, - ) - - @classmethod - def settings_customise_sources( - cls, - settings_cls: type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> tuple[PydanticBaseSettingsSource, ...]: - """Customise the sources of settings for the connector. - - This method is called by the Pydantic BaseSettings class to determine the order of sources. - The configuration come in this order either from: - 1. Environment variables - 2. YAML file - 3. .env file - 4. Default values - - The variables loading order will remain the same as in `pycti.get_config_variable()`: - 1. If a config.yml file is found, the order will be: `ENV VAR` → config.yml → default value - 2. If a .env file is found, the order will be: `ENV VAR` → .env → default value - """ - _main_path = os.curdir - - settings_cls.model_config["env_file"] = f"{_main_path}/../.env" - - if not settings_cls.model_config["yaml_file"]: - if Path(f"{_main_path}/config.yml").is_file(): - settings_cls.model_config["yaml_file"] = f"{_main_path}/config.yml" - if Path(f"{_main_path}/../config.yml").is_file(): - settings_cls.model_config["yaml_file"] = f"{_main_path}/../config.yml" - - if Path(settings_cls.model_config["yaml_file"] or "").is_file(): # type: ignore - return ( - env_settings, - YamlConfigSettingsSource(settings_cls), - ) - if Path(settings_cls.model_config["env_file"] or "").is_file(): # type: ignore - return ( - env_settings, - DotEnvSettingsSource(settings_cls), - ) - return (env_settings,) - - -LogLevelToLower = Annotated[ - Literal["debug", "info", "warn", "error"], - PlainSerializer(lambda v: "".join(v), return_type=str), -] - -HttpUrlToString = Annotated[HttpUrl, PlainSerializer(str, return_type=str)] -TimedeltaInSeconds = Annotated[ - timedelta, PlainSerializer(lambda v: int(v.total_seconds()), return_type=int) +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "BaseConfigModel", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "HttpUrlToString", + "LogLevelToLower", + "SettingsLoader", + "TimedeltaInSeconds", ] +_deprecated_getattr = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) -class ConfigLoaderOAEV(BaseConfigModel): - """OpenAEV/OpenAEV platform configuration settings. - - Contains URL and authentication token for connecting to the OpenAEV platform. - """ - - url: HttpUrlToString = Field( - description="The OpenAEV platform URL.", - ) - token: str = Field( - description="The token for the OpenAEV platform.", - ) - tenant_id: UUID | None = Field( - default=None, - description="Identifier of the tenant within the OpenAEV platform. Used in multi-tenant environments to scope " - "API requests and ensure data isolation between different tenants.", - ) - - -class ConfigLoaderCollector(BaseConfigModel): - """Base collector configuration settings. - - Contains common collector settings including identification, logging, - scheduling, and platform information. - """ - - id: str = Field(description="ID of the collector.") - - name: str = Field(description="Name of the collector") - log_level: LogLevelToLower | None = Field( - default="error", - description="Determines the verbosity of the logs.", - ) - period: timedelta | None = Field( - default=timedelta(minutes=1), - description="Duration between two scheduled runs of the collector (ISO 8601 format).", - ) - icon_filepath: str | None = Field( - description="Path to the icon file of the collector.", - ) - author: str | None = Field( - default=None, - description="Optional author override for the connector's payloads and contracts. " - "When absent, the platform attributes them to the connector's name.", - ) - platform_description: str | None = Field( - default=None, - description="Description applied to the security platform auto-created by the " - "collector, so its read-only card is not empty in the platform UI.", - ) - platform_tags: list[str] | str | None = Field( - default=None, - description="Tag names applied to the security platform auto-created by the " - "collector. Accepts a list of names or a single comma-separated string " - "(the usual shape when configured through an environment variable).", - ) +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) diff --git a/pyoaev/configuration/sources.py b/pyoaev/configuration/sources.py index aeb53850..10007a62 100644 --- a/pyoaev/configuration/sources.py +++ b/pyoaev/configuration/sources.py @@ -1,47 +1,25 @@ -import os +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -class EnvironmentSource: - """A utility for fecthing a value in the env vars.""" +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr - @classmethod - def get(cls, env_var: str) -> str | None: - """Gets the value for the specified env var +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "DictionarySource", + "EnvironmentSource", +] - :param env_var: the name of the env var to query - :type env_var: str +_deprecated_getattr = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) - :return: value of the env var, or None if not found - :rtype: str | None - """ - return os.getenv(env_var) if env_var else None - -class DictionarySource: - """A utility for fetching a value from within a JSON-like (nested dict) structure""" - - # this is quite hacky - # it only strictly handles two levels of keys in a dict - @classmethod - def get(cls, config_key_path: list[str] | None, source_dict: dict) -> str | None: - """Gets the value for the specified env var - - :param config_key_path: the two-level dictionary path to the config key - :type config_key_path: list[str] - :param source_dict: JSON-like (nested dict) structure containing config values. - :type source_dict: dict - - :return: value for the config key at specified path, or None if not found - :rtype: str | None - """ - if config_key_path is None: - return None - - assert ( - isinstance(config_key_path, list) - and len(config_key_path) == 2 - and all([len(path_part) > 0 for path_part in config_key_path]) - ) - return source_dict.get(config_key_path[0], {config_key_path[1]: None}).get( - config_key_path[1] - ) +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) diff --git a/pyoaev/daemons/base_daemon.py b/pyoaev/daemons/base_daemon.py index 9728a872..e2d5868f 100644 --- a/pyoaev/daemons/base_daemon.py +++ b/pyoaev/daemons/base_daemon.py @@ -4,8 +4,9 @@ from types import FunctionType from uuid import UUID +from xtm_oaev_sdk import Configuration + from pyoaev.client import OpenAEV -from pyoaev.configuration import Configuration from pyoaev.exceptions import OpenAEVError from pyoaev.utils import logger diff --git a/pyoaev/helpers.py b/pyoaev/helpers.py index 6867ba32..23b3cb34 100644 --- a/pyoaev/helpers.py +++ b/pyoaev/helpers.py @@ -13,9 +13,9 @@ import pika from thefuzz import fuzz +from xtm_oaev_sdk import Configuration from pyoaev import OpenAEV, utils -from pyoaev.configuration import Configuration from pyoaev.daemons import CollectorDaemon from pyoaev.exceptions import ConfigurationError diff --git a/pyproject.toml b/pyproject.toml index 5a7bb308..37be9a69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,8 @@ dependencies = [ "requests (>=2.33.0,<2.34.0)", "requests-toolbelt (>=1.0.0,<1.1.0)", "thefuzz (>=0.22,<0.23)", + # OpenAEV SDKs + "xtm-oaev-sdk (>=0.1.0,<0.2.0)", ] [project.optional-dependencies] From acbcbd69387ffa4d6045de7fbc2d9ed9bf2292fc Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 12:42:15 +0200 Subject: [PATCH 02/22] :sparkles: feat(sdk): migrate common foundation to xtm-oaev-sdk with PEP 562 pyoaev shims --- libs/xtm-oaev-sdk/pyproject.toml | 1 + libs/xtm-oaev-sdk/test/common/__init__.py | 0 .../xtm-oaev-sdk/test/common/shim/__init__.py | 0 .../test/common/shim}/test_exceptions.py | 0 .../test/common/shim/test_shim_contract.py | 146 +++++++ .../test/common/shim/test_utils.py | 141 ++++++ .../test/common/test_exceptions.py | 62 +++ .../test/common/test_ssl_utils.py | 92 ++++ libs/xtm-oaev-sdk/test/common/test_utils.py | 401 +++++++++++++++++ .../test/configuration/__init__.py | 0 libs/xtm-oaev-sdk/test/test_boundaries.py | 42 ++ libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 3 + .../xtm_oaev_sdk/_core/asset_types.py | 135 ++++++ .../xtm-oaev-sdk/xtm_oaev_sdk/_core/errors.py | 315 ++++++++++++++ .../_core/security_domain/__init__.py | 9 + .../_core/security_domain/types.py | 45 ++ .../xtm_oaev_sdk/_core/ssl_utils.py | 79 ++++ libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/utils.py | 404 ++++++++++++++++++ .../xtm_oaev_sdk/public/__init__.py | 74 ++++ .../xtm_oaev_sdk/seam/__init__.py | 2 + .../xtm_oaev_sdk/seam/common/__init__.py | 85 ++++ pyoaev/__init__.py | 15 +- pyoaev/apis/ai_target.py | 61 +-- pyoaev/apis/attack_pattern.py | 3 +- pyoaev/apis/collector.py | 5 +- pyoaev/apis/cve.py | 3 +- pyoaev/apis/document.py | 3 +- pyoaev/apis/endpoint.py | 5 +- pyoaev/apis/inject.py | 3 +- .../inject_expectation/inject_expectation.py | 5 +- pyoaev/apis/inject_expectation_trace.py | 5 +- pyoaev/apis/injector.py | 3 +- pyoaev/apis/injector_contract.py | 5 +- pyoaev/apis/kill_chain_phase.py | 3 +- pyoaev/apis/payload.py | 3 +- pyoaev/apis/security_platform.py | 5 +- pyoaev/apis/signature.py | 6 +- pyoaev/apis/tag.py | 3 +- pyoaev/apis/team.py | 5 +- pyoaev/apis/user.py | 5 +- pyoaev/apis/vulnerability.py | 3 +- pyoaev/asset_types.py | 164 ++----- pyoaev/base.py | 9 +- pyoaev/client.py | 10 +- pyoaev/contracts/contract_config.py | 9 +- pyoaev/daemons/base_daemon.py | 4 +- pyoaev/daemons/collector_daemon.py | 3 +- pyoaev/exceptions.py | 240 ++--------- pyoaev/helpers.py | 85 ++-- pyoaev/mixins.py | 12 +- pyoaev/security_domain/types.py | 42 +- pyoaev/signatures/signature_manager.py | 2 +- pyoaev/signatures/signature_match.py | 3 +- pyoaev/utils.py | 184 ++------ test/test_utils.py | 150 ------- 55 files changed, 2289 insertions(+), 813 deletions(-) create mode 100644 libs/xtm-oaev-sdk/test/common/__init__.py create mode 100644 libs/xtm-oaev-sdk/test/common/shim/__init__.py rename {test => libs/xtm-oaev-sdk/test/common/shim}/test_exceptions.py (100%) create mode 100644 libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py create mode 100644 libs/xtm-oaev-sdk/test/common/shim/test_utils.py create mode 100644 libs/xtm-oaev-sdk/test/common/test_exceptions.py create mode 100644 libs/xtm-oaev-sdk/test/common/test_ssl_utils.py create mode 100644 libs/xtm-oaev-sdk/test/common/test_utils.py create mode 100644 libs/xtm-oaev-sdk/test/configuration/__init__.py create mode 100644 libs/xtm-oaev-sdk/test/test_boundaries.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/asset_types.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/errors.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/types.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/ssl_utils.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/utils.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py diff --git a/libs/xtm-oaev-sdk/pyproject.toml b/libs/xtm-oaev-sdk/pyproject.toml index 0deaf771..073822e5 100644 --- a/libs/xtm-oaev-sdk/pyproject.toml +++ b/libs/xtm-oaev-sdk/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "pydantic (>=2.13.3,<2.14.0)", "pydantic-settings (>=2.14.0,<2.15.0)", "PyYAML (>=6.0,<6.1)", + "python_json_logger (>=3.3.0,<3.4.0)", ] [build-system] diff --git a/libs/xtm-oaev-sdk/test/common/__init__.py b/libs/xtm-oaev-sdk/test/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/test/common/shim/__init__.py b/libs/xtm-oaev-sdk/test/common/shim/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/test_exceptions.py b/libs/xtm-oaev-sdk/test/common/shim/test_exceptions.py similarity index 100% rename from test/test_exceptions.py rename to libs/xtm-oaev-sdk/test/common/shim/test_exceptions.py diff --git a/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py new file mode 100644 index 00000000..39e01ba0 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py @@ -0,0 +1,146 @@ +"""Contract of the pyoaev shims for the shared (common) capability. + +Legacy imports must keep working, return the *identical* xtm_oaev_sdk object, +and warn; importing pyoaev's own modules and using the SDK directly must stay +silent. +""" + +import subprocess +import sys +import warnings + +import pytest +import xtm_oaev_sdk as sdk + +_ERRORS = [ + "ConfigurationError", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SignatureTransmissionError", + "on_http_error", +] +_UTILS = [ + "AppLogger", + "CustomJsonFormatter", + "EncodedId", + "EnhancedJSONEncoder", + "RequiredOptional", + "copy_dict", + "logger", + "remove_none_from_dict", + "setup_logging_config", +] +_SSL = [ + "data_to_temp_file", + "is_memory_certificate", + "ssl_cert_chain", + "ssl_verify_locations", +] +_ASSET_TYPES = [ + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "CloudProvider", +] + +LEGACY_NAMES = ( + [("pyoaev.exceptions", name) for name in _ERRORS] + + [("pyoaev.utils", name) for name in _UTILS] + + [("pyoaev.helpers", name) for name in _SSL] + + [("pyoaev.asset_types", name) for name in _ASSET_TYPES] + + [("pyoaev.security_domain.types", "SecurityDomains")] + + [("pyoaev", name) for name in _ASSET_TYPES] + + [ + ("pyoaev", name) + for name in _ERRORS + if name not in ("OpenAEVError", "on_http_error") + ] +) + + +def _given_fresh_interpreter_script(script: str) -> list[str]: + return [sys.executable, "-W", "error::DeprecationWarning", "-c", script] + + +def _when_run(command: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(command, capture_output=True, text=True) + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(f"from {module} import {name} as _legacy", namespace) + return namespace["_legacy"], caught + + +def _then_succeeds(result: subprocess.CompletedProcess) -> None: + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize(("module", "name"), LEGACY_NAMES) +def test_legacy_import_warns_and_returns_the_sdk_object(module, name) -> None: + """It warns towards xtm_oaev_sdk and returns the identical object.""" + legacy, caught = _when_import_legacy_name(module, name) + assert legacy is getattr(sdk, name) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert deprecations + assert f"from xtm_oaev_sdk import {name}" in str(deprecations[0].message) + + +def test_pyoaev_all_still_lists_legacy_exceptions() -> None: + """It keeps the legacy star-export surface of pyoaev.""" + import pyoaev + + for name in _ERRORS: + if name not in ("OpenAEVError", "on_http_error"): + assert name in pyoaev.__all__ + for name in _ASSET_TYPES: + assert name in pyoaev.__all__ + + +def test_importing_pyoaev_modules_is_silent() -> None: + """It imports pyoaev and its own modules without any DeprecationWarning.""" + script = ( + "import pyoaev, pyoaev.apis, pyoaev.base, pyoaev.client, pyoaev.mixins\n" + "import pyoaev.helpers, pyoaev.daemons, pyoaev.contracts, pyoaev.signatures\n" + "import pyoaev.utils, pyoaev.exceptions, pyoaev.asset_types\n" + "import pyoaev.security_domain.types\n" + ) + result = _when_run(_given_fresh_interpreter_script(script)) + _then_succeeds(result) + + +def test_sdk_objects_stay_silent_once_pyoaev_is_loaded() -> None: + """It never alters SDK objects: native use stays silent next to pyoaev.""" + script = ( + "import pyoaev, pyoaev.exceptions, pyoaev.utils, pyoaev.helpers\n" + "import xtm_oaev_sdk as sdk\n" + "sdk.EncodedId('a b')\n" + "sdk.RequiredOptional(required=('a',)).validate_attrs(data={'a': 1})\n" + "sdk.AssetCategory('WEB_APPLICATION')\n" + "try:\n" + " raise sdk.OpenAEVHttpError('boom', 500)\n" + "except sdk.OpenAEVError:\n" + " pass\n" + ) + result = _when_run(_given_fresh_interpreter_script(script)) + _then_succeeds(result) + + +@pytest.mark.parametrize( + "module", ["pyoaev.exceptions", "pyoaev.utils", "pyoaev.helpers"] +) +def test_unknown_legacy_name_still_raises(module: str) -> None: + """It does not swallow genuine misses.""" + with pytest.raises(ImportError): + exec(f"from {module} import DoesNotExist", {}) diff --git a/libs/xtm-oaev-sdk/test/common/shim/test_utils.py b/libs/xtm-oaev-sdk/test/common/shim/test_utils.py new file mode 100644 index 00000000..5a9b32f8 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/common/shim/test_utils.py @@ -0,0 +1,141 @@ +import dataclasses +import json +import logging +import unittest +from typing import Any, cast +from unittest.mock import MagicMock, patch + +from pythonjsonlogger.json import JsonFormatter + +import pyoaev.utils as module + + +@dataclasses.dataclass +class _SampleData: + value: int + + +class TestUtils(unittest.TestCase): + def test_custom_json_formatter_inherits_non_deprecated_formatter(self): + self.assertTrue(issubclass(module.CustomJsonFormatter, JsonFormatter)) + + def test_copy_dict_flattens_nested_dict(self): + destination = {} + module.copy_dict( + src={"a": 1, "meta": {"x": "y", "n": 2}}, + dest=destination, + ) + self.assertEqual(destination, {"a": 1, "meta[x]": "y", "meta[n]": 2}) + + def test_remove_none_from_dict(self): + self.assertEqual(module.remove_none_from_dict({"a": 1, "b": None}), {"a": 1}) + + def test_encoded_id_from_existing_instance_returns_same_object(self): + encoded_id = module.EncodedId("with space") + self.assertIs(module.EncodedId(encoded_id), encoded_id) + + def test_encoded_id_encodes_string_and_keeps_int(self): + self.assertEqual(module.EncodedId("a/b c"), "a%2Fb%20c") + self.assertEqual(module.EncodedId(42), "42") + + def test_encoded_id_rejects_unsupported_type(self): + with self.assertRaises(TypeError): + module.EncodedId(cast(Any, ["bad"])) + + def test_enhanced_json_encoder_serializes_not_dataclasses(self): + data = {"test": "this is a test", "other": 3} + classic_json_dump = json.dumps(data) + enhanced_json_dump = json.dumps(data, cls=module.EnhancedJSONEncoder) + self.assertFalse(dataclasses.is_dataclass(data)) + self.assertEqual( + classic_json_dump, + enhanced_json_dump, + ) + + def test_enhanced_json_encoder_serializes_dataclasses(self): + self.assertEqual( + json.dumps(_SampleData(value=3), cls=module.EnhancedJSONEncoder), + '{"value": 3}', + ) + + def test_required_optional_required_attribute_missing(self): + rules = module.RequiredOptional(required=("name",)) + with self.assertRaises(AttributeError): + rules.validate_attrs(data={}) + + def test_required_optional_excludes_required_attribute(self): + rules = module.RequiredOptional(required=("name",)) + rules.validate_attrs(data={}, excludes=["name"]) + + def test_required_optional_exclusive_allows_only_one_key(self): + rules = module.RequiredOptional(exclusive=("a", "b")) + with self.assertRaises(AttributeError): + rules.validate_attrs(data={"a": 1, "b": 2}) + + def test_required_optional_exclusive_requires_one_key(self): + rules = module.RequiredOptional(exclusive=("a", "b")) + with self.assertRaises(AttributeError): + rules.validate_attrs(data={}) + + def test_required_optional_exclusive_with_single_key_is_valid(self): + rules = module.RequiredOptional(exclusive=("a", "b")) + rules.validate_attrs(data={"a": 1}) + + def test_custom_json_formatter_add_fields_sets_timestamp_and_level(self): + formatter = module.CustomJsonFormatter("%(message)s") + record = logging.LogRecord( + name="test", + level=logging.INFO, + pathname=__file__, + lineno=1, + msg="hello", + args=(), + exc_info=None, + ) + log_record = {} + + formatter.add_fields(log_record, record, {}) + + self.assertIn("timestamp", log_record) + self.assertEqual(log_record["level"], "INFO") + + def test_setup_logging_config_json_logging_true_uses_custom_formatter(self): + with patch("xtm_oaev_sdk._core.utils.logging.basicConfig") as mock_basic_config: + module.setup_logging_config(logging.INFO, json_logging=True) + + kwargs = mock_basic_config.call_args.kwargs + self.assertEqual(kwargs["level"], logging.INFO) + self.assertIn("handlers", kwargs) + self.assertIsInstance( + kwargs["handlers"][0].formatter, module.CustomJsonFormatter + ) + + def test_setup_logging_config_json_logging_false_calls_basic_config(self): + with patch("xtm_oaev_sdk._core.utils.logging.basicConfig") as mock_basic_config: + module.setup_logging_config(logging.WARNING, json_logging=False) + + mock_basic_config.assert_called_once_with(level=logging.WARNING) + + def test_app_logger_methods_delegate_to_local_logger(self): + with patch("xtm_oaev_sdk._core.utils.setup_logging_config"): + app_logger = module.AppLogger(logging.INFO) + app_logger.local_logger = MagicMock() + + app_logger.debug("d", {"x": 1}) + app_logger.info("i") + app_logger.warning("w") + app_logger.error("e") + + self.assertTrue(app_logger.local_logger.debug.called) + self.assertTrue(app_logger.local_logger.info.called) + self.assertTrue(app_logger.local_logger.warning.called) + self.assertTrue(app_logger.local_logger.error.called) + + def test_logger_helper_returns_app_logger(self): + with patch("xtm_oaev_sdk._core.utils.setup_logging_config"): + helper = module.logger(logging.INFO) + self.assertIsInstance(helper, module.AppLogger) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/common/test_exceptions.py b/libs/xtm-oaev-sdk/test/common/test_exceptions.py new file mode 100644 index 00000000..a98afda0 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/common/test_exceptions.py @@ -0,0 +1,62 @@ +import unittest +from unittest.mock import MagicMock + +import xtm_oaev_sdk as module + + +class TestExceptions(unittest.TestCase): + def test_openaeverror_init(self): + error_message = MagicMock() + response_code = MagicMock() + response_body = MagicMock() + + exception = module.OpenAEVError(error_message, response_code, response_body) + + self.assertEqual(exception.response_code, response_code) + self.assertEqual(exception.response_body, response_body) + self.assertEqual(exception.error_message, error_message.decode.return_value) + + def test_openaeverror_init_auto_decode(self): + response_code = MagicMock() + response_body = MagicMock() + + exception = module.OpenAEVError(b"test", response_code, response_body) + + self.assertEqual(exception.error_message, "test") + + exception = module.OpenAEVError("test", response_code, response_body) + + self.assertEqual(exception.error_message, "test") + + def test_openaeverror_strcast_no_message(self): + error_message = None + response_code = 418 + response_body = MagicMock() + + exception = module.OpenAEVError(error_message, response_code, response_body) + + strdata = str(exception) + + self.assertEqual(strdata, "418: Unknown error") + + def test_openaeverror_strcast_no_message_no_code(self): + error_message = None + response_code = None + response_body = MagicMock() + + exception = module.OpenAEVError(error_message, response_code, response_body) + + strdata = str(exception) + + self.assertEqual(strdata, "Unknown error") + + def test_openaeverror_strcast_with_body(self): + error_message = None + response_code = 418 + response_body = b'{"message": "I am a teapot"}' + + exception = module.OpenAEVError(error_message, response_code, response_body) + + strdata = str(exception) + + self.assertEqual(strdata, "418: I am a teapot") diff --git a/libs/xtm-oaev-sdk/test/common/test_ssl_utils.py b/libs/xtm-oaev-sdk/test/common/test_ssl_utils.py new file mode 100644 index 00000000..7df6755d --- /dev/null +++ b/libs/xtm-oaev-sdk/test/common/test_ssl_utils.py @@ -0,0 +1,92 @@ +"""Tests for the SSL helpers.""" + +import os +from typing import Any +from unittest.mock import MagicMock + +import xtm_oaev_sdk as sdk + +PEM_CERT = "-----BEGIN CERTIFICATE-----\nAAAA\n-----END CERTIFICATE-----\n" +PEM_KEY = "-----BEGIN PRIVATE KEY-----\nBBBB\n-----END PRIVATE KEY-----\n" + + +def _given_ssl_context() -> MagicMock: + return MagicMock() + + +def _given_recording_context(recorded: dict[str, Any]) -> MagicMock: + def _record(cert: str, key: str | None, passphrase: str | None) -> None: + recorded["cert"] = cert + recorded["key"] = key + recorded["passphrase"] = passphrase + with open(cert) as handle: + recorded["cert_content"] = handle.read() + + context = MagicMock() + context.load_cert_chain.side_effect = _record + return context + + +def test_data_to_temp_file_writes_content_owner_only() -> None: + """It writes the data to a new file readable only by its owner.""" + path = sdk.data_to_temp_file("secret") + try: + with open(path) as handle: + assert handle.read() == "secret" + assert os.stat(path).st_mode & 0o077 == 0 + finally: + os.unlink(path) + + +def test_is_memory_certificate_detects_pem_data() -> None: + """It distinguishes in-memory PEM data from a file path.""" + assert sdk.is_memory_certificate(PEM_CERT) is True + assert sdk.is_memory_certificate("/etc/ssl/cert.pem") is False + + +def test_ssl_verify_locations_ignores_none() -> None: + """It leaves the context untouched without CA data.""" + context = _given_ssl_context() + sdk.ssl_verify_locations(context, None) + context.load_verify_locations.assert_not_called() + + +def test_ssl_verify_locations_loads_memory_ca_as_cadata() -> None: + """It loads in-memory CA data through cadata.""" + context = _given_ssl_context() + sdk.ssl_verify_locations(context, PEM_CERT) + context.load_verify_locations.assert_called_once_with(cadata=PEM_CERT) + + +def test_ssl_verify_locations_loads_ca_path_as_cafile() -> None: + """It loads a CA path through cafile.""" + context = _given_ssl_context() + sdk.ssl_verify_locations(context, "/etc/ssl/ca.pem") + context.load_verify_locations.assert_called_once_with(cafile="/etc/ssl/ca.pem") + + +def test_ssl_cert_chain_ignores_missing_certificate() -> None: + """It does nothing without certificate data.""" + context = _given_ssl_context() + sdk.ssl_cert_chain(context, None, PEM_KEY, None) + context.load_cert_chain.assert_not_called() + + +def test_ssl_cert_chain_passes_paths_through() -> None: + """It loads certificate and key paths as given.""" + context = _given_ssl_context() + sdk.ssl_cert_chain(context, "/tmp/cert.pem", "/tmp/key.pem", "pass") + context.load_cert_chain.assert_called_once_with( + "/tmp/cert.pem", "/tmp/key.pem", "pass" + ) + + +def test_ssl_cert_chain_bridges_memory_pem_through_removed_temp_files() -> None: + """It writes in-memory PEM data to temp files, loads them, then removes them.""" + recorded: dict[str, Any] = {} + context = _given_recording_context(recorded) + sdk.ssl_cert_chain(context, PEM_CERT, PEM_KEY, "pass") + assert recorded["cert_content"] == PEM_CERT + assert recorded["passphrase"] == "pass" + assert not os.path.exists(recorded["cert"]) + assert not os.path.exists(recorded["key"]) diff --git a/libs/xtm-oaev-sdk/test/common/test_utils.py b/libs/xtm-oaev-sdk/test/common/test_utils.py new file mode 100644 index 00000000..4a850ce5 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/common/test_utils.py @@ -0,0 +1,401 @@ +"""Tests for xtm_oaev_sdk utils public surface.""" + +import dataclasses +import logging +import subprocess +import sys +import warnings +from typing import Any + +import pytest +import xtm_oaev_sdk as sdk + + +@dataclasses.dataclass +class _SampleDataClass: + name: str + value: int + + +class _UnsupportedType: + pass + + +class _RecordingLogger: + def __init__(self) -> None: + self.calls: list[dict[str, Any]] = [] + + def debug(self, message: str, **kwargs: Any) -> None: + self.calls.append({"level": "debug", "message": message, "kwargs": kwargs}) + + def info(self, message: str, **kwargs: Any) -> None: + self.calls.append({"level": "info", "message": message, "kwargs": kwargs}) + + def warning(self, message: str, **kwargs: Any) -> None: + self.calls.append({"level": "warning", "message": message, "kwargs": kwargs}) + + def error(self, message: str, **kwargs: Any) -> None: + self.calls.append({"level": "error", "message": message, "kwargs": kwargs}) + + +def _given_public_symbol(name: str) -> Any: + return getattr(sdk, name) + + +def _given_app_logger() -> Any: + app_logger_cls = _given_public_symbol("AppLogger") + return app_logger_cls(logging.INFO, json_logging=False, name="initial") + + +def _given_app_logger_with_recording_logger() -> tuple[Any, _RecordingLogger]: + app_logger = _given_app_logger() + recorder = _RecordingLogger() + app_logger.local_logger = recorder + return app_logger, recorder + + +def _given_required_optional(**kwargs: Any) -> Any: + required_optional_cls = _given_public_symbol("RequiredOptional") + return required_optional_cls(**kwargs) + + +def _given_exclusive_spec() -> Any: + return _given_required_optional(exclusive=("id", "key")) + + +def _given_json_encoder() -> Any: + encoder_cls = _given_public_symbol("EnhancedJSONEncoder") + return encoder_cls() + + +def _given_clean_root_logging_state() -> logging.Logger: + root_logger = logging.getLogger() + root_logger.handlers.clear() + root_logger.setLevel(logging.NOTSET) + return root_logger + + +def _when_call_logger_with_name(app_logger: Any, name: str) -> Any: + return app_logger(name) + + +def _when_call_app_logger_method( + app_logger: Any, method_name: str, message: str, meta: dict[str, Any] +) -> None: + getattr(app_logger, method_name)(message, meta) + + +def _when_set_third_party_logger_level(lib: str, level: int) -> None: + app_logger_cls = _given_public_symbol("AppLogger") + app_logger_cls.setup_logger_level(lib, level) + + +def _when_encode_with_default(encoder: Any, value: Any) -> Any: + return encoder.default(value) + + +def _when_encode_unsupported_type_raises(encoder: Any, value: Any) -> Exception: + with pytest.raises(TypeError) as exc_info: + encoder.default(value) + return exc_info.value + + +def _when_validate_attrs_raises( + validator: Any, data: dict[str, Any], excludes: list[str] | None = None +) -> Exception: + with pytest.raises(AttributeError) as exc_info: + validator.validate_attrs(data=data, excludes=excludes) + return exc_info.value + + +def _when_validate_attrs( + validator: Any, data: dict[str, Any], excludes: list[str] | None = None +) -> None: + validator.validate_attrs(data=data, excludes=excludes) + + +def _when_set_required_optional_field_raises( + instance: Any, field_name: str, value: Any +) -> Exception: + with pytest.raises(dataclasses.FrozenInstanceError) as exc_info: + setattr(instance, field_name, value) + return exc_info.value + + +def _when_encode_id(value: Any) -> Any: + encoded_id_cls = _given_public_symbol("EncodedId") + return encoded_id_cls(value) + + +def _when_encode_invalid_id_raises(value: Any) -> Exception: + with pytest.raises(TypeError) as exc_info: + _when_encode_id(value) + return exc_info.value + + +def _when_copy_dict(src: dict[str, Any], dest: dict[str, Any]) -> None: + copy_dict_fn = _given_public_symbol("copy_dict") + copy_dict_fn(src=src, dest=dest) + + +def _when_remove_none_from_dict(data: dict[str, Any]) -> dict[str, Any]: + remove_none_fn = _given_public_symbol("remove_none_from_dict") + return remove_none_fn(data) + + +def _when_call_logger(level: int, json_logging: bool) -> Any: + logger_fn = _given_public_symbol("logger") + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + return logger_fn(level, json_logging) + + +def _when_setup_logging(level: int, json_logging: bool) -> None: + setup_fn = _given_public_symbol("setup_logging_config") + setup_fn(level, json_logging=json_logging) + + +def _then_logger_name_switched_and_self_returned( + original: Any, switched: Any, expected_name: str +) -> None: + assert switched is original + assert original.local_logger.name == expected_name + + +def _then_recording_logger_has_call( + recorder: _RecordingLogger, + expected_level: str, + expected_message: str, + expected_meta: dict[str, Any], +) -> None: + assert len(recorder.calls) == 1 + call = recorder.calls[0] + assert call["level"] == expected_level + assert call["message"] == expected_message + assert call["kwargs"]["extra"] == {"attributes": expected_meta} + if expected_level == "error": + assert call["kwargs"]["exc_info"] == 1 + + +def _then_third_party_logger_level_is(lib: str, expected_level: int) -> None: + assert logging.getLogger(lib).level == expected_level + + +def _then_runtime_checkable_protocol_satisfied(instance: Any) -> None: + protocol_cls = _given_public_symbol("AppLoggerProtocol") + assert isinstance(instance, protocol_cls) + + +def _then_encoded_object_equals(encoded: Any, expected: Any) -> None: + assert encoded == expected + + +def _then_contains_text(exc: Exception, text: str) -> None: + assert text in str(exc) + + +def _then_error_is_frozen(exc: Exception) -> None: + assert isinstance(exc, dataclasses.FrozenInstanceError) + + +def _then_type_error_for_unsupported_type(exc: Exception) -> None: + assert "not json serializable" in str(exc).lower() + + +def _then_idempotent_reencode(value: Any) -> None: + encoded_id_cls = _given_public_symbol("EncodedId") + assert encoded_id_cls(encoded_id_cls(value)) == encoded_id_cls(value) + + +def _then_dict_equals(actual: dict[str, Any], expected: dict[str, Any]) -> None: + assert actual == expected + + +def _then_is_app_logger_instance(instance: Any) -> None: + app_logger_cls = _given_public_symbol("AppLogger") + assert isinstance(instance, app_logger_cls) + + +def _then_root_logger_has_json_formatter(expected_level: int) -> None: + custom_formatter_cls = _given_public_symbol("CustomJsonFormatter") + root_logger = logging.getLogger() + assert root_logger.handlers + assert isinstance(root_logger.handlers[0].formatter, custom_formatter_cls) + assert root_logger.level == expected_level + + +def _then_root_logger_uses_basic_config_level(expected_level: int) -> None: + root_logger = logging.getLogger() + assert root_logger.level == expected_level + + +def test_app_logger_call_switches_name_and_returns_self() -> None: + """Verify AppLogger.__call__ switches logger name and returns self.""" + app_logger = _given_app_logger() + switched = _when_call_logger_with_name(app_logger, "switched") + _then_logger_name_switched_and_self_returned(app_logger, switched, "switched") + + +@pytest.mark.parametrize("level_name", ["debug", "info", "warning", "error"]) +def test_app_logger_logs_with_meta_dict(level_name: str) -> None: + """Verify AppLogger log methods accept metadata and call the right level.""" + app_logger, recorder = _given_app_logger_with_recording_logger() + _when_call_app_logger_method(app_logger, level_name, "message", {"k": "v"}) + _then_recording_logger_has_call(recorder, level_name, "message", {"k": "v"}) + + +def test_app_logger_setup_logger_level_adjusts_third_party_logger() -> None: + """Verify AppLogger.setup_logger_level updates external logger level.""" + _when_set_third_party_logger_level("third.party.lib", logging.ERROR) + _then_third_party_logger_level_is("third.party.lib", logging.ERROR) + + +def test_app_logger_protocol_runtime_checkable() -> None: + """Verify AppLogger satisfies AppLoggerProtocol runtime isinstance check.""" + app_logger = _given_app_logger() + _then_runtime_checkable_protocol_satisfied(app_logger) + + +def test_enhanced_json_encoder_serializes_dataclass() -> None: + """Verify EnhancedJSONEncoder serializes dataclass instances via asdict.""" + encoder = _given_json_encoder() + encoded = _when_encode_with_default(encoder, _SampleDataClass(name="n", value=1)) + _then_encoded_object_equals(encoded, {"name": "n", "value": 1}) + + +def test_enhanced_json_encoder_raises_type_error_for_unsupported_types() -> None: + """Verify EnhancedJSONEncoder raises TypeError on unsupported objects.""" + encoder = _given_json_encoder() + exc = _when_encode_unsupported_type_raises(encoder, _UnsupportedType()) + _then_type_error_for_unsupported_type(exc) + + +def test_required_optional_is_frozen() -> None: + """Verify RequiredOptional is immutable and rejects field assignment.""" + required_optional = _given_required_optional(required=("name",)) + exc = _when_set_required_optional_field_raises( + required_optional, "required", ("id",) + ) + _then_error_is_frozen(exc) + + +def test_required_optional_validate_attrs_missing_required_raises() -> None: + """Verify RequiredOptional.validate_attrs raises for missing required fields.""" + required_optional = _given_required_optional(required=("name", "type")) + exc = _when_validate_attrs_raises(required_optional, {"name": "sdk"}) + _then_contains_text(exc, "Missing attributes") + + +@pytest.mark.parametrize( + ("data", "expected_text"), + [ + ({}, "Must provide one of these attributes"), + ({"id": "1", "key": "2"}, "Provide only one of these attributes"), + ], +) +def test_required_optional_validate_attrs_exclusive_violations_raise( + data: dict[str, Any], expected_text: str +) -> None: + """Verify RequiredOptional.validate_attrs raises for exclusive rule violations.""" + required_optional = _given_exclusive_spec() + exc = _when_validate_attrs_raises(required_optional, data) + _then_contains_text(exc, expected_text) + + +def test_required_optional_validate_attrs_excludes_skip_required_check() -> None: + """Verify RequiredOptional.validate_attrs excludes bypass required checks.""" + required_optional = _given_required_optional(required=("name", "type")) + _when_validate_attrs(required_optional, {"name": "sdk"}, excludes=["type"]) + + +def test_required_optional_validate_attrs_valid_data_passes() -> None: + """Verify RequiredOptional.validate_attrs accepts valid required/exclusive data.""" + required_optional = _given_required_optional( + required=("name",), exclusive=("id", "key") + ) + _when_validate_attrs(required_optional, {"name": "sdk", "id": "123"}) + + +def test_encoded_id_url_encodes_strings() -> None: + """Verify EncodedId percent-encodes unsafe string characters.""" + encoded = _when_encode_id("hello world") + _then_encoded_object_equals(encoded, "hello%20world") + + +def test_encoded_id_is_idempotent_when_reencoded() -> None: + """Verify EncodedId(EncodedId(x)) yields the same encoded value.""" + _then_idempotent_reencode("hello world") + + +def test_encoded_id_rejects_non_str_non_int_values() -> None: + """Verify EncodedId raises TypeError for unsupported value types.""" + exc = _when_encode_invalid_id_raises(3.14) + _then_contains_text(exc, "Unsupported type received") + + +def test_encoded_id_accepts_int_and_returns_string_representation() -> None: + """Verify EncodedId passes integer values through as string content.""" + encoded = _when_encode_id(42) + _then_encoded_object_equals(encoded, "42") + + +def test_copy_dict_flattens_nested_dict_keys() -> None: + """Verify copy_dict flattens nested dict keys to bracket notation.""" + destination: dict[str, Any] = {} + _when_copy_dict(src={"filters": {"type": "malware"}}, dest=destination) + _then_dict_equals(destination, {"filters[type]": "malware"}) + + +def test_copy_dict_copies_non_dict_values_directly() -> None: + """Verify copy_dict copies non-dict values without transformation.""" + destination: dict[str, Any] = {} + _when_copy_dict(src={"name": "sdk", "count": 2}, dest=destination) + _then_dict_equals(destination, {"name": "sdk", "count": 2}) + + +def test_remove_none_from_dict_strips_none_valued_keys() -> None: + """Verify remove_none_from_dict removes entries with None values.""" + result = _when_remove_none_from_dict({"a": 1, "b": None, "c": "x"}) + _then_dict_equals(result, {"a": 1, "c": "x"}) + + +def test_remove_none_from_dict_keeps_non_none_values() -> None: + """Verify remove_none_from_dict keeps entries with non-None values.""" + result = _when_remove_none_from_dict({"a": 0, "b": False, "c": ""}) + _then_dict_equals(result, {"a": 0, "b": False, "c": ""}) + + +def test_logger_function_returns_app_logger() -> None: + """Verify logger() returns an AppLogger without warning.""" + logger_instance = _when_call_logger(logging.INFO, json_logging=False) + _then_is_app_logger_instance(logger_instance) + + +def test_setup_logging_config_json_logging_true_sets_json_handler() -> None: + """Verify setup_logging_config configures JSON formatter when enabled.""" + _given_clean_root_logging_state() + _when_setup_logging(logging.INFO, json_logging=True) + _then_root_logger_has_json_formatter(logging.INFO) + + +def test_setup_logging_config_json_logging_false_uses_basic_config() -> None: + """Verify setup_logging_config uses basicConfig when JSON logging is disabled.""" + _given_clean_root_logging_state() + _when_setup_logging(logging.DEBUG, json_logging=False) + _then_root_logger_uses_basic_config_level(logging.DEBUG) + + +def test_importing_utils_does_not_raise_jsonlogger_deprecation_warning() -> None: + """Verify the JSON formatter uses the non-deprecated pythonjsonlogger path.""" + result = subprocess.run( + [ + sys.executable, + "-W", + "error::DeprecationWarning", + "-c", + "import xtm_oaev_sdk._core.utils", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/libs/xtm-oaev-sdk/test/configuration/__init__.py b/libs/xtm-oaev-sdk/test/configuration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/test/test_boundaries.py b/libs/xtm-oaev-sdk/test/test_boundaries.py new file mode 100644 index 00000000..6c57f6ac --- /dev/null +++ b/libs/xtm-oaev-sdk/test/test_boundaries.py @@ -0,0 +1,42 @@ +"""SDK dependency and source-boundary assertions.""" + +import pathlib +import re + +import pytest + +_SDK_ROOT = pathlib.Path(__file__).resolve().parents[1] / "xtm_oaev_sdk" + + +def _when_searching_sdk_sources(pattern: str) -> list[tuple[str, str]]: + regex = re.compile(pattern) + return [ + (str(path.relative_to(_SDK_ROOT)), line) + for path in sorted(_SDK_ROOT.rglob("*.py")) + for line in path.read_text(encoding="utf-8").splitlines() + if regex.search(line) + ] + + +@pytest.mark.parametrize("package", ["pyoaev", "requests", "pika", "thefuzz"]) +def test_sdk_sources_do_not_import(package: str) -> None: + """It never imports the legacy client nor its transport dependencies.""" + pattern = rf"^\s*(?:from|import)\s+{package}(?:\.|\b)" + assert _when_searching_sdk_sources(pattern) == [] + + +@pytest.mark.parametrize( + "artifact", + [ + "class ListenQueue", + "class PingAlive", + "class OpenAEVConfigHelper", + "class OpenAEVCollectorHelper", + "class OpenAEVInjectorHelper", + "class OpenAEVDetectionHelper", + "def create_mq_ssl_context", + ], +) +def test_sdk_sources_do_not_define_client_helpers(artifact: str) -> None: + """It leaves transport and daemon helpers to the client.""" + assert _when_searching_sdk_sources(rf"^\s*{artifact}\b") == [] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py index 2c899e20..0698318c 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -1,7 +1,10 @@ """xtm-oaev-sdk — OpenAEV SDK core (DDD + Light Hex Architecture). Architecture: + _core/ Shared implementations (errors, utils, ssl_utils, + asset_types, security_domain) _core/configuration/ Configuration feature implementation + seam/common/ Shared errors, enums, types, and utilities seam/configuration/ Configuration I/O surface seam/deprecation.py PEP 562 deprecation primitive public/ User-facing re-exports diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/asset_types.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/asset_types.py new file mode 100644 index 00000000..49cea094 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/asset_types.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +"""Canonical asset taxonomy for OpenAEV (mirrors the backend enums). + +These give producers (collectors, custom integrations) a single, typed vocabulary for the +two-level asset categorization (category + subcategory), criticality and cloud provider, instead +of free-form strings. They are plain ``str`` enums so they can be dropped straight into the dict +payloads sent to the API. +""" + +from enum import Enum + + +class AssetCategory(str, Enum): + HOST = "HOST" + CONTAINER_WORKLOAD = "CONTAINER_WORKLOAD" + CLOUD_RESOURCE = "CLOUD_RESOURCE" + WEB_APPLICATION = "WEB_APPLICATION" + NETWORK_DEVICE = "NETWORK_DEVICE" + MOBILE_DEVICE = "MOBILE_DEVICE" + IOT_OT_DEVICE = "IOT_OT_DEVICE" + IDENTITY = "IDENTITY" + SAAS_APPLICATION = "SAAS_APPLICATION" + AI_TARGET = "AI_TARGET" + SECURITY_PLATFORM = "SECURITY_PLATFORM" + GENERIC_ASSET = "GENERIC_ASSET" + + +class AssetSubCategory(str, Enum): + # HOST + SERVER = "SERVER" + WORKSTATION = "WORKSTATION" + LAPTOP = "LAPTOP" + VIRTUAL_MACHINE = "VIRTUAL_MACHINE" + HYPERVISOR = "HYPERVISOR" + MAINFRAME = "MAINFRAME" + THIN_CLIENT = "THIN_CLIENT" + # CONTAINER_WORKLOAD + CONTAINER = "CONTAINER" + CONTAINER_IMAGE = "CONTAINER_IMAGE" + KUBERNETES_POD = "KUBERNETES_POD" + KUBERNETES_CLUSTER = "KUBERNETES_CLUSTER" + KUBERNETES_NODE = "KUBERNETES_NODE" + SERVERLESS_FUNCTION = "SERVERLESS_FUNCTION" + # CLOUD_RESOURCE + COMPUTE = "COMPUTE" + STORAGE = "STORAGE" + DATABASE = "DATABASE" + NETWORKING = "NETWORKING" + SERVERLESS = "SERVERLESS" + CONTAINER_REGISTRY = "CONTAINER_REGISTRY" + KUBERNETES = "KUBERNETES" + IAM_PRINCIPAL = "IAM_PRINCIPAL" + SECRETS_KEY_MGMT = "SECRETS_KEY_MGMT" + MESSAGING_QUEUE = "MESSAGING_QUEUE" + ANALYTICS_DATA = "ANALYTICS_DATA" + AI_ML_SERVICE = "AI_ML_SERVICE" + IAC_TEMPLATE = "IAC_TEMPLATE" + CLOUD_OTHER = "CLOUD_OTHER" + # WEB_APPLICATION + WEBSITE = "WEBSITE" + WEB_API = "WEB_API" + SINGLE_PAGE_APP = "SINGLE_PAGE_APP" + GRAPHQL_API = "GRAPHQL_API" + WEB_SERVICE = "WEB_SERVICE" + MICROSERVICE = "MICROSERVICE" + # NETWORK_DEVICE + ROUTER = "ROUTER" + SWITCH = "SWITCH" + FIREWALL = "FIREWALL" + LOAD_BALANCER = "LOAD_BALANCER" + VPN_GATEWAY = "VPN_GATEWAY" + WIRELESS_AP = "WIRELESS_AP" + PROXY = "PROXY" + DNS_SERVER = "DNS_SERVER" + DHCP_SERVER = "DHCP_SERVER" + SAN_NAS = "SAN_NAS" + NETWORK_OTHER = "NETWORK_OTHER" + # MOBILE_DEVICE + SMARTPHONE = "SMARTPHONE" + TABLET = "TABLET" + # IOT_OT_DEVICE + IOT_SENSOR = "IOT_SENSOR" + IP_CAMERA = "IP_CAMERA" + GATEWAY = "GATEWAY" + POINT_OF_SALE = "POINT_OF_SALE" + MEDIA_DEVICE = "MEDIA_DEVICE" + PLC = "PLC" + RTU = "RTU" + HMI = "HMI" + SCADA_HISTORIAN = "SCADA_HISTORIAN" + MEDICAL_DEVICE = "MEDICAL_DEVICE" + PRINTER_PERIPHERAL = "PRINTER_PERIPHERAL" + BUILDING_MGMT = "BUILDING_MGMT" + # IDENTITY + USER_ACCOUNT = "USER_ACCOUNT" + SERVICE_ACCOUNT = "SERVICE_ACCOUNT" + GROUP = "GROUP" + ROLE = "ROLE" + SHARED_MAILBOX = "SHARED_MAILBOX" + NON_HUMAN_IDENTITY = "NON_HUMAN_IDENTITY" + # SAAS_APPLICATION + SAAS_APP = "SAAS_APP" + SAAS_TENANT = "SAAS_TENANT" + # AI_TARGET + LLM_MODEL = "LLM_MODEL" + AI_AGENT = "AI_AGENT" + MCP_SERVER = "MCP_SERVER" + RAG_PIPELINE = "RAG_PIPELINE" + # SECURITY_PLATFORM + EDR = "EDR" + XDR = "XDR" + SIEM = "SIEM" + SOAR = "SOAR" + NDR = "NDR" + ISPM = "ISPM" + LLM_FIREWALL = "LLM_FIREWALL" + AI_GATEWAY = "AI_GATEWAY" + + +class CloudProvider(str, Enum): + AWS = "AWS" + AZURE = "AZURE" + GCP = "GCP" + OCI = "OCI" + ALIBABA = "ALIBABA" + KUBERNETES = "KUBERNETES" + OTHER = "OTHER" + + +class AssetCriticality(str, Enum): + VERY_HIGH = "VERY_HIGH" + HIGH = "HIGH" + MEDIUM = "MEDIUM" + LOW = "LOW" + UNKNOWN = "UNKNOWN" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/errors.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/errors.py new file mode 100644 index 00000000..0db3758b --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/errors.py @@ -0,0 +1,315 @@ +"""xtm-oaev-sdk error hierarchy. + +All SDK exceptions derive from OpenAEVError. Consumers catch specific +subclasses for granular handling or catch the base class as a catch-all. + +Exception tree:: + + OpenAEVError (base) + ├── OpenAEVAuthenticationError — credential/token failures + ├── OpenAEVHttpError — generic HTTP transport errors + │ └── (wrapped by on_http_error decorator into caller-specific types) + ├── OpenAEVParsingError — response deserialization failures + ├── RedirectError — unexpected HTTP redirects + ├── OpenAEVHeadError — HEAD request failures + ├── OpenAEVGetError — GET request failures + ├── OpenAEVUpdateError — PUT/PATCH request failures + ├── OpenAEVListError — list/pagination failures + ├── OpenAEVCreateError — POST creation failures + ├── SignatureTransmissionError — signature delivery failures + └── ConfigurationError — configuration loading/validation failures + +Example: + >>> from xtm_oaev_sdk import OpenAEVError, OpenAEVHttpError + >>> try: + ... api.call() + ... except OpenAEVHttpError as e: + ... print(f"HTTP {e.response_code}: {e.error_message}") + ... except OpenAEVError as e: + ... print(f"SDK error: {e}") +""" + +import functools +from typing import TYPE_CHECKING, Any, Callable, Optional, Type, TypeVar, Union, cast + + +class OpenAEVError(Exception): + """Base exception for all xtm-oaev-sdk errors. + + Provides structured error context including HTTP status codes and + response bodies when available. Automatically extracts meaningful + error messages from JSON response payloads. + + Args: + error_message: Human-readable error description. Accepts str or bytes. + response_code: HTTP status code if the error originated from an API call. + response_body: Raw HTTP response body for deeper error extraction. + """ + + def __init__( + self, + error_message: Union[str, bytes] = "", + response_code: Optional[int] = None, + response_body: Optional[bytes] = None, + ) -> None: + Exception.__init__(self, error_message) + self.response_code = response_code + self.response_body = response_body + try: + if TYPE_CHECKING: + assert isinstance(error_message, bytes) + self.error_message = error_message.decode() + except Exception: + if TYPE_CHECKING: + assert isinstance(error_message, str) + self.error_message = error_message + + def __str__(self) -> str: + message = self.error_message + + generic_messages = ( + "Internal Server Error", + "Bad Request", + "Not Found", + "Unauthorized", + "Forbidden", + "Service Unavailable", + "Gateway Timeout", + "Unknown error", + "Validation Failed", + ) + + if ( + not message or (message in generic_messages and len(message) < 30) + ) and self.response_body is not None: + try: + import json + + body = self.response_body.decode(errors="ignore") + data = json.loads(body) + extracted_msg = None + + if isinstance(data, dict): + if "error" in data: + err = data.get("error") + if isinstance(err, dict) and "message" in err: + extracted_msg = err.get("message") + elif isinstance(err, str): + extracted_msg = err + elif "message" in data: + extracted_msg = data.get("message") + elif "execution_message" in data: + extracted_msg = data.get("execution_message") + elif "detail" in data: + extracted_msg = data.get("detail") + elif "errors" in data: + errs = data.get("errors") + if isinstance(errs, list) and errs: + parts = [] + for item in errs: + if isinstance(item, dict) and "message" in item: + parts.append(str(item.get("message"))) + else: + parts.append(str(item)) + extracted_msg = "; ".join(parts) + elif isinstance(errs, dict): + if "children" in errs: + validation_errors = [] + children = errs.get("children", {}) + for field, field_errors in children.items(): + if ( + isinstance(field_errors, dict) + and "errors" in field_errors + ): + field_error_list = field_errors.get( + "errors", [] + ) + if field_error_list: + for err_msg in field_error_list: + validation_errors.append( + f"{field}: {err_msg}" + ) + if validation_errors: + base_msg = data.get("message", "Validation Failed") + extracted_msg = ( + f"{base_msg}: {'; '.join(validation_errors)}" + ) + else: + parts = [] + for key, value in errs.items(): + if value: + parts.append(f"{key}: {value}") + if parts: + extracted_msg = "; ".join(parts) + elif isinstance(errs, str): + extracted_msg = errs + + if extracted_msg and extracted_msg not in generic_messages: + message = str(extracted_msg) + elif not message: + message = body[:500] + + except json.JSONDecodeError: + if not message or message in generic_messages: + try: + decoded = self.response_body.decode(errors="ignore")[:500] + if decoded and decoded not in generic_messages: + message = decoded + except Exception: + pass + except Exception: + pass + + if not message: + message = "Unknown error" + + message = " ".join(message.split()) + + if self.response_code is not None: + return f"{self.response_code}: {message}" + return f"{message}" + + +class OpenAEVAuthenticationError(OpenAEVError): + """Authentication or authorization failure. + + Raised when credentials are invalid, tokens are expired, or the + authenticated user lacks permission for the requested operation. + """ + + pass + + +class OpenAEVHttpError(OpenAEVError): + """Generic HTTP transport error. + + Raised on unexpected HTTP responses. Use the ``on_http_error`` + decorator to automatically wrap these into operation-specific types. + """ + + pass + + +class OpenAEVParsingError(OpenAEVError): + """Response deserialization failure. + + Raised when an API response cannot be parsed into the expected format + (malformed JSON, unexpected schema, encoding issues). + """ + + pass + + +class RedirectError(OpenAEVError): + """Unexpected HTTP redirect encountered.""" + + pass + + +class OpenAEVHeadError(OpenAEVError): + """HEAD request failure.""" + + pass + + +class OpenAEVGetError(OpenAEVError): + """GET request failure.""" + + pass + + +class OpenAEVUpdateError(OpenAEVError): + """PUT/PATCH update request failure.""" + + pass + + +class OpenAEVListError(OpenAEVError): + """List/pagination request failure.""" + + pass + + +class OpenAEVCreateError(OpenAEVError): + """POST creation request failure.""" + + pass + + +class OpenAEVDeleteError(OpenAEVError): + """DELETE request failure.""" + + pass + + +class SignatureTransmissionError(OpenAEVError): + """Signature delivery failure. + + Raised when signatures cannot be transmitted to the platform. + Validation rejected them, 4xx slammed the door, or retries ran dry. + """ + + pass + + +class ConfigurationError(OpenAEVError): + """Configuration loading or validation failure. + + Raised when required configuration keys are missing, values fail + validation, or configuration sources are unreachable. + """ + + pass + + +_F = TypeVar("_F", bound=Callable[..., Any]) + + +def on_http_error(error: Type[Exception]) -> Callable[[_F], _F]: + """Decorator that wraps OpenAEVHttpError into a caller-specified type. + + Use on API methods to convert generic HTTP errors into operation-specific + exception types while preserving error context. + + Args: + error: The exception class to raise instead of OpenAEVHttpError. + Must accept (error_message, response_code, response_body) args. + + Returns: + A decorator that wraps the target function. + + Example: + >>> @on_http_error(OpenAEVCreateError) + ... def create_entity(self, data): + ... return self._post("/entities", data) + """ + + def wrap(f: _F) -> _F: + @functools.wraps(f) + def wrapped_f(*args: Any, **kwargs: Any) -> Any: + try: + return f(*args, **kwargs) + except OpenAEVHttpError as e: + raise error(e.error_message, e.response_code, e.response_body) from e + + return cast(_F, wrapped_f) + + return wrap + + +__all__ = [ + "OpenAEVError", + "OpenAEVAuthenticationError", + "OpenAEVHttpError", + "OpenAEVParsingError", + "RedirectError", + "OpenAEVHeadError", + "OpenAEVGetError", + "OpenAEVUpdateError", + "OpenAEVListError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "SignatureTransmissionError", + "ConfigurationError", + "on_http_error", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/__init__.py new file mode 100644 index 00000000..c23cf7fc --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/__init__.py @@ -0,0 +1,9 @@ +"""Security domain enums and types. + +Re-exports the SecurityDomains enum for internal engine use. +Public consumers import from xtm_oaev_sdk directly. +""" + +from .types import SecurityDomains + +__all__ = ["SecurityDomains"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/types.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/types.py new file mode 100644 index 00000000..eb201bd2 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/types.py @@ -0,0 +1,45 @@ +"""Security domain classification types. + +Defines the taxonomy of security domains used to categorize signatures, +injections, and detection scenarios across the OpenAEV platform. + +Each domain carries a display name and color for UI rendering. + +Example: + >>> from xtm_oaev_sdk import SecurityDomains + >>> domain = SecurityDomains.ENDPOINT + >>> domain.value["domain_name"] + 'Endpoint' + >>> domain.value["domain_color"] + '#389CFF' +""" + +from enum import Enum + +__all__ = ["SecurityDomains"] + + +class SecurityDomains(Enum): + """Security domain classification for signatures and injections. + + Each member maps to a dict with: + - ``domain_name``: Human-readable display name. + - ``domain_color``: Hex color code for UI rendering. + """ + + ENDPOINT = {"domain_name": "Endpoint", "domain_color": "#389CFF"} + NETWORK = {"domain_name": "Network", "domain_color": "#009933"} + WEB_APP = {"domain_name": "Web App", "domain_color": "#FF9933"} + EMAIL_INFILTRATION = { + "domain_name": "E-mail Infiltration", + "domain_color": "#FF6666", + } + DATA_EXFILTRATION = {"domain_name": "Data Exfiltration", "domain_color": "#9933CC"} + URL_FILTERING = {"domain_name": "URL Filtering", "domain_color": "#66CCFF"} + CLOUD = {"domain_name": "Cloud", "domain_color": "#9999CC"} + TABLE_TOP = {"domain_name": "Tabletop", "domain_color": "#FFCC33"} + ARTIFICIAL_INTELLIGENCE = { + "domain_name": "Artificial Intelligence", + "domain_color": "#7C4DFF", + } + TOCLASSIFY = {"domain_name": "To classify", "domain_color": "#FFFFFF"} diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/ssl_utils.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/ssl_utils.py new file mode 100644 index 00000000..8f78e748 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/ssl_utils.py @@ -0,0 +1,79 @@ +"""SSL utility functions for OpenAEV SDK. + +Loading certificates into an ``ssl.SSLContext``, including PEM data held in +memory (see https://bugs.python.org/issue16487). +""" + +import os +import ssl +import tempfile + +__all__ = [ + "data_to_temp_file", + "is_memory_certificate", + "ssl_cert_chain", + "ssl_verify_locations", +] + + +# As cert must be written in files to be loaded in ssl context +# Creates a temporary file in the most secure manner possible +def data_to_temp_file(data: str) -> str: + """Write data to a secure temporary file and return its path.""" + # The file is readable and writable only by the creating user ID. + # If the operating system uses permission bits to indicate whether a + # file is executable, the file is executable by no one. The file + # descriptor is not inherited by children of this process. + file_descriptor, file_path = tempfile.mkstemp() + with os.fdopen(file_descriptor, "w") as open_file: + open_file.write(data) + open_file.close() + return file_path + + +def is_memory_certificate(certificate: str) -> bool: + """Return True if the certificate is PEM-encoded in memory (not a file path).""" + return certificate.startswith("-----BEGIN") + + +def ssl_cert_chain( + ssl_context: ssl.SSLContext, + cert_data: str | None, + key_data: str | None, + passphrase: str | None, +) -> None: + """Load a certificate chain into an SSL context, handling in-memory PEM data.""" + if cert_data is None: + return + + cert_file_path = None + key_file_path = None + + # Cert loading + if is_memory_certificate(cert_data): + cert_file_path = data_to_temp_file(cert_data) + cert = cert_file_path if cert_file_path is not None else cert_data + + # Key loading + if key_data is not None and is_memory_certificate(key_data): + key_file_path = data_to_temp_file(key_data) + key = key_file_path if key_file_path is not None else key_data + + # Load cert + ssl_context.load_cert_chain(cert, key, passphrase) + # Remove temp files + if cert_file_path is not None: + os.unlink(cert_file_path) + if key_file_path is not None: + os.unlink(key_file_path) + + +def ssl_verify_locations(ssl_context: ssl.SSLContext, certdata: str | None) -> None: + """Load CA verification locations into an SSL context.""" + if certdata is None: + return + + if is_memory_certificate(certdata): + ssl_context.load_verify_locations(cadata=certdata) + else: + ssl_context.load_verify_locations(cafile=certdata) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/utils.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/utils.py new file mode 100644 index 00000000..5e23aa85 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/utils.py @@ -0,0 +1,404 @@ +"""xtm-oaev-sdk shared utilities. + +Core utility classes and functions used across the SDK engine modules. +Provides logging infrastructure, JSON encoding, data validation helpers, +and dict manipulation utilities. + +Public symbols (re-exported from xtm_oaev_sdk): + - AppLogger: Structured JSON logger with level-aware output. + - AppLoggerProtocol: Protocol for logger behavioral contract. + - EnhancedJSONEncoder: JSON encoder supporting dataclasses. + - RequiredOptional: Immutable field-presence validator. + - EncodedId: URL-safe encoded identifier type. + - CustomJsonFormatter: Timestamped JSON log formatter. + - setup_logging_config: Configure root logger with JSON or plain output. + - copy_dict: Flatten nested dict into bracket-notation keys. + - remove_none_from_dict: Strip None-valued keys from a dict. + - logger: Factory returning an AppLogger. +""" + +import dataclasses +import datetime +import json +import logging +import urllib.parse +from typing import Any, Protocol, runtime_checkable + +from pythonjsonlogger.json import JsonFormatter + +__all__ = [ + "AppLogger", + "AppLoggerProtocol", + "CustomJsonFormatter", + "EncodedId", + "EnhancedJSONEncoder", + "RequiredOptional", + "copy_dict", + "logger", + "remove_none_from_dict", + "setup_logging_config", +] + + +@runtime_checkable +class AppLoggerProtocol(Protocol): + """Behavioral contract for SDK logging services. + + Any logger implementation used by SDK consumers must satisfy this + protocol. The built-in AppLogger class implements it. + + The __call__ method allows switching the logger name (factory pattern): + >>> log = AppLogger(logging.DEBUG) + >>> module_log = log("my_module") # returns self with new name + """ + + def __call__(self, name: str) -> "AppLoggerProtocol": + """Switch logger name and return self.""" + ... + + def debug(self, message: str, meta: Any = None) -> None: + """Log at DEBUG level.""" + ... + + def info(self, message: str, meta: Any = None) -> None: + """Log at INFO level.""" + ... + + def warning(self, message: str, meta: Any = None) -> None: + """Log at WARNING level.""" + ... + + def error(self, message: str, meta: Any = None) -> None: + """Log at ERROR level with traceback.""" + ... + + @staticmethod + def setup_logger_level(lib: str, log_level: int) -> None: + """Override log level for a third-party library.""" + ... + + +class EnhancedJSONEncoder(json.JSONEncoder): + """JSON encoder that serializes ``@dataclass`` instances via + ``dataclasses.asdict()``. + + Example: + >>> import json + >>> from dataclasses import dataclass + >>> @dataclass + ... class Point: + ... x: int + ... y: int + >>> json.dumps(Point(1, 2), cls=EnhancedJSONEncoder) + '{"x": 1, "y": 2}' + """ + + def default(self, o: Any) -> Any: + """Serialize dataclasses to dicts. + + Args: + o: Object to serialize. + + Returns: + A JSON-serializable representation. + + Raises: + TypeError: If the object is not serializable. + """ + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + return super().default(o) + + +@dataclasses.dataclass(frozen=True) +class RequiredOptional: + """Immutable field-presence validator for API payloads. + + Validates that required fields are present, optional fields are allowed, + and exclusive fields follow mutual exclusion rules. + + Args: + required: Tuple of field names that must be present. + optional: Tuple of field names that may be present. + exclusive: Tuple of field names where exactly one must be present. + + Example: + >>> ro = RequiredOptional(required=("name", "type"), exclusive=("id", "key")) + >>> ro.validate_attrs(data={"name": "x", "type": "y", "id": "123"}) # OK + >>> ro.validate_attrs(data={"name": "x"}) # raises AttributeError + """ + + required: tuple[str, ...] = () + optional: tuple[str, ...] = () + exclusive: tuple[str, ...] = () + + def validate_attrs( + self, + *, + data: dict[str, Any], + excludes: list[str] | None = None, + ) -> None: + """Validate field presence against the configured rules. + + Args: + data: Dict of field names to values to validate. + excludes: Field names to skip during required-field checks. + + Raises: + AttributeError: If required fields are missing, or exclusive + field constraints are violated. + """ + if excludes is None: + excludes = [] + + if self.required: + required = [k for k in self.required if k not in excludes] + missing = [attr for attr in required if attr not in data] + if missing: + raise AttributeError(f"Missing attributes: {', '.join(missing)}") + + if self.exclusive: + exclusives = [attr for attr in data if attr in self.exclusive] + if len(exclusives) > 1: + raise AttributeError( + f"Provide only one of these attributes: {', '.join(exclusives)}" + ) + if not exclusives: + raise AttributeError( + f"Must provide one of these attributes: " + f"{', '.join(self.exclusive)}" + ) + + +class EncodedId(str): + """URL-safe encoded identifier. + + Wraps a string or integer value with URL percent-encoding applied. + Idempotent: encoding an already-encoded value returns it unchanged. + + Args: + value: The raw identifier to encode. Accepts str, int, or EncodedId. + + Raises: + TypeError: If value is not str, int, or EncodedId. + + Example: + >>> EncodedId("hello world") + 'hello%20world' + >>> EncodedId(42) + '42' + """ + + def __new__(cls, value: "str | int | EncodedId") -> "EncodedId": + if isinstance(value, EncodedId): + return value + + if not isinstance(value, (int, str)): + raise TypeError(f"Unsupported type received: {type(value)}") + if isinstance(value, str): + value = urllib.parse.quote(value, safe="") + return super().__new__(cls, value) + + +def remove_none_from_dict(data: dict[str, Any]) -> dict[str, Any]: + """Remove all keys with None values from a dict. + + Args: + data: Source dictionary. + + Returns: + New dictionary with None-valued entries removed. + + Example: + >>> remove_none_from_dict({"a": 1, "b": None, "c": "x"}) + {'a': 1, 'c': 'x'} + """ + return {k: v for k, v in data.items() if v is not None} + + +def copy_dict(*, src: dict[str, Any], dest: dict[str, Any]) -> None: + """Flatten nested dict values into bracket-notation keys. + + For nested dicts, expands ``{"key": {"sub": "val"}}`` into + ``dest["key[sub]"] = "val"``. Non-dict values copy directly. + + Args: + src: Source dictionary to copy from. + dest: Destination dictionary to copy into (mutated in place). + + Example: + >>> dest = {} + >>> copy_dict(src={"filters": {"type": "malware"}}, dest=dest) + >>> dest + {'filters[type]': 'malware'} + """ + for k, v in src.items(): + if isinstance(v, dict): + for dict_k, dict_v in v.items(): + dest[f"{k}[{dict_k}]"] = dict_v + else: + dest[k] = v + + +class CustomJsonFormatter(JsonFormatter): + """JSON log formatter with ISO timestamp and uppercase level. + + Extends pythonjsonlogger's JsonFormatter to add: + - UTC ISO 8601 timestamp if not already present. + - Uppercase level field normalization. + """ + + def add_fields( + self, + log_record: dict[str, Any], + record: logging.LogRecord, + message_dict: dict[str, Any], + ) -> None: + """Add timestamp and level fields to the log record. + + Args: + log_record: The output dict being built. + record: The stdlib LogRecord. + message_dict: Extra message fields. + """ + super().add_fields(log_record, record, message_dict) + if not log_record.get("timestamp"): + # This doesn't use record.created, so it is slightly off + now = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + log_record["timestamp"] = now + if log_record.get("level"): + log_record["level"] = log_record["level"].upper() + else: + log_record["level"] = record.levelname + + +def setup_logging_config(level: int, json_logging: bool = True) -> None: + """Configure root logger with JSON or plain text output. + + Silences noisy third-party loggers (urllib3, pika) and sets up + the root logger with either structured JSON or basic text formatting. + + Args: + level: Logging level (e.g., logging.DEBUG, logging.INFO). + json_logging: If True, use CustomJsonFormatter for structured output. + If False, use basicConfig plain text. + """ + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("pika").setLevel(logging.ERROR) + if json_logging: + log_handler = logging.StreamHandler() + log_handler.setLevel(level) + formatter = CustomJsonFormatter("%(timestamp)s %(level)s %(name)s %(message)s") + log_handler.setFormatter(formatter) + logging.basicConfig(handlers=[log_handler], level=level, force=True) + else: + logging.basicConfig(level=level) + + +class AppLogger: + """Structured application logger with JSON support. + + Provides a simple logging interface with optional JSON formatting + and metadata attachment. Supports name-switching via __call__ for + per-module logger instances. + + Args: + level: Logging level (e.g., logging.DEBUG). + json_logging: Whether to use JSON-formatted output. + name: Logger name (defaults to module name). + + Example: + >>> log = AppLogger(logging.INFO) + >>> log.info("Server started", {"port": 8080}) + >>> module_log = log("my_module") + >>> module_log.debug("Processing request") + """ + + def __init__(self, level: int, json_logging: bool = True, name: str = __name__): + self.log_level = level + self.json_logging = json_logging + setup_logging_config(self.log_level, self.json_logging) + self.local_logger = logging.getLogger(name) + + def __call__(self, name: str) -> "AppLogger": + """Switch to a named logger instance. + + Args: + name: New logger name. + + Returns: + Self with updated logger name. + """ + self.local_logger = logging.getLogger(name) + return self + + @staticmethod + def prepare_meta(meta: Any = None) -> dict[str, Any] | None: + """Wrap metadata for structured logging extra field. + + Args: + meta: Arbitrary metadata to attach to log entry. + + Returns: + Dict with 'attributes' key, or None if no metadata. + """ + return None if meta is None else {"attributes": meta} + + @staticmethod + def setup_logger_level(lib: str, log_level: int) -> None: + """Override log level for a third-party library. + + Args: + lib: Library logger name (e.g., "urllib3"). + log_level: Desired logging level. + """ + logging.getLogger(lib).setLevel(log_level) + + def debug(self, message: str, meta: Any = None) -> None: + """Log at DEBUG level. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.debug(message, extra=AppLogger.prepare_meta(meta)) + + def info(self, message: str, meta: Any = None) -> None: + """Log at INFO level. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.info(message, extra=AppLogger.prepare_meta(meta)) + + def warning(self, message: str, meta: Any = None) -> None: + """Log at WARNING level. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.warning(message, extra=AppLogger.prepare_meta(meta)) + + def error(self, message: str, meta: Any = None) -> None: + """Log at ERROR level with traceback. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.error(message, exc_info=1, extra=AppLogger.prepare_meta(meta)) + + +def logger(level: int, json_logging: bool = True) -> AppLogger: + """Create an AppLogger instance. + + Args: + level: Logging level. + json_logging: Whether to use JSON output. + + Returns: + An AppLogger instance. + """ + return AppLogger(level, json_logging) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index ba499ef2..94883204 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -2,6 +2,43 @@ __version__ = "0.1.0" +# --- Common (shared) --- +from xtm_oaev_sdk.seam.common import ( + AppLogger, + AppLoggerProtocol, + AssetCategory, + AssetCriticality, + AssetSubCategory, + CloudProvider, + ConfigurationError, + CustomJsonFormatter, + EncodedId, + EnhancedJSONEncoder, + OpenAEVAuthenticationError, + OpenAEVCreateError, + OpenAEVDeleteError, + OpenAEVError, + OpenAEVGetError, + OpenAEVHeadError, + OpenAEVHttpError, + OpenAEVListError, + OpenAEVParsingError, + OpenAEVUpdateError, + RedirectError, + RequiredOptional, + SecurityDomains, + SignatureTransmissionError, + copy_dict, + data_to_temp_file, + is_memory_certificate, + logger, + on_http_error, + remove_none_from_dict, + setup_logging_config, + ssl_cert_chain, + ssl_verify_locations, +) + # --- Configuration --- from xtm_oaev_sdk.seam.configuration import ( CONFIGURATION_TYPES, @@ -27,6 +64,43 @@ __all__ = [ "__version__", + # Common: errors + "ConfigurationError", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SignatureTransmissionError", + "on_http_error", + # Common: utils + "AppLogger", + "AppLoggerProtocol", + "CustomJsonFormatter", + "EncodedId", + "EnhancedJSONEncoder", + "RequiredOptional", + "copy_dict", + "logger", + "remove_none_from_dict", + "setup_logging_config", + # Common: ssl + "data_to_temp_file", + "is_memory_certificate", + "ssl_cert_chain", + "ssl_verify_locations", + # Common: enums + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "CloudProvider", + "SecurityDomains", # Configuration "BaseConfigModel", "CONFIGURATION_TYPES", diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py index 9523baa1..f2dfca6c 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -3,6 +3,8 @@ Everything in this layer pins a stable interface so that the two sides of it can evolve independently without leaking either: +- **``common``** — shared errors, enums, types, and utilities: the + platform-facing vocabulary both sides import. - **``configuration``** — the configuration capability's I/O models and conversion primitives. - **``deprecation``** — the PEP 562 deprecation primitive diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py new file mode 100644 index 00000000..bc5955f9 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py @@ -0,0 +1,85 @@ +"""Shared contracts: errors, enums, type aliases, and the concrete utility +layer.""" + +from xtm_oaev_sdk._core.asset_types import ( + AssetCategory, + AssetCriticality, + AssetSubCategory, + CloudProvider, +) +from xtm_oaev_sdk._core.errors import ( + ConfigurationError, + OpenAEVAuthenticationError, + OpenAEVCreateError, + OpenAEVDeleteError, + OpenAEVError, + OpenAEVGetError, + OpenAEVHeadError, + OpenAEVHttpError, + OpenAEVListError, + OpenAEVParsingError, + OpenAEVUpdateError, + RedirectError, + SignatureTransmissionError, + on_http_error, +) +from xtm_oaev_sdk._core.security_domain import SecurityDomains +from xtm_oaev_sdk._core.ssl_utils import ( + data_to_temp_file, + is_memory_certificate, + ssl_cert_chain, + ssl_verify_locations, +) +from xtm_oaev_sdk._core.utils import ( + AppLogger, + AppLoggerProtocol, + CustomJsonFormatter, + EncodedId, + EnhancedJSONEncoder, + RequiredOptional, + copy_dict, + logger, + remove_none_from_dict, + setup_logging_config, +) + +__all__ = [ + # Errors + "ConfigurationError", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SignatureTransmissionError", + "on_http_error", + # Protocols + "AppLoggerProtocol", + # Types + "EncodedId", + "RequiredOptional", + # Enums + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "CloudProvider", + "SecurityDomains", + # Behaviour (concrete adapters) + "AppLogger", + "CustomJsonFormatter", + "EnhancedJSONEncoder", + "copy_dict", + "data_to_temp_file", + "is_memory_certificate", + "logger", + "remove_none_from_dict", + "setup_logging_config", + "ssl_cert_chain", + "ssl_verify_locations", +] diff --git a/pyoaev/__init__.py b/pyoaev/__init__.py index 14a7f808..db714285 100644 --- a/pyoaev/__init__.py +++ b/pyoaev/__init__.py @@ -4,7 +4,7 @@ import xtm_oaev_sdk as _sdk from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr -from pyoaev import configuration # noqa: F401 +from pyoaev import asset_types, configuration, exceptions # noqa: F401 from pyoaev._version import ( # noqa: F401 __author__, __copyright__, @@ -12,18 +12,11 @@ __license__, __title__, ) -from pyoaev.asset_types import ( # noqa: F401 - AssetCategory, - AssetCriticality, - AssetSubCategory, - CloudProvider, -) from pyoaev.client import OpenAEV # noqa: F401 from pyoaev.contracts import * # noqa: F401,F403,F405 -from pyoaev.exceptions import * # noqa: F401,F403,F405 from pyoaev.signatures import * # noqa: F401,F403,F405 -__all__ = [ +__all__ = [ # noqa: F405 (moved names are served by the PEP 562 __getattr__ below) "__author__", "__copyright__", "__email__", @@ -36,14 +29,14 @@ "CloudProvider", "AssetCriticality", ] -__all__.extend(exceptions.__all__) # noqa: F405 +__all__.extend(exceptions.__all__) # Names moved to xtm_oaev_sdk: served lazily (PEP 562) so that `import pyoaev` # stays silent and only legacy access to a moved name warns. _deprecated_getattr = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) - for name in configuration.__all__ + for name in (*asset_types.__all__, *configuration.__all__, *exceptions.__all__) }, module_name=__name__, ) diff --git a/pyoaev/apis/ai_target.py b/pyoaev/apis/ai_target.py index c1dfdf91..56ae892b 100644 --- a/pyoaev/apis/ai_target.py +++ b/pyoaev/apis/ai_target.py @@ -1,30 +1,31 @@ -from pyoaev.base import RESTManager, RESTObject -from pyoaev.mixins import CreateMixin, DeleteMixin, GetMixin, ListMixin, UpdateMixin -from pyoaev.utils import RequiredOptional - - -class AiTarget(RESTObject): - _id_attr = "asset_id" - - -class AiTargetManager( - GetMixin, ListMixin, CreateMixin, UpdateMixin, DeleteMixin, RESTManager -): - """Manage AI Target assets (LLM endpoints / AI agents under adversarial test).""" - - _path = "/ai_targets" - _obj_cls = AiTarget - _create_attrs = RequiredOptional( - required=("asset_name", "ai_target_provider"), - optional=( - "asset_description", - "asset_tags", - "asset_external_reference", - "ai_target_endpoint", - "ai_target_model", - "ai_target_modality", - "ai_target_system_prompt", - "ai_target_token", - "ai_target_configuration", - ), - ) +from xtm_oaev_sdk import RequiredOptional + +from pyoaev.base import RESTManager, RESTObject +from pyoaev.mixins import CreateMixin, DeleteMixin, GetMixin, ListMixin, UpdateMixin + + +class AiTarget(RESTObject): + _id_attr = "asset_id" + + +class AiTargetManager( + GetMixin, ListMixin, CreateMixin, UpdateMixin, DeleteMixin, RESTManager +): + """Manage AI Target assets (LLM endpoints / AI agents under adversarial test).""" + + _path = "/ai_targets" + _obj_cls = AiTarget + _create_attrs = RequiredOptional( + required=("asset_name", "ai_target_provider"), + optional=( + "asset_description", + "asset_tags", + "asset_external_reference", + "ai_target_endpoint", + "ai_target_model", + "ai_target_modality", + "ai_target_system_prompt", + "ai_target_token", + "ai_target_configuration", + ), + ) diff --git a/pyoaev/apis/attack_pattern.py b/pyoaev/apis/attack_pattern.py index 89b800b9..2d96fde5 100644 --- a/pyoaev/apis/attack_pattern.py +++ b/pyoaev/apis/attack_pattern.py @@ -1,6 +1,7 @@ from typing import Any, Dict, List -from pyoaev import exceptions as exc +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/collector.py b/pyoaev/apis/collector.py index 69ff43fb..a134bff4 100644 --- a/pyoaev/apis/collector.py +++ b/pyoaev/apis/collector.py @@ -1,9 +1,10 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk import RequiredOptional +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, GetMixin, ListMixin, UpdateMixin -from pyoaev.utils import RequiredOptional class Collector(RESTObject): diff --git a/pyoaev/apis/cve.py b/pyoaev/apis/cve.py index 45145f2b..50a9d8fc 100644 --- a/pyoaev/apis/cve.py +++ b/pyoaev/apis/cve.py @@ -1,6 +1,7 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/document.py b/pyoaev/apis/document.py index 38f3ab8a..98c41225 100644 --- a/pyoaev/apis/document.py +++ b/pyoaev/apis/document.py @@ -1,6 +1,7 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/endpoint.py b/pyoaev/apis/endpoint.py index 543ccb14..2a1b5452 100644 --- a/pyoaev/apis/endpoint.py +++ b/pyoaev/apis/endpoint.py @@ -1,9 +1,10 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk import RequiredOptional +from xtm_oaev_sdk.seam import common as exc + from pyoaev.apis.inputs.search import SearchPaginationInput from pyoaev.base import RESTManager, RESTObject -from pyoaev.utils import RequiredOptional class Endpoint(RESTObject): diff --git a/pyoaev/apis/inject.py b/pyoaev/apis/inject.py index 7fb6add1..73a1abe9 100644 --- a/pyoaev/apis/inject.py +++ b/pyoaev/apis/inject.py @@ -1,6 +1,7 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/inject_expectation/inject_expectation.py b/pyoaev/apis/inject_expectation/inject_expectation.py index be1c453a..370b4ef8 100644 --- a/pyoaev/apis/inject_expectation/inject_expectation.py +++ b/pyoaev/apis/inject_expectation/inject_expectation.py @@ -1,6 +1,8 @@ from typing import Any, Dict, List -from pyoaev import exceptions as exc +from xtm_oaev_sdk import RequiredOptional +from xtm_oaev_sdk.seam import common as exc + from pyoaev.apis.inject_expectation.model import ( DetectionExpectation, ExpectationTypeEnum, @@ -8,7 +10,6 @@ ) from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import ListMixin, UpdateMixin -from pyoaev.utils import RequiredOptional class InjectExpectation(RESTObject): diff --git a/pyoaev/apis/inject_expectation_trace.py b/pyoaev/apis/inject_expectation_trace.py index ac9e154d..2792839e 100644 --- a/pyoaev/apis/inject_expectation_trace.py +++ b/pyoaev/apis/inject_expectation_trace.py @@ -1,9 +1,10 @@ from typing import Any, Dict, List -from pyoaev import exceptions as exc +from xtm_oaev_sdk import RequiredOptional +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin -from pyoaev.utils import RequiredOptional class InjectExpectationTrace(RESTObject): diff --git a/pyoaev/apis/injector.py b/pyoaev/apis/injector.py index c6ea0d8d..f760dc4b 100644 --- a/pyoaev/apis/injector.py +++ b/pyoaev/apis/injector.py @@ -1,6 +1,7 @@ +from xtm_oaev_sdk import RequiredOptional + from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, GetMixin, ListMixin, UpdateMixin -from pyoaev.utils import RequiredOptional class Injector(RESTObject): diff --git a/pyoaev/apis/injector_contract.py b/pyoaev/apis/injector_contract.py index 8be83c61..2147a289 100644 --- a/pyoaev/apis/injector_contract.py +++ b/pyoaev/apis/injector_contract.py @@ -1,10 +1,11 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk import RequiredOptional +from xtm_oaev_sdk.seam import common as exc + from pyoaev.apis.inputs.search import InjectorContractSearchPaginationInput from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, DeleteMixin, UpdateMixin -from pyoaev.utils import RequiredOptional class InjectorContract(RESTObject): diff --git a/pyoaev/apis/kill_chain_phase.py b/pyoaev/apis/kill_chain_phase.py index a40ebb22..9a565051 100644 --- a/pyoaev/apis/kill_chain_phase.py +++ b/pyoaev/apis/kill_chain_phase.py @@ -1,6 +1,7 @@ from typing import Any, Dict, List -from pyoaev import exceptions as exc +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/payload.py b/pyoaev/apis/payload.py index 3de5b0f3..cad3e645 100644 --- a/pyoaev/apis/payload.py +++ b/pyoaev/apis/payload.py @@ -1,6 +1,7 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/security_platform.py b/pyoaev/apis/security_platform.py index 184474d8..2c22edd8 100644 --- a/pyoaev/apis/security_platform.py +++ b/pyoaev/apis/security_platform.py @@ -1,9 +1,10 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk import RequiredOptional +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, GetMixin, ListMixin, UpdateMixin -from pyoaev.utils import RequiredOptional class SecurityPlatform(RESTObject): diff --git a/pyoaev/apis/signature.py b/pyoaev/apis/signature.py index 0fe19719..b03793f7 100644 --- a/pyoaev/apis/signature.py +++ b/pyoaev/apis/signature.py @@ -6,10 +6,10 @@ from typing import Any from pydantic import ValidationError +from xtm_oaev_sdk import SignatureTransmissionError +from xtm_oaev_sdk.seam import common as exc -from pyoaev import exceptions as exc from pyoaev.base import RESTManager, RESTObject -from pyoaev.exceptions import SignatureTransmissionError from pyoaev.signatures.models import ( ExecutionDetails, SignatureCallbackPayload, @@ -243,7 +243,7 @@ def _send_with_retry( Raises: SignatureTransmissionError: 4xx error or all retries exhausted. """ - from pyoaev.exceptions import OpenAEVError + from xtm_oaev_sdk import OpenAEVError effective_logger = logger if logger is not None else self._logger last_error: Exception | None = None diff --git a/pyoaev/apis/tag.py b/pyoaev/apis/tag.py index e112b0c6..8ee34127 100644 --- a/pyoaev/apis/tag.py +++ b/pyoaev/apis/tag.py @@ -1,6 +1,7 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/team.py b/pyoaev/apis/team.py index a4c68a19..82b9a00a 100644 --- a/pyoaev/apis/team.py +++ b/pyoaev/apis/team.py @@ -1,9 +1,10 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk import RequiredOptional +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, ListMixin, UpdateMixin -from pyoaev.utils import RequiredOptional class Team(RESTObject): diff --git a/pyoaev/apis/user.py b/pyoaev/apis/user.py index 64ac0db0..e3fff082 100644 --- a/pyoaev/apis/user.py +++ b/pyoaev/apis/user.py @@ -1,9 +1,10 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk import RequiredOptional +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, ListMixin, UpdateMixin -from pyoaev.utils import RequiredOptional class User(RESTObject): diff --git a/pyoaev/apis/vulnerability.py b/pyoaev/apis/vulnerability.py index 5418afce..587001ac 100644 --- a/pyoaev/apis/vulnerability.py +++ b/pyoaev/apis/vulnerability.py @@ -1,6 +1,7 @@ from typing import Any, Dict -from pyoaev import exceptions as exc +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/asset_types.py b/pyoaev/asset_types.py index b8e0b418..89f1250b 100644 --- a/pyoaev/asset_types.py +++ b/pyoaev/asset_types.py @@ -1,135 +1,29 @@ -# -*- coding: utf-8 -*- -"""Canonical asset taxonomy for OpenAEV (mirrors the backend enums). - -These give producers (collectors, custom integrations) a single, typed vocabulary for the -two-level asset categorization (category + subcategory), criticality and cloud provider, instead -of free-form strings. They are plain ``str`` enums so they can be dropped straight into the dict -payloads sent to the API. -""" - -from enum import Enum - - -class AssetCategory(str, Enum): - HOST = "HOST" - CONTAINER_WORKLOAD = "CONTAINER_WORKLOAD" - CLOUD_RESOURCE = "CLOUD_RESOURCE" - WEB_APPLICATION = "WEB_APPLICATION" - NETWORK_DEVICE = "NETWORK_DEVICE" - MOBILE_DEVICE = "MOBILE_DEVICE" - IOT_OT_DEVICE = "IOT_OT_DEVICE" - IDENTITY = "IDENTITY" - SAAS_APPLICATION = "SAAS_APPLICATION" - AI_TARGET = "AI_TARGET" - SECURITY_PLATFORM = "SECURITY_PLATFORM" - GENERIC_ASSET = "GENERIC_ASSET" - - -class AssetSubCategory(str, Enum): - # HOST - SERVER = "SERVER" - WORKSTATION = "WORKSTATION" - LAPTOP = "LAPTOP" - VIRTUAL_MACHINE = "VIRTUAL_MACHINE" - HYPERVISOR = "HYPERVISOR" - MAINFRAME = "MAINFRAME" - THIN_CLIENT = "THIN_CLIENT" - # CONTAINER_WORKLOAD - CONTAINER = "CONTAINER" - CONTAINER_IMAGE = "CONTAINER_IMAGE" - KUBERNETES_POD = "KUBERNETES_POD" - KUBERNETES_CLUSTER = "KUBERNETES_CLUSTER" - KUBERNETES_NODE = "KUBERNETES_NODE" - SERVERLESS_FUNCTION = "SERVERLESS_FUNCTION" - # CLOUD_RESOURCE - COMPUTE = "COMPUTE" - STORAGE = "STORAGE" - DATABASE = "DATABASE" - NETWORKING = "NETWORKING" - SERVERLESS = "SERVERLESS" - CONTAINER_REGISTRY = "CONTAINER_REGISTRY" - KUBERNETES = "KUBERNETES" - IAM_PRINCIPAL = "IAM_PRINCIPAL" - SECRETS_KEY_MGMT = "SECRETS_KEY_MGMT" - MESSAGING_QUEUE = "MESSAGING_QUEUE" - ANALYTICS_DATA = "ANALYTICS_DATA" - AI_ML_SERVICE = "AI_ML_SERVICE" - IAC_TEMPLATE = "IAC_TEMPLATE" - CLOUD_OTHER = "CLOUD_OTHER" - # WEB_APPLICATION - WEBSITE = "WEBSITE" - WEB_API = "WEB_API" - SINGLE_PAGE_APP = "SINGLE_PAGE_APP" - GRAPHQL_API = "GRAPHQL_API" - WEB_SERVICE = "WEB_SERVICE" - MICROSERVICE = "MICROSERVICE" - # NETWORK_DEVICE - ROUTER = "ROUTER" - SWITCH = "SWITCH" - FIREWALL = "FIREWALL" - LOAD_BALANCER = "LOAD_BALANCER" - VPN_GATEWAY = "VPN_GATEWAY" - WIRELESS_AP = "WIRELESS_AP" - PROXY = "PROXY" - DNS_SERVER = "DNS_SERVER" - DHCP_SERVER = "DHCP_SERVER" - SAN_NAS = "SAN_NAS" - NETWORK_OTHER = "NETWORK_OTHER" - # MOBILE_DEVICE - SMARTPHONE = "SMARTPHONE" - TABLET = "TABLET" - # IOT_OT_DEVICE - IOT_SENSOR = "IOT_SENSOR" - IP_CAMERA = "IP_CAMERA" - GATEWAY = "GATEWAY" - POINT_OF_SALE = "POINT_OF_SALE" - MEDIA_DEVICE = "MEDIA_DEVICE" - PLC = "PLC" - RTU = "RTU" - HMI = "HMI" - SCADA_HISTORIAN = "SCADA_HISTORIAN" - MEDICAL_DEVICE = "MEDICAL_DEVICE" - PRINTER_PERIPHERAL = "PRINTER_PERIPHERAL" - BUILDING_MGMT = "BUILDING_MGMT" - # IDENTITY - USER_ACCOUNT = "USER_ACCOUNT" - SERVICE_ACCOUNT = "SERVICE_ACCOUNT" - GROUP = "GROUP" - ROLE = "ROLE" - SHARED_MAILBOX = "SHARED_MAILBOX" - NON_HUMAN_IDENTITY = "NON_HUMAN_IDENTITY" - # SAAS_APPLICATION - SAAS_APP = "SAAS_APP" - SAAS_TENANT = "SAAS_TENANT" - # AI_TARGET - LLM_MODEL = "LLM_MODEL" - AI_AGENT = "AI_AGENT" - MCP_SERVER = "MCP_SERVER" - RAG_PIPELINE = "RAG_PIPELINE" - # SECURITY_PLATFORM - EDR = "EDR" - XDR = "XDR" - SIEM = "SIEM" - SOAR = "SOAR" - NDR = "NDR" - ISPM = "ISPM" - LLM_FIREWALL = "LLM_FIREWALL" - AI_GATEWAY = "AI_GATEWAY" - - -class CloudProvider(str, Enum): - AWS = "AWS" - AZURE = "AZURE" - GCP = "GCP" - OCI = "OCI" - ALIBABA = "ALIBABA" - KUBERNETES = "KUBERNETES" - OTHER = "OTHER" - - -class AssetCriticality(str, Enum): - VERY_HIGH = "VERY_HIGH" - HIGH = "HIGH" - MEDIUM = "MEDIUM" - LOW = "LOW" - UNKNOWN = "UNKNOWN" +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "AssetCategory", + "AssetSubCategory", + "CloudProvider", + "AssetCriticality", +] + +_MOVED = __all__ + +_deprecated_getattr = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in _MOVED + }, + module_name=__name__, +) + + +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) diff --git a/pyoaev/base.py b/pyoaev/base.py index 10fd2f20..8576eed5 100644 --- a/pyoaev/base.py +++ b/pyoaev/base.py @@ -15,9 +15,8 @@ get_type_hints, ) -from pyoaev.exceptions import OpenAEVParsingError +from xtm_oaev_sdk import EncodedId, OpenAEVParsingError -from . import utils from .client import OpenAEV, OpenAEVList __all__ = [ @@ -215,7 +214,7 @@ def encoded_id(self) -> Optional[Union[int, str]]: path""" obj_id = self.get_id() if isinstance(obj_id, str): - obj_id = utils.EncodedId(obj_id) + obj_id = EncodedId(obj_id) return obj_id @@ -316,12 +315,12 @@ def _compute_path(self, path: Optional[str] = None) -> Optional[str]: if self._parent is None or not self._from_parent_attrs: return path - data: Dict[str, Optional[utils.EncodedId]] = {} + data: Dict[str, Optional[EncodedId]] = {} for self_attr, parent_attr in self._from_parent_attrs.items(): if not hasattr(self._parent, parent_attr): data[self_attr] = None continue - data[self_attr] = utils.EncodedId(getattr(self._parent, parent_attr)) + data[self_attr] = EncodedId(getattr(self._parent, parent_attr)) self._parent_attrs = data return path.format(**data) diff --git a/pyoaev/client.py b/pyoaev/client.py index f0f295f7..3f39144b 100644 --- a/pyoaev/client.py +++ b/pyoaev/client.py @@ -3,8 +3,10 @@ from uuid import UUID import requests +from xtm_oaev_sdk import copy_dict +from xtm_oaev_sdk.seam import common as exceptions -from pyoaev import exceptions, utils +from pyoaev import utils from pyoaev._version import __version__ # noqa: F401 REDIRECT_MSG = ( @@ -170,17 +172,17 @@ def http_request( # parse user-provided URL params to ensure we don't add our own duplicates parsed = parse.urlparse(raw_url) params = parse.parse_qs(parsed.query) - utils.copy_dict(src=query_data, dest=params) + copy_dict(src=query_data, dest=params) url = parse.urlunparse(parsed._replace(query="")) if "query_parameters" in kwargs: - utils.copy_dict(src=kwargs["query_parameters"], dest=params) + copy_dict(src=kwargs["query_parameters"], dest=params) for arg in ("per_page", "page"): if arg in kwargs: params[arg] = kwargs[arg] else: - utils.copy_dict(src=kwargs, dest=params) + copy_dict(src=kwargs, dest=params) opts = self._get_session_opts() diff --git a/pyoaev/contracts/contract_config.py b/pyoaev/contracts/contract_config.py index 7d5e95e6..8217194d 100644 --- a/pyoaev/contracts/contract_config.py +++ b/pyoaev/contracts/contract_config.py @@ -4,7 +4,8 @@ from enum import Enum from typing import Dict, List, Optional -from pyoaev import utils +from xtm_oaev_sdk import EnhancedJSONEncoder + from pyoaev.contracts.contract_utils import ContractCardinality, ContractVariable from pyoaev.contracts.variable_helper import VariableHelper from pyoaev.credential.types import CredentialType @@ -276,7 +277,7 @@ def to_contract_add_input(self, source_id: str): "contract_labels": self.label, "contract_attack_patterns_external_ids": self.contract_attack_patterns_external_ids, "contract_vulnerability_external_ids": self.contract_vulnerability_external_ids, - "contract_content": json.dumps(self, cls=utils.EnhancedJSONEncoder), + "contract_content": json.dumps(self, cls=EnhancedJSONEncoder), "is_atomic_testing": self.is_atomic_testing, "contract_platforms": self.platforms, "contract_domains": self.domains, @@ -288,7 +289,7 @@ def to_contract_update_input(self): "contract_labels": self.label, "contract_attack_patterns_external_ids": self.contract_attack_patterns_external_ids, "contract_vulnerability_external_ids": self.contract_vulnerability_external_ids, - "contract_content": json.dumps(self, cls=utils.EnhancedJSONEncoder), + "contract_content": json.dumps(self, cls=EnhancedJSONEncoder), "is_atomic_testing": self.is_atomic_testing, "contract_platforms": self.platforms, "contract_domains": self.domains, @@ -318,7 +319,7 @@ def prepare_contracts(contracts): "contract_id": c.contract_id, "contract_labels": c.label, "contract_attack_patterns_external_ids": c.contract_attack_patterns_external_ids, - "contract_content": json.dumps(c, cls=utils.EnhancedJSONEncoder), + "contract_content": json.dumps(c, cls=EnhancedJSONEncoder), "contract_platforms": c.platforms, "contract_domains": c.domains, }, diff --git a/pyoaev/daemons/base_daemon.py b/pyoaev/daemons/base_daemon.py index e2d5868f..049bed5b 100644 --- a/pyoaev/daemons/base_daemon.py +++ b/pyoaev/daemons/base_daemon.py @@ -4,11 +4,9 @@ from types import FunctionType from uuid import UUID -from xtm_oaev_sdk import Configuration +from xtm_oaev_sdk import Configuration, OpenAEVError, logger from pyoaev.client import OpenAEV -from pyoaev.exceptions import OpenAEVError -from pyoaev.utils import logger class BaseDaemon(ABC): diff --git a/pyoaev/daemons/collector_daemon.py b/pyoaev/daemons/collector_daemon.py index 642f7441..55473144 100644 --- a/pyoaev/daemons/collector_daemon.py +++ b/pyoaev/daemons/collector_daemon.py @@ -1,7 +1,8 @@ import sched import time -from pyoaev import Configuration +from xtm_oaev_sdk import Configuration + from pyoaev.daemons import BaseDaemon from pyoaev.utils import PingAlive diff --git a/pyoaev/exceptions.py b/pyoaev/exceptions.py index cca8e842..889cb6fd 100644 --- a/pyoaev/exceptions.py +++ b/pyoaev/exceptions.py @@ -1,222 +1,16 @@ -import functools -from typing import TYPE_CHECKING, Any, Callable, Optional, Type, TypeVar, Union, cast +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -class OpenAEVError(Exception): - def __init__( - self, - error_message: Union[str, bytes] = "", - response_code: Optional[int] = None, - response_body: Optional[bytes] = None, - ) -> None: - Exception.__init__(self, error_message) - # Http status code - self.response_code = response_code - # Full http response - self.response_body = response_body - try: - # if we receive str/bytes we try to convert to unicode/str to have - # consistent message types (see #616) - if TYPE_CHECKING: - assert isinstance(error_message, bytes) - self.error_message = error_message.decode() - except Exception: - if TYPE_CHECKING: - assert isinstance(error_message, str) - self.error_message = error_message +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr - def __str__(self) -> str: - # Start with the provided error message - message = self.error_message - - # List of generic HTTP status messages that indicate we should look deeper - generic_messages = ( - "Internal Server Error", - "Bad Request", - "Not Found", - "Unauthorized", - "Forbidden", - "Service Unavailable", - "Gateway Timeout", - "Unknown error", - "Validation Failed", - ) - - # Only try to extract from response body if message is truly generic - # Don't override if we already have a specific error message - if ( - not message or (message in generic_messages and len(message) < 30) - ) and self.response_body is not None: - try: - import json - - body = self.response_body.decode(errors="ignore") - data = json.loads(body) - extracted_msg = None - - if isinstance(data, dict): - # Try various common error fields - if "error" in data: - err = data.get("error") - if isinstance(err, dict) and "message" in err: - extracted_msg = err.get("message") - elif isinstance(err, str): - extracted_msg = err - elif "message" in data: - extracted_msg = data.get("message") - elif "execution_message" in data: - extracted_msg = data.get("execution_message") - elif "detail" in data: - extracted_msg = data.get("detail") - elif "errors" in data: - errs = data.get("errors") - if isinstance(errs, list) and errs: - # Join any messages in the list - parts = [] - for item in errs: - if isinstance(item, dict) and "message" in item: - parts.append(str(item.get("message"))) - else: - parts.append(str(item)) - extracted_msg = "; ".join(parts) - elif isinstance(errs, dict): - # Handle nested validation errors structure - if "children" in errs: - validation_errors = [] - children = errs.get("children", {}) - for field, field_errors in children.items(): - if ( - isinstance(field_errors, dict) - and "errors" in field_errors - ): - field_error_list = field_errors.get( - "errors", [] - ) - if field_error_list: - for err_msg in field_error_list: - validation_errors.append( - f"{field}: {err_msg}" - ) - if validation_errors: - base_msg = data.get("message", "Validation Failed") - extracted_msg = ( - f"{base_msg}: {'; '.join(validation_errors)}" - ) - else: - # Try to get any string representation - parts = [] - for key, value in errs.items(): - if value: - parts.append(f"{key}: {value}") - if parts: - extracted_msg = "; ".join(parts) - elif isinstance(errs, str): - extracted_msg = errs - - # Use extracted message if it's better than what we have - if extracted_msg and extracted_msg not in generic_messages: - message = str(extracted_msg) - elif not message: - # Last resort: use the raw body - message = body[:500] - - except json.JSONDecodeError: - # Not JSON, use raw text if we don't have a good message - if not message or message in generic_messages: - try: - decoded = self.response_body.decode(errors="ignore")[:500] - if decoded and decoded not in generic_messages: - message = decoded - except Exception: - pass - except Exception: - pass - - # Final fallback - if not message: - message = "Unknown error" - - # Clean up the message - remove extra whitespace and newlines - message = " ".join(message.split()) - - if self.response_code is not None: - return f"{self.response_code}: {message}" - return f"{message}" - - -class OpenAEVAuthenticationError(OpenAEVError): - pass - - -class OpenAEVHttpError(OpenAEVError): - pass - - -class OpenAEVParsingError(OpenAEVError): - pass - - -class RedirectError(OpenAEVError): - pass - - -class OpenAEVHeadError(OpenAEVError): - pass - - -class OpenAEVGetError(OpenAEVError): - pass - - -class OpenAEVUpdateError(OpenAEVError): - pass - - -class OpenAEVListError(OpenAEVError): - pass - - -class OpenAEVCreateError(OpenAEVError): - pass - - -class OpenAEVDeleteError(OpenAEVError): - pass - - -class SignatureTransmissionError(OpenAEVError): - """Signatures didn't make it. Validation rejected them, 4xx slammed the door, or retries ran dry.""" - - pass - - -class ConfigurationError(OpenAEVError): - pass - - -# For an explanation of how these type-hints work see: -# https://mypy.readthedocs.io/en/stable/generics.html#declaring-decorators -# -# The goal here is that functions which get decorated will retain their types. -__F = TypeVar("__F", bound=Callable[..., Any]) - - -def on_http_error(error: Type[Exception]) -> Callable[[__F], __F]: - def wrap(f: __F) -> __F: - @functools.wraps(f) - def wrapped_f(*args: Any, **kwargs: Any) -> Any: - try: - return f(*args, **kwargs) - except OpenAEVHttpError as e: - raise error(e.error_message, e.response_code, e.response_body) from e - - return cast(__F, wrapped_f) - - return wrap - - -# Export manually to keep mypy happy -__all__ = [ +# Unchanged public export list of the legacy module (also extended into +# ``pyoaev.__all__``); ``OpenAEVError`` and ``on_http_error`` were reachable +# but never exported. +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) "ConfigurationError", "OpenAEVAuthenticationError", "OpenAEVHttpError", @@ -230,3 +24,17 @@ def wrapped_f(*args: Any, **kwargs: Any) -> Any: "OpenAEVDeleteError", "SignatureTransmissionError", ] + +_MOVED = [*__all__, "OpenAEVError", "on_http_error"] + +_deprecated_getattr = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in _MOVED + }, + module_name=__name__, +) + + +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) diff --git a/pyoaev/helpers.py b/pyoaev/helpers.py index 23b3cb34..0eb36abd 100644 --- a/pyoaev/helpers.py +++ b/pyoaev/helpers.py @@ -5,76 +5,45 @@ import re import sched import ssl -import tempfile import threading import time import traceback from typing import Callable, Dict, List import pika +import xtm_oaev_sdk as _sdk from thefuzz import fuzz -from xtm_oaev_sdk import Configuration +from xtm_oaev_sdk import ( + Configuration, + ConfigurationError, + logger, + make_deprecated_getattr, +) from pyoaev import OpenAEV, utils from pyoaev.daemons import CollectorDaemon -from pyoaev.exceptions import ConfigurationError TRUTHY: List[str] = ["yes", "true", "True"] FALSY: List[str] = ["no", "false", "False"] +# SSL helpers moved to xtm_oaev_sdk: legacy names resolve to the identical SDK +# objects and warn on access (PEP 562). +_deprecated_getattr = make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in ( + "data_to_temp_file", + "is_memory_certificate", + "ssl_cert_chain", + "ssl_verify_locations", + ) + }, + module_name=__name__, +) -# As cert must be written in files to be loaded in ssl context -# Creates a temporary file in the most secure manner possible -def data_to_temp_file(data): - # The file is readable and writable only by the creating user ID. - # If the operating system uses permission bits to indicate whether a - # file is executable, the file is executable by no one. The file - # descriptor is not inherited by children of this process. - file_descriptor, file_path = tempfile.mkstemp() - with os.fdopen(file_descriptor, "w") as open_file: - open_file.write(data) - open_file.close() - return file_path - - -def is_memory_certificate(certificate): - return certificate.startswith("-----BEGIN") - - -def ssl_cert_chain(ssl_context, cert_data, key_data, passphrase): - if cert_data is None: - return - - cert_file_path = None - key_file_path = None - - # Cert loading - if cert_data is not None and is_memory_certificate(cert_data): - cert_file_path = data_to_temp_file(cert_data) - cert = cert_file_path if cert_file_path is not None else cert_data - - # Key loading - if key_data is not None and is_memory_certificate(key_data): - key_file_path = data_to_temp_file(key_data) - key = key_file_path if key_file_path is not None else key_data - - # Load cert - ssl_context.load_cert_chain(cert, key, passphrase) - # Remove temp files - if cert_file_path is not None: - os.unlink(cert_file_path) - if key_file_path is not None: - os.unlink(key_file_path) - - -def ssl_verify_locations(ssl_context, certdata): - if certdata is None: - return - if is_memory_certificate(certdata): - ssl_context.load_verify_locations(cadata=certdata) - else: - ssl_context.load_verify_locations(cafile=certdata) +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) def create_mq_ssl_context(config) -> ssl.SSLContext: @@ -115,10 +84,10 @@ def create_mq_ssl_context(config) -> ssl.SSLContext: if not use_ssl_reject_unauthorized: # noinspection PyUnresolvedReferences,PyProtectedMember ssl_context = ssl._create_unverified_context() - ssl_verify_locations(ssl_context, use_ssl_ca) + _sdk.ssl_verify_locations(ssl_context, use_ssl_ca) # Thanks to https://bugs.python.org/issue16487 is not possible today to easily use memory pem # in SSL context. We need to write it to a temporary file before - ssl_cert_chain(ssl_context, use_ssl_cert, use_ssl_key, use_ssl_passphrase) + _sdk.ssl_cert_chain(ssl_context, use_ssl_cert, use_ssl_key, use_ssl_passphrase) return ssl_context @@ -294,7 +263,7 @@ def __init__( f"DEPRECATED: this collector should be migrated to use {CollectorDaemon}." ) - self.logger_class = utils.logger( + self.logger_class = logger( config.get_conf("collector_log_level", default="error").upper(), config.get_conf("collector_json_logging", default=True), ) @@ -344,7 +313,7 @@ def __init__(self, config: OpenAEVConfigHelper, icon) -> None: "injector_author": config.get_conf("injector_author", default=None), } - self.logger_class = utils.logger( + self.logger_class = logger( config.get_conf("injector_log_level", default="error").upper(), config.get_conf("injector_json_logging", default=True), ) diff --git a/pyoaev/mixins.py b/pyoaev/mixins.py index 6309e311..c07bea11 100644 --- a/pyoaev/mixins.py +++ b/pyoaev/mixins.py @@ -12,11 +12,11 @@ ) import requests +from xtm_oaev_sdk import EncodedId +from xtm_oaev_sdk.seam import common as exc import pyoaev from pyoaev import base -from pyoaev import exceptions as exc -from pyoaev import utils __all__ = [ "GetMixin", @@ -42,7 +42,7 @@ def head( path = self.path if id is not None: - path = f"{path}/{utils.EncodedId(id)}" + path = f"{path}/{EncodedId(id)}" return self.openaev.http_head(path, **kwargs) @@ -60,7 +60,7 @@ class GetMixin(HeadMixin, _RestManagerBase): @exc.on_http_error(exc.OpenAEVGetError) def get(self, id: Union[str, int], **kwargs: Any) -> base.RESTObject: if isinstance(id, str): - id = utils.EncodedId(id) + id = EncodedId(id) path = f"{self.path}/{id}" if TYPE_CHECKING: assert self._obj_cls is not None @@ -171,7 +171,7 @@ def update( if id is None: path = self.path else: - path = f"{self.path}/{utils.EncodedId(id)}" + path = f"{self.path}/{EncodedId(id)}" # excludes = [] # if self._obj_cls is not None and self._obj_cls._id_attr is not None: @@ -234,7 +234,7 @@ def delete( if id is None: path = self.path else: - path = f"{self.path}/{utils.EncodedId(id)}" + path = f"{self.path}/{EncodedId(id)}" result = self.openaev.http_delete(path, **kwargs) if TYPE_CHECKING: diff --git a/pyoaev/security_domain/types.py b/pyoaev/security_domain/types.py index c48e48c1..41920222 100644 --- a/pyoaev/security_domain/types.py +++ b/pyoaev/security_domain/types.py @@ -1,20 +1,26 @@ -from enum import Enum +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -class SecurityDomains(Enum): - ENDPOINT = {"domain_name": "Endpoint", "domain_color": "#389CFF"} - NETWORK = {"domain_name": "Network", "domain_color": "#009933"} - WEB_APP = {"domain_name": "Web App", "domain_color": "#FF9933"} - EMAIL_INFILTRATION = { - "domain_name": "E-mail Infiltration", - "domain_color": "#FF6666", - } - DATA_EXFILTRATION = {"domain_name": "Data Exfiltration", "domain_color": "#9933CC"} - URL_FILTERING = {"domain_name": "URL Filtering", "domain_color": "#66CCFF"} - CLOUD = {"domain_name": "Cloud", "domain_color": "#9999CC"} - TABLE_TOP = {"domain_name": "Tabletop", "domain_color": "#FFCC33"} - ARTIFICIAL_INTELLIGENCE = { - "domain_name": "Artificial Intelligence", - "domain_color": "#7C4DFF", - } - TOCLASSIFY = {"domain_name": "To classify", "domain_color": "#FFFFFF"} +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "SecurityDomains", +] + +_MOVED = __all__ + +_deprecated_getattr = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in _MOVED + }, + module_name=__name__, +) + + +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) diff --git a/pyoaev/signatures/signature_manager.py b/pyoaev/signatures/signature_manager.py index 28452342..51857f7d 100644 --- a/pyoaev/signatures/signature_manager.py +++ b/pyoaev/signatures/signature_manager.py @@ -8,8 +8,8 @@ from typing import TYPE_CHECKING, Any from pydantic import ValidationError +from xtm_oaev_sdk import OpenAEVError -from pyoaev.exceptions import OpenAEVError from pyoaev.signatures.models import ( CloudInjectorConfig, ExecutionDetails, diff --git a/pyoaev/signatures/signature_match.py b/pyoaev/signatures/signature_match.py index 75ce54d7..4f8fa830 100644 --- a/pyoaev/signatures/signature_match.py +++ b/pyoaev/signatures/signature_match.py @@ -1,4 +1,5 @@ -from pyoaev.exceptions import OpenAEVError +from xtm_oaev_sdk import OpenAEVError + from pyoaev.signatures.types import MatchTypes diff --git a/pyoaev/utils.py b/pyoaev/utils.py index 21e7c7bd..15db0fec 100644 --- a/pyoaev/utils.py +++ b/pyoaev/utils.py @@ -1,14 +1,36 @@ -import dataclasses -import datetime +"""Utilities specific to the pyoaev HTTP client and daemons. + +The shared utilities moved to :mod:`xtm_oaev_sdk`; their legacy names here +resolve to the *identical* SDK objects and emit a :class:`DeprecationWarning` +on access. +""" + import email.message -import json -import logging import threading -import urllib.parse -from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union +from typing import Any, Callable, Iterator, Optional, Union import requests -from pythonjsonlogger.json import JsonFormatter +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +_MOVED = [ + "AppLogger", + "CustomJsonFormatter", + "EncodedId", + "EnhancedJSONEncoder", + "RequiredOptional", + "copy_dict", + "logger", + "remove_none_from_dict", + "setup_logging_config", +] + +__all__ = [ # noqa: F822 (moved names are served by the PEP 562 __getattr__ below) + "get_content_type", + "response_content", + "PingAlive", + *_MOVED, +] class _StdoutStream: @@ -46,141 +68,6 @@ def response_content( return None -def copy_dict( - *, - src: Dict[str, Any], - dest: Dict[str, Any], -) -> None: - for k, v in src.items(): - if isinstance(v, dict): - for dict_k, dict_v in v.items(): - dest[f"{k}[{dict_k}]"] = dict_v - else: - dest[k] = v - - -class EncodedId(str): - def __new__(cls, value: Union[str, int, "EncodedId"]) -> "EncodedId": - if isinstance(value, EncodedId): - return value - - if not isinstance(value, (int, str)): - raise TypeError(f"Unsupported type received: {type(value)}") - if isinstance(value, str): - value = urllib.parse.quote(value, safe="") - return super().__new__(cls, value) - - -def remove_none_from_dict(data: Dict[str, Any]) -> Dict[str, Any]: - return {k: v for k, v in data.items() if v is not None} - - -class EnhancedJSONEncoder(json.JSONEncoder): - def default(self, o): - if dataclasses.is_dataclass(o): - return dataclasses.asdict(o) - return super().default(o) - - -@dataclasses.dataclass(frozen=True) -class RequiredOptional: - required: Tuple[str, ...] = () - optional: Tuple[str, ...] = () - exclusive: Tuple[str, ...] = () - - def validate_attrs( - self, - *, - data: Dict[str, Any], - excludes: Optional[List[str]] = None, - ) -> None: - if excludes is None: - excludes = [] - - if self.required: - required = [k for k in self.required if k not in excludes] - missing = [attr for attr in required if attr not in data] - if missing: - raise AttributeError(f"Missing attributes: {', '.join(missing)}") - - if self.exclusive: - exclusives = [attr for attr in data if attr in self.exclusive] - if len(exclusives) > 1: - raise AttributeError( - f"Provide only one of these attributes: {', '.join(exclusives)}" - ) - if not exclusives: - raise AttributeError( - f"Must provide one of these attributes: " - f"{', '.join(self.exclusive)}" - ) - - -class CustomJsonFormatter(JsonFormatter): - def add_fields(self, log_record, record, message_dict): - super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict) - if not log_record.get("timestamp"): - # This doesn't use record.created, so it is slightly off - now = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") - log_record["timestamp"] = now - if log_record.get("level"): - log_record["level"] = log_record["level"].upper() - else: - log_record["level"] = record.levelname - - -def setup_logging_config(level, json_logging=True): - logging.getLogger("urllib3").setLevel(logging.WARNING) - logging.getLogger("pika").setLevel(logging.ERROR) - # Exceptions - if json_logging: - log_handler = logging.StreamHandler() - log_handler.setLevel(level) - formatter = CustomJsonFormatter("%(timestamp)s %(level)s %(name)s %(message)s") - log_handler.setFormatter(formatter) - logging.basicConfig(handlers=[log_handler], level=level, force=True) - else: - logging.basicConfig(level=level) - - -class AppLogger: - def __init__(self, level, json_logging=True, name: str = __name__): - self.log_level = level - self.json_logging = json_logging - setup_logging_config(self.log_level, self.json_logging) - self.local_logger = logging.getLogger(name) - - def __call__(self, name): - self.local_logger = logging.getLogger(name) - return self - - @staticmethod - def prepare_meta(meta=None): - return None if meta is None else {"attributes": meta} - - @staticmethod - def setup_logger_level(lib, log_level): - logging.getLogger(lib).setLevel(log_level) - - def debug(self, message, meta=None): - self.local_logger.debug(message, extra=AppLogger.prepare_meta(meta)) - - def info(self, message, meta=None): - self.local_logger.info(message, extra=AppLogger.prepare_meta(meta)) - - def warning(self, message, meta=None): - self.local_logger.warning(message, extra=AppLogger.prepare_meta(meta)) - - def error(self, message, meta=None): - # noinspection PyTypeChecker - self.local_logger.error(message, exc_info=1, extra=AppLogger.prepare_meta(meta)) - - -# DEPRECATED: compatibility -def logger(level, json_logging=True): - return AppLogger(level, json_logging) - - class PingAlive(threading.Thread): def __init__(self, api, config, logger, ping_type) -> None: threading.Thread.__init__(self) @@ -216,3 +103,16 @@ def stop(self) -> None: {"tenant_id": str(self.tenant_id) if self.tenant_id else None}, ) self.exit_event.set() + + +_deprecated_getattr = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in _MOVED + }, + module_name=__name__, +) + + +def __getattr__(name: str): # PEP 562 + return _deprecated_getattr(name) diff --git a/test/test_utils.py b/test/test_utils.py index e5279b76..395cb979 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,111 +1,16 @@ -import dataclasses -import importlib -import json -import logging import unittest -import warnings -from typing import Any, cast from unittest.mock import MagicMock, patch -from pythonjsonlogger.json import JsonFormatter - import pyoaev.utils as module -@dataclasses.dataclass -class _SampleData: - value: int - - class TestUtils(unittest.TestCase): - def test_custom_json_formatter_inherits_non_deprecated_formatter(self): - self.assertTrue(issubclass(module.CustomJsonFormatter, JsonFormatter)) - - def test_reloading_utils_does_not_raise_jsonlogger_deprecation_warning(self): - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - importlib.reload(module) - - deprecation_messages = [ - str(warning.message) - for warning in caught - if issubclass(warning.category, DeprecationWarning) - ] - self.assertFalse( - any( - "pythonjsonlogger.jsonlogger has been moved" in message - for message in deprecation_messages - ) - ) - def test_get_content_type(self): self.assertEqual( module.get_content_type("application/json; charset=utf-8"), "application/json", ) - def test_copy_dict_flattens_nested_dict(self): - destination = {} - module.copy_dict( - src={"a": 1, "meta": {"x": "y", "n": 2}}, - dest=destination, - ) - self.assertEqual(destination, {"a": 1, "meta[x]": "y", "meta[n]": 2}) - - def test_remove_none_from_dict(self): - self.assertEqual(module.remove_none_from_dict({"a": 1, "b": None}), {"a": 1}) - - def test_encoded_id_from_existing_instance_returns_same_object(self): - encoded_id = module.EncodedId("with space") - self.assertIs(module.EncodedId(encoded_id), encoded_id) - - def test_encoded_id_encodes_string_and_keeps_int(self): - self.assertEqual(module.EncodedId("a/b c"), "a%2Fb%20c") - self.assertEqual(module.EncodedId(42), "42") - - def test_encoded_id_rejects_unsupported_type(self): - with self.assertRaises(TypeError): - module.EncodedId(cast(Any, ["bad"])) - - def test_enhanced_json_encoder_serializes_not_dataclasses(self): - data = {"test": "this is a test", "other": 3} - classic_json_dump = json.dumps(data) - enhanced_json_dump = json.dumps(data, cls=module.EnhancedJSONEncoder) - self.assertFalse(module.dataclasses.is_dataclass(data)) - self.assertEqual( - classic_json_dump, - enhanced_json_dump, - ) - - def test_enhanced_json_encoder_serializes_dataclasses(self): - self.assertEqual( - json.dumps(_SampleData(value=3), cls=module.EnhancedJSONEncoder), - '{"value": 3}', - ) - - def test_required_optional_required_attribute_missing(self): - rules = module.RequiredOptional(required=("name",)) - with self.assertRaises(AttributeError): - rules.validate_attrs(data={}) - - def test_required_optional_excludes_required_attribute(self): - rules = module.RequiredOptional(required=("name",)) - rules.validate_attrs(data={}, excludes=["name"]) - - def test_required_optional_exclusive_allows_only_one_key(self): - rules = module.RequiredOptional(exclusive=("a", "b")) - with self.assertRaises(AttributeError): - rules.validate_attrs(data={"a": 1, "b": 2}) - - def test_required_optional_exclusive_requires_one_key(self): - rules = module.RequiredOptional(exclusive=("a", "b")) - with self.assertRaises(AttributeError): - rules.validate_attrs(data={}) - - def test_required_optional_exclusive_with_single_key_is_valid(self): - rules = module.RequiredOptional(exclusive=("a", "b")) - rules.validate_attrs(data={"a": 1}) - def test_response_content_returns_iterator_when_requested(self): response = MagicMock() response.iter_content.return_value = iter([b"a", b"b"]) @@ -167,61 +72,6 @@ def test_response_content_streamed_defaults_to_stdout_stream(self): mock_print.assert_called_once_with(b"visible") - def test_custom_json_formatter_add_fields_sets_timestamp_and_level(self): - formatter = module.CustomJsonFormatter("%(message)s") - record = logging.LogRecord( - name="test", - level=logging.INFO, - pathname=__file__, - lineno=1, - msg="hello", - args=(), - exc_info=None, - ) - log_record = {} - - formatter.add_fields(log_record, record, {}) - - self.assertIn("timestamp", log_record) - self.assertEqual(log_record["level"], "INFO") - - def test_setup_logging_config_json_logging_true_uses_custom_formatter(self): - with patch("pyoaev.utils.logging.basicConfig") as mock_basic_config: - module.setup_logging_config(logging.INFO, json_logging=True) - - kwargs = mock_basic_config.call_args.kwargs - self.assertEqual(kwargs["level"], logging.INFO) - self.assertIn("handlers", kwargs) - self.assertIsInstance( - kwargs["handlers"][0].formatter, module.CustomJsonFormatter - ) - - def test_setup_logging_config_json_logging_false_calls_basic_config(self): - with patch("pyoaev.utils.logging.basicConfig") as mock_basic_config: - module.setup_logging_config(logging.WARNING, json_logging=False) - - mock_basic_config.assert_called_once_with(level=logging.WARNING) - - def test_app_logger_methods_delegate_to_local_logger(self): - with patch("pyoaev.utils.setup_logging_config"): - app_logger = module.AppLogger(logging.INFO) - app_logger.local_logger = MagicMock() - - app_logger.debug("d", {"x": 1}) - app_logger.info("i") - app_logger.warning("w") - app_logger.error("e") - - self.assertTrue(app_logger.local_logger.debug.called) - self.assertTrue(app_logger.local_logger.info.called) - self.assertTrue(app_logger.local_logger.warning.called) - self.assertTrue(app_logger.local_logger.error.called) - - def test_logger_helper_returns_app_logger(self): - with patch("pyoaev.utils.setup_logging_config"): - helper = module.logger(logging.INFO) - self.assertIsInstance(helper, module.AppLogger) - def test_pingalive_ping_uses_injector_branch(self): api = MagicMock() logger = MagicMock() From 382a40379f9e76ce972308f14dba6306a0f503e5 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 13:14:31 +0200 Subject: [PATCH 03/22] :sparkles: feat(sdk): migrate signatures to xtm-oaev-sdk with versioned seams and pyoaev shims --- .../test/common/shim/test_shim_contract.py | 10 +- .../configuration/shim/test_shim_contract.py | 10 +- libs/xtm-oaev-sdk/test/seam/__init__.py | 0 .../test/seam/fixtures/port_io_schemas.json | 9 + .../fixtures/signatures_seam_schemas.json | 8 + .../test/seam/generate_port_io_golden.py | 47 ++ .../seam/generate_signatures_seam_golden.py | 45 ++ .../test/seam/test_port_io_schema_pin.py | 97 ++++ .../seam/test_signatures_seam_schema_pin.py | 101 ++++ libs/xtm-oaev-sdk/test/signatures/__init__.py | 0 ..._manager_ip_resolution_constraints.feature | 8 + ...manager_post_execution_constraints.feature | 39 ++ ..._manager_pre_execution_constraints.feature | 15 + .../signature_manager_ip_resolution.feature | 12 + .../signature_manager_post_execution.feature | 28 ++ .../signature_manager_pre_execution.feature | 83 ++++ .../test/signatures/shim/__init__.py | 0 ...manager_post_execution_constraints.feature | 0 ..._manager_pre_execution_constraints.feature | 0 .../signature_manager_post_execution.feature | 0 .../signature_manager_pre_execution.feature | 0 .../test/signatures/shim}/test_ai_marker.py | 0 .../shim}/test_ai_signature_types.py | 0 .../signatures/shim/test_shim_contract.py | 149 ++++++ .../test_signature_manager_post_execution.py | 0 .../test_signature_manager_pre_execution.py | 0 .../signatures/shim}/test_signature_match.py | 0 .../signatures/shim}/test_signature_type.py | 0 .../signatures/test_ai_signature_types.py | 28 ++ .../test_signature_manager_ip_resolution.py | 110 +++++ .../test_signature_manager_post_execution.py | 223 +++++++++ .../test_signature_manager_pre_execution.py | 466 ++++++++++++++++++ .../test/signatures/test_signature_match.py | 23 + .../test/signatures/test_signature_type.py | 56 +++ libs/xtm-oaev-sdk/test/test_boundaries.py | 23 + libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 3 + .../xtm_oaev_sdk/_core/signatures/__init__.py | 71 +++ .../_core/signatures/ai_marker.py | 17 + .../xtm_oaev_sdk/_core/signatures/models.py | 129 +++++ .../_core/signatures/signature_manager.py | 359 ++++++++++++++ .../_core/signatures/signature_match.py | 31 ++ .../_core/signatures/signature_type.py | 54 ++ .../xtm_oaev_sdk/_core/signatures/types.py | 65 +++ .../xtm_oaev_sdk/public/__init__.py | 67 +++ .../xtm_oaev_sdk/seam/__init__.py | 11 +- .../xtm_oaev_sdk/seam/port_io/__init__.py | 50 ++ .../xtm_oaev_sdk/seam/port_io/models.py | 299 +++++++++++ .../xtm_oaev_sdk/seam/signatures/__init__.py | 71 +++ .../xtm_oaev_sdk/seam/signatures/seam.py | 146 ++++++ pyoaev/__init__.py | 23 +- .../inject_expectation/model/expectation.py | 4 +- pyoaev/apis/signature.py | 10 +- pyoaev/asset_types.py | 6 +- pyoaev/configuration/__init__.py | 6 +- pyoaev/configuration/configuration.py | 6 +- .../connector_config_schema_generator.py | 6 +- pyoaev/configuration/settings_loader.py | 6 +- pyoaev/configuration/sources.py | 6 +- pyoaev/exceptions.py | 6 +- pyoaev/helpers.py | 14 +- pyoaev/security_domain/types.py | 6 +- pyoaev/signatures/__init__.py | 54 +- pyoaev/signatures/ai_marker.py | 35 +- pyoaev/signatures/models.py | 405 ++------------- pyoaev/signatures/signature_manager.py | 342 +++---------- pyoaev/signatures/signature_match.py | 29 +- pyoaev/signatures/signature_type.py | 74 +-- pyoaev/signatures/types.py | 105 ++-- pyoaev/utils.py | 6 +- 69 files changed, 3208 insertions(+), 904 deletions(-) create mode 100644 libs/xtm-oaev-sdk/test/seam/__init__.py create mode 100644 libs/xtm-oaev-sdk/test/seam/fixtures/port_io_schemas.json create mode 100644 libs/xtm-oaev-sdk/test/seam/fixtures/signatures_seam_schemas.json create mode 100644 libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py create mode 100644 libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py create mode 100644 libs/xtm-oaev-sdk/test/seam/test_port_io_schema_pin.py create mode 100644 libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py create mode 100644 libs/xtm-oaev-sdk/test/signatures/__init__.py create mode 100644 libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_ip_resolution_constraints.feature create mode 100644 libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_post_execution_constraints.feature create mode 100644 libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_pre_execution_constraints.feature create mode 100644 libs/xtm-oaev-sdk/test/signatures/features/signature_manager_ip_resolution.feature create mode 100644 libs/xtm-oaev-sdk/test/signatures/features/signature_manager_post_execution.feature create mode 100644 libs/xtm-oaev-sdk/test/signatures/features/signature_manager_pre_execution.feature create mode 100644 libs/xtm-oaev-sdk/test/signatures/shim/__init__.py rename {test/signatures => libs/xtm-oaev-sdk/test/signatures/shim}/constraints/signature_manager_post_execution_constraints.feature (100%) rename {test/signatures => libs/xtm-oaev-sdk/test/signatures/shim}/constraints/signature_manager_pre_execution_constraints.feature (100%) rename {test/signatures => libs/xtm-oaev-sdk/test/signatures/shim}/features/signature_manager_post_execution.feature (100%) rename {test/signatures => libs/xtm-oaev-sdk/test/signatures/shim}/features/signature_manager_pre_execution.feature (100%) rename {test/signatures => libs/xtm-oaev-sdk/test/signatures/shim}/test_ai_marker.py (100%) rename {test/signatures => libs/xtm-oaev-sdk/test/signatures/shim}/test_ai_signature_types.py (100%) create mode 100644 libs/xtm-oaev-sdk/test/signatures/shim/test_shim_contract.py rename {test/signatures => libs/xtm-oaev-sdk/test/signatures/shim}/test_signature_manager_post_execution.py (100%) rename {test/signatures => libs/xtm-oaev-sdk/test/signatures/shim}/test_signature_manager_pre_execution.py (100%) rename {test/signatures => libs/xtm-oaev-sdk/test/signatures/shim}/test_signature_match.py (100%) rename {test/signatures => libs/xtm-oaev-sdk/test/signatures/shim}/test_signature_type.py (100%) create mode 100644 libs/xtm-oaev-sdk/test/signatures/test_ai_signature_types.py create mode 100644 libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py create mode 100644 libs/xtm-oaev-sdk/test/signatures/test_signature_manager_post_execution.py create mode 100644 libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py create mode 100644 libs/xtm-oaev-sdk/test/signatures/test_signature_match.py create mode 100644 libs/xtm-oaev-sdk/test/signatures/test_signature_type.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/ai_marker.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/models.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_type.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/types.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py diff --git a/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py index 39e01ba0..96ab33a5 100644 --- a/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py +++ b/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py @@ -67,6 +67,9 @@ ) +_CONSUMER = "legacy_consumer.py" + + def _given_fresh_interpreter_script(script: str) -> list[str]: return [sys.executable, "-W", "error::DeprecationWarning", "-c", script] @@ -79,7 +82,10 @@ def _when_import_legacy_name(module: str, name: str): namespace: dict = {} with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - exec(f"from {module} import {name} as _legacy", namespace) + exec( + compile(f"from {module} import {name} as _legacy", _CONSUMER, "exec"), + namespace, + ) return namespace["_legacy"], caught @@ -95,6 +101,8 @@ def test_legacy_import_warns_and_returns_the_sdk_object(module, name) -> None: deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] assert deprecations assert f"from xtm_oaev_sdk import {name}" in str(deprecations[0].message) + # attributed to the consumer's line, so default warning filters show it + assert any(w.filename == _CONSUMER for w in deprecations) def test_pyoaev_all_still_lists_legacy_exceptions() -> None: diff --git a/libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py index 19d188e2..7bd77c42 100644 --- a/libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py +++ b/libs/xtm-oaev-sdk/test/configuration/shim/test_shim_contract.py @@ -46,6 +46,9 @@ ] +_CONSUMER = "legacy_consumer.py" + + def _given_fresh_interpreter_script(script: str) -> list[str]: return [sys.executable, "-W", "error::DeprecationWarning", "-c", script] @@ -58,7 +61,10 @@ def _when_import_legacy_name(module: str, name: str): namespace: dict = {} with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - exec(f"from {module} import {name} as _legacy", namespace) + exec( + compile(f"from {module} import {name} as _legacy", _CONSUMER, "exec"), + namespace, + ) return namespace["_legacy"], caught @@ -74,6 +80,8 @@ def test_legacy_import_warns_and_returns_the_sdk_object(module, name) -> None: deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] assert deprecations assert f"from xtm_oaev_sdk import {name}" in str(deprecations[0].message) + # attributed to the consumer's line, so default warning filters show it + assert any(w.filename == _CONSUMER for w in deprecations) def test_importing_pyoaev_is_silent() -> None: diff --git a/libs/xtm-oaev-sdk/test/seam/__init__.py b/libs/xtm-oaev-sdk/test/seam/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/test/seam/fixtures/port_io_schemas.json b/libs/xtm-oaev-sdk/test/seam/fixtures/port_io_schemas.json new file mode 100644 index 00000000..a3e010fb --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/fixtures/port_io_schemas.json @@ -0,0 +1,9 @@ +{ + "ExecutionDetails": "4cc109aef2950c555a6d0dce2e637e3ff709b86588a08fe0e459c5eb5328f939", + "ExpectationSignatureGroup": "8258ae33685d9826dc3b090186c6fa76935353a0fabd12ca9b6974184df251ff", + "SignatureOutputStructure": "4291aed5c4a354327c70db42534cd90d38bfbf7b39cc241cacf946bb0b70c066", + "SignaturePayload": "5f7451f4640c22c3f51cc1dab1fe0e65ca85d31f67349b2c42151111980d1467", + "SignatureTarget": "52a8848d8b60e2f68aaa8769f303ec886beb3b77a9088b98ad783690512e9ea9", + "SignatureValue": "dd21d79b3127f06390e5e17120bd6b662c11b04bb45f7db9c8c7848ca33d3d95", + "TargetSignatures": "490debc49e8fd5352b6afa91fe018935b17c8a4497c35fcc35ac1ebaf79242df" +} diff --git a/libs/xtm-oaev-sdk/test/seam/fixtures/signatures_seam_schemas.json b/libs/xtm-oaev-sdk/test/seam/fixtures/signatures_seam_schemas.json new file mode 100644 index 00000000..d4202b8e --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/fixtures/signatures_seam_schemas.json @@ -0,0 +1,8 @@ +{ + "ToolErrorInfo": "5c7a5427eec8e163b614c69c5a3b4bd0e17bd5e483f2ca1820965561601fa413", + "ToolTimeoutInfo": "6a6da8f7280971640ff0ae8ae50f844c241e36bfb35a73a682e255e6758a36b2", + "ToolOutput": "9269914b152c6c42bd77110a483595b70fe3811a50cd0b8880770b7b9b686161", + "ExecutionSignature": "e9655dffea0daee5ad1c242f4d43138235ae7f4f55eb357b70752614fa68daf0", + "NetworkInjectorConfig": "68a8f930e3e606bf95efed0c4c386332714e0c8b540918d440ef58b4f36caf0d", + "CloudInjectorConfig": "908a129f91d32fd6e1f553c445ac92ec07c2ebeb065f51a5e955c1d78fce6ad9" +} diff --git a/libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py b/libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py new file mode 100644 index 00000000..31df41d2 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py @@ -0,0 +1,47 @@ +"""Regenerate the golden schema hashes for the port I/O wire contract. + +Run ``python libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py``. The +golden file pins the JSON schema of every wire model; it is only regenerated +on an intentional, version-bumped contract change. +""" + +import hashlib +import json +from pathlib import Path + +from xtm_oaev_sdk.seam.port_io import ( + ExecutionDetails, + ExpectationSignatureGroup, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, +) + +GOLDEN_PATH = Path(__file__).parent / "fixtures" / "port_io_schemas.json" + +CONTRACT_MODELS = { + "ExecutionDetails": ExecutionDetails, + "ExpectationSignatureGroup": ExpectationSignatureGroup, + "SignatureOutputStructure": SignatureOutputStructure, + "SignaturePayload": SignaturePayload, + "SignatureTarget": SignatureTarget, + "SignatureValue": SignatureValue, + "TargetSignatures": TargetSignatures, +} + + +def main() -> None: + out: dict[str, str] = {} + for name, cls in CONTRACT_MODELS.items(): + canonical = json.dumps(cls.model_json_schema(), sort_keys=True) + out[name] = hashlib.sha256(canonical.encode()).hexdigest() + with GOLDEN_PATH.open("w") as f: + json.dump(out, f, indent=2) + f.write("\n") + print(f"wrote {GOLDEN_PATH}") + + +if __name__ == "__main__": + main() diff --git a/libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py b/libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py new file mode 100644 index 00000000..572ade4d --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py @@ -0,0 +1,45 @@ +"""Regenerate the golden schema hashes for the signatures inbound seam. + +Run ``python libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py``. The golden file pins the +JSON schema of every seam model; it is only regenerated on an +intentional, version-bumped contract change. +""" + +import hashlib +import json +from pathlib import Path + +from xtm_oaev_sdk.seam.signatures.seam import ( + CloudInjectorConfig, + ExecutionSignature, + NetworkInjectorConfig, + ToolErrorInfo, + ToolOutput, + ToolTimeoutInfo, +) + +GOLDEN_PATH = Path(__file__).parent / "fixtures" / "signatures_seam_schemas.json" + +SEAM_MODELS = { + "ToolErrorInfo": ToolErrorInfo, + "ToolTimeoutInfo": ToolTimeoutInfo, + "ToolOutput": ToolOutput, + "ExecutionSignature": ExecutionSignature, + "NetworkInjectorConfig": NetworkInjectorConfig, + "CloudInjectorConfig": CloudInjectorConfig, +} + + +def main() -> None: + out: dict[str, str] = {} + for name, cls in SEAM_MODELS.items(): + canonical = json.dumps(cls.model_json_schema(), sort_keys=True) + out[name] = hashlib.sha256(canonical.encode()).hexdigest() + with GOLDEN_PATH.open("w") as f: + json.dump(out, f, indent=2) + f.write("\n") + print(f"wrote {GOLDEN_PATH}") + + +if __name__ == "__main__": + main() diff --git a/libs/xtm-oaev-sdk/test/seam/test_port_io_schema_pin.py b/libs/xtm-oaev-sdk/test/seam/test_port_io_schema_pin.py new file mode 100644 index 00000000..a5931ee9 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/test_port_io_schema_pin.py @@ -0,0 +1,97 @@ +"""Schema-pinning guard for the port I/O data contract. + +Hashes each contract model's ``model_json_schema()`` and compares against +the committed golden file. Any drift in the wire schema fails this test. + +To intentionally update the contract: + 1. Bump ``PORT_IO_CONTRACT_VERSION`` in ``seam/port_io/models.py``. + 2. Regenerate the golden file: + ``python libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py`` + 3. Commit both changes together. +""" + +import ast +import hashlib +import json +from pathlib import Path + +from xtm_oaev_sdk.seam.port_io import ( + PORT_IO_CONTRACT_VERSION, + ExecutionDetails, + ExpectationSignatureGroup, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, +) +from xtm_oaev_sdk.seam.port_io import models as port_io_models + +GOLDEN_PATH = Path(__file__).parent / "fixtures" / "port_io_schemas.json" + +CONTRACT_MODELS = { + "ExecutionDetails": ExecutionDetails, + "ExpectationSignatureGroup": ExpectationSignatureGroup, + "SignatureOutputStructure": SignatureOutputStructure, + "SignaturePayload": SignaturePayload, + "SignatureTarget": SignatureTarget, + "SignatureValue": SignatureValue, + "TargetSignatures": TargetSignatures, +} + + +def _schema_hash(model_cls) -> str: + """SHA-256 of the canonical JSON Schema for a pydantic model class.""" + schema = model_cls.model_json_schema() + canonical = json.dumps(schema, sort_keys=True) + return hashlib.sha256(canonical.encode()).hexdigest() + + +def _load_golden() -> dict[str, str]: + with GOLDEN_PATH.open() as f: + return json.load(f) + + +def test_port_io_contract_version_is_pinned(): + """The version constant must be a semver string, not a placeholder.""" + assert PORT_IO_CONTRACT_VERSION == "1.0.0" + + +def test_port_io_wire_schema_matches_golden(): + """Every contract model's JSON Schema must match the golden file. + + This is the anti-breaking guard: if a field is added, removed, renamed, + or retyped in any of the 7 wire models, this test fails. + """ + golden = _load_golden() + + # Guard against accidental golden file truncation. + assert len(golden) == len(CONTRACT_MODELS), ( + f"Golden file has {len(golden)} entries; expected {len(CONTRACT_MODELS)}. " + f"Missing: {set(CONTRACT_MODELS) - set(golden)}" + ) + + for name, cls in sorted(CONTRACT_MODELS.items()): + current = _schema_hash(cls) + expected = golden[name] + assert current == expected, ( + f"Schema drift detected in {name} (port I/O contract " + f"v{PORT_IO_CONTRACT_VERSION}).\n" + f" expected: {expected}\n" + f" actual: {current}\n" + f"If intentional, bump PORT_IO_CONTRACT_VERSION and regenerate " + f"the golden file." + ) + + +def test_port_io_models_import_only_the_signatures_seam(): + """The wire contract is closed: no import of the capability internals.""" + source = Path(port_io_models.__file__).read_text() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.ImportFrom) and (node.module or "").startswith( + "xtm_oaev_sdk" + ): + assert node.module == "xtm_oaev_sdk.seam.signatures.seam", node.module + elif isinstance(node, ast.Import): + for alias in node.names: + assert not alias.name.startswith("xtm_oaev_sdk"), alias.name diff --git a/libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py b/libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py new file mode 100644 index 00000000..674262a7 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py @@ -0,0 +1,101 @@ +"""Schema-pinning guard for the signatures inbound seam. + +Hashes each seam model's ``model_json_schema()`` and compares against +the committed golden file. Any drift in the seam schema fails this test. + +The seam is the frozen boundary both sides of the capability share, so +this guard also enforces the dependency rule (the seam imports nothing +from the capability) and the identity contract (every re-export layer +shares the exact same objects as the seam). + +To intentionally update the seam: + 1. Bump ``SEAM_VERSION`` in ``seam/signatures/seam.py``. + 2. Regenerate the golden file: + ``python libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py`` + 3. Commit both changes together. +""" + +import ast +import hashlib +import json +from pathlib import Path + +from xtm_oaev_sdk.seam.signatures import ( + SEAM_VERSION, + CloudInjectorConfig, + ExecutionSignature, + NetworkInjectorConfig, + ToolErrorInfo, + ToolOutput, + ToolTimeoutInfo, +) +from xtm_oaev_sdk.seam.signatures import seam as seam_module + +GOLDEN_PATH = Path(__file__).parent / "fixtures" / "signatures_seam_schemas.json" + +SEAM_MODELS = { + "ToolErrorInfo": ToolErrorInfo, + "ToolTimeoutInfo": ToolTimeoutInfo, + "ToolOutput": ToolOutput, + "ExecutionSignature": ExecutionSignature, + "NetworkInjectorConfig": NetworkInjectorConfig, + "CloudInjectorConfig": CloudInjectorConfig, +} + + +def _schema_hash(model_cls) -> str: + """SHA-256 of the canonical JSON Schema for a pydantic model class.""" + schema = model_cls.model_json_schema() + canonical = json.dumps(schema, sort_keys=True) + return hashlib.sha256(canonical.encode()).hexdigest() + + +def test_seam_version_is_pinned(): + """The version constant must be the pinned semver string.""" + assert SEAM_VERSION == "1.0.0" + + +def test_seam_schema_is_pinned(): + """Every seam model's JSON schema must match the committed golden file.""" + with GOLDEN_PATH.open() as f: + golden = json.load(f) + for name, cls in SEAM_MODELS.items(): + assert name in golden, f"{name} missing from golden file" + actual = _schema_hash(cls) + assert actual == golden[name], ( + f"{name}: seam schema drift detected\n" + f" golden {golden[name]}\n" + f" actual {actual}\n" + "If this was intentional, bump SEAM_VERSION and regenerate the golden file." + ) + + +def test_seam_imports_nothing_from_capability(): + """The seam is pure data: no imports from xtm_oaev_sdk itself.""" + source = Path(seam_module.__file__).read_text() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + for alias in node.names: + assert not alias.name.startswith( + "xtm_oaev_sdk" + ), f"seam imports capability module {alias.name!r} — pure-data violation" + elif isinstance(node, ast.ImportFrom): + module = node.module or "" + assert not module.startswith( + "xtm_oaev_sdk" + ), f"seam imports from capability module {module!r} — pure-data violation" + + +def test_reexports_preserve_identity(): + """The seam, the port's models module, and the public surface share one object.""" + import xtm_oaev_sdk as sdk + import xtm_oaev_sdk._core.signatures.models as models + + for name, cls in SEAM_MODELS.items(): + assert ( + getattr(models, name) is cls + ), f"{name}: models.py re-export broke identity" + assert ( + getattr(sdk, name) is cls + ), f"{name}: public surface re-export broke identity" + assert sdk.InjectorConfig is models.InjectorConfig diff --git a/libs/xtm-oaev-sdk/test/signatures/__init__.py b/libs/xtm-oaev-sdk/test/signatures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_ip_resolution_constraints.feature b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_ip_resolution_constraints.feature new file mode 100644 index 00000000..30ab2524 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_ip_resolution_constraints.feature @@ -0,0 +1,8 @@ +Feature: SignatureManager IP resolution constraints + + Scenario: resolve_container_ip returns unknown and emits exactly one warning when all strategies fail + Given all IP resolution strategies are mocked to fail + When I call resolve_container_ip + Then the returned value is the string "unknown" + And exactly 1 WARNING log message is emitted + And no exception propagates from resolve_container_ip diff --git a/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_post_execution_constraints.feature b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_post_execution_constraints.feature new file mode 100644 index 00000000..ec7b4dc0 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_post_execution_constraints.feature @@ -0,0 +1,39 @@ +Feature: SignatureManager post-execution constraints + As an injector using the OpenAEV client + I want post-execution compilation to handle failure and timeout edge cases + So that incomplete executions still produce valid signature records + + Background: + Given a SignatureManager initialised with constructor SignatureManager(transport, logger) + And a execution_signatures object containing: + | key | value | + | source_ipv4 | 172.17.0.2 | + | target_ipv4 | 10.0.0.1 | + | target_hostname | host-a.internal | + | start_time | 2024-06-26T06:00:00Z | + And a execution_details object containing: + | key | value | + | start_time | 2024-06-26T06:00:00Z | + + Scenario: Tool crash sets execution_status to failed and uses crash timestamp as end_time + Given a tool_output containing error_info with exit_code=1 and crash_timestamp="2024-06-26T06:05:00Z" + When I call post_execution_updates with the execution_details, execution_signatures and tool_output + Then execution_status equals "ERROR" + And end_time equals "2024-06-26T06:05:00Z" + And the execution signature model contains every previous parameter unchanged + And the execution details model contain every previous parameter pair unchanged + + Scenario: Timeout sets execution_status to timeout and includes available partial results + Given a tool_output containing timeout_info with partial_results=["result-A", "result-B"] + When I call post_execution_updates with the execution_details, execution_signatures and tool_output + Then execution_status equals "TIMEOUT" + And the returned dict contains the partial results ["result-A", "result-B"] from timeout_info + And the execution signature model contains every previous parameter unchanged + And the execution details model contain every previous parameter pair unchanged + + Scenario: Timeout with no partial results still sets execution_status to timeout + Given a tool_output containing timeout_info with no partial results available + When I call post_execution_updates with the execution_details, execution_signatures and tool_output + Then execution_status equals "TIMEOUT" + And the execution signature model contains every previous parameter unchanged + And the execution details model contain every previous parameter pair unchanged diff --git a/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_pre_execution_constraints.feature b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_pre_execution_constraints.feature new file mode 100644 index 00000000..ddff8521 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/constraints/signature_manager_pre_execution_constraints.feature @@ -0,0 +1,15 @@ +Feature: SignatureManager pre-execution constraints + As an injector using the OpenAEV client + I want pre-execution compilation to handle timing edge cases correctly + So that signatures always reflect the actual moment of execution + + Background: + Given a SignatureManager initialised with constructor SignatureManager(transport, logger) + + Scenario: start_time is generated at method-call time not at class instantiation time + Given a SignatureManager that was instantiated at timestamp T0 + And 5 seconds elapse after instantiation + And a NetworkInjectorConfig with target_ipv4="192.168.1.10" + When I call build_execution_signatures with the config at timestamp T1 + Then the start_time in the returned dict equals T1 within 1 second tolerance + And start_time does not equal T0 diff --git a/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_ip_resolution.feature b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_ip_resolution.feature new file mode 100644 index 00000000..0751541b --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_ip_resolution.feature @@ -0,0 +1,12 @@ +Feature: SignatureManager container IP resolution + + Scenario Outline: resolve_container_ip returns a valid IPv4 in each supported execution environment + Given a SignatureManager running in a "" environment + When I call resolve_container_ip + Then the returned value is a non-empty valid IPv4 address string + + Examples: + | environment | + | Docker | + | Kubernetes | + | bare-metal | diff --git a/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_post_execution.feature b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_post_execution.feature new file mode 100644 index 00000000..746c5a81 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_post_execution.feature @@ -0,0 +1,28 @@ +Feature: SignatureManager post-execution execution elements update + As an injector using the OpenAEV client + I want to update both execution signatures and execution details with execution results + So that each inject has a complete execution record including outcome and timing + + Background: + Given a SignatureManager initialised with constructor SignatureManager(transport, logger) + And a execution_signatures object containing: + | key | value | + | source_ipv4 | 172.17.0.2 | + | target_ipv4 | 10.0.0.1 | + | target_hostname | host-a.internal | + | start_time | 2024-06-26T06:00:00Z | + And a execution_details object containing: + | key | value | + | start_time | 2024-06-26T06:00:00Z | + + Scenario: Successful execution updates end_time and execution_status in execution signatures and execution details + Given a tool_output indicating successful completion with no errors and no timeout + When I call post_execution_updates with the execution_details, execution_signatures and tool_output + Then the execution signature model contains every previous parameter unchanged + And the end_time parameter in the execution signature model is a UTC ISO 8601 string + And this end_time is chronologically greater than or equal to start_time "2024-06-26T06:00:00Z" + And the execution details model contain every previous parameter pair unchanged + And the end_time parameter in the execution details model is a datetime object + And this end_time is chronologically greater than or equal to start_time "2024-06-26T06:00:00Z" + And the execution_status parameter in the execution details model is equal to "EXECUTED" + And the execution_action parameter in the execution details model is equal to "data" diff --git a/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_pre_execution.feature b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_pre_execution.feature new file mode 100644 index 00000000..0f1edf49 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/features/signature_manager_pre_execution.feature @@ -0,0 +1,83 @@ +Feature: SignatureManager pre-execution signature compilation + As an injector using the OpenAEV client + I want to compile category-specific pre-execution signatures + So that each inject has a correct, typed signature payload before execution begins + + Background: + Given a SignatureManager initialised with constructor SignatureManager(transport, logger) + + Scenario: Network category returns required IP and timing fields and no cloud or query fields + Given a NetworkInjectorConfig with target_ipv4="192.168.1.10" + And the running container has a resolvable IPv4 address + When I call build_execution_signatures with the config + Then the returned dict contains source_ipv4 as a non-empty valid IPv4 address string + And the returned dict contains start_time as a UTC ISO 8601 string + And the returned dict contains target_ipv4 equal to "192.168.1.10" + But the returned dict does not contain cloud_provider + And the returned dict does not contain cloud_account_id + And the returned dict does not contain cloud_region + And the returned dict does not contain target_service + And the returned dict does not contain query + + Scenario: Network hostname target returns hostname and no target_ipv4 + Given a NetworkInjectorConfig with target_hostname="target.example.com" + And the running container has a resolvable IPv4 address + When I call build_execution_signatures with the config + Then the returned dict contains target_hostname equal to "target.example.com" + And the returned dict contains source_ipv4 as a non-empty valid IPv4 address string + But the returned dict does not contain target_ipv4 + + Scenario: Cloud category returns required cloud identity fields and no IP fields + Given a CloudInjectorConfig with cloud_provider="aws", cloud_account_id="123456789012", cloud_region="eu-west-1", and target_service="ec2" + When I call build_execution_signatures with the config + Then the returned dict contains cloud_provider equal to "aws" + And the returned dict contains cloud_account_id equal to "123456789012" + And the returned dict contains cloud_region equal to "eu-west-1" + And the returned dict contains target_service equal to "ec2" + And the returned dict contains start_time as a UTC ISO 8601 string + But the returned dict does not contain source_ipv4 + And the returned dict does not contain source_ipv6 + And the returned dict does not contain target_ipv4 + And the returned dict does not contain target_ipv6 + + Scenario Outline: Network multi-target returns one dict per target with a shared source IP + Given a list of 3 NetworkInjectorConfig with target_ipv4 "10.0.0.1", "10.0.0.2", "10.0.0.3" + And the running container has a resolvable IPv4 address "172.17.0.2" + When I call build_execution_signatures with the config list + Then the return value is a list of exactly 3 dicts + And the dict at position contains target_ipv4 equal to "" + And the dict at position contains source_ipv4 equal to "172.17.0.2" + + Examples: + | index | target_ip | + | 0 | 10.0.0.1 | + | 1 | 10.0.0.2 | + | 2 | 10.0.0.3 | + + Scenario: All network multi-target dicts share the same source_ipv4 + Given a list of 3 NetworkInjectorConfig built from default IPv4 targets + And the running container has a resolvable IPv4 address + When I call build_execution_signatures with the config list + Then the return value is a list of 3 dicts + And all 3 dicts contain the same source_ipv4 value + + Scenario Outline: Cloud multi-region returns one dict per region with a shared account ID + Given a list of 3 CloudInjectorConfig with cloud_account_id="123456789012" and regions "us-east-1", "eu-west-1", "ap-southeast-1" + When I call build_execution_signatures with the config list + Then the return value is a list of exactly 3 dicts + And the dict at position contains cloud_region equal to "" + And the dict at position contains cloud_account_id equal to "123456789012" + + Examples: + | index | region | + | 0 | us-east-1 | + | 1 | eu-west-1 | + | 2 | ap-southeast-1 | + + Scenario: Builder classifies a mixed list of targets into typed configs + Given a raw mixed target list "10.0.0.1", "2001:db8::1", "target.example.com" + When I build network configs from the raw list + Then the builder returns 3 NetworkInjectorConfig + And the config at position 0 has target_ipv4 equal to "10.0.0.1" + And the config at position 1 has target_ipv6 equal to "2001:db8::1" + And the config at position 2 has target_hostname equal to "target.example.com" diff --git a/libs/xtm-oaev-sdk/test/signatures/shim/__init__.py b/libs/xtm-oaev-sdk/test/signatures/shim/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/signatures/constraints/signature_manager_post_execution_constraints.feature b/libs/xtm-oaev-sdk/test/signatures/shim/constraints/signature_manager_post_execution_constraints.feature similarity index 100% rename from test/signatures/constraints/signature_manager_post_execution_constraints.feature rename to libs/xtm-oaev-sdk/test/signatures/shim/constraints/signature_manager_post_execution_constraints.feature diff --git a/test/signatures/constraints/signature_manager_pre_execution_constraints.feature b/libs/xtm-oaev-sdk/test/signatures/shim/constraints/signature_manager_pre_execution_constraints.feature similarity index 100% rename from test/signatures/constraints/signature_manager_pre_execution_constraints.feature rename to libs/xtm-oaev-sdk/test/signatures/shim/constraints/signature_manager_pre_execution_constraints.feature diff --git a/test/signatures/features/signature_manager_post_execution.feature b/libs/xtm-oaev-sdk/test/signatures/shim/features/signature_manager_post_execution.feature similarity index 100% rename from test/signatures/features/signature_manager_post_execution.feature rename to libs/xtm-oaev-sdk/test/signatures/shim/features/signature_manager_post_execution.feature diff --git a/test/signatures/features/signature_manager_pre_execution.feature b/libs/xtm-oaev-sdk/test/signatures/shim/features/signature_manager_pre_execution.feature similarity index 100% rename from test/signatures/features/signature_manager_pre_execution.feature rename to libs/xtm-oaev-sdk/test/signatures/shim/features/signature_manager_pre_execution.feature diff --git a/test/signatures/test_ai_marker.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_ai_marker.py similarity index 100% rename from test/signatures/test_ai_marker.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_ai_marker.py diff --git a/test/signatures/test_ai_signature_types.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_ai_signature_types.py similarity index 100% rename from test/signatures/test_ai_signature_types.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_ai_signature_types.py diff --git a/libs/xtm-oaev-sdk/test/signatures/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_shim_contract.py new file mode 100644 index 00000000..fe4e380e --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/shim/test_shim_contract.py @@ -0,0 +1,149 @@ +"""Contract of the pyoaev signatures shims. + +Legacy imports must keep working and warn. They return the *identical* +xtm_oaev_sdk object, except for the renamed ``ExpectationType`` (now +``SignatureExpectationType``) and ``SignatureManager``, which resolves to a +subclass keeping the client-based constructor. +""" + +import subprocess +import sys +import warnings +from unittest.mock import MagicMock + +import pytest +import xtm_oaev_sdk as sdk + +_CONSUMER = "legacy_consumer.py" +_RENAMED = {"ExpectationType": "SignatureExpectationType"} + +_PACKAGE = [ + "CloudInjectorConfig", + "ExpectationSignatureGroup", + "ExpectationType", + "ExtraSignatureData", + "InjectorConfig", + "InjectExecutionActions", + "MatchTypes", + "NetworkInjectorConfig", + "SignatureCallbackPayload", + "SignaturePayload", + "SignatureTarget", + "SignatureTypes", + "SignatureValue", + "TargetSignatures", + "build_network_configs", +] +_MODELS = [ + "SignatureValue", + "ExpectationSignatureGroup", + "SignatureTarget", + "TargetSignatures", + "SignaturePayload", + "SignatureCallbackPayload", + "ExecutionSignature", + "ToolErrorInfo", + "ToolTimeoutInfo", + "ToolOutput", + "NetworkInjectorConfig", + "CloudInjectorConfig", + "InjectorConfig", + "build_network_configs", + "ExecutionDetails", + "ExtraSignatureData", + "SignatureOutputStructure", + "ExecutionStatus", + "ExpectationType", + "InjectExecutionActions", +] +_TYPES = [ + "ExpectationType", + "ExecutionStatus", + "InjectExecutionActions", + "MatchTypes", + "SignatureTypes", +] + +LEGACY_NAMES = ( + [("pyoaev", name) for name in _PACKAGE] + + [("pyoaev.signatures", name) for name in _PACKAGE] + + [("pyoaev.signatures.models", name) for name in _MODELS] + + [("pyoaev.signatures.types", name) for name in _TYPES] + + [("pyoaev.signatures.signature_match", "SignatureMatch")] + + [("pyoaev.signatures.signature_type", "SignatureType")] + + [("pyoaev.signatures.ai_marker", "build_marker")] + + [("pyoaev.signatures.signature_manager", "ExecutionDetails")] + + [("pyoaev.signatures.signature_manager", "OpenAEVError")] +) +LEGACY_MANAGERS = [ + "pyoaev", + "pyoaev.signatures", + "pyoaev.signatures.signature_manager", +] + + +def _given_fresh_interpreter_script(script: str) -> list[str]: + return [sys.executable, "-W", "error::DeprecationWarning", "-c", script] + + +def _when_run(command: list[str]) -> subprocess.CompletedProcess: + return subprocess.run(command, capture_output=True, text=True) + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + source = f"from {module} import {name} as _legacy" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(compile(source, _CONSUMER, "exec"), namespace) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + return namespace["_legacy"], deprecations + + +def _then_succeeds(result: subprocess.CompletedProcess) -> None: + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize(("module", "name"), LEGACY_NAMES) +def test_legacy_import_warns_and_returns_the_sdk_object(module, name) -> None: + """It warns towards xtm_oaev_sdk and returns the identical object.""" + sdk_name = _RENAMED.get(name, name) + legacy, deprecations = _when_import_legacy_name(module, name) + assert legacy is getattr(sdk, sdk_name) + assert deprecations + assert f"from xtm_oaev_sdk import {sdk_name}" in str(deprecations[0].message) + # attributed to the consumer's line, so default warning filters show it + assert any(w.filename == _CONSUMER for w in deprecations) + + +@pytest.mark.parametrize("module", LEGACY_MANAGERS) +def test_legacy_signature_manager_keeps_the_client_constructor(module) -> None: + """It warns and sends through ``client.signature`` like before.""" + legacy_manager, deprecations = _when_import_legacy_name(module, "SignatureManager") + assert deprecations + assert issubclass(legacy_manager, sdk.SignatureManager) + + client = MagicMock() + manager = legacy_manager(client, logger=None) + + assert manager.client is client + assert manager.transport is client.signature + + +def test_importing_pyoaev_modules_is_silent() -> None: + """It imports pyoaev and its signatures modules without any warning.""" + script = ( + "import pyoaev, pyoaev.apis, pyoaev.helpers, pyoaev.daemons\n" + "import pyoaev.signatures, pyoaev.signatures.models\n" + "import pyoaev.signatures.signature_manager, pyoaev.signatures.types\n" + "import pyoaev.signatures.signature_match\n" + "import pyoaev.signatures.signature_type, pyoaev.signatures.ai_marker\n" + ) + result = _when_run(_given_fresh_interpreter_script(script)) + _then_succeeds(result) + + +def test_unknown_legacy_name_still_raises() -> None: + """It does not swallow genuine misses.""" + with pytest.raises(ImportError): + exec("from pyoaev.signatures import DoesNotExist", {}) diff --git a/test/signatures/test_signature_manager_post_execution.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_signature_manager_post_execution.py similarity index 100% rename from test/signatures/test_signature_manager_post_execution.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_signature_manager_post_execution.py diff --git a/test/signatures/test_signature_manager_pre_execution.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_signature_manager_pre_execution.py similarity index 100% rename from test/signatures/test_signature_manager_pre_execution.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_signature_manager_pre_execution.py diff --git a/test/signatures/test_signature_match.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_signature_match.py similarity index 100% rename from test/signatures/test_signature_match.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_signature_match.py diff --git a/test/signatures/test_signature_type.py b/libs/xtm-oaev-sdk/test/signatures/shim/test_signature_type.py similarity index 100% rename from test/signatures/test_signature_type.py rename to libs/xtm-oaev-sdk/test/signatures/shim/test_signature_type.py diff --git a/libs/xtm-oaev-sdk/test/signatures/test_ai_signature_types.py b/libs/xtm-oaev-sdk/test/signatures/test_ai_signature_types.py new file mode 100644 index 00000000..c64a9298 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_ai_signature_types.py @@ -0,0 +1,28 @@ +import unittest + +from xtm_oaev_sdk import MatchTypes, SignatureType, SignatureTypes + + +class TestAiSignatureTypes(unittest.TestCase): + def test_ai_signature_type_values(self): + self.assertEqual( + SignatureTypes.SIG_TYPE_AI_REQUEST_MARKER.value, "ai_request_marker" + ) + self.assertEqual( + SignatureTypes.SIG_TYPE_AI_TARGET_ENDPOINT.value, "ai_target_endpoint" + ) + + def test_ai_request_marker_usable_in_signature_type(self): + signature_type = SignatureType( + label=SignatureTypes.SIG_TYPE_AI_REQUEST_MARKER, + match_type=MatchTypes.MATCH_TYPE_SIMPLE, + ) + + struct = signature_type.make_struct_for_matching(data="oaevdeadbeef") + + self.assertEqual(struct.get("type"), MatchTypes.MATCH_TYPE_SIMPLE.value) + self.assertEqual(struct.get("data"), "oaevdeadbeef") + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py new file mode 100644 index 00000000..4f488771 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py @@ -0,0 +1,110 @@ +import ipaddress +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from pytest_bdd import given, parsers, scenario, then, when +from xtm_oaev_sdk import SignatureManager + + +@scenario( + "features/signature_manager_ip_resolution.feature", + "resolve_container_ip returns a valid IPv4 in each supported execution environment", +) +def test_resolve_container_ip_returns_valid_ipv4(): + pass + + +@scenario( + "constraints/signature_manager_ip_resolution_constraints.feature", + "resolve_container_ip returns unknown and emits exactly one warning when all strategies fail", +) +def test_resolve_container_ip_returns_unknown_when_all_strategies_fail(): + pass + + +@pytest.fixture +def context(): + logger = MagicMock() + return { + "logger": logger, + "signature_manager": SignatureManager(transport=MagicMock(), logger=logger), + } + + +@given(parsers.parse('a SignatureManager running in a "{environment}" environment')) +def signature_manager_environment(context, monkeypatch, environment): + if environment == "Docker": + monkeypatch.setattr( + "xtm_oaev_sdk._core.signatures.signature_manager.socket.gethostbyname", + lambda _: "172.17.0.2", + ) + monkeypatch.setattr( + "xtm_oaev_sdk._core.signatures.signature_manager.subprocess.run", + lambda *args, **kwargs: SimpleNamespace( + returncode=0, + stdout="172.17.0.2\n", + ), + ) + elif environment == "Kubernetes": + monkeypatch.setattr( + "xtm_oaev_sdk._core.signatures.signature_manager.socket.gethostbyname", + lambda _: (_ for _ in ()).throw(OSError("socket fail")), + ) + monkeypatch.setattr( + "xtm_oaev_sdk._core.signatures.signature_manager.subprocess.run", + lambda *args, **kwargs: SimpleNamespace( + returncode=0, + stdout="10.244.0.8\n", + ), + ) + else: + monkeypatch.setattr( + "xtm_oaev_sdk._core.signatures.signature_manager.socket.gethostbyname", + lambda _: "192.0.2.20", + ) + + +@given("all IP resolution strategies are mocked to fail") +def all_ip_resolution_strategies_fail(context, monkeypatch): + monkeypatch.setattr( + "xtm_oaev_sdk._core.signatures.signature_manager.socket.gethostbyname", + lambda _: (_ for _ in ()).throw(OSError("socket fail")), + ) + monkeypatch.setattr( + "xtm_oaev_sdk._core.signatures.signature_manager.subprocess.run", + lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("command fail")), + ) + + +@when("I call resolve_container_ip") +def call_resolve_container_ip(context): + context["resolve_exception"] = None + try: + context["resolved_ip"] = context["signature_manager"].resolve_container_ip() + except Exception as exc: + context["resolve_exception"] = exc + + +@then("the returned value is a non-empty valid IPv4 address string") +def assert_returned_value_valid_ipv4(context): + assert context["resolve_exception"] is None + resolved_ip = context["resolved_ip"] + assert isinstance(resolved_ip, str) + assert resolved_ip.strip() != "" + assert ipaddress.ip_address(resolved_ip).version == 4 + + +@then(parsers.parse('the returned value is the string "{expected_value}"')) +def assert_returned_value_matches(context, expected_value): + assert context["resolved_ip"] == expected_value + + +@then(parsers.parse("exactly {count:d} WARNING log message is emitted")) +def assert_warning_count(context, count): + assert context["logger"].warning.call_count == count + + +@then("no exception propagates from resolve_container_ip") +def assert_no_exception_from_resolve_container_ip(context): + assert context["resolve_exception"] is None diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_post_execution.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_post_execution.py new file mode 100644 index 00000000..32c1f905 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_post_execution.py @@ -0,0 +1,223 @@ +from datetime import datetime +from unittest.mock import MagicMock + +import pytest +from pytest_bdd import given, parsers, scenario, then, when +from xtm_oaev_sdk import ( + ExecutionDetails, + ExecutionSignature, + ExecutionStatus, + SignatureManager, +) + + +@scenario( + "features/signature_manager_post_execution.feature", + "Successful execution updates end_time and execution_status in execution signatures and execution details", +) +def test_successful_execution_merges_post_execution_fields(): + pass + + +@scenario( + "constraints/signature_manager_post_execution_constraints.feature", + "Tool crash sets execution_status to failed and uses crash timestamp as end_time", +) +def test_tool_crash_sets_failed_status_and_crash_timestamp_end_time(): + pass + + +@scenario( + "constraints/signature_manager_post_execution_constraints.feature", + "Timeout sets execution_status to timeout and includes available partial results", +) +def test_timeout_sets_timeout_status_and_includes_partial_results(): + pass + + +@scenario( + "constraints/signature_manager_post_execution_constraints.feature", + "Timeout with no partial results still sets execution_status to timeout", +) +def test_timeout_without_partial_results_still_sets_timeout_status(): + pass + + +@pytest.fixture +def context(): + return {} + + +def _parse_iso8601_utc(value: str) -> datetime: + return datetime.fromisoformat(value.replace("Z", "+00:00")) + + +@given( + "a SignatureManager initialised with constructor SignatureManager(transport, logger)" +) +def signature_manager(context): + context["signature_manager"] = SignatureManager( + transport=MagicMock(), logger=MagicMock() + ) + + +@given( + "a execution_signatures object containing:", + target_fixture="execution_signatures", +) +def execution_signatures(): + return ExecutionSignature( + source_ipv4="172.17.0.2", + target_ipv4="10.0.0.1", + target_hostname="host-a.internal", + start_time="2024-06-26T06:00:00Z", + ) + + +@given( + "a execution_details object containing:", + target_fixture="execution_details", +) +def execution_details(): + return ExecutionDetails( + execution_status=ExecutionStatus.INFO, + start_time=datetime.strptime("2024-06-26T06:00:00Z", "%Y-%m-%dT%H:%M:%SZ"), + ) + + +@given( + "a tool_output indicating successful completion with no errors and no timeout", + target_fixture="tool_output", +) +def successful_tool_output(): + return {"status": "EXECUTED"} + + +@given( + 'a tool_output containing error_info with exit_code=1 and crash_timestamp="2024-06-26T06:05:00Z"', + target_fixture="tool_output", +) +def crashed_tool_output(): + return { + "error_info": { + "exit_code": 1, + "crash_timestamp": "2024-06-26T06:05:00Z", + } + } + + +@given( + 'a tool_output containing timeout_info with partial_results=["result-A", "result-B"]', + target_fixture="tool_output", +) +def timeout_tool_output_with_partial_results(): + return {"timeout_info": {"partial_results": ["result-A", "result-B"]}} + + +@given( + "a tool_output containing timeout_info with no partial results available", + target_fixture="tool_output", +) +def timeout_tool_output_with_no_partial_results(): + return {"timeout_info": {"partial_results": []}} + + +@when( + "I call post_execution_updates with the execution_details, execution_signatures and tool_output" +) +def post_execution_update( + context, execution_details, execution_signatures, tool_output +): + context["signature_manager"].post_execution_updates( + execution_details, execution_signatures, tool_output + ) + context["execution_details_result"] = execution_details + context["execution_signatures_result"] = execution_signatures + + +@then("the execution signature model contains every previous parameter unchanged") +def execution_signatures_unchanged(context, execution_signatures): + basic_exec_sig = execution_signatures + exec_sig_result = context["execution_signatures_result"] + + assert exec_sig_result.source_ipv4 == basic_exec_sig.source_ipv4 + assert exec_sig_result.source_ipv6 == basic_exec_sig.source_ipv6 + assert exec_sig_result.target_ipv4 == basic_exec_sig.target_ipv4 + assert exec_sig_result.target_ipv6 == basic_exec_sig.target_ipv6 + assert exec_sig_result.target_hostname == basic_exec_sig.target_hostname + assert exec_sig_result.cloud_provider == basic_exec_sig.cloud_provider + assert exec_sig_result.cloud_account_id == basic_exec_sig.cloud_account_id + assert exec_sig_result.cloud_region == basic_exec_sig.cloud_region + assert exec_sig_result.target_service == basic_exec_sig.target_service + + +@then( + "the end_time parameter in the execution signature model is a UTC ISO 8601 string" +) +def result_contains_iso8601_end_time(context): + end_time = context["execution_signatures_result"].end_time + assert isinstance(end_time, str) + _parse_iso8601_utc(end_time) + + +@then( + parsers.parse( + 'this end_time is chronologically greater than or equal to start_time "{start_time}"' + ) +) +def end_time_at_or_after_start_time(context, start_time): + end_time_dt = _parse_iso8601_utc(context["execution_signatures_result"].end_time) + start_time_dt = _parse_iso8601_utc(start_time) + assert end_time_dt >= start_time_dt + + +@then(parsers.parse('end_time equals "{expected_end_time}"')) +def end_time_equals(context, expected_end_time): + assert context["execution_signatures_result"].end_time == expected_end_time + + +@then( + 'the returned dict contains the partial results ["result-A", "result-B"] from timeout_info' +) +def contains_timeout_partial_results(context): + assert context["execution_signatures_result"].partial_results == [ + "result-A", + "result-B", + ] + + +@then("the execution details model contain every previous parameter pair unchanged") +def execution_details_unchanged(context, execution_details): + basic_exec_details = execution_details + exec_details_result = context["execution_details_result"] + + assert exec_details_result.start_time == basic_exec_details.start_time + assert exec_details_result.execution_message == basic_exec_details.execution_message + + +@then("the end_time parameter in the execution details model is a datetime object") +def result_contains_datetime_end_time(context): + end_time = context["execution_details_result"].end_time + assert isinstance(end_time, datetime) + + +@then( + parsers.parse( + 'the execution_status parameter in the execution details model is equal to "{status}"' + ) +) +@then(parsers.parse('execution_status equals "{status}"')) +def execution_status_equals(context, status): + assert context["execution_details_result"].execution_status == ExecutionStatus( + status + ) + + +@then( + parsers.parse( + 'the execution_action parameter in the execution details model is equal to "{action}"' + ) +) +@then(parsers.parse('execution_action equals "{action}"')) +def execution_action_equals(context, action): + assert context["execution_details_result"].execution_action == action diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py new file mode 100644 index 00000000..22f967f4 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py @@ -0,0 +1,466 @@ +import ipaddress +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock, patch + +import pytest +from pytest_bdd import given, parsers, scenario, then, when +from xtm_oaev_sdk import ( + CloudInjectorConfig, + ExecutionSignature, + NetworkInjectorConfig, + SignatureManager, + build_network_configs, +) + +# -------------------------------------------------- +# SCENARIOS +# -------------------------------------------------- + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Network category returns required IP and timing fields and no cloud or query fields", +) +def test_network_category_required_fields(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Network hostname target returns hostname and no target_ipv4", +) +def test_network_hostname_target(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Cloud category returns required cloud identity fields and no IP fields", +) +def test_cloud_category_required_fields(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Network multi-target returns one dict per target with a shared source IP", +) +def test_network_multi_target(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "All network multi-target dicts share the same source_ipv4", +) +def test_network_multi_target_shared_source(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Cloud multi-region returns one dict per region with a shared account ID", +) +def test_cloud_multi_region(): + pass + + +@scenario( + "features/signature_manager_pre_execution.feature", + "Builder classifies a mixed list of targets into typed configs", +) +def test_builder_classifies_mixed_targets(): + pass + + +@scenario( + "constraints/signature_manager_pre_execution_constraints.feature", + "start_time is generated at method-call time not at class instantiation time", +) +def test_start_time_generated_at_call_time(): + pass + + +# -------------------------------------------------- +# FIXTURE CONTEXT +# -------------------------------------------------- + + +@pytest.fixture +def context(): + return {} + + +# -------------------------------------------------- +# HELPERS +# -------------------------------------------------- + + +def parse_utc_iso8601(value): + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + return parsed.astimezone(timezone.utc) + + +# -------------------------------------------------- +# GIVEN +# -------------------------------------------------- + + +@given( + "a SignatureManager initialised with constructor SignatureManager(transport, logger)", + target_fixture="signature_manager", +) +def signature_manager(): + return SignatureManager(transport=MagicMock(), logger=None) + + +@given( + parsers.parse('a NetworkInjectorConfig with target_ipv4="{target_ipv4}"'), + target_fixture="config", +) +def network_config_ipv4(target_ipv4): + return NetworkInjectorConfig(target_ipv4=target_ipv4) + + +@given( + parsers.parse('a NetworkInjectorConfig with target_hostname="{target_hostname}"'), + target_fixture="config", +) +def network_config_hostname(target_hostname): + return NetworkInjectorConfig(target_hostname=target_hostname) + + +@given( + parsers.parse( + 'a CloudInjectorConfig with cloud_provider="{cloud_provider}", ' + 'cloud_account_id="{cloud_account_id}", cloud_region="{cloud_region}", ' + 'and target_service="{target_service}"' + ), + target_fixture="config", +) +def cloud_config_single( + cloud_provider, + cloud_account_id, + cloud_region, + target_service, +): + return CloudInjectorConfig( + cloud_provider=cloud_provider, + cloud_account_id=cloud_account_id, + cloud_region=cloud_region, + target_service=target_service, + ) + + +@given( + parsers.parse( + "a list of 3 NetworkInjectorConfig with target_ipv4 " + '"{ip_1}", "{ip_2}", "{ip_3}"' + ), + target_fixture="config", +) +def network_config_list(ip_1, ip_2, ip_3): + return [ + NetworkInjectorConfig(target_ipv4=ip_1), + NetworkInjectorConfig(target_ipv4=ip_2), + NetworkInjectorConfig(target_ipv4=ip_3), + ] + + +@given( + "a list of 3 NetworkInjectorConfig built from default IPv4 targets", + target_fixture="config", +) +def network_config_list_default(): + return [ + NetworkInjectorConfig(target_ipv4="10.0.0.1"), + NetworkInjectorConfig(target_ipv4="10.0.0.2"), + NetworkInjectorConfig(target_ipv4="10.0.0.3"), + ] + + +@given( + parsers.parse( + 'a list of 3 CloudInjectorConfig with cloud_account_id="{cloud_account_id}" ' + 'and regions "{region_1}", "{region_2}", "{region_3}"' + ), + target_fixture="config", +) +def cloud_config_list(cloud_account_id, region_1, region_2, region_3): + return [ + CloudInjectorConfig( + cloud_provider="aws", + cloud_account_id=cloud_account_id, + cloud_region=region, + ) + for region in (region_1, region_2, region_3) + ] + + +@given( + parsers.parse('a raw mixed target list "{value_1}", "{value_2}", "{value_3}"'), + target_fixture="raw_targets", +) +def raw_mixed_target_list(value_1, value_2, value_3): + return [value_1, value_2, value_3] + + +@given( + "the running container has a resolvable IPv4 address", + target_fixture="source_ipv4", +) +def resolvable_container_ipv4(request): + patcher = patch( + "xtm_oaev_sdk._core.signatures.signature_manager.SignatureManager.resolve_container_ip", + return_value="172.17.0.2", + ) + patcher.start() + request.addfinalizer(patcher.stop) + return "172.17.0.2" + + +@given( + parsers.parse( + 'the running container has a resolvable IPv4 address "{source_ipv4}"' + ), + target_fixture="source_ipv4", +) +def resolvable_container_ipv4_explicit(request, source_ipv4): + patcher = patch( + "xtm_oaev_sdk._core.signatures.signature_manager.SignatureManager.resolve_container_ip", + return_value=source_ipv4, + ) + patcher.start() + request.addfinalizer(patcher.stop) + return source_ipv4 + + +@given( + "a SignatureManager that was instantiated at timestamp T0", + target_fixture="signature_manager", +) +def signature_manager_at_t0(context): + t0 = datetime(2024, 6, 26, 6, 6, 1, tzinfo=timezone.utc) + context["t0"] = t0 + manager = SignatureManager(transport=MagicMock(), logger=None) + manager._test_t0 = t0 + return manager + + +@given( + "5 seconds elapse after instantiation", + target_fixture="t1", +) +def elapsed_5_seconds(context): + t1 = context["t0"] + timedelta(seconds=5) + context["t1"] = t1 + return t1 + + +# -------------------------------------------------- +# WHEN +# -------------------------------------------------- + + +@when( + "I call build_execution_signatures with the config", + target_fixture="result", +) +def call_build_with_config(signature_manager, config): + return signature_manager.build_execution_signatures(config=config) + + +@when( + "I call build_execution_signatures with the config list", + target_fixture="result", +) +def call_build_with_config_list(signature_manager, config): + return signature_manager.build_execution_signatures(config=config) + + +@when( + "I call build_execution_signatures with the config at timestamp T1", + target_fixture="result", +) +def call_build_at_t1(signature_manager, config, t1): + with patch.object(signature_manager, "_utcnow", return_value=t1): + return signature_manager.build_execution_signatures(config=config) + + +@when( + "I build network configs from the raw list", + target_fixture="built_configs", +) +def build_configs_from_raw(raw_targets): + return build_network_configs(raw_targets) + + +# -------------------------------------------------- +# THEN +# -------------------------------------------------- + + +@then("the returned dict contains source_ipv4 as a non-empty valid IPv4 address string") +def source_ipv4_is_valid(result): + source_ipv4 = result.source_ipv4 + assert source_ipv4 + ipaddress.IPv4Address(source_ipv4) + + +@then("the returned dict contains start_time as a UTC ISO 8601 string") +def start_time_is_utc_iso8601(result): + start_time = result.start_time + parsed = parse_utc_iso8601(start_time) + assert parsed.tzinfo is not None + + +@then(parsers.parse('the returned dict contains target_ipv4 equal to "{value}"')) +def returned_dict_target_ipv4(result, value): + assert result.target_ipv4 == value + + +@then(parsers.parse('the returned dict contains target_hostname equal to "{value}"')) +def returned_dict_target_hostname(result, value): + assert result.target_hostname == value + + +@then(parsers.parse('the returned dict contains cloud_provider equal to "{value}"')) +def returned_dict_cloud_provider(result, value): + assert result.cloud_provider == value + + +@then(parsers.parse('the returned dict contains cloud_account_id equal to "{value}"')) +def returned_dict_cloud_account_id(result, value): + assert result.cloud_account_id == value + + +@then(parsers.parse('the returned dict contains cloud_region equal to "{value}"')) +def returned_dict_cloud_region(result, value): + assert result.cloud_region == value + + +@then(parsers.parse('the returned dict contains target_service equal to "{value}"')) +def returned_dict_target_service(result, value): + assert result.target_service == value + + +@then(parsers.parse('the returned dict contains query equal to "{value}"')) +def returned_dict_query(result, value): + assert result.query == value + + +@then(parsers.parse("the returned dict does not contain {field}")) +def returned_dict_does_not_contain_field(result, field): + assert field not in result + + +@then("the return value is a list of exactly 3 dicts") +def return_value_is_list_of_three_dicts(result): + assert isinstance(result, list) + assert len(result) == 3 + assert all(isinstance(item, ExecutionSignature) for item in result) + + +@then(parsers.parse("the return value is a list of {count:d} dicts")) +def return_value_is_list_of_n_dicts(result, count): + assert isinstance(result, list) + assert len(result) == count + assert all(isinstance(item, ExecutionSignature) for item in result) + + +@then( + parsers.parse( + 'the dict at position {index:d} contains target_ipv4 equal to "{target_ip}"' + ) +) +def list_dict_contains_target_ipv4_at_position(result, index, target_ip): + assert result[index].target_ipv4 == target_ip + + +@then( + parsers.parse( + 'the dict at position {index:d} contains source_ipv4 equal to "{source_ipv4}"' + ) +) +def list_dict_contains_source_ipv4_at_position( + result, + index, + source_ipv4, +): + assert result[index].source_ipv4 == source_ipv4 + + +@then( + parsers.parse( + 'the dict at position {index:d} contains cloud_region equal to "{region}"' + ) +) +def list_dict_contains_cloud_region_at_position(result, index, region): + assert result[index].cloud_region == region + + +@then( + parsers.parse( + 'the dict at position {index:d} contains cloud_account_id equal to "{account_id}"' + ) +) +def list_dict_contains_cloud_account_id_at_position(result, index, account_id): + assert result[index].cloud_account_id == account_id + + +@then("all 3 dicts contain the same source_ipv4 value") +def all_dicts_share_same_source_ipv4(result): + assert isinstance(result, list) + assert len(result) == 3 + source_values = {item.source_ipv4 for item in result} + assert len(source_values) == 1 + ipaddress.IPv4Address(next(iter(source_values))) + + +@then("the start_time in the returned dict equals T1 within 1 second tolerance") +def start_time_equals_t1_with_tolerance(result, t1): + start_time = parse_utc_iso8601(result.start_time) + delta_seconds = abs((start_time - t1).total_seconds()) + assert delta_seconds <= 1 + + +@then("start_time does not equal T0") +def start_time_not_equal_t0(result, signature_manager): + start_time = parse_utc_iso8601(result.start_time) + assert start_time != signature_manager._test_t0 + + +@then(parsers.parse("the builder returns {count:d} NetworkInjectorConfig")) +def builder_returns_n_configs(built_configs, count): + assert isinstance(built_configs, list) + assert len(built_configs) == count + assert all(isinstance(c, NetworkInjectorConfig) for c in built_configs) + + +@then( + parsers.parse('the config at position {index:d} has target_ipv4 equal to "{value}"') +) +def config_has_target_ipv4(built_configs, index, value): + assert built_configs[index].target_ipv4 == value + assert built_configs[index].target_ipv6 is None + assert built_configs[index].target_hostname is None + + +@then( + parsers.parse('the config at position {index:d} has target_ipv6 equal to "{value}"') +) +def config_has_target_ipv6(built_configs, index, value): + assert built_configs[index].target_ipv6 == value + assert built_configs[index].target_ipv4 is None + assert built_configs[index].target_hostname is None + + +@then( + parsers.parse( + 'the config at position {index:d} has target_hostname equal to "{value}"' + ) +) +def config_has_target_hostname(built_configs, index, value): + assert built_configs[index].target_hostname == value + assert built_configs[index].target_ipv4 is None + assert built_configs[index].target_ipv6 is None diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_match.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_match.py new file mode 100644 index 00000000..ce5f2234 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_match.py @@ -0,0 +1,23 @@ +import unittest + +from xtm_oaev_sdk import MatchTypes, OpenAEVError, SignatureMatch + + +class TestSignatureMatch(unittest.TestCase): + def test_non_simple_match_with_non_null_score_throws(self): + score = None + self.assertRaises( + OpenAEVError, SignatureMatch, MatchTypes.MATCH_TYPE_FUZZY, score + ) + + def test_simple_match_with_null_score_does_not_throw(self): + score = None + SignatureMatch(match_type=MatchTypes.MATCH_TYPE_SIMPLE, match_score=score) + + def test_fuzzy_match_with_0_score_does_not_throw(self): + score = 0 + SignatureMatch(match_type=MatchTypes.MATCH_TYPE_FUZZY, match_score=score) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_type.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_type.py new file mode 100644 index 00000000..1747b813 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_type.py @@ -0,0 +1,56 @@ +import unittest + +from xtm_oaev_sdk import MatchTypes, SignatureType, SignatureTypes + + +class TestSignatureType(unittest.TestCase): + def test_make_struct_create_expected_struct_for_simple_sig_type(self): + simple_signature_type_label = SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS + simple_signature_type = SignatureType( + label=simple_signature_type_label, match_type=MatchTypes.MATCH_TYPE_SIMPLE + ) + + data = "just a simple string" + simple_struct = simple_signature_type.make_struct_for_matching(data=data) + + self.assertEqual(simple_struct.get("type"), MatchTypes.MATCH_TYPE_SIMPLE.value) + self.assertEqual(simple_struct.get("data"), data) + self.assertFalse("score" in simple_struct.keys()) + + def test_make_struct_create_expected_struct_for_fuzzy_sig_type(self): + fuzzy_signature_type_label = SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS + fuzzy_signature_type_score = 50 + fuzzy_signature_type = SignatureType( + label=fuzzy_signature_type_label, + match_type=MatchTypes.MATCH_TYPE_FUZZY, + match_score=fuzzy_signature_type_score, + ) + + data = "just another simple string" + simple_struct = fuzzy_signature_type.make_struct_for_matching(data=data) + + self.assertEqual(simple_struct.get("type"), MatchTypes.MATCH_TYPE_FUZZY.value) + self.assertEqual(simple_struct.get("data"), data) + self.assertEqual(simple_struct.get("score"), fuzzy_signature_type_score) + + def test_make_struct_create_expected_struct_for_fuzzy_sig_type_when_score_is_0( + self, + ): + fuzzy_signature_type_label = SignatureTypes.SIG_TYPE_TARGET_HOSTNAME_ADDRESS + fuzzy_signature_type_score = 0 + fuzzy_signature_type = SignatureType( + label=fuzzy_signature_type_label, + match_type=MatchTypes.MATCH_TYPE_FUZZY, + match_score=fuzzy_signature_type_score, + ) + + data = "just another simple string" + simple_struct = fuzzy_signature_type.make_struct_for_matching(data=data) + + self.assertEqual(simple_struct.get("type"), MatchTypes.MATCH_TYPE_FUZZY.value) + self.assertEqual(simple_struct.get("data"), data) + self.assertEqual(simple_struct.get("score"), fuzzy_signature_type_score) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/test_boundaries.py b/libs/xtm-oaev-sdk/test/test_boundaries.py index 6c57f6ac..f26f4429 100644 --- a/libs/xtm-oaev-sdk/test/test_boundaries.py +++ b/libs/xtm-oaev-sdk/test/test_boundaries.py @@ -2,6 +2,8 @@ import pathlib import re +import subprocess +import sys import pytest @@ -40,3 +42,24 @@ def test_sdk_sources_do_not_import(package: str) -> None: def test_sdk_sources_do_not_define_client_helpers(artifact: str) -> None: """It leaves transport and daemon helpers to the client.""" assert _when_searching_sdk_sources(rf"^\s*{artifact}\b") == [] + + +def _sdk_modules() -> list[str]: + package_root = _SDK_ROOT.parent + return sorted( + ".".join(path.relative_to(package_root).with_suffix("").parts).removesuffix( + ".__init__" + ) + for path in _SDK_ROOT.rglob("*.py") + ) + + +@pytest.mark.parametrize("module", _sdk_modules()) +def test_every_sdk_module_imports_first_in_a_fresh_interpreter(module: str) -> None: + """It has no import cycle hidden by a lucky import order.""" + result = subprocess.run( + [sys.executable, "-W", "error", "-c", f"import {module}"], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py index 0698318c..62a7b8e3 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -4,8 +4,11 @@ _core/ Shared implementations (errors, utils, ssl_utils, asset_types, security_domain) _core/configuration/ Configuration feature implementation + _core/signatures/ Signature feature implementation seam/common/ Shared errors, enums, types, and utilities seam/configuration/ Configuration I/O surface + seam/signatures/ Signatures inbound I/O seam (versioned) + seam/port_io/ Port I/O wire schema (versioned, seam-owned) seam/deprecation.py PEP 562 deprecation primitive public/ User-facing re-exports diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py new file mode 100644 index 00000000..b4eed062 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py @@ -0,0 +1,71 @@ +"""Public surface for the signatures subpackage. + +This module re-exports the canonical signature types, models, and manager +protocols used by the SDK engine signatures package. +""" + +from .ai_marker import build_marker +from .models import ( + CloudInjectorConfig, + ExecutionDetails, + ExecutionSignature, + ExpectationSignatureGroup, + ExtraSignatureData, + InjectorConfig, + NetworkInjectorConfig, + SignatureCallbackPayload, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, + ToolErrorInfo, + ToolOutput, + ToolTimeoutInfo, + build_network_configs, +) +from .signature_manager import ( + SignatureManager, + SignatureManagerProtocol, + SignatureTransportProtocol, +) +from .signature_match import SignatureMatch +from .signature_type import SignatureType +from .types import ( + ExecutionStatus, + InjectExecutionActions, + MatchTypes, + SignatureExpectationType, + SignatureTypes, +) + +__all__ = [ + "CloudInjectorConfig", + "ExecutionDetails", + "ExecutionSignature", + "ExecutionStatus", + "ExpectationSignatureGroup", + "ExtraSignatureData", + "InjectExecutionActions", + "InjectorConfig", + "MatchTypes", + "NetworkInjectorConfig", + "SignatureCallbackPayload", + "SignatureExpectationType", + "SignatureManager", + "SignatureManagerProtocol", + "SignatureMatch", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureTransportProtocol", + "SignatureType", + "SignatureTypes", + "SignatureValue", + "TargetSignatures", + "ToolErrorInfo", + "ToolOutput", + "ToolTimeoutInfo", + "build_marker", + "build_network_configs", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/ai_marker.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/ai_marker.py new file mode 100644 index 00000000..67cd8638 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/ai_marker.py @@ -0,0 +1,17 @@ +"""Deterministic per-inject canary marker shared by the AI red-team injector and the AI defense +collectors. + +The marker is derived purely from the inject (and optional agent) id, so the injector that sends the +attack and the collector that validates an AI defense response compute the same value independently, +without the platform having to store it. It is emitted by the injector (request header + in-prompt +token) and matched by collectors against guardrail / firewall logs. +""" + +import hashlib + +__all__ = ["build_marker"] + + +def build_marker(inject_id: str, agent_id: str = "") -> str: + seed = f"{inject_id}:{agent_id}".encode("utf-8") + return "oaev" + hashlib.sha256(seed).hexdigest()[:16] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/models.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/models.py new file mode 100644 index 00000000..9d3756d1 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/models.py @@ -0,0 +1,129 @@ +"""Pydantic schemas pinning every shape SignatureManager touches. + +The seam models live in the architectural seam layer and are re-exported +here so this module keeps its historical surface: + +- the inbound seam (the ``InjectorConfig`` family, ``ExecutionSignature``, + and the ``ToolOutput`` family) in + ``xtm_oaev_sdk.seam.signatures.seam`` (versioned ``SEAM_VERSION``); +- the observation / callback wire models (the port I/O contract) in + ``xtm_oaev_sdk.seam.port_io.models`` (versioned + ``PORT_IO_CONTRACT_VERSION``). + +Only ``ExtraSignatureData`` and the target-builder helpers are defined here +— they are the port's own shapes, internal to the capability. +""" + +import ipaddress +from typing import Any + +from pydantic import BaseModel, Field, JsonValue +from xtm_oaev_sdk.seam.port_io.models import ( + ExecutionDetails, + ExpectationSignatureGroup, + SignatureCallbackPayload, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, +) +from xtm_oaev_sdk.seam.signatures.seam import ( + CloudInjectorConfig, + ExecutionSignature, + InjectorConfig, + NetworkInjectorConfig, + ToolErrorInfo, + ToolOutput, + ToolTimeoutInfo, +) + + +class ExtraSignatureData(BaseModel): + """Format for extra signatures added to the default signatures.""" + + detection: dict[str, JsonValue] = Field(default_factory=dict) + prevention: dict[str, JsonValue] = Field(default_factory=dict) + vulnerability: dict[str, JsonValue] = Field(default_factory=dict) + + def get_extra(self, expectation_type: str) -> dict[str, JsonValue]: + if expectation_type.lower() == "detection": + return self.detection + if expectation_type.lower() == "prevention": + return self.prevention + if expectation_type.lower() == "vulnerability": + return self.vulnerability + raise ValueError( + f"Expectation type should be one of the available parameters: {list(self.model_fields.keys())}" + ) + + +# --------------------------------------------------------------------------- +# Builders. Cheap helpers to turn raw injector input into typed configs. +# --------------------------------------------------------------------------- + + +def _classify_network_target(value: str) -> NetworkInjectorConfig: + """Decide whether ``value`` is an IPv4, IPv6, or hostname and wrap it.""" + try: + addr = ipaddress.ip_address(value) + except ValueError: + return NetworkInjectorConfig(target_hostname=value) + if isinstance(addr, ipaddress.IPv4Address): + return NetworkInjectorConfig(target_ipv4=value) + return NetworkInjectorConfig(target_ipv6=value) + + +def build_network_configs( + targets: list[str | dict[str, Any] | NetworkInjectorConfig], +) -> list[NetworkInjectorConfig]: + """Forge a list of `NetworkInjectorConfig` from a heterogeneous target list. + + Each item is one distinct asset. Accepted shapes: + + - `NetworkInjectorConfig`: passed through unchanged. + - `dict`: validated against :class:`NetworkInjectorConfig`. + - `str`: auto-classified into IPv4 / IPv6 / hostname. + + Args: + targets: Raw target list straight out of the injector. + + Returns: + One `NetworkInjectorConfig` per input target, order preserved. + + Raises: + TypeError: An item is not one of the accepted shapes. + ValidationError: A dict item fails the one-identity invariant. + """ + configs: list[NetworkInjectorConfig] = [] + for target in targets: + if isinstance(target, NetworkInjectorConfig): + configs.append(target) + elif isinstance(target, dict): + configs.append(NetworkInjectorConfig(**target)) + elif isinstance(target, str): + configs.append(_classify_network_target(target)) + else: + raise TypeError(f"unsupported network target type: {type(target).__name__}") + return configs + + +__all__ = [ + "SignatureValue", + "ExpectationSignatureGroup", + "ExtraSignatureData", + "SignatureTarget", + "TargetSignatures", + "SignaturePayload", + "SignatureOutputStructure", + "ExecutionDetails", + "SignatureCallbackPayload", + "ExecutionSignature", + "ToolErrorInfo", + "ToolTimeoutInfo", + "ToolOutput", + "NetworkInjectorConfig", + "CloudInjectorConfig", + "InjectorConfig", + "build_network_configs", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py new file mode 100644 index 00000000..997b89e7 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py @@ -0,0 +1,359 @@ +"""Signature lifecycle for OpenAEV injectors: compile pre, merge post, ship to backend.""" + +import logging +import os +import socket +import subprocess +from datetime import datetime, timezone +from typing import Any, Protocol, runtime_checkable + +from pydantic import ValidationError +from xtm_oaev_sdk._core.errors import OpenAEVError +from xtm_oaev_sdk._core.signatures.models import ( + CloudInjectorConfig, + ExecutionDetails, + ExecutionSignature, + ExpectationSignatureGroup, + ExtraSignatureData, + InjectorConfig, + NetworkInjectorConfig, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, + ToolOutput, +) + +__all__ = [ + "SignatureManager", + "SignatureManagerProtocol", + "SignatureTransportProtocol", +] + + +@runtime_checkable +class SignatureTransportProtocol(Protocol): + """Transport contract for signature delivery. + + pyoaev implements this via its Signature API manager. + The SDK only knows this interface, never the concrete client. + """ + + def send_signatures( + self, + inject_id: str, + sig_output: SignatureOutputStructure, + execution_details: ExecutionDetails, + *, + max_payload_size: int, + logger: logging.Logger, + ) -> None: + """Transmit signatures to the platform callback endpoint.""" + ... + + +@runtime_checkable +class SignatureManagerProtocol(Protocol): + """Behavioral contract for the signature lifecycle manager. + + Covers the full compile → merge → transmit pipeline. + """ + + def build_execution_signatures( + self, + config: InjectorConfig | list[InjectorConfig], + ) -> ExecutionSignature | list[ExecutionSignature]: + """Build pre-execution signatures from injector configs.""" + ... + + def post_execution_updates( + self, + execution_details: ExecutionDetails, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + tool_output: dict[str, Any], + ) -> None: + """Update execution metadata after tool completion.""" + ... + + def send_signatures( + self, + inject_id: str, + execution_details: ExecutionDetails, + signatures: dict[str, Any], + ) -> None: + """Ship signatures to the callback endpoint.""" + ... + + def resolve_container_ip(self) -> str: + """Resolve the container's primary IPv4 address.""" + ... + + +class SignatureManager: + """End-to-end signature pipeline: compile, merge, transmit. One class, three jobs.""" + + DEFAULT_MAX_PAYLOAD_SIZE = 5_242_880 # 5 MiB + + def __init__( + self, + transport: SignatureTransportProtocol, + logger: logging.Logger | None = None, + max_payload_size: int = DEFAULT_MAX_PAYLOAD_SIZE, + ) -> None: + """Initialize manager with a transport implementation for signature delivery.""" + self.transport = transport + self.logger = logger or logging.getLogger(__name__) + self.max_payload_size = max_payload_size + self._cached_ipv4: str | None = None + self._cached_ipv6: str | None = None + + def _utcnow(self) -> datetime: + """Current UTC time. Carved out so tests can pin the clock.""" + return datetime.now(timezone.utc) + + def build_execution_signatures( + self, + config: InjectorConfig | list[InjectorConfig], + ) -> ExecutionSignature | list[ExecutionSignature]: + """Build pre-execution signature dicts from one or more typed injector configs. + + The category is carried by the config type itself + (:class:`NetworkInjectorConfig`, :class:`CloudInjectorConfig`), + so no separate ``category`` flag is needed. + + Args: + config: A single injector config or a homogeneous list of them. + Multi-target injects must be expressed as a list. + + Returns: + One ExecutionSignature object when a single config is given, + otherwise a list of ExecutionSignature in input order. + + Raises: + ValueError: Empty list, or mixed config types in a single call. + TypeError: Unknown injector config type. + """ + configs = list(config) if isinstance(config, list) else [config] + if not configs: + raise ValueError("build_execution_signatures requires at least one config") + + first_type = type(configs[0]) + for c in configs: + if not isinstance(c, first_type): + raise ValueError( + "build_execution_signatures does not mix injector config types; " + f"got {sorted({type(c).__name__ for c in configs})}" + ) + + start_time = self._utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") + results = [self._build_one(cfg, start_time) for cfg in configs] + return results[0] if len(results) == 1 else results + + def _build_one(self, config: InjectorConfig, start_time: str) -> ExecutionSignature: + """Project a single injector config into a flat pre-execution signature dict. + + Common pipeline for every category: + 1. Seed the base dict with ``start_time`` and category-specific context + (network gets resolved source IPs; cloud add nothing). + 2. Layer the config's own fields on top. + 3. Run it through :class:`ExecutionSignature` for validation + and emit JSON-ready output stripped of ``None``\\s. + """ + base: dict[str, Any] = {"start_time": start_time} + base.update(self._source_context(config)) + base.update(config.model_dump(exclude_none=True)) + return ExecutionSignature(**base) + + def _source_context(self, config: InjectorConfig) -> dict[str, Any]: + """Return the source identity bits injected for the config's category. + + Only network signatures need the running container's source IPs; + cloud rows have no source identity to carry. + """ + if isinstance(config, NetworkInjectorConfig): + return { + "source_ipv4": self.resolve_container_ip(), + "source_ipv6": self._cached_ipv6, + } + if isinstance(config, CloudInjectorConfig): + return {} + raise TypeError(f"unsupported injector config type: {type(config).__name__}") + + def post_execution_updates( + self, + execution_details: ExecutionDetails, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + tool_output: dict[str, Any], + ) -> None: + """ + Update both execution details and execution signatures according to tool output + """ + try: + tool_output = ToolOutput.model_validate(tool_output or {}) + except ValidationError as exc: + raise OpenAEVError( + error_message=f"Invalid tool_output: {exc}", + ) from exc + + now = self._utcnow() + + execution_details.post_execution_update(tool_output, now) + + if isinstance(execution_signatures, list): + for exec_sig in execution_signatures: + exec_sig.post_execution_update(tool_output, now) + else: + execution_signatures.post_execution_update(tool_output, now) + + @staticmethod + def build_payload( + execution_signatures: ExecutionSignature | list[ExecutionSignature], + targets_meta: Any | list[Any], + expectation_types: list[str], + extra_signatures: ExtraSignatureData = ExtraSignatureData(), + ) -> dict[str, Any]: + """Build the nested wire payload from flat post-execution signatures. + + Bridges the gap between compile_post_execution_signatures output (flat dicts) + and send_signatures input (nested wire format). + + Args: + execution_signatures: A single post-execution ExecutionSignature or a list (multi-targets). + targets_meta: Target metadata dict(s) with keys like agent, asset, asset_group. + expectation_types: The 1+ expectation type labels (e.g. ['DETECTION', 'PREVENTION']). + extra_signatures: Optional mapping of expectation types to additional signature fields that will be merged + into the base post_signatures. + + Returns: + A payload dict ready for send_signatures. + """ + if not isinstance(execution_signatures, list): + execution_signatures = [execution_signatures] + if isinstance(targets_meta, dict): + targets_meta = [targets_meta] * len(execution_signatures) + + targets = [] + for signature, target in zip(execution_signatures, targets_meta): + signature_values = [] + for expectation_type in expectation_types: + signature_data = signature.model_dump(exclude_none=True) + signature_data.update(extra_signatures.get_extra(expectation_type)) + values = [ + SignatureValue(signature_type=key, signature_value=value) + for key, value in signature_data.items() + ] + signature_values.append( + ExpectationSignatureGroup( + expectation_type=expectation_type, + values=values, + ) + ) + targets.append( + TargetSignatures( + signature_target=SignatureTarget( + agent=target.agent_id, + asset=target.asset_id, + asset_group=target.asset_group_id, + ), + signature_values=signature_values, + ) + ) + + return SignaturePayload(targets=targets).model_dump() + + def send_signatures( + self, + inject_id: str, + execution_details: ExecutionDetails, + signatures: dict[str, Any], + ) -> None: + """Ship signatures to the callback endpoint via the configured transport. + + Constructs typed ``SignatureOutputStructure`` and ``ExecutionDetails`` + models, then delegates transport (retry, envelope splitting, validation) + to the configured signature transport. + + Args: + inject_id: Inject UUID. + execution_details: execution-related metadata as an ExecutionDetails object + signatures: Full signatures dict with a ``targets`` list. + + Raises: + SignatureTransmissionError: Validation failed, 4xx hit, or retries exhausted. + """ + sig_output = SignatureOutputStructure( + expectation_signatures=SignaturePayload(**signatures) + ) + + self.transport.send_signatures( + inject_id, + sig_output, + execution_details, + max_payload_size=self.max_payload_size, + logger=self.logger, + ) + + def resolve_container_ip(self) -> str: + """Sniff the container's primary IPv4. Env var, hostname, then ``hostname -i``. + + Returns: + The IPv4 string, or ``'unknown'`` with a warning when all strategies fail. + """ + if self._cached_ipv4 and self._cached_ipv4 != "unknown": + return self._cached_ipv4 + + env_ip = os.environ.get("CONTAINER_IP") + if env_ip: + self._cached_ipv4 = env_ip + self._resolve_ipv6() + return env_ip + + try: + ip = socket.gethostbyname(socket.gethostname()) + if ip and ip != "127.0.0.1": + self._cached_ipv4 = ip + self._resolve_ipv6() + return ip + except (socket.gaierror, OSError): + pass + + try: + result = subprocess.run( + ["hostname", "-i"], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + if result.returncode == 0: + ip = result.stdout.strip().split()[0] + if ip and ip != "127.0.0.1": + self._cached_ipv4 = ip + self._resolve_ipv6() + return ip + except (OSError, RuntimeError, subprocess.TimeoutExpired): + pass + + self.logger.warning("Could not resolve container IP; returning 'unknown'") + self._cached_ipv4 = "unknown" + return "unknown" + + def _resolve_ipv6(self) -> None: + """Best-effort IPv6 sniff. Silent on failure, no exceptions escape.""" + try: + infos = socket.getaddrinfo( + socket.gethostname(), None, socket.AF_INET6, socket.SOCK_STREAM + ) + for info in infos: + addr = info[4][0] + if ( + isinstance(addr, str) + and addr + and not addr.startswith("::1") + and not addr.startswith("fe80") + ): + self._cached_ipv6 = addr + return + except (socket.gaierror, OSError): + pass diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py new file mode 100644 index 00000000..31c5c252 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py @@ -0,0 +1,31 @@ +"""Defines signature match result objects. + +This module provides the `SignatureMatch` class, which stores the selected +match type and its optional score. +""" + +from xtm_oaev_sdk._core.errors import OpenAEVError +from xtm_oaev_sdk._core.signatures.types import MatchTypes + +__all__ = ["SignatureMatch"] + + +class SignatureMatch: + """Represents a signature matching outcome. + + Args: + match_type: The matching strategy/result type. + match_score: Optional score produced by matching. + + Raises: + OpenAEVError: If `match_score` is not provided for non-simple match + types. + """ + + def __init__(self, match_type: MatchTypes, match_score: int | None): + if match_score is None and match_type != MatchTypes.MATCH_TYPE_SIMPLE: + raise OpenAEVError( + f"Match type {match_type} requires score to be set, found score = {match_score}" + ) + self.match_type = match_type + self.match_score = match_score diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_type.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_type.py new file mode 100644 index 00000000..e6da710b --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_type.py @@ -0,0 +1,54 @@ +"""Signature type model and matching-structure builder. + +This module defines ``SignatureType``, which couples a signature label with a +matching policy and can generate helper structures for signature matching. +""" + +from typing import Any + +from xtm_oaev_sdk._core.signatures.signature_match import SignatureMatch +from xtm_oaev_sdk._core.signatures.types import MatchTypes, SignatureTypes + +__all__ = ["SignatureType"] + + +class SignatureType: + """Describe a signature type and its matching policy. + + Args: + label: Type specifier. + match_type: Matching policy used when trying to match this signature + type, for example fuzzy or simple. + match_score: Threshold score used when ``match_type`` is fuzzy. + """ + + def __init__( + self, + label: SignatureTypes, + match_type: MatchTypes = MatchTypes.MATCH_TYPE_SIMPLE, + match_score: int | None = None, + ) -> None: + self.label = label + self.match_policy = SignatureMatch(match_type, match_score) + + def make_struct_for_matching(self, data: Any) -> dict[str, Any]: + """Format matching metadata for helper-based signature matching. + + Args: + data: Arbitrary data, most often a string or numeric primitive. + + Returns: + A dictionary with matching specifiers containing: + - ``type``: Matching type string. + - ``data``: Provided data payload. + - ``score``: Optional score threshold when configured. + """ + struct: dict[str, Any] = { + "type": self.match_policy.match_type.value, + "data": data, + } + + if self.match_policy.match_score is not None: + struct["score"] = self.match_policy.match_score + + return struct diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/types.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/types.py new file mode 100644 index 00000000..99d07bcb --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/types.py @@ -0,0 +1,65 @@ +"""Enumerations used by the signature engine. + +The wire vocabulary — the three enums that appear in the port I/O JSON +schema (``SignatureExpectationType``, ``ExecutionStatus``, +``InjectExecutionActions``) — is owned by the seam +(``xtm_oaev_sdk.seam.port_io.models``) and re-exported here so internal +import paths stay stable. The matching-engine vocabulary (``MatchTypes``, +``SignatureTypes``) never crosses the wire and stays defined here. +""" + +from enum import Enum + +from xtm_oaev_sdk.seam.port_io.models import ( + ExecutionStatus, + InjectExecutionActions, + SignatureExpectationType, +) + +__all__ = [ + "SignatureExpectationType", + "ExecutionStatus", + "InjectExecutionActions", + "MatchTypes", + "SignatureTypes", +] + + +class MatchTypes(str, Enum): + """Supported match algorithm types for signature evaluation.""" + + MATCH_TYPE_FUZZY = "fuzzy" + MATCH_TYPE_SIMPLE = "simple" + + +class SignatureTypes(str, Enum): + """Supported signature field identifiers.""" + + SIG_TYPE_PROCESS_NAME = "process_name" + SIG_TYPE_PARENT_PROCESS_NAME = "parent_process_name" + SIG_TYPE_COMMAND_LINE = "command_line" + SIG_TYPE_IPV4_ADDRESS = "ipv4_address" + SIG_TYPE_IPV6_ADDRESS = "ipv6_address" + SIG_TYPE_SOURCE_IPV4_ADDRESS = "source_ipv4_address" + SIG_TYPE_SOURCE_IPV6_ADDRESS = "source_ipv6_address" + SIG_TYPE_TARGET_IPV4_ADDRESS = "target_ipv4_address" + SIG_TYPE_TARGET_IPV6_ADDRESS = "target_ipv6_address" + SIG_TYPE_HOSTNAME = "hostname" + SIG_TYPE_TARGET_HOSTNAME_ADDRESS = "target_hostname_address" + SIG_TYPE_START_DATE = "start_date" + SIG_TYPE_END_DATE = "end_date" + SIG_TYPE_CLOUD_PROVIDER = "cloud_provider" + SIG_TYPE_CLOUD_ACCOUNT_ID = "cloud_account_id" + SIG_TYPE_CLOUD_REGION = "cloud_region" + SIG_TYPE_TARGET_SERVICE = "target_service" + SIG_TYPE_QUERY = "query" + SIG_TYPE_URL_HASH = "url_hash" + SIG_TYPE_FILE_NAME = "file_name" + SIG_TYPE_FILE_HASH = "file_hash" + SIG_TYPE_SOURCE_EMAIL = "source_email" + SIG_TYPE_TARGET_EMAIL = "target_email" + SIG_TYPE_EMAIL_CUSTOM_HEADER = "email_custom_header" + # AI adversarial validation: correlate AI defense (LLM firewall / guardrail) events back to a + # specific AI inject execution. + SIG_TYPE_AI_REQUEST_MARKER = "ai_request_marker" + SIG_TYPE_AI_TARGET_ENDPOINT = "ai_target_endpoint" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index 94883204..8a23e533 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -62,6 +62,43 @@ # --- Seam / compatibility --- from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr +# --- Port I/O contract (versioned wire schema) --- +from xtm_oaev_sdk.seam.port_io import ( + PORT_IO_CONTRACT_VERSION, + ExecutionDetails, + ExecutionStatus, + ExpectationSignatureGroup, + InjectExecutionActions, + SignatureCallbackPayload, + SignatureExpectationType, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, +) + +# --- Signatures --- +from xtm_oaev_sdk.seam.signatures import ( + CloudInjectorConfig, + ExecutionSignature, + ExtraSignatureData, + InjectorConfig, + MatchTypes, + NetworkInjectorConfig, + SignatureManager, + SignatureManagerProtocol, + SignatureMatch, + SignatureTransportProtocol, + SignatureType, + SignatureTypes, + ToolErrorInfo, + ToolOutput, + ToolTimeoutInfo, + build_marker, + build_network_configs, +) + __all__ = [ "__version__", # Common: errors @@ -118,6 +155,36 @@ "TimedeltaInSeconds", "is_falsy", "is_truthy", + # Signatures + "CloudInjectorConfig", + "ExecutionDetails", + "ExecutionSignature", + "ExecutionStatus", + "ExpectationSignatureGroup", + "ExtraSignatureData", + "InjectExecutionActions", + "InjectorConfig", + "MatchTypes", + "NetworkInjectorConfig", + "SignatureCallbackPayload", + "SignatureExpectationType", + "SignatureManager", + "SignatureManagerProtocol", + "SignatureMatch", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureTransportProtocol", + "SignatureType", + "SignatureTypes", + "SignatureValue", + "TargetSignatures", + "ToolErrorInfo", + "ToolOutput", + "ToolTimeoutInfo", + "build_marker", + "build_network_configs", # Seam / compatibility + "PORT_IO_CONTRACT_VERSION", "make_deprecated_getattr", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py index f2dfca6c..aad7835c 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -7,11 +7,20 @@ platform-facing vocabulary both sides import. - **``configuration``** — the configuration capability's I/O models and conversion primitives. +- **``signatures``** — the signatures capability's inbound seam + (``seam/signatures/seam.py``), versioned ``SEAM_VERSION``. +- **``port_io``** — the port I/O wire schema crossing the SDK↔platform + boundary; the seam **owns** the models (``seam/port_io/models.py``), + versioned ``PORT_IO_CONTRACT_VERSION``. - **``deprecation``** — the PEP 562 deprecation primitive (:func:`make_deprecated_getattr`) for retiring public names without breaking importers. -The package ``__init__`` surfaces present ``_core``'s stable vocabulary +``seam/signatures/seam.py`` and ``seam/port_io/models.py`` are closed: they +import only the standard library, pydantic, and sibling seam modules — never +``_core``. The capability's implementation imports and re-exports from them. + +The other package ``__init__`` surfaces present ``_core``'s stable vocabulary through the boundary: they import ``_core`` so the public layer can. The seam is the **sole door** between ``public/`` and ``_core/``: every public import of a ``_core`` symbol crosses it. The seam never imports the public layer, diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py new file mode 100644 index 00000000..fc6375a9 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py @@ -0,0 +1,50 @@ +"""Port I/O data contract: the versioned wire schema crossing the SDK↔platform seam. + +These models — and the three wire-vocabulary enums they serialise — define +the exact JSON shapes transmitted between the SDK's transport port +(``SignatureTransportProtocol``) and the platform's callback endpoint. This package is the re-export surface; the seam **owns** the +models (``xtm_oaev_sdk.seam.port_io.models``), and the capability's port +(``_core/signatures``) re-exports them back for its historical surface. + +They form a pinnable, anti-breaking boundary: + +- contract unchanged + port (internal) changes → no breaking change +- contract unchanged + adapter (public) changes → retrocompat holds +- contract changes → **breaking change** + +Any modification to the JSON schema of a model in this module is a breaking +change for consumers and must bump :data:`PORT_IO_CONTRACT_VERSION`. + +The schema-pinning guard (``tests/test_port_io_schema_pin.py``) hashes each +model's ``model_json_schema()`` against a golden file and fails on drift. +""" + +from xtm_oaev_sdk.seam.port_io.models import ( + PORT_IO_CONTRACT_VERSION, + ExecutionDetails, + ExecutionStatus, + ExpectationSignatureGroup, + InjectExecutionActions, + SignatureCallbackPayload, + SignatureExpectationType, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, +) + +__all__ = [ + "PORT_IO_CONTRACT_VERSION", + "ExecutionDetails", + "ExpectationSignatureGroup", + "ExecutionStatus", + "InjectExecutionActions", + "SignatureCallbackPayload", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureValue", + "SignatureExpectationType", + "TargetSignatures", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py new file mode 100644 index 00000000..5b641ef9 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py @@ -0,0 +1,299 @@ +"""The port I/O wire schema — the seam-owned models crossing the SDK↔platform boundary. + +These eight pydantic models — together with the three wire-vocabulary enums +they serialise (``SignatureExpectationType``, ``ExecutionStatus``, +``InjectExecutionActions``) — are the exact JSON shapes transmitted between +the SDK's transport port (``SignatureTransportProtocol``) and the +platform's callback endpoint. The seam is their **owner**: the capability's +port (``_core/signatures``) imports and re-exports from here, never the +reverse, so the two sides of the boundary can evolve independently. + +- contract unchanged + port (internal) changes → no breaking change +- contract unchanged + adapter (public) changes → retrocompat holds +- contract changes → **breaking change** + +Any modification to the JSON schema of a model in this module is a breaking +change for consumers and must bump :data:`PORT_IO_CONTRACT_VERSION`. + +The schema-pinning guard (``tests/test_port_io_schema_pin.py``) hashes each +model's ``model_json_schema()`` against a golden file and fails on drift. + +The module is **closed**: at runtime it imports only the standard library and +pydantic. ``ToolOutput`` (from the signatures inbound seam, +``seam/signatures/seam.py``) is referenced for type checking only, by +``ExecutionDetails.post_execution_update``: importing it at runtime would run +``seam/signatures/__init__`` and, through it, ``_core`` — an import cycle back +into this module. The wire contract therefore has no hidden dependency on +internal layout. +""" + +import math +from collections import defaultdict +from datetime import datetime, timezone +from enum import Enum +from typing import TYPE_CHECKING, Any + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + JsonValue, + TypeAdapter, + computed_field, + field_validator, +) + +if TYPE_CHECKING: + from xtm_oaev_sdk.seam.signatures.seam import ToolOutput + +#: Version of the port I/O wire schema. Bump on any breaking contract change. +PORT_IO_CONTRACT_VERSION = "1.0.0" + +# --- Wire vocabulary ----------------------------------------------------------- +# The three enums that appear in the JSON contract below. They live here, in +# the seam, because they are part of the wire schema; the internal +# implementation imports them from here (``_core/signatures/types`` +# re-exports them for its historical import paths). + + +class SignatureExpectationType(str, Enum): + """Expectation categories for a signature.""" + + DETECTION = "DETECTION" + PREVENTION = "PREVENTION" + VULNERABILITY = "VULNERABILITY" + + +class ExecutionStatus(str, Enum): + """Outcome status of a single inject tool execution. + + A ``str`` enum so it serialises to its string form in callback payloads. + Mirrors the platform's execution-status taxonomy label-for-label. + """ + + EXECUTED = "EXECUTED" + EXECUTED_WITH_CLEANUP_FAILURE = "EXECUTED_WITH_CLEANUP_FAILURE" + WARNING = "WARNING" + ACCESS_DENIED = "ACCESS_DENIED" + ERROR = "ERROR" + COMMAND_NOT_FOUND = "COMMAND_NOT_FOUND" + COMMAND_CANNOT_BE_EXECUTED = "COMMAND_CANNOT_BE_EXECUTED" + PREREQUISITE_FAILED = "PREREQUISITE_FAILED" + INVALID_USAGE = "INVALID_USAGE" + TIMEOUT = "TIMEOUT" + INTERRUPTED = "INTERRUPTED" + ASSET_AGENTLESS = "ASSET_AGENTLESS" + AGENT_INACTIVE = "AGENT_INACTIVE" + AGENT_OVERLOADED = "AGENT_OVERLOADED" + INFO = "INFO" + PARTIAL = "PARTIAL" + MAYBE_PREVENTED = "MAYBE_PREVENTED" + MAYBE_PARTIAL_PREVENTED = "MAYBE_PARTIAL_PREVENTED" + + +class InjectExecutionActions(str, Enum): + """Execution action phases used by injection signatures.""" + + PREREQUISITE_CHECK = "prerequisite_check" + PREREQUISITE_EXECUTION = "prerequisite_execution" + CLEANUP_EXECUTION = "cleanup_execution" + COMMAND_EXECUTION = "command_execution" + DNS_RESOLUTION = "dns_resolution" + FILE_EXECUTION = "file_execution" + FILE_DROP = "file_drop" + DATA = "data" + COMPLETE = "complete" + + +class SignatureValue(BaseModel): + """One signature observation: a type and the value it carries.""" + + model_config = ConfigDict(extra="allow") + + signature_type: str + signature_value: JsonValue + + +class ExpectationSignatureGroup(BaseModel): + """Values bound to a single expectation type (DETECTION, PREVENTION, ...).""" + + model_config = ConfigDict(extra="allow") + + expectation_type: str + values: list[SignatureValue] + + @field_validator("expectation_type", mode="after") + @classmethod + def is_expectation_type(cls, value: str) -> str: + SignatureExpectationType(value.upper()) + return value + + +class SignatureTarget(BaseModel): + """Target identity on the wire.""" + + model_config = ConfigDict(extra="forbid") + + agent: str | None = None + asset: str | None = None + asset_group: str | None = None + + +class TargetSignatures(BaseModel): + """A target plus everything observed about it, grouped by expectation.""" + + model_config = ConfigDict(extra="allow") + + signature_target: SignatureTarget + signature_values: list[ExpectationSignatureGroup] + + +class SignaturePayload(BaseModel): + """Inner ``signatures`` body: a list of targets, nothing else.""" + + model_config = ConfigDict(extra="allow") + + targets: list[TargetSignatures] + + +class SignatureOutputStructure(BaseModel): + """Structured output to be serialized as a str in the callback payload yet data has to follow model.""" + + model_config = ConfigDict(populate_by_name=True, extra="forbid") + + expectation_signatures: SignaturePayload + + def normalize_signature_payload(self) -> None: + """ + Regroup signature_values by expectation_type within each target. + """ + normalized_targets: list[TargetSignatures] = [] + + for target in self.expectation_signatures.targets: + if not target.signature_values: + normalized_targets.append(target) + continue + + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + order: list[str] = [] + + for entry in target.signature_values: + if entry.expectation_type not in order: + order.append(entry.expectation_type) + grouped[entry.expectation_type].extend(entry.values) + + normalized_target = TargetSignatures( + signature_target=target.signature_target, + signature_values=[ + ExpectationSignatureGroup( + expectation_type=expectation_type, + values=grouped[expectation_type], + ) + for expectation_type in order + ], + ) + + normalized_targets.append(normalized_target) + + self.expectation_signatures.targets = normalized_targets + + +class ExecutionDetails(BaseModel): + """Helper to wrap the execution-related details for the callback payload.""" + + model_config = ConfigDict(extra="forbid") + + start_time: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + end_time: datetime | None = None + + execution_status: ExecutionStatus | None = None + execution_action: InjectExecutionActions | None = None + + @computed_field + @property + def execution_message(self) -> str: + action = self.execution_action.value if self.execution_action else "unknown" + status = self.execution_action.value if self.execution_action else "unknown" + return f"Current action: {action} - Current status: {status}" + + @computed_field + @property + def execution_duration(self) -> float: + try: + return (self.end_time - self.start_time).total_seconds() + except: + return 0.0 + + def post_execution_update(self, tool_output: "ToolOutput", now: datetime) -> None: + """ + Update execution-related metadata according to tool output and now timestamp. + """ + self.end_time = now + + if tool_output.error_info and tool_output.error_info.exit_code != 0: + self.execution_status = ExecutionStatus.ERROR + if tool_output.error_info.crash_timestamp: + self.end_time = datetime.strptime( + tool_output.error_info.crash_timestamp, "%Y-%m-%dT%H:%M:%SZ" + ) + elif tool_output.timeout_info: + self.execution_status = ExecutionStatus.TIMEOUT + elif tool_output.status == "partial": + self.execution_status = ExecutionStatus.ERROR + else: + self.execution_status = ExecutionStatus.EXECUTED + + self.execution_action = InjectExecutionActions("data") + + +class SignatureCallbackPayload(BaseModel): + """Outer POST envelope validated by ``SignatureApiManager`` before wire transmission.""" + + model_config = ConfigDict(populate_by_name=True, extra="forbid") + + execution_message: str + execution_output_structured: str | None = None + execution_status: str + execution_duration: int | None = None + execution_action: InjectExecutionActions | None = None + + @field_validator("execution_output_structured", mode="after") + @classmethod + def is_proper_signature_output_structure(cls, value: str) -> str | None: + if value is None: + return None + TypeAdapter(SignatureOutputStructure).validate_json(value) + return value + + @classmethod + def build_from_models( + cls, signatures: SignatureOutputStructure, execution_details: ExecutionDetails + ): + """Producing a SignatureCallbackPayload from the data of a SignatureOutputStructure and of a ExecutionDetails.""" + return cls( + execution_message=execution_details.execution_message, + execution_output_structured=signatures.model_dump_json(exclude_none=True), + execution_status=execution_details.execution_status, + execution_duration=( + math.ceil(execution_details.execution_duration) + if execution_details.execution_duration is not None + else None + ), + execution_action=execution_details.execution_action, + ) + + +__all__ = [ + "PORT_IO_CONTRACT_VERSION", + "SignatureExpectationType", + "ExecutionStatus", + "InjectExecutionActions", + "SignatureValue", + "ExpectationSignatureGroup", + "SignatureTarget", + "TargetSignatures", + "SignaturePayload", + "SignatureOutputStructure", + "ExecutionDetails", + "SignatureCallbackPayload", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py new file mode 100644 index 00000000..2704e300 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py @@ -0,0 +1,71 @@ +"""Signature feature data models. + +The inbound seam (the versioned ``InjectorConfig`` / ``ExecutionSignature`` / +``ToolOutput`` I/O models) is declared in :mod:`xtm_oaev_sdk.seam.signatures.seam`; +everything else in this package is the port's interface, re-export surface, +and the concrete matching-engine layer. +""" + +from xtm_oaev_sdk._core.signatures import ( + CloudInjectorConfig, + ExecutionDetails, + ExecutionSignature, + ExecutionStatus, + ExpectationSignatureGroup, + ExtraSignatureData, + InjectExecutionActions, + InjectorConfig, + MatchTypes, + NetworkInjectorConfig, + SignatureCallbackPayload, + SignatureExpectationType, + SignatureManager, + SignatureManagerProtocol, + SignatureMatch, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureTransportProtocol, + SignatureType, + SignatureTypes, + SignatureValue, + TargetSignatures, + ToolErrorInfo, + ToolOutput, + ToolTimeoutInfo, + build_marker, + build_network_configs, +) +from xtm_oaev_sdk.seam.signatures.seam import SEAM_VERSION + +__all__ = [ + "SEAM_VERSION", + "CloudInjectorConfig", + "ExecutionDetails", + "ExecutionSignature", + "ExecutionStatus", + "ExpectationSignatureGroup", + "ExtraSignatureData", + "InjectExecutionActions", + "InjectorConfig", + "MatchTypes", + "NetworkInjectorConfig", + "SignatureCallbackPayload", + "SignatureExpectationType", + "SignatureManager", + "SignatureManagerProtocol", + "SignatureMatch", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureTransportProtocol", + "SignatureType", + "SignatureTypes", + "SignatureValue", + "TargetSignatures", + "ToolErrorInfo", + "ToolOutput", + "ToolTimeoutInfo", + "build_marker", + "build_network_configs", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py new file mode 100644 index 00000000..13219309 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py @@ -0,0 +1,146 @@ +"""The signatures capability's inbound seam: the versioned I/O models. + +The frozen boundary both sides of the signatures capability share: + +- **input face** — what is handed to the injector to execute: the + ``InjectorConfig`` family (one network target, or one cloud target row) + and the ``ExecutionSignature`` (the execution instruction and its + outcome fields). +- **tool result face** — what the injected tool reports back: + ``ToolOutput`` and its ``ToolErrorInfo`` / ``ToolTimeoutInfo``. + +Pure data: this module imports nothing from the capability. The +dependency rule holds — the port imports the seam, never the other way, +and this module is the only thing both sides may import. + +Any modification to the JSON schema of a model in this module is a +breaking change for consumers and must bump :data:`SEAM_VERSION`. + +The schema-pinning guard (``tests/test_signatures_seam_schema_pin.py``) +hashes each model's ``model_json_schema()`` against a golden file and +fails on drift. +""" + +from datetime import datetime, timezone + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +#: Version of the signatures inbound seam. Bump on any breaking contract change. +SEAM_VERSION = "1.0.0" + + +class ToolErrorInfo(BaseModel): + """Crash report. Non-zero exit code and a timestamp if the tool left one behind.""" + + model_config = ConfigDict(extra="allow") + + exit_code: int = 0 + crash_timestamp: str | None = None + + +class ToolTimeoutInfo(BaseModel): + """Timeout report. Whatever partial loot was rescued before the kill signal.""" + + model_config = ConfigDict(extra="allow") + + partial_results: list[str] = Field(default_factory=list) + + +class ToolOutput(BaseModel): + """Whatever the tool spat out: status, error info, timeout info, or injector extras.""" + + model_config = ConfigDict(extra="allow") + + status: str | None = None + error_info: ToolErrorInfo | None = None + timeout_info: ToolTimeoutInfo | None = None + + +class ExecutionSignature(BaseModel): + """ + Execution signature data. Field set varies by category: network, cloud. Plus outcome, end_time, and any partial results. + """ + + model_config = ConfigDict(extra="allow") + + # Timing always emitted at call time. + start_time: str = Field( + default_factory=lambda: datetime.now(timezone.utc).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + ) + end_time: str | None = None + partial_results: list[str] | None = None + + # Network identity + source_ipv4: str | None = None + source_ipv6: str | None = None + target_ipv4: str | None = None + target_ipv6: str | None = None + target_hostname: str | None = None + + # Cloud identity + cloud_provider: str | None = None + cloud_account_id: str | None = None + cloud_region: str | None = None + target_service: str | None = None + + def post_execution_update(self, tool_output: ToolOutput, now: datetime) -> None: + """Update timing and partial results from tool output.""" + self.end_time = now.strftime("%Y-%m-%dT%H:%M:%SZ") + + if tool_output.error_info and tool_output.error_info.crash_timestamp: + self.end_time = tool_output.error_info.crash_timestamp + + if tool_output.timeout_info and tool_output.timeout_info.partial_results: + self.partial_results = tool_output.timeout_info.partial_results + + +class NetworkInjectorConfig(BaseModel): + """A single network target. Exactly one of ``target_ipv4``, ``target_ipv6``, or ``target_hostname``.""" + + model_config = ConfigDict(extra="forbid") + + target_ipv4: str | None = None + target_ipv6: str | None = None + target_hostname: str | None = None + + @model_validator(mode="before") + @classmethod + def check_one(cls, data): + """Ensure exactly one target identity field is provided.""" + assert ( + sum( + value is not None + for key, value in data.items() + if key in ["target_ipv4", "target_ipv6", "target_hostname"] + ) + == 1 + ) + return data + + +class CloudInjectorConfig(BaseModel): + """A single cloud target row. One config per region; fan out by passing a list.""" + + model_config = ConfigDict(extra="forbid") + + cloud_provider: str + cloud_account_id: str + cloud_region: str + target_service: str | None = None + + +InjectorConfig = NetworkInjectorConfig | CloudInjectorConfig + + +__all__ = [ + "SEAM_VERSION", + "ToolErrorInfo", + "ToolTimeoutInfo", + "ToolOutput", + "ExecutionSignature", + "NetworkInjectorConfig", + "CloudInjectorConfig", + "InjectorConfig", +] diff --git a/pyoaev/__init__.py b/pyoaev/__init__.py index db714285..3fddaa5c 100644 --- a/pyoaev/__init__.py +++ b/pyoaev/__init__.py @@ -4,7 +4,7 @@ import xtm_oaev_sdk as _sdk from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr -from pyoaev import asset_types, configuration, exceptions # noqa: F401 +from pyoaev import asset_types, configuration, exceptions, signatures # noqa: F401 from pyoaev._version import ( # noqa: F401 __author__, __copyright__, @@ -14,7 +14,6 @@ ) from pyoaev.client import OpenAEV # noqa: F401 from pyoaev.contracts import * # noqa: F401,F403,F405 -from pyoaev.signatures import * # noqa: F401,F403,F405 __all__ = [ # noqa: F405 (moved names are served by the PEP 562 __getattr__ below) "__author__", @@ -33,14 +32,20 @@ # Names moved to xtm_oaev_sdk: served lazily (PEP 562) so that `import pyoaev` # stays silent and only legacy access to a moved name warns. -_deprecated_getattr = _make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { - name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) - for name in (*asset_types.__all__, *configuration.__all__, *exceptions.__all__) + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in ( + *asset_types.__all__, + *configuration.__all__, + *exceptions.__all__, + ) + }, + **signatures._DEPRECATED, }, module_name=__name__, ) - - -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) diff --git a/pyoaev/apis/inject_expectation/model/expectation.py b/pyoaev/apis/inject_expectation/model/expectation.py index 3e64ce0d..3b290c10 100644 --- a/pyoaev/apis/inject_expectation/model/expectation.py +++ b/pyoaev/apis/inject_expectation/model/expectation.py @@ -4,9 +4,7 @@ from pydantic import BaseModel, Field, field_validator from thefuzz import fuzz - -from pyoaev.signatures.signature_type import SignatureType -from pyoaev.signatures.types import MatchTypes, SignatureTypes +from xtm_oaev_sdk import MatchTypes, SignatureType, SignatureTypes class ExpectationTypeEnum(str, Enum): diff --git a/pyoaev/apis/signature.py b/pyoaev/apis/signature.py index b03793f7..80dbbed6 100644 --- a/pyoaev/apis/signature.py +++ b/pyoaev/apis/signature.py @@ -6,15 +6,15 @@ from typing import Any from pydantic import ValidationError -from xtm_oaev_sdk import SignatureTransmissionError -from xtm_oaev_sdk.seam import common as exc - -from pyoaev.base import RESTManager, RESTObject -from pyoaev.signatures.models import ( +from xtm_oaev_sdk import ( ExecutionDetails, SignatureCallbackPayload, SignatureOutputStructure, + SignatureTransmissionError, ) +from xtm_oaev_sdk.seam import common as exc + +from pyoaev.base import RESTManager, RESTObject class Signature(RESTObject): diff --git a/pyoaev/asset_types.py b/pyoaev/asset_types.py index 89f1250b..616815fc 100644 --- a/pyoaev/asset_types.py +++ b/pyoaev/asset_types.py @@ -16,14 +16,10 @@ _MOVED = __all__ -_deprecated_getattr = _make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) for name in _MOVED }, module_name=__name__, ) - - -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) diff --git a/pyoaev/configuration/__init__.py b/pyoaev/configuration/__init__.py index 750e9c12..b28d0462 100644 --- a/pyoaev/configuration/__init__.py +++ b/pyoaev/configuration/__init__.py @@ -15,14 +15,10 @@ "SettingsLoader", ] -_deprecated_getattr = _make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) for name in __all__ }, module_name=__name__, ) - - -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) diff --git a/pyoaev/configuration/configuration.py b/pyoaev/configuration/configuration.py index 72d651bd..a5348569 100644 --- a/pyoaev/configuration/configuration.py +++ b/pyoaev/configuration/configuration.py @@ -18,14 +18,10 @@ "is_truthy", ] -_deprecated_getattr = _make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) for name in __all__ }, module_name=__name__, ) - - -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) diff --git a/pyoaev/configuration/connector_config_schema_generator.py b/pyoaev/configuration/connector_config_schema_generator.py index d6b045eb..14b6bf3c 100644 --- a/pyoaev/configuration/connector_config_schema_generator.py +++ b/pyoaev/configuration/connector_config_schema_generator.py @@ -11,14 +11,10 @@ "ConnectorConfigSchemaGenerator", ] -_deprecated_getattr = _make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) for name in __all__ }, module_name=__name__, ) - - -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) diff --git a/pyoaev/configuration/settings_loader.py b/pyoaev/configuration/settings_loader.py index 49ecea35..39d52a2c 100644 --- a/pyoaev/configuration/settings_loader.py +++ b/pyoaev/configuration/settings_loader.py @@ -17,14 +17,10 @@ "TimedeltaInSeconds", ] -_deprecated_getattr = _make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) for name in __all__ }, module_name=__name__, ) - - -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) diff --git a/pyoaev/configuration/sources.py b/pyoaev/configuration/sources.py index 10007a62..9481637b 100644 --- a/pyoaev/configuration/sources.py +++ b/pyoaev/configuration/sources.py @@ -12,14 +12,10 @@ "EnvironmentSource", ] -_deprecated_getattr = _make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) for name in __all__ }, module_name=__name__, ) - - -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) diff --git a/pyoaev/exceptions.py b/pyoaev/exceptions.py index 889cb6fd..b92a2a4d 100644 --- a/pyoaev/exceptions.py +++ b/pyoaev/exceptions.py @@ -27,14 +27,10 @@ _MOVED = [*__all__, "OpenAEVError", "on_http_error"] -_deprecated_getattr = _make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) for name in _MOVED }, module_name=__name__, ) - - -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) diff --git a/pyoaev/helpers.py b/pyoaev/helpers.py index 0eb36abd..bae04d3d 100644 --- a/pyoaev/helpers.py +++ b/pyoaev/helpers.py @@ -13,12 +13,8 @@ import pika import xtm_oaev_sdk as _sdk from thefuzz import fuzz -from xtm_oaev_sdk import ( - Configuration, - ConfigurationError, - logger, - make_deprecated_getattr, -) +from xtm_oaev_sdk import Configuration, ConfigurationError, logger +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr from pyoaev import OpenAEV, utils from pyoaev.daemons import CollectorDaemon @@ -28,7 +24,7 @@ # SSL helpers moved to xtm_oaev_sdk: legacy names resolve to the identical SDK # objects and warn on access (PEP 562). -_deprecated_getattr = make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) for name in ( @@ -42,10 +38,6 @@ ) -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) - - def create_mq_ssl_context(config) -> ssl.SSLContext: config_obj = Configuration( config_hints={ diff --git a/pyoaev/security_domain/types.py b/pyoaev/security_domain/types.py index 41920222..157aa100 100644 --- a/pyoaev/security_domain/types.py +++ b/pyoaev/security_domain/types.py @@ -13,14 +13,10 @@ _MOVED = __all__ -_deprecated_getattr = _make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) for name in _MOVED }, module_name=__name__, ) - - -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) diff --git a/pyoaev/signatures/__init__.py b/pyoaev/signatures/__init__.py index dc2e0616..422f7b61 100644 --- a/pyoaev/signatures/__init__.py +++ b/pyoaev/signatures/__init__.py @@ -1,23 +1,16 @@ -from pyoaev.signatures.models import ( - CloudInjectorConfig, - ExpectationSignatureGroup, - ExtraSignatureData, - InjectorConfig, - NetworkInjectorConfig, - SignatureCallbackPayload, - SignaturePayload, - SignatureTarget, - SignatureValue, - TargetSignatures, - build_network_configs, -) -from pyoaev.signatures.signature_manager import SignatureManager -from pyoaev.signatures.types import ( - ExpectationType, - InjectExecutionActions, - MatchTypes, - SignatureTypes, -) +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access, with +two exceptions: ``ExpectationType`` was renamed ``SignatureExpectationType``, +and ``SignatureManager`` resolves to the client-based adapter described in +:mod:`pyoaev.signatures.signature_manager`. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +from pyoaev.signatures.signature_manager import _ClientSignatureManager __all__ = [ "CloudInjectorConfig", @@ -37,3 +30,24 @@ "TargetSignatures", "build_network_configs", ] + +_RENAMED = {"ExpectationType": "SignatureExpectationType"} + +# Also served from the pyoaev package root, which used to star-import it. +_DEPRECATED = { + "SignatureManager": ( + "Use 'from xtm_oaev_sdk import SignatureManager' with a " + "SignatureTransportProtocol transport (e.g. client.signature) instead.", + _ClientSignatureManager, + ), + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {_RENAMED.get(name, name)}' instead.", + getattr(_sdk, _RENAMED.get(name, name)), + ) + for name in __all__ + if name != "SignatureManager" + }, +} + +__getattr__ = _make_deprecated_getattr(_DEPRECATED, module_name=__name__) diff --git a/pyoaev/signatures/ai_marker.py b/pyoaev/signatures/ai_marker.py index 51165068..61d6e5fa 100644 --- a/pyoaev/signatures/ai_marker.py +++ b/pyoaev/signatures/ai_marker.py @@ -1,15 +1,20 @@ -"""Deterministic per-inject canary marker shared by the AI red-team injector and the AI defense -collectors. - -The marker is derived purely from the inject (and optional agent) id, so the injector that sends the -attack and the collector that validates an AI defense response compute the same value independently, -without the platform having to store it. It is emitted by the injector (request header + in-prompt -token) and matched by collectors against guardrail / firewall logs. -""" - -import hashlib - - -def build_marker(inject_id: str, agent_id: str = "") -> str: - seed = f"{inject_id}:{agent_id}".encode("utf-8") - return "oaev" + hashlib.sha256(seed).hexdigest()[:16] +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "build_marker", +] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/models.py b/pyoaev/signatures/models.py index c8e914f9..aee7504d 100644 --- a/pyoaev/signatures/models.py +++ b/pyoaev/signatures/models.py @@ -1,381 +1,13 @@ -"""Pydantic schemas pinning every shape SignatureManager touches.""" +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -import ipaddress -import math -from collections import defaultdict -from datetime import datetime, timezone -from typing import Any +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -from pydantic import ( - BaseModel, - ConfigDict, - Field, - JsonValue, - TypeAdapter, - computed_field, - field_validator, - model_validator, -) - -from pyoaev.signatures.types import ( - ExecutionStatus, - ExpectationType, - InjectExecutionActions, -) - - -class ToolErrorInfo(BaseModel): - """Crash report. Non-zero exit code and a timestamp if the tool left one behind.""" - - model_config = ConfigDict(extra="allow") - - exit_code: int = 0 - crash_timestamp: str | None = None - - -class ToolTimeoutInfo(BaseModel): - """Timeout report. Whatever partial loot was rescued before the kill signal.""" - - model_config = ConfigDict(extra="allow") - - partial_results: list[str] = Field(default_factory=list) - - -class ToolOutput(BaseModel): - """Whatever the tool spat out: status, error info, timeout info, or injector extras.""" - - model_config = ConfigDict(extra="allow") - - status: str | None = None - error_info: ToolErrorInfo | None = None - timeout_info: ToolTimeoutInfo | None = None - - -class SignatureValue(BaseModel): - """One signature observation: a type and the value it carries.""" - - model_config = ConfigDict(extra="allow") - - signature_type: str - signature_value: JsonValue - - -class ExpectationSignatureGroup(BaseModel): - """Values bound to a single expectation type (DETECTION, PREVENTION, ...).""" - - model_config = ConfigDict(extra="allow") - - expectation_type: str - values: list[SignatureValue] - - @field_validator("expectation_type", mode="after") - @classmethod - def is_expectation_type(cls, value: str) -> str: - ExpectationType(value.upper()) - return value - - -class ExtraSignatureData(BaseModel): - """Format for extra signatures added to the default signatures""" - - detection: dict[str, JsonValue] = Field(default_factory=dict) - prevention: dict[str, JsonValue] = Field(default_factory=dict) - vulnerability: dict[str, JsonValue] = Field(default_factory=dict) - - def get_extra(self, expectation_type: str) -> dict[str, JsonValue]: - if expectation_type.lower() == "detection": - return self.detection - if expectation_type.lower() == "prevention": - return self.prevention - if expectation_type.lower() == "vulnerability": - return self.vulnerability - raise ValueError( - f"Expectation type should be one of the available parameters: {list(self.model_fields.keys())}" - ) - - -class SignatureTarget(BaseModel): - """Target identity on the wire.""" - - model_config = ConfigDict(extra="forbid") - - agent: str | None = None - asset: str | None = None - asset_group: str | None = None - - -class TargetSignatures(BaseModel): - """A target plus everything observed about it, grouped by expectation.""" - - model_config = ConfigDict(extra="allow") - - signature_target: SignatureTarget - signature_values: list[ExpectationSignatureGroup] - - -class SignaturePayload(BaseModel): - """Inner ``signatures`` body: a list of targets, nothing else.""" - - model_config = ConfigDict(extra="allow") - - targets: list[TargetSignatures] - - -class SignatureOutputStructure(BaseModel): - """Structured output to be serialized as a str in the callback payload yet data has to follow model.""" - - model_config = ConfigDict(populate_by_name=True, extra="forbid") - - expectation_signatures: SignaturePayload - - def normalize_signature_payload(self) -> None: - """ - Regroup signature_values by expectation_type within each target. - """ - normalized_targets: list[TargetSignatures] = [] - - for target in self.expectation_signatures.targets: - if not target.signature_values: - normalized_targets.append(target) - continue - - grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) - order: list[str] = [] - - for entry in target.signature_values: - if entry.expectation_type not in order: - order.append(entry.expectation_type) - grouped[entry.expectation_type].extend(entry.values) - - normalized_target = TargetSignatures( - signature_target=target.signature_target, - signature_values=[ - ExpectationSignatureGroup( - expectation_type=expectation_type, - values=grouped[expectation_type], - ) - for expectation_type in order - ], - ) - - normalized_targets.append(normalized_target) - - self.expectation_signatures.targets = normalized_targets - - -class ExecutionDetails(BaseModel): - """Helper to wrap the execution-related details for the callback payload""" - - model_config = ConfigDict(extra="forbid") +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr - start_time: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) - end_time: datetime | None = None - - execution_status: ExecutionStatus | None = None - execution_action: InjectExecutionActions | None = None - - @computed_field - @property - def execution_message(self) -> str: - action = self.execution_action.value if self.execution_action else "unknown" - status = self.execution_action.value if self.execution_action else "unknown" - return f"Current action: {action} - Current status: {status}" - - @computed_field - @property - def execution_duration(self) -> float: - try: - return (self.end_time - self.start_time).total_seconds() - except: - return 0.0 - - def post_execution_update(self, tool_output: ToolOutput, now: datetime) -> None: - """ - Update execution-related metadata according to tool output and now timestamp - """ - self.end_time = now - - if tool_output.error_info and tool_output.error_info.exit_code != 0: - self.execution_status = ExecutionStatus.ERROR - if tool_output.error_info.crash_timestamp: - self.end_time = datetime.strptime( - tool_output.error_info.crash_timestamp, "%Y-%m-%dT%H:%M:%SZ" - ) - elif tool_output.timeout_info: - self.execution_status = ExecutionStatus.TIMEOUT - elif tool_output.status == "partial": - self.execution_status = ExecutionStatus.ERROR - else: - self.execution_status = ExecutionStatus.EXECUTED - - self.execution_action = InjectExecutionActions("data") - - -class SignatureCallbackPayload(BaseModel): - """Outer POST envelope validated by ``SignatureApiManager`` before wire transmission.""" - - model_config = ConfigDict(populate_by_name=True, extra="forbid") - - execution_message: str - execution_output_structured: str | None = None - execution_status: str - execution_duration: int | None = None - execution_action: InjectExecutionActions | None = None - - @field_validator("execution_output_structured", mode="after") - @classmethod - def is_proper_signature_output_structure(cls, value: str) -> str | None: - if value is None: - return None - TypeAdapter(SignatureOutputStructure).validate_json(value) - return value - - @classmethod - def build_from_models( - cls, signatures: SignatureOutputStructure, execution_details: ExecutionDetails - ): - """Producing a SignatureCallbackPayload from the data of a SignatureOutputStructure and of a ExecutionDetails.""" - return cls( - execution_message=execution_details.execution_message, - execution_output_structured=signatures.model_dump_json(exclude_none=True), - execution_status=execution_details.execution_status, - execution_duration=( - math.ceil(execution_details.execution_duration) - if execution_details.execution_duration is not None - else None - ), - execution_action=execution_details.execution_action, - ) - - -class ExecutionSignature(BaseModel): - """ - Execution signature data. Field set varies by category: network, cloud. Plus outcome, end_time, and any partial results. - """ - - model_config = ConfigDict(extra="allow") - - # Timing always emitted at call time. - start_time: str = Field( - default_factory=lambda: datetime.now(timezone.utc).strftime( - "%Y-%m-%dT%H:%M:%SZ" - ) - ) - end_time: str | None = None - partial_results: list[str] | None = None - - # Network identity - source_ipv4: str | None = None - source_ipv6: str | None = None - target_ipv4: str | None = None - target_ipv6: str | None = None - target_hostname: str | None = None - - # Cloud identity - cloud_provider: str | None = None - cloud_account_id: str | None = None - cloud_region: str | None = None - target_service: str | None = None - - def post_execution_update(self, tool_output: ToolOutput, now: datetime) -> None: - """ """ - self.end_time = now.strftime("%Y-%m-%dT%H:%M:%SZ") - - if tool_output.error_info and tool_output.error_info.crash_timestamp: - self.end_time = tool_output.error_info.crash_timestamp - - if tool_output.timeout_info and tool_output.timeout_info.partial_results: - self.partial_results = tool_output.timeout_info.partial_results - - -class NetworkInjectorConfig(BaseModel): - """A single network target. Exactly one of ``target_ipv4``, ``target_ipv6``, or ``target_hostname``.""" - - model_config = ConfigDict(extra="forbid") - - target_ipv4: str | None = None - target_ipv6: str | None = None - target_hostname: str | None = None - - @model_validator(mode="before") - @classmethod - def check_one(cls, data): - assert ( - sum( - value is not None - for key, value in data.items() - if key in ["target_ipv4", "target_ipv6", "target_hostname"] - ) - == 1 - ) - return data - - -class CloudInjectorConfig(BaseModel): - """A single cloud target row. One config per region; fan out by passing a list.""" - - model_config = ConfigDict(extra="forbid") - - cloud_provider: str - cloud_account_id: str - cloud_region: str - target_service: str | None = None - - -InjectorConfig = NetworkInjectorConfig | CloudInjectorConfig - - -# --------------------------------------------------------------------------- -# Builders. Cheap helpers to turn raw injector input into typed configs. -# --------------------------------------------------------------------------- - - -def _classify_network_target(value: str) -> NetworkInjectorConfig: - """Decide whether ``value`` is an IPv4, IPv6, or hostname and wrap it.""" - try: - addr = ipaddress.ip_address(value) - except ValueError: - return NetworkInjectorConfig(target_hostname=value) - if isinstance(addr, ipaddress.IPv4Address): - return NetworkInjectorConfig(target_ipv4=value) - return NetworkInjectorConfig(target_ipv6=value) - - -def build_network_configs( - targets: list[str | dict[str, Any] | NetworkInjectorConfig], -) -> list[NetworkInjectorConfig]: - """Forge a list of `NetworkInjectorConfig` from a heterogeneous target list. - - Each item is one distinct asset. Accepted shapes: - - - `NetworkInjectorConfig`: passed through unchanged. - - `dict`: validated against :class:`NetworkInjectorConfig`. - - `str`: auto-classified into IPv4 / IPv6 / hostname. - - Args: - targets: Raw target list straight out of the injector. - - Returns: - One `NetworkInjectorConfig` per input target, order preserved. - - Raises: - TypeError: An item is not one of the accepted shapes. - ValidationError: A dict item fails the one-identity invariant. - """ - configs: list[NetworkInjectorConfig] = [] - for target in targets: - if isinstance(target, NetworkInjectorConfig): - configs.append(target) - elif isinstance(target, dict): - configs.append(NetworkInjectorConfig(**target)) - elif isinstance(target, str): - configs.append(_classify_network_target(target)) - else: - raise TypeError(f"unsupported network target type: {type(target).__name__}") - return configs - - -__all__ = [ +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) "SignatureValue", "ExpectationSignatureGroup", "SignatureTarget", @@ -391,3 +23,26 @@ def build_network_configs( "InjectorConfig", "build_network_configs", ] + +_RENAMED = {"ExpectationType": "SignatureExpectationType"} + +_MOVED = [ + *__all__, + "ExecutionDetails", + "ExtraSignatureData", + "SignatureOutputStructure", + "ExecutionStatus", + "ExpectationType", + "InjectExecutionActions", +] + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from xtm_oaev_sdk import {_RENAMED.get(name, name)}' instead.", + getattr(_sdk, _RENAMED.get(name, name)), + ) + for name in _MOVED + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/signature_manager.py b/pyoaev/signatures/signature_manager.py index 51857f7d..8054eac1 100644 --- a/pyoaev/signatures/signature_manager.py +++ b/pyoaev/signatures/signature_manager.py @@ -1,298 +1,78 @@ -"""Signature lifecycle for OpenAEV injectors: compile pre, merge post, ship to backend.""" +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -import logging -import os -import socket -import subprocess -from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any +The SDK ``SignatureManager`` takes a ``SignatureTransportProtocol`` instead of +an ``OpenAEV`` client. The legacy name resolves to a thin subclass that keeps +the historical ``SignatureManager(client, logger=None, max_payload_size=...)`` +constructor by using ``client.signature`` as the transport. Every other name +resolves to the *identical* xtm_oaev_sdk object. All names emit a +:class:`DeprecationWarning` on access. -from pydantic import ValidationError -from xtm_oaev_sdk import OpenAEVError +``socket`` and ``subprocess`` stay importable here so that legacy patch targets +such as ``pyoaev.signatures.signature_manager.socket.gethostbyname`` keep +reaching the shared standard-library modules the SDK uses. +""" -from pyoaev.signatures.models import ( - CloudInjectorConfig, - ExecutionDetails, - ExecutionSignature, - ExpectationSignatureGroup, - ExtraSignatureData, - InjectorConfig, - NetworkInjectorConfig, - SignatureOutputStructure, - SignaturePayload, - SignatureTarget, - SignatureValue, - TargetSignatures, - ToolOutput, -) +import socket # noqa: F401 +import subprocess # noqa: F401 +from typing import TYPE_CHECKING + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr if TYPE_CHECKING: from pyoaev.client import OpenAEV -class SignatureManager: - """End-to-end signature pipeline: compile, merge, transmit. One class, three jobs.""" - - DEFAULT_MAX_PAYLOAD_SIZE = 5_242_880 # 5 MiB +class _ClientSignatureManager(_sdk.SignatureManager): + """SDK SignatureManager driven by a pyoaev client (legacy constructor).""" def __init__( self, client: "OpenAEV", - logger: logging.Logger | None = None, - max_payload_size: int = DEFAULT_MAX_PAYLOAD_SIZE, - ) -> None: - self.client = client - self.logger = logger or logging.getLogger(__name__) - self.max_payload_size = max_payload_size - self._cached_ipv4: str | None = None - self._cached_ipv6: str | None = None - - def _utcnow(self) -> datetime: - """Current UTC time. Carved out so tests can pin the clock.""" - return datetime.now(timezone.utc) - - def build_execution_signatures( - self, - config: InjectorConfig | list[InjectorConfig], - ) -> ExecutionSignature | list[ExecutionSignature]: - """Build pre-execution signature dicts from one or more typed injector configs. - - The category is carried by the config type itself - (:class:`NetworkInjectorConfig`, :class:`CloudInjectorConfig`), - so no separate ``category`` flag is needed. - - Args: - config: A single injector config or a homogeneous list of them. - Multi-target injects must be expressed as a list. - - Returns: - One ExecutionSignature object when a single config is given, - otherwise a list of ExecutionSignature in input order. - - Raises: - ValueError: Empty list, or mixed config types in a single call. - TypeError: Unknown injector config type. - """ - configs = list(config) if isinstance(config, list) else [config] - if not configs: - raise ValueError("build_execution_signatures requires at least one config") - - first_type = type(configs[0]) - for c in configs: - if not isinstance(c, first_type): - raise ValueError( - "build_execution_signatures does not mix injector config types; " - f"got {sorted({type(c).__name__ for c in configs})}" - ) - - start_time = self._utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - results = [self._build_one(cfg, start_time) for cfg in configs] - return results[0] if len(results) == 1 else results - - def _build_one(self, config: InjectorConfig, start_time: str) -> ExecutionSignature: - """Project a single injector config into a flat pre-execution signature dict. - - Common pipeline for every category: - 1. Seed the base dict with ``start_time`` and category-specific context - (network gets resolved source IPs; cloud add nothing). - 2. Layer the config's own fields on top. - 3. Run it through :class:`ExecutionSignature` for validation - and emit JSON-ready output stripped of ``None``\\ s. - """ - base: dict[str, Any] = {"start_time": start_time} - base.update(self._source_context(config)) - base.update(config.model_dump(exclude_none=True)) - return ExecutionSignature(**base) - - def _source_context(self, config: InjectorConfig) -> dict[str, Any]: - """Return the source identity bits injected for the config's category. - - Only network signatures need the running container's source IPs; - cloud rows have no source identity to carry. - """ - if isinstance(config, NetworkInjectorConfig): - return { - "source_ipv4": self.resolve_container_ip(), - "source_ipv6": self._cached_ipv6, - } - if isinstance(config, CloudInjectorConfig): - return {} - raise TypeError(f"unsupported injector config type: {type(config).__name__}") - - def post_execution_updates( - self, - execution_details: ExecutionDetails, - execution_signatures: ExecutionSignature | list[ExecutionSignature], - tool_output: dict[str, Any], - ) -> None: - """ - Update both execution details and execution signatures according to tool output - """ - try: - tool_output = ToolOutput.model_validate(tool_output or {}) - except ValidationError as exc: - raise OpenAEVError( - error_message=f"Invalid tool_output: {exc}", - ) from exc - - now = self._utcnow() - - execution_details.post_execution_update(tool_output, now) - - if isinstance(execution_signatures, list): - for exec_sig in execution_signatures: - exec_sig.post_execution_update(tool_output, now) - else: - execution_signatures.post_execution_update(tool_output, now) - - @staticmethod - def build_payload( - execution_signatures: ExecutionSignature | list[ExecutionSignature], - targets_meta: Any | list[Any], - expectation_types: list[str], - extra_signatures: ExtraSignatureData = ExtraSignatureData(), - ) -> dict[str, Any]: - """Build the nested wire payload from flat post-execution signatures. - - Bridges the gap between compile_post_execution_signatures output (flat dicts) - and send_signatures input (nested wire format). - - Args: - execution_signatures: A single post-execution ExecutionSignature or a list (multi-targets). - targets_meta: Target metadata dict(s) with keys like agent, asset, asset_group. - expectation_types: The 1+ expectation type labels (e.g. ['DETECTION', 'PREVENTION']). - extra_signatures: Optional mapping of expectation types to additional signature fields that will be merged - into the base post_signatures. - - Returns: - A payload dict ready for send_signatures. - """ - if not isinstance(execution_signatures, list): - execution_signatures = [execution_signatures] - if isinstance(targets_meta, dict): - targets_meta = [targets_meta] * len(execution_signatures) - - targets = [] - for signature, target in zip(execution_signatures, targets_meta): - signature_values = [] - for expectation_type in expectation_types: - signature_data = signature.model_dump(exclude_none=True) - signature_data.update(extra_signatures.get_extra(expectation_type)) - values = [ - SignatureValue(signature_type=key, signature_value=value) - for key, value in signature_data.items() - ] - signature_values.append( - ExpectationSignatureGroup( - expectation_type=expectation_type, - values=values, - ) - ) - targets.append( - TargetSignatures( - signature_target=SignatureTarget( - agent=target.agent_id, - asset=target.asset_id, - asset_group=target.asset_group_id, - ), - signature_values=signature_values, - ) - ) - - return SignaturePayload(targets=targets).model_dump() - - def send_signatures( - self, - inject_id: str, - execution_details: ExecutionDetails, - signatures: dict[str, Any], + logger=None, + max_payload_size: int = _sdk.SignatureManager.DEFAULT_MAX_PAYLOAD_SIZE, ) -> None: - """Ship signatures to the callback endpoint via the Signature API manager. - - Constructs typed ``SignatureOutputStructure`` and ``ExecutionDetails`` - models, then delegates transport (retry, envelope splitting, validation) - to ``client.signature``. - - Args: - inject_id: Inject UUID. - execution_details: execution-related metadata as an ExecutionDetails object - signatures: Full signatures dict with a ``targets`` list. - - Raises: - SignatureTransmissionError: Validation failed, 4xx hit, or retries exhausted. - """ - sig_output = SignatureOutputStructure( - expectation_signatures=SignaturePayload(**signatures) - ) - - self.client.signature.send_signatures( - inject_id, - sig_output, - execution_details, - max_payload_size=self.max_payload_size, - logger=self.logger, + super().__init__( + transport=client.signature, + logger=logger, + max_payload_size=max_payload_size, ) + self.client = client - def resolve_container_ip(self) -> str: - """Sniff the container's primary IPv4. Env var, hostname, then ``hostname -i``. - - Returns: - The IPv4 string, or ``'unknown'`` with a warning when all strategies fail. - """ - if self._cached_ipv4 and self._cached_ipv4 != "unknown": - return self._cached_ipv4 - - env_ip = os.environ.get("CONTAINER_IP") - if env_ip: - self._cached_ipv4 = env_ip - self._resolve_ipv6() - return env_ip - - try: - ip = socket.gethostbyname(socket.gethostname()) - if ip and ip != "127.0.0.1": - self._cached_ipv4 = ip - self._resolve_ipv6() - return ip - except (socket.gaierror, OSError): - pass - - try: - result = subprocess.run( - ["hostname", "-i"], - capture_output=True, - text=True, - timeout=5, - check=False, - ) - if result.returncode == 0: - ip = result.stdout.strip().split()[0] - if ip and ip != "127.0.0.1": - self._cached_ipv4 = ip - self._resolve_ipv6() - return ip - except (OSError, RuntimeError, subprocess.TimeoutExpired): - pass - - self.logger.warning("Could not resolve container IP; returning 'unknown'") - self._cached_ipv4 = "unknown" - return "unknown" - def _resolve_ipv6(self) -> None: - """Best-effort IPv6 sniff. Silent on failure, no exceptions escape.""" - try: - infos = socket.getaddrinfo( - socket.gethostname(), None, socket.AF_INET6, socket.SOCK_STREAM +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "SignatureManager", + "OpenAEVError", + "CloudInjectorConfig", + "ExecutionDetails", + "ExecutionSignature", + "ExpectationSignatureGroup", + "ExtraSignatureData", + "InjectorConfig", + "NetworkInjectorConfig", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureValue", + "TargetSignatures", + "ToolOutput", +] + +__getattr__ = _make_deprecated_getattr( + { + "SignatureManager": ( + "Use 'from xtm_oaev_sdk import SignatureManager' with a " + "SignatureTransportProtocol transport (e.g. client.signature) instead.", + _ClientSignatureManager, + ), + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), ) - for info in infos: - addr = info[4][0] - if ( - isinstance(addr, str) - and addr - and not addr.startswith("::1") - and not addr.startswith("fe80") - ): - self._cached_ipv6 = addr - return - except (socket.gaierror, OSError): - pass + for name in __all__ + if name != "SignatureManager" + }, + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/signature_match.py b/pyoaev/signatures/signature_match.py index 4f8fa830..b52b9a72 100644 --- a/pyoaev/signatures/signature_match.py +++ b/pyoaev/signatures/signature_match.py @@ -1,13 +1,22 @@ -from xtm_oaev_sdk import OpenAEVError +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -from pyoaev.signatures.types import MatchTypes +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr -class SignatureMatch: - def __init__(self, match_type: MatchTypes, match_score: int | None): - if match_score is None and match_type != MatchTypes.MATCH_TYPE_SIMPLE: - raise OpenAEVError( - f"Match type {match_type} requires score to be set, found score = {match_score}" - ) - self.match_type = match_type - self.match_score = match_score +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "SignatureMatch", + "OpenAEVError", + "MatchTypes", +] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/signature_type.py b/pyoaev/signatures/signature_type.py index bab72dbd..c40f2f32 100644 --- a/pyoaev/signatures/signature_type.py +++ b/pyoaev/signatures/signature_type.py @@ -1,51 +1,23 @@ -from pyoaev.signatures.signature_match import SignatureMatch -from pyoaev.signatures.types import MatchTypes, SignatureTypes - - -class SignatureType: - """Describes a signature of some time and a matching policy - - :param label: Type specifier - :type label: SignatureTypes - :param match_type: the matching policy to use when trying - to match this signature type, e.g. fuzzy, simple... - :type match_type: MatchTypes - :param match_score: if the matching type is fuzzy, this is - the score to use as threshold, defaults to None - :type match_score: int, optional - """ - - def __init__( - self, - label: SignatureTypes, - match_type: MatchTypes = MatchTypes.MATCH_TYPE_SIMPLE, - match_score: int | None = None, - ): - self.label = label - self.match_policy = SignatureMatch(match_type, match_score) - - def make_struct_for_matching(self, data): - """Provided some `data`, formats a dictionary specifying the matching - policy to use by the helper to match expected signatures (from expectations) - with actual, alert signatures (from the security software) - - :param data: arbitrary data, but most often string or a number primitive - :type: Any - - :return: dictionary of matching specifiers:: - { - "type": str, - "data": any, - "score": (optional) int - } - :rtype: dict - """ - struct = { - "type": self.match_policy.match_type.value, - "data": data, - } - - if self.match_policy.match_score is not None: - struct["score"] = self.match_policy.match_score - - return struct +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "SignatureType", + "SignatureMatch", + "MatchTypes", + "SignatureTypes", +] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/signatures/types.py b/pyoaev/signatures/types.py index 5a5102ec..c76b946c 100644 --- a/pyoaev/signatures/types.py +++ b/pyoaev/signatures/types.py @@ -1,76 +1,29 @@ -from enum import Enum - - -class ExpectationType(str, Enum): - DETECTION = "DETECTION" - PREVENTION = "PREVENTION" - VULNERABILITY = "VULNERABILITY" - - -class ExecutionStatus(str, Enum): - EXECUTED = "EXECUTED" - EXECUTED_WITH_CLEANUP_FAILURE = "EXECUTED_WITH_CLEANUP_FAILURE" - WARNING = "WARNING" - ACCESS_DENIED = "ACCESS_DENIED" - ERROR = "ERROR" - COMMAND_NOT_FOUND = "COMMAND_NOT_FOUND" - COMMAND_CANNOT_BE_EXECUTED = "COMMAND_CANNOT_BE_EXECUTED" - PREREQUISITE_FAILED = "PREREQUISITE_FAILED" - INVALID_USAGE = "INVALID_USAGE" - TIMEOUT = "TIMEOUT" - INTERRUPTED = "INTERRUPTED" - ASSET_AGENTLESS = "ASSET_AGENTLESS" - AGENT_INACTIVE = "AGENT_INACTIVE" - AGENT_OVERLOADED = "AGENT_OVERLOADED" - INFO = "INFO" - PARTIAL = "PARTIAL" - MAYBE_PREVENTED = "MAYBE_PREVENTED" - MAYBE_PARTIAL_PREVENTED = "MAYBE_PARTIAL_PREVENTED" - - -class InjectExecutionActions(str, Enum): - PREREQUISITE_CHECK = "prerequisite_check" - PREREQUISITE_EXECUTION = "prerequisite_execution" - CLEANUP_EXECUTION = "cleanup_execution" - COMMAND_EXECUTION = "command_execution" - DNS_RESOLUTION = "dns_resolution" - FILE_EXECUTION = "file_execution" - FILE_DROP = "file_drop" - DATA = "data" - COMPLETE = "complete" - - -class MatchTypes(str, Enum): - MATCH_TYPE_FUZZY = "fuzzy" - MATCH_TYPE_SIMPLE = "simple" - - -class SignatureTypes(str, Enum): - SIG_TYPE_PROCESS_NAME = "process_name" - SIG_TYPE_PARENT_PROCESS_NAME = "parent_process_name" - SIG_TYPE_COMMAND_LINE = "command_line" - SIG_TYPE_IPV4_ADDRESS = "ipv4_address" - SIG_TYPE_IPV6_ADDRESS = "ipv6_address" - SIG_TYPE_SOURCE_IPV4_ADDRESS = "source_ipv4_address" - SIG_TYPE_SOURCE_IPV6_ADDRESS = "source_ipv6_address" - SIG_TYPE_TARGET_IPV4_ADDRESS = "target_ipv4_address" - SIG_TYPE_TARGET_IPV6_ADDRESS = "target_ipv6_address" - SIG_TYPE_HOSTNAME = "hostname" - SIG_TYPE_TARGET_HOSTNAME_ADDRESS = "target_hostname_address" - SIG_TYPE_START_DATE = "start_date" - SIG_TYPE_END_DATE = "end_date" - SIG_TYPE_CLOUD_PROVIDER = "cloud_provider" - SIG_TYPE_CLOUD_ACCOUNT_ID = "cloud_account_id" - SIG_TYPE_CLOUD_REGION = "cloud_region" - SIG_TYPE_TARGET_SERVICE = "target_service" - SIG_TYPE_QUERY = "query" - SIG_TYPE_URL_HASH = "url_hash" - SIG_TYPE_FILE_NAME = "file_name" - SIG_TYPE_FILE_HASH = "file_hash" - SIG_TYPE_SOURCE_EMAIL = "source_email" - SIG_TYPE_TARGET_EMAIL = "target_email" - SIG_TYPE_EMAIL_CUSTOM_HEADER = "email_custom_header" - # AI adversarial validation: correlate AI defense (LLM firewall / guardrail) events back to a - # specific AI inject execution. - SIG_TYPE_AI_REQUEST_MARKER = "ai_request_marker" - SIG_TYPE_AI_TARGET_ENDPOINT = "ai_target_endpoint" +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ExpectationType", + "ExecutionStatus", + "InjectExecutionActions", + "MatchTypes", + "SignatureTypes", +] + +_RENAMED = {"ExpectationType": "SignatureExpectationType"} + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from xtm_oaev_sdk import {_RENAMED.get(name, name)}' instead.", + getattr(_sdk, _RENAMED.get(name, name)), + ) + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/utils.py b/pyoaev/utils.py index 15db0fec..7a03ce31 100644 --- a/pyoaev/utils.py +++ b/pyoaev/utils.py @@ -105,14 +105,10 @@ def stop(self) -> None: self.exit_event.set() -_deprecated_getattr = _make_deprecated_getattr( +__getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) for name in _MOVED }, module_name=__name__, ) - - -def __getattr__(name: str): # PEP 562 - return _deprecated_getattr(name) From b220f9e44bbca55b0f585446c3e434fea75c06af Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 13:28:34 +0200 Subject: [PATCH 04/22] :sparkles: feat(sdk): migrate contracts to xtm-oaev-sdk with PEP 562 pyoaev shims --- .../xtm-oaev-sdk/test}/contracts/__init__.py | 0 .../test/contracts/shim}/__init__.py | 0 .../test_contract_element_argument_type.py | 0 .../shim}/test_contract_expectations.py | 0 .../shim}/test_contract_output_types.py | 0 .../test_contract_referenced_credential.py | 0 .../contracts/shim}/test_credential_types.py | 0 .../contracts/shim}/test_credential_utils.py | 0 .../contracts/shim}/test_primitive_type.py | 0 .../test/contracts/shim/test_shim_contract.py | 153 +++++ .../test_contract_element_argument_type.py | 41 ++ .../contracts/test_contract_expectations.py | 165 +++++ .../contracts/test_contract_output_types.py | 22 + .../test_contract_referenced_credential.py | 53 ++ .../test/contracts/test_contracts.py | 538 +++++++++++++++ .../test/contracts/test_credential_types.py | 27 + .../test/contracts/test_credential_utils.py | 59 ++ .../test/contracts/test_primitive_type.py | 77 +++ libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 3 + .../xtm_oaev_sdk/_core/contracts/__init__.py | 82 +++ .../_core/contracts/contract_builder.py | 172 +++++ .../_core/contracts/contract_config.py | 624 ++++++++++++++++++ .../_core/contracts/contract_utils.py | 57 ++ .../_core/contracts/credential_types.py | 18 + .../_core/contracts/credentials.py | 43 ++ .../_core/contracts/variable_helper.py | 149 +++++ .../xtm_oaev_sdk/contracts/__init__.py | 6 + .../contracts/contracts/__init__.py | 73 ++ .../xtm_oaev_sdk/public/__init__.py | 81 +++ .../xtm_oaev_sdk/seam/__init__.py | 6 + .../xtm_oaev_sdk/seam/contracts/__init__.py | 27 + pyoaev/__init__.py | 12 +- pyoaev/contracts/__init__.py | 24 +- pyoaev/contracts/contract_builder.py | 71 +- pyoaev/contracts/contract_config.py | 524 ++------------- pyoaev/contracts/contract_utils.py | 41 +- pyoaev/contracts/variable_helper.py | 163 ++--- pyoaev/credential/__init__.py | 29 +- pyoaev/credential/types.py | 31 +- pyoaev/credential/utils.py | 62 +- 40 files changed, 2726 insertions(+), 707 deletions(-) rename {test => libs/xtm-oaev-sdk/test}/contracts/__init__.py (100%) rename {test/credential => libs/xtm-oaev-sdk/test/contracts/shim}/__init__.py (100%) rename {test/contracts => libs/xtm-oaev-sdk/test/contracts/shim}/test_contract_element_argument_type.py (100%) rename {test/contracts => libs/xtm-oaev-sdk/test/contracts/shim}/test_contract_expectations.py (100%) rename {test/contracts => libs/xtm-oaev-sdk/test/contracts/shim}/test_contract_output_types.py (100%) rename {test/contracts => libs/xtm-oaev-sdk/test/contracts/shim}/test_contract_referenced_credential.py (100%) rename {test/credential => libs/xtm-oaev-sdk/test/contracts/shim}/test_credential_types.py (100%) rename {test/credential => libs/xtm-oaev-sdk/test/contracts/shim}/test_credential_utils.py (100%) rename {test/contracts => libs/xtm-oaev-sdk/test/contracts/shim}/test_primitive_type.py (100%) create mode 100644 libs/xtm-oaev-sdk/test/contracts/shim/test_shim_contract.py create mode 100644 libs/xtm-oaev-sdk/test/contracts/test_contract_element_argument_type.py create mode 100644 libs/xtm-oaev-sdk/test/contracts/test_contract_expectations.py create mode 100644 libs/xtm-oaev-sdk/test/contracts/test_contract_output_types.py create mode 100644 libs/xtm-oaev-sdk/test/contracts/test_contract_referenced_credential.py create mode 100644 libs/xtm-oaev-sdk/test/contracts/test_contracts.py create mode 100644 libs/xtm-oaev-sdk/test/contracts/test_credential_types.py create mode 100644 libs/xtm-oaev-sdk/test/contracts/test_credential_utils.py create mode 100644 libs/xtm-oaev-sdk/test/contracts/test_primitive_type.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_builder.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_config.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_utils.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credential_types.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variable_helper.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/contracts/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py diff --git a/test/contracts/__init__.py b/libs/xtm-oaev-sdk/test/contracts/__init__.py similarity index 100% rename from test/contracts/__init__.py rename to libs/xtm-oaev-sdk/test/contracts/__init__.py diff --git a/test/credential/__init__.py b/libs/xtm-oaev-sdk/test/contracts/shim/__init__.py similarity index 100% rename from test/credential/__init__.py rename to libs/xtm-oaev-sdk/test/contracts/shim/__init__.py diff --git a/test/contracts/test_contract_element_argument_type.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_contract_element_argument_type.py similarity index 100% rename from test/contracts/test_contract_element_argument_type.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_contract_element_argument_type.py diff --git a/test/contracts/test_contract_expectations.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_contract_expectations.py similarity index 100% rename from test/contracts/test_contract_expectations.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_contract_expectations.py diff --git a/test/contracts/test_contract_output_types.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_contract_output_types.py similarity index 100% rename from test/contracts/test_contract_output_types.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_contract_output_types.py diff --git a/test/contracts/test_contract_referenced_credential.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_contract_referenced_credential.py similarity index 100% rename from test/contracts/test_contract_referenced_credential.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_contract_referenced_credential.py diff --git a/test/credential/test_credential_types.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_credential_types.py similarity index 100% rename from test/credential/test_credential_types.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_credential_types.py diff --git a/test/credential/test_credential_utils.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_credential_utils.py similarity index 100% rename from test/credential/test_credential_utils.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_credential_utils.py diff --git a/test/contracts/test_primitive_type.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_primitive_type.py similarity index 100% rename from test/contracts/test_primitive_type.py rename to libs/xtm-oaev-sdk/test/contracts/shim/test_primitive_type.py diff --git a/libs/xtm-oaev-sdk/test/contracts/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/contracts/shim/test_shim_contract.py new file mode 100644 index 00000000..ca56f1b2 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/shim/test_shim_contract.py @@ -0,0 +1,153 @@ +"""Contract of the pyoaev contracts shims. + +Legacy imports must keep working, return the *identical* xtm_oaev_sdk object +(``ExpectationType`` was renamed ``ContractExpectationType``), warn, and be +attributed to the consumer's line. +""" + +import subprocess +import sys +import warnings + +import pytest +import xtm_oaev_sdk as sdk + +_CONSUMER = "legacy_consumer.py" +_RENAMED = {"ExpectationType": "ContractExpectationType"} + +_CONTRACT_CONFIG = [ + "SupportedLanguage", + "ContractFieldType", + "ContractFieldKey", + "ContractOutputType", + "PrimitiveType", + "ExpectationType", + "SecurityPlatformType", + "Expectation", + "LinkedFieldModel", + "ContractElement", + "ContractCardinalityElement", + "ContractOutputElement", + "ContractConfig", + "Domain", + "Contract", + "ContractTeam", + "ContractText", + "prepare_contracts", + "ContractTuple", + "ContractTextArea", + "ContractCheckbox", + "ContractAttachment", + "ContractExpectations", + "ContractSelect", + "ContractAsset", + "ContractAssetGroup", + "ContractAiTarget", + "ContractPayload", + "ContractReferencedCredential", + "CredentialType", + "EnhancedJSONEncoder", + "ContractCardinality", + "ContractVariable", + "VariableHelper", +] + +LEGACY_NAMES = ( + [("pyoaev", "ContractBuilder"), ("pyoaev.contracts", "ContractBuilder")] + + [ + ("pyoaev.contracts.contract_builder", name) + for name in ("ContractBuilder", "ContractElement", "ContractOutputElement") + ] + + [("pyoaev.contracts.contract_config", name) for name in _CONTRACT_CONFIG] + + [ + ("pyoaev.contracts.contract_utils", name) + for name in ("ContractCardinality", "VariableType", "ContractVariable") + ] + + [ + ("pyoaev.contracts.variable_helper", name) + for name in ( + "VariableHelper", + "ContractCardinality", + "ContractVariable", + "VariableType", + ) + ] + + [ + ("pyoaev.credential", name) + for name in ("CredentialType", "build_single_referenced_credential_element") + ] + + [("pyoaev.credential.types", "CredentialType")] + + [ + ("pyoaev.credential.utils", name) + for name in ( + "ContractReferencedCredential", + "CredentialType", + "build_single_referenced_credential_element", + ) + ] +) + +LEGACY_CONSTANTS = { + "USER": "user", + "EXERCISE": "exercise", + "TEAMS": "teams", + "COMCHECK": "comcheck", + "PLAYER_URI": "player_uri", + "CHALLENGES_URI": "challenges_uri", + "SCOREBOARD_URI": "scoreboard_uri", + "LESSONS_URI": "lessons_uri", +} + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + source = f"from {module} import {name} as _legacy" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(compile(source, _CONSUMER, "exec"), namespace) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + return namespace["_legacy"], deprecations + + +@pytest.mark.parametrize(("module", "name"), LEGACY_NAMES) +def test_legacy_import_warns_and_returns_the_sdk_object(module, name) -> None: + """It warns towards xtm_oaev_sdk and returns the identical object.""" + sdk_name = _RENAMED.get(name, name) + legacy, deprecations = _when_import_legacy_name(module, name) + assert legacy is getattr(sdk, sdk_name) + assert deprecations + assert f"from xtm_oaev_sdk import {sdk_name}" in str(deprecations[0].message) + # attributed to the consumer's line, so default warning filters show it + assert any(w.filename == _CONSUMER for w in deprecations) + + +@pytest.mark.parametrize(("name", "value"), LEGACY_CONSTANTS.items()) +def test_legacy_variable_constants_keep_their_value_and_warn(name, value) -> None: + """It keeps the unexported module constants working, with a warning.""" + legacy, deprecations = _when_import_legacy_name( + "pyoaev.contracts.variable_helper", name + ) + assert legacy == value + assert deprecations + + +def test_importing_pyoaev_contracts_is_silent() -> None: + """It imports pyoaev and its contracts modules without any warning.""" + script = ( + "import pyoaev, pyoaev.contracts, pyoaev.contracts.contract_builder\n" + "import pyoaev.contracts.contract_config, pyoaev.contracts.contract_utils\n" + "import pyoaev.contracts.variable_helper\n" + "import pyoaev.credential, pyoaev.credential.types, pyoaev.credential.utils\n" + ) + result = subprocess.run( + [sys.executable, "-W", "error::DeprecationWarning", "-c", script], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + +def test_unknown_legacy_name_still_raises() -> None: + """It does not swallow genuine misses.""" + with pytest.raises(ImportError): + exec("from pyoaev.contracts.contract_config import DoesNotExist", {}) diff --git a/libs/xtm-oaev-sdk/test/contracts/test_contract_element_argument_type.py b/libs/xtm-oaev-sdk/test/contracts/test_contract_element_argument_type.py new file mode 100644 index 00000000..4cad01c4 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_contract_element_argument_type.py @@ -0,0 +1,41 @@ +import json +import unittest + +import xtm_oaev_sdk as utils +from xtm_oaev_sdk import ContractText, PrimitiveType + + +def _serialize(field): + return json.loads(json.dumps(field, cls=utils.EnhancedJSONEncoder)) + + +class ContractElementArgumentTypeTest(unittest.TestCase): + def test_defaults_to_none_when_not_declared(self): + """Existing contracts that never pass argumentType keep working + unchanged — the platform normalizes a null value to PrimitiveType.Text + on its own, so this is not a breaking change for anything already + deployed.""" + untyped = ContractText(key="uri", label="URL") + self.assertIsNone(untyped.argumentType) + self.assertIsNone(_serialize(untyped)["argumentType"]) + + def test_explicit_argument_type_round_trips(self): + typed = ContractText( + key="basicUser", label="Username", argumentType=PrimitiveType.Username + ) + self.assertEqual(typed.argumentType, PrimitiveType.Username) + self.assertEqual(_serialize(typed)["argumentType"], "username") + + def test_argument_type_is_independent_of_widget_type(self): + """A field's rendering (ContractFieldType, on `type`) and its chaining + semantics (PrimitiveType, on `argumentType`) are separate axes.""" + typed = ContractText( + key="basicUser", label="Username", argumentType=PrimitiveType.Username + ) + serialized = _serialize(typed) + self.assertEqual(serialized["type"], "text") + self.assertEqual(serialized["argumentType"], "username") + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_contract_expectations.py b/libs/xtm-oaev-sdk/test/contracts/test_contract_expectations.py new file mode 100644 index 00000000..90b9be8b --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_contract_expectations.py @@ -0,0 +1,165 @@ +import json +import unittest + +import xtm_oaev_sdk as utils +from xtm_oaev_sdk import ContractCardinality, ContractExpectations +from xtm_oaev_sdk import ContractExpectationType as ExpectationType +from xtm_oaev_sdk import Expectation, SecurityPlatformType + + +def _expectation(expectation_type, name, is_predefined, expected_platforms=None): + # Only pass the field when declared so the default (None) path exercises + # the dataclass default_factory instead of an explicit empty list. + kwargs = {} + if expected_platforms is not None: + kwargs["expectation_expected_security_platform_types"] = expected_platforms + return Expectation( + expectation_type=expectation_type, + expectation_name=name, + expectation_description="", + expectation_score=100, + expectation_expectation_group=False, + expectation_is_predefined=is_predefined, + **kwargs, + ) + + +def _serialize(field): + return json.loads(json.dumps(field, cls=utils.EnhancedJSONEncoder)) + + +class TestContractExpectations(unittest.TestCase): + def test_predefined_derived_from_flag(self): + """ + The platform pre-fills expectations from the field-level + ``predefinedExpectations`` array, so it must be populated from the + expectations flagged ``expectation_is_predefined=True``. + """ + field = ContractExpectations( + key="expectations", + label="Expectations", + cardinality=ContractCardinality.Multiple, + availableExpectations=[ + _expectation(ExpectationType.detection, "Detection", True), + _expectation(ExpectationType.prevention, "Prevention", True), + _expectation(ExpectationType.vulnerability, "Not vulnerable", False), + ], + ) + + data = _serialize(field) + + self.assertEqual("expectation", data["type"]) + self.assertEqual( + ["DETECTION", "PREVENTION", "VULNERABILITY"], + [e["expectation_type"] for e in data["availableExpectations"]], + ) + # Only the flagged ones are pre-filled. + self.assertEqual( + ["DETECTION", "PREVENTION"], + [e["expectation_type"] for e in data["predefinedExpectations"]], + ) + + def test_explicit_predefined_is_kept(self): + detection = _expectation(ExpectationType.detection, "Detection", False) + field = ContractExpectations( + key="expectations", + label="Expectations", + availableExpectations=[detection], + predefinedExpectations=[detection], + ) + + data = _serialize(field) + + self.assertEqual( + ["DETECTION"], + [e["expectation_type"] for e in data["predefinedExpectations"]], + ) + + def test_explicit_empty_list_overrides_flags(self): + """ + An explicit empty ``predefinedExpectations`` list must suppress the + derivation even when some available expectations carry the predefined + flag, while still serializing as an (empty) array. + """ + field = ContractExpectations( + key="expectations", + label="Expectations", + availableExpectations=[ + _expectation(ExpectationType.detection, "Detection", True), + ], + predefinedExpectations=[], + ) + + data = _serialize(field) + + self.assertEqual([], data["predefinedExpectations"]) + + def test_no_flag_yields_no_predefined(self): + field = ContractExpectations( + key="expectations", + label="Expectations", + availableExpectations=[ + _expectation(ExpectationType.vulnerability, "Not vulnerable", False), + ], + ) + + data = _serialize(field) + + self.assertEqual([], data["predefinedExpectations"]) + + def test_expected_security_platform_types_default_empty(self): + """An expectation without declared platforms serializes an empty list.""" + field = ContractExpectations( + key="expectations", + label="Expectations", + availableExpectations=[ + _expectation(ExpectationType.detection, "Detection", True), + ], + ) + + data = _serialize(field) + + self.assertEqual( + [], + data["availableExpectations"][0][ + "expectation_expected_security_platform_types" + ], + ) + + def test_expected_security_platform_types_serialize(self): + """Declared platform types serialize as their string values on both + the available and predefined arrays.""" + field = ContractExpectations( + key="expectations", + label="Expectations", + availableExpectations=[ + _expectation( + ExpectationType.detection, + "Detection", + True, + expected_platforms=[ + SecurityPlatformType.EDR, + SecurityPlatformType.XDR, + ], + ), + ], + ) + + data = _serialize(field) + + self.assertEqual( + ["EDR", "XDR"], + data["availableExpectations"][0][ + "expectation_expected_security_platform_types" + ], + ) + self.assertEqual( + ["EDR", "XDR"], + data["predefinedExpectations"][0][ + "expectation_expected_security_platform_types" + ], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_contract_output_types.py b/libs/xtm-oaev-sdk/test/contracts/test_contract_output_types.py new file mode 100644 index 00000000..5e49c7ce --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_contract_output_types.py @@ -0,0 +1,22 @@ +import unittest + +from xtm_oaev_sdk import ContractOutputType + + +class ContractOutputTypeTest(unittest.TestCase): + def test_file_wire_label(self): + # The wire label is a public contract shared with the platform enum and every + # injector that declares a `file` output; it must stay exactly "file". + self.assertEqual(ContractOutputType.File.value, "file") + self.assertEqual(ContractOutputType.File, "file") + + def test_action_output_wire_label(self): + # The wire label is a public contract shared with the platform enum + # (io.openaev.database.model.ContractOutputType.ActionOutput); it must stay + # exactly "action_output". + self.assertEqual(ContractOutputType.ActionOutput.value, "action_output") + self.assertEqual(ContractOutputType.ActionOutput, "action_output") + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_contract_referenced_credential.py b/libs/xtm-oaev-sdk/test/contracts/test_contract_referenced_credential.py new file mode 100644 index 00000000..302abb2d --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_contract_referenced_credential.py @@ -0,0 +1,53 @@ +import json +import unittest + +from xtm_oaev_sdk import ( + ContractFieldKey, + ContractFieldType, + ContractReferencedCredential, + CredentialType, + EnhancedJSONEncoder, +) + + +def _serialize(field): + return json.loads(json.dumps(field, cls=EnhancedJSONEncoder)) + + +class ContractReferencedCredentialTest(unittest.TestCase): + def test_defaults_match_platform_contract_expectations(self): + field = ContractReferencedCredential() + serialized = _serialize(field) + + self.assertEqual(field.key, ContractFieldKey.CredentialReference.value) + self.assertEqual(field.label, "Select a credential reference") + self.assertFalse(field.mandatory) + self.assertTrue(field.multiple) + self.assertIsNone(field.credential_reference_type) + self.assertEqual(field.type, ContractFieldType.CredentialReference.value) + self.assertEqual(serialized["key"], "credential_reference") + self.assertEqual(serialized["type"], "credential-reference") + self.assertEqual(serialized["label"], "Select a credential reference") + self.assertFalse(serialized["mandatory"]) + self.assertTrue(serialized["multiple"]) + self.assertIsNone(serialized["credential_reference_type"]) + + def test_explicit_credential_type_serializes_to_platform_label(self): + field = ContractReferencedCredential( + credential_reference_type=CredentialType.CLOUD_AZURE + ) + + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AZURE) + self.assertEqual(_serialize(field)["credential_reference_type"], "CLOUD_AZURE") + + def test_identity_credential_type_serializes_to_platform_label(self): + field = ContractReferencedCredential( + credential_reference_type=CredentialType.IDENTITY + ) + + self.assertEqual(field.credential_reference_type, CredentialType.IDENTITY) + self.assertEqual(_serialize(field)["credential_reference_type"], "IDENTITY") + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_contracts.py b/libs/xtm-oaev-sdk/test/contracts/test_contracts.py new file mode 100644 index 00000000..ada58710 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_contracts.py @@ -0,0 +1,538 @@ +"""Tests for the contracts public surface exposed by xtm_oaev_sdk.""" + +import json + +import pytest +import xtm_oaev_sdk as sdk + + +def _given_public_symbol(name: str): + return getattr(sdk, name, None) + + +def _then_public_symbol_exists(symbol, name: str) -> None: + assert ( + symbol is not None + ), f"Expected xtm_oaev_sdk.{name} to exist on public surface" + + +def _given_existing_public_symbol(name: str): + symbol = _given_public_symbol(name) + _then_public_symbol_exists(symbol, name) + return symbol + + +def _given_contract_text(key: str = "text_key", label: str = "Text label"): + contract_text = _given_existing_public_symbol("ContractText") + return contract_text(key=key, label=label) + + +def _given_contract_checkbox(key: str = "checkbox_key", label: str = "Checkbox label"): + contract_checkbox = _given_existing_public_symbol("ContractCheckbox") + return contract_checkbox(key=key, label=label) + + +def _given_contract_tuple(key: str = "tuple_key", label: str = "Tuple label"): + contract_tuple = _given_existing_public_symbol("ContractTuple") + return contract_tuple(key=key, label=label) + + +def _given_contract_select(key: str = "select_key", label: str = "Select label"): + contract_select = _given_existing_public_symbol("ContractSelect") + return contract_select(key=key, label=label) + + +def _given_contract_attachment( + key: str = "attachment_key", label: str = "Attachment label" +): + contract_attachment = _given_existing_public_symbol("ContractAttachment") + return contract_attachment(key=key, label=label) + + +def _given_contract_textarea(key: str = "textarea_key", label: str = "Textarea label"): + contract_textarea = _given_existing_public_symbol("ContractTextArea") + return contract_textarea(key=key, label=label) + + +def _given_contract_team(key: str = "team_key", label: str = "Team label"): + contract_team = _given_existing_public_symbol("ContractTeam") + return contract_team(key=key, label=label) + + +def _given_contract_expectations( + key: str = "expectations_key", label: str = "Expectations label" +): + contract_expectations = _given_existing_public_symbol("ContractExpectations") + return contract_expectations(key=key, label=label) + + +def _given_contract_payload(key: str = "payload_key", label: str = "Payload label"): + contract_payload = _given_existing_public_symbol("ContractPayload") + return contract_payload(key=key, label=label) + + +def _given_contract_asset(label: str = "Asset label"): + contract_asset = _given_existing_public_symbol("ContractAsset") + return contract_asset(label=label) + + +def _given_contract_asset_group(label: str = "Asset Group label"): + contract_asset_group = _given_existing_public_symbol("ContractAssetGroup") + return contract_asset_group(label=label) + + +def _given_contract_output_element(): + output_element = _given_existing_public_symbol("ContractOutputElement") + contract_output_type = _given_existing_public_symbol("ContractOutputType") + return output_element( + type=contract_output_type.Text.value, + field="output_field", + labels=["Output field"], + isFindingCompatible=True, + isMultiple=False, + ) + + +def _given_contract_config(): + contract_config = _given_existing_public_symbol("ContractConfig") + supported_language = _given_existing_public_symbol("SupportedLanguage") + return contract_config( + type="injector", + expose=True, + label={supported_language.en: "Injector", supported_language.fr: "Injecteur"}, + color_dark="#111111", + color_light="#eeeeee", + ) + + +def _given_contract_instance(): + contract = _given_existing_public_symbol("Contract") + supported_language = _given_existing_public_symbol("SupportedLanguage") + return contract( + contract_id="contract-1", + label={supported_language.en: "Contract", supported_language.fr: "Contrat"}, + fields=[_given_contract_text()], + outputs=[_given_contract_output_element()], + config=_given_contract_config(), + manual=False, + ) + + +def _given_contract_builder_instance(): + contract_builder = _given_existing_public_symbol("ContractBuilder") + return contract_builder() + + +def _when_building_fluent_contract_builder_sequence(builder): + first = builder.mandatory(_given_contract_text("f1", "F1")) + second = builder.optional(_given_contract_checkbox("f2", "F2")) + third = builder.mandatory_group( + [_given_contract_text("f3", "F3"), _given_contract_text("f4", "F4")] + ) + fourth = builder.add_fields([_given_contract_select("f5", "F5")]) + fifth = builder.add_outputs([_given_contract_output_element()]) + built_fields = builder.build_fields() + built_outputs = builder.build_outputs() + return first, second, third, fourth, fifth, built_fields, built_outputs + + +def _then_builder_fluent_sequence_is_chainable(builder, call_results) -> None: + for result in call_results[:5]: + assert result is builder + + +def _then_builder_accumulates_expected_fields_and_outputs( + built_fields, built_outputs +) -> None: + assert [field.key for field in built_fields] == ["f1", "f2", "f3", "f4", "f5"] + assert len(built_outputs) == 1 + + +def _when_checking_builder_protocol(builder): + contract_builder_protocol = _given_existing_public_symbol("ContractBuilderProtocol") + return isinstance(builder, contract_builder_protocol) + + +def _then_value_is_true(value: bool) -> None: + assert value is True + + +def _when_reading_element_type(element): + return element.type + + +def _then_element_type_equals(actual: str, expected: str) -> None: + assert actual == expected + + +def _when_reading_tuple_cardinality(contract_tuple): + return contract_tuple.cardinality + + +def _then_cardinality_is_multiple(cardinality) -> None: + contract_cardinality = _given_existing_public_symbol("ContractCardinality") + assert cardinality == contract_cardinality.Multiple + + +def _when_reading_key(element): + return element.key + + +def _then_key_equals(actual: str, expected: str) -> None: + assert actual == expected + + +def _when_serializing_contract_add_input(contract_instance): + return contract_instance.to_contract_add_input("injector-123") + + +def _when_serializing_contract_update_input(contract_instance): + return contract_instance.to_contract_update_input() + + +def _then_add_input_contains_injector_and_contract_content(serialized: dict) -> None: + assert serialized["injector_id"] == "injector-123" + assert "contract_content" in serialized + assert isinstance(serialized["contract_content"], str) + assert json.loads(serialized["contract_content"])["contract_id"] == "contract-1" + + +def _then_update_input_omits_injector_id(serialized: dict) -> None: + assert "injector_id" not in serialized + assert "contract_content" in serialized + + +def _when_adding_attack_pattern(contract_instance): + contract_instance.add_attack_pattern("T1234") + return contract_instance.contract_attack_patterns_external_ids + + +def _then_list_contains_single_value(values: list[str], expected: str) -> None: + assert values == [expected] + + +def _when_adding_vulnerability(contract_instance): + contract_instance.add_vulnerability("CVE-2024-0001") + return contract_instance.contract_vulnerability_external_ids + + +def _when_adding_variable(contract_instance): + contract_variable = _given_existing_public_symbol("ContractVariable") + variable_type = _given_existing_public_symbol("VariableType") + contract_cardinality = _given_existing_public_symbol("ContractCardinality") + new_variable = contract_variable( + key="custom", + label="Custom variable", + type=variable_type.String, + cardinality=contract_cardinality.One, + ) + contract_instance.add_variable(new_variable) + return contract_instance.variables + + +def _then_variables_include_custom_key(variables) -> None: + assert any(variable.key == "custom" for variable in variables) + + +def _when_reading_default_contract_variables(contract_instance): + return contract_instance.variables + + +def _then_default_variables_include_expected_keys(variables) -> None: + keys = [variable.key for variable in variables] + assert len(keys) == 7 + assert "user" in keys + assert "exercise" in keys + assert "teams" in keys + assert "player_uri" in keys + assert "challenges_uri" in keys + assert "scoreboard_uri" in keys + assert "lessons_uri" in keys + + +def _when_building_user_variable(): + variable_helper = _given_existing_public_symbol("VariableHelper") + return variable_helper.user_variable() + + +def _when_building_exercise_variable(): + variable_helper = _given_existing_public_symbol("VariableHelper") + return variable_helper.exercise_variable() + + +def _when_building_team_variable(): + variable_helper = _given_existing_public_symbol("VariableHelper") + return variable_helper.team_variable() + + +def _when_building_uri_variables(): + variable_helper = _given_existing_public_symbol("VariableHelper") + return variable_helper.uri_variables() + + +def _then_variable_has_children_count(variable, expected: int) -> None: + assert len(variable.children) == expected + + +def _then_variable_cardinality_is_multiple(variable) -> None: + contract_cardinality = _given_existing_public_symbol("ContractCardinality") + assert variable.cardinality == contract_cardinality.Multiple + + +def _then_uri_variables_have_expected_length(uri_variables) -> None: + assert len(uri_variables) == 4 + + +def _when_preparing_contracts(contracts): + prepare_contracts = _given_existing_public_symbol("prepare_contracts") + return prepare_contracts(contracts) + + +def _then_prepared_contracts_have_expected_shape(serialized_contracts) -> None: + assert len(serialized_contracts) == 1 + assert serialized_contracts[0]["contract_id"] == "contract-1" + assert "contract_content" in serialized_contracts[0] + assert isinstance(serialized_contracts[0]["contract_content"], str) + + +def _when_reading_enum_values(): + contract_cardinality = _given_existing_public_symbol("ContractCardinality") + variable_type = _given_existing_public_symbol("VariableType") + return contract_cardinality, variable_type + + +def _then_enum_values_match(contract_cardinality, variable_type) -> None: + assert contract_cardinality.One.value == "1" + assert contract_cardinality.Multiple.value == "n" + assert variable_type.String.value == "String" + assert variable_type.Object.value == "Object" + + +def _when_reading_contract_expectation_members(): + contract_expectation_type = _given_existing_public_symbol("ContractExpectationType") + return list(contract_expectation_type) + + +def _when_reading_supported_language_members(): + supported_language = _given_existing_public_symbol("SupportedLanguage") + return list(supported_language) + + +def _then_collection_has_length(collection, expected: int) -> None: + assert len(collection) == expected + + +def _when_building_mandatory_group(builder): + first = _given_contract_text("group_1", "Group 1") + second = _given_contract_text("group_2", "Group 2") + builder.mandatory_group([first, second]) + return first, second + + +def _then_elements_have_shared_mandatory_groups(elements) -> None: + keys = [element.key for element in elements] + for element in elements: + assert element.mandatory is True + assert element.mandatoryGroups == keys + + +@pytest.mark.parametrize( + ("factory_name", "expected_type"), + [ + ("_given_contract_text", "text"), + ("_given_contract_checkbox", "checkbox"), + ("_given_contract_tuple", "tuple"), + ("_given_contract_asset", "asset"), + ("_given_contract_asset_group", "asset-group"), + ("_given_contract_attachment", "attachment"), + ("_given_contract_select", "select"), + ("_given_contract_textarea", "textarea"), + ("_given_contract_team", "team"), + ("_given_contract_expectations", "expectation"), + ("_given_contract_payload", "payload"), + ], +) +def test_contract_element_subclasses_set_type_from_get_type( + factory_name: str, expected_type: str +) -> None: + """Verifies each ContractElement subclass sets its type via __post_init__.""" + element = globals()[factory_name]() + actual_type = _when_reading_element_type(element) + _then_element_type_equals(actual_type, expected_type) + + +@pytest.mark.parametrize( + "symbol_name", + [ + "ContractBuilder", + "ContractBuilderProtocol", + "ContractCardinality", + "ContractVariable", + "VariableType", + "VariableHelper", + "Contract", + "ContractConfig", + "ContractElement", + "ContractText", + "ContractCheckbox", + "ContractTuple", + "ContractAsset", + "ContractAssetGroup", + "ContractExpectationType", + "ContractExpectations", + "ContractPayload", + "ContractSelect", + "ContractAttachment", + "ContractTextArea", + "ContractTeam", + "Expectation", + "Domain", + "LinkedFieldModel", + "SupportedLanguage", + "prepare_contracts", + "ContractOutputElement", + "ContractOutputType", + "ContractFieldType", + "ContractFieldKey", + "ContractCardinalityElement", + ], +) +def test_contract_public_surface_exposes_expected_symbols(symbol_name: str) -> None: + """Verifies the xtm_oaev_sdk public surface exposes required contract symbols.""" + symbol = _given_public_symbol(symbol_name) + _then_public_symbol_exists(symbol, symbol_name) + + +def test_contract_builder_fluent_api_is_chainable_and_accumulates_fields() -> None: + """Verifies ContractBuilder fluent methods return self and collect fields/outputs.""" + builder = _given_contract_builder_instance() + call_results = _when_building_fluent_contract_builder_sequence(builder) + _then_builder_fluent_sequence_is_chainable(builder, call_results) + _then_builder_accumulates_expected_fields_and_outputs( + call_results[5], call_results[6] + ) + + +def test_contract_builder_satisfies_protocol() -> None: + """Verifies ContractBuilder satisfies ContractBuilderProtocol runtime checks.""" + builder = _given_contract_builder_instance() + result = _when_checking_builder_protocol(builder) + _then_value_is_true(result) + + +def test_contract_tuple_forces_multiple_cardinality() -> None: + """Verifies ContractTuple forces cardinality to ContractCardinality.Multiple.""" + contract_tuple = _given_contract_tuple() + cardinality = _when_reading_tuple_cardinality(contract_tuple) + _then_cardinality_is_multiple(cardinality) + + +def test_contract_asset_key_is_auto_set_to_assets() -> None: + """Verifies ContractAsset key is automatically set to assets.""" + contract_asset = _given_contract_asset() + key = _when_reading_key(contract_asset) + _then_key_equals(key, "assets") + + +def test_contract_asset_group_key_is_auto_set_to_asset_groups() -> None: + """Verifies ContractAssetGroup key is automatically set to asset_groups.""" + contract_asset_group = _given_contract_asset_group() + key = _when_reading_key(contract_asset_group) + _then_key_equals(key, "asset_groups") + + +def test_contract_to_contract_add_input_includes_injector_id_and_content() -> None: + """Verifies contract add payload includes injector_id and serialized content.""" + contract_instance = _given_contract_instance() + serialized = _when_serializing_contract_add_input(contract_instance) + _then_add_input_contains_injector_and_contract_content(serialized) + + +def test_contract_to_contract_update_input_omits_injector_id() -> None: + """Verifies contract update payload omits injector_id.""" + contract_instance = _given_contract_instance() + serialized = _when_serializing_contract_update_input(contract_instance) + _then_update_input_omits_injector_id(serialized) + + +def test_contract_add_attack_pattern_appends_value() -> None: + """Verifies add_attack_pattern appends values to the attack pattern list.""" + contract_instance = _given_contract_instance() + values = _when_adding_attack_pattern(contract_instance) + _then_list_contains_single_value(values, "T1234") + + +def test_contract_add_vulnerability_appends_value() -> None: + """Verifies add_vulnerability appends values to the vulnerability list.""" + contract_instance = _given_contract_instance() + values = _when_adding_vulnerability(contract_instance) + _then_list_contains_single_value(values, "CVE-2024-0001") + + +def test_contract_add_variable_appends_to_variables_list() -> None: + """Verifies add_variable appends a custom variable to contract variables.""" + contract_instance = _given_contract_instance() + variables = _when_adding_variable(contract_instance) + _then_variables_include_custom_key(variables) + + +def test_contract_default_variables_include_expected_core_and_uri_variables() -> None: + """Verifies default contract variables include user, exercise, teams, and 4 URIs.""" + contract_instance = _given_contract_instance() + variables = _when_reading_default_contract_variables(contract_instance) + _then_default_variables_include_expected_keys(variables) + + +def test_variable_helper_user_variable_contains_five_children() -> None: + """Verifies VariableHelper.user_variable returns a variable with five children.""" + user_variable = _when_building_user_variable() + _then_variable_has_children_count(user_variable, 5) + + +def test_variable_helper_exercise_variable_contains_three_children() -> None: + """Verifies VariableHelper.exercise_variable returns a variable with three children.""" + exercise_variable = _when_building_exercise_variable() + _then_variable_has_children_count(exercise_variable, 3) + + +def test_variable_helper_team_variable_has_multiple_cardinality() -> None: + """Verifies VariableHelper.team_variable returns a variable with multiple cardinality.""" + team_variable = _when_building_team_variable() + _then_variable_cardinality_is_multiple(team_variable) + + +def test_variable_helper_uri_variables_returns_four_items() -> None: + """Verifies VariableHelper.uri_variables returns four URI contract variables.""" + uri_variables = _when_building_uri_variables() + _then_uri_variables_have_expected_length(uri_variables) + + +def test_prepare_contracts_serializes_contracts_to_dict_list() -> None: + """Verifies prepare_contracts serializes Contract instances to dict payloads.""" + serialized_contracts = _when_preparing_contracts([_given_contract_instance()]) + _then_prepared_contracts_have_expected_shape(serialized_contracts) + + +def test_contract_enums_expose_expected_values() -> None: + """Verifies ContractCardinality and VariableType expose expected values.""" + contract_cardinality, variable_type = _when_reading_enum_values() + _then_enum_values_match(contract_cardinality, variable_type) + + +def test_contract_expectation_type_exposes_eight_members() -> None: + """Verifies ContractExpectationType contains exactly eight members.""" + members = _when_reading_contract_expectation_members() + _then_collection_has_length(members, 8) + + +def test_supported_language_exposes_fr_and_en_members() -> None: + """Verifies SupportedLanguage contains exactly fr and en members.""" + members = _when_reading_supported_language_members() + _then_collection_has_length(members, 2) + + +def test_contract_builder_mandatory_group_sets_all_group_keys() -> None: + """Verifies mandatory_group sets shared mandatoryGroups across all grouped elements.""" + builder = _given_contract_builder_instance() + elements = _when_building_mandatory_group(builder) + _then_elements_have_shared_mandatory_groups(elements) diff --git a/libs/xtm-oaev-sdk/test/contracts/test_credential_types.py b/libs/xtm-oaev-sdk/test/contracts/test_credential_types.py new file mode 100644 index 00000000..76f02bc3 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_credential_types.py @@ -0,0 +1,27 @@ +import unittest + +from xtm_oaev_sdk import CredentialType + + +class CredentialTypeTest(unittest.TestCase): + def test_every_value_matches_the_supported_platform_labels(self): + expected_labels = { + "IDENTITY", + "CLOUD_AWS", + "CLOUD_AZURE", + "CLOUD_GCP", + } + actual_labels = {member.value for member in CredentialType} + self.assertEqual(actual_labels, expected_labels) + + def test_identity_wire_label(self): + self.assertEqual(CredentialType.IDENTITY.value, "IDENTITY") + self.assertEqual(CredentialType.IDENTITY, "IDENTITY") + + def test_aws_wire_label(self): + self.assertEqual(CredentialType.CLOUD_AWS.value, "CLOUD_AWS") + self.assertEqual(CredentialType.CLOUD_AWS, "CLOUD_AWS") + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_credential_utils.py b/libs/xtm-oaev-sdk/test/contracts/test_credential_utils.py new file mode 100644 index 00000000..295db924 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_credential_utils.py @@ -0,0 +1,59 @@ +import json +import unittest + +from xtm_oaev_sdk import ( + ContractReferencedCredential, + CredentialType, + EnhancedJSONEncoder, + build_single_referenced_credential_element, +) + + +def _serialize(field): + return json.loads(json.dumps(field, cls=EnhancedJSONEncoder)) + + +class CredentialUtilsTest(unittest.TestCase): + def test_aws_provider_maps_to_aws_credential_reference(self): + field = build_single_referenced_credential_element("aws") + + self.assertIsInstance(field, ContractReferencedCredential) + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AWS) + + def test_eks_provider_maps_to_aws_credential_reference(self): + field = build_single_referenced_credential_element("eks") + + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AWS) + + def test_azure_provider_maps_to_azure_credential_reference(self): + field = build_single_referenced_credential_element("azure") + + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AZURE) + + def test_gcp_provider_maps_to_gcp_credential_reference(self): + field = build_single_referenced_credential_element("gcp") + + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_GCP) + + def test_provider_mapping_is_case_insensitive(self): + field = build_single_referenced_credential_element("AWS") + + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AWS) + + def test_package_public_helper_builds_the_same_field(self): + field = build_single_referenced_credential_element("azure") + + self.assertIsInstance(field, ContractReferencedCredential) + self.assertEqual(field.credential_reference_type, CredentialType.CLOUD_AZURE) + self.assertEqual(_serialize(field)["credential_reference_type"], "CLOUD_AZURE") + self.assertFalse(field.multiple) + + def test_unknown_provider_leaves_credential_type_empty(self): + field = build_single_referenced_credential_element("openstack") + + self.assertIsNone(field.credential_reference_type) + self.assertIsNone(_serialize(field)["credential_reference_type"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/contracts/test_primitive_type.py b/libs/xtm-oaev-sdk/test/contracts/test_primitive_type.py new file mode 100644 index 00000000..76863d7f --- /dev/null +++ b/libs/xtm-oaev-sdk/test/contracts/test_primitive_type.py @@ -0,0 +1,77 @@ +import unittest + +from xtm_oaev_sdk import PrimitiveType + + +class PrimitiveTypeTest(unittest.TestCase): + def test_username_wire_label(self): + # The wire label is a public contract shared with the platform enum + # (io.openaev.database.model.PrimitiveType.Username); it must stay + # exactly "username". + self.assertEqual(PrimitiveType.Username.value, "username") + self.assertEqual(PrimitiveType.Username, "username") + + def test_host_wire_label(self): + self.assertEqual(PrimitiveType.Host.value, "host") + self.assertEqual(PrimitiveType.Host, "host") + + def test_text_wire_label(self): + # The platform's own normalizer defaults a missing argumentType to + # exactly this value — it must stay "text". + self.assertEqual(PrimitiveType.Text.value, "text") + self.assertEqual(PrimitiveType.Text, "text") + + def test_action_output_wire_label(self): + # Shared with ContractOutputType.ActionOutput; both enums describe the + # same platform-side value from two different angles (an output + # producing it vs. an input typed to receive it) and must not drift + # apart. + self.assertEqual(PrimitiveType.ActionOutput.value, "action_output") + + def test_every_value_matches_the_openaev_platform_enum(self): + # io.openaev.database.model.PrimitiveType, transcribed label-for-label. + # Keep this set in sync with that file, not just the four spot-checked + # above. + expected_labels = { + "account_with_password_not_required", + "action_output", + "admin_username", + "asreproastable_account", + "asset_group_id", + "asset_id", + "computer_name", + "cve", + "delegation_account", + "document", + "domain", + "file_name", + "file_path", + "group_name", + "hash", + "host", + "ipv4", + "ipv6", + "ip_subnet", + "kerberoastable_account", + "key", + "number", + "password", + "permissions", + "port", + "service", + "severity", + "share_name", + "sid", + "targeted-asset", + "text", + "username", + "value", + "vulnerability_name", + "vulnerability_status", + } + actual_labels = {member.value for member in PrimitiveType} + self.assertEqual(actual_labels, expected_labels) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py index 62a7b8e3..9f5fa44c 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -4,12 +4,15 @@ _core/ Shared implementations (errors, utils, ssl_utils, asset_types, security_domain) _core/configuration/ Configuration feature implementation + _core/contracts/ OpenAEV scenarios feature implementation _core/signatures/ Signature feature implementation seam/common/ Shared errors, enums, types, and utilities seam/configuration/ Configuration I/O surface + seam/contracts/ Scenarios capability behaviour + builder protocol seam/signatures/ Signatures inbound I/O seam (versioned) seam/port_io/ Port I/O wire schema (versioned, seam-owned) seam/deprecation.py PEP 562 deprecation primitive + contracts/contracts/ OpenAEV scenario data models (business layer) public/ User-facing re-exports The seam is the compatibility boundary and the sole door between the public diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py new file mode 100644 index 00000000..6688b45d --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py @@ -0,0 +1,82 @@ +"""Contract building: field definitions, validation, and serialization. + +Re-exports the public contract surface for internal engine use. +Public consumers import from xtm_oaev_sdk directly. +""" + +from .contract_builder import ContractBuilder, ContractBuilderProtocol +from .contract_config import ( + Contract, + ContractAiTarget, + ContractAsset, + ContractAssetGroup, + ContractAttachment, + ContractCardinalityElement, + ContractCheckbox, + ContractConfig, + ContractElement, + ContractExpectations, + ContractExpectationType, + ContractFieldKey, + ContractFieldType, + ContractOutputElement, + ContractOutputType, + ContractPayload, + ContractReferencedCredential, + ContractSelect, + ContractTeam, + ContractText, + ContractTextArea, + ContractTuple, + Domain, + Expectation, + LinkedFieldModel, + PrimitiveType, + SecurityPlatformType, + SupportedLanguage, + prepare_contracts, +) +from .contract_utils import ContractCardinality, ContractVariable, VariableType +from .credential_types import CredentialType +from .credentials import build_single_referenced_credential_element +from .variable_helper import VariableHelper + +__all__ = [ + "Contract", + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractBuilder", + "ContractBuilderProtocol", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractElement", + "ContractExpectationType", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "CredentialType", + "Domain", + "Expectation", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableHelper", + "VariableType", + "build_single_referenced_credential_element", + "prepare_contracts", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_builder.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_builder.py new file mode 100644 index 00000000..bb5374aa --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_builder.py @@ -0,0 +1,172 @@ +"""Contract builder: fluent API for constructing contract field sets. + +Provides the ContractBuilder class and its Protocol for building +contract input field and output declarations with a chainable API. + +Example: + >>> builder = ContractBuilder() + >>> builder.mandatory(ContractText(key="name", label="Name")) + >>> builder.optional(ContractCheckbox(key="verbose", label="Verbose")) + >>> fields = builder.build_fields() +""" + +from typing import Protocol, runtime_checkable + +from xtm_oaev_sdk._core.contracts.contract_config import ( + ContractElement, + ContractOutputElement, +) + +__all__ = [ + "ContractBuilder", + "ContractBuilderProtocol", +] + + +@runtime_checkable +class ContractBuilderProtocol(Protocol): + """Behavioral contract for contract field builders. + + Defines the chainable API for constructing contract field sets. + Consumers can code against this protocol for testability. + """ + + def add_fields(self, fields: list[ContractElement]) -> "ContractBuilderProtocol": + """Add multiple fields at once.""" + ... + + def add_outputs( + self, outputs: list[ContractOutputElement] + ) -> "ContractBuilderProtocol": + """Add multiple output declarations at once.""" + ... + + def mandatory(self, element: ContractElement) -> "ContractBuilderProtocol": + """Add a mandatory field.""" + ... + + def optional(self, element: ContractElement) -> "ContractBuilderProtocol": + """Add an optional field.""" + ... + + def mandatory_group( + self, elements: list[ContractElement] + ) -> "ContractBuilderProtocol": + """Add a group of mutually-required fields.""" + ... + + def build_fields(self) -> list[ContractElement]: + """Return the accumulated fields.""" + ... + + def build_outputs(self) -> list[ContractOutputElement]: + """Return the accumulated outputs.""" + ... + + +class ContractBuilder: + """Fluent builder for contract field and output declarations. + + Provides a chainable API to construct contract definitions + with mandatory, optional, and grouped fields. + + Example: + >>> b = ContractBuilder() + >>> b.mandatory(ContractText(key="target", label="Target URL")) + >>> b.optional(ContractCheckbox(key="verify_ssl", label="Verify SSL")) + >>> fields = b.build_fields() + >>> len(fields) + 2 + """ + + fields: list[ContractElement] + outputs: list[ContractOutputElement] + + def __init__(self) -> None: + self.fields = [] + self.outputs = [] + + def add_fields(self, fields: list[ContractElement]) -> "ContractBuilder": + """Add multiple fields at once. + + Args: + fields: List of contract elements to add. + + Returns: + Self for chaining. + """ + self.fields = self.fields + fields + return self + + def add_outputs(self, outputs: list[ContractOutputElement]) -> "ContractBuilder": + """Add multiple output declarations at once. + + Args: + outputs: List of output elements to add. + + Returns: + Self for chaining. + """ + self.outputs = self.outputs + outputs + return self + + def mandatory(self, element: ContractElement) -> "ContractBuilder": + """Add a mandatory field. + + Args: + element: The contract element to mark as mandatory. + + Returns: + Self for chaining. + """ + element.mandatory = True + self.fields.append(element) + return self + + def optional(self, element: ContractElement) -> "ContractBuilder": + """Add an optional field. + + Args: + element: The contract element to mark as optional. + + Returns: + Self for chaining. + """ + element.mandatory = False + self.fields.append(element) + return self + + def mandatory_group(self, elements: list[ContractElement]) -> "ContractBuilder": + """Add a group of mutually-required fields. + + All fields in the group reference each other's keys and are + marked mandatory. + + Args: + elements: List of contract elements forming the group. + + Returns: + Self for chaining. + """ + keys: list[str] = list(map(lambda iterable: iterable.key, elements)) + for element in elements: + element.mandatory = True + element.mandatoryGroups = keys + self.fields.append(element) + return self + + def build_fields(self) -> list[ContractElement]: + """Return all accumulated contract fields. + + Returns: + List of ContractElement instances added via this builder. + """ + return self.fields + + def build_outputs(self) -> list[ContractOutputElement]: + """Return all accumulated output declarations. + + Returns: + List of ContractOutputElement instances added via this builder. + """ + return self.outputs diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_config.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_config.py new file mode 100644 index 00000000..ce83ff8a --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_config.py @@ -0,0 +1,624 @@ +"""Contract configuration: field types, elements, and contract definitions. + +Defines the full contract model hierarchy used by injectors and collectors +to declare their configuration schemas, input fields, and output types. + +Key types: + - ContractElement (ABC): Base for all contract field types. + - Contract: A complete contract definition with fields, outputs, and metadata. + - ContractExpectationType: Expectation classification for contract fields. +""" + +import json +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Optional + +from xtm_oaev_sdk._core.contracts.contract_utils import ( + ContractCardinality, + ContractVariable, +) +from xtm_oaev_sdk._core.contracts.credential_types import CredentialType +from xtm_oaev_sdk._core.contracts.variable_helper import VariableHelper +from xtm_oaev_sdk._core.utils import EnhancedJSONEncoder + +__all__ = [ + "Contract", + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractElement", + "ContractExpectationType", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "Domain", + "Expectation", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "prepare_contracts", +] + + +class SupportedLanguage(str, Enum): + """Supported languages for contract label localization.""" + + fr: str = "fr" + en: str = "en" + + +class ContractFieldType(str, Enum): + """Available field types for contract input definitions.""" + + Text: str = "text" + Number: str = "number" + Tuple: str = "tuple" + Checkbox: str = "checkbox" + Textarea: str = "textarea" + Select: str = "select" + Article: str = "article" + Challenge: str = "challenge" + DependencySelect: str = "dependency-select" + Attachment: str = "attachment" + Team: str = "team" + Expectation: str = "expectation" + Asset: str = "asset" + AssetGroup: str = "asset-group" + AiTarget: str = "ai-target" + Payload: str = "payload" + CredentialReference: str = "credential-reference" + + +class ContractFieldKey(str, Enum): + """Predefined field keys for special contract fields.""" + + Asset: str = "assets" + AssetGroup: str = "asset_groups" + CredentialReference: str = "credential_reference" + + +class ContractOutputType(str, Enum): + """Available output types for contract result declarations.""" + + Text: str = "text" + ActionOutput: str = "action_output" + Number: str = "number" + Port: str = "port" + PortsScan: str = "portscan" + IPv4: str = "ipv4" + IPv6: str = "ipv6" + CVE: str = "cve" + Asset: str = "asset" + Credentials: str = "credentials" + Username: str = "username" + Share: str = "share" + File: str = "file" + AdminUsername: str = "admin_username" + Group: str = "group" + Computer: str = "computer" + PasswordPolicy: str = "password_policy" + Delegation: str = "delegation" + Sid: str = "sid" + Vulnerability: str = "vulnerability" + AccountWithPasswordNotRequired: str = "account_with_password_not_required" + ExpectationSignature: str = "expectation_signature" + AsreproastableAccount: str = "asreproastable_account" + KerberoastableAccount: str = "kerberoastable_account" + + +class PrimitiveType(str, Enum): + """Semantic type of a contract argument's value, independent of how the field + renders in the UI (see ``ContractFieldType``). Mirrors the openaev platform's + own ``PrimitiveType`` label-for-label so a contract pushed from here needs no + translation on the other end. + """ + + AccountWithPasswordNotRequired: str = "account_with_password_not_required" + ActionOutput: str = "action_output" + AdminUsername: str = "admin_username" + AsreproastableAccount: str = "asreproastable_account" + AssetGroupId: str = "asset_group_id" + AssetId: str = "asset_id" + ComputerName: str = "computer_name" + CVE: str = "cve" + DelegationAccount: str = "delegation_account" + Document: str = "document" + Domain: str = "domain" + FileName: str = "file_name" + FilePath: str = "file_path" + GroupName: str = "group_name" + Hash: str = "hash" + Host: str = "host" + IPv4: str = "ipv4" + IPv6: str = "ipv6" + IpSubnet: str = "ip_subnet" + KerberoastableAccount: str = "kerberoastable_account" + Key: str = "key" + Number: str = "number" + Password: str = "password" + Permissions: str = "permissions" + Port: str = "port" + Service: str = "service" + Severity: str = "severity" + ShareName: str = "share_name" + SID: str = "sid" + TargetedAsset: str = "targeted-asset" + Text: str = "text" + Username: str = "username" + Value: str = "value" + VulnerabilityName: str = "vulnerability_name" + VulnerabilityStatus: str = "vulnerability_status" + + +class ContractExpectationType(str, Enum): + """Expectation type classification for contract fields. + + Defines the 8 expectation categories available in contract definitions. + Renamed from ExpectationType to avoid collision with signatures module. + """ + + text: str = "TEXT" + document: str = "DOCUMENT" + article: str = "ARTICLE" + challenge: str = "CHALLENGE" + manual: str = "MANUAL" + prevention: str = "PREVENTION" + detection: str = "DETECTION" + vulnerability: str = "VULNERABILITY" + + +class SecurityPlatformType(str, Enum): + """Categories of security platform expected to fulfil a technical expectation. + + When an expectation declares one or more of these, the platform focuses the + technical (DETECTION / PREVENTION / VULNERABILITY) result on collectors of + those types only (instead of every connected security platform). An empty + list means "any security platform". + """ + + EDR: str = "EDR" + XDR: str = "XDR" + SIEM: str = "SIEM" + SOAR: str = "SOAR" + NDR: str = "NDR" + ISPM: str = "ISPM" + LLM_FIREWALL: str = "LLM_FIREWALL" + AI_GATEWAY: str = "AI_GATEWAY" + + +@dataclass +class Expectation: + """An expectation entry within a contract expectations field. + + Args: + expectation_type: The category of expectation. + expectation_name: Human-readable name. + expectation_description: Detailed description. + expectation_score: Score weight for this expectation. + expectation_expectation_group: Whether this belongs to a group. + """ + + expectation_type: ContractExpectationType + expectation_name: str + expectation_description: str + expectation_score: int + expectation_expectation_group: bool + expectation_is_predefined: bool = False + expectation_is_multi_selectable: bool = False + # Security platform types expected to fulfil this expectation. Empty = any + # platform (unchanged behaviour). Typically set for technical expectations + # (DETECTION / PREVENTION / VULNERABILITY), left empty for MANUAL ones. + expectation_expected_security_platform_types: list[SecurityPlatformType] = field( + default_factory=list + ) + + +@dataclass +class LinkedFieldModel: + """A linked field reference within a contract element.""" + + key: str + type: ContractFieldType + + +@dataclass +class ContractElement(ABC): + """Abstract base for all contract field elements. + + Subclass this to define new field types. Each subclass must + implement the ``get_type`` property returning the field type string. + + Args: + key: Unique field identifier. + label: Human-readable field label. + mandatory: Whether this field is required. + readOnly: Whether this field is read-only in the UI. + """ + + key: str + label: str + type: str = field(default="", init=False) + mandatoryGroups: list[str] = field(default_factory=list) + mandatoryConditionFields: list[str] = field(default_factory=list) + mandatoryConditionValues: dict[str, Any] = field(default_factory=list) + visibleConditionFields: list[str] = field(default_factory=list) + visibleConditionValues: dict[str, Any] = field(default_factory=list) + linkedFields: list[str] = field(default_factory=list) + mandatory: bool = False + readOnly: bool = False + # The argument's chaining/semantic type (``PrimitiveType``) — matches the + # platform's own ``argumentType`` label-for-label. ``None`` for fields with + # no established type; the platform normalizes a missing value to ``Text``. + argumentType: PrimitiveType | None = None + + @property + @abstractmethod + def get_type(self) -> str: + """Return the contract field type string.""" + pass + + def __post_init__(self) -> None: + self.type = self.get_type + + +@dataclass +class ContractCardinalityElement(ContractElement, ABC): + """Contract element with cardinality (single vs multiple values).""" + + cardinality: str = ContractCardinality.One + defaultValue: list[str] = field(default_factory=list) + + +@dataclass +class ContractOutputElement(ABC): + """An output field declaration in a contract. + + Args: + type: Output type (from ContractOutputType). + field: Field identifier. + labels: Localized labels for the output. + isFindingCompatible: Whether output maps to a finding. + isMultiple: Whether multiple values are produced. + """ + + type: str + field: str + labels: list[str] + isFindingCompatible: bool + isMultiple: bool + + +@dataclass +class ContractConfig: + """Contract display configuration. + + Args: + type: Contract type identifier. + expose: Whether this contract is visible in the UI. + label: Localized labels. + color_dark: Hex color for dark theme. + color_light: Hex color for light theme. + """ + + type: str + expose: bool + label: dict[SupportedLanguage, str] + color_dark: str + color_light: str + + +@dataclass +class Domain: + """A domain classification for a contract.""" + + domain_id: str + domain_name: str + domain_color: str + domain_created_at: str + domain_updated_at: str + + +@dataclass +class Contract: + """A complete contract definition for an injector or collector. + + Contains fields (inputs), outputs, configuration, and metadata + for registering the contract with the OpenAEV platform. + + Args: + contract_id: Unique contract identifier. + label: Localized contract labels. + fields: Input field definitions. + outputs: Output field declarations. + config: Display configuration. + manual: Whether this is a manual-execution contract. + variables: Template variables available for injection. + """ + + contract_id: str + label: dict[SupportedLanguage, str] + fields: list[ContractElement] + outputs: list[ContractOutputElement] + config: ContractConfig + manual: bool + variables: list[ContractVariable] = field( + default_factory=lambda: ( + [ + VariableHelper.user_variable(), + VariableHelper.exercise_variable(), + VariableHelper.team_variable(), + ] + + VariableHelper.uri_variables() + ) + ) + contract_attack_patterns_external_ids: list[str] = field(default_factory=list) + contract_vulnerability_external_ids: list[str] = field(default_factory=list) + is_atomic_testing: bool = True + platforms: list[str] = field(default_factory=list) + external_id: str | None = None + domains: list[Domain] | None = None + + def add_attack_pattern(self, var: str) -> None: + """Append an attack pattern external ID to this contract.""" + self.contract_attack_patterns_external_ids.append(var) + + def add_vulnerability(self, var: str) -> None: + """Append a vulnerability external ID to this contract.""" + self.contract_vulnerability_external_ids.append(var) + + def add_variable(self, var: ContractVariable) -> None: + """Add a custom template variable to this contract.""" + self.variables.append(var) + + def to_contract_add_input(self, source_id: str) -> dict[str, Any]: + """Serialize this contract for the platform creation API. + + Args: + source_id: The injector ID that owns this contract. + + Returns: + Dict payload for the contract creation endpoint. + """ + return { + "contract_id": self.contract_id, + "external_contract_id": self.external_id, + "injector_id": source_id, + "contract_manual": self.manual, + "contract_labels": self.label, + "contract_attack_patterns_external_ids": self.contract_attack_patterns_external_ids, + "contract_vulnerability_external_ids": self.contract_vulnerability_external_ids, + "contract_content": json.dumps(self, cls=EnhancedJSONEncoder), + "is_atomic_testing": self.is_atomic_testing, + "contract_platforms": self.platforms, + "contract_domains": self.domains, + } + + def to_contract_update_input(self) -> dict[str, Any]: + """Serialize this contract for the platform update API. + + Returns: + Dict payload for the contract update endpoint. + """ + return { + "contract_manual": self.manual, + "contract_labels": self.label, + "contract_attack_patterns_external_ids": self.contract_attack_patterns_external_ids, + "contract_vulnerability_external_ids": self.contract_vulnerability_external_ids, + "contract_content": json.dumps(self, cls=EnhancedJSONEncoder), + "is_atomic_testing": self.is_atomic_testing, + "contract_platforms": self.platforms, + "contract_domains": self.domains, + } + + +@dataclass +class ContractTeam(ContractCardinalityElement): + """Team selection field.""" + + @property + def get_type(self) -> str: + return ContractFieldType.Team.value + + +@dataclass +class ContractText(ContractCardinalityElement): + """Text input field with optional default value.""" + + defaultValue: str = "" + + @property + def get_type(self) -> str: + return ContractFieldType.Text.value + + +def prepare_contracts(contracts: list[Contract]) -> list[dict[str, Any]]: + """Serialize a list of contracts for bulk platform registration. + + Args: + contracts: List of Contract instances to serialize. + + Returns: + List of dicts suitable for the bulk contract registration API. + """ + return list( + map( + lambda c: { + "contract_id": c.contract_id, + "contract_labels": c.label, + "contract_attack_patterns_external_ids": c.contract_attack_patterns_external_ids, + "contract_content": json.dumps(c, cls=EnhancedJSONEncoder), + "contract_platforms": c.platforms, + "contract_domains": c.domains, + }, + contracts, + ) + ) + + +@dataclass +class ContractTuple(ContractCardinalityElement): + """Tuple (multi-value) input field with optional file attachment.""" + + def __post_init__(self) -> None: + super().__post_init__() + self.cardinality = ContractCardinality.Multiple + + attachmentKey: str | None = None + contractAttachment: bool = False + tupleFilePrefix: str = "file :: " + + @property + def get_type(self) -> str: + return ContractFieldType.Tuple.value + + +@dataclass +class ContractTextArea(ContractCardinalityElement): + """Textarea input field with optional rich text support.""" + + defaultValue: str = "" + richText: bool = False + + @property + def get_type(self) -> str: + return ContractFieldType.Textarea.value + + +@dataclass +class ContractCheckbox(ContractElement): + """Checkbox (boolean) input field.""" + + defaultValue: bool = False + + @property + def get_type(self) -> str: + return ContractFieldType.Checkbox.value + + +@dataclass +class ContractAttachment(ContractCardinalityElement): + """File attachment field.""" + + @property + def get_type(self) -> str: + return ContractFieldType.Attachment.value + + +@dataclass +class ContractReferencedCredential(ContractElement): + """Credential reference field, optionally restricted to one credential type.""" + + key: str = field(default=ContractFieldKey.CredentialReference.value, init=False) + label: str = "Select a credential reference" + mandatory: bool = False + credential_reference_type: Optional[CredentialType] = None + multiple: bool = True + + @property + def get_type(self) -> str: + return ContractFieldType.CredentialReference.value + + +@dataclass +class ContractExpectations(ContractCardinalityElement): + """Expectation selection field with predefined options. + + ``availableExpectations`` is the full set the user may pick from; + ``predefinedExpectations`` is the subset pre-filled by default. When + ``predefinedExpectations`` is not provided explicitly it is derived from + the available expectations flagged ``expectation_is_predefined=True``. + """ + + cardinality: str = ContractCardinality.Multiple + # Full set the user may choose from in the "add expectation" picker. + availableExpectations: list[Expectation] = field(default_factory=list) + # Subset pre-filled by default. ``None`` means "not provided": the list is + # then derived from the available expectations flagged predefined. + predefinedExpectations: list[Expectation] | None = None + + def __post_init__(self) -> None: + super().__post_init__() + if self.predefinedExpectations is None: + self.predefinedExpectations = [ + expectation + for expectation in self.availableExpectations + if expectation.expectation_is_predefined + ] + + @property + def get_type(self) -> str: + return ContractFieldType.Expectation.value + + +@dataclass +class ContractSelect(ContractCardinalityElement): + """Dropdown select field with predefined choices.""" + + choices: dict[str, str] | None = None + + @property + def get_type(self) -> str: + return ContractFieldType.Select.value + + +@dataclass +class ContractAsset(ContractCardinalityElement): + """Asset selection field (key auto-set to 'assets').""" + + key: str = field(default=ContractFieldKey.Asset.value, init=False) + + @property + def get_type(self) -> str: + return ContractFieldType.Asset.value + + +@dataclass +class ContractAssetGroup(ContractCardinalityElement): + """Asset group selection field (key auto-set to 'asset_groups').""" + + key: str = field(default=ContractFieldKey.AssetGroup.value, init=False) + + @property + def get_type(self) -> str: + return ContractFieldType.AssetGroup.value + + +@dataclass +class ContractAiTarget(ContractCardinalityElement): + """Selector for a pre-configured AiTarget asset (LLM endpoint / AI agent under test). + + Renders in the UI as an entity autocomplete (backed by the platform ``/ai_targets`` + options endpoint), analogous to the asset / asset group pickers. The selected value is + the AI target's ``asset_id`` (AI targets are AI_TARGET-category assets), resolved by the + executing injector at run time. Unlike ``ContractAsset`` the key is free-form so a + contract can name the field as it wishes. + """ + + @property + def get_type(self) -> str: + return ContractFieldType.AiTarget.value + + +@dataclass +class ContractPayload(ContractCardinalityElement): + """Payload selection field.""" + + @property + def get_type(self) -> str: + return ContractFieldType.Payload.value diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_utils.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_utils.py new file mode 100644 index 00000000..a795617e --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_utils.py @@ -0,0 +1,57 @@ +"""Contract utility types: cardinality and variable definitions. + +Provides foundational types used by the contract building system +for defining field cardinality and template variables. +""" + +from dataclasses import dataclass, field +from enum import Enum + +__all__ = [ + "ContractCardinality", + "ContractVariable", + "VariableType", +] + + +class ContractCardinality(str, Enum): + """Field cardinality in a contract definition. + + Attributes: + One: Single value expected. + Multiple: Multiple values (list) expected. + """ + + One: str = "1" + Multiple: str = "n" + + +class VariableType(str, Enum): + """Variable type classification for contract template variables. + + Attributes: + String: Simple string value. + Object: Nested object with children. + """ + + String: str = "String" + Object: str = "Object" + + +@dataclass +class ContractVariable: + """A template variable available for injection into contract fields. + + Args: + key: Variable identifier (dot-notation for nested, e.g. "user.email"). + label: Human-readable description of the variable. + type: Variable type (String or Object). + cardinality: Whether this variable holds one or many values. + children: Nested child variables (for Object type). + """ + + key: str + label: str + type: VariableType + cardinality: ContractCardinality + children: list["ContractVariable"] = field(default_factory=list) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credential_types.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credential_types.py new file mode 100644 index 00000000..282b67cc --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credential_types.py @@ -0,0 +1,18 @@ +"""Credential reference types shared with the OpenAEV platform. + +This enum is the Python mirror of the OpenAEV backend credential-type enum +used by credential secret references. Values must stay label-for-label in sync +with the platform so contracts can declare credential references without any +translation layer. +""" + +from enum import Enum + +__all__ = ["CredentialType"] + + +class CredentialType(str, Enum): + IDENTITY = "IDENTITY" + CLOUD_AWS = "CLOUD_AWS" + CLOUD_AZURE = "CLOUD_AZURE" + CLOUD_GCP = "CLOUD_GCP" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py new file mode 100644 index 00000000..f342b350 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py @@ -0,0 +1,43 @@ +"""Helpers centralizing how injectors declare credential-reference fields. + +Injectors can call ``build_single_referenced_credential_element`` instead of +re-implementing provider-to-credential-type mapping in each project. This keeps +contract generation consistent across the Python ecosystem and aligns the +serialized contract payload with the OpenAEV platform's expected values. +""" + +from typing import Optional + +from xtm_oaev_sdk._core.contracts.contract_config import ContractReferencedCredential +from xtm_oaev_sdk._core.contracts.credential_types import CredentialType + +__all__ = ["build_single_referenced_credential_element"] + +_PROVIDER_TO_CREDENTIAL_TYPE = { + "aws": CredentialType.CLOUD_AWS, + "eks": CredentialType.CLOUD_AWS, + "azure": CredentialType.CLOUD_AZURE, + "gcp": CredentialType.CLOUD_GCP, +} + + +def _resolve_credential_type(provider_name: str) -> Optional[CredentialType]: + normalized_provider = provider_name.casefold() + return _PROVIDER_TO_CREDENTIAL_TYPE.get(normalized_provider) + + +def build_single_referenced_credential_element( + provider_name: str, +) -> ContractReferencedCredential: + """Build a credential-reference field for a provider-specific contract + with ``multiple`` value at ``False``. + + This is the centralized entry point injectors should use when they need the + OpenAEV inject form to ask for one referenced credential. The helper keeps + provider-to-``CredentialType`` mapping consistent across injector projects. + """ + + return ContractReferencedCredential( + credential_reference_type=_resolve_credential_type(provider_name), + multiple=False, + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variable_helper.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variable_helper.py new file mode 100644 index 00000000..d1be8c4a --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variable_helper.py @@ -0,0 +1,149 @@ +"""Variable helper: pre-built contract template variables. + +Provides factory methods for standard template variables (user, exercise, +team, URIs) that are injected into contract fields during simulation. +""" + +from xtm_oaev_sdk._core.contracts.contract_utils import ( + ContractCardinality, + ContractVariable, + VariableType, +) + +__all__ = ["VariableHelper"] + +USER = "user" +EXERCISE = "exercise" +TEAMS = "teams" +COMCHECK = "comcheck" +PLAYER_URI = "player_uri" +CHALLENGES_URI = "challenges_uri" +SCOREBOARD_URI = "scoreboard_uri" +LESSONS_URI = "lessons_uri" + + +class VariableHelper: + """Factory for standard contract template variables. + + Provides pre-built variable definitions for common injection + targets (users, exercises, teams, platform URIs). + + Example: + >>> variables = [ + ... VariableHelper.user_variable(), + ... VariableHelper.exercise_variable(), + ... VariableHelper.team_variable(), + ... ] + VariableHelper.uri_variables() + """ + + @staticmethod + def user_variable() -> ContractVariable: + """Create the user template variable with standard children.""" + return ContractVariable( + key=USER, + label="User that will receive the injection", + type=VariableType.String, + cardinality=ContractCardinality.One, + children=[ + ContractVariable( + key=USER + ".id", + label="Id of the user in the platform", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=USER + ".email", + label="Email of the user", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=USER + ".firstname", + label="Firstname of the user", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=USER + ".lastname", + label="Lastname of the user", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=USER + ".lang", + label="Lang of the user", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ], + ) + + @staticmethod + def exercise_variable() -> ContractVariable: + """Create the exercise template variable with standard children.""" + return ContractVariable( + key=EXERCISE, + label="Exercise of the current injection", + type=VariableType.Object, + cardinality=ContractCardinality.One, + children=[ + ContractVariable( + key=EXERCISE + ".id", + label="Id of the exercise in the platform", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=EXERCISE + ".name", + label="Name of the exercise", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=EXERCISE + ".description", + label="Description of the exercise", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ], + ) + + @staticmethod + def team_variable() -> ContractVariable: + """Create the teams template variable.""" + return ContractVariable( + key=TEAMS, + label="List of team name for the injection", + type=VariableType.String, + cardinality=ContractCardinality.Multiple, + ) + + @staticmethod + def uri_variables() -> list[ContractVariable]: + """Create platform URI template variables.""" + return [ + ContractVariable( + key=PLAYER_URI, + label="Player interface platform link", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=CHALLENGES_URI, + label="Challenges interface platform link", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=SCOREBOARD_URI, + label="Scoreboard interface platform link", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ContractVariable( + key=LESSONS_URI, + label="Lessons learned interface platform link", + type=VariableType.String, + cardinality=ContractCardinality.One, + ), + ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/__init__.py new file mode 100644 index 00000000..83be2c06 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/__init__.py @@ -0,0 +1,6 @@ +"""Business contracts layer: the OpenAEV scenario data models. + +The live member is :mod:`xtm_oaev_sdk.contracts.contracts` — the domain +models describing OpenAEV scenarios / contracts. It is a domain concept, +distinct from the architectural :mod:`xtm_oaev_sdk.seam` boundary. +""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/contracts/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/contracts/__init__.py new file mode 100644 index 00000000..05bc962e --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/contracts/__init__.py @@ -0,0 +1,73 @@ +"""OpenAEV contracts/scenarios data models.""" + +from xtm_oaev_sdk._core.contracts import ( + Contract, + ContractAiTarget, + ContractAsset, + ContractAssetGroup, + ContractAttachment, + ContractBuilderProtocol, + ContractCardinality, + ContractCardinalityElement, + ContractCheckbox, + ContractConfig, + ContractElement, + ContractExpectations, + ContractExpectationType, + ContractFieldKey, + ContractFieldType, + ContractOutputElement, + ContractOutputType, + ContractPayload, + ContractReferencedCredential, + ContractSelect, + ContractTeam, + ContractText, + ContractTextArea, + ContractTuple, + ContractVariable, + CredentialType, + Domain, + Expectation, + LinkedFieldModel, + PrimitiveType, + SecurityPlatformType, + SupportedLanguage, + VariableType, +) + +__all__ = [ + "Contract", + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractBuilderProtocol", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractElement", + "ContractExpectationType", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "CredentialType", + "Domain", + "Expectation", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableType", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index 8a23e533..c3bbc52b 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -2,6 +2,42 @@ __version__ = "0.1.0" +# --- Contracts (scenarios) --- +from xtm_oaev_sdk.contracts.contracts import ( + Contract, + ContractAiTarget, + ContractAsset, + ContractAssetGroup, + ContractAttachment, + ContractCardinality, + ContractCardinalityElement, + ContractCheckbox, + ContractConfig, + ContractElement, + ContractExpectations, + ContractExpectationType, + ContractFieldKey, + ContractFieldType, + ContractOutputElement, + ContractOutputType, + ContractPayload, + ContractReferencedCredential, + ContractSelect, + ContractTeam, + ContractText, + ContractTextArea, + ContractTuple, + ContractVariable, + CredentialType, + Domain, + Expectation, + LinkedFieldModel, + PrimitiveType, + SecurityPlatformType, + SupportedLanguage, + VariableType, +) + # --- Common (shared) --- from xtm_oaev_sdk.seam.common import ( AppLogger, @@ -58,6 +94,13 @@ is_falsy, is_truthy, ) +from xtm_oaev_sdk.seam.contracts import ( + ContractBuilder, + ContractBuilderProtocol, + VariableHelper, + build_single_referenced_credential_element, + prepare_contracts, +) # --- Seam / compatibility --- from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr @@ -155,6 +198,44 @@ "TimedeltaInSeconds", "is_falsy", "is_truthy", + # Contracts (scenarios) + "Contract", + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractBuilder", + "ContractBuilderProtocol", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractElement", + "ContractExpectationType", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "CredentialType", + "Domain", + "Expectation", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableHelper", + "VariableType", + "build_single_referenced_credential_element", + "prepare_contracts", # Signatures "CloudInjectorConfig", "ExecutionDetails", diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py index aad7835c..71e283ae 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -7,6 +7,8 @@ platform-facing vocabulary both sides import. - **``configuration``** — the configuration capability's I/O models and conversion primitives. +- **``contracts``** — the scenarios capability's behaviour layer (builder, + variable helper, preparation entry point) and its extension protocol. - **``signatures``** — the signatures capability's inbound seam (``seam/signatures/seam.py``), versioned ``SEAM_VERSION``. - **``port_io``** — the port I/O wire schema crossing the SDK↔platform @@ -25,6 +27,10 @@ is the **sole door** between ``public/`` and ``_core/``: every public import of a ``_core`` symbol crosses it. The seam never imports the public layer, nor does ``_core``. + +Distinct from the *business* :mod:`xtm_oaev_sdk.contracts` layer, which holds +the OpenAEV scenario data models — a domain concept, unrelated to the +boundary. """ from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py new file mode 100644 index 00000000..602cc640 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py @@ -0,0 +1,27 @@ +"""The scenarios (contracts) capability's seam surface. + +Re-presents the capability's behaviour layer — the concrete builder, the +variable helper, and the preparation entry point — together with the +capability's extension protocol, so that the public firewall never imports +``_core`` directly. + +The *data models* of the scenarios capability are a business concern and +live in :mod:`xtm_oaev_sdk.contracts`; this sub-package carries the +behaviour and the protocol only. +""" + +from xtm_oaev_sdk._core.contracts import ( + ContractBuilder, + ContractBuilderProtocol, + VariableHelper, + build_single_referenced_credential_element, + prepare_contracts, +) + +__all__ = [ + "ContractBuilder", + "ContractBuilderProtocol", + "VariableHelper", + "build_single_referenced_credential_element", + "prepare_contracts", +] diff --git a/pyoaev/__init__.py b/pyoaev/__init__.py index 3fddaa5c..62415734 100644 --- a/pyoaev/__init__.py +++ b/pyoaev/__init__.py @@ -4,7 +4,13 @@ import xtm_oaev_sdk as _sdk from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr -from pyoaev import asset_types, configuration, exceptions, signatures # noqa: F401 +from pyoaev import ( # noqa: F401 + asset_types, + configuration, + contracts, + exceptions, + signatures, +) from pyoaev._version import ( # noqa: F401 __author__, __copyright__, @@ -13,9 +19,8 @@ __title__, ) from pyoaev.client import OpenAEV # noqa: F401 -from pyoaev.contracts import * # noqa: F401,F403,F405 -__all__ = [ # noqa: F405 (moved names are served by the PEP 562 __getattr__ below) +__all__ = [ # moved names are served by the PEP 562 __getattr__ below "__author__", "__copyright__", "__email__", @@ -42,6 +47,7 @@ for name in ( *asset_types.__all__, *configuration.__all__, + *contracts.__all__, *exceptions.__all__, ) }, diff --git a/pyoaev/contracts/__init__.py b/pyoaev/contracts/__init__.py index a9b7be78..730c4484 100644 --- a/pyoaev/contracts/__init__.py +++ b/pyoaev/contracts/__init__.py @@ -1,5 +1,25 @@ -from .contract_builder import ContractBuilder # noqa: F401,F403 +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -__all__ = [ +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) "ContractBuilder", ] + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/contracts/contract_builder.py b/pyoaev/contracts/contract_builder.py index d7b583ea..0e1b380c 100644 --- a/pyoaev/contracts/contract_builder.py +++ b/pyoaev/contracts/contract_builder.py @@ -1,44 +1,27 @@ -from typing import List - -from pyoaev.contracts.contract_config import ContractElement, ContractOutputElement - - -class ContractBuilder: - fields: List[ContractElement] - outputs: List[ContractOutputElement] - - def __init__(self): - self.fields = [] - self.outputs = [] - - def add_fields(self, fields: List[ContractElement]): - self.fields = self.fields + fields - return self - - def add_outputs(self, outputs: List[ContractOutputElement]): - self.outputs = self.outputs + outputs - return self - - def mandatory(self, element: ContractElement): - element.mandatory = True - self.fields.append(element) - return self - - def optional(self, element: ContractElement): - element.mandatory = False - self.fields.append(element) - return self - - def mandatory_group(self, elements: List[ContractElement]): - keys: List[str] = list(map(lambda iterable: iterable.key, elements)) - for element in elements: - element.mandatory = True - element.mandatoryGroups = keys - self.fields.append(element) - return self - - def build_fields(self) -> List[ContractElement]: - return self.fields - - def build_outputs(self) -> List[ContractOutputElement]: - return self.outputs +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ContractBuilder", + "ContractElement", + "ContractOutputElement", +] + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/contracts/contract_config.py b/pyoaev/contracts/contract_config.py index 8217194d..18623977 100644 --- a/pyoaev/contracts/contract_config.py +++ b/pyoaev/contracts/contract_config.py @@ -1,463 +1,61 @@ -import json -from abc import ABC, abstractmethod -from dataclasses import dataclass, field -from enum import Enum -from typing import Dict, List, Optional - -from xtm_oaev_sdk import EnhancedJSONEncoder - -from pyoaev.contracts.contract_utils import ContractCardinality, ContractVariable -from pyoaev.contracts.variable_helper import VariableHelper -from pyoaev.credential.types import CredentialType - - -class SupportedLanguage(str, Enum): - fr: str = "fr" - en: str = "en" - - -class ContractFieldType(str, Enum): - Text: str = "text" - Number: str = "number" - Tuple: str = "tuple" - Checkbox: str = "checkbox" - Textarea: str = "textarea" - Select: str = "select" - Article: str = "article" - Challenge: str = "challenge" - DependencySelect: str = "dependency-select" - Attachment: str = "attachment" - Team: str = "team" - Expectation: str = "expectation" - Asset: str = "asset" - AssetGroup: str = "asset-group" - AiTarget: str = "ai-target" - Payload: str = "payload" - CredentialReference: str = "credential-reference" - - -class ContractFieldKey(str, Enum): - Asset: str = "assets" - AssetGroup: str = "asset_groups" - CredentialReference: str = "credential_reference" - - -class ContractOutputType(str, Enum): - Text: str = "text" - ActionOutput: str = "action_output" - Number: str = "number" - Port: str = "port" - PortsScan: str = "portscan" - IPv4: str = "ipv4" - IPv6: str = "ipv6" - CVE: str = "cve" - Asset: str = "asset" - Credentials: str = "credentials" - Username: str = "username" - Share: str = "share" - File: str = "file" - AdminUsername: str = "admin_username" - Group: str = "group" - Computer: str = "computer" - PasswordPolicy: str = "password_policy" - Delegation: str = "delegation" - Sid: str = "sid" - Vulnerability: str = "vulnerability" - AccountWithPasswordNotRequired: str = "account_with_password_not_required" - ExpectationSignature: str = "expectation_signature" - AsreproastableAccount: str = "asreproastable_account" - KerberoastableAccount: str = "kerberoastable_account" - - -class PrimitiveType(str, Enum): - """The semantic type of a contract argument's value, independent of how the - field renders in the UI (see ``ContractFieldType``) and independent of - chaining specifically — an argument can carry a ``PrimitiveType`` and be - filled by hand, exactly as it can be auto-linked from a prior step's - matching output. Mirrors ``io.openaev.database.model.PrimitiveType`` - (the openaev platform's own enum) label-for-label; every value here must - stay in sync with it. - """ - - AccountWithPasswordNotRequired: str = "account_with_password_not_required" - ActionOutput: str = "action_output" - AdminUsername: str = "admin_username" - AsreproastableAccount: str = "asreproastable_account" - AssetGroupId: str = "asset_group_id" - AssetId: str = "asset_id" - ComputerName: str = "computer_name" - CVE: str = "cve" - DelegationAccount: str = "delegation_account" - Document: str = "document" - Domain: str = "domain" - FileName: str = "file_name" - FilePath: str = "file_path" - GroupName: str = "group_name" - Hash: str = "hash" - Host: str = "host" - IPv4: str = "ipv4" - IPv6: str = "ipv6" - IpSubnet: str = "ip_subnet" - KerberoastableAccount: str = "kerberoastable_account" - Key: str = "key" - Number: str = "number" - Password: str = "password" - Permissions: str = "permissions" - Port: str = "port" - Service: str = "service" - Severity: str = "severity" - ShareName: str = "share_name" - SID: str = "sid" - TargetedAsset: str = "targeted-asset" - Text: str = "text" - Username: str = "username" - Value: str = "value" - VulnerabilityName: str = "vulnerability_name" - VulnerabilityStatus: str = "vulnerability_status" - - -class ExpectationType(str, Enum): - text: str = "TEXT" - document: str = "DOCUMENT" - article: str = "ARTICLE" - challenge: str = "CHALLENGE" - manual: str = "MANUAL" - prevention: str = "PREVENTION" - detection: str = "DETECTION" - vulnerability: str = "VULNERABILITY" - - -class SecurityPlatformType(str, Enum): - """Categories of security platform expected to fulfil a technical expectation. - - When an expectation declares one or more of these, the platform focuses the - technical (DETECTION / PREVENTION / VULNERABILITY) result on collectors of - those types only (instead of every connected security platform). An empty - list means "any security platform". - """ - - EDR: str = "EDR" - XDR: str = "XDR" - SIEM: str = "SIEM" - SOAR: str = "SOAR" - NDR: str = "NDR" - ISPM: str = "ISPM" - LLM_FIREWALL: str = "LLM_FIREWALL" - AI_GATEWAY: str = "AI_GATEWAY" - - -@dataclass -class Expectation: - expectation_type: ExpectationType - expectation_name: str - expectation_description: str - expectation_score: int - expectation_expectation_group: bool - expectation_is_predefined: bool = False - expectation_is_multi_selectable: bool = False - # Security platform types expected to fulfil this expectation. Empty = any - # platform (unchanged behaviour). Typically set for technical expectations - # (DETECTION / PREVENTION / VULNERABILITY), left empty for MANUAL ones. - expectation_expected_security_platform_types: List[SecurityPlatformType] = field( - default_factory=list - ) - - -@dataclass -class LinkedFieldModel: - key: str - type: ContractFieldType - - -@dataclass -class ContractElement(ABC): - key: str - label: str - type: str = field(default="", init=False) - mandatoryGroups: List[str] = field(default_factory=list) - mandatoryConditionFields: List[str] = field(default_factory=list) - mandatoryConditionValues: Dict[str, any] = field(default_factory=list) - visibleConditionFields: List[str] = field(default_factory=list) - visibleConditionValues: Dict[str, any] = field(default_factory=list) - linkedFields: List[str] = field(default_factory=list) - mandatory: bool = False - readOnly: bool = False - # The argument's chaining/semantic type (``PrimitiveType``), e.g. "username" - # or "host" — matches the platform's own ``argumentType`` field - # (``io.openaev.database.model.ContractElement``) label-for-label so a - # contract pushed from here needs no translation on the other end. Left - # unset (``None``) for fields with no established type: the platform - # normalizes a missing/null/empty value to ``PrimitiveType.Text`` on its - # own, so omitting this is always safe and never a breaking change for - # existing contracts. - argumentType: Optional[PrimitiveType] = None - - @property - @abstractmethod - def get_type(self) -> str: - pass - - def __post_init__(self): - self.type = self.get_type - - -@dataclass -class ContractCardinalityElement(ContractElement, ABC): - cardinality: str = ContractCardinality.One - defaultValue: List[str] = field(default_factory=list) - - -@dataclass -class ContractOutputElement(ABC): - type: str - field: str - labels: List[str] - isFindingCompatible: bool - isMultiple: bool - - -@dataclass -class ContractConfig: - type: str - expose: bool - label: dict[SupportedLanguage, str] - color_dark: str - color_light: str - - -@dataclass -class Domain: - domain_id: str - domain_name: str - domain_color: str - domain_created_at: str - domain_updated_at: str - - -@dataclass -class Contract: - contract_id: str - label: dict[SupportedLanguage, str] - fields: List[ContractElement] - outputs: List[ContractOutputElement] - config: ContractConfig - manual: bool - variables: List[ContractVariable] = field( - default_factory=lambda: ( - [ - VariableHelper.user_variable(), - VariableHelper.exercise_variable(), - VariableHelper.team_variable(), - ] - + VariableHelper.uri_variables() - ) - ) - contract_attack_patterns_external_ids: List[str] = field(default_factory=list) - contract_vulnerability_external_ids: List[str] = field(default_factory=list) - is_atomic_testing: bool = True - platforms: List[str] = field(default_factory=list) - external_id: str = None - domains: List[Domain] = None - - def add_attack_pattern(self, var: str): - self.contract_attack_patterns_external_ids.append(var) - - def add_vulnerability(self, var: str): - self.contract_vulnerability_external_ids.append(var) - - def add_variable(self, var: ContractVariable): - self.variables.append(var) - - def to_contract_add_input(self, source_id: str): - return { - "contract_id": self.contract_id, - "external_contract_id": self.external_id, - "injector_id": source_id, - "contract_manual": self.manual, - "contract_labels": self.label, - "contract_attack_patterns_external_ids": self.contract_attack_patterns_external_ids, - "contract_vulnerability_external_ids": self.contract_vulnerability_external_ids, - "contract_content": json.dumps(self, cls=EnhancedJSONEncoder), - "is_atomic_testing": self.is_atomic_testing, - "contract_platforms": self.platforms, - "contract_domains": self.domains, - } - - def to_contract_update_input(self): - return { - "contract_manual": self.manual, - "contract_labels": self.label, - "contract_attack_patterns_external_ids": self.contract_attack_patterns_external_ids, - "contract_vulnerability_external_ids": self.contract_vulnerability_external_ids, - "contract_content": json.dumps(self, cls=EnhancedJSONEncoder), - "is_atomic_testing": self.is_atomic_testing, - "contract_platforms": self.platforms, - "contract_domains": self.domains, - } - - -@dataclass -class ContractTeam(ContractCardinalityElement): - @property - def get_type(self) -> str: - return ContractFieldType.Team.value - - -@dataclass -class ContractText(ContractCardinalityElement): - defaultValue: str = "" - - @property - def get_type(self) -> str: - return ContractFieldType.Text.value - - -def prepare_contracts(contracts): - return list( - map( - lambda c: { - "contract_id": c.contract_id, - "contract_labels": c.label, - "contract_attack_patterns_external_ids": c.contract_attack_patterns_external_ids, - "contract_content": json.dumps(c, cls=EnhancedJSONEncoder), - "contract_platforms": c.platforms, - "contract_domains": c.domains, - }, - contracts, - ) - ) - - -@dataclass -class ContractTuple(ContractCardinalityElement): - def __post_init__(self): - super().__post_init__() - self.cardinality = ContractCardinality.Multiple - - attachmentKey: str = None - contractAttachment: bool = attachmentKey is not None - tupleFilePrefix: str = "file :: " - - @property - def get_type(self) -> str: - return ContractFieldType.Tuple.value - - -@dataclass -class ContractTextArea(ContractCardinalityElement): - defaultValue: str = "" - richText: bool = False - - @property - def get_type(self) -> str: - return ContractFieldType.Textarea.value - - -@dataclass -class ContractCheckbox(ContractElement): - defaultValue: bool = False - - @property - def get_type(self) -> str: - return ContractFieldType.Checkbox.value - - -@dataclass -class ContractAttachment(ContractCardinalityElement): - @property - def get_type(self) -> str: - return ContractFieldType.Attachment.value - - -@dataclass -class ContractReferencedCredential(ContractElement): - key: str = field(default=ContractFieldKey.CredentialReference.value, init=False) - label: str = "Select a credential reference" - mandatory: bool = False - credential_reference_type: Optional[CredentialType] = None - multiple: bool = True - - @property - def get_type(self) -> str: - return ContractFieldType.CredentialReference.value - - -@dataclass -class ContractExpectations(ContractCardinalityElement): - cardinality: str = ContractCardinality.Multiple - # Full set the user may choose from in the "add expectation" picker. - availableExpectations: List[Expectation] = field(default_factory=list) - # Subset pre-filled by default when the inject / atomic testing is created. - # The OpenAEV platform reads this field-level array (not the per-expectation - # flag) to pre-populate expectations, so it must be emitted in the contract. - # None means "not provided": the list is then derived from the expectations - # flagged expectation_is_predefined=True. Pass an explicit list (possibly - # empty) to override the derivation. - predefinedExpectations: Optional[List[Expectation]] = None - - def __post_init__(self): - super().__post_init__() - # When the caller did not pass an explicit predefined list, derive it from - # the expectations flagged predefined in the available set. This keeps the - # ergonomic single-list API (flag each Expectation with - # expectation_is_predefined=True) working end-to-end on the platform. - if self.predefinedExpectations is None: - self.predefinedExpectations = [ - expectation - for expectation in self.availableExpectations - if expectation.expectation_is_predefined - ] - - @property - def get_type(self) -> str: - return ContractFieldType.Expectation.value - - -@dataclass -class ContractSelect(ContractCardinalityElement): - choices: dict[str, str] = None - - @property - def get_type(self) -> str: - return ContractFieldType.Select.value - - -@dataclass -class ContractAsset(ContractCardinalityElement): - key: str = field(default=ContractFieldKey.Asset.value, init=False) - - @property - def get_type(self) -> str: - return ContractFieldType.Asset.value - - -@dataclass -class ContractAssetGroup(ContractCardinalityElement): - key: str = field(default=ContractFieldKey.AssetGroup.value, init=False) - - @property - def get_type(self) -> str: - return ContractFieldType.AssetGroup.value - - -@dataclass -class ContractAiTarget(ContractCardinalityElement): - """Selector for a pre-configured AiTarget asset (LLM endpoint / AI agent under test). - - Renders in the UI as an entity autocomplete (backed by the platform ``/ai_targets`` - options endpoint), analogous to the asset / asset group pickers. The selected value is - the AI target's ``asset_id`` (AI targets are AI_TARGET-category assets, see - ``AiTarget._id_attr``), resolved by the executing injector at run time. Unlike - ``ContractAsset`` the key is free-form so a contract can name the field as it wishes. - """ - - @property - def get_type(self) -> str: - return ContractFieldType.AiTarget.value - - -@dataclass -class ContractPayload(ContractCardinalityElement): - @property - def get_type(self) -> str: - return ContractFieldType.Payload.value +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +``ExpectationType`` was renamed ``ContractExpectationType``. +""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "SupportedLanguage", + "ContractFieldType", + "ContractFieldKey", + "ContractOutputType", + "PrimitiveType", + "ExpectationType", + "SecurityPlatformType", + "Expectation", + "LinkedFieldModel", + "ContractElement", + "ContractCardinalityElement", + "ContractOutputElement", + "ContractConfig", + "Domain", + "Contract", + "ContractTeam", + "ContractText", + "prepare_contracts", + "ContractTuple", + "ContractTextArea", + "ContractCheckbox", + "ContractAttachment", + "ContractExpectations", + "ContractSelect", + "ContractAsset", + "ContractAssetGroup", + "ContractAiTarget", + "ContractPayload", + "ContractReferencedCredential", + "CredentialType", + "EnhancedJSONEncoder", + "ContractCardinality", + "ContractVariable", + "VariableHelper", +] + +_RENAMED = {"ExpectationType": "ContractExpectationType"} + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {_RENAMED.get(name, name)}' instead.", + getattr(_sdk, _RENAMED.get(name, name)), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/contracts/contract_utils.py b/pyoaev/contracts/contract_utils.py index 903f6c4c..d31c0220 100644 --- a/pyoaev/contracts/contract_utils.py +++ b/pyoaev/contracts/contract_utils.py @@ -1,22 +1,27 @@ -from dataclasses import dataclass, field -from enum import Enum -from typing import List +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -class ContractCardinality(str, Enum): - One: str = "1" - Multiple: str = "n" +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ContractCardinality", + "VariableType", + "ContractVariable", +] -class VariableType(str, Enum): - String: str = "String" - Object: str = "Object" - - -@dataclass -class ContractVariable: - key: str - label: str - type: VariableType - cardinality: ContractCardinality - children: List["ContractVariable"] = field(default_factory=list) +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/contracts/variable_helper.py b/pyoaev/contracts/variable_helper.py index 7649d84b..4018a182 100644 --- a/pyoaev/contracts/variable_helper.py +++ b/pyoaev/contracts/variable_helper.py @@ -1,124 +1,49 @@ -from pyoaev.contracts.contract_utils import ( - ContractCardinality, - ContractVariable, - VariableType, -) - -USER = "user" -EXERCISE = "exercise" -TEAMS = "teams" -COMCHECK = "comcheck" -PLAYER_URI = "player_uri" -CHALLENGES_URI = "challenges_uri" -SCOREBOARD_URI = "scoreboard_uri" -LESSONS_URI = "lessons_uri" +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +The module constants (``USER``, ``EXERCISE``, ...) are not part of the SDK +public surface; they keep their values here and warn. +""" -class VariableHelper: - @staticmethod - def user_variable(): - return ContractVariable( - key=USER, - label="User that will receive the injection", - type=VariableType.String, - cardinality=ContractCardinality.One, - children=[ - ContractVariable( - key=USER + ".id", - label="Id of the user in the platform", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=USER + ".email", - label="Email of the user", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=USER + ".firstname", - label="Firstname of the user", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=USER + ".lastname", - label="Lastname of the user", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=USER + ".lang", - label="Lang of the user", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ], - ) +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr - @staticmethod - def exercise_variable(): - return ContractVariable( - key=EXERCISE, - label="Exercise of the current injection", - type=VariableType.Object, - cardinality=ContractCardinality.One, - children=[ - ContractVariable( - key=EXERCISE + ".id", - label="Id of the exercise in the platform", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=EXERCISE + ".name", - label="Name of the exercise", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=EXERCISE + ".description", - label="Description of the exercise", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ], - ) +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "VariableHelper", + "ContractCardinality", + "ContractVariable", + "VariableType", +] - @staticmethod - def team_variable(): - return ContractVariable( - key=TEAMS, - label="List of team name for the injection", - type=VariableType.String, - cardinality=ContractCardinality.Multiple, - ) +_CONSTANTS = { + "USER": "user", + "EXERCISE": "exercise", + "TEAMS": "teams", + "COMCHECK": "comcheck", + "PLAYER_URI": "player_uri", + "CHALLENGES_URI": "challenges_uri", + "SCOREBOARD_URI": "scoreboard_uri", + "LESSONS_URI": "lessons_uri", +} - @staticmethod - def uri_variables(): - return [ - ContractVariable( - key=PLAYER_URI, - label="Player interface platform link", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=CHALLENGES_URI, - label="Challenges interface platform link", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=SCOREBOARD_URI, - label="Scoreboard interface platform link", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ContractVariable( - key=LESSONS_URI, - label="Lessons learned interface platform link", - type=VariableType.String, - cardinality=ContractCardinality.One, - ), - ] +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + **{ + name: ( + f"'{name}' is deprecated and has no public replacement in " + "xtm_oaev_sdk; use the VariableHelper built-in variables instead.", + value, + ) + for name, value in _CONSTANTS.items() + }, + }, + module_name=__name__, +) diff --git a/pyoaev/credential/__init__.py b/pyoaev/credential/__init__.py index 5086830e..abcc1536 100644 --- a/pyoaev/credential/__init__.py +++ b/pyoaev/credential/__init__.py @@ -1,15 +1,26 @@ -from .types import CredentialType +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +""" -def build_single_referenced_credential_element(provider_name: str): - from .utils import ( - build_single_referenced_credential_element as _build_single_referenced_credential_element, - ) +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr - return _build_single_referenced_credential_element(provider_name) - - -__all__ = [ +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) "CredentialType", "build_single_referenced_credential_element", ] + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/credential/types.py b/pyoaev/credential/types.py index f2600a4d..a018bb10 100644 --- a/pyoaev/credential/types.py +++ b/pyoaev/credential/types.py @@ -1,16 +1,25 @@ -"""Credential reference types shared with the OpenAEV platform. +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -This enum is the Python mirror of the OpenAEV backend credential-type enum -used by credential secret references. Values must stay label-for-label in sync -with the platform so contracts can declare credential references without any -translation layer. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. """ -from enum import Enum +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "CredentialType", +] -class CredentialType(str, Enum): - IDENTITY = "IDENTITY" - CLOUD_AWS = "CLOUD_AWS" - CLOUD_AZURE = "CLOUD_AZURE" - CLOUD_GCP = "CLOUD_GCP" +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) diff --git a/pyoaev/credential/utils.py b/pyoaev/credential/utils.py index 3975e71f..bf7a91d9 100644 --- a/pyoaev/credential/utils.py +++ b/pyoaev/credential/utils.py @@ -1,41 +1,27 @@ -"""Helpers centralizing how injectors declare credential-reference fields. +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -Injectors can call ``build_single_referenced_credential_element`` instead of -re-implementing provider-to-credential-type mapping in each project. This keeps -contract generation consistent across the Python ecosystem and aligns the -serialized contract payload with the OpenAEV platform's expected values. +Backward-compatibility shim: every name below resolves to the *identical* +xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. """ -from typing import Optional - -from pyoaev.contracts.contract_config import ContractReferencedCredential -from pyoaev.credential.types import CredentialType - -_PROVIDER_TO_CREDENTIAL_TYPE = { - "aws": CredentialType.CLOUD_AWS, - "eks": CredentialType.CLOUD_AWS, - "azure": CredentialType.CLOUD_AZURE, - "gcp": CredentialType.CLOUD_GCP, -} - - -def _resolve_credential_type(provider_name: str) -> Optional[CredentialType]: - normalized_provider = provider_name.casefold() - return _PROVIDER_TO_CREDENTIAL_TYPE.get(normalized_provider) - - -def build_single_referenced_credential_element( - provider_name: str, -) -> ContractReferencedCredential: - """Build a credential-reference field for a provider-specific contract - with ``multiple`` value at ``False``. - - This is the centralized entry point injectors should use when they need the - OpenAEV inject form to ask for one referenced credential. The helper keeps - provider-to-``CredentialType`` mapping consistent across injector projects. - """ - - return ContractReferencedCredential( - credential_reference_type=_resolve_credential_type(provider_name), - multiple=False, - ) +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "ContractReferencedCredential", + "CredentialType", + "build_single_referenced_credential_element", +] + +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in __all__ + }, + }, + module_name=__name__, +) From ca1b4a0d6ecf88910968902d2db158a5ca66d25b Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 13:50:44 +0200 Subject: [PATCH 05/22] :sparkles: feat(sdk): add the shared daemon runtime to xtm-oaev-sdk --- libs/xtm-oaev-sdk/test/daemons/__init__.py | 0 .../test/daemons/shim/__init__.py | 0 .../test/daemons/shim}/test_base_daemon.py | 0 .../test/daemons/shim/test_ping_alive.py | 57 +++++++ .../test/daemons/shim/test_shim_contract.py | 99 +++++++++++ .../test/daemons/test_base_daemon.py | 160 ++++++++++++++++++ .../test/daemons/test_ping_alive.py | 59 +++++++ .../test/daemons/test_protocols.py | 109 ++++++++++++ libs/xtm-oaev-sdk/test/test_boundaries.py | 1 - libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 4 +- .../xtm_oaev_sdk/_core/client_protocol.py | 81 +++++++++ .../xtm_oaev_sdk/_core/daemon_protocol.py | 39 +++++ .../xtm_oaev_sdk/_core/daemons/__init__.py | 6 + .../xtm_oaev_sdk/_core/daemons/base_daemon.py | 149 ++++++++++++++++ .../xtm_oaev_sdk/_core/daemons/ping_alive.py | 42 +++++ .../xtm_oaev_sdk/public/__init__.py | 11 ++ .../xtm_oaev_sdk/seam/__init__.py | 2 + .../xtm_oaev_sdk/seam/common/__init__.py | 8 +- .../xtm_oaev_sdk/seam/daemons/__init__.py | 6 + pyoaev/daemons/__init__.py | 15 +- pyoaev/daemons/base_daemon.py | 151 +++-------------- pyoaev/daemons/collector_daemon.py | 7 +- pyoaev/helpers.py | 4 +- pyoaev/utils.py | 42 +---- test/test_utils.py | 47 ----- 25 files changed, 874 insertions(+), 225 deletions(-) create mode 100644 libs/xtm-oaev-sdk/test/daemons/__init__.py create mode 100644 libs/xtm-oaev-sdk/test/daemons/shim/__init__.py rename {test/daemons => libs/xtm-oaev-sdk/test/daemons/shim}/test_base_daemon.py (100%) create mode 100644 libs/xtm-oaev-sdk/test/daemons/shim/test_ping_alive.py create mode 100644 libs/xtm-oaev-sdk/test/daemons/shim/test_shim_contract.py create mode 100644 libs/xtm-oaev-sdk/test/daemons/test_base_daemon.py create mode 100644 libs/xtm-oaev-sdk/test/daemons/test_ping_alive.py create mode 100644 libs/xtm-oaev-sdk/test/daemons/test_protocols.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/ping_alive.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py diff --git a/libs/xtm-oaev-sdk/test/daemons/__init__.py b/libs/xtm-oaev-sdk/test/daemons/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/libs/xtm-oaev-sdk/test/daemons/shim/__init__.py b/libs/xtm-oaev-sdk/test/daemons/shim/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/daemons/test_base_daemon.py b/libs/xtm-oaev-sdk/test/daemons/shim/test_base_daemon.py similarity index 100% rename from test/daemons/test_base_daemon.py rename to libs/xtm-oaev-sdk/test/daemons/shim/test_base_daemon.py diff --git a/libs/xtm-oaev-sdk/test/daemons/shim/test_ping_alive.py b/libs/xtm-oaev-sdk/test/daemons/shim/test_ping_alive.py new file mode 100644 index 00000000..ecffa164 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/daemons/shim/test_ping_alive.py @@ -0,0 +1,57 @@ +import unittest +from unittest.mock import MagicMock + +import pyoaev.utils as module + + +class TestPingAlive(unittest.TestCase): + def test_pingalive_ping_uses_injector_branch(self): + api = MagicMock() + logger = MagicMock() + ping_alive = module.PingAlive( + api=api, config={"id": 1}, logger=logger, ping_type="injector" + ) + ping_alive.exit_event.is_set = MagicMock(side_effect=[False, True]) + ping_alive.exit_event.wait = MagicMock() + + ping_alive.ping() + + api.injector.create.assert_called_once_with({"id": 1}, False) + ping_alive.exit_event.wait.assert_called_once_with(40) + + def test_pingalive_ping_uses_collector_branch_and_logs_errors(self): + api = MagicMock() + api.collector.create.side_effect = Exception("boom") + logger = MagicMock() + ping_alive = module.PingAlive( + api=api, config={}, logger=logger, ping_type="collector" + ) + ping_alive.exit_event.is_set = MagicMock(side_effect=[False, True]) + ping_alive.exit_event.wait = MagicMock() + + ping_alive.ping() + + logger.error.assert_called_once() + ping_alive.exit_event.wait.assert_called_once_with(40) + + def test_pingalive_run_and_stop(self): + ping_alive = module.PingAlive( + api=MagicMock(), config={}, logger=MagicMock(), ping_type="collector" + ) + ping_alive.ping = MagicMock() + + ping_alive.run() + ping_alive.stop() + + ping_alive.logger.info.assert_any_call( + "Starting PingAlive thread", {"tenant_id": str(ping_alive.tenant_id)} + ) + ping_alive.logger.info.assert_any_call( + "Preparing PingAlive for clean shutdown", + {"tenant_id": str(ping_alive.tenant_id)}, + ) + self.assertTrue(ping_alive.exit_event.is_set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/daemons/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/daemons/shim/test_shim_contract.py new file mode 100644 index 00000000..b898a31a --- /dev/null +++ b/libs/xtm-oaev-sdk/test/daemons/shim/test_shim_contract.py @@ -0,0 +1,99 @@ +"""Contract of the pyoaev daemon runtime shims. + +``PingAlive`` resolves to the *identical* xtm_oaev_sdk object. ``BaseDaemon`` +resolves to a subclass that keeps building the OpenAEV client when none is +injected. Both warn, attributed to the consumer's line. +""" + +import subprocess +import sys +import warnings +from unittest.mock import MagicMock, patch + +import pytest +import xtm_oaev_sdk as sdk + +_CONSUMER = "legacy_consumer.py" + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + source = f"from {module} import {name} as _legacy" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(compile(source, _CONSUMER, "exec"), namespace) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + return namespace["_legacy"], deprecations + + +def _then_warns_at_consumer(deprecations, sdk_name: str) -> None: + assert deprecations + assert f"from xtm_oaev_sdk import {sdk_name}" in str(deprecations[0].message) + assert any(w.filename == _CONSUMER for w in deprecations) + + +@pytest.mark.parametrize("module", ["pyoaev.utils"]) +def test_legacy_ping_alive_is_the_sdk_object(module: str) -> None: + """It warns and returns the identical PingAlive.""" + legacy, deprecations = _when_import_legacy_name(module, "PingAlive") + assert legacy is sdk.PingAlive + _then_warns_at_consumer(deprecations, "PingAlive") + + +@pytest.mark.parametrize("module", ["pyoaev.daemons", "pyoaev.daemons.base_daemon"]) +def test_legacy_base_daemon_builds_the_openaev_client(module: str) -> None: + """It warns and still builds the OpenAEV client from configuration.""" + legacy, deprecations = _when_import_legacy_name(module, "BaseDaemon") + _then_warns_at_consumer(deprecations, "BaseDaemon") + assert issubclass(legacy, sdk.BaseDaemon) + + class _Daemon(legacy): + def _setup(self): + pass + + def _start_loop(self): + pass + + configuration = sdk.Configuration( + config_hints={ + "openaev_url": {"data": "http://example.com"}, + "openaev_token": {"data": "token"}, + "openaev_tenant_id": {"data": "tenant"}, + } + ) + with patch("pyoaev.daemons.base_daemon.OpenAEV") as client_class: + daemon = _Daemon(configuration=configuration, logger=MagicMock()) + + client_class.assert_called_once_with( + url="http://example.com", token="token", tenant_id="tenant" + ) + assert daemon.api is client_class.return_value + + +def test_legacy_collector_daemon_is_still_a_legacy_base_daemon() -> None: + """It keeps the legacy class hierarchy.""" + from pyoaev.daemons import CollectorDaemon + + legacy_base, _ = _when_import_legacy_name("pyoaev.daemons", "BaseDaemon") + assert issubclass(CollectorDaemon, legacy_base) + + +def test_openaev_client_satisfies_base_client() -> None: + """It accepts the real pyoaev client as an injected API client.""" + from pyoaev import OpenAEV + + assert isinstance(OpenAEV(url="http://example.com", token="t"), sdk.BaseClient) + + +def test_importing_pyoaev_daemon_modules_is_silent() -> None: + """It imports the daemon runtime modules without any warning.""" + script = ( + "import pyoaev, pyoaev.daemons, pyoaev.daemons.base_daemon\n" + "import pyoaev.daemons.collector_daemon, pyoaev.helpers, pyoaev.utils\n" + ) + result = subprocess.run( + [sys.executable, "-W", "error::DeprecationWarning", "-c", script], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/libs/xtm-oaev-sdk/test/daemons/test_base_daemon.py b/libs/xtm-oaev-sdk/test/daemons/test_base_daemon.py new file mode 100644 index 00000000..8077df55 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/daemons/test_base_daemon.py @@ -0,0 +1,160 @@ +"""Tests for the SDK BaseDaemon (API client injected).""" + +import unittest +import unittest.mock + +import xtm_oaev_sdk as sdk + +TEST_DAEMON_CONFIGURATION = sdk.Configuration( + config_hints={ + "log_level": {"data": "info"}, + "name": {"data": "my test daemon"}, + "collector_id": {"data": "collector-id"}, + } +) + + +class DaemonForTest(sdk.BaseDaemon): + def _setup(self): + pass + + def _start_loop(self): + pass + + +def create_mock_daemon(callback: callable = None): + mock_setup = unittest.mock.MagicMock() + mock_start_loop = unittest.mock.MagicMock() + inner_mock_func = unittest.mock.MagicMock() + + class MockDaemon(sdk.BaseDaemon): + def _setup(self): + mock_setup() + + def _start_loop(self): + mock_start_loop() + + def bound_method(self): + inner_mock_func() + + return ( + MockDaemon( + configuration=TEST_DAEMON_CONFIGURATION, + callback=callback, + logger=unittest.mock.MagicMock(), + api_client=unittest.mock.MagicMock(), + ), + mock_setup, + mock_start_loop, + inner_mock_func, + ) + + +@unittest.mock.patch("argparse.ArgumentParser") +class TestBaseDaemon(unittest.TestCase): + def test_injected_api_client_is_used(self, _): + client = unittest.mock.MagicMock() + daemon = DaemonForTest( + configuration=TEST_DAEMON_CONFIGURATION, api_client=client + ) + + self.assertIs(daemon.api, client) + + def test_without_api_client_ctor_raises(self, _): + with self.assertRaises(sdk.OpenAEVError): + DaemonForTest(configuration=TEST_DAEMON_CONFIGURATION) + + def test_without_api_client_subclass_hook_builds_it(self, _): + client = unittest.mock.MagicMock() + + class HookedDaemon(DaemonForTest): + def _create_default_api_client(self): + return client + + daemon = HookedDaemon(configuration=TEST_DAEMON_CONFIGURATION) + + self.assertIs(daemon.api, client) + + def test_default_logger_is_named_from_configuration(self, _): + daemon = DaemonForTest( + configuration=TEST_DAEMON_CONFIGURATION, + api_client=unittest.mock.MagicMock(), + ) + + self.assertIsInstance(daemon.logger, sdk.AppLogger) + + def test_get_id_reads_legacy_keys(self, _): + daemon, *_ = create_mock_daemon() + + self.assertEqual(daemon.get_id(), "collector-id") + + def test_when_no_callback_daemon_cant_start(self, m_argparser): + m_argparser.return_value.parse_args.return_value = unittest.mock.MagicMock( + dump_config_schema=False + ) + daemon, mock_setup, mock_start_loop, _ = create_mock_daemon() + + with self.assertRaises(sdk.OpenAEVError): + daemon.start() + + mock_setup.assert_not_called() + mock_start_loop.assert_not_called() + + def test_when_callback_daemon_can_start(self, m_argparser): + m_argparser.return_value.parse_args.return_value = unittest.mock.MagicMock( + dump_config_schema=False + ) + daemon, mock_setup, mock_start_loop, _ = create_mock_daemon(lambda: None) + + daemon.start() + + mock_setup.assert_called_once() + mock_start_loop.assert_called_once() + + def test_when_dump_config_schema_daemon_prints_schema_and_stops(self, m_argparser): + m_argparser.return_value.parse_args.return_value = unittest.mock.MagicMock( + dump_config_schema=True + ) + daemon, mock_setup, mock_start_loop, _ = create_mock_daemon(lambda: None) + + with unittest.mock.patch.object( + TEST_DAEMON_CONFIGURATION, "schema", return_value={} + ), unittest.mock.patch("builtins.print") as mock_print: + daemon.start() + + mock_print.assert_called_once_with({}) + mock_setup.assert_not_called() + mock_start_loop.assert_not_called() + + def test_when_callback_is_bound_method_daemon_can_call(self, _): + daemon, _, _, inner_mock_func = create_mock_daemon() + daemon.set_callback(daemon.bound_method) + + daemon._try_callback() + + inner_mock_func.assert_called_once() + + def test_when_callback_is_func_with_collector_parameter_daemon_can_call(self, _): + daemon, *_ = create_mock_daemon() + inner_mock_func = unittest.mock.MagicMock() + daemon.set_callback(lambda collector: inner_mock_func(collector)) + + daemon._try_callback() + + inner_mock_func.assert_called_once_with(daemon) + + def test_when_callback_raises_daemon_logs_and_survives(self, _): + daemon, *_ = create_mock_daemon() + + def failing(): + raise RuntimeError("boom") + + daemon.set_callback(failing) + + daemon._try_callback() + + daemon.logger.error.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/daemons/test_ping_alive.py b/libs/xtm-oaev-sdk/test/daemons/test_ping_alive.py new file mode 100644 index 00000000..67501a31 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/daemons/test_ping_alive.py @@ -0,0 +1,59 @@ +"""Tests for the PingAlive heartbeat thread.""" + +import unittest +from unittest.mock import MagicMock + +import xtm_oaev_sdk as sdk + + +class TestPingAlive(unittest.TestCase): + def _given_ping(self, ping_type: str, api=None, config=None) -> sdk.PingAlive: + ping = sdk.PingAlive( + api=api or MagicMock(), + config=config if config is not None else {}, + logger=MagicMock(), + ping_type=ping_type, + ) + ping.exit_event.is_set = MagicMock(side_effect=[False, True]) + ping.exit_event.wait = MagicMock() + return ping + + def test_injector_ping_registers_the_injector_every_40_seconds(self): + api = MagicMock() + ping = self._given_ping("injector", api=api, config={"id": 1}) + + ping.ping() + + api.injector.create.assert_called_once_with({"id": 1}, False) + ping.exit_event.wait.assert_called_once_with(40) + + def test_collector_ping_logs_api_errors_and_keeps_going(self): + api = MagicMock() + api.collector.create.side_effect = Exception("boom") + ping = self._given_ping("collector", api=api) + + ping.ping() + + ping.logger.error.assert_called_once() + ping.exit_event.wait.assert_called_once_with(40) + + def test_tenant_id_comes_from_the_client(self): + api = MagicMock() + api.tenant_id = "tenant-1" + + ping = sdk.PingAlive(api=api, config={}, logger=MagicMock(), ping_type="x") + + self.assertEqual(ping.tenant_id, "tenant-1") + + def test_stop_sets_the_exit_event(self): + ping = sdk.PingAlive( + api=MagicMock(), config={}, logger=MagicMock(), ping_type="collector" + ) + + ping.stop() + + self.assertTrue(ping.exit_event.is_set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/libs/xtm-oaev-sdk/test/daemons/test_protocols.py b/libs/xtm-oaev-sdk/test/daemons/test_protocols.py new file mode 100644 index 00000000..23d14cdc --- /dev/null +++ b/libs/xtm-oaev-sdk/test/daemons/test_protocols.py @@ -0,0 +1,109 @@ +"""Tests for the BaseClient and DaemonProtocol runtime protocols.""" + +from typing import Any +from unittest.mock import MagicMock + +import pytest +import xtm_oaev_sdk as sdk + +_CLIENT_METHODS = [ + "http_delete", + "http_get", + "http_head", + "http_list", + "http_patch", + "http_post", + "http_put", +] +_CLIENT_PROPERTIES = [ + "backend", + "headers", + "order_by", + "pagination", + "per_page", + "session", + "ssl_verify", + "tenant_id", + "timeout", + "url", +] +_DAEMON_METHODS = ["get_id", "set_callback", "start"] + + +def _given_object_with(members: list[str]) -> Any: + instance = MagicMock(spec=members) + for name in members: + setattr(instance, name, MagicMock()) + return instance + + +def _when_reading_members(protocol: type, *, properties: bool) -> list[str]: + return sorted( + name + for name, value in protocol.__dict__.items() + if not name.startswith("_") + and isinstance(value, property) is properties + and (properties or callable(value)) + ) + + +@pytest.mark.parametrize("protocol", [sdk.BaseClient, sdk.DaemonProtocol]) +def test_protocols_are_runtime_checkable(protocol: type) -> None: + """It allows isinstance checks against the protocol.""" + assert getattr(protocol, "_is_runtime_protocol", False) is True + + +def test_base_client_declares_the_http_surface() -> None: + """It declares exactly the HTTP methods and client properties.""" + assert _when_reading_members(sdk.BaseClient, properties=False) == _CLIENT_METHODS + assert _when_reading_members(sdk.BaseClient, properties=True) == _CLIENT_PROPERTIES + + +def test_daemon_protocol_declares_the_lifecycle_surface() -> None: + """It declares exactly the public daemon lifecycle methods.""" + assert _when_reading_members(sdk.DaemonProtocol, properties=False) == ( + _DAEMON_METHODS + ) + + +@pytest.mark.parametrize( + ("protocol", "members"), + [ + (sdk.BaseClient, _CLIENT_METHODS + _CLIENT_PROPERTIES), + (sdk.DaemonProtocol, _DAEMON_METHODS), + ], +) +def test_protocols_accept_full_implementations(protocol: type, members) -> None: + """It accepts an object exposing every member.""" + assert isinstance(_given_object_with(members), protocol) + + +@pytest.mark.parametrize( + ("protocol", "members", "missing"), + [(sdk.BaseClient, _CLIENT_METHODS + _CLIENT_PROPERTIES, m) for m in _CLIENT_METHODS] + + [(sdk.DaemonProtocol, _DAEMON_METHODS, m) for m in _DAEMON_METHODS], +) +def test_protocols_reject_incomplete_implementations( + protocol: type, members, missing: str +) -> None: + """It rejects an object missing a member.""" + incomplete = [m for m in members if m != missing] + assert not isinstance(_given_object_with(incomplete), protocol) + + +def test_base_daemon_satisfies_daemon_protocol() -> None: + """It implements the daemon lifecycle protocol.""" + + class _Daemon(sdk.BaseDaemon): + def _setup(self): + pass + + def _start_loop(self): + pass + + daemon = _Daemon( + configuration=sdk.Configuration(config_hints={}), + api_client=MagicMock(), + logger=MagicMock(), + ) + assert isinstance(daemon, sdk.DaemonProtocol) diff --git a/libs/xtm-oaev-sdk/test/test_boundaries.py b/libs/xtm-oaev-sdk/test/test_boundaries.py index f26f4429..b77990f0 100644 --- a/libs/xtm-oaev-sdk/test/test_boundaries.py +++ b/libs/xtm-oaev-sdk/test/test_boundaries.py @@ -31,7 +31,6 @@ def test_sdk_sources_do_not_import(package: str) -> None: "artifact", [ "class ListenQueue", - "class PingAlive", "class OpenAEVConfigHelper", "class OpenAEVCollectorHelper", "class OpenAEVInjectorHelper", diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py index 9f5fa44c..4b8d0170 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -1,14 +1,16 @@ """xtm-oaev-sdk — OpenAEV SDK core (DDD + Light Hex Architecture). Architecture: - _core/ Shared implementations (errors, utils, ssl_utils, + _core/ Shared implementations (errors, utils, ssl_utils, protocols, asset_types, security_domain) _core/configuration/ Configuration feature implementation _core/contracts/ OpenAEV scenarios feature implementation + _core/daemons/ Daemon runtime (base daemon, heartbeat) _core/signatures/ Signature feature implementation seam/common/ Shared errors, enums, types, and utilities seam/configuration/ Configuration I/O surface seam/contracts/ Scenarios capability behaviour + builder protocol + seam/daemons/ Daemon runtime (API client injected) seam/signatures/ Signatures inbound I/O seam (versioned) seam/port_io/ Port I/O wire schema (versioned, seam-owned) seam/deprecation.py PEP 562 deprecation primitive diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py new file mode 100644 index 00000000..bd69a795 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py @@ -0,0 +1,81 @@ +"""BaseClient Protocol — behavioral contract for OpenAEV HTTP client implementations.""" + +from typing import Any, Protocol, runtime_checkable + +__all__ = ["BaseClient"] + + +@runtime_checkable +class BaseClient(Protocol): + """Protocol defining the behavioral contract for an OpenAEV HTTP client. + + Any class implementing all methods and properties defined here + will pass isinstance(obj, BaseClient) checks at runtime. + + IMPORTANT: Properties are used (not bare attributes) to ensure + isinstance checks work on Python 3.11 where @runtime_checkable + only inspects methods, not data attributes. + """ + + @property + def per_page(self) -> int: + ... + + @property + def pagination(self) -> str: + ... + + @property + def order_by(self) -> str: + ... + + @property + def url(self) -> str: + ... + + @property + def headers(self) -> dict[str, str]: + ... + + @property + def ssl_verify(self) -> bool | str: + ... + + @property + def timeout(self) -> int | float | None: + ... + + @property + def tenant_id(self) -> str | None: + ... + + @property + def session(self) -> Any: + """The underlying HTTP session (e.g. requests.Session).""" + ... + + @property + def backend(self) -> Any: + """The HTTP backend implementation handling transport.""" + ... + + def http_head(self, path: str, **kwargs: Any) -> Any: + ... + + def http_get(self, path: str, **kwargs: Any) -> Any: + ... + + def http_post(self, path: str, **kwargs: Any) -> Any: + ... + + def http_patch(self, path: str, **kwargs: Any) -> Any: + ... + + def http_put(self, path: str, **kwargs: Any) -> Any: + ... + + def http_delete(self, path: str, **kwargs: Any) -> Any: + ... + + def http_list(self, path: str, **kwargs: Any) -> Any: + ... diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py new file mode 100644 index 00000000..d6b06812 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py @@ -0,0 +1,39 @@ +"""DaemonProtocol — behavioral contract for daemon implementations. + +Defines the shared lifecycle surface that both collector and injector +daemons expose. Extension SDKs type-hint against this Protocol instead +of a concrete daemon class. +""" + +from typing import Any, Callable, Protocol, runtime_checkable + +__all__ = ["DaemonProtocol"] + + +@runtime_checkable +class DaemonProtocol(Protocol): + """Protocol defining the behavioral contract for a daemon runtime. + + Any class implementing all methods defined here will pass + isinstance(obj, DaemonProtocol) checks at runtime. + + The three methods capture the public lifecycle surface shared by + CollectorDaemon and injector daemon runtimes: + + - ``start()`` — run setup then enter the main loop. + - ``set_callback(callback)`` — configure the periodic callback. + - ``get_id()`` — return the daemon instance identifier. + + IMPORTANT: Only public methods are part of this contract. + Implementation details (_setup, _start_loop, _try_callback) + are excluded — they belong to the concrete daemon class. + """ + + def start(self) -> None: + ... + + def set_callback(self, callback: Callable[..., Any]) -> None: + ... + + def get_id(self) -> str | None: + ... diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py new file mode 100644 index 00000000..842d589c --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py @@ -0,0 +1,6 @@ +"""Daemon runtime shared by collectors and injectors.""" + +from .base_daemon import BaseDaemon +from .ping_alive import PingAlive + +__all__ = ["BaseDaemon", "PingAlive"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py new file mode 100644 index 00000000..97c07f93 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py @@ -0,0 +1,149 @@ +"""Base daemon: the shared runtime of collector and injector daemons.""" + +import argparse +from abc import ABC, abstractmethod +from inspect import signature +from types import FunctionType + +from xtm_oaev_sdk._core.client_protocol import BaseClient +from xtm_oaev_sdk._core.configuration import Configuration +from xtm_oaev_sdk._core.errors import OpenAEVError +from xtm_oaev_sdk._core.utils import logger + +__all__ = ["BaseDaemon"] + + +class BaseDaemon(ABC): + """A base class for implementing a kind of daemon that periodically polls + a given callback. + + :param configuration: configuration to provide the daemon + (allowing for looking up values within the callback for example) + :type configuration: Configuration + :param callback: a method or function to periodically call, defaults to None. + :type callback: callable, optional + :param logger: a logger object, to log events. if not supplied, a default logger + will be spawned to provide this functionality. + :type logger: Any + :param api_client: an API client that will provide connectivity with other systems, + conforming to the ``BaseClient`` protocol. When omitted, the client is built by + ``_create_default_api_client()``, which the SDK cannot do on its own: it never + depends on a concrete client. Inject one, or override the hook. + :type api_client: BaseClient, optional + """ + + def __init__( + self, + configuration: Configuration, + callback: callable = None, + logger=None, + api_client: BaseClient | None = None, + ): + self._configuration = configuration + self._callback = callback + self.api = api_client or self._create_default_api_client() + + # logging + # compatibility layer: in order for older configs to still work, search for legacy names + actual_log_level = ( + self._configuration.get("log_level") + or self._configuration.get("collector_log_level") + or self._configuration.get("injector_log_level") + or "info" + ) + actual_log_name = ( + self._configuration.get("name") + or self._configuration.get("collector_name") + or self._configuration.get("injector_name") + or "daemon" + ) + self.logger = logger or BaseDaemon.__get_default_logger( + actual_log_level.upper(), + actual_log_name, + ) + + @abstractmethod + def _setup(self): + """A run-once method that inheritors must implement. This serves to instantiate + all useful objects and functionality for the implementor to run. + """ + pass + + @abstractmethod + def _start_loop(self): + """Starts the daemon's main execution loop. Implementors should implement + the main execution logic in here. + """ + pass + + def _try_callback(self): + """Tries to call the configured callback. Note that if any error is thrown, + it is immediately swallowed (but still logged) allowing the collector to keep + running. This is useful for any transient issue (e.g. API endpoint down...). + """ + try: + # this is some black magic to allow injecting the collector daemon instance + # into an arbitrary callback that has a specific argument name + # this allow for avoiding subclassing the CollectorDaemon class just to provide the callback + # Example: + # + # def standalone_func(collector): + # collector.api.call_openaev() + # + # CollectorDaemon(config=, standalone_func).start() + if ( + isinstance(self._callback, FunctionType) + and "collector" in signature(self._callback).parameters + ): + self._callback(collector=self) + else: + self._callback() + except Exception as err: # pylint: disable=broad-except + self.logger.error(f"Error calling: {err}") + + def start(self): + """Start the daemon. This will run the implementor's run-once setup method and + follow-up with the main execution loop. Note that at this point, if there is no + configured callback, the method will abort and kill the daemon. + """ + parser = argparse.ArgumentParser(description="parse daemon options") + parser.add_argument("--dump-config-schema", action="store_true") + args = parser.parse_args() + if args.dump_config_schema: + print(self._configuration.schema()) + return + + if self._callback is None: + raise OpenAEVError("This daemon has no configured callback.") + self._setup() + self._start_loop() + + def set_callback(self, callback: callable): + """Configures a callback to call in the main execution loop. If the callback + was not provided in the daemon's ctor, this should be set before calling start(). + """ + self._callback = callback + + def get_id(self): + """Returns the daemon instance's ID contained in configuration. Configuration + must define any of these keys: `id`, `collector_id`, `injector_id`. + """ + return ( + self._configuration.get("id") + or self._configuration.get("collector_id") + or self._configuration.get("injector_id") + ) + + def _create_default_api_client(self) -> BaseClient: + """Builds the API client used when none was injected. The SDK has no + concrete client to build: client libraries override this hook (e.g. to + construct one from the ``openaev_url``, ``openaev_token`` and + ``openaev_tenant_id`` configuration keys). + """ + raise OpenAEVError( + "No api_client was provided to the daemon; inject an OpenAEV API client." + ) + + @classmethod + def __get_default_logger(cls, log_level, name): + return logger(log_level)(name) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/ping_alive.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/ping_alive.py new file mode 100644 index 00000000..65f354e8 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/ping_alive.py @@ -0,0 +1,42 @@ +"""Heartbeat thread keeping a daemon registered on the platform.""" + +import threading + +__all__ = ["PingAlive"] + + +class PingAlive(threading.Thread): + def __init__(self, api, config, logger, ping_type) -> None: + threading.Thread.__init__(self) + self.ping_type = ping_type + self.api = api + self.tenant_id = getattr(self.api, "tenant_id", None) + self.config = config + self.logger = logger + self.in_error = False + self.exit_event = threading.Event() + + def ping(self) -> None: + while not self.exit_event.is_set(): + try: + if self.ping_type == "injector": + self.api.injector.create(self.config, False) + else: + self.api.collector.create(self.config, False) + except Exception as err: # pylint: disable=broad-except + self.logger.error("Error pinging the API: " + str(err)) + self.exit_event.wait(40) + + def run(self) -> None: + self.logger.info( + "Starting PingAlive thread", + {"tenant_id": str(self.tenant_id) if self.tenant_id else None}, + ) + self.ping() + + def stop(self) -> None: + self.logger.info( + "Preparing PingAlive for clean shutdown", + {"tenant_id": str(self.tenant_id) if self.tenant_id else None}, + ) + self.exit_event.set() diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index c3bbc52b..5e44c646 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -45,9 +45,11 @@ AssetCategory, AssetCriticality, AssetSubCategory, + BaseClient, CloudProvider, ConfigurationError, CustomJsonFormatter, + DaemonProtocol, EncodedId, EnhancedJSONEncoder, OpenAEVAuthenticationError, @@ -102,6 +104,9 @@ prepare_contracts, ) +# --- Daemons --- +from xtm_oaev_sdk.seam.daemons import BaseDaemon, PingAlive + # --- Seam / compatibility --- from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr @@ -170,6 +175,9 @@ "logger", "remove_none_from_dict", "setup_logging_config", + # Common: protocols + "BaseClient", + "DaemonProtocol", # Common: ssl "data_to_temp_file", "is_memory_certificate", @@ -236,6 +244,9 @@ "VariableType", "build_single_referenced_credential_element", "prepare_contracts", + # Daemons + "BaseDaemon", + "PingAlive", # Signatures "CloudInjectorConfig", "ExecutionDetails", diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py index 71e283ae..6e72e26f 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -9,6 +9,8 @@ conversion primitives. - **``contracts``** — the scenarios capability's behaviour layer (builder, variable helper, preparation entry point) and its extension protocol. +- **``daemons``** — the daemon runtime shared by collectors and injectors; + the API client is injected through the ``BaseClient`` protocol. - **``signatures``** — the signatures capability's inbound seam (``seam/signatures/seam.py``), versioned ``SEAM_VERSION``. - **``port_io``** — the port I/O wire schema crossing the SDK↔platform diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py index bc5955f9..4896a87f 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py @@ -1,5 +1,5 @@ -"""Shared contracts: errors, enums, type aliases, and the concrete utility -layer.""" +"""Shared contracts: errors, protocols, enums, type aliases, and the concrete +utility layer.""" from xtm_oaev_sdk._core.asset_types import ( AssetCategory, @@ -7,6 +7,8 @@ AssetSubCategory, CloudProvider, ) +from xtm_oaev_sdk._core.client_protocol import BaseClient +from xtm_oaev_sdk._core.daemon_protocol import DaemonProtocol from xtm_oaev_sdk._core.errors import ( ConfigurationError, OpenAEVAuthenticationError, @@ -61,6 +63,8 @@ "on_http_error", # Protocols "AppLoggerProtocol", + "BaseClient", + "DaemonProtocol", # Types "EncodedId", "RequiredOptional", diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py new file mode 100644 index 00000000..1ec21077 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py @@ -0,0 +1,6 @@ +"""Daemon runtime: the base daemon and its heartbeat, shared by the collector +and injector daemons. The API client is injected (``BaseClient``).""" + +from xtm_oaev_sdk._core.daemons import BaseDaemon, PingAlive + +__all__ = ["BaseDaemon", "PingAlive"] diff --git a/pyoaev/daemons/__init__.py b/pyoaev/daemons/__init__.py index 4a0be2a9..1884747e 100644 --- a/pyoaev/daemons/__init__.py +++ b/pyoaev/daemons/__init__.py @@ -1,4 +1,17 @@ -from .base_daemon import BaseDaemon +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +from .base_daemon import _ClientBaseDaemon from .collector_daemon import CollectorDaemon __all__ = ["BaseDaemon", "CollectorDaemon"] + +__getattr__ = _make_deprecated_getattr( + { + "BaseDaemon": ( + "Use 'from xtm_oaev_sdk import BaseDaemon' and pass " + "api_client=OpenAEV(...) instead.", + _ClientBaseDaemon, + ), + }, + module_name=__name__, +) diff --git a/pyoaev/daemons/base_daemon.py b/pyoaev/daemons/base_daemon.py index 049bed5b..544682c5 100644 --- a/pyoaev/daemons/base_daemon.py +++ b/pyoaev/daemons/base_daemon.py @@ -1,140 +1,37 @@ -import argparse -from abc import ABC, abstractmethod -from inspect import signature -from types import FunctionType -from uuid import UUID +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. -from xtm_oaev_sdk import Configuration, OpenAEVError, logger +The SDK ``BaseDaemon`` takes an injected API client. The legacy name resolves +to a thin subclass that keeps building the ``OpenAEV`` client from the +``openaev_url``, ``openaev_token`` and ``openaev_tenant_id`` configuration +keys when none is passed, and emits a :class:`DeprecationWarning` on access. +""" -from pyoaev.client import OpenAEV +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr +from pyoaev.client import OpenAEV -class BaseDaemon(ABC): - """A base class for implementing a kind of daemon that periodically polls - a given callback. - :param configuration: configuration to provide the daemon - (allowing for looking up values within the callback for example) - :type configuration: Configuration - :param callback: a method or function to periodically call, defaults to None. - :type callback: callable, optional - :param logger: a logger object, to log events. if not supplied, a default logger - will be spawned to provide this functionality. - :type logger: Any - :param api_client: an API client that will provide connectivity with other systems. - :type api_client: Any - """ +class _ClientBaseDaemon(_sdk.BaseDaemon): + """SDK BaseDaemon building an OpenAEV client when none is injected.""" - def __init__( - self, - configuration: Configuration, - callback: callable = None, - logger=None, - api_client=None, - ): - self._configuration = configuration - self._callback = callback - self.api = api_client or BaseDaemon.__get_default_api_client( + def _create_default_api_client(self) -> OpenAEV: + return OpenAEV( url=self._configuration.get("openaev_url"), token=self._configuration.get("openaev_token"), tenant_id=self._configuration.get("openaev_tenant_id"), ) - # logging - # compatibility layer: in order for older configs to still work, search for legacy names - actual_log_level = ( - self._configuration.get("log_level") - or self._configuration.get("collector_log_level") - or self._configuration.get("injector_log_level") - or "info" - ) - actual_log_name = ( - self._configuration.get("name") - or self._configuration.get("collector_name") - or self._configuration.get("injector_name") - or "daemon" - ) - self.logger = logger or BaseDaemon.__get_default_logger( - actual_log_level.upper(), - actual_log_name, - ) - - @abstractmethod - def _setup(self): - """A run-once method that inheritors must implement. This serves to instantiate - all useful objects and functionality for the implementor to run. - """ - pass - - @abstractmethod - def _start_loop(self): - """Starts the daemon's main execution loop. Implementors should implement - the main execution logic in here. - """ - pass - - def _try_callback(self): - """Tries to call the configured callback. Note that if any error is thrown, - it is immediately swallowed (but still logged) allowing the collector to keep - running. This is useful for any transient issue (e.g. API endpoint down...). - """ - try: - # this is some black magic to allow injecting the collector daemon instance - # into an arbitrary callback that has a specific argument name - # this allow for avoiding subclassing the CollectorDaemon class just to provide the callback - # Example: - # - # def standalone_func(collector): - # collector.api.call_openaev() - # - # CollectorDaemon(config=, standalone_func).start() - if ( - isinstance(self._callback, FunctionType) - and "collector" in signature(self._callback).parameters - ): - self._callback(collector=self) - else: - self._callback() - except Exception as err: # pylint: disable=broad-except - self.logger.error(f"Error calling: {err}") - - def start(self): - """Start the daemon. This will run the implementor's run-once setup method and - follow-up with the main execution loop. Note that at this point, if there is no - configured callback, the method will abort and kill the daemon. - """ - parser = argparse.ArgumentParser(description="parse daemon options") - parser.add_argument("--dump-config-schema", action="store_true") - args = parser.parse_args() - if args.dump_config_schema: - print(self._configuration.schema()) - return - - if self._callback is None: - raise OpenAEVError("This daemon has no configured callback.") - self._setup() - self._start_loop() - - def set_callback(self, callback: callable): - """Configures a callback to call in the main execution loop. If the callback - was not provided in the daemon's ctor, this should be set before calling start(). - """ - self._callback = callback - - def get_id(self): - """Returns the daemon instance's ID contained in configuration. Configuration - must define any of these keys: `id`, `collector_id`, `injector_id`. - """ - return ( - self._configuration.get("id") - or self._configuration.get("collector_id") - or self._configuration.get("injector_id") - ) - @classmethod - def __get_default_api_client(cls, url, token, tenant_id: UUID | None): - return OpenAEV(url=url, token=token, tenant_id=tenant_id) +__all__ = ["BaseDaemon"] # noqa: F822 (served by the PEP 562 __getattr__ below) - @classmethod - def __get_default_logger(cls, log_level, name): - return logger(log_level)(name) +__getattr__ = _make_deprecated_getattr( + { + "BaseDaemon": ( + "Use 'from xtm_oaev_sdk import BaseDaemon' and pass " + "api_client=OpenAEV(...) instead.", + _ClientBaseDaemon, + ), + }, + module_name=__name__, +) diff --git a/pyoaev/daemons/collector_daemon.py b/pyoaev/daemons/collector_daemon.py index 55473144..17fc1d81 100644 --- a/pyoaev/daemons/collector_daemon.py +++ b/pyoaev/daemons/collector_daemon.py @@ -1,10 +1,9 @@ import sched import time -from xtm_oaev_sdk import Configuration +from xtm_oaev_sdk import Configuration, PingAlive -from pyoaev.daemons import BaseDaemon -from pyoaev.utils import PingAlive +from pyoaev.daemons.base_daemon import _ClientBaseDaemon DEFAULT_PERIOD_SECONDS = 60 @@ -13,7 +12,7 @@ DEFAULT_PLATFORM_TAG_COLOR = "#6b7280" -class CollectorDaemon(BaseDaemon): +class CollectorDaemon(_ClientBaseDaemon): """Implementation of a daemon of Collector type. Note that it requires specific configuration keys to run its setup. `collector_icon_filepath`: relative path to an icon image (preferably PNG) diff --git a/pyoaev/helpers.py b/pyoaev/helpers.py index bae04d3d..e8e14300 100644 --- a/pyoaev/helpers.py +++ b/pyoaev/helpers.py @@ -16,7 +16,7 @@ from xtm_oaev_sdk import Configuration, ConfigurationError, logger from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr -from pyoaev import OpenAEV, utils +from pyoaev import OpenAEV from pyoaev.daemons import CollectorDaemon TRUTHY: List[str] = ["yes", "true", "True"] @@ -183,7 +183,7 @@ def stop(self): self.thread.join() -class PingAlive(utils.PingAlive): +class PingAlive(_sdk.PingAlive): pass diff --git a/pyoaev/utils.py b/pyoaev/utils.py index 7a03ce31..9fe4586a 100644 --- a/pyoaev/utils.py +++ b/pyoaev/utils.py @@ -1,4 +1,4 @@ -"""Utilities specific to the pyoaev HTTP client and daemons. +"""Utilities specific to the pyoaev HTTP client. The shared utilities moved to :mod:`xtm_oaev_sdk`; their legacy names here resolve to the *identical* SDK objects and emit a :class:`DeprecationWarning` @@ -6,7 +6,6 @@ """ import email.message -import threading from typing import Any, Callable, Iterator, Optional, Union import requests @@ -23,12 +22,12 @@ "logger", "remove_none_from_dict", "setup_logging_config", + "PingAlive", ] __all__ = [ # noqa: F822 (moved names are served by the PEP 562 __getattr__ below) "get_content_type", "response_content", - "PingAlive", *_MOVED, ] @@ -68,43 +67,6 @@ def response_content( return None -class PingAlive(threading.Thread): - def __init__(self, api, config, logger, ping_type) -> None: - threading.Thread.__init__(self) - self.ping_type = ping_type - self.api = api - self.tenant_id = getattr(self.api, "tenant_id", None) - self.config = config - self.logger = logger - self.in_error = False - self.exit_event = threading.Event() - - def ping(self) -> None: - while not self.exit_event.is_set(): - try: - if self.ping_type == "injector": - self.api.injector.create(self.config, False) - else: - self.api.collector.create(self.config, False) - except Exception as err: # pylint: disable=broad-except - self.logger.error("Error pinging the API: " + str(err)) - self.exit_event.wait(40) - - def run(self) -> None: - self.logger.info( - "Starting PingAlive thread", - {"tenant_id": str(self.tenant_id) if self.tenant_id else None}, - ) - self.ping() - - def stop(self) -> None: - self.logger.info( - "Preparing PingAlive for clean shutdown", - {"tenant_id": str(self.tenant_id) if self.tenant_id else None}, - ) - self.exit_event.set() - - __getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) diff --git a/test/test_utils.py b/test/test_utils.py index 395cb979..d1d1d090 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -72,53 +72,6 @@ def test_response_content_streamed_defaults_to_stdout_stream(self): mock_print.assert_called_once_with(b"visible") - def test_pingalive_ping_uses_injector_branch(self): - api = MagicMock() - logger = MagicMock() - ping_alive = module.PingAlive( - api=api, config={"id": 1}, logger=logger, ping_type="injector" - ) - ping_alive.exit_event.is_set = MagicMock(side_effect=[False, True]) - ping_alive.exit_event.wait = MagicMock() - - ping_alive.ping() - - api.injector.create.assert_called_once_with({"id": 1}, False) - ping_alive.exit_event.wait.assert_called_once_with(40) - - def test_pingalive_ping_uses_collector_branch_and_logs_errors(self): - api = MagicMock() - api.collector.create.side_effect = Exception("boom") - logger = MagicMock() - ping_alive = module.PingAlive( - api=api, config={}, logger=logger, ping_type="collector" - ) - ping_alive.exit_event.is_set = MagicMock(side_effect=[False, True]) - ping_alive.exit_event.wait = MagicMock() - - ping_alive.ping() - - logger.error.assert_called_once() - ping_alive.exit_event.wait.assert_called_once_with(40) - - def test_pingalive_run_and_stop(self): - ping_alive = module.PingAlive( - api=MagicMock(), config={}, logger=MagicMock(), ping_type="collector" - ) - ping_alive.ping = MagicMock() - - ping_alive.run() - ping_alive.stop() - - ping_alive.logger.info.assert_any_call( - "Starting PingAlive thread", {"tenant_id": str(ping_alive.tenant_id)} - ) - ping_alive.logger.info.assert_any_call( - "Preparing PingAlive for clean shutdown", - {"tenant_id": str(ping_alive.tenant_id)}, - ) - self.assertTrue(ping_alive.exit_event.is_set()) - if __name__ == "__main__": unittest.main() From 937a4b47fa42dc8858dccba7f5e8a526890480bd Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 14:16:55 +0200 Subject: [PATCH 06/22] :recycle: refactor(pyoaev): shim the daemon helpers and expectations to collectors-sdk and injectors-sdk --- .circleci/config.yml | 8 +- .github/workflows/coverage.yml | 2 +- libs/sdk-requirements.txt | 7 + libs/xtm-oaev-sdk/pyproject.toml | 2 +- .../xtm_oaev_sdk/_core/client_protocol.py | 30 +- .../xtm_oaev_sdk/_core/daemon_protocol.py | 6 +- .../inject_expectation/inject_expectation.py | 8 +- .../apis/inject_expectation/model/__init__.py | 30 +- .../inject_expectation/model/expectation.py | 206 +------- pyoaev/daemons/__init__.py | 11 +- pyoaev/daemons/collector_daemon.py | 184 ++----- pyoaev/helpers.py | 456 ++++-------------- pyproject.toml | 10 +- test/test_sdk_split_shims.py | 197 ++++++++ 14 files changed, 429 insertions(+), 728 deletions(-) create mode 100644 libs/sdk-requirements.txt create mode 100644 test/test_sdk_split_shims.py diff --git a/.circleci/config.yml b/.circleci/config.yml index aea511d9..206de606 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ jobs: - checkout - run: name: install dependencies - command: pip3 install --user ./libs/xtm-oaev-sdk .[dev] + command: pip3 install --user ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt .[dev] - run: name: confirm black version command: black --version @@ -33,7 +33,7 @@ jobs: - checkout - run: name: install dependencies - command: pip3 install --user ./libs/xtm-oaev-sdk '.[dev]' + command: pip3 install --user ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt '.[dev]' - run: name: run tests command: python -m pytest @@ -60,7 +60,7 @@ jobs: - run: name: install dependencies command: > - pip3 install --user .[dev] + pip3 install --user ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt .[dev] - run: name: check version command: | @@ -92,7 +92,7 @@ jobs: at: ~/openaev-client - run: name: install dependencies - command: pip3 install --user . + command: pip3 install --user ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt . - run: name: install twine command: pip3 install twine diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 1cad7be8..a54f4e9f 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | - uv venv && uv pip install -e ./libs/xtm-oaev-sdk -e '.[dev]' coverage + uv venv && uv pip install -e ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt -e '.[dev]' coverage - name: Run tests with coverage run: uv run coverage run -m pytest --junitxml="junit.xml" diff --git a/libs/sdk-requirements.txt b/libs/sdk-requirements.txt new file mode 100644 index 00000000..aefde822 --- /dev/null +++ b/libs/sdk-requirements.txt @@ -0,0 +1,7 @@ +# SDKs hosted in sibling repositories, required by pyoaev while it keeps the +# deprecation shims of its former daemon helpers. Install together with the +# local xtm-oaev-sdk, e.g.: +# pip install ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt '.[dev]' +# Switch to version pins once the SDKs are published on PyPI. +collectors-sdk @ git+https://github.com/OpenAEV-Platform/collectors.git@feat/303-sdk-init#subdirectory=libs/collectors-sdk +injectors-sdk @ git+https://github.com/OpenAEV-Platform/injectors.git@feat/303-sdk-init#subdirectory=libs/injectors-sdk diff --git a/libs/xtm-oaev-sdk/pyproject.toml b/libs/xtm-oaev-sdk/pyproject.toml index 073822e5..99f745f2 100644 --- a/libs/xtm-oaev-sdk/pyproject.toml +++ b/libs/xtm-oaev-sdk/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ ] dependencies = [ "pydantic (>=2.13.3,<2.14.0)", - "pydantic-settings (>=2.14.0,<2.15.0)", + "pydantic-settings (>=2.15.0,<2.16.0)", "PyYAML (>=6.0,<6.1)", "python_json_logger (>=3.3.0,<3.4.0)", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py index bd69a795..92ce15d8 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py @@ -19,35 +19,35 @@ class BaseClient(Protocol): @property def per_page(self) -> int: - ... + """Number of items per page requested from the API.""" @property def pagination(self) -> str: - ... + """Pagination mode requested from the API.""" @property def order_by(self) -> str: - ... + """Sort key requested from the API.""" @property def url(self) -> str: - ... + """Base URL of the OpenAEV API.""" @property def headers(self) -> dict[str, str]: - ... + """HTTP headers sent with every request.""" @property def ssl_verify(self) -> bool | str: - ... + """TLS verification flag or CA bundle path.""" @property def timeout(self) -> int | float | None: - ... + """Request timeout in seconds.""" @property def tenant_id(self) -> str | None: - ... + """Tenant scoping the API requests, if any.""" @property def session(self) -> Any: @@ -60,22 +60,22 @@ def backend(self) -> Any: ... def http_head(self, path: str, **kwargs: Any) -> Any: - ... + """Send a HEAD request.""" def http_get(self, path: str, **kwargs: Any) -> Any: - ... + """Send a GET request.""" def http_post(self, path: str, **kwargs: Any) -> Any: - ... + """Send a POST request.""" def http_patch(self, path: str, **kwargs: Any) -> Any: - ... + """Send a PATCH request.""" def http_put(self, path: str, **kwargs: Any) -> Any: - ... + """Send a PUT request.""" def http_delete(self, path: str, **kwargs: Any) -> Any: - ... + """Send a DELETE request.""" def http_list(self, path: str, **kwargs: Any) -> Any: - ... + """Send a paginated GET request.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py index d6b06812..00f86760 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py @@ -30,10 +30,10 @@ class DaemonProtocol(Protocol): """ def start(self) -> None: - ... + """Run the daemon setup, then its main loop.""" def set_callback(self, callback: Callable[..., Any]) -> None: - ... + """Configure the callback run by the main loop.""" def get_id(self) -> str | None: - ... + """Return the daemon instance identifier.""" diff --git a/pyoaev/apis/inject_expectation/inject_expectation.py b/pyoaev/apis/inject_expectation/inject_expectation.py index 370b4ef8..9fede64d 100644 --- a/pyoaev/apis/inject_expectation/inject_expectation.py +++ b/pyoaev/apis/inject_expectation/inject_expectation.py @@ -1,13 +1,13 @@ from typing import Any, Dict, List -from xtm_oaev_sdk import RequiredOptional -from xtm_oaev_sdk.seam import common as exc - -from pyoaev.apis.inject_expectation.model import ( +from collectors_sdk import ( DetectionExpectation, ExpectationTypeEnum, PreventionExpectation, ) +from xtm_oaev_sdk import RequiredOptional +from xtm_oaev_sdk.seam import common as exc + from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import ListMixin, UpdateMixin diff --git a/pyoaev/apis/inject_expectation/model/__init__.py b/pyoaev/apis/inject_expectation/model/__init__.py index 8cedb2d1..7b605b57 100644 --- a/pyoaev/apis/inject_expectation/model/__init__.py +++ b/pyoaev/apis/inject_expectation/model/__init__.py @@ -1,7 +1,25 @@ -from .expectation import ( - DetectionExpectation, - ExpectationTypeEnum, - PreventionExpectation, -) +"""DEPRECATED: moved to :mod:`collectors_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +collectors_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import collectors_sdk as _collectors_sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr -__all__ = ["DetectionExpectation", "ExpectationTypeEnum", "PreventionExpectation"] +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "DetectionExpectation", + "ExpectationTypeEnum", + "PreventionExpectation", +] + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in (*__all__, "Expectation", "ExpectationSignature") + }, + module_name=__name__, +) diff --git a/pyoaev/apis/inject_expectation/model/expectation.py b/pyoaev/apis/inject_expectation/model/expectation.py index 3b290c10..04f2959c 100644 --- a/pyoaev/apis/inject_expectation/model/expectation.py +++ b/pyoaev/apis/inject_expectation/model/expectation.py @@ -1,181 +1,27 @@ -from enum import Enum -from typing import Any, List -from uuid import UUID - -from pydantic import BaseModel, Field, field_validator -from thefuzz import fuzz -from xtm_oaev_sdk import MatchTypes, SignatureType, SignatureTypes - - -class ExpectationTypeEnum(str, Enum): - """Types of Expectations""" - - Detection = "DETECTION" - Prevention = "PREVENTION" - Vulnerability = "VULNERABILITY" - Other = "other" - - @classmethod - def _missing_(cls, value): - return cls.Other - - -class ExpectationSignature(BaseModel): - """An expectation signature describes a known marker potentially - found in alerting data in security software. For example, an - expectation signature can be a process image name, a command - line, or any other relevant piece of data. - """ - - type: SignatureTypes - value: str - - -class Expectation(BaseModel): - """An expectation represents an expected outcome of a BAS run. - For example, in the case of running an attack command line, the - expectation may be that security software has _detected_ it, while - another expectation may be that the attack was _prevented_. - """ - - inject_expectation_id: UUID - inject_expectation_signatures: List[ExpectationSignature] = Field( - default_factory=list - ) - - success_label: str = "Success" - failure_label: str = "Failure" - - @field_validator("inject_expectation_signatures", mode="before") - @classmethod - def _coerce_none_signatures_to_empty(cls, value: Any) -> Any: - # The platform serializes this field as null when an expectation has no - # signatures yet. Without this, a single such expectation raises a - # ValidationError that aborts the whole batch fetch (collectors#490). - return [] if value is None else value - - def __init__(self, *a, **kw): - super().__init__(*a, **kw) - self.__api_client = kw["api_client"] - - def update(self, success, sender_id, metadata): - """Update the expectation object in OpenAEV with the supplied outcome. - - :param success: whether the expectation was fulfilled (true) or not (false) - :type success: bool - :param sender_id: identifier of the collector that is updating the expectation - :type sender_id: string - :param metadata: arbitrary dictionary of additional data relevant to updating the expectation - :type metadata: dict[string,string] - """ - self.__api_client.update( - self.inject_expectation_id, - inject_expectation={ - "collector_id": sender_id, - "result": (self.success_label if success else self.failure_label), - "is_success": success, - "metadata": metadata, - }, +"""DEPRECATED: moved to :mod:`collectors_sdk`. + +Backward-compatibility shim: every name below resolves to the *identical* +collectors_sdk object and emits a :class:`DeprecationWarning` on access. +""" + +import collectors_sdk as _collectors_sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "DetectionExpectation", + "Expectation", + "ExpectationSignature", + "ExpectationTypeEnum", + "PreventionExpectation", +] + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), ) - - def match_alert(self, relevant_signature_types: list[SignatureType], alert_data): - """Matches an alert's data against the current expectation signatures - to see if the alert is relevant to the current expectation's inject, - i.e. this alert was triggered by the execution of the inject to which - belongs the expectation. - - :param relevant_signature_types: filter of signature types that we want to consider. - Only the signature types listed in this collection may be checked for matching. - :type relevant_signature_types: list[SignatureType] - :param alert_data: list of possibly relevant markers found in an alert. - :type alert_data: dict[SignatureTypes, dict] - - :return: whether the alert matches the expectation signatures or not. - :rtype: bool - """ - relevant_expectation_signatures = [ - signature - for signature in self.inject_expectation_signatures - if signature.type in [type.label for type in relevant_signature_types] - ] - if not any(relevant_expectation_signatures): - return False - - for relevant_expectation_signature in relevant_expectation_signatures: - if not ( - alert_signature_for_type := alert_data.get( - relevant_expectation_signature.type.value - ) - ): - return False - - if alert_signature_for_type[ - "type" - ] == MatchTypes.MATCH_TYPE_FUZZY and not self.match_fuzzy( - alert_signature_for_type["data"], - relevant_expectation_signature.value, - alert_signature_for_type["score"], - ): - return False - if alert_signature_for_type[ - "type" - ] == MatchTypes.MATCH_TYPE_SIMPLE and not self.match_simple( - alert_signature_for_type["data"], relevant_expectation_signature.value - ): - return False - - return True - - @staticmethod - def match_fuzzy(tested: list[str], reference: str, threshold: int): - """Applies a fuzzy match against a known reference to a list of candidates - - :param tested: list of strings candidate for fuzzy matching - :type tested: list[str] - :param reference: the reference against which to try to fuzzy match - :type reference: str - :param threshold: string overlap percentage threshold above which to declare a match - :type threshold: int - - :return: whether any of the candidate is a match against the reference - :rtype: bool - """ - actual_tested = [tested] if isinstance(tested, str) else tested - for value in actual_tested: - ratio = fuzz.ratio(value, reference) - if ratio >= threshold: - return True - return False - - @staticmethod - def match_simple(tested: list[str], reference: str): - """A simple strict, case-sensitive string matching between a list of - candidates and a reference. - - :param tested: list of strings candidate for fuzzy matching - :type tested: list[str] - :param reference: the reference against which to try to fuzzy match - :type reference: str - - :return: whether any of the candidate is a match against the reference - :rtype: bool - """ - return Expectation.match_fuzzy(tested, reference, threshold=100) - - -class DetectionExpectation(Expectation): - """An expectation that is specific to Detection, i.e. that is used - by OpenAEV to assert that an inject's execution was detected. - """ - - success_label: str = "Detected" - failure_label: str = "Not Detected" - - -class PreventionExpectation(Expectation): - """An expectation that is specific to Prevention, i.e. that is used - by OpenAEV to assert that an inject's execution was prevented. - """ - - success_label: str = "Prevented" - failure_label: str = "Not Prevented" + for name in __all__ + }, + module_name=__name__, +) diff --git a/pyoaev/daemons/__init__.py b/pyoaev/daemons/__init__.py index 1884747e..2586f74d 100644 --- a/pyoaev/daemons/__init__.py +++ b/pyoaev/daemons/__init__.py @@ -1,9 +1,15 @@ +"""DEPRECATED: the daemon runtime moved to :mod:`xtm_oaev_sdk` and +:mod:`collectors_sdk`; the legacy names keep building the OpenAEV client.""" + from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr from .base_daemon import _ClientBaseDaemon -from .collector_daemon import CollectorDaemon +from .collector_daemon import _COLLECTOR_DAEMON_MESSAGE, _ClientCollectorDaemon -__all__ = ["BaseDaemon", "CollectorDaemon"] +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "BaseDaemon", + "CollectorDaemon", +] __getattr__ = _make_deprecated_getattr( { @@ -12,6 +18,7 @@ "api_client=OpenAEV(...) instead.", _ClientBaseDaemon, ), + "CollectorDaemon": (_COLLECTOR_DAEMON_MESSAGE, _ClientCollectorDaemon), }, module_name=__name__, ) diff --git a/pyoaev/daemons/collector_daemon.py b/pyoaev/daemons/collector_daemon.py index 17fc1d81..78f332a6 100644 --- a/pyoaev/daemons/collector_daemon.py +++ b/pyoaev/daemons/collector_daemon.py @@ -1,144 +1,50 @@ -import sched -import time +"""DEPRECATED: moved to :mod:`collectors_sdk`. -from xtm_oaev_sdk import Configuration, PingAlive +The collectors-sdk ``CollectorDaemon`` takes an injected API client. The legacy +name resolves to a thin subclass that keeps building the ``OpenAEV`` client +from configuration when none is passed. Every other name resolves to the +*identical* SDK object. All names emit a :class:`DeprecationWarning` on access. +""" -from pyoaev.daemons.base_daemon import _ClientBaseDaemon - -DEFAULT_PERIOD_SECONDS = 60 - -# Neutral grey used for tags auto-created alongside the security platform, -# consistent with the tag colors already used across collectors. -DEFAULT_PLATFORM_TAG_COLOR = "#6b7280" - - -class CollectorDaemon(_ClientBaseDaemon): - """Implementation of a daemon of Collector type. Note that it requires - specific configuration keys to run its setup. - `collector_icon_filepath`: relative path to an icon image (preferably PNG) - `collector_id`: unique identifier for the collector (UUIDv4) - `collector_period`: time to wait in seconds between each loop execution; note - that this time is added to the time the loop takes to run, so the actual total - time between each loop start is time_of_loop+period. - `collector_author` (optional): source-declared author override for the - collector's payloads and contracts; when absent, the platform attributes - them to the collector's name. - `collector_platform_description` (optional): description applied to the - security platform auto-created when `collector_platform` is set, so the - read-only card is not empty in the UI. - `collector_platform_tags` (optional): list of tag names (or a - comma-separated string) applied to the auto-created security platform. - """ +import collectors_sdk as _collectors_sdk +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr - def __init__( - self, - configuration: Configuration, - callback: callable = None, - logger=None, - api_client=None, - collector_type=None, - ): - super().__init__(configuration, callback, logger, api_client) - if collector_type is None: - raise ValueError("Must define a value for collector type") - self.collector_type = collector_type - - def _setup(self): - if self._configuration.get("collector_period") is None: - self._configuration.set("collector_period", DEFAULT_PERIOD_SECONDS) - icon_path = self._configuration.get("collector_icon_filepath") - icon_name = self._configuration.get("collector_id") + ".png" - with open(icon_path, "rb") as icon_file_handle: - collector_icon = (icon_name, icon_file_handle, "image/png") - document = self.api.document.upsert(document={}, file=collector_icon) - if self._configuration.get("collector_platform") is not None: - security_platform_input = { - "asset_name": self._configuration.get("collector_name"), - "asset_external_reference": self._configuration.get("collector_id"), - "security_platform_type": self._configuration.get( - "collector_platform" - ), - "security_platform_logo_light": document.get("document_id"), - "security_platform_logo_dark": document.get("document_id"), - } - description = self._configuration.get("collector_platform_description") - if description: - security_platform_input["asset_description"] = description - tag_ids = self.__upsert_platform_tags( - self._configuration.get("collector_platform_tags") - ) - if tag_ids: - security_platform_input["asset_tags"] = tag_ids - security_platform = self.api.security_platform.upsert( - security_platform_input - ) - else: - security_platform = {} - security_platform_id = security_platform.get("asset_id") - config = { - "collector_id": self._configuration.get("collector_id"), - "collector_name": self._configuration.get("collector_name"), - "collector_type": self.collector_type, - "collector_period": self._configuration.get("collector_period"), - "collector_security_platform": security_platform_id, - # Optional author override; None lets the platform fall back to - # the collector name. - "collector_author": self._configuration.get("collector_author"), - } - with open(icon_path, "rb") as icon_file_handle: - collector_icon = (icon_name, icon_file_handle, "image/png") - self.api.collector.create(config, collector_icon) - - PingAlive(self.api, config, self.logger, "collector").start() - - def __upsert_platform_tags(self, tag_names) -> list: - """Upserts the configured security platform tags and returns their ids. - - Accepts either a list of tag names or a comma-separated string; tag - upsert failures are logged and skipped so the collector setup never - fails because of a tag. - """ - if not tag_names: - return [] - if isinstance(tag_names, str): - tag_names = tag_names.split(",") - tag_ids = [] - for raw_name in tag_names: - # Strip whitespace for both string and list inputs (YAML list values - # can carry stray spaces too). - tag_name = raw_name.strip() if isinstance(raw_name, str) else raw_name - if not tag_name: - continue - try: - tag = self.api.tag.upsert( - {"tag_name": tag_name, "tag_color": DEFAULT_PLATFORM_TAG_COLOR} - ) - tag_id = tag.get("tag_id") - if tag_id: - tag_ids.append(tag_id) - except Exception as err: # noqa: BLE001 - self.logger.warning( - f"Could not upsert security platform tag {tag_name}: {err}" - ) - return tag_ids +from pyoaev.daemons.base_daemon import _ClientBaseDaemon - def _start_loop(self): - scheduler = sched.scheduler(time.time, time.sleep) - delay = self._configuration.get("collector_period") - self._try_callback() - scheduler.enter( - delay=delay, - priority=1, - action=self.__schedule, - argument=(scheduler, self._try_callback, delay), - ) - scheduler.run() - def __schedule(self, scheduler, callback, delay): - callback() - scheduler.enter( - delay=delay, - priority=1, - action=self.__schedule, - argument=(scheduler, callback, delay), - ) +class _ClientCollectorDaemon(_collectors_sdk.CollectorDaemon, _ClientBaseDaemon): + """collectors-sdk CollectorDaemon building an OpenAEV client when none is injected.""" + + +_COLLECTOR_DAEMON_MESSAGE = ( + "Use 'from collectors_sdk import CollectorDaemon' and pass " + "api_client=OpenAEV(...) instead." +) + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "CollectorDaemon", + "DEFAULT_PERIOD_SECONDS", + "DEFAULT_PLATFORM_TAG_COLOR", +] + +__getattr__ = _make_deprecated_getattr( + { + "CollectorDaemon": (_COLLECTOR_DAEMON_MESSAGE, _ClientCollectorDaemon), + **{ + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in ("DEFAULT_PERIOD_SECONDS", "DEFAULT_PLATFORM_TAG_COLOR") + }, + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in ("Configuration", "PingAlive") + }, + }, + module_name=__name__, +) diff --git a/pyoaev/helpers.py b/pyoaev/helpers.py index e8e14300..a10a666a 100644 --- a/pyoaev/helpers.py +++ b/pyoaev/helpers.py @@ -1,238 +1,43 @@ -import base64 -import json -import os -import os.path -import re -import sched -import ssl -import threading -import time -import traceback -from typing import Callable, Dict, List - -import pika +"""DEPRECATED: the daemon helpers moved to :mod:`collectors_sdk`, +:mod:`injectors_sdk` and :mod:`xtm_oaev_sdk`. + +Backward-compatibility shim: every legacy name resolves to the *identical* SDK +object and emits a :class:`DeprecationWarning` on access, with two adapters: +``OpenAEVInjectorHelper`` keeps building the ``OpenAEV`` client when none is +injected, and ``OpenAEVCollectorHelper`` (already deprecated) stays here as +legacy glue over the collectors-sdk ``CollectorDaemon``. +""" + +import collectors_sdk as _collectors_sdk +import injectors_sdk as _injectors_sdk import xtm_oaev_sdk as _sdk -from thefuzz import fuzz -from xtm_oaev_sdk import Configuration, ConfigurationError, logger +from collectors_sdk.seam import detection as _detection from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr from pyoaev import OpenAEV -from pyoaev.daemons import CollectorDaemon - -TRUTHY: List[str] = ["yes", "true", "True"] -FALSY: List[str] = ["no", "false", "False"] - -# SSL helpers moved to xtm_oaev_sdk: legacy names resolve to the identical SDK -# objects and warn on access (PEP 562). -__getattr__ = _make_deprecated_getattr( - { - name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) - for name in ( - "data_to_temp_file", - "is_memory_certificate", - "ssl_cert_chain", - "ssl_verify_locations", - ) - }, - module_name=__name__, +from pyoaev.daemons.collector_daemon import ( + _COLLECTOR_DAEMON_MESSAGE, + _ClientCollectorDaemon, ) -def create_mq_ssl_context(config) -> ssl.SSLContext: - config_obj = Configuration( - config_hints={ - "MQ_USE_SSL_CA": { - "env": "MQ_USE_SSL_CA", - "file_path": ["mq", "use_ssl_ca"], - }, - "MQ_USE_SSL_CERT": { - "env": "MQ_USE_SSL_CERT", - "file_path": ["mq", "use_ssl_cert"], - }, - "MQ_USE_SSL_KEY": { - "env": "MQ_USE_SSL_KEY", - "file_path": ["mq", "use_ssl_key"], - }, - "MQ_USE_SSL_REJECT_UNAUTHORIZED": { - "env": "MQ_USE_SSL_REJECT_UNAUTHORIZED", - "file_path": ["mq", "use_ssl_reject_unauthorized"], - "is_number": False, - "default": False, - }, - "MQ_USE_SSL_PASSPHRASE": { - "env": "MQ_USE_SSL_PASSPHRASE", - "file_path": ["mq", "use_ssl_passphrase"], - }, - }, - config_values=config, - ) - use_ssl_ca = config_obj.get("MQ_USE_SSL_CA") - use_ssl_cert = config_obj.get("MQ_USE_SSL_CERT") - use_ssl_key = config_obj.get("MQ_USE_SSL_KEY") - use_ssl_reject_unauthorized = config_obj.get("MQ_USE_SSL_REJECT_UNAUTHORIZED") - use_ssl_passphrase = config_obj.get("MQ_USE_SSL_PASSPHRASE") - ssl_context = ssl.create_default_context() - # If no rejection allowed, use private function to generate unverified context - if not use_ssl_reject_unauthorized: - # noinspection PyUnresolvedReferences,PyProtectedMember - ssl_context = ssl._create_unverified_context() - _sdk.ssl_verify_locations(ssl_context, use_ssl_ca) - # Thanks to https://bugs.python.org/issue16487 is not possible today to easily use memory pem - # in SSL context. We need to write it to a temporary file before - _sdk.ssl_cert_chain(ssl_context, use_ssl_cert, use_ssl_key, use_ssl_passphrase) - return ssl_context - - -class ListenQueue(threading.Thread): - def __init__( - self, - config: Dict, - injector_config, - logger, - callback, - ) -> None: - threading.Thread.__init__(self) - self.pika_credentials = None - self.pika_parameters = None - self.pika_connection = None - self.channel = None - - self.callback = callback - self.config = config - self.logger = logger - self.host = injector_config.connection["host"] - self.vhost = injector_config.connection["vhost"] - self.use_ssl = injector_config.connection["use_ssl"] - self.port = injector_config.connection["port"] - self.user = injector_config.connection["user"] - self.password = injector_config.connection["pass"] - self.queue_name = injector_config.listen - self.exit_event = threading.Event() - self.thread = None - - # noinspection PyUnusedLocal - def _process_message(self, channel, method, properties, body) -> None: - """process a message from the rabbit queue - - :param channel: channel instance - :type channel: callable - :param method: message methods - :type method: callable - :param properties: unused - :type properties: str - :param body: message body (data) - :type body: str or bytes or bytearray - """ - json_data = json.loads(body) - # Message should be ack before processing as we don't own the processing - # Not ACK the message here may lead to infinite re-deliver if the connector is broken - # Also ACK, will not have any impact on the blocking aspect of the following functions - channel.basic_ack(delivery_tag=method.delivery_tag) - self.thread = threading.Thread(target=self._data_handler, args=[json_data]) - self.thread.start() +class _ClientInjectorHelper(_injectors_sdk.OpenAEVInjectorHelper): + """injectors-sdk OpenAEVInjectorHelper building an OpenAEV client when none + is injected.""" - def _data_handler(self, json_data) -> None: - self.callback(json_data) - - def run(self) -> None: - self.logger.info("Starting ListenQueue thread") - while not self.exit_event.is_set(): - try: - self.logger.info("ListenQueue connecting to RabbitMQ.") - # Connect the broker - self.pika_credentials = pika.PlainCredentials(self.user, self.password) - self.pika_parameters = pika.ConnectionParameters( - host=self.host, - port=self.port, - virtual_host=self.vhost, - credentials=self.pika_credentials, - ssl_options=( - pika.SSLOptions(create_mq_ssl_context(self.config), self.host) - if self.use_ssl - else None - ), - ) - self.pika_connection = pika.BlockingConnection(self.pika_parameters) - self.channel = self.pika_connection.channel() - try: - # confirm_delivery is only for cluster mode rabbitMQ - # when not in cluster mode this line raise an exception - self.channel.confirm_delivery() - except Exception as err: # pylint: disable=broad-except - self.logger.error(str(err)) - self.channel.basic_qos(prefetch_count=1) - assert self.channel is not None - self.channel.basic_consume( - queue=self.queue_name, on_message_callback=self._process_message - ) - self.channel.start_consuming() - except Exception: # pylint: disable=broad-except - try: - self.pika_connection.close() - except Exception as errInException: - self.logger.error(str(errInException)) - traceback.print_exc() - # Wait some time and then retry ListenQueue again. - time.sleep(10) - - def stop(self): - self.logger.info("Preparing ListenQueue for clean shutdown") - self.exit_event.set() - self.pika_connection.close() - if self.thread: - self.thread.join() - - -class PingAlive(_sdk.PingAlive): - pass - - -### DEPRECATED -class OpenAEVConfigHelper: - def __init__(self, base_path, variables: Dict | None, config_obj: Configuration): - if config_obj is not None: - self.__config_obj = config_obj - else: - self.__config_obj = Configuration( - config_hints=variables, - config_file_path=os.path.join( - os.path.dirname(os.path.abspath(base_path)), "config.yml" - ), - ) - - @staticmethod - def from_configuration_object(config: Configuration): - return OpenAEVConfigHelper(None, None, config) - - def get_config_obj(self) -> Configuration: - return self.__config_obj - - def get_conf(self, variable, is_number=None, default=None, required=None): - result = None - try: - result = ( - self.__config_obj.get(variable) - if (self.__config_obj.get(variable) is not None) - else default - ) - except ConfigurationError: - result = default - finally: - if result is None and default is None and required: - raise ValueError( - f"Could not find required key {variable} with no available default." - ) - return result - - def to_configuration(self): - return self.__config_obj + def _create_default_api_client(self, config) -> OpenAEV: + return OpenAEV( + url=config.get_conf("openaev_url"), + token=config.get_conf("openaev_token"), + tenant_id=config.get_conf("openaev_tenant_id"), + ) ### DEPRECATED -class OpenAEVCollectorHelper: +class _OpenAEVCollectorHelper: def __init__( self, - config: OpenAEVConfigHelper, + config: _injectors_sdk.OpenAEVConfigHelper, icon, collector_type=None, security_platform_type=None, @@ -245,17 +50,17 @@ def __init__( if security_platform_type is not None: config_obj.set("collector_platform", security_platform_type) - self.__daemon = CollectorDaemon( + self.__daemon = _ClientCollectorDaemon( configuration=config_obj, callback=None, collector_type=collector_type, ) self.__daemon.logger.warning( - f"DEPRECATED: this collector should be migrated to use {CollectorDaemon}." + f"DEPRECATED: this collector should be migrated to use {_ClientCollectorDaemon}." ) - self.logger_class = logger( + self.logger_class = _sdk.logger( config.get_conf("collector_log_level", default="error").upper(), config.get_conf("collector_json_logging", default=True), ) @@ -278,148 +83,65 @@ def schedule(self, message_callback, delay): self.__daemon.start() -class OpenAEVInjectorHelper: - def __init__(self, config: OpenAEVConfigHelper, icon) -> None: - self.api = OpenAEV( - url=config.get_conf("openaev_url"), - token=config.get_conf("openaev_token"), - tenant_id=config.get_conf("openaev_tenant_id"), - ) - # Get the mq configuration from api - self.config = { - "injector_id": config.get_conf("injector_id"), - "injector_name": config.get_conf("injector_name"), - "injector_type": config.get_conf("injector_type"), - "injector_contracts": config.get_conf("injector_contracts"), - "injector_custom_contracts": config.get_conf( - "injector_custom_contracts", default=False - ), - "injector_category": config.get_conf("injector_category", default=None), - "injector_executor_commands": config.get_conf( - "injector_executor_commands", default=None - ), - "injector_executor_clear_commands": config.get_conf( - "injector_executor_clear_commands", default=None - ), - # Publisher/author of the injector's contracts (source-declared). - "injector_author": config.get_conf("injector_author", default=None), - } - - self.logger_class = logger( - config.get_conf("injector_log_level", default="error").upper(), - config.get_conf("injector_json_logging", default=True), - ) - self.injector_logger = self.logger_class(config.get_conf("injector_name")) +_FROM_XTM = [ + "Configuration", + "ConfigurationError", + "PingAlive", + "data_to_temp_file", + "is_memory_certificate", + "logger", + "ssl_cert_chain", + "ssl_verify_locations", +] +_FROM_INJECTORS = ["ListenQueue", "OpenAEVConfigHelper", "create_mq_ssl_context"] +_TRUTHINESS = { + "TRUTHY": (["yes", "true", "True"], "is_truthy"), + "FALSY": (["no", "false", "False"], "is_falsy"), +} - icon_name = config.get_conf("injector_type") + ".png" - injector_icon = (icon_name, icon, "image/png") - self.injector_config = self.api.injector.create(self.config, injector_icon) - self.connect_run_and_terminate = False - self.scheduler = sched.scheduler(time.time, time.sleep) - # Start ping thread - if not self.connect_run_and_terminate: - self.ping = PingAlive( - self.api, self.config, self.injector_logger, "injector" +__getattr__ = _make_deprecated_getattr( + { + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), ) - self.ping.start() - self.listen_queue = None - - def listen(self, message_callback: Callable[[Dict], None]) -> None: - self.listen_queue = ListenQueue( - self.config, self.injector_config, self.injector_logger, message_callback - ) - self.listen_queue.start() - - -class OpenAEVDetectionHelper: - def __init__(self, logger, relevant_signatures_types) -> None: - self.logger = logger - self.relevant_signatures_types = relevant_signatures_types - - def match_alert_element_fuzzy(self, signature_value, alert_values, fuzzy_scoring): - for alert_value in alert_values: - self.logger.info( - "Comparing alert value (" + alert_value + ", " + signature_value + ")" + for name in _FROM_XTM + }, + **{ + name: ( + f"Use 'from injectors_sdk import {name}' instead.", + getattr(_injectors_sdk, name), ) - ratio = fuzz.ratio(alert_value, signature_value) - if ratio > fuzzy_scoring: - self.logger.info("MATCHING! (score: " + str(ratio) + ")") - return True - return False - - def match_alert_elements(self, signatures, alert_data): - return self._match_alert_elements_original( - signatures, alert_data - ) or self._match_alert_elements_for_command_line(signatures, alert_data) - - def _match_alert_elements_original(self, signatures, alert_data): - # Example for alert_data - # {"process_name": {"list": ["xx", "yy"], "fuzzy": 90}} - relevant_signatures = [ - s for s in signatures if s["type"] in self.relevant_signatures_types - ] - - # Matching logics - signatures_number = len(relevant_signatures) - matching_number = 0 - for signature in relevant_signatures: - alert_data_for_signature = alert_data[signature["type"]] - signature_result = False - if alert_data_for_signature["type"] == "fuzzy": - signature_result = self.match_alert_element_fuzzy( - signature["value"], - alert_data_for_signature["data"], - alert_data_for_signature["score"], - ) - elif alert_data_for_signature["type"] == "simple": - signature_result = signature["value"] in str( - alert_data_for_signature["data"] - ) - - if signature_result: - matching_number = matching_number + 1 - - if signatures_number == matching_number: - return True - return False - - def _match_alert_elements_for_command_line(self, signatures, alert_data): - command_line_signatures = [ - signature - for signature in signatures - if signature.get("type") == "command_line" - ] - if len(command_line_signatures) == 0: - return False - key_types = ["command_line", "process_name", "file_name"] - alert_datas = [alert_data.get(key) for key in key_types if key in alert_data] - for signature in command_line_signatures: - signature_result = False - signature_value = self._decode_value(signature["value"]).strip().lower() - for alert_data in alert_datas: - trimmed_lowered_datas = [s.strip().lower() for s in alert_data["data"]] - signature_result = any( - data in signature_value for data in trimmed_lowered_datas - ) - if signature_result: - return True - return False - - def _decode_value(self, signature_value): - if _is_base64_encoded(signature_value): - try: - decoded_bytes = base64.b64decode(signature_value) - decoded_str = decoded_bytes.decode("utf-8") - return decoded_str - except Exception as e: - self.logger.error(str(e)) - else: - return signature_value - - -def _is_base64_encoded(str_maybe_base64): - # Check if the length is a multiple of 4 and matches the Base64 character set - base64_pattern = re.compile(r"^[A-Za-z0-9+/]*={0,2}$") - return len(str_maybe_base64) % 4 == 0 and bool( - base64_pattern.match(str_maybe_base64) - ) + for name in _FROM_INJECTORS + }, + "OpenAEVInjectorHelper": ( + "Use 'from injectors_sdk import OpenAEVInjectorHelper' and pass " + "api_client=OpenAEV(...) instead.", + _ClientInjectorHelper, + ), + "OpenAEVDetectionHelper": ( + "Use 'from collectors_sdk import OpenAEVDetectionHelper' instead.", + _collectors_sdk.OpenAEVDetectionHelper, + ), + "_is_base64_encoded": ( + "'_is_base64_encoded' is private and will be removed.", + _detection._is_base64_encoded, + ), + "CollectorDaemon": (_COLLECTOR_DAEMON_MESSAGE, _ClientCollectorDaemon), + "OpenAEVCollectorHelper": ( + "OpenAEVCollectorHelper is deprecated; use " + "'from collectors_sdk import CollectorDaemon' instead.", + _OpenAEVCollectorHelper, + ), + **{ + name: ( + f"'{name}' is deprecated; use 'from xtm_oaev_sdk import {helper}' " + "instead.", + value, + ) + for name, (value, helper) in _TRUTHINESS.items() + }, + }, + module_name=__name__, +) diff --git a/pyproject.toml b/pyproject.toml index 37be9a69..deb46a72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,16 +25,14 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules" ] dependencies = [ - "pika (>=1.4.4,<1.5.0)", - "python_json_logger (>=3.3.0,<3.4.0)", - "PyYAML (>=6.0,<6.1)", "pydantic (>=2.13.3,<2.14.0)", - "pydantic-settings (>=2.15.0,<2.16.0)", "requests (>=2.33.0,<2.34.0)", "requests-toolbelt (>=1.0.0,<1.1.0)", - "thefuzz (>=0.22,<0.23)", - # OpenAEV SDKs + # OpenAEV SDKs (collectors-sdk and injectors-sdk host the former daemon + # helpers and stay required while pyoaev keeps their deprecation shims) "xtm-oaev-sdk (>=0.1.0,<0.2.0)", + "collectors-sdk (>=0.1.0,<0.2.0)", + "injectors-sdk (>=0.1.0,<0.2.0)", ] [project.optional-dependencies] diff --git a/test/test_sdk_split_shims.py b/test/test_sdk_split_shims.py new file mode 100644 index 00000000..d5e1633a --- /dev/null +++ b/test/test_sdk_split_shims.py @@ -0,0 +1,197 @@ +"""Contract of the pyoaev shims toward collectors-sdk and injectors-sdk. + +Legacy imports and patch targets of collectors and injectors must keep working: +names resolve to the *identical* SDK object, or to a thin subclass that keeps +building the OpenAEV client, and warn at the consumer's line. +""" + +import subprocess +import sys +import warnings +from unittest.mock import MagicMock, mock_open, patch + +import collectors_sdk +import injectors_sdk +import pytest +import xtm_oaev_sdk + +_CONSUMER = "legacy_consumer.py" + +IDENTICAL = [ + ("pyoaev.helpers", "OpenAEVDetectionHelper", collectors_sdk), + ("pyoaev.helpers", "ListenQueue", injectors_sdk), + ("pyoaev.helpers", "OpenAEVConfigHelper", injectors_sdk), + ("pyoaev.helpers", "create_mq_ssl_context", injectors_sdk), + ("pyoaev.helpers", "PingAlive", xtm_oaev_sdk), + ("pyoaev.helpers", "Configuration", xtm_oaev_sdk), + ("pyoaev.helpers", "ConfigurationError", xtm_oaev_sdk), + ("pyoaev.helpers", "logger", xtm_oaev_sdk), + ("pyoaev.daemons.collector_daemon", "DEFAULT_PERIOD_SECONDS", collectors_sdk), + ("pyoaev.daemons.collector_daemon", "DEFAULT_PLATFORM_TAG_COLOR", collectors_sdk), + ("pyoaev.apis.inject_expectation.model", "DetectionExpectation", collectors_sdk), + ("pyoaev.apis.inject_expectation.model", "PreventionExpectation", collectors_sdk), + ("pyoaev.apis.inject_expectation.model", "ExpectationTypeEnum", collectors_sdk), + ("pyoaev.apis.inject_expectation.model", "Expectation", collectors_sdk), + ("pyoaev.apis.inject_expectation.model.expectation", "Expectation", collectors_sdk), + ( + "pyoaev.apis.inject_expectation.model.expectation", + "ExpectationSignature", + collectors_sdk, + ), +] + + +def _when_import_legacy_name(module: str, name: str): + namespace: dict = {} + source = f"from {module} import {name} as _legacy" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(compile(source, _CONSUMER, "exec"), namespace) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + return namespace["_legacy"], deprecations + + +def _then_warns_at_consumer(deprecations) -> None: + assert deprecations + assert any(w.filename == _CONSUMER for w in deprecations) + + +@pytest.mark.parametrize(("module", "name", "sdk"), IDENTICAL) +def test_legacy_name_is_the_sdk_object(module, name, sdk) -> None: + """It warns and returns the identical SDK object.""" + legacy, deprecations = _when_import_legacy_name(module, name) + assert legacy is getattr(sdk, name) + _then_warns_at_consumer(deprecations) + assert sdk.__name__ in str(deprecations[0].message) + + +@pytest.mark.parametrize( + "module", ["pyoaev.daemons", "pyoaev.daemons.collector_daemon", "pyoaev.helpers"] +) +def test_legacy_collector_daemon_is_one_client_building_subclass(module) -> None: + """It resolves every path to the same class, so class patches still apply.""" + from pyoaev.daemons.collector_daemon import _ClientCollectorDaemon + + legacy, deprecations = _when_import_legacy_name(module, "CollectorDaemon") + _then_warns_at_consumer(deprecations) + assert legacy is _ClientCollectorDaemon + assert issubclass(legacy, collectors_sdk.CollectorDaemon) + + +@patch("pyoaev.apis.DocumentManager.upsert", return_value={}) +@patch("pyoaev.apis.CollectorManager.create", return_value={}) +@patch("builtins.open", new_callable=mock_open, read_data="data") +@patch("pyoaev.utils.PingAlive.start") +def test_legacy_collector_daemon_builds_the_openaev_client(*_) -> None: + """It still works with only a configuration, as legacy collectors use it.""" + from pyoaev.client import OpenAEV + from pyoaev.configuration import Configuration + from pyoaev.daemons import CollectorDaemon + + config = Configuration( + config_hints={ + "openaev_url": {"data": "http://example.com"}, + "openaev_token": {"data": "token"}, + "collector_id": {"data": "collector-id"}, + } + ) + collector = CollectorDaemon(configuration=config, collector_type="test") + + assert isinstance(collector.api, OpenAEV) + collector._setup() + + +def test_legacy_collector_daemon_setup_patch_target_still_applies() -> None: + """It honours patches on pyoaev.daemons.collector_daemon.CollectorDaemon._setup.""" + from pyoaev.daemons import CollectorDaemon + + with patch("pyoaev.daemons.collector_daemon.CollectorDaemon._setup") as setup: + collector = CollectorDaemon( + configuration=xtm_oaev_sdk.Configuration(config_hints={}), + api_client=MagicMock(), + collector_type="test", + ) + collector._setup() + + setup.assert_called_once() + + +@patch("xtm_oaev_sdk.PingAlive.start") +def test_legacy_injector_helper_builds_the_openaev_client(_) -> None: + """It still builds the client from the config helper when none is injected.""" + from pyoaev.helpers import OpenAEVConfigHelper, OpenAEVInjectorHelper + + config = OpenAEVConfigHelper.from_configuration_object( + xtm_oaev_sdk.Configuration( + config_hints={ + "openaev_url": {"data": "http://example.com"}, + "openaev_token": {"data": "token"}, + "openaev_tenant_id": {"data": "tenant"}, + "injector_type": {"data": "openaev_test"}, + "injector_log_level": {"data": "info"}, + } + ) + ) + with patch("pyoaev.helpers.OpenAEV") as client_class: + helper = OpenAEVInjectorHelper(config, MagicMock()) + + client_class.assert_called_once_with( + url="http://example.com", token="token", tenant_id="tenant" + ) + assert helper.api is client_class.return_value + assert isinstance(helper, injectors_sdk.OpenAEVInjectorHelper) + + +@pytest.mark.parametrize( + ("name", "value"), + [("TRUTHY", ["yes", "true", "True"]), ("FALSY", ["no", "false", "False"])], +) +def test_legacy_truthiness_constants_keep_their_value(name, value) -> None: + """It keeps the unused legacy constants available, with a warning.""" + legacy, deprecations = _when_import_legacy_name("pyoaev.helpers", name) + assert legacy == value + _then_warns_at_consumer(deprecations) + + +def test_legacy_collector_helper_still_resolves() -> None: + """It keeps the already deprecated OpenAEVCollectorHelper available.""" + legacy, deprecations = _when_import_legacy_name( + "pyoaev.helpers", "OpenAEVCollectorHelper" + ) + _then_warns_at_consumer(deprecations) + assert callable(legacy) + + +def test_expectation_manager_builds_sdk_models() -> None: + """It hands out collectors-sdk expectation models.""" + from pyoaev.apis import InjectExpectationManager + + manager = InjectExpectationManager(MagicMock()) + manager.expectations_assets_for_source = MagicMock( + return_value=[ + { + "inject_expectation_id": "4a1bb2c4-6a59-4b4b-a3c0-0d36a4f8e1c1", + "inject_expectation_type": "DETECTION", + "inject_expectation_signatures": [], + } + ] + ) + + (model,) = manager.expectations_models_for_source(source_id="collector") + + assert isinstance(model, collectors_sdk.DetectionExpectation) + + +def test_importing_pyoaev_modules_is_silent() -> None: + """It imports pyoaev and its shim modules without any warning.""" + script = ( + "import pyoaev, pyoaev.helpers, pyoaev.daemons, pyoaev.daemons.collector_daemon\n" + "import pyoaev.apis, pyoaev.apis.inject_expectation.model\n" + "import pyoaev.apis.inject_expectation.model.expectation\n" + ) + result = subprocess.run( + [sys.executable, "-W", "error::DeprecationWarning", "-c", script], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr From 78e74540adac57ada751da6785b26be10c0eeab2 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 14:22:43 +0200 Subject: [PATCH 07/22] :white_check_mark: test(pyoaev): guard the legacy public surface of pyoaev --- pyoaev/apis/inject_expectation/__init__.py | 1 + .../inject_expectation/model/expectation.py | 20 +- pyoaev/daemons/base_daemon.py | 7 + pyoaev/daemons/collector_daemon.py | 5 + test/fixtures/legacy_surface.json | 643 ++++++++++++++++++ test/generate_legacy_surface.py | 64 ++ test/test_legacy_surface.py | 67 ++ 7 files changed, 802 insertions(+), 5 deletions(-) create mode 100644 test/fixtures/legacy_surface.json create mode 100644 test/generate_legacy_surface.py create mode 100644 test/test_legacy_surface.py diff --git a/pyoaev/apis/inject_expectation/__init__.py b/pyoaev/apis/inject_expectation/__init__.py index 183bc0e2..2bbeaa66 100644 --- a/pyoaev/apis/inject_expectation/__init__.py +++ b/pyoaev/apis/inject_expectation/__init__.py @@ -1 +1,2 @@ +from . import model # noqa: F401 (legacy: exposed as pyoaev.apis.model) from .inject_expectation import * # noqa: F401,F403 diff --git a/pyoaev/apis/inject_expectation/model/expectation.py b/pyoaev/apis/inject_expectation/model/expectation.py index 04f2959c..94080ba3 100644 --- a/pyoaev/apis/inject_expectation/model/expectation.py +++ b/pyoaev/apis/inject_expectation/model/expectation.py @@ -5,6 +5,7 @@ """ import collectors_sdk as _collectors_sdk +import xtm_oaev_sdk as _sdk from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr __all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) @@ -17,11 +18,20 @@ __getattr__ = _make_deprecated_getattr( { - name: ( - f"Use 'from collectors_sdk import {name}' instead.", - getattr(_collectors_sdk, name), - ) - for name in __all__ + **{ + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in __all__ + }, + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in ("MatchTypes", "SignatureType", "SignatureTypes") + }, }, module_name=__name__, ) diff --git a/pyoaev/daemons/base_daemon.py b/pyoaev/daemons/base_daemon.py index 544682c5..2e98d162 100644 --- a/pyoaev/daemons/base_daemon.py +++ b/pyoaev/daemons/base_daemon.py @@ -32,6 +32,13 @@ def _create_default_api_client(self) -> OpenAEV: "api_client=OpenAEV(...) instead.", _ClientBaseDaemon, ), + **{ + name: ( + f"Use 'from xtm_oaev_sdk import {name}' instead.", + getattr(_sdk, name), + ) + for name in ("Configuration", "OpenAEVError", "logger") + }, }, module_name=__name__, ) diff --git a/pyoaev/daemons/collector_daemon.py b/pyoaev/daemons/collector_daemon.py index 78f332a6..9d4e26d1 100644 --- a/pyoaev/daemons/collector_daemon.py +++ b/pyoaev/daemons/collector_daemon.py @@ -31,6 +31,11 @@ class _ClientCollectorDaemon(_collectors_sdk.CollectorDaemon, _ClientBaseDaemon) __getattr__ = _make_deprecated_getattr( { "CollectorDaemon": (_COLLECTOR_DAEMON_MESSAGE, _ClientCollectorDaemon), + "BaseDaemon": ( + "Use 'from xtm_oaev_sdk import BaseDaemon' and pass " + "api_client=OpenAEV(...) instead.", + _ClientBaseDaemon, + ), **{ name: ( f"Use 'from collectors_sdk import {name}' instead.", diff --git a/test/fixtures/legacy_surface.json b/test/fixtures/legacy_surface.json new file mode 100644 index 00000000..44b4bbfa --- /dev/null +++ b/test/fixtures/legacy_surface.json @@ -0,0 +1,643 @@ +{ + "modules": { + "pyoaev": [ + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "BaseConfigModel", + "CloudInjectorConfig", + "CloudProvider", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", + "ConfigurationError", + "ContractBuilder", + "ExpectationSignatureGroup", + "ExpectationType", + "ExtraSignatureData", + "InjectExecutionActions", + "MatchTypes", + "NetworkInjectorConfig", + "OpenAEV", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SettingsLoader", + "SignatureCallbackPayload", + "SignatureManager", + "SignaturePayload", + "SignatureTarget", + "SignatureTransmissionError", + "SignatureTypes", + "SignatureValue", + "TargetSignatures", + "__author__", + "__copyright__", + "__email__", + "__license__", + "__title__", + "__version__", + "build_network_configs" + ], + "pyoaev._version": [], + "pyoaev.apis": [ + "AiTarget", + "AiTargetManager", + "Any", + "AttackPattern", + "AttackPatternManager", + "Collector", + "CollectorManager", + "CreateMixin", + "Cve", + "CveManager", + "DeleteMixin", + "DetectionExpectation", + "Dict", + "Document", + "DocumentManager", + "Endpoint", + "EndpointManager", + "ExecutionDetails", + "ExpectationTypeEnum", + "GetMixin", + "GetWithoutIdMixin", + "Inject", + "InjectExpectation", + "InjectExpectationManager", + "InjectExpectationTrace", + "InjectExpectationTraceManager", + "InjectManager", + "Injector", + "InjectorContract", + "InjectorContractManager", + "InjectorContractSearchPaginationInput", + "InjectorManager", + "KillChainPhase", + "KillChainPhaseManager", + "List", + "ListMixin", + "Me", + "MeManager", + "Organization", + "OrganizationManager", + "Payload", + "PayloadManager", + "PreventionExpectation", + "RESTManager", + "RESTObject", + "RequiredOptional", + "SearchPaginationInput", + "SecurityPlatform", + "SecurityPlatformManager", + "Signature", + "SignatureApiManager", + "SignatureCallbackPayload", + "SignatureOutputStructure", + "SignatureTransmissionError", + "Tag", + "TagManager", + "Team", + "TeamManager", + "UpdateMixin", + "User", + "UserManager", + "ValidationError", + "Vulnerability", + "VulnerabilityManager", + "ai_target", + "attack_pattern", + "cast", + "collector", + "cve", + "document", + "endpoint", + "exc", + "inject", + "inject_expectation", + "inject_expectation_trace", + "injector", + "injector_contract", + "inputs", + "json", + "kill_chain_phase", + "logging", + "me", + "model", + "organization", + "payload", + "security_platform", + "signature", + "tag", + "team", + "time", + "user", + "vulnerability" + ], + "pyoaev.apis.ai_target": [ + "AiTarget", + "AiTargetManager", + "CreateMixin", + "DeleteMixin", + "GetMixin", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.attack_pattern": [ + "AttackPattern", + "AttackPatternManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.collector": [ + "Collector", + "CollectorManager", + "CreateMixin", + "GetMixin", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.cve": [ + "Cve", + "CveManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.document": [ + "Document", + "DocumentManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.endpoint": [ + "Endpoint", + "EndpointManager", + "RESTManager", + "RESTObject", + "RequiredOptional", + "SearchPaginationInput" + ], + "pyoaev.apis.inject": [ + "Inject", + "InjectManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.inject_expectation": [ + "DetectionExpectation", + "ExpectationTypeEnum", + "InjectExpectation", + "InjectExpectationManager", + "ListMixin", + "PreventionExpectation", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.inject_expectation.inject_expectation": [ + "DetectionExpectation", + "ExpectationTypeEnum", + "InjectExpectation", + "InjectExpectationManager", + "ListMixin", + "PreventionExpectation", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.inject_expectation.model": [ + "DetectionExpectation", + "ExpectationTypeEnum", + "PreventionExpectation" + ], + "pyoaev.apis.inject_expectation.model.expectation": [ + "DetectionExpectation", + "Expectation", + "ExpectationSignature", + "ExpectationTypeEnum", + "MatchTypes", + "PreventionExpectation", + "SignatureType", + "SignatureTypes" + ], + "pyoaev.apis.inject_expectation_trace": [ + "CreateMixin", + "InjectExpectationTrace", + "InjectExpectationTraceManager", + "RESTManager", + "RESTObject", + "RequiredOptional" + ], + "pyoaev.apis.injector": [ + "CreateMixin", + "GetMixin", + "Injector", + "InjectorManager", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.injector_contract": [ + "CreateMixin", + "DeleteMixin", + "InjectorContract", + "InjectorContractManager", + "InjectorContractSearchPaginationInput", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin" + ], + "pyoaev.apis.inputs": [], + "pyoaev.apis.inputs.search": [ + "Filter", + "FilterGroup", + "InjectorContractSearchPaginationInput", + "SearchPaginationInput" + ], + "pyoaev.apis.kill_chain_phase": [ + "KillChainPhase", + "KillChainPhaseManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.me": [ + "GetWithoutIdMixin", + "Me", + "MeManager", + "RESTManager", + "RESTObject", + "UpdateMixin" + ], + "pyoaev.apis.organization": [ + "ListMixin", + "Organization", + "OrganizationManager", + "RESTManager", + "RESTObject", + "UpdateMixin" + ], + "pyoaev.apis.payload": [ + "Payload", + "PayloadManager", + "RESTManager", + "RESTObject" + ], + "pyoaev.apis.security_platform": [ + "CreateMixin", + "GetMixin", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "SecurityPlatform", + "SecurityPlatformManager", + "UpdateMixin" + ], + "pyoaev.apis.signature": [ + "ExecutionDetails", + "RESTManager", + "RESTObject", + "Signature", + "SignatureApiManager", + "SignatureCallbackPayload", + "SignatureOutputStructure", + "SignatureTransmissionError" + ], + "pyoaev.apis.tag": [ + "RESTManager", + "RESTObject", + "Tag", + "TagManager" + ], + "pyoaev.apis.team": [ + "CreateMixin", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "Team", + "TeamManager", + "UpdateMixin" + ], + "pyoaev.apis.user": [ + "CreateMixin", + "ListMixin", + "RESTManager", + "RESTObject", + "RequiredOptional", + "UpdateMixin", + "User", + "UserManager" + ], + "pyoaev.apis.vulnerability": [ + "RESTManager", + "RESTObject", + "Vulnerability", + "VulnerabilityManager" + ], + "pyoaev.asset_types": [ + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "CloudProvider" + ], + "pyoaev.backends": [ + "DefaultBackend", + "DefaultResponse", + "RequestsBackend", + "RequestsResponse", + "TokenAuth" + ], + "pyoaev.backends.backend": [ + "Auth", + "RequestsBackend", + "RequestsResponse", + "SendData", + "TokenAuth" + ], + "pyoaev.backends.protocol": [ + "Backend", + "BackendResponse" + ], + "pyoaev.base": [ + "OpenAEV", + "OpenAEVList", + "OpenAEVParsingError", + "RESTManager", + "RESTObject", + "RESTObjectList" + ], + "pyoaev.client": [ + "OpenAEV", + "OpenAEVList", + "REDIRECT_MSG" + ], + "pyoaev.configuration": [ + "BaseConfigModel", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", + "SettingsLoader" + ], + "pyoaev.configuration.configuration": [ + "Configuration", + "ConfigurationHint", + "ConnectorConfigSchemaGenerator", + "DictionarySource", + "EnvironmentSource", + "is_falsy", + "is_truthy" + ], + "pyoaev.configuration.connector_config_schema_generator": [ + "ConnectorConfigSchemaGenerator" + ], + "pyoaev.configuration.settings_loader": [ + "BaseConfigModel", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "SettingsLoader" + ], + "pyoaev.configuration.sources": [ + "DictionarySource", + "EnvironmentSource" + ], + "pyoaev.contracts": [ + "ContractBuilder" + ], + "pyoaev.contracts.contract_builder": [ + "ContractBuilder", + "ContractElement", + "ContractOutputElement" + ], + "pyoaev.contracts.contract_config": [ + "Contract", + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractElement", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "Domain", + "Expectation", + "ExpectationType", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableHelper", + "prepare_contracts" + ], + "pyoaev.contracts.contract_utils": [ + "ContractCardinality", + "ContractVariable", + "VariableType" + ], + "pyoaev.contracts.variable_helper": [ + "CHALLENGES_URI", + "COMCHECK", + "ContractCardinality", + "ContractVariable", + "EXERCISE", + "LESSONS_URI", + "PLAYER_URI", + "SCOREBOARD_URI", + "TEAMS", + "USER", + "VariableHelper", + "VariableType" + ], + "pyoaev.daemons": [ + "BaseDaemon", + "CollectorDaemon" + ], + "pyoaev.daemons.base_daemon": [ + "BaseDaemon", + "Configuration", + "OpenAEV", + "OpenAEVError", + "logger" + ], + "pyoaev.daemons.collector_daemon": [ + "BaseDaemon", + "CollectorDaemon", + "Configuration", + "DEFAULT_PERIOD_SECONDS", + "DEFAULT_PLATFORM_TAG_COLOR", + "PingAlive" + ], + "pyoaev.exceptions": [ + "ConfigurationError", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "SignatureTransmissionError", + "on_http_error" + ], + "pyoaev.helpers": [ + "CollectorDaemon", + "Configuration", + "ConfigurationError", + "FALSY", + "ListenQueue", + "OpenAEV", + "OpenAEVCollectorHelper", + "OpenAEVConfigHelper", + "OpenAEVDetectionHelper", + "OpenAEVInjectorHelper", + "PingAlive", + "TRUTHY", + "create_mq_ssl_context", + "data_to_temp_file", + "is_memory_certificate", + "ssl_cert_chain", + "ssl_verify_locations" + ], + "pyoaev.mixins": [ + "CreateMixin", + "DeleteMixin", + "GetMixin", + "GetWithoutIdMixin", + "HeadMixin", + "ListMixin", + "UpdateMethod", + "UpdateMixin" + ], + "pyoaev.security_domain": [], + "pyoaev.security_domain.types": [ + "SecurityDomains" + ], + "pyoaev.signatures": [ + "CloudInjectorConfig", + "ExpectationSignatureGroup", + "ExpectationType", + "ExtraSignatureData", + "InjectExecutionActions", + "InjectorConfig", + "MatchTypes", + "NetworkInjectorConfig", + "SignatureCallbackPayload", + "SignatureManager", + "SignaturePayload", + "SignatureTarget", + "SignatureTypes", + "SignatureValue", + "TargetSignatures", + "build_network_configs" + ], + "pyoaev.signatures.ai_marker": [ + "build_marker" + ], + "pyoaev.signatures.models": [ + "CloudInjectorConfig", + "ExecutionDetails", + "ExecutionSignature", + "ExecutionStatus", + "ExpectationSignatureGroup", + "ExpectationType", + "ExtraSignatureData", + "InjectExecutionActions", + "InjectorConfig", + "NetworkInjectorConfig", + "SignatureCallbackPayload", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureValue", + "TargetSignatures", + "ToolErrorInfo", + "ToolOutput", + "ToolTimeoutInfo", + "build_network_configs" + ], + "pyoaev.signatures.signature_manager": [ + "CloudInjectorConfig", + "ExecutionDetails", + "ExecutionSignature", + "ExpectationSignatureGroup", + "ExtraSignatureData", + "NetworkInjectorConfig", + "OpenAEVError", + "SignatureManager", + "SignatureOutputStructure", + "SignaturePayload", + "SignatureTarget", + "SignatureValue", + "TargetSignatures", + "ToolOutput" + ], + "pyoaev.signatures.signature_match": [ + "MatchTypes", + "OpenAEVError", + "SignatureMatch" + ], + "pyoaev.signatures.signature_type": [ + "MatchTypes", + "SignatureMatch", + "SignatureType", + "SignatureTypes" + ], + "pyoaev.signatures.types": [ + "ExecutionStatus", + "ExpectationType", + "InjectExecutionActions", + "MatchTypes", + "SignatureTypes" + ], + "pyoaev.utils": [ + "AppLogger", + "CustomJsonFormatter", + "EncodedId", + "EnhancedJSONEncoder", + "PingAlive", + "RequiredOptional", + "copy_dict", + "get_content_type", + "logger", + "remove_none_from_dict", + "response_content", + "setup_logging_config" + ] + }, + "pyoaev_version": "3.260923.0" +} diff --git a/test/generate_legacy_surface.py b/test/generate_legacy_surface.py new file mode 100644 index 00000000..bc1500ea --- /dev/null +++ b/test/generate_legacy_surface.py @@ -0,0 +1,64 @@ +"""Record the public surface of an installed pyoaev release. + +The legacy surface guard (``test_legacy_surface.py``) asserts that every name a +released pyoaev exposed still resolves, so the SDK split never breaks existing +code. Regenerate the golden from the *released* package, never from the +working tree:: + + python -m venv /tmp/pyoaev-release + /tmp/pyoaev-release/bin/pip install "pyoaev==" + /tmp/pyoaev-release/bin/python test/generate_legacy_surface.py + +A public name is anything in a module's ``__all__``, any class or function +defined in pyoaev (including re-exports from other pyoaev modules), and any +upper-case data constant. +""" + +import importlib +import json +import pkgutil +import types +import warnings +from pathlib import Path + +import pyoaev + +GOLDEN_PATH = Path(__file__).parent / "fixtures" / "legacy_surface.json" + +_DATA_CONSTANT_TYPES = (str, int, float, list, tuple, dict) + + +def _is_public(name: str, obj: object, exported: set[str]) -> bool: + if name in exported: + return True + if name.startswith("_") or isinstance(obj, types.ModuleType): + return False + if (getattr(obj, "__module__", None) or "").startswith("pyoaev"): + return True + is_data = isinstance(obj, _DATA_CONSTANT_TYPES) and not isinstance(obj, bool) + return name.isupper() and is_data + + +def _public_names(module: types.ModuleType) -> list[str]: + exported = set(getattr(module, "__all__", ())) + names = [] + for name in dir(module): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + obj = getattr(module, name, None) + if _is_public(name, obj, exported): + names.append(name) + return sorted(names) + + +def main() -> None: + modules = {"pyoaev": _public_names(pyoaev)} + for info in pkgutil.walk_packages(pyoaev.__path__, "pyoaev."): + modules[info.name] = _public_names(importlib.import_module(info.name)) + surface = {"pyoaev_version": pyoaev.__version__, "modules": modules} + GOLDEN_PATH.write_text(json.dumps(surface, indent=2, sort_keys=True) + "\n") + print(f"wrote {GOLDEN_PATH}") + + +if __name__ == "__main__": + main() diff --git a/test/test_legacy_surface.py b/test/test_legacy_surface.py new file mode 100644 index 00000000..c5a135cb --- /dev/null +++ b/test/test_legacy_surface.py @@ -0,0 +1,67 @@ +"""Legacy surface guard: the SDK split must never break existing code. + +Every public name the released pyoaev exposed (recorded in +``fixtures/legacy_surface.json`` by ``generate_legacy_surface.py``) must still +resolve from the same module, possibly through a deprecation shim. +""" + +import importlib +import json +import warnings +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest + +GOLDEN_PATH = Path(__file__).parent / "fixtures" / "legacy_surface.json" + + +def _given_legacy_surface() -> dict[str, Any]: + return json.loads(GOLDEN_PATH.read_text(encoding="utf-8")) + + +def _given_legacy_names() -> list[tuple[str, str]]: + modules = _given_legacy_surface()["modules"] + return [(module, name) for module, names in modules.items() for name in names] + + +def _when_importing_module(module: str) -> ModuleType: + return importlib.import_module(module) + + +def _when_resolving_name(module: ModuleType, name: str) -> bool: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + try: + getattr(module, name) + except AttributeError: + return False + return True + + +def _then_name_resolves(resolved: bool, module: str, name: str) -> None: + assert resolved, ( + f"'{module}.{name}' no longer resolves: legacy code importing it breaks. " + "Serve it from the module's deprecation shim." + ) + + +def test_golden_records_a_released_pyoaev() -> None: + """It pins the surface of a released pyoaev version.""" + surface = _given_legacy_surface() + assert surface["pyoaev_version"] + assert surface["modules"]["pyoaev"] + + +@pytest.mark.parametrize("module", sorted(_given_legacy_surface()["modules"])) +def test_every_legacy_module_still_imports(module: str) -> None: + """It keeps every legacy module importable.""" + assert _when_importing_module(module).__name__ == module + + +@pytest.mark.parametrize(("module", "name"), _given_legacy_names()) +def test_every_legacy_name_still_resolves(module: str, name: str) -> None: + """It keeps every legacy name reachable from its original module.""" + resolved = _when_resolving_name(_when_importing_module(module), name) + _then_name_resolves(resolved, module, name) From 4ece485f346f08be22f47c3fddd05a79280b09be Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 15:02:38 +0200 Subject: [PATCH 08/22] :recycle: refactor(sdk): enhancing port-adapter public boundaries for configuration --- libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 16 +- .../_core/configuration/__init__.py | 51 +-- .../_core/configuration/configuration.py | 258 ------------ .../_core/configuration/resolver.py | 103 +++++ ...r_config_schema_generator.py => schema.py} | 100 +---- .../_core/configuration/settings_sources.py | 44 ++ .../_core/configuration/sources.py | 66 +-- .../_core/configuration/truthiness.py | 13 + .../xtm_oaev_sdk/_core/daemons/base_daemon.py | 7 +- .../xtm_oaev_sdk/public/__init__.py | 40 +- .../xtm_oaev_sdk/public/configuration.py | 380 ++++++++++++++++++ .../xtm_oaev_sdk/seam/__init__.py | 21 +- .../seam/configuration/__init__.py | 46 ++- .../configuration/models.py} | 131 ++---- .../xtm_oaev_sdk/seam/configuration/ports.py | 140 +++++++ 15 files changed, 830 insertions(+), 586 deletions(-) delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/configuration.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/resolver.py rename libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/{connector_config_schema_generator.py => schema.py} (53%) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_sources.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/truthiness.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/public/configuration.py rename libs/xtm-oaev-sdk/xtm_oaev_sdk/{_core/configuration/settings_loader.py => seam/configuration/models.py} (55%) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/ports.py diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py index 4b8d0170..e2fc883d 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -8,18 +8,26 @@ _core/daemons/ Daemon runtime (base daemon, heartbeat) _core/signatures/ Signature feature implementation seam/common/ Shared errors, enums, types, and utilities - seam/configuration/ Configuration I/O surface + seam/configuration/ Configuration contracts and ports (closed, versioned) seam/contracts/ Scenarios capability behaviour + builder protocol seam/daemons/ Daemon runtime (API client injected) seam/signatures/ Signatures inbound I/O seam (versioned) seam/port_io/ Port I/O wire schema (versioned, seam-owned) seam/deprecation.py PEP 562 deprecation primitive contracts/contracts/ OpenAEV scenario data models (business layer) + public/configuration Configuration adapters public/ User-facing re-exports -The seam is the compatibility boundary and the sole door between the public -surface and ``_core``: the public layer reaches ``_core`` exclusively through -the seam, and nothing in ``_core`` imports ``public``. +Port/adapter layering, capability by capability (see ``seam``): + +- ``seam`` holds the contracts — data models, enums, errors and the ``Protocol`` + ports — and imports nothing from ``_core`` nor ``public``; +- ``_core`` implements the seam ports and never imports ``public``; +- ``public`` exposes adapters that speak seam types and delegate to ``_core``, + plus the seam contracts themselves — never a core object. + +Capabilities not migrated yet still re-export ``_core`` through their seam +package. """ __version__ = "0.1.0" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py index f5c2924d..e892e1a8 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/__init__.py @@ -1,44 +1,15 @@ -"""Configuration management: loading, hints, and schema generation. +"""Configuration core: implementations of the configuration seam ports.""" -Re-exports the public configuration surface for internal engine use. -Public consumers import from xtm_oaev_sdk directly. -""" - -from .configuration import ( - CONFIGURATION_TYPES, - Configuration, - ConfigurationHint, - ConfigurationProtocol, - is_falsy, - is_truthy, -) -from .connector_config_schema_generator import ConnectorConfigSchemaGenerator -from .settings_loader import ( - BaseConfigModel, - ConfigLoaderCollector, - ConfigLoaderOAEV, - HttpUrlToString, - LogLevelToLower, - SettingsLoader, - TimedeltaInSeconds, -) -from .sources import DictionarySource, EnvironmentSource +from .resolver import ConfigurationResolver +from .schema import ConnectorSchemaTransformer +from .settings_sources import SettingsSourceSelector +from .sources import ConfigurationSources +from .truthiness import Truthiness __all__ = [ - "BaseConfigModel", - "CONFIGURATION_TYPES", - "ConfigLoaderCollector", - "ConfigLoaderOAEV", - "Configuration", - "ConfigurationHint", - "ConfigurationProtocol", - "ConnectorConfigSchemaGenerator", - "DictionarySource", - "EnvironmentSource", - "HttpUrlToString", - "LogLevelToLower", - "SettingsLoader", - "TimedeltaInSeconds", - "is_falsy", - "is_truthy", + "ConfigurationResolver", + "ConfigurationSources", + "ConnectorSchemaTransformer", + "SettingsSourceSelector", + "Truthiness", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/configuration.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/configuration.py deleted file mode 100644 index d29bc3da..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/configuration.py +++ /dev/null @@ -1,258 +0,0 @@ -"""Configuration management: hint-based value resolution. - -Provides the Configuration class that resolves values from multiple -sources (environment, files, overrides) using a hint-based lookup system. - -The Configuration class is the primary interface for SDK consumers to -access typed configuration values. - -Example: - >>> config = Configuration({ - ... "log_level": {"env": "LOG_LEVEL", "default": "info"}, - ... "api_url": {"env": "API_URL", "file_path": ["openaev", "url"]}, - ... }) - >>> config.get("log_level") - 'info' - >>> config.set("log_level", "debug") - >>> config.get("log_level") - 'debug' -""" - -import os -import os.path -from typing import Any, Protocol, runtime_checkable - -import yaml -from pydantic import BaseModel, Field -from pydantic_settings import BaseSettings -from xtm_oaev_sdk._core.configuration.connector_config_schema_generator import ( - ConnectorConfigSchemaGenerator, -) -from xtm_oaev_sdk._core.configuration.sources import DictionarySource, EnvironmentSource - -__all__ = [ - "CONFIGURATION_TYPES", - "Configuration", - "ConfigurationHint", - "ConfigurationProtocol", - "is_falsy", - "is_truthy", -] - -CONFIGURATION_TYPES = str | int | bool | Any | None -"""Union type for configuration values.""" - - -def is_truthy(value: str) -> bool: - """Asserts whether a given string signals a "True" value. - - :param value: value to test - :type value: str - - :return: whether the string represents True or not. - :rtype: bool - """ - return value.lower() in ["yes", "true"] - - -def is_falsy(value: str) -> bool: - """Asserts whether a given string signals a "False" value. - - :param value: value to test - :type value: str - - :return: whether the string represents False or not. - :rtype: bool - """ - return value.lower() in ["no", "false"] - - -@runtime_checkable -class ConfigurationProtocol(Protocol): - """Behavioral contract for configuration services. - - Any configuration implementation used by SDK consumers must satisfy - this protocol. The built-in Configuration class implements it. - - Methods: - get: Retrieve a typed configuration value by key. - set: Override a configuration value at runtime. - schema: Generate the connector configuration JSON schema. - """ - - def get(self, config_key: str) -> CONFIGURATION_TYPES: - """Get configuration value by key.""" - ... - - def set(self, config_key: str, value: CONFIGURATION_TYPES) -> None: - """Set a runtime override for a configuration key.""" - ... - - def schema(self) -> dict[str, Any]: - """Generate the configuration JSON schema.""" - ... - - -class ConfigurationHint(BaseModel): - """An individual configuration hint defining where a value can be found. - - Hints specify the resolution strategy for a configuration key: - which environment variable to check, which file path to follow, - and what default to use if no source provides a value. - - The model is mutable: ``Configuration.set()`` assigns ``data`` in place. - - Args: - data: Override value. When set, this value is returned directly. - env: Environment variable name to read. - file_path: JSON path (nested keys) in the config file. - is_number: Whether to interpret the resolved value as an integer. - default: Fallback value when no source provides one. - - Example: - >>> hint = ConfigurationHint(env="LOG_LEVEL", default="info") - >>> hint.data = "debug" # override - """ - - data: CONFIGURATION_TYPES = Field(default=None) - env: str | None = Field(default=None) - file_path: list[str] | None = Field(default=None) - is_number: bool | None = Field(default=False) - default: CONFIGURATION_TYPES = Field(default=None) - - -class Configuration: - """Hint-based configuration resolver with multi-source lookup. - - Resolves configuration values by checking (in priority order): - 1. Explicit override via ``.set()`` - 2. Environment variable (if ``env`` hint provided) - 3. Config file value (if ``file_path`` hint provided) - 4. Default value (if ``default`` hint provided) - - Args: - config_hints: Mapping of config keys to hint definitions. - Values can be dicts (parsed as ConfigurationHint) or strings - (treated as default values). - config_values: Pre-loaded configuration dict (e.g., from YAML). - config_file_path: Path to YAML config file. Defaults to ./config.yml. - config_base_model: Optional Pydantic BaseSettings for schema generation. - - Example: - >>> config = Configuration({ - ... "url": {"env": "API_URL", "default": "http://localhost"}, - ... "timeout": {"env": "TIMEOUT", "is_number": True, "default": "30"}, - ... }) - >>> config.get("url") - 'http://localhost' - >>> config.get("timeout") - 30 - """ - - def __init__( - self, - config_hints: dict[str, dict[str, Any] | str], - config_values: dict[str, Any] | None = None, - config_file_path: str = os.path.join(os.curdir, "config.yml"), - config_base_model: BaseSettings | None = None, - ): - self.__config_hints: dict[str, ConfigurationHint] = { - key: ( - ConfigurationHint(**value) - if isinstance(value, dict) - else ConfigurationHint(default=value) - ) - for key, value in config_hints.items() - } - - file_contents = ( - yaml.load(open(config_file_path), Loader=yaml.FullLoader) - if os.path.isfile(config_file_path) - else {} - ) - - self.__config_values = (config_values or {}) | file_contents - self.__base_model = config_base_model - - def get(self, config_key: str) -> CONFIGURATION_TYPES: - """Get the resolved value for a configuration key. - - Resolution order: override → env var → file → default. - - Args: - config_key: The configuration key to look up. - - Returns: - The resolved value, or None if the key is unknown or - no source provides a value. - """ - config = self.__config_hints.get(config_key) - if config is None: - return None - - return self.__process_value_to_type( - ( - self.__dig_config_sources_for_key(config) - if config.data is None - else config.data - ), - config.is_number, - ) - - def set(self, config_key: str, value: CONFIGURATION_TYPES) -> None: - """Set a runtime override for a configuration key. - - After calling this, ``get(config_key)`` will return the - overridden value regardless of environment or file sources. - - Args: - config_key: The configuration key to override. - value: The new value to return for this key. - """ - if config_key not in self.__config_hints: - self.__config_hints[config_key] = ConfigurationHint(data=value) - else: - self.__config_hints[config_key].data = value - - def schema(self) -> dict[str, Any]: - """Generate the connector configuration JSON schema. - - Uses the configured base model's Pydantic schema generation - with the ConnectorConfigSchemaGenerator for flattened output. - - Returns: - A JSON schema dict describing all configuration properties. - - Raises: - AttributeError: If no config_base_model was provided. - """ - return self.__base_model.model_json_schema( # type: ignore[union-attr] - by_alias=False, - schema_generator=ConnectorConfigSchemaGenerator, - mode="validation", - ) - - @staticmethod - def __process_value_to_type( - value: CONFIGURATION_TYPES, is_number_hint: bool | None - ) -> CONFIGURATION_TYPES: - if value is None: - return value - if isinstance(value, int) or is_number_hint: - return int(value) - if isinstance(value, str): - if is_truthy(value): - return True - if is_falsy(value): - return False - if len(value) == 0: - return None - return value - - def __dig_config_sources_for_key( - self, config: ConfigurationHint - ) -> CONFIGURATION_TYPES: - result = EnvironmentSource.get(config.env) or DictionarySource.get( - config.file_path, self.__config_values - ) - return result or config.default diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/resolver.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/resolver.py new file mode 100644 index 00000000..26fa5aa5 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/resolver.py @@ -0,0 +1,103 @@ +"""Configuration resolution: override, then env var, then file, then default.""" + +import os.path +from typing import Any + +import yaml +from pydantic_settings import BaseSettings +from xtm_oaev_sdk._core.configuration.sources import ConfigurationSources +from xtm_oaev_sdk._core.configuration.truthiness import Truthiness +from xtm_oaev_sdk.seam.configuration import ( + CONFIGURATION_TYPES, + ConfigurationHint, + ConfigurationSourcesPort, + TruthinessPort, +) + +__all__ = ["ConfigurationResolver"] + + +class ConfigurationResolver: + """Implements ``ConfigurationResolverPort``.""" + + def __init__( + self, + sources: ConfigurationSourcesPort | None = None, + truthiness: TruthinessPort | None = None, + ) -> None: + self._sources = sources or ConfigurationSources() + self._truthiness = truthiness or Truthiness() + + def build_hints( + self, config_hints: dict[str, dict[str, Any] | str] + ) -> dict[str, ConfigurationHint]: + return { + key: ( + ConfigurationHint(**value) + if isinstance(value, dict) + else ConfigurationHint(default=value) + ) + for key, value in config_hints.items() + } + + def load_values( + self, config_values: dict[str, Any] | None, config_file_path: str + ) -> dict[str, Any]: + file_contents = ( + yaml.load(open(config_file_path), Loader=yaml.FullLoader) + if os.path.isfile(config_file_path) + else {} + ) + return (config_values or {}) | file_contents + + def resolve( + self, hint: ConfigurationHint, values: dict[str, Any] + ) -> CONFIGURATION_TYPES: + return self._process_value_to_type( + (self._dig_sources(hint, values) if hint.data is None else hint.data), + hint.is_number, + ) + + def override( + self, + hints: dict[str, ConfigurationHint], + config_key: str, + value: CONFIGURATION_TYPES, + ) -> None: + if config_key not in hints: + hints[config_key] = ConfigurationHint(data=value) + else: + hints[config_key].data = value + + def generate_schema( + self, base_model: BaseSettings | None, schema_generator: type + ) -> dict[str, Any]: + return base_model.model_json_schema( # type: ignore[union-attr] + by_alias=False, + schema_generator=schema_generator, + mode="validation", + ) + + def _process_value_to_type( + self, value: CONFIGURATION_TYPES, is_number_hint: bool | None + ) -> CONFIGURATION_TYPES: + if value is None: + return value + if isinstance(value, int) or is_number_hint: + return int(value) + if isinstance(value, str): + if self._truthiness.is_truthy(value): + return True + if self._truthiness.is_falsy(value): + return False + if len(value) == 0: + return None + return value + + def _dig_sources( + self, hint: ConfigurationHint, values: dict[str, Any] + ) -> CONFIGURATION_TYPES: + result = self._sources.from_environment( + hint.env + ) or self._sources.from_dictionary(hint.file_path, values) + return result or hint.default diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/connector_config_schema_generator.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/schema.py similarity index 53% rename from libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/connector_config_schema_generator.py rename to libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/schema.py index fc250818..f9754739 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/connector_config_schema_generator.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/schema.py @@ -1,55 +1,32 @@ -"""Connector configuration JSON schema generator. - -Provides a custom Pydantic JSON schema generator that produces flattened, -dereferenced schemas suitable for connector configuration manifests. +"""Connector manifest schema: flat, dereferenced, filtered JSON schema. Adapted from the OpenCTI connectors configuration schema generator: https://github.com/OpenCTI-Platform/connectors/blob/5c8cf1235f62f5651c9c08d0b67f1bd182662c8a/shared/tools/composer/generate_connectors_config_schemas/generate_connector_config_json_schema.py.sample """ from copy import deepcopy -from typing import Any, override - -from pydantic.json_schema import GenerateJsonSchema +from typing import Any -__all__ = ["ConnectorConfigSchemaGenerator"] +__all__ = ["ConnectorSchemaTransformer"] +# attributes filtered from the connector configuration before generating the manifest _FILTERED_ATTRIBUTES = [ + # connector id is generated "CONNECTOR_ID", ] -class ConnectorConfigSchemaGenerator(GenerateJsonSchema): - """Custom JSON schema generator for connector configurations. - - Produces a flat schema where all config variables are at root level, - with ``$ref`` entries fully resolved and filtered attributes removed. - - This is used by ``Configuration.schema()`` to produce manifests - compatible with the OpenAEV platform connector registration API. - """ - - @staticmethod - def dereference_schema(schema_with_refs: dict[str, Any]) -> dict[str, Any]: - """Resolve all internal ``$ref`` entries in a JSON schema. - - Args: - schema_with_refs: A JSON schema dict potentially containing - ``$ref`` pointers to ``$defs``. - - Returns: - A new schema with all references inlined. - - Raises: - ValueError: If a ``$ref`` uses an unsupported format. - """ +class ConnectorSchemaTransformer: + """Implements ``ConnectorSchemaPort``.""" + def dereference_schema(self, schema_with_refs: dict[str, Any]) -> dict[str, Any]: def _resolve(schema: Any, root: dict[str, Any]) -> Any: if isinstance(schema, dict): if "$ref" in schema: ref_path = schema["$ref"] if ref_path.startswith("#/$defs/"): def_name = ref_path.split("/")[-1] + # Deep copy to avoid mutating $defs resolved = deepcopy(root["$defs"][def_name]) return _resolve(resolved, root) else: @@ -66,20 +43,9 @@ def _resolve(schema: Any, root: dict[str, Any]) -> Any: return _resolve(deepcopy(schema_with_refs), schema_with_refs) - @staticmethod - def flatten_config_loader_schema(root_schema: dict[str, Any]) -> dict[str, Any]: - """Flatten a hierarchical config schema to root-level properties. - - Transforms nested namespace properties into flat - ``NAMESPACE_VARIABLE`` format suitable for environment variable - mapping. - - Args: - root_schema: The dereferenced schema to flatten. - - Returns: - A flat schema with all properties at root level. - """ + def flatten_config_loader_schema( + self, root_schema: dict[str, Any] + ) -> dict[str, Any]: flat_json_schema: dict[str, Any] = { "$schema": root_schema["$schema"], "$id": root_schema["$id"], @@ -110,16 +76,7 @@ def flatten_config_loader_schema(root_schema: dict[str, Any]) -> dict[str, Any]: return flat_json_schema - @staticmethod - def filter_schema(schema: dict[str, Any]) -> dict[str, Any]: - """Remove filtered attributes from the schema. - - Args: - schema: The flat schema to filter. - - Returns: - The schema with filtered attributes removed. - """ + def filter_schema(self, schema: dict[str, Any]) -> dict[str, Any]: for filtered_attribute in _FILTERED_ATTRIBUTES: if filtered_attribute in schema["properties"]: del schema["properties"][filtered_attribute] @@ -135,40 +92,15 @@ def filter_schema(schema: dict[str, Any]) -> dict[str, Any]: return schema - @override - def generate(self, schema: Any, mode: str = "validation") -> dict[str, Any]: - """Generate a flat, dereferenced, filtered JSON schema. - - Args: - schema: The Pydantic core schema to process. - mode: Schema generation mode. - - Returns: - The final connector configuration schema. - """ - json_schema = super().generate(schema, mode=mode) - - json_schema["$schema"] = self.schema_dialect + def finalize(self, json_schema: dict[str, Any], dialect: str) -> dict[str, Any]: + json_schema["$schema"] = dialect json_schema["$id"] = "config.schema.json" dereferenced_schema = self.dereference_schema(json_schema) flattened_schema = self.flatten_config_loader_schema(dereferenced_schema) return self.filter_schema(flattened_schema) - @override - def nullable_schema(self, schema: Any) -> dict[str, Any]: - """Generate schema for nullable fields without ``anyOf``. - - Overrides the default behavior to produce cleaner schemas - that don't use the ``anyOf`` keyword for optional fields. - - Args: - schema: The core schema for the nullable type. - - Returns: - The generated JSON schema for the nullable type. - """ + def collapse_nullable(self, inner_json_schema: dict[str, Any]) -> dict[str, Any]: null_schema = {"type": "null"} - inner_json_schema = self.generate_inner(schema["schema"]) if inner_json_schema == null_schema: return null_schema diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_sources.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_sources.py new file mode 100644 index 00000000..57a994c2 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_sources.py @@ -0,0 +1,44 @@ +"""Settings sources: environment variables first, then the YAML or .env file.""" + +import os +from pathlib import Path + +from pydantic_settings import ( + BaseSettings, + DotEnvSettingsSource, + PydanticBaseSettingsSource, + YamlConfigSettingsSource, +) + +__all__ = ["SettingsSourceSelector"] + + +class SettingsSourceSelector: + """Implements ``SettingsSourcesPort``.""" + + def select_sources( + self, + settings_cls: type[BaseSettings], + env_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + _main_path = os.curdir + + settings_cls.model_config["env_file"] = f"{_main_path}/../.env" + + if not settings_cls.model_config["yaml_file"]: + if Path(f"{_main_path}/config.yml").is_file(): + settings_cls.model_config["yaml_file"] = f"{_main_path}/config.yml" + if Path(f"{_main_path}/../config.yml").is_file(): + settings_cls.model_config["yaml_file"] = f"{_main_path}/../config.yml" + + if Path(settings_cls.model_config["yaml_file"] or "").is_file(): # type: ignore + return ( + env_settings, + YamlConfigSettingsSource(settings_cls), + ) + if Path(settings_cls.model_config["env_file"] or "").is_file(): # type: ignore + return ( + env_settings, + DotEnvSettingsSource(settings_cls), + ) + return (env_settings,) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py index 263215f4..9f48aac4 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/sources.py @@ -1,69 +1,22 @@ -"""Configuration value sources: environment variables and dictionaries. - -Provides abstractions for resolving configuration values from different -backing stores without coupling the Configuration class to specific I/O. -""" +"""Configuration sources: environment variables and nested dictionaries.""" import os from typing import Any -__all__ = ["DictionarySource", "EnvironmentSource"] - - -class EnvironmentSource: - """Resolves configuration values from environment variables. +__all__ = ["ConfigurationSources"] - Example: - >>> EnvironmentSource.get("HOME") - '/home/user' - >>> EnvironmentSource.get(None) is None - True - """ - @classmethod - def get(cls, env_var: str | None) -> str | None: - """Get the value of an environment variable. +class ConfigurationSources: + """Implements ``ConfigurationSourcesPort``.""" - Args: - env_var: Name of the environment variable to read. - Returns None if env_var is None or empty. - - Returns: - The environment variable value, or None if not set. - """ + def from_environment(self, env_var: str | None) -> str | None: return os.getenv(env_var) if env_var else None - -class DictionarySource: - """Resolves configuration values from nested dictionary structures. - - Handles two-level key paths into a JSON-like configuration dict, - typically loaded from a YAML or JSON config file. - - Example: - >>> source = {"openaev": {"url": "http://localhost:8080"}} - >>> DictionarySource.get(["openaev", "url"], source) - 'http://localhost:8080' - """ - - @classmethod - def get( - cls, config_key_path: list[str] | None, source_dict: dict[str, Any] + # this is quite hacky + # it only strictly handles two levels of keys in a dict + def from_dictionary( + self, config_key_path: list[str] | None, source_dict: dict[str, Any] ) -> str | None: - """Get a value from a nested dict using a two-level key path. - - Args: - config_key_path: A list of exactly 2 non-empty strings representing - the path ``[section, key]`` into the source dict. - source_dict: The nested dictionary to search. - - Returns: - The value at the specified path, or None if not found. - - Raises: - AssertionError: If config_key_path is not a 2-element list of - non-empty strings. - """ if config_key_path is None: return None @@ -72,6 +25,7 @@ def get( and len(config_key_path) == 2 and all([len(path_part) > 0 for path_part in config_key_path]) ) + return source_dict.get(config_key_path[0], {config_key_path[1]: None}).get( config_key_path[1] ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/truthiness.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/truthiness.py new file mode 100644 index 00000000..8eeb16b2 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/truthiness.py @@ -0,0 +1,13 @@ +"""Truthiness: the strings signalling a boolean configuration value.""" + +__all__ = ["Truthiness"] + + +class Truthiness: + """Implements ``TruthinessPort``.""" + + def is_truthy(self, value: str) -> bool: + return value.lower() in ["yes", "true"] + + def is_falsy(self, value: str) -> bool: + return value.lower() in ["no", "false"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py index 97c07f93..09c25f78 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py @@ -4,12 +4,15 @@ from abc import ABC, abstractmethod from inspect import signature from types import FunctionType +from typing import TYPE_CHECKING from xtm_oaev_sdk._core.client_protocol import BaseClient -from xtm_oaev_sdk._core.configuration import Configuration from xtm_oaev_sdk._core.errors import OpenAEVError from xtm_oaev_sdk._core.utils import logger +if TYPE_CHECKING: # annotation only: the core never imports the public layer + from xtm_oaev_sdk.public.configuration import Configuration + __all__ = ["BaseDaemon"] @@ -34,7 +37,7 @@ class BaseDaemon(ABC): def __init__( self, - configuration: Configuration, + configuration: "Configuration", callback: callable = None, logger=None, api_client: BaseClient | None = None, diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index 5e44c646..9b19b22d 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -38,6 +38,26 @@ VariableType, ) +# --- Configuration --- +from xtm_oaev_sdk.public.configuration import ( + CONFIGURATION_TYPES, + BaseConfigModel, + ConfigLoaderCollector, + ConfigLoaderOAEV, + Configuration, + ConfigurationHint, + ConfigurationProtocol, + ConnectorConfigSchemaGenerator, + DictionarySource, + EnvironmentSource, + HttpUrlToString, + LogLevelToLower, + SettingsLoader, + TimedeltaInSeconds, + is_falsy, + is_truthy, +) + # --- Common (shared) --- from xtm_oaev_sdk.seam.common import ( AppLogger, @@ -76,26 +96,6 @@ ssl_cert_chain, ssl_verify_locations, ) - -# --- Configuration --- -from xtm_oaev_sdk.seam.configuration import ( - CONFIGURATION_TYPES, - BaseConfigModel, - ConfigLoaderCollector, - ConfigLoaderOAEV, - Configuration, - ConfigurationHint, - ConfigurationProtocol, - ConnectorConfigSchemaGenerator, - DictionarySource, - EnvironmentSource, - HttpUrlToString, - LogLevelToLower, - SettingsLoader, - TimedeltaInSeconds, - is_falsy, - is_truthy, -) from xtm_oaev_sdk.seam.contracts import ( ContractBuilder, ContractBuilderProtocol, diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/configuration.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/configuration.py new file mode 100644 index 00000000..15447541 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/configuration.py @@ -0,0 +1,380 @@ +"""Configuration adapters: hint-based value resolution and settings loading. + +Each adapter speaks the configuration seam contracts and delegates to the core +implementation of its port. + +Example: + >>> config = Configuration({ + ... "log_level": {"env": "LOG_LEVEL", "default": "info"}, + ... "api_url": {"env": "API_URL", "file_path": ["openaev", "url"]}, + ... }) + >>> config.get("log_level") + 'info' + >>> config.set("log_level", "debug") + >>> config.get("log_level") + 'debug' +""" + +import os +from typing import Any, override + +from pydantic.json_schema import GenerateJsonSchema +from pydantic_settings import ( + BaseSettings, + PydanticBaseSettingsSource, + SettingsConfigDict, +) +from xtm_oaev_sdk._core.configuration import ( + ConfigurationResolver, + ConfigurationSources, + ConnectorSchemaTransformer, + SettingsSourceSelector, + Truthiness, +) +from xtm_oaev_sdk.seam.configuration import ( + CONFIGURATION_TYPES, + BaseConfigModel, + ConfigLoaderCollector, + ConfigLoaderOAEV, + ConfigurationHint, + ConfigurationProtocol, + ConfigurationResolverPort, + ConfigurationSourcesPort, + ConnectorSchemaPort, + HttpUrlToString, + LogLevelToLower, + SettingsSourcesPort, + TimedeltaInSeconds, + TruthinessPort, +) + +__all__ = [ + "BaseConfigModel", + "CONFIGURATION_TYPES", + "ConfigLoaderCollector", + "ConfigLoaderOAEV", + "Configuration", + "ConfigurationHint", + "ConfigurationProtocol", + "ConnectorConfigSchemaGenerator", + "DictionarySource", + "EnvironmentSource", + "HttpUrlToString", + "LogLevelToLower", + "SettingsLoader", + "TimedeltaInSeconds", + "is_falsy", + "is_truthy", +] + +_truthiness: TruthinessPort = Truthiness() +_sources: ConfigurationSourcesPort = ConfigurationSources() +_resolver: ConfigurationResolverPort = ConfigurationResolver(_sources, _truthiness) +_schema: ConnectorSchemaPort = ConnectorSchemaTransformer() +_settings_sources: SettingsSourcesPort = SettingsSourceSelector() + + +def is_truthy(value: str) -> bool: + """Asserts whether a given string signals a "True" value. + + :param value: value to test + :type value: str + + :return: whether the string represents True or not. + :rtype: bool + """ + return _truthiness.is_truthy(value) + + +def is_falsy(value: str) -> bool: + """Asserts whether a given string signals a "False" value. + + :param value: value to test + :type value: str + + :return: whether the string represents False or not. + :rtype: bool + """ + return _truthiness.is_falsy(value) + + +class EnvironmentSource: + """Resolves configuration values from environment variables. + + Example: + >>> EnvironmentSource.get("HOME") + '/home/user' + >>> EnvironmentSource.get(None) is None + True + """ + + @classmethod + def get(cls, env_var: str | None) -> str | None: + """Get the value of an environment variable. + + Args: + env_var: Name of the environment variable to read. + Returns None if env_var is None or empty. + + Returns: + The environment variable value, or None if not set. + """ + return _sources.from_environment(env_var) + + +class DictionarySource: + """Resolves configuration values from nested dictionary structures. + + Handles two-level key paths into a JSON-like configuration dict, + typically loaded from a YAML or JSON config file. + + Example: + >>> source = {"openaev": {"url": "http://localhost:8080"}} + >>> DictionarySource.get(["openaev", "url"], source) + 'http://localhost:8080' + """ + + @classmethod + def get( + cls, config_key_path: list[str] | None, source_dict: dict[str, Any] + ) -> str | None: + """Get a value from a nested dict using a two-level key path. + + Args: + config_key_path: A list of exactly 2 non-empty strings representing + the path ``[section, key]`` into the source dict. + source_dict: The nested dictionary to search. + + Returns: + The value at the specified path, or None if not found. + + Raises: + AssertionError: If config_key_path is not a 2-element list of + non-empty strings. + """ + return _sources.from_dictionary(config_key_path, source_dict) + + +class ConnectorConfigSchemaGenerator(GenerateJsonSchema): + """Custom JSON schema generator for connector configurations. + + Produces a flat schema where all config variables are at root level, + with ``$ref`` entries fully resolved and filtered attributes removed. + + This is used by ``Configuration.schema()`` to produce manifests + compatible with the OpenAEV platform connector registration API. + """ + + @staticmethod + def dereference_schema(schema_with_refs: dict[str, Any]) -> dict[str, Any]: + """Resolve all internal ``$ref`` entries in a JSON schema. + + Args: + schema_with_refs: A JSON schema dict potentially containing + ``$ref`` pointers to ``$defs``. + + Returns: + A new schema with all references inlined. + + Raises: + ValueError: If a ``$ref`` uses an unsupported format. + """ + return _schema.dereference_schema(schema_with_refs) + + @staticmethod + def flatten_config_loader_schema(root_schema: dict[str, Any]) -> dict[str, Any]: + """Flatten a hierarchical config schema to root-level properties. + + Transforms nested namespace properties into flat + ``NAMESPACE_VARIABLE`` format suitable for environment variable + mapping. + + Args: + root_schema: The dereferenced schema to flatten. + + Returns: + A flat schema with all properties at root level. + """ + return _schema.flatten_config_loader_schema(root_schema) + + @staticmethod + def filter_schema(schema: dict[str, Any]) -> dict[str, Any]: + """Remove filtered attributes from the schema. + + Args: + schema: The flat schema to filter. + + Returns: + The schema with filtered attributes removed. + """ + return _schema.filter_schema(schema) + + @override + def generate(self, schema: Any, mode: str = "validation") -> dict[str, Any]: + """Generate a flat, dereferenced, filtered JSON schema. + + Args: + schema: The Pydantic core schema to process. + mode: Schema generation mode. + + Returns: + The final connector configuration schema. + """ + json_schema = super().generate(schema, mode=mode) + return _schema.finalize(json_schema, self.schema_dialect) + + @override + def nullable_schema(self, schema: Any) -> dict[str, Any]: + """Generate schema for nullable fields without ``anyOf``. + + Overrides the default behavior to produce cleaner schemas + that don't use the ``anyOf`` keyword for optional fields. + + Args: + schema: The core schema for the nullable type. + + Returns: + The generated JSON schema for the nullable type. + """ + return _schema.collapse_nullable(self.generate_inner(schema["schema"])) + + +class Configuration: + """Hint-based configuration resolver with multi-source lookup. + + Resolves configuration values by checking (in priority order): + 1. Explicit override via ``.set()`` + 2. Environment variable (if ``env`` hint provided) + 3. Config file value (if ``file_path`` hint provided) + 4. Default value (if ``default`` hint provided) + + Args: + config_hints: Mapping of config keys to hint definitions. + Values can be dicts (parsed as ConfigurationHint) or strings + (treated as default values). + config_values: Pre-loaded configuration dict (e.g., from YAML). + config_file_path: Path to YAML config file. Defaults to ./config.yml. + config_base_model: Optional Pydantic BaseSettings for schema generation. + + Example: + >>> config = Configuration({ + ... "url": {"env": "API_URL", "default": "http://localhost"}, + ... "timeout": {"env": "TIMEOUT", "is_number": True, "default": "30"}, + ... }) + >>> config.get("url") + 'http://localhost' + >>> config.get("timeout") + 30 + """ + + def __init__( + self, + config_hints: dict[str, dict[str, Any] | str], + config_values: dict[str, Any] | None = None, + config_file_path: str = os.path.join(os.curdir, "config.yml"), + config_base_model: BaseSettings | None = None, + ): + self.__config_hints: dict[str, ConfigurationHint] = _resolver.build_hints( + config_hints + ) + self.__config_values = _resolver.load_values(config_values, config_file_path) + self.__base_model = config_base_model + + def get(self, config_key: str) -> CONFIGURATION_TYPES: + """Get the resolved value for a configuration key. + + Resolution order: override → env var → file → default. + + Args: + config_key: The configuration key to look up. + + Returns: + The resolved value, or None if the key is unknown or + no source provides a value. + """ + config = self.__config_hints.get(config_key) + if config is None: + return None + + return _resolver.resolve(config, self.__config_values) + + def set(self, config_key: str, value: CONFIGURATION_TYPES) -> None: + """Set a runtime override for a configuration key. + + After calling this, ``get(config_key)`` will return the + overridden value regardless of environment or file sources. + + Args: + config_key: The configuration key to override. + value: The new value to return for this key. + """ + _resolver.override(self.__config_hints, config_key, value) + + def schema(self) -> dict[str, Any]: + """Generate the connector configuration JSON schema. + + Uses the configured base model's Pydantic schema generation + with the ConnectorConfigSchemaGenerator for flattened output. + + Returns: + A JSON schema dict describing all configuration properties. + + Raises: + AttributeError: If no config_base_model was provided. + """ + return _resolver.generate_schema( + self.__base_model, schema_generator=ConnectorConfigSchemaGenerator + ) + + +class SettingsLoader(BaseSettings): + """Base settings class with prioritized multi-source loading. + + Automatically discovers and loads configuration from environment + variables, YAML files, and .env files with a defined priority order. + + Subclass this to define your connector's configuration schema. + + Example: + >>> class MySettings(SettingsLoader): + ... model_config = SettingsConfigDict(yaml_file="config.yml") + ... api_url: str + ... timeout: int = 30 + """ + + model_config = SettingsConfigDict( + frozen=True, + extra="allow", + env_nested_delimiter="_", + env_nested_max_split=1, + enable_decoding=False, + ) + + @classmethod + def settings_customise_sources( + cls, + settings_cls: type[BaseSettings], + init_settings: PydanticBaseSettingsSource, + env_settings: PydanticBaseSettingsSource, + dotenv_settings: PydanticBaseSettingsSource, + file_secret_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Configure source resolution order. + + Resolution priority: + 1. Environment variables (always checked first) + 2. YAML file (if config.yml exists) + 3. .env file (if .env exists and no YAML found) + 4. Default values (implicit via Pydantic) + + Args: + settings_cls: The settings class being configured. + init_settings: Init-based settings source. + env_settings: Environment variable source. + dotenv_settings: .env file source. + file_secret_settings: Secrets file source. + + Returns: + Tuple of settings sources in priority order. + """ + return _settings_sources.select_sources(settings_cls, env_settings) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py index 6e72e26f..a08651c1 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -5,8 +5,9 @@ - **``common``** — shared errors, enums, types, and utilities: the platform-facing vocabulary both sides import. -- **``configuration``** — the configuration capability's I/O models and - conversion primitives. +- **``configuration``** — the configuration contracts (hints, settings + models, serialization types) and the ports its public adapters delegate + to; closed and versioned (``SEAM_VERSION``). - **``contracts``** — the scenarios capability's behaviour layer (builder, variable helper, preparation entry point) and its extension protocol. - **``daemons``** — the daemon runtime shared by collectors and injectors; @@ -20,15 +21,15 @@ (:func:`make_deprecated_getattr`) for retiring public names without breaking importers. -``seam/signatures/seam.py`` and ``seam/port_io/models.py`` are closed: they -import only the standard library, pydantic, and sibling seam modules — never -``_core``. The capability's implementation imports and re-exports from them. +A seam is **closed**: it imports only the standard library, third-party data +libraries and sibling seam modules — never ``_core`` nor ``public``. The core +implements its ports; the public adapters speak its types and delegate to the +core. As long as the core keeps answering the seam, the public surface keeps +working; a seam change is a contract change (``SEAM_VERSION`` bump, changelog). -The other package ``__init__`` surfaces present ``_core``'s stable vocabulary -through the boundary: they import ``_core`` so the public layer can. The seam -is the **sole door** between ``public/`` and ``_core/``: every public import -of a ``_core`` symbol crosses it. The seam never imports the public layer, -nor does ``_core``. +``configuration``, ``seam/signatures/seam.py`` and ``seam/port_io/models.py`` +are closed today; the other packages still re-export ``_core`` until their +capability is migrated. Distinct from the *business* :mod:`xtm_oaev_sdk.contracts` layer, which holds the OpenAEV scenario data models — a domain concept, unrelated to the diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py index f3934a9e..0e41fca0 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/__init__.py @@ -1,40 +1,48 @@ -"""Configuration feature contracts: the I/O surface plus the concrete -capability layer.""" +"""Configuration seam: the contract between the public adapters and the core. -from xtm_oaev_sdk._core.configuration import ( +Closed: it imports nothing from ``xtm_oaev_sdk._core`` nor +``xtm_oaev_sdk.public``. Any change to these contracts is a breaking change for +one side of the boundary and must bump :data:`SEAM_VERSION`. +""" + +from xtm_oaev_sdk.seam.configuration.models import ( CONFIGURATION_TYPES, BaseConfigModel, ConfigLoaderCollector, ConfigLoaderOAEV, - Configuration, ConfigurationHint, - ConfigurationProtocol, - ConnectorConfigSchemaGenerator, - DictionarySource, - EnvironmentSource, HttpUrlToString, LogLevelToLower, - SettingsLoader, TimedeltaInSeconds, - is_falsy, - is_truthy, +) +from xtm_oaev_sdk.seam.configuration.ports import ( + ConfigurationProtocol, + ConfigurationResolverPort, + ConfigurationSourcesPort, + ConnectorSchemaPort, + SettingsSourcesPort, + TruthinessPort, ) +#: Version of the configuration seam. Bump on any contract change. +SEAM_VERSION = "1.0.0" + __all__ = [ + "SEAM_VERSION", + # Data contracts "BaseConfigModel", "CONFIGURATION_TYPES", "ConfigLoaderCollector", "ConfigLoaderOAEV", - "Configuration", "ConfigurationHint", - "ConfigurationProtocol", - "ConnectorConfigSchemaGenerator", - "DictionarySource", - "EnvironmentSource", "HttpUrlToString", "LogLevelToLower", - "SettingsLoader", "TimedeltaInSeconds", - "is_falsy", - "is_truthy", + # Ports + "ConfigurationProtocol", + "ConfigurationResolverPort", + "ConfigurationSourcesPort", + "ConnectorSchemaPort", + "SettingsSourcesPort", + "TruthinessPort", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_loader.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/models.py similarity index 55% rename from libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_loader.py rename to libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/models.py index 31b260e8..6a25398e 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/configuration/settings_loader.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/models.py @@ -1,41 +1,59 @@ -"""Settings loader: Pydantic-based configuration from env, YAML, and .env files. +"""Configuration data contracts: hints, base models and serialization types. -Provides base models and settings classes for structured, validated -configuration loading with prioritized source resolution. - -Source priority (highest to lowest): - 1. Environment variables - 2. YAML config file (config.yml) - 3. .env file - 4. Default values +Settings models are frozen, allow extra fields for forward compatibility and +validate their defaults. Source priority when loading settings (highest to +lowest): environment variables, YAML config file, .env file, defaults. """ -import os from abc import ABC from datetime import timedelta -from pathlib import Path -from typing import Annotated, Literal +from typing import Annotated, Any, Literal from uuid import UUID from pydantic import BaseModel, ConfigDict, Field, HttpUrl, PlainSerializer -from pydantic_settings import ( - BaseSettings, - DotEnvSettingsSource, - PydanticBaseSettingsSource, - SettingsConfigDict, - YamlConfigSettingsSource, -) __all__ = [ "BaseConfigModel", + "CONFIGURATION_TYPES", "ConfigLoaderCollector", "ConfigLoaderOAEV", + "ConfigurationHint", "HttpUrlToString", "LogLevelToLower", - "SettingsLoader", "TimedeltaInSeconds", ] +CONFIGURATION_TYPES = str | int | bool | Any | None +"""Union type for configuration values.""" + + +class ConfigurationHint(BaseModel): + """An individual configuration hint defining where a value can be found. + + Hints specify the resolution strategy for a configuration key: + which environment variable to check, which file path to follow, + and what default to use if no source provides a value. + + The model is mutable: ``Configuration.set()`` assigns ``data`` in place. + + Args: + data: Override value. When set, this value is returned directly. + env: Environment variable name to read. + file_path: JSON path (nested keys) in the config file. + is_number: Whether to interpret the resolved value as an integer. + default: Fallback value when no source provides one. + + Example: + >>> hint = ConfigurationHint(env="LOG_LEVEL", default="info") + >>> hint.data = "debug" # override + """ + + data: CONFIGURATION_TYPES = Field(default=None) + env: str | None = Field(default=None) + file_path: list[str] | None = Field(default=None) + is_number: bool | None = Field(default=False) + default: CONFIGURATION_TYPES = Field(default=None) + class BaseConfigModel(BaseModel, ABC): """Abstract base for frozen configuration models. @@ -55,79 +73,6 @@ class BaseConfigModel(BaseModel, ABC): model_config = ConfigDict(extra="allow", frozen=True, validate_default=True) -class SettingsLoader(BaseSettings): - """Base settings class with prioritized multi-source loading. - - Automatically discovers and loads configuration from environment - variables, YAML files, and .env files with a defined priority order. - - Subclass this to define your connector's configuration schema. - - Example: - >>> class MySettings(SettingsLoader): - ... model_config = SettingsConfigDict(yaml_file="config.yml") - ... api_url: str - ... timeout: int = 30 - """ - - model_config = SettingsConfigDict( - frozen=True, - extra="allow", - env_nested_delimiter="_", - env_nested_max_split=1, - enable_decoding=False, - ) - - @classmethod - def settings_customise_sources( - cls, - settings_cls: type[BaseSettings], - init_settings: PydanticBaseSettingsSource, - env_settings: PydanticBaseSettingsSource, - dotenv_settings: PydanticBaseSettingsSource, - file_secret_settings: PydanticBaseSettingsSource, - ) -> tuple[PydanticBaseSettingsSource, ...]: - """Configure source resolution order. - - Resolution priority: - 1. Environment variables (always checked first) - 2. YAML file (if config.yml exists) - 3. .env file (if .env exists and no YAML found) - 4. Default values (implicit via Pydantic) - - Args: - settings_cls: The settings class being configured. - init_settings: Init-based settings source. - env_settings: Environment variable source. - dotenv_settings: .env file source. - file_secret_settings: Secrets file source. - - Returns: - Tuple of settings sources in priority order. - """ - _main_path = os.curdir - - settings_cls.model_config["env_file"] = f"{_main_path}/../.env" - - if not settings_cls.model_config["yaml_file"]: - if Path(f"{_main_path}/config.yml").is_file(): - settings_cls.model_config["yaml_file"] = f"{_main_path}/config.yml" - if Path(f"{_main_path}/../config.yml").is_file(): - settings_cls.model_config["yaml_file"] = f"{_main_path}/../config.yml" - - if Path(settings_cls.model_config["yaml_file"] or "").is_file(): # type: ignore - return ( - env_settings, - YamlConfigSettingsSource(settings_cls), - ) - if Path(settings_cls.model_config["env_file"] or "").is_file(): # type: ignore - return ( - env_settings, - DotEnvSettingsSource(settings_cls), - ) - return (env_settings,) - - # Annotated type aliases for common configuration field patterns LogLevelToLower = Annotated[ diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/ports.py new file mode 100644 index 00000000..d5360bec --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/configuration/ports.py @@ -0,0 +1,140 @@ +"""Configuration ports: the behaviour the public layer expects from the core. + +``ConfigurationProtocol`` is the consumer-facing contract every configuration +service satisfies. The other protocols are the ports the public adapters +delegate to; the core implements them. +""" + +from typing import Any, Protocol, runtime_checkable + +from pydantic_settings import BaseSettings, PydanticBaseSettingsSource +from xtm_oaev_sdk.seam.configuration.models import ( + CONFIGURATION_TYPES, + ConfigurationHint, +) + +__all__ = [ + "ConfigurationProtocol", + "ConfigurationResolverPort", + "ConfigurationSourcesPort", + "ConnectorSchemaPort", + "SettingsSourcesPort", + "TruthinessPort", +] + + +@runtime_checkable +class ConfigurationProtocol(Protocol): + """Behavioral contract for configuration services. + + Any configuration implementation used by SDK consumers must satisfy + this protocol. The built-in Configuration class implements it. + + Methods: + get: Retrieve a typed configuration value by key. + set: Override a configuration value at runtime. + schema: Generate the connector configuration JSON schema. + """ + + def get(self, config_key: str) -> CONFIGURATION_TYPES: + """Get configuration value by key.""" + ... + + def set(self, config_key: str, value: CONFIGURATION_TYPES) -> None: + """Set a runtime override for a configuration key.""" + ... + + def schema(self) -> dict[str, Any]: + """Generate the configuration JSON schema.""" + ... + + +@runtime_checkable +class TruthinessPort(Protocol): + """Interprets the strings signalling a boolean value.""" + + def is_truthy(self, value: str) -> bool: + """Whether ``value`` signals True.""" + + def is_falsy(self, value: str) -> bool: + """Whether ``value`` signals False.""" + + +@runtime_checkable +class ConfigurationSourcesPort(Protocol): + """Reads raw configuration values from their backing stores.""" + + def from_environment(self, env_var: str | None) -> str | None: + """Value of an environment variable, None when unset or unnamed.""" + + def from_dictionary( + self, config_key_path: list[str] | None, source_dict: dict[str, Any] + ) -> str | None: + """Value at a two-level ``[section, key]`` path of a nested dict.""" + + +@runtime_checkable +class ConfigurationResolverPort(Protocol): + """Resolves configuration values from hints and loaded values.""" + + def build_hints( + self, config_hints: dict[str, dict[str, Any] | str] + ) -> dict[str, ConfigurationHint]: + """Hints from their definitions (a dict, or a bare default value).""" + + def load_values( + self, config_values: dict[str, Any] | None, config_file_path: str + ) -> dict[str, Any]: + """Given values merged with the YAML file contents, when it exists.""" + + def resolve( + self, hint: ConfigurationHint, values: dict[str, Any] + ) -> CONFIGURATION_TYPES: + """Typed value of a hint: override, then env var, file, default.""" + + def override( + self, + hints: dict[str, ConfigurationHint], + config_key: str, + value: CONFIGURATION_TYPES, + ) -> None: + """Set a runtime override, creating the hint when unknown.""" + + def generate_schema( + self, base_model: BaseSettings | None, schema_generator: type + ) -> dict[str, Any]: + """Connector configuration JSON schema of ``base_model``.""" + + +@runtime_checkable +class ConnectorSchemaPort(Protocol): + """Transforms a pydantic JSON schema into a connector manifest schema.""" + + def dereference_schema(self, schema_with_refs: dict[str, Any]) -> dict[str, Any]: + """Schema with every internal ``$ref`` inlined.""" + + def flatten_config_loader_schema( + self, root_schema: dict[str, Any] + ) -> dict[str, Any]: + """Schema with every config variable at root level.""" + + def filter_schema(self, schema: dict[str, Any]) -> dict[str, Any]: + """Schema without the attributes generated by the platform.""" + + def finalize(self, json_schema: dict[str, Any], dialect: str) -> dict[str, Any]: + """Flat, dereferenced, filtered manifest schema.""" + + def collapse_nullable(self, inner_json_schema: dict[str, Any]) -> dict[str, Any]: + """Schema of a nullable field, without ``anyOf``.""" + + +@runtime_checkable +class SettingsSourcesPort(Protocol): + """Chooses where settings are loaded from, in priority order.""" + + def select_sources( + self, + settings_cls: type[BaseSettings], + env_settings: PydanticBaseSettingsSource, + ) -> tuple[PydanticBaseSettingsSource, ...]: + """Environment variables first, then the YAML or .env file found.""" From ffdfac23e88d198bd05573286e78d2084d68b2a7 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 15:11:24 +0200 Subject: [PATCH 09/22] :recycle: refactor(sdk): enhancing port-adapter public boundaries for contracts --- libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 4 +- .../xtm_oaev_sdk/_core/contracts/__init__.py | 87 +---- .../xtm_oaev_sdk/_core/contracts/assembly.py | 27 ++ .../_core/contracts/contract_builder.py | 172 ---------- .../_core/contracts/contract_utils.py | 57 ---- .../_core/contracts/credential_types.py | 18 - .../_core/contracts/credentials.py | 44 ++- .../_core/contracts/serializer.py | 76 +++++ .../{variable_helper.py => variables.py} | 45 +-- .../xtm_oaev_sdk/contracts/__init__.py | 6 - .../contracts/contracts/__init__.py | 73 ----- .../xtm_oaev_sdk/public/__init__.py | 54 ++- .../xtm_oaev_sdk/public/contracts.py | 308 ++++++++++++++++++ .../xtm_oaev_sdk/seam/__init__.py | 14 +- .../xtm_oaev_sdk/seam/contracts/__init__.py | 103 +++++- .../contracts/models.py} | 165 ++++------ .../xtm_oaev_sdk/seam/contracts/ports.py | 125 +++++++ 17 files changed, 770 insertions(+), 608 deletions(-) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/assembly.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_builder.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_utils.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credential_types.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py rename libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/{variable_helper.py => variables.py} (80%) delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/__init__.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/contracts/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/public/contracts.py rename libs/xtm-oaev-sdk/xtm_oaev_sdk/{_core/contracts/contract_config.py => seam/contracts/models.py} (81%) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/ports.py diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py index e2fc883d..c01f44b5 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -9,13 +9,13 @@ _core/signatures/ Signature feature implementation seam/common/ Shared errors, enums, types, and utilities seam/configuration/ Configuration contracts and ports (closed, versioned) - seam/contracts/ Scenarios capability behaviour + builder protocol + seam/contracts/ Scenario contracts and ports (closed, versioned) seam/daemons/ Daemon runtime (API client injected) seam/signatures/ Signatures inbound I/O seam (versioned) seam/port_io/ Port I/O wire schema (versioned, seam-owned) seam/deprecation.py PEP 562 deprecation primitive - contracts/contracts/ OpenAEV scenario data models (business layer) public/configuration Configuration adapters + public/contracts Scenario contract adapters public/ User-facing re-exports Port/adapter layering, capability by capability (see ``seam``): diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py index 6688b45d..72299378 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/__init__.py @@ -1,82 +1,13 @@ -"""Contract building: field definitions, validation, and serialization. +"""Contracts core: implementations of the contracts seam ports.""" -Re-exports the public contract surface for internal engine use. -Public consumers import from xtm_oaev_sdk directly. -""" - -from .contract_builder import ContractBuilder, ContractBuilderProtocol -from .contract_config import ( - Contract, - ContractAiTarget, - ContractAsset, - ContractAssetGroup, - ContractAttachment, - ContractCardinalityElement, - ContractCheckbox, - ContractConfig, - ContractElement, - ContractExpectations, - ContractExpectationType, - ContractFieldKey, - ContractFieldType, - ContractOutputElement, - ContractOutputType, - ContractPayload, - ContractReferencedCredential, - ContractSelect, - ContractTeam, - ContractText, - ContractTextArea, - ContractTuple, - Domain, - Expectation, - LinkedFieldModel, - PrimitiveType, - SecurityPlatformType, - SupportedLanguage, - prepare_contracts, -) -from .contract_utils import ContractCardinality, ContractVariable, VariableType -from .credential_types import CredentialType -from .credentials import build_single_referenced_credential_element -from .variable_helper import VariableHelper +from .assembly import ContractAssembly +from .credentials import CredentialReferences +from .serializer import ContractSerializer +from .variables import BuiltinContractVariables __all__ = [ - "Contract", - "ContractAiTarget", - "ContractAsset", - "ContractAssetGroup", - "ContractAttachment", - "ContractBuilder", - "ContractBuilderProtocol", - "ContractCardinality", - "ContractCardinalityElement", - "ContractCheckbox", - "ContractConfig", - "ContractElement", - "ContractExpectationType", - "ContractExpectations", - "ContractFieldKey", - "ContractFieldType", - "ContractOutputElement", - "ContractOutputType", - "ContractPayload", - "ContractReferencedCredential", - "ContractSelect", - "ContractTeam", - "ContractText", - "ContractTextArea", - "ContractTuple", - "ContractVariable", - "CredentialType", - "Domain", - "Expectation", - "LinkedFieldModel", - "PrimitiveType", - "SecurityPlatformType", - "SupportedLanguage", - "VariableHelper", - "VariableType", - "build_single_referenced_credential_element", - "prepare_contracts", + "BuiltinContractVariables", + "ContractAssembly", + "ContractSerializer", + "CredentialReferences", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/assembly.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/assembly.py new file mode 100644 index 00000000..ef0092b7 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/assembly.py @@ -0,0 +1,27 @@ +"""Contract assembly: what a builder does to the fields it collects.""" + +from typing import Any + +from xtm_oaev_sdk.seam.contracts import ContractElement + +__all__ = ["ContractAssembly"] + + +class ContractAssembly: + """Implements ``ContractAssemblyPort``.""" + + def concat(self, existing: list[Any], extra: list[Any]) -> list[Any]: + return existing + extra + + def set_mandatory( + self, element: ContractElement, mandatory: bool + ) -> ContractElement: + element.mandatory = mandatory + return element + + def require_group(self, elements: list[ContractElement]) -> list[ContractElement]: + keys: list[str] = list(map(lambda iterable: iterable.key, elements)) + for element in elements: + element.mandatory = True + element.mandatoryGroups = keys + return elements diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_builder.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_builder.py deleted file mode 100644 index bb5374aa..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_builder.py +++ /dev/null @@ -1,172 +0,0 @@ -"""Contract builder: fluent API for constructing contract field sets. - -Provides the ContractBuilder class and its Protocol for building -contract input field and output declarations with a chainable API. - -Example: - >>> builder = ContractBuilder() - >>> builder.mandatory(ContractText(key="name", label="Name")) - >>> builder.optional(ContractCheckbox(key="verbose", label="Verbose")) - >>> fields = builder.build_fields() -""" - -from typing import Protocol, runtime_checkable - -from xtm_oaev_sdk._core.contracts.contract_config import ( - ContractElement, - ContractOutputElement, -) - -__all__ = [ - "ContractBuilder", - "ContractBuilderProtocol", -] - - -@runtime_checkable -class ContractBuilderProtocol(Protocol): - """Behavioral contract for contract field builders. - - Defines the chainable API for constructing contract field sets. - Consumers can code against this protocol for testability. - """ - - def add_fields(self, fields: list[ContractElement]) -> "ContractBuilderProtocol": - """Add multiple fields at once.""" - ... - - def add_outputs( - self, outputs: list[ContractOutputElement] - ) -> "ContractBuilderProtocol": - """Add multiple output declarations at once.""" - ... - - def mandatory(self, element: ContractElement) -> "ContractBuilderProtocol": - """Add a mandatory field.""" - ... - - def optional(self, element: ContractElement) -> "ContractBuilderProtocol": - """Add an optional field.""" - ... - - def mandatory_group( - self, elements: list[ContractElement] - ) -> "ContractBuilderProtocol": - """Add a group of mutually-required fields.""" - ... - - def build_fields(self) -> list[ContractElement]: - """Return the accumulated fields.""" - ... - - def build_outputs(self) -> list[ContractOutputElement]: - """Return the accumulated outputs.""" - ... - - -class ContractBuilder: - """Fluent builder for contract field and output declarations. - - Provides a chainable API to construct contract definitions - with mandatory, optional, and grouped fields. - - Example: - >>> b = ContractBuilder() - >>> b.mandatory(ContractText(key="target", label="Target URL")) - >>> b.optional(ContractCheckbox(key="verify_ssl", label="Verify SSL")) - >>> fields = b.build_fields() - >>> len(fields) - 2 - """ - - fields: list[ContractElement] - outputs: list[ContractOutputElement] - - def __init__(self) -> None: - self.fields = [] - self.outputs = [] - - def add_fields(self, fields: list[ContractElement]) -> "ContractBuilder": - """Add multiple fields at once. - - Args: - fields: List of contract elements to add. - - Returns: - Self for chaining. - """ - self.fields = self.fields + fields - return self - - def add_outputs(self, outputs: list[ContractOutputElement]) -> "ContractBuilder": - """Add multiple output declarations at once. - - Args: - outputs: List of output elements to add. - - Returns: - Self for chaining. - """ - self.outputs = self.outputs + outputs - return self - - def mandatory(self, element: ContractElement) -> "ContractBuilder": - """Add a mandatory field. - - Args: - element: The contract element to mark as mandatory. - - Returns: - Self for chaining. - """ - element.mandatory = True - self.fields.append(element) - return self - - def optional(self, element: ContractElement) -> "ContractBuilder": - """Add an optional field. - - Args: - element: The contract element to mark as optional. - - Returns: - Self for chaining. - """ - element.mandatory = False - self.fields.append(element) - return self - - def mandatory_group(self, elements: list[ContractElement]) -> "ContractBuilder": - """Add a group of mutually-required fields. - - All fields in the group reference each other's keys and are - marked mandatory. - - Args: - elements: List of contract elements forming the group. - - Returns: - Self for chaining. - """ - keys: list[str] = list(map(lambda iterable: iterable.key, elements)) - for element in elements: - element.mandatory = True - element.mandatoryGroups = keys - self.fields.append(element) - return self - - def build_fields(self) -> list[ContractElement]: - """Return all accumulated contract fields. - - Returns: - List of ContractElement instances added via this builder. - """ - return self.fields - - def build_outputs(self) -> list[ContractOutputElement]: - """Return all accumulated output declarations. - - Returns: - List of ContractOutputElement instances added via this builder. - """ - return self.outputs diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_utils.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_utils.py deleted file mode 100644 index a795617e..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_utils.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Contract utility types: cardinality and variable definitions. - -Provides foundational types used by the contract building system -for defining field cardinality and template variables. -""" - -from dataclasses import dataclass, field -from enum import Enum - -__all__ = [ - "ContractCardinality", - "ContractVariable", - "VariableType", -] - - -class ContractCardinality(str, Enum): - """Field cardinality in a contract definition. - - Attributes: - One: Single value expected. - Multiple: Multiple values (list) expected. - """ - - One: str = "1" - Multiple: str = "n" - - -class VariableType(str, Enum): - """Variable type classification for contract template variables. - - Attributes: - String: Simple string value. - Object: Nested object with children. - """ - - String: str = "String" - Object: str = "Object" - - -@dataclass -class ContractVariable: - """A template variable available for injection into contract fields. - - Args: - key: Variable identifier (dot-notation for nested, e.g. "user.email"). - label: Human-readable description of the variable. - type: Variable type (String or Object). - cardinality: Whether this variable holds one or many values. - children: Nested child variables (for Object type). - """ - - key: str - label: str - type: VariableType - cardinality: ContractCardinality - children: list["ContractVariable"] = field(default_factory=list) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credential_types.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credential_types.py deleted file mode 100644 index 282b67cc..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credential_types.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Credential reference types shared with the OpenAEV platform. - -This enum is the Python mirror of the OpenAEV backend credential-type enum -used by credential secret references. Values must stay label-for-label in sync -with the platform so contracts can declare credential references without any -translation layer. -""" - -from enum import Enum - -__all__ = ["CredentialType"] - - -class CredentialType(str, Enum): - IDENTITY = "IDENTITY" - CLOUD_AWS = "CLOUD_AWS" - CLOUD_AZURE = "CLOUD_AZURE" - CLOUD_GCP = "CLOUD_GCP" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py index f342b350..5ec25383 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/credentials.py @@ -1,17 +1,16 @@ -"""Helpers centralizing how injectors declare credential-reference fields. +"""Credential-reference fields: provider-to-credential-type mapping. -Injectors can call ``build_single_referenced_credential_element`` instead of -re-implementing provider-to-credential-type mapping in each project. This keeps -contract generation consistent across the Python ecosystem and aligns the +Injectors call ``build_single_referenced_credential_element`` instead of +re-implementing the provider-to-credential-type mapping in each project. This +keeps contract generation consistent across the Python ecosystem and aligns the serialized contract payload with the OpenAEV platform's expected values. """ from typing import Optional -from xtm_oaev_sdk._core.contracts.contract_config import ContractReferencedCredential -from xtm_oaev_sdk._core.contracts.credential_types import CredentialType +from xtm_oaev_sdk.seam.contracts import ContractReferencedCredential, CredentialType -__all__ = ["build_single_referenced_credential_element"] +__all__ = ["CredentialReferences"] _PROVIDER_TO_CREDENTIAL_TYPE = { "aws": CredentialType.CLOUD_AWS, @@ -21,23 +20,18 @@ } -def _resolve_credential_type(provider_name: str) -> Optional[CredentialType]: - normalized_provider = provider_name.casefold() - return _PROVIDER_TO_CREDENTIAL_TYPE.get(normalized_provider) +class CredentialReferences: + """Implements ``ContractCredentialsPort``.""" + def build_single_referenced_credential_element( + self, provider_name: str + ) -> ContractReferencedCredential: + return ContractReferencedCredential( + credential_reference_type=self._resolve_credential_type(provider_name), + multiple=False, + ) -def build_single_referenced_credential_element( - provider_name: str, -) -> ContractReferencedCredential: - """Build a credential-reference field for a provider-specific contract - with ``multiple`` value at ``False``. - - This is the centralized entry point injectors should use when they need the - OpenAEV inject form to ask for one referenced credential. The helper keeps - provider-to-``CredentialType`` mapping consistent across injector projects. - """ - - return ContractReferencedCredential( - credential_reference_type=_resolve_credential_type(provider_name), - multiple=False, - ) + @staticmethod + def _resolve_credential_type(provider_name: str) -> Optional[CredentialType]: + normalized_provider = provider_name.casefold() + return _PROVIDER_TO_CREDENTIAL_TYPE.get(normalized_provider) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py new file mode 100644 index 00000000..8c31b137 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py @@ -0,0 +1,76 @@ +"""Contract serialization into platform API payloads.""" + +import json +from typing import Any + +from xtm_oaev_sdk._core.utils import EnhancedJSONEncoder +from xtm_oaev_sdk.seam.contracts import ContractData + +__all__ = ["ContractSerializer"] + + +class ContractSerializer: + """Implements ``ContractSerializerPort``.""" + + def to_add_input(self, contract: ContractData, source_id: str) -> dict[str, Any]: + """Serialize a contract for the platform creation API. + + Args: + source_id: The injector ID that owns this contract. + + Returns: + Dict payload for the contract creation endpoint. + """ + return { + "contract_id": contract.contract_id, + "external_contract_id": contract.external_id, + "injector_id": source_id, + "contract_manual": contract.manual, + "contract_labels": contract.label, + "contract_attack_patterns_external_ids": contract.contract_attack_patterns_external_ids, + "contract_vulnerability_external_ids": contract.contract_vulnerability_external_ids, + "contract_content": json.dumps(contract, cls=EnhancedJSONEncoder), + "is_atomic_testing": contract.is_atomic_testing, + "contract_platforms": contract.platforms, + "contract_domains": contract.domains, + } + + def to_update_input(self, contract: ContractData) -> dict[str, Any]: + """Serialize a contract for the platform update API. + + Returns: + Dict payload for the contract update endpoint. + """ + return { + "contract_manual": contract.manual, + "contract_labels": contract.label, + "contract_attack_patterns_external_ids": contract.contract_attack_patterns_external_ids, + "contract_vulnerability_external_ids": contract.contract_vulnerability_external_ids, + "contract_content": json.dumps(contract, cls=EnhancedJSONEncoder), + "is_atomic_testing": contract.is_atomic_testing, + "contract_platforms": contract.platforms, + "contract_domains": contract.domains, + } + + def prepare(self, contracts: list[ContractData]) -> list[dict[str, Any]]: + """Serialize a list of contracts for bulk platform registration. + + Args: + contracts: List of Contract instances to serialize. + + Returns: + List of dicts suitable for the bulk contract registration API. + """ + return list( + map( + lambda c: { + "contract_id": c.contract_id, + "contract_labels": c.label, + "contract_attack_patterns_external_ids": c.contract_attack_patterns_external_ids, + "contract_content": json.dumps(c, cls=EnhancedJSONEncoder), + "contract_platforms": c.platforms, + "contract_domains": c.domains, + }, + contracts, + ) + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variable_helper.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variables.py similarity index 80% rename from libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variable_helper.py rename to libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variables.py index d1be8c4a..bd859cf1 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variable_helper.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/variables.py @@ -1,16 +1,12 @@ -"""Variable helper: pre-built contract template variables. +"""Built-in contract template variables (user, exercise, team, platform URIs).""" -Provides factory methods for standard template variables (user, exercise, -team, URIs) that are injected into contract fields during simulation. -""" - -from xtm_oaev_sdk._core.contracts.contract_utils import ( +from xtm_oaev_sdk.seam.contracts import ( ContractCardinality, ContractVariable, VariableType, ) -__all__ = ["VariableHelper"] +__all__ = ["BuiltinContractVariables"] USER = "user" EXERCISE = "exercise" @@ -22,22 +18,10 @@ LESSONS_URI = "lessons_uri" -class VariableHelper: - """Factory for standard contract template variables. - - Provides pre-built variable definitions for common injection - targets (users, exercises, teams, platform URIs). - - Example: - >>> variables = [ - ... VariableHelper.user_variable(), - ... VariableHelper.exercise_variable(), - ... VariableHelper.team_variable(), - ... ] + VariableHelper.uri_variables() - """ +class BuiltinContractVariables: + """Implements ``ContractVariablesPort``.""" - @staticmethod - def user_variable() -> ContractVariable: + def user_variable(self) -> ContractVariable: """Create the user template variable with standard children.""" return ContractVariable( key=USER, @@ -78,8 +62,7 @@ def user_variable() -> ContractVariable: ], ) - @staticmethod - def exercise_variable() -> ContractVariable: + def exercise_variable(self) -> ContractVariable: """Create the exercise template variable with standard children.""" return ContractVariable( key=EXERCISE, @@ -108,8 +91,7 @@ def exercise_variable() -> ContractVariable: ], ) - @staticmethod - def team_variable() -> ContractVariable: + def team_variable(self) -> ContractVariable: """Create the teams template variable.""" return ContractVariable( key=TEAMS, @@ -118,8 +100,7 @@ def team_variable() -> ContractVariable: cardinality=ContractCardinality.Multiple, ) - @staticmethod - def uri_variables() -> list[ContractVariable]: + def uri_variables(self) -> list[ContractVariable]: """Create platform URI template variables.""" return [ ContractVariable( @@ -147,3 +128,11 @@ def uri_variables() -> list[ContractVariable]: cardinality=ContractCardinality.One, ), ] + + def default_variables(self) -> list[ContractVariable]: + """Create the variables every contract starts with.""" + return [ + self.user_variable(), + self.exercise_variable(), + self.team_variable(), + ] + self.uri_variables() diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/__init__.py deleted file mode 100644 index 83be2c06..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Business contracts layer: the OpenAEV scenario data models. - -The live member is :mod:`xtm_oaev_sdk.contracts.contracts` — the domain -models describing OpenAEV scenarios / contracts. It is a domain concept, -distinct from the architectural :mod:`xtm_oaev_sdk.seam` boundary. -""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/contracts/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/contracts/__init__.py deleted file mode 100644 index 05bc962e..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/contracts/contracts/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -"""OpenAEV contracts/scenarios data models.""" - -from xtm_oaev_sdk._core.contracts import ( - Contract, - ContractAiTarget, - ContractAsset, - ContractAssetGroup, - ContractAttachment, - ContractBuilderProtocol, - ContractCardinality, - ContractCardinalityElement, - ContractCheckbox, - ContractConfig, - ContractElement, - ContractExpectations, - ContractExpectationType, - ContractFieldKey, - ContractFieldType, - ContractOutputElement, - ContractOutputType, - ContractPayload, - ContractReferencedCredential, - ContractSelect, - ContractTeam, - ContractText, - ContractTextArea, - ContractTuple, - ContractVariable, - CredentialType, - Domain, - Expectation, - LinkedFieldModel, - PrimitiveType, - SecurityPlatformType, - SupportedLanguage, - VariableType, -) - -__all__ = [ - "Contract", - "ContractAiTarget", - "ContractAsset", - "ContractAssetGroup", - "ContractAttachment", - "ContractBuilderProtocol", - "ContractCardinality", - "ContractCardinalityElement", - "ContractCheckbox", - "ContractConfig", - "ContractElement", - "ContractExpectationType", - "ContractExpectations", - "ContractFieldKey", - "ContractFieldType", - "ContractOutputElement", - "ContractOutputType", - "ContractPayload", - "ContractReferencedCredential", - "ContractSelect", - "ContractTeam", - "ContractText", - "ContractTextArea", - "ContractTuple", - "ContractVariable", - "CredentialType", - "Domain", - "Expectation", - "LinkedFieldModel", - "PrimitiveType", - "SecurityPlatformType", - "SupportedLanguage", - "VariableType", -] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index 9b19b22d..e2d29fb7 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -2,13 +2,35 @@ __version__ = "0.1.0" +# --- Configuration --- +from xtm_oaev_sdk.public.configuration import ( + CONFIGURATION_TYPES, + BaseConfigModel, + ConfigLoaderCollector, + ConfigLoaderOAEV, + Configuration, + ConfigurationHint, + ConfigurationProtocol, + ConnectorConfigSchemaGenerator, + DictionarySource, + EnvironmentSource, + HttpUrlToString, + LogLevelToLower, + SettingsLoader, + TimedeltaInSeconds, + is_falsy, + is_truthy, +) + # --- Contracts (scenarios) --- -from xtm_oaev_sdk.contracts.contracts import ( +from xtm_oaev_sdk.public.contracts import ( Contract, ContractAiTarget, ContractAsset, ContractAssetGroup, ContractAttachment, + ContractBuilder, + ContractBuilderProtocol, ContractCardinality, ContractCardinalityElement, ContractCheckbox, @@ -35,27 +57,10 @@ PrimitiveType, SecurityPlatformType, SupportedLanguage, + VariableHelper, VariableType, -) - -# --- Configuration --- -from xtm_oaev_sdk.public.configuration import ( - CONFIGURATION_TYPES, - BaseConfigModel, - ConfigLoaderCollector, - ConfigLoaderOAEV, - Configuration, - ConfigurationHint, - ConfigurationProtocol, - ConnectorConfigSchemaGenerator, - DictionarySource, - EnvironmentSource, - HttpUrlToString, - LogLevelToLower, - SettingsLoader, - TimedeltaInSeconds, - is_falsy, - is_truthy, + build_single_referenced_credential_element, + prepare_contracts, ) # --- Common (shared) --- @@ -96,13 +101,6 @@ ssl_cert_chain, ssl_verify_locations, ) -from xtm_oaev_sdk.seam.contracts import ( - ContractBuilder, - ContractBuilderProtocol, - VariableHelper, - build_single_referenced_credential_element, - prepare_contracts, -) # --- Daemons --- from xtm_oaev_sdk.seam.daemons import BaseDaemon, PingAlive diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/contracts.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/contracts.py new file mode 100644 index 00000000..8cfdd063 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/contracts.py @@ -0,0 +1,308 @@ +"""Contract adapters: contract definitions, field builder, template variables. + +Each adapter speaks the contracts seam types and delegates to the core +implementation of its port. + +Example: + >>> builder = ContractBuilder() + >>> builder.mandatory(ContractText(key="name", label="Name")) + >>> builder.optional(ContractCheckbox(key="verbose", label="Verbose")) + >>> fields = builder.build_fields() +""" + +from dataclasses import dataclass, field +from typing import Any + +from xtm_oaev_sdk._core.contracts import ( + BuiltinContractVariables, + ContractAssembly, + ContractSerializer, + CredentialReferences, +) +from xtm_oaev_sdk.seam.contracts import ( + ContractAiTarget, + ContractAssemblyPort, + ContractAsset, + ContractAssetGroup, + ContractAttachment, + ContractBuilderProtocol, + ContractCardinality, + ContractCardinalityElement, + ContractCheckbox, + ContractConfig, + ContractCredentialsPort, + ContractData, + ContractElement, + ContractExpectations, + ContractExpectationType, + ContractFieldKey, + ContractFieldType, + ContractOutputElement, + ContractOutputType, + ContractPayload, + ContractReferencedCredential, + ContractSelect, + ContractSerializerPort, + ContractTeam, + ContractText, + ContractTextArea, + ContractTuple, + ContractVariable, + ContractVariablesPort, + CredentialType, + Domain, + Expectation, + LinkedFieldModel, + PrimitiveType, + SecurityPlatformType, + SupportedLanguage, + VariableType, +) + +__all__ = [ + "Contract", + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractBuilder", + "ContractBuilderProtocol", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractElement", + "ContractExpectationType", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "CredentialType", + "Domain", + "Expectation", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableHelper", + "VariableType", + "build_single_referenced_credential_element", + "prepare_contracts", +] + +_assembly: ContractAssemblyPort = ContractAssembly() +_variables: ContractVariablesPort = BuiltinContractVariables() +_serializer: ContractSerializerPort = ContractSerializer() +_credentials: ContractCredentialsPort = CredentialReferences() + + +class VariableHelper: + """Factory for standard contract template variables. + + Provides pre-built variable definitions for common injection + targets (users, exercises, teams, platform URIs). + + Example: + >>> variables = [ + ... VariableHelper.user_variable(), + ... VariableHelper.exercise_variable(), + ... VariableHelper.team_variable(), + ... ] + VariableHelper.uri_variables() + """ + + @staticmethod + def user_variable() -> ContractVariable: + """Create the user template variable with standard children.""" + return _variables.user_variable() + + @staticmethod + def exercise_variable() -> ContractVariable: + """Create the exercise template variable with standard children.""" + return _variables.exercise_variable() + + @staticmethod + def team_variable() -> ContractVariable: + """Create the teams template variable.""" + return _variables.team_variable() + + @staticmethod + def uri_variables() -> list[ContractVariable]: + """Create platform URI template variables.""" + return _variables.uri_variables() + + +@dataclass +class Contract(ContractData): + """A complete contract definition for an injector or collector. + + Contains fields (inputs), outputs, configuration, and metadata + for registering the contract with the OpenAEV platform. + + Args: + contract_id: Unique contract identifier. + label: Localized contract labels. + fields: Input field definitions. + outputs: Output field declarations. + config: Display configuration. + manual: Whether this is a manual-execution contract. + variables: Template variables available for injection. + """ + + variables: list[ContractVariable] = field( + default_factory=lambda: _variables.default_variables() + ) + + def to_contract_add_input(self, source_id: str) -> dict[str, Any]: + """Serialize this contract for the platform creation API. + + Args: + source_id: The injector ID that owns this contract. + + Returns: + Dict payload for the contract creation endpoint. + """ + return _serializer.to_add_input(self, source_id) + + def to_contract_update_input(self) -> dict[str, Any]: + """Serialize this contract for the platform update API. + + Returns: + Dict payload for the contract update endpoint. + """ + return _serializer.to_update_input(self) + + +def prepare_contracts(contracts: list[Contract]) -> list[dict[str, Any]]: + """Serialize a list of contracts for bulk platform registration. + + Args: + contracts: List of Contract instances to serialize. + + Returns: + List of dicts suitable for the bulk contract registration API. + """ + return _serializer.prepare(contracts) + + +def build_single_referenced_credential_element( + provider_name: str, +) -> ContractReferencedCredential: + """Build a credential-reference field for a provider-specific contract + with ``multiple`` value at ``False``. + + This is the centralized entry point injectors should use when they need the + OpenAEV inject form to ask for one referenced credential. The helper keeps + provider-to-``CredentialType`` mapping consistent across injector projects. + """ + return _credentials.build_single_referenced_credential_element(provider_name) + + +class ContractBuilder: + """Fluent builder for contract field and output declarations. + + Provides a chainable API to construct contract definitions + with mandatory, optional, and grouped fields. + + Example: + >>> b = ContractBuilder() + >>> b.mandatory(ContractText(key="target", label="Target URL")) + >>> b.optional(ContractCheckbox(key="verify_ssl", label="Verify SSL")) + >>> fields = b.build_fields() + >>> len(fields) + 2 + """ + + fields: list[ContractElement] + outputs: list[ContractOutputElement] + + def __init__(self) -> None: + self.fields = [] + self.outputs = [] + + def add_fields(self, fields: list[ContractElement]) -> "ContractBuilder": + """Add multiple fields at once. + + Args: + fields: List of contract elements to add. + + Returns: + Self for chaining. + """ + self.fields = _assembly.concat(self.fields, fields) + return self + + def add_outputs(self, outputs: list[ContractOutputElement]) -> "ContractBuilder": + """Add multiple output declarations at once. + + Args: + outputs: List of output elements to add. + + Returns: + Self for chaining. + """ + self.outputs = _assembly.concat(self.outputs, outputs) + return self + + def mandatory(self, element: ContractElement) -> "ContractBuilder": + """Add a mandatory field. + + Args: + element: The contract element to mark as mandatory. + + Returns: + Self for chaining. + """ + self.fields.append(_assembly.set_mandatory(element, True)) + return self + + def optional(self, element: ContractElement) -> "ContractBuilder": + """Add an optional field. + + Args: + element: The contract element to mark as optional. + + Returns: + Self for chaining. + """ + self.fields.append(_assembly.set_mandatory(element, False)) + return self + + def mandatory_group(self, elements: list[ContractElement]) -> "ContractBuilder": + """Add a group of mutually-required fields. + + All fields in the group reference each other's keys and are + marked mandatory. + + Args: + elements: List of contract elements forming the group. + + Returns: + Self for chaining. + """ + self.fields.extend(_assembly.require_group(elements)) + return self + + def build_fields(self) -> list[ContractElement]: + """Return all accumulated contract fields. + + Returns: + List of ContractElement instances added via this builder. + """ + return self.fields + + def build_outputs(self) -> list[ContractOutputElement]: + """Return all accumulated output declarations. + + Returns: + List of ContractOutputElement instances added via this builder. + """ + return self.outputs diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py index a08651c1..5921c88d 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -8,8 +8,10 @@ - **``configuration``** — the configuration contracts (hints, settings models, serialization types) and the ports its public adapters delegate to; closed and versioned (``SEAM_VERSION``). -- **``contracts``** — the scenarios capability's behaviour layer (builder, - variable helper, preparation entry point) and its extension protocol. +- **``contracts``** — the OpenAEV scenario data contracts (contract + definitions, field elements, enums, template variables), the builder + protocol and the ports its public adapters delegate to; closed and + versioned (``SEAM_VERSION``). - **``daemons``** — the daemon runtime shared by collectors and injectors; the API client is injected through the ``BaseClient`` protocol. - **``signatures``** — the signatures capability's inbound seam @@ -27,13 +29,9 @@ core. As long as the core keeps answering the seam, the public surface keeps working; a seam change is a contract change (``SEAM_VERSION`` bump, changelog). -``configuration``, ``seam/signatures/seam.py`` and ``seam/port_io/models.py`` -are closed today; the other packages still re-export ``_core`` until their +``configuration``, ``contracts``, ``seam/signatures/seam.py`` and +``seam/port_io/models.py`` are closed today; the other packages still re-export ``_core`` until their capability is migrated. - -Distinct from the *business* :mod:`xtm_oaev_sdk.contracts` layer, which holds -the OpenAEV scenario data models — a domain concept, unrelated to the -boundary. """ from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py index 602cc640..8b1b31da 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/__init__.py @@ -1,27 +1,94 @@ -"""The scenarios (contracts) capability's seam surface. +"""Contracts seam: the contract between the public adapters and the core. -Re-presents the capability's behaviour layer — the concrete builder, the -variable helper, and the preparation entry point — together with the -capability's extension protocol, so that the public firewall never imports -``_core`` directly. - -The *data models* of the scenarios capability are a business concern and -live in :mod:`xtm_oaev_sdk.contracts`; this sub-package carries the -behaviour and the protocol only. +Closed: it imports nothing from ``xtm_oaev_sdk._core`` nor +``xtm_oaev_sdk.public``. Any change to these contracts is a breaking change for +one side of the boundary and must bump :data:`SEAM_VERSION`. """ -from xtm_oaev_sdk._core.contracts import ( - ContractBuilder, +from xtm_oaev_sdk.seam.contracts.models import ( + ContractAiTarget, + ContractAsset, + ContractAssetGroup, + ContractAttachment, + ContractCardinality, + ContractCardinalityElement, + ContractCheckbox, + ContractConfig, + ContractData, + ContractElement, + ContractExpectations, + ContractExpectationType, + ContractFieldKey, + ContractFieldType, + ContractOutputElement, + ContractOutputType, + ContractPayload, + ContractReferencedCredential, + ContractSelect, + ContractTeam, + ContractText, + ContractTextArea, + ContractTuple, + ContractVariable, + CredentialType, + Domain, + Expectation, + LinkedFieldModel, + PrimitiveType, + SecurityPlatformType, + SupportedLanguage, + VariableType, +) +from xtm_oaev_sdk.seam.contracts.ports import ( + ContractAssemblyPort, ContractBuilderProtocol, - VariableHelper, - build_single_referenced_credential_element, - prepare_contracts, + ContractCredentialsPort, + ContractSerializerPort, + ContractVariablesPort, ) +#: Version of the contracts seam. Bump on any contract change. +SEAM_VERSION = "1.0.0" + __all__ = [ - "ContractBuilder", + "SEAM_VERSION", + # Data contracts + "ContractAiTarget", + "ContractAsset", + "ContractAssetGroup", + "ContractAttachment", + "ContractCardinality", + "ContractCardinalityElement", + "ContractCheckbox", + "ContractConfig", + "ContractData", + "ContractElement", + "ContractExpectationType", + "ContractExpectations", + "ContractFieldKey", + "ContractFieldType", + "ContractOutputElement", + "ContractOutputType", + "ContractPayload", + "ContractReferencedCredential", + "ContractSelect", + "ContractTeam", + "ContractText", + "ContractTextArea", + "ContractTuple", + "ContractVariable", + "CredentialType", + "Domain", + "Expectation", + "LinkedFieldModel", + "PrimitiveType", + "SecurityPlatformType", + "SupportedLanguage", + "VariableType", + # Ports + "ContractAssemblyPort", "ContractBuilderProtocol", - "VariableHelper", - "build_single_referenced_credential_element", - "prepare_contracts", + "ContractCredentialsPort", + "ContractSerializerPort", + "ContractVariablesPort", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_config.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/models.py similarity index 81% rename from libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_config.py rename to libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/models.py index ce83ff8a..2a3b39aa 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/contract_config.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/models.py @@ -1,37 +1,25 @@ -"""Contract configuration: field types, elements, and contract definitions. +"""Contract data contracts: field types, elements and contract definitions. -Defines the full contract model hierarchy used by injectors and collectors -to declare their configuration schemas, input fields, and output types. - -Key types: - - ContractElement (ABC): Base for all contract field types. - - Contract: A complete contract definition with fields, outputs, and metadata. - - ContractExpectationType: Expectation classification for contract fields. +The contract model hierarchy injectors and collectors use to declare their +input fields, output types, expectations and template variables. Elements own +their invariants (derived ``type``, cardinality, predefined expectations). """ -import json from abc import ABC, abstractmethod from dataclasses import dataclass, field from enum import Enum from typing import Any, Optional -from xtm_oaev_sdk._core.contracts.contract_utils import ( - ContractCardinality, - ContractVariable, -) -from xtm_oaev_sdk._core.contracts.credential_types import CredentialType -from xtm_oaev_sdk._core.contracts.variable_helper import VariableHelper -from xtm_oaev_sdk._core.utils import EnhancedJSONEncoder - __all__ = [ - "Contract", "ContractAiTarget", "ContractAsset", "ContractAssetGroup", "ContractAttachment", + "ContractCardinality", "ContractCardinalityElement", "ContractCheckbox", "ContractConfig", + "ContractData", "ContractElement", "ContractExpectationType", "ContractExpectations", @@ -46,16 +34,76 @@ "ContractText", "ContractTextArea", "ContractTuple", + "ContractVariable", + "CredentialType", "Domain", "Expectation", "LinkedFieldModel", "PrimitiveType", "SecurityPlatformType", "SupportedLanguage", - "prepare_contracts", + "VariableType", ] +class ContractCardinality(str, Enum): + """Field cardinality in a contract definition. + + Attributes: + One: Single value expected. + Multiple: Multiple values (list) expected. + """ + + One: str = "1" + Multiple: str = "n" + + +class VariableType(str, Enum): + """Variable type classification for contract template variables. + + Attributes: + String: Simple string value. + Object: Nested object with children. + """ + + String: str = "String" + Object: str = "Object" + + +@dataclass +class ContractVariable: + """A template variable available for injection into contract fields. + + Args: + key: Variable identifier (dot-notation for nested, e.g. "user.email"). + label: Human-readable description of the variable. + type: Variable type (String or Object). + cardinality: Whether this variable holds one or many values. + children: Nested child variables (for Object type). + """ + + key: str + label: str + type: VariableType + cardinality: ContractCardinality + children: list["ContractVariable"] = field(default_factory=list) + + +class CredentialType(str, Enum): + """Credential reference types shared with the OpenAEV platform. + + The Python mirror of the OpenAEV backend credential-type enum used by + credential secret references. Values must stay label-for-label in sync with + the platform so contracts can declare credential references without any + translation layer. + """ + + IDENTITY = "IDENTITY" + CLOUD_AWS = "CLOUD_AWS" + CLOUD_AZURE = "CLOUD_AZURE" + CLOUD_GCP = "CLOUD_GCP" + + class SupportedLanguage(str, Enum): """Supported languages for contract label localization.""" @@ -335,8 +383,8 @@ class Domain: @dataclass -class Contract: - """A complete contract definition for an injector or collector. +class ContractData: + """Data of a complete contract definition for an injector or collector. Contains fields (inputs), outputs, configuration, and metadata for registering the contract with the OpenAEV platform. @@ -357,16 +405,7 @@ class Contract: outputs: list[ContractOutputElement] config: ContractConfig manual: bool - variables: list[ContractVariable] = field( - default_factory=lambda: ( - [ - VariableHelper.user_variable(), - VariableHelper.exercise_variable(), - VariableHelper.team_variable(), - ] - + VariableHelper.uri_variables() - ) - ) + variables: list[ContractVariable] = field(default_factory=list) contract_attack_patterns_external_ids: list[str] = field(default_factory=list) contract_vulnerability_external_ids: list[str] = field(default_factory=list) is_atomic_testing: bool = True @@ -386,46 +425,6 @@ def add_variable(self, var: ContractVariable) -> None: """Add a custom template variable to this contract.""" self.variables.append(var) - def to_contract_add_input(self, source_id: str) -> dict[str, Any]: - """Serialize this contract for the platform creation API. - - Args: - source_id: The injector ID that owns this contract. - - Returns: - Dict payload for the contract creation endpoint. - """ - return { - "contract_id": self.contract_id, - "external_contract_id": self.external_id, - "injector_id": source_id, - "contract_manual": self.manual, - "contract_labels": self.label, - "contract_attack_patterns_external_ids": self.contract_attack_patterns_external_ids, - "contract_vulnerability_external_ids": self.contract_vulnerability_external_ids, - "contract_content": json.dumps(self, cls=EnhancedJSONEncoder), - "is_atomic_testing": self.is_atomic_testing, - "contract_platforms": self.platforms, - "contract_domains": self.domains, - } - - def to_contract_update_input(self) -> dict[str, Any]: - """Serialize this contract for the platform update API. - - Returns: - Dict payload for the contract update endpoint. - """ - return { - "contract_manual": self.manual, - "contract_labels": self.label, - "contract_attack_patterns_external_ids": self.contract_attack_patterns_external_ids, - "contract_vulnerability_external_ids": self.contract_vulnerability_external_ids, - "contract_content": json.dumps(self, cls=EnhancedJSONEncoder), - "is_atomic_testing": self.is_atomic_testing, - "contract_platforms": self.platforms, - "contract_domains": self.domains, - } - @dataclass class ContractTeam(ContractCardinalityElement): @@ -447,30 +446,6 @@ def get_type(self) -> str: return ContractFieldType.Text.value -def prepare_contracts(contracts: list[Contract]) -> list[dict[str, Any]]: - """Serialize a list of contracts for bulk platform registration. - - Args: - contracts: List of Contract instances to serialize. - - Returns: - List of dicts suitable for the bulk contract registration API. - """ - return list( - map( - lambda c: { - "contract_id": c.contract_id, - "contract_labels": c.label, - "contract_attack_patterns_external_ids": c.contract_attack_patterns_external_ids, - "contract_content": json.dumps(c, cls=EnhancedJSONEncoder), - "contract_platforms": c.platforms, - "contract_domains": c.domains, - }, - contracts, - ) - ) - - @dataclass class ContractTuple(ContractCardinalityElement): """Tuple (multi-value) input field with optional file attachment.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/ports.py new file mode 100644 index 00000000..29cf6721 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/contracts/ports.py @@ -0,0 +1,125 @@ +"""Contract ports: the behaviour the public layer expects from the core. + +``ContractBuilderProtocol`` is the consumer-facing contract of field builders. +The other protocols are the ports the public adapters delegate to; the core +implements them. +""" + +from typing import Any, Protocol, runtime_checkable + +from xtm_oaev_sdk.seam.contracts.models import ( + ContractData, + ContractElement, + ContractOutputElement, + ContractReferencedCredential, + ContractVariable, +) + +__all__ = [ + "ContractAssemblyPort", + "ContractBuilderProtocol", + "ContractCredentialsPort", + "ContractSerializerPort", + "ContractVariablesPort", +] + + +@runtime_checkable +class ContractBuilderProtocol(Protocol): + """Behavioral contract for contract field builders. + + Defines the chainable API for constructing contract field sets. + Consumers can code against this protocol for testability. + """ + + def add_fields(self, fields: list[ContractElement]) -> "ContractBuilderProtocol": + """Add multiple fields at once.""" + ... + + def add_outputs( + self, outputs: list[ContractOutputElement] + ) -> "ContractBuilderProtocol": + """Add multiple output declarations at once.""" + ... + + def mandatory(self, element: ContractElement) -> "ContractBuilderProtocol": + """Add a mandatory field.""" + ... + + def optional(self, element: ContractElement) -> "ContractBuilderProtocol": + """Add an optional field.""" + ... + + def mandatory_group( + self, elements: list[ContractElement] + ) -> "ContractBuilderProtocol": + """Add a group of mutually-required fields.""" + ... + + def build_fields(self) -> list[ContractElement]: + """Return the accumulated fields.""" + ... + + def build_outputs(self) -> list[ContractOutputElement]: + """Return the accumulated outputs.""" + ... + + +@runtime_checkable +class ContractAssemblyPort(Protocol): + """Assembles the field and output lists of a contract.""" + + def concat(self, existing: list[Any], extra: list[Any]) -> list[Any]: + """New list made of ``existing`` followed by ``extra``.""" + + def set_mandatory( + self, element: ContractElement, mandatory: bool + ) -> ContractElement: + """The element, marked mandatory or optional.""" + + def require_group(self, elements: list[ContractElement]) -> list[ContractElement]: + """The elements, mandatory and referencing each other's keys.""" + + +@runtime_checkable +class ContractVariablesPort(Protocol): + """Provides the built-in contract template variables.""" + + def user_variable(self) -> ContractVariable: + """The user template variable with its standard children.""" + + def exercise_variable(self) -> ContractVariable: + """The exercise template variable with its standard children.""" + + def team_variable(self) -> ContractVariable: + """The teams template variable.""" + + def uri_variables(self) -> list[ContractVariable]: + """The platform URI template variables.""" + + def default_variables(self) -> list[ContractVariable]: + """The variables every contract starts with.""" + + +@runtime_checkable +class ContractSerializerPort(Protocol): + """Serializes contracts into platform API payloads.""" + + def to_add_input(self, contract: ContractData, source_id: str) -> dict[str, Any]: + """Payload of the contract creation endpoint.""" + + def to_update_input(self, contract: ContractData) -> dict[str, Any]: + """Payload of the contract update endpoint.""" + + def prepare(self, contracts: list[ContractData]) -> list[dict[str, Any]]: + """Payloads of the bulk contract registration endpoint.""" + + +@runtime_checkable +class ContractCredentialsPort(Protocol): + """Builds the credential-reference fields of a contract.""" + + def build_single_referenced_credential_element( + self, provider_name: str + ) -> ContractReferencedCredential: + """Credential-reference field asking for one credential of the provider's type.""" From 9128fb2afee479b0323732364e534f3f61d403b5 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 15:18:57 +0200 Subject: [PATCH 10/22] :recycle: refactor(sdk): enhancing port-adapter public boundaries for common --- .../test/common/shim/test_utils.py | 8 +- libs/xtm-oaev-sdk/test/common/test_utils.py | 2 +- libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 7 +- .../xtm_oaev_sdk/_core/client_protocol.py | 81 ---- .../xtm_oaev_sdk/_core/common/__init__.py | 15 + .../xtm_oaev_sdk/_core/common/certificates.py | 73 ++++ .../xtm_oaev_sdk/_core/common/dicts.py | 20 + .../xtm_oaev_sdk/_core/common/encoding.py | 18 + .../xtm_oaev_sdk/_core/common/http_errors.py | 23 + .../_core/common/logging_setup.py | 49 +++ .../_core/contracts/serializer.py | 17 +- .../xtm_oaev_sdk/_core/daemon_protocol.py | 39 -- .../xtm_oaev_sdk/_core/daemons/base_daemon.py | 7 +- .../_core/security_domain/__init__.py | 9 - .../_core/signatures/signature_manager.py | 2 +- .../_core/signatures/signature_match.py | 2 +- .../xtm_oaev_sdk/_core/ssl_utils.py | 79 ---- libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/utils.py | 404 ------------------ .../xtm_oaev_sdk/public/__init__.py | 78 ++-- .../xtm_oaev_sdk/public/common.py | 365 ++++++++++++++++ .../xtm_oaev_sdk/seam/__init__.py | 7 +- .../xtm_oaev_sdk/seam/common/__init__.py | 81 ++-- .../{_core => seam/common}/asset_types.py | 7 + .../{_core => seam/common}/errors.py | 72 +--- .../xtm_oaev_sdk/seam/common/ports.py | 98 +++++ .../xtm_oaev_sdk/seam/common/protocols.py | 152 +++++++ .../common/security_domains.py} | 0 .../xtm_oaev_sdk/seam/common/values.py | 98 +++++ pyoaev/apis/attack_pattern.py | 2 +- pyoaev/apis/collector.py | 2 +- pyoaev/apis/cve.py | 2 +- pyoaev/apis/document.py | 2 +- pyoaev/apis/endpoint.py | 2 +- pyoaev/apis/inject.py | 2 +- .../inject_expectation/inject_expectation.py | 2 +- pyoaev/apis/inject_expectation_trace.py | 2 +- pyoaev/apis/injector_contract.py | 2 +- pyoaev/apis/kill_chain_phase.py | 2 +- pyoaev/apis/payload.py | 2 +- pyoaev/apis/security_platform.py | 2 +- pyoaev/apis/signature.py | 2 +- pyoaev/apis/tag.py | 2 +- pyoaev/apis/team.py | 2 +- pyoaev/apis/user.py | 2 +- pyoaev/apis/vulnerability.py | 2 +- pyoaev/client.py | 2 +- pyoaev/mixins.py | 2 +- 47 files changed, 1063 insertions(+), 788 deletions(-) delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/certificates.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/dicts.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/encoding.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/http_errors.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/logging_setup.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/__init__.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/ssl_utils.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/utils.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py rename libs/xtm-oaev-sdk/xtm_oaev_sdk/{_core => seam/common}/asset_types.py (97%) rename libs/xtm-oaev-sdk/xtm_oaev_sdk/{_core => seam/common}/errors.py (88%) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/protocols.py rename libs/xtm-oaev-sdk/xtm_oaev_sdk/{_core/security_domain/types.py => seam/common/security_domains.py} (100%) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/values.py diff --git a/libs/xtm-oaev-sdk/test/common/shim/test_utils.py b/libs/xtm-oaev-sdk/test/common/shim/test_utils.py index 5a9b32f8..3bc1dc74 100644 --- a/libs/xtm-oaev-sdk/test/common/shim/test_utils.py +++ b/libs/xtm-oaev-sdk/test/common/shim/test_utils.py @@ -100,7 +100,7 @@ def test_custom_json_formatter_add_fields_sets_timestamp_and_level(self): self.assertEqual(log_record["level"], "INFO") def test_setup_logging_config_json_logging_true_uses_custom_formatter(self): - with patch("xtm_oaev_sdk._core.utils.logging.basicConfig") as mock_basic_config: + with patch("logging.basicConfig") as mock_basic_config: module.setup_logging_config(logging.INFO, json_logging=True) kwargs = mock_basic_config.call_args.kwargs @@ -111,13 +111,13 @@ def test_setup_logging_config_json_logging_true_uses_custom_formatter(self): ) def test_setup_logging_config_json_logging_false_calls_basic_config(self): - with patch("xtm_oaev_sdk._core.utils.logging.basicConfig") as mock_basic_config: + with patch("logging.basicConfig") as mock_basic_config: module.setup_logging_config(logging.WARNING, json_logging=False) mock_basic_config.assert_called_once_with(level=logging.WARNING) def test_app_logger_methods_delegate_to_local_logger(self): - with patch("xtm_oaev_sdk._core.utils.setup_logging_config"): + with patch("xtm_oaev_sdk.public.common.setup_logging_config"): app_logger = module.AppLogger(logging.INFO) app_logger.local_logger = MagicMock() @@ -132,7 +132,7 @@ def test_app_logger_methods_delegate_to_local_logger(self): self.assertTrue(app_logger.local_logger.error.called) def test_logger_helper_returns_app_logger(self): - with patch("xtm_oaev_sdk._core.utils.setup_logging_config"): + with patch("xtm_oaev_sdk.public.common.setup_logging_config"): helper = module.logger(logging.INFO) self.assertIsInstance(helper, module.AppLogger) diff --git a/libs/xtm-oaev-sdk/test/common/test_utils.py b/libs/xtm-oaev-sdk/test/common/test_utils.py index 4a850ce5..49691327 100644 --- a/libs/xtm-oaev-sdk/test/common/test_utils.py +++ b/libs/xtm-oaev-sdk/test/common/test_utils.py @@ -393,7 +393,7 @@ def test_importing_utils_does_not_raise_jsonlogger_deprecation_warning() -> None "-W", "error::DeprecationWarning", "-c", - "import xtm_oaev_sdk._core.utils", + "import xtm_oaev_sdk", ], capture_output=True, text=True, diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py index c01f44b5..27f32458 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -1,19 +1,20 @@ """xtm-oaev-sdk — OpenAEV SDK core (DDD + Light Hex Architecture). Architecture: - _core/ Shared implementations (errors, utils, ssl_utils, protocols, - asset_types, security_domain) + _core/common/ Common implementations (HTTP errors, logging, JSON + encoding, dict tools, certificates) _core/configuration/ Configuration feature implementation _core/contracts/ OpenAEV scenarios feature implementation _core/daemons/ Daemon runtime (base daemon, heartbeat) _core/signatures/ Signature feature implementation - seam/common/ Shared errors, enums, types, and utilities + seam/common/ Shared errors, enums, value objects, protocols, ports seam/configuration/ Configuration contracts and ports (closed, versioned) seam/contracts/ Scenario contracts and ports (closed, versioned) seam/daemons/ Daemon runtime (API client injected) seam/signatures/ Signatures inbound I/O seam (versioned) seam/port_io/ Port I/O wire schema (versioned, seam-owned) seam/deprecation.py PEP 562 deprecation primitive + public/common Common adapters public/configuration Configuration adapters public/contracts Scenario contract adapters public/ User-facing re-exports diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py deleted file mode 100644 index 92ce15d8..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/client_protocol.py +++ /dev/null @@ -1,81 +0,0 @@ -"""BaseClient Protocol — behavioral contract for OpenAEV HTTP client implementations.""" - -from typing import Any, Protocol, runtime_checkable - -__all__ = ["BaseClient"] - - -@runtime_checkable -class BaseClient(Protocol): - """Protocol defining the behavioral contract for an OpenAEV HTTP client. - - Any class implementing all methods and properties defined here - will pass isinstance(obj, BaseClient) checks at runtime. - - IMPORTANT: Properties are used (not bare attributes) to ensure - isinstance checks work on Python 3.11 where @runtime_checkable - only inspects methods, not data attributes. - """ - - @property - def per_page(self) -> int: - """Number of items per page requested from the API.""" - - @property - def pagination(self) -> str: - """Pagination mode requested from the API.""" - - @property - def order_by(self) -> str: - """Sort key requested from the API.""" - - @property - def url(self) -> str: - """Base URL of the OpenAEV API.""" - - @property - def headers(self) -> dict[str, str]: - """HTTP headers sent with every request.""" - - @property - def ssl_verify(self) -> bool | str: - """TLS verification flag or CA bundle path.""" - - @property - def timeout(self) -> int | float | None: - """Request timeout in seconds.""" - - @property - def tenant_id(self) -> str | None: - """Tenant scoping the API requests, if any.""" - - @property - def session(self) -> Any: - """The underlying HTTP session (e.g. requests.Session).""" - ... - - @property - def backend(self) -> Any: - """The HTTP backend implementation handling transport.""" - ... - - def http_head(self, path: str, **kwargs: Any) -> Any: - """Send a HEAD request.""" - - def http_get(self, path: str, **kwargs: Any) -> Any: - """Send a GET request.""" - - def http_post(self, path: str, **kwargs: Any) -> Any: - """Send a POST request.""" - - def http_patch(self, path: str, **kwargs: Any) -> Any: - """Send a PATCH request.""" - - def http_put(self, path: str, **kwargs: Any) -> Any: - """Send a PUT request.""" - - def http_delete(self, path: str, **kwargs: Any) -> Any: - """Send a DELETE request.""" - - def http_list(self, path: str, **kwargs: Any) -> Any: - """Send a paginated GET request.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py new file mode 100644 index 00000000..adff0d9e --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py @@ -0,0 +1,15 @@ +"""Common core: implementations of the common seam ports.""" + +from .certificates import Certificates +from .dicts import DictTools +from .encoding import JsonEncoding +from .http_errors import HttpErrorTranslator +from .logging_setup import LoggingSetup + +__all__ = [ + "Certificates", + "DictTools", + "HttpErrorTranslator", + "JsonEncoding", + "LoggingSetup", +] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/certificates.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/certificates.py new file mode 100644 index 00000000..73b7d8fb --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/certificates.py @@ -0,0 +1,73 @@ +"""Certificates: load files or in-memory PEM data into SSL contexts. + +Thanks to https://bugs.python.org/issue16487 it is not possible to load +in-memory PEM data into an SSL context: it is written to a temporary file first. +""" + +import os +import ssl +import tempfile + +__all__ = ["Certificates"] + + +class Certificates: + """Implements ``CertificatesPort``.""" + + # As cert must be written in files to be loaded in ssl context + # Creates a temporary file in the most secure manner possible + def data_to_temp_file(self, data: str) -> str: + # The file is readable and writable only by the creating user ID. + # If the operating system uses permission bits to indicate whether a + # file is executable, the file is executable by no one. The file + # descriptor is not inherited by children of this process. + file_descriptor, file_path = tempfile.mkstemp() + with os.fdopen(file_descriptor, "w") as open_file: + open_file.write(data) + open_file.close() + return file_path + + def is_memory_certificate(self, certificate: str) -> bool: + return certificate.startswith("-----BEGIN") + + def ssl_cert_chain( + self, + ssl_context: ssl.SSLContext, + cert_data: str | None, + key_data: str | None, + passphrase: str | None, + ) -> None: + if cert_data is None: + return + + cert_file_path = None + key_file_path = None + + # Cert loading + if self.is_memory_certificate(cert_data): + cert_file_path = self.data_to_temp_file(cert_data) + cert = cert_file_path if cert_file_path is not None else cert_data + + # Key loading + if key_data is not None and self.is_memory_certificate(key_data): + key_file_path = self.data_to_temp_file(key_data) + key = key_file_path if key_file_path is not None else key_data + + # Load cert + ssl_context.load_cert_chain(cert, key, passphrase) + # Remove temp files + if cert_file_path is not None: + os.unlink(cert_file_path) + if key_file_path is not None: + os.unlink(key_file_path) + + def ssl_verify_locations( + self, ssl_context: ssl.SSLContext, certdata: str | None + ) -> None: + if certdata is None: + return + + if self.is_memory_certificate(certdata): + ssl_context.load_verify_locations(cadata=certdata) + else: + ssl_context.load_verify_locations(cafile=certdata) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/dicts.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/dicts.py new file mode 100644 index 00000000..12e87a0c --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/dicts.py @@ -0,0 +1,20 @@ +"""Dict tools: reshape the dictionaries exchanged with the platform.""" + +from typing import Any + +__all__ = ["DictTools"] + + +class DictTools: + """Implements ``DictToolsPort``.""" + + def copy_dict(self, *, src: dict[str, Any], dest: dict[str, Any]) -> None: + for k, v in src.items(): + if isinstance(v, dict): + for dict_k, dict_v in v.items(): + dest[f"{k}[{dict_k}]"] = dict_v + else: + dest[k] = v + + def remove_none_from_dict(self, data: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in data.items() if v is not None} diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/encoding.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/encoding.py new file mode 100644 index 00000000..b49315dd --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/encoding.py @@ -0,0 +1,18 @@ +"""JSON encoding of the objects the standard encoder does not support.""" + +import dataclasses +from typing import Any + +__all__ = ["JsonEncoding"] + + +class JsonEncoding: + """Implements ``JsonEncodingPort``.""" + + def encode_default(self, o: Any) -> Any: + if dataclasses.is_dataclass(o): + return dataclasses.asdict(o) + # same error as json.JSONEncoder.default + raise TypeError( + f"Object of type {o.__class__.__name__} is not JSON serializable" + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/http_errors.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/http_errors.py new file mode 100644 index 00000000..be4d46c4 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/http_errors.py @@ -0,0 +1,23 @@ +"""HTTP error translation: generic HTTP errors into operation-specific ones.""" + +from typing import Any, Callable + +from xtm_oaev_sdk.seam.common import OpenAEVHttpError + +__all__ = ["HttpErrorTranslator"] + + +class HttpErrorTranslator: + """Implements ``HttpErrorPort``.""" + + def call( + self, + function: Callable[..., Any], + error: type[Exception], + *args: Any, + **kwargs: Any, + ) -> Any: + try: + return function(*args, **kwargs) + except OpenAEVHttpError as e: + raise error(e.error_message, e.response_code, e.response_body) from e diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/logging_setup.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/logging_setup.py new file mode 100644 index 00000000..cbe3298c --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/logging_setup.py @@ -0,0 +1,49 @@ +"""Logging setup: root logger configuration and structured record fields.""" + +import datetime +import logging +from typing import Any + +__all__ = ["LoggingSetup"] + + +class LoggingSetup: + """Implements ``LoggingPort``.""" + + def setup_logging_config( + self, + level: int, + json_logging: bool, + formatter_class: type[logging.Formatter], + ) -> None: + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("pika").setLevel(logging.ERROR) + if json_logging: + log_handler = logging.StreamHandler() + log_handler.setLevel(level) + formatter = formatter_class("%(timestamp)s %(level)s %(name)s %(message)s") + log_handler.setFormatter(formatter) + logging.basicConfig(handlers=[log_handler], level=level, force=True) + else: + logging.basicConfig(level=level) + + def get_logger(self, name: str) -> logging.Logger: + return logging.getLogger(name) + + def prepare_meta(self, meta: Any) -> dict[str, Any] | None: + return None if meta is None else {"attributes": meta} + + def set_library_level(self, lib: str, log_level: int) -> None: + logging.getLogger(lib).setLevel(log_level) + + def complete_record( + self, log_record: dict[str, Any], record: logging.LogRecord + ) -> None: + if not log_record.get("timestamp"): + # This doesn't use record.created, so it is slightly off + now = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + log_record["timestamp"] = now + if log_record.get("level"): + log_record["level"] = log_record["level"].upper() + else: + log_record["level"] = record.levelname diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py index 8c31b137..99a583a9 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/contracts/serializer.py @@ -3,7 +3,7 @@ import json from typing import Any -from xtm_oaev_sdk._core.utils import EnhancedJSONEncoder +from xtm_oaev_sdk._core.common import JsonEncoding from xtm_oaev_sdk.seam.contracts import ContractData __all__ = ["ContractSerializer"] @@ -12,6 +12,9 @@ class ContractSerializer: """Implements ``ContractSerializerPort``.""" + def __init__(self) -> None: + self._encoding = JsonEncoding() + def to_add_input(self, contract: ContractData, source_id: str) -> dict[str, Any]: """Serialize a contract for the platform creation API. @@ -29,7 +32,9 @@ def to_add_input(self, contract: ContractData, source_id: str) -> dict[str, Any] "contract_labels": contract.label, "contract_attack_patterns_external_ids": contract.contract_attack_patterns_external_ids, "contract_vulnerability_external_ids": contract.contract_vulnerability_external_ids, - "contract_content": json.dumps(contract, cls=EnhancedJSONEncoder), + "contract_content": json.dumps( + contract, default=self._encoding.encode_default + ), "is_atomic_testing": contract.is_atomic_testing, "contract_platforms": contract.platforms, "contract_domains": contract.domains, @@ -46,7 +51,9 @@ def to_update_input(self, contract: ContractData) -> dict[str, Any]: "contract_labels": contract.label, "contract_attack_patterns_external_ids": contract.contract_attack_patterns_external_ids, "contract_vulnerability_external_ids": contract.contract_vulnerability_external_ids, - "contract_content": json.dumps(contract, cls=EnhancedJSONEncoder), + "contract_content": json.dumps( + contract, default=self._encoding.encode_default + ), "is_atomic_testing": contract.is_atomic_testing, "contract_platforms": contract.platforms, "contract_domains": contract.domains, @@ -67,7 +74,9 @@ def prepare(self, contracts: list[ContractData]) -> list[dict[str, Any]]: "contract_id": c.contract_id, "contract_labels": c.label, "contract_attack_patterns_external_ids": c.contract_attack_patterns_external_ids, - "contract_content": json.dumps(c, cls=EnhancedJSONEncoder), + "contract_content": json.dumps( + c, default=self._encoding.encode_default + ), "contract_platforms": c.platforms, "contract_domains": c.domains, }, diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py deleted file mode 100644 index 00f86760..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemon_protocol.py +++ /dev/null @@ -1,39 +0,0 @@ -"""DaemonProtocol — behavioral contract for daemon implementations. - -Defines the shared lifecycle surface that both collector and injector -daemons expose. Extension SDKs type-hint against this Protocol instead -of a concrete daemon class. -""" - -from typing import Any, Callable, Protocol, runtime_checkable - -__all__ = ["DaemonProtocol"] - - -@runtime_checkable -class DaemonProtocol(Protocol): - """Protocol defining the behavioral contract for a daemon runtime. - - Any class implementing all methods defined here will pass - isinstance(obj, DaemonProtocol) checks at runtime. - - The three methods capture the public lifecycle surface shared by - CollectorDaemon and injector daemon runtimes: - - - ``start()`` — run setup then enter the main loop. - - ``set_callback(callback)`` — configure the periodic callback. - - ``get_id()`` — return the daemon instance identifier. - - IMPORTANT: Only public methods are part of this contract. - Implementation details (_setup, _start_loop, _try_callback) - are excluded — they belong to the concrete daemon class. - """ - - def start(self) -> None: - """Run the daemon setup, then its main loop.""" - - def set_callback(self, callback: Callable[..., Any]) -> None: - """Configure the callback run by the main loop.""" - - def get_id(self) -> str | None: - """Return the daemon instance identifier.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py index 09c25f78..03773326 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py @@ -6,9 +6,7 @@ from types import FunctionType from typing import TYPE_CHECKING -from xtm_oaev_sdk._core.client_protocol import BaseClient -from xtm_oaev_sdk._core.errors import OpenAEVError -from xtm_oaev_sdk._core.utils import logger +from xtm_oaev_sdk.seam.common import BaseClient, OpenAEVError if TYPE_CHECKING: # annotation only: the core never imports the public layer from xtm_oaev_sdk.public.configuration import Configuration @@ -149,4 +147,7 @@ def _create_default_api_client(self) -> BaseClient: @classmethod def __get_default_logger(cls, log_level, name): + # transitional until the daemons capability is layered + from xtm_oaev_sdk.public.common import logger + return logger(log_level)(name) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/__init__.py deleted file mode 100644 index c23cf7fc..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Security domain enums and types. - -Re-exports the SecurityDomains enum for internal engine use. -Public consumers import from xtm_oaev_sdk directly. -""" - -from .types import SecurityDomains - -__all__ = ["SecurityDomains"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py index 997b89e7..d1eae55e 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py @@ -8,7 +8,6 @@ from typing import Any, Protocol, runtime_checkable from pydantic import ValidationError -from xtm_oaev_sdk._core.errors import OpenAEVError from xtm_oaev_sdk._core.signatures.models import ( CloudInjectorConfig, ExecutionDetails, @@ -24,6 +23,7 @@ TargetSignatures, ToolOutput, ) +from xtm_oaev_sdk.seam.common import OpenAEVError __all__ = [ "SignatureManager", diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py index 31c5c252..9a7438ce 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py @@ -4,8 +4,8 @@ match type and its optional score. """ -from xtm_oaev_sdk._core.errors import OpenAEVError from xtm_oaev_sdk._core.signatures.types import MatchTypes +from xtm_oaev_sdk.seam.common import OpenAEVError __all__ = ["SignatureMatch"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/ssl_utils.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/ssl_utils.py deleted file mode 100644 index 8f78e748..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/ssl_utils.py +++ /dev/null @@ -1,79 +0,0 @@ -"""SSL utility functions for OpenAEV SDK. - -Loading certificates into an ``ssl.SSLContext``, including PEM data held in -memory (see https://bugs.python.org/issue16487). -""" - -import os -import ssl -import tempfile - -__all__ = [ - "data_to_temp_file", - "is_memory_certificate", - "ssl_cert_chain", - "ssl_verify_locations", -] - - -# As cert must be written in files to be loaded in ssl context -# Creates a temporary file in the most secure manner possible -def data_to_temp_file(data: str) -> str: - """Write data to a secure temporary file and return its path.""" - # The file is readable and writable only by the creating user ID. - # If the operating system uses permission bits to indicate whether a - # file is executable, the file is executable by no one. The file - # descriptor is not inherited by children of this process. - file_descriptor, file_path = tempfile.mkstemp() - with os.fdopen(file_descriptor, "w") as open_file: - open_file.write(data) - open_file.close() - return file_path - - -def is_memory_certificate(certificate: str) -> bool: - """Return True if the certificate is PEM-encoded in memory (not a file path).""" - return certificate.startswith("-----BEGIN") - - -def ssl_cert_chain( - ssl_context: ssl.SSLContext, - cert_data: str | None, - key_data: str | None, - passphrase: str | None, -) -> None: - """Load a certificate chain into an SSL context, handling in-memory PEM data.""" - if cert_data is None: - return - - cert_file_path = None - key_file_path = None - - # Cert loading - if is_memory_certificate(cert_data): - cert_file_path = data_to_temp_file(cert_data) - cert = cert_file_path if cert_file_path is not None else cert_data - - # Key loading - if key_data is not None and is_memory_certificate(key_data): - key_file_path = data_to_temp_file(key_data) - key = key_file_path if key_file_path is not None else key_data - - # Load cert - ssl_context.load_cert_chain(cert, key, passphrase) - # Remove temp files - if cert_file_path is not None: - os.unlink(cert_file_path) - if key_file_path is not None: - os.unlink(key_file_path) - - -def ssl_verify_locations(ssl_context: ssl.SSLContext, certdata: str | None) -> None: - """Load CA verification locations into an SSL context.""" - if certdata is None: - return - - if is_memory_certificate(certdata): - ssl_context.load_verify_locations(cadata=certdata) - else: - ssl_context.load_verify_locations(cafile=certdata) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/utils.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/utils.py deleted file mode 100644 index 5e23aa85..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/utils.py +++ /dev/null @@ -1,404 +0,0 @@ -"""xtm-oaev-sdk shared utilities. - -Core utility classes and functions used across the SDK engine modules. -Provides logging infrastructure, JSON encoding, data validation helpers, -and dict manipulation utilities. - -Public symbols (re-exported from xtm_oaev_sdk): - - AppLogger: Structured JSON logger with level-aware output. - - AppLoggerProtocol: Protocol for logger behavioral contract. - - EnhancedJSONEncoder: JSON encoder supporting dataclasses. - - RequiredOptional: Immutable field-presence validator. - - EncodedId: URL-safe encoded identifier type. - - CustomJsonFormatter: Timestamped JSON log formatter. - - setup_logging_config: Configure root logger with JSON or plain output. - - copy_dict: Flatten nested dict into bracket-notation keys. - - remove_none_from_dict: Strip None-valued keys from a dict. - - logger: Factory returning an AppLogger. -""" - -import dataclasses -import datetime -import json -import logging -import urllib.parse -from typing import Any, Protocol, runtime_checkable - -from pythonjsonlogger.json import JsonFormatter - -__all__ = [ - "AppLogger", - "AppLoggerProtocol", - "CustomJsonFormatter", - "EncodedId", - "EnhancedJSONEncoder", - "RequiredOptional", - "copy_dict", - "logger", - "remove_none_from_dict", - "setup_logging_config", -] - - -@runtime_checkable -class AppLoggerProtocol(Protocol): - """Behavioral contract for SDK logging services. - - Any logger implementation used by SDK consumers must satisfy this - protocol. The built-in AppLogger class implements it. - - The __call__ method allows switching the logger name (factory pattern): - >>> log = AppLogger(logging.DEBUG) - >>> module_log = log("my_module") # returns self with new name - """ - - def __call__(self, name: str) -> "AppLoggerProtocol": - """Switch logger name and return self.""" - ... - - def debug(self, message: str, meta: Any = None) -> None: - """Log at DEBUG level.""" - ... - - def info(self, message: str, meta: Any = None) -> None: - """Log at INFO level.""" - ... - - def warning(self, message: str, meta: Any = None) -> None: - """Log at WARNING level.""" - ... - - def error(self, message: str, meta: Any = None) -> None: - """Log at ERROR level with traceback.""" - ... - - @staticmethod - def setup_logger_level(lib: str, log_level: int) -> None: - """Override log level for a third-party library.""" - ... - - -class EnhancedJSONEncoder(json.JSONEncoder): - """JSON encoder that serializes ``@dataclass`` instances via - ``dataclasses.asdict()``. - - Example: - >>> import json - >>> from dataclasses import dataclass - >>> @dataclass - ... class Point: - ... x: int - ... y: int - >>> json.dumps(Point(1, 2), cls=EnhancedJSONEncoder) - '{"x": 1, "y": 2}' - """ - - def default(self, o: Any) -> Any: - """Serialize dataclasses to dicts. - - Args: - o: Object to serialize. - - Returns: - A JSON-serializable representation. - - Raises: - TypeError: If the object is not serializable. - """ - if dataclasses.is_dataclass(o): - return dataclasses.asdict(o) - return super().default(o) - - -@dataclasses.dataclass(frozen=True) -class RequiredOptional: - """Immutable field-presence validator for API payloads. - - Validates that required fields are present, optional fields are allowed, - and exclusive fields follow mutual exclusion rules. - - Args: - required: Tuple of field names that must be present. - optional: Tuple of field names that may be present. - exclusive: Tuple of field names where exactly one must be present. - - Example: - >>> ro = RequiredOptional(required=("name", "type"), exclusive=("id", "key")) - >>> ro.validate_attrs(data={"name": "x", "type": "y", "id": "123"}) # OK - >>> ro.validate_attrs(data={"name": "x"}) # raises AttributeError - """ - - required: tuple[str, ...] = () - optional: tuple[str, ...] = () - exclusive: tuple[str, ...] = () - - def validate_attrs( - self, - *, - data: dict[str, Any], - excludes: list[str] | None = None, - ) -> None: - """Validate field presence against the configured rules. - - Args: - data: Dict of field names to values to validate. - excludes: Field names to skip during required-field checks. - - Raises: - AttributeError: If required fields are missing, or exclusive - field constraints are violated. - """ - if excludes is None: - excludes = [] - - if self.required: - required = [k for k in self.required if k not in excludes] - missing = [attr for attr in required if attr not in data] - if missing: - raise AttributeError(f"Missing attributes: {', '.join(missing)}") - - if self.exclusive: - exclusives = [attr for attr in data if attr in self.exclusive] - if len(exclusives) > 1: - raise AttributeError( - f"Provide only one of these attributes: {', '.join(exclusives)}" - ) - if not exclusives: - raise AttributeError( - f"Must provide one of these attributes: " - f"{', '.join(self.exclusive)}" - ) - - -class EncodedId(str): - """URL-safe encoded identifier. - - Wraps a string or integer value with URL percent-encoding applied. - Idempotent: encoding an already-encoded value returns it unchanged. - - Args: - value: The raw identifier to encode. Accepts str, int, or EncodedId. - - Raises: - TypeError: If value is not str, int, or EncodedId. - - Example: - >>> EncodedId("hello world") - 'hello%20world' - >>> EncodedId(42) - '42' - """ - - def __new__(cls, value: "str | int | EncodedId") -> "EncodedId": - if isinstance(value, EncodedId): - return value - - if not isinstance(value, (int, str)): - raise TypeError(f"Unsupported type received: {type(value)}") - if isinstance(value, str): - value = urllib.parse.quote(value, safe="") - return super().__new__(cls, value) - - -def remove_none_from_dict(data: dict[str, Any]) -> dict[str, Any]: - """Remove all keys with None values from a dict. - - Args: - data: Source dictionary. - - Returns: - New dictionary with None-valued entries removed. - - Example: - >>> remove_none_from_dict({"a": 1, "b": None, "c": "x"}) - {'a': 1, 'c': 'x'} - """ - return {k: v for k, v in data.items() if v is not None} - - -def copy_dict(*, src: dict[str, Any], dest: dict[str, Any]) -> None: - """Flatten nested dict values into bracket-notation keys. - - For nested dicts, expands ``{"key": {"sub": "val"}}`` into - ``dest["key[sub]"] = "val"``. Non-dict values copy directly. - - Args: - src: Source dictionary to copy from. - dest: Destination dictionary to copy into (mutated in place). - - Example: - >>> dest = {} - >>> copy_dict(src={"filters": {"type": "malware"}}, dest=dest) - >>> dest - {'filters[type]': 'malware'} - """ - for k, v in src.items(): - if isinstance(v, dict): - for dict_k, dict_v in v.items(): - dest[f"{k}[{dict_k}]"] = dict_v - else: - dest[k] = v - - -class CustomJsonFormatter(JsonFormatter): - """JSON log formatter with ISO timestamp and uppercase level. - - Extends pythonjsonlogger's JsonFormatter to add: - - UTC ISO 8601 timestamp if not already present. - - Uppercase level field normalization. - """ - - def add_fields( - self, - log_record: dict[str, Any], - record: logging.LogRecord, - message_dict: dict[str, Any], - ) -> None: - """Add timestamp and level fields to the log record. - - Args: - log_record: The output dict being built. - record: The stdlib LogRecord. - message_dict: Extra message fields. - """ - super().add_fields(log_record, record, message_dict) - if not log_record.get("timestamp"): - # This doesn't use record.created, so it is slightly off - now = datetime.datetime.now(datetime.UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") - log_record["timestamp"] = now - if log_record.get("level"): - log_record["level"] = log_record["level"].upper() - else: - log_record["level"] = record.levelname - - -def setup_logging_config(level: int, json_logging: bool = True) -> None: - """Configure root logger with JSON or plain text output. - - Silences noisy third-party loggers (urllib3, pika) and sets up - the root logger with either structured JSON or basic text formatting. - - Args: - level: Logging level (e.g., logging.DEBUG, logging.INFO). - json_logging: If True, use CustomJsonFormatter for structured output. - If False, use basicConfig plain text. - """ - logging.getLogger("urllib3").setLevel(logging.WARNING) - logging.getLogger("pika").setLevel(logging.ERROR) - if json_logging: - log_handler = logging.StreamHandler() - log_handler.setLevel(level) - formatter = CustomJsonFormatter("%(timestamp)s %(level)s %(name)s %(message)s") - log_handler.setFormatter(formatter) - logging.basicConfig(handlers=[log_handler], level=level, force=True) - else: - logging.basicConfig(level=level) - - -class AppLogger: - """Structured application logger with JSON support. - - Provides a simple logging interface with optional JSON formatting - and metadata attachment. Supports name-switching via __call__ for - per-module logger instances. - - Args: - level: Logging level (e.g., logging.DEBUG). - json_logging: Whether to use JSON-formatted output. - name: Logger name (defaults to module name). - - Example: - >>> log = AppLogger(logging.INFO) - >>> log.info("Server started", {"port": 8080}) - >>> module_log = log("my_module") - >>> module_log.debug("Processing request") - """ - - def __init__(self, level: int, json_logging: bool = True, name: str = __name__): - self.log_level = level - self.json_logging = json_logging - setup_logging_config(self.log_level, self.json_logging) - self.local_logger = logging.getLogger(name) - - def __call__(self, name: str) -> "AppLogger": - """Switch to a named logger instance. - - Args: - name: New logger name. - - Returns: - Self with updated logger name. - """ - self.local_logger = logging.getLogger(name) - return self - - @staticmethod - def prepare_meta(meta: Any = None) -> dict[str, Any] | None: - """Wrap metadata for structured logging extra field. - - Args: - meta: Arbitrary metadata to attach to log entry. - - Returns: - Dict with 'attributes' key, or None if no metadata. - """ - return None if meta is None else {"attributes": meta} - - @staticmethod - def setup_logger_level(lib: str, log_level: int) -> None: - """Override log level for a third-party library. - - Args: - lib: Library logger name (e.g., "urllib3"). - log_level: Desired logging level. - """ - logging.getLogger(lib).setLevel(log_level) - - def debug(self, message: str, meta: Any = None) -> None: - """Log at DEBUG level. - - Args: - message: Log message. - meta: Optional structured metadata. - """ - self.local_logger.debug(message, extra=AppLogger.prepare_meta(meta)) - - def info(self, message: str, meta: Any = None) -> None: - """Log at INFO level. - - Args: - message: Log message. - meta: Optional structured metadata. - """ - self.local_logger.info(message, extra=AppLogger.prepare_meta(meta)) - - def warning(self, message: str, meta: Any = None) -> None: - """Log at WARNING level. - - Args: - message: Log message. - meta: Optional structured metadata. - """ - self.local_logger.warning(message, extra=AppLogger.prepare_meta(meta)) - - def error(self, message: str, meta: Any = None) -> None: - """Log at ERROR level with traceback. - - Args: - message: Log message. - meta: Optional structured metadata. - """ - self.local_logger.error(message, exc_info=1, extra=AppLogger.prepare_meta(meta)) - - -def logger(level: int, json_logging: bool = True) -> AppLogger: - """Create an AppLogger instance. - - Args: - level: Logging level. - json_logging: Whether to use JSON output. - - Returns: - An AppLogger instance. - """ - return AppLogger(level, json_logging) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index e2d29fb7..f8b476e7 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -2,6 +2,45 @@ __version__ = "0.1.0" +# --- Common (shared) --- +from xtm_oaev_sdk.public.common import ( + AppLogger, + AppLoggerProtocol, + AssetCategory, + AssetCriticality, + AssetSubCategory, + BaseClient, + CloudProvider, + ConfigurationError, + CustomJsonFormatter, + DaemonProtocol, + EncodedId, + EnhancedJSONEncoder, + OpenAEVAuthenticationError, + OpenAEVCreateError, + OpenAEVDeleteError, + OpenAEVError, + OpenAEVGetError, + OpenAEVHeadError, + OpenAEVHttpError, + OpenAEVListError, + OpenAEVParsingError, + OpenAEVUpdateError, + RedirectError, + RequiredOptional, + SecurityDomains, + SignatureTransmissionError, + copy_dict, + data_to_temp_file, + is_memory_certificate, + logger, + on_http_error, + remove_none_from_dict, + setup_logging_config, + ssl_cert_chain, + ssl_verify_locations, +) + # --- Configuration --- from xtm_oaev_sdk.public.configuration import ( CONFIGURATION_TYPES, @@ -63,45 +102,6 @@ prepare_contracts, ) -# --- Common (shared) --- -from xtm_oaev_sdk.seam.common import ( - AppLogger, - AppLoggerProtocol, - AssetCategory, - AssetCriticality, - AssetSubCategory, - BaseClient, - CloudProvider, - ConfigurationError, - CustomJsonFormatter, - DaemonProtocol, - EncodedId, - EnhancedJSONEncoder, - OpenAEVAuthenticationError, - OpenAEVCreateError, - OpenAEVDeleteError, - OpenAEVError, - OpenAEVGetError, - OpenAEVHeadError, - OpenAEVHttpError, - OpenAEVListError, - OpenAEVParsingError, - OpenAEVUpdateError, - RedirectError, - RequiredOptional, - SecurityDomains, - SignatureTransmissionError, - copy_dict, - data_to_temp_file, - is_memory_certificate, - logger, - on_http_error, - remove_none_from_dict, - setup_logging_config, - ssl_cert_chain, - ssl_verify_locations, -) - # --- Daemons --- from xtm_oaev_sdk.seam.daemons import BaseDaemon, PingAlive diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py new file mode 100644 index 00000000..8e3ededa --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py @@ -0,0 +1,365 @@ +"""Common adapters: HTTP error translation, logging, JSON encoding, dict and +certificate helpers, plus the shared errors, enums, value objects and +protocols of the common seam. + +Each adapter speaks the common seam types and delegates to the core +implementation of its port. +""" + +import functools +import json +import logging +import ssl +from typing import Any, Callable, Type, TypeVar, cast + +from pythonjsonlogger.json import JsonFormatter +from xtm_oaev_sdk._core.common import ( + Certificates, + DictTools, + HttpErrorTranslator, + JsonEncoding, + LoggingSetup, +) +from xtm_oaev_sdk.seam.common import ( + AppLoggerProtocol, + AssetCategory, + AssetCriticality, + AssetSubCategory, + BaseClient, + CertificatesPort, + CloudProvider, + ConfigurationError, + DaemonProtocol, + DictToolsPort, + EncodedId, + HttpErrorPort, + JsonEncodingPort, + LoggingPort, + OpenAEVAuthenticationError, + OpenAEVCreateError, + OpenAEVDeleteError, + OpenAEVError, + OpenAEVGetError, + OpenAEVHeadError, + OpenAEVHttpError, + OpenAEVListError, + OpenAEVParsingError, + OpenAEVUpdateError, + RedirectError, + RequiredOptional, + SecurityDomains, + SignatureTransmissionError, +) + +__all__ = [ + "AppLogger", + "AppLoggerProtocol", + "AssetCategory", + "AssetCriticality", + "AssetSubCategory", + "BaseClient", + "CloudProvider", + "ConfigurationError", + "CustomJsonFormatter", + "DaemonProtocol", + "EncodedId", + "EnhancedJSONEncoder", + "OpenAEVAuthenticationError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "OpenAEVError", + "OpenAEVGetError", + "OpenAEVHeadError", + "OpenAEVHttpError", + "OpenAEVListError", + "OpenAEVParsingError", + "OpenAEVUpdateError", + "RedirectError", + "RequiredOptional", + "SecurityDomains", + "SignatureTransmissionError", + "copy_dict", + "data_to_temp_file", + "is_memory_certificate", + "logger", + "on_http_error", + "remove_none_from_dict", + "setup_logging_config", + "ssl_cert_chain", + "ssl_verify_locations", +] + +_http_errors: HttpErrorPort = HttpErrorTranslator() +_encoding: JsonEncodingPort = JsonEncoding() +_logging: LoggingPort = LoggingSetup() +_dicts: DictToolsPort = DictTools() +_certificates: CertificatesPort = Certificates() + +_F = TypeVar("_F", bound=Callable[..., Any]) + + +def on_http_error(error: Type[Exception]) -> Callable[[_F], _F]: + """Decorator that wraps OpenAEVHttpError into a caller-specified type. + + Use on API methods to convert generic HTTP errors into operation-specific + exception types while preserving error context. + + Args: + error: The exception class to raise instead of OpenAEVHttpError. + Must accept (error_message, response_code, response_body) args. + + Returns: + A decorator that wraps the target function. + + Example: + >>> @on_http_error(OpenAEVCreateError) + ... def create_entity(self, data): + ... return self._post("/entities", data) + """ + + def wrap(f: _F) -> _F: + @functools.wraps(f) + def wrapped_f(*args: Any, **kwargs: Any) -> Any: + return _http_errors.call(f, error, *args, **kwargs) + + return cast(_F, wrapped_f) + + return wrap + + +class EnhancedJSONEncoder(json.JSONEncoder): + """JSON encoder that serializes ``@dataclass`` instances via + ``dataclasses.asdict()``. + + Example: + >>> import json + >>> from dataclasses import dataclass + >>> @dataclass + ... class Point: + ... x: int + ... y: int + >>> json.dumps(Point(1, 2), cls=EnhancedJSONEncoder) + '{"x": 1, "y": 2}' + """ + + def default(self, o: Any) -> Any: + """Serialize dataclasses to dicts. + + Args: + o: Object to serialize. + + Returns: + A JSON-serializable representation. + + Raises: + TypeError: If the object is not serializable. + """ + return _encoding.encode_default(o) + + +def remove_none_from_dict(data: dict[str, Any]) -> dict[str, Any]: + """Remove all keys with None values from a dict. + + Args: + data: Source dictionary. + + Returns: + New dictionary with None-valued entries removed. + + Example: + >>> remove_none_from_dict({"a": 1, "b": None, "c": "x"}) + {'a': 1, 'c': 'x'} + """ + return _dicts.remove_none_from_dict(data) + + +def copy_dict(*, src: dict[str, Any], dest: dict[str, Any]) -> None: + """Flatten nested dict values into bracket-notation keys. + + For nested dicts, expands ``{"key": {"sub": "val"}}`` into + ``dest["key[sub]"] = "val"``. Non-dict values copy directly. + + Args: + src: Source dictionary to copy from. + dest: Destination dictionary to copy into (mutated in place). + + Example: + >>> dest = {} + >>> copy_dict(src={"filters": {"type": "malware"}}, dest=dest) + >>> dest + {'filters[type]': 'malware'} + """ + _dicts.copy_dict(src=src, dest=dest) + + +class CustomJsonFormatter(JsonFormatter): + """JSON log formatter with ISO timestamp and uppercase level. + + Extends pythonjsonlogger's JsonFormatter to add: + - UTC ISO 8601 timestamp if not already present. + - Uppercase level field normalization. + """ + + def add_fields( + self, + log_record: dict[str, Any], + record: logging.LogRecord, + message_dict: dict[str, Any], + ) -> None: + """Add timestamp and level fields to the log record. + + Args: + log_record: The output dict being built. + record: The stdlib LogRecord. + message_dict: Extra message fields. + """ + super().add_fields(log_record, record, message_dict) + _logging.complete_record(log_record, record) + + +def setup_logging_config(level: int, json_logging: bool = True) -> None: + """Configure root logger with JSON or plain text output. + + Silences noisy third-party loggers (urllib3, pika) and sets up + the root logger with either structured JSON or basic text formatting. + + Args: + level: Logging level (e.g., logging.DEBUG, logging.INFO). + json_logging: If True, use CustomJsonFormatter for structured output. + If False, use basicConfig plain text. + """ + _logging.setup_logging_config(level, json_logging, CustomJsonFormatter) + + +class AppLogger: + """Structured application logger with JSON support. + + Provides a simple logging interface with optional JSON formatting + and metadata attachment. Supports name-switching via __call__ for + per-module logger instances. + + Args: + level: Logging level (e.g., logging.DEBUG). + json_logging: Whether to use JSON-formatted output. + name: Logger name (defaults to module name). + + Example: + >>> log = AppLogger(logging.INFO) + >>> log.info("Server started", {"port": 8080}) + >>> module_log = log("my_module") + >>> module_log.debug("Processing request") + """ + + def __init__(self, level: int, json_logging: bool = True, name: str = __name__): + self.log_level = level + self.json_logging = json_logging + setup_logging_config(self.log_level, self.json_logging) + self.local_logger = _logging.get_logger(name) + + def __call__(self, name: str) -> "AppLogger": + """Switch to a named logger instance. + + Args: + name: New logger name. + + Returns: + Self with updated logger name. + """ + self.local_logger = _logging.get_logger(name) + return self + + @staticmethod + def prepare_meta(meta: Any = None) -> dict[str, Any] | None: + """Wrap metadata for structured logging extra field. + + Args: + meta: Arbitrary metadata to attach to log entry. + + Returns: + Dict with 'attributes' key, or None if no metadata. + """ + return _logging.prepare_meta(meta) + + @staticmethod + def setup_logger_level(lib: str, log_level: int) -> None: + """Override log level for a third-party library. + + Args: + lib: Library logger name (e.g., "urllib3"). + log_level: Desired logging level. + """ + _logging.set_library_level(lib, log_level) + + def debug(self, message: str, meta: Any = None) -> None: + """Log at DEBUG level. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.debug(message, extra=AppLogger.prepare_meta(meta)) + + def info(self, message: str, meta: Any = None) -> None: + """Log at INFO level. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.info(message, extra=AppLogger.prepare_meta(meta)) + + def warning(self, message: str, meta: Any = None) -> None: + """Log at WARNING level. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.warning(message, extra=AppLogger.prepare_meta(meta)) + + def error(self, message: str, meta: Any = None) -> None: + """Log at ERROR level with traceback. + + Args: + message: Log message. + meta: Optional structured metadata. + """ + self.local_logger.error(message, exc_info=1, extra=AppLogger.prepare_meta(meta)) + + +def logger(level: int, json_logging: bool = True) -> AppLogger: + """Create an AppLogger instance. + + Args: + level: Logging level. + json_logging: Whether to use JSON output. + + Returns: + An AppLogger instance. + """ + return AppLogger(level, json_logging) + + +def data_to_temp_file(data: str) -> str: + """Write data to a secure temporary file and return its path.""" + return _certificates.data_to_temp_file(data) + + +def is_memory_certificate(certificate: str) -> bool: + """Return True if the certificate is PEM-encoded in memory (not a file path).""" + return _certificates.is_memory_certificate(certificate) + + +def ssl_cert_chain( + ssl_context: ssl.SSLContext, + cert_data: str | None, + key_data: str | None, + passphrase: str | None, +) -> None: + """Load a certificate chain into an SSL context, handling in-memory PEM data.""" + _certificates.ssl_cert_chain(ssl_context, cert_data, key_data, passphrase) + + +def ssl_verify_locations(ssl_context: ssl.SSLContext, certdata: str | None) -> None: + """Load CA verification locations into an SSL context.""" + _certificates.ssl_verify_locations(ssl_context, certdata) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py index 5921c88d..b6551d5b 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -3,8 +3,9 @@ Everything in this layer pins a stable interface so that the two sides of it can evolve independently without leaking either: -- **``common``** — shared errors, enums, types, and utilities: the - platform-facing vocabulary both sides import. +- **``common``** — the shared vocabulary (errors, enums, value objects, + consumer protocols) and the common ports; closed and versioned + (``SEAM_VERSION``). - **``configuration``** — the configuration contracts (hints, settings models, serialization types) and the ports its public adapters delegate to; closed and versioned (``SEAM_VERSION``). @@ -29,7 +30,7 @@ core. As long as the core keeps answering the seam, the public surface keeps working; a seam change is a contract change (``SEAM_VERSION`` bump, changelog). -``configuration``, ``contracts``, ``seam/signatures/seam.py`` and +``common``, ``configuration``, ``contracts``, ``seam/signatures/seam.py`` and ``seam/port_io/models.py`` are closed today; the other packages still re-export ``_core`` until their capability is migrated. """ diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py index 4896a87f..7457caf2 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py @@ -1,15 +1,18 @@ -"""Shared contracts: errors, protocols, enums, type aliases, and the concrete -utility layer.""" +"""Common seam: the shared vocabulary and the common ports. -from xtm_oaev_sdk._core.asset_types import ( +Errors, enums, value objects and consumer protocols every capability speaks, +plus the ports the common public adapters delegate to. Closed: it imports +nothing from ``xtm_oaev_sdk._core`` nor ``xtm_oaev_sdk.public``. Any change to +these contracts must bump :data:`SEAM_VERSION`. +""" + +from xtm_oaev_sdk.seam.common.asset_types import ( AssetCategory, AssetCriticality, AssetSubCategory, CloudProvider, ) -from xtm_oaev_sdk._core.client_protocol import BaseClient -from xtm_oaev_sdk._core.daemon_protocol import DaemonProtocol -from xtm_oaev_sdk._core.errors import ( +from xtm_oaev_sdk.seam.common.errors import ( ConfigurationError, OpenAEVAuthenticationError, OpenAEVCreateError, @@ -23,29 +26,27 @@ OpenAEVUpdateError, RedirectError, SignatureTransmissionError, - on_http_error, ) -from xtm_oaev_sdk._core.security_domain import SecurityDomains -from xtm_oaev_sdk._core.ssl_utils import ( - data_to_temp_file, - is_memory_certificate, - ssl_cert_chain, - ssl_verify_locations, +from xtm_oaev_sdk.seam.common.ports import ( + CertificatesPort, + DictToolsPort, + HttpErrorPort, + JsonEncodingPort, + LoggingPort, ) -from xtm_oaev_sdk._core.utils import ( - AppLogger, +from xtm_oaev_sdk.seam.common.protocols import ( AppLoggerProtocol, - CustomJsonFormatter, - EncodedId, - EnhancedJSONEncoder, - RequiredOptional, - copy_dict, - logger, - remove_none_from_dict, - setup_logging_config, + BaseClient, + DaemonProtocol, ) +from xtm_oaev_sdk.seam.common.security_domains import SecurityDomains +from xtm_oaev_sdk.seam.common.values import EncodedId, RequiredOptional + +#: Version of the common seam. Bump on any contract change. +SEAM_VERSION = "1.0.0" __all__ = [ + "SEAM_VERSION", # Errors "ConfigurationError", "OpenAEVAuthenticationError", @@ -60,30 +61,24 @@ "OpenAEVUpdateError", "RedirectError", "SignatureTransmissionError", - "on_http_error", - # Protocols - "AppLoggerProtocol", - "BaseClient", - "DaemonProtocol", - # Types - "EncodedId", - "RequiredOptional", # Enums "AssetCategory", "AssetCriticality", "AssetSubCategory", "CloudProvider", + # Enums (security domains) "SecurityDomains", - # Behaviour (concrete adapters) - "AppLogger", - "CustomJsonFormatter", - "EnhancedJSONEncoder", - "copy_dict", - "data_to_temp_file", - "is_memory_certificate", - "logger", - "remove_none_from_dict", - "setup_logging_config", - "ssl_cert_chain", - "ssl_verify_locations", + # Value objects + "EncodedId", + "RequiredOptional", + # Consumer protocols + "AppLoggerProtocol", + "BaseClient", + "DaemonProtocol", + # Ports + "CertificatesPort", + "DictToolsPort", + "HttpErrorPort", + "JsonEncodingPort", + "LoggingPort", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/asset_types.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/asset_types.py similarity index 97% rename from libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/asset_types.py rename to libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/asset_types.py index 49cea094..769cdb3e 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/asset_types.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/asset_types.py @@ -9,6 +9,13 @@ from enum import Enum +__all__ = [ + "AssetCategory", + "AssetSubCategory", + "CloudProvider", + "AssetCriticality", +] + class AssetCategory(str, Enum): HOST = "HOST" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/errors.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/errors.py similarity index 88% rename from libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/errors.py rename to libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/errors.py index 0db3758b..8cca2995 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/errors.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/errors.py @@ -29,8 +29,23 @@ ... print(f"SDK error: {e}") """ -import functools -from typing import TYPE_CHECKING, Any, Callable, Optional, Type, TypeVar, Union, cast +from typing import TYPE_CHECKING, Optional, Union + +__all__ = [ + "OpenAEVError", + "OpenAEVAuthenticationError", + "OpenAEVHttpError", + "OpenAEVParsingError", + "RedirectError", + "OpenAEVHeadError", + "OpenAEVGetError", + "OpenAEVUpdateError", + "OpenAEVListError", + "OpenAEVCreateError", + "OpenAEVDeleteError", + "SignatureTransmissionError", + "ConfigurationError", +] class OpenAEVError(Exception): @@ -260,56 +275,3 @@ class ConfigurationError(OpenAEVError): """ pass - - -_F = TypeVar("_F", bound=Callable[..., Any]) - - -def on_http_error(error: Type[Exception]) -> Callable[[_F], _F]: - """Decorator that wraps OpenAEVHttpError into a caller-specified type. - - Use on API methods to convert generic HTTP errors into operation-specific - exception types while preserving error context. - - Args: - error: The exception class to raise instead of OpenAEVHttpError. - Must accept (error_message, response_code, response_body) args. - - Returns: - A decorator that wraps the target function. - - Example: - >>> @on_http_error(OpenAEVCreateError) - ... def create_entity(self, data): - ... return self._post("/entities", data) - """ - - def wrap(f: _F) -> _F: - @functools.wraps(f) - def wrapped_f(*args: Any, **kwargs: Any) -> Any: - try: - return f(*args, **kwargs) - except OpenAEVHttpError as e: - raise error(e.error_message, e.response_code, e.response_body) from e - - return cast(_F, wrapped_f) - - return wrap - - -__all__ = [ - "OpenAEVError", - "OpenAEVAuthenticationError", - "OpenAEVHttpError", - "OpenAEVParsingError", - "RedirectError", - "OpenAEVHeadError", - "OpenAEVGetError", - "OpenAEVUpdateError", - "OpenAEVListError", - "OpenAEVCreateError", - "OpenAEVDeleteError", - "SignatureTransmissionError", - "ConfigurationError", - "on_http_error", -] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py new file mode 100644 index 00000000..aa150713 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py @@ -0,0 +1,98 @@ +"""Common ports: the behaviour the public adapters expect from the core.""" + +import logging +import ssl +from typing import Any, Callable, Protocol, runtime_checkable + +__all__ = [ + "CertificatesPort", + "DictToolsPort", + "HttpErrorPort", + "JsonEncodingPort", + "LoggingPort", +] + + +@runtime_checkable +class HttpErrorPort(Protocol): + """Converts generic HTTP errors into operation-specific ones.""" + + def call( + self, + function: Callable[..., Any], + error: type[Exception], + *args: Any, + **kwargs: Any, + ) -> Any: + """Result of ``function``, raising ``error`` instead of an HTTP error.""" + + +@runtime_checkable +class JsonEncodingPort(Protocol): + """Encodes the objects the standard JSON encoder does not support.""" + + def encode_default(self, o: Any) -> Any: + """JSON-serializable form of ``o``; TypeError when unsupported.""" + + +@runtime_checkable +class LoggingPort(Protocol): + """Configures and feeds the standard logging machinery.""" + + def setup_logging_config( + self, + level: int, + json_logging: bool, + formatter_class: type[logging.Formatter], + ) -> None: + """Configure the root logger, as JSON with ``formatter_class`` or text.""" + + def get_logger(self, name: str) -> logging.Logger: + """The named standard logger.""" + + def prepare_meta(self, meta: Any) -> dict[str, Any] | None: + """The ``extra`` mapping carrying the structured metadata.""" + + def set_library_level(self, lib: str, log_level: int) -> None: + """Override the log level of a third-party library.""" + + def complete_record( + self, log_record: dict[str, Any], record: logging.LogRecord + ) -> None: + """Add the timestamp and uppercase level fields of a JSON record.""" + + +@runtime_checkable +class DictToolsPort(Protocol): + """Reshapes the dictionaries exchanged with the platform.""" + + def copy_dict(self, *, src: dict[str, Any], dest: dict[str, Any]) -> None: + """Copy ``src`` into ``dest``, flattening nested dicts as ``key[sub]``.""" + + def remove_none_from_dict(self, data: dict[str, Any]) -> dict[str, Any]: + """New dict without the None-valued entries.""" + + +@runtime_checkable +class CertificatesPort(Protocol): + """Loads certificates, files or in-memory PEM data, into SSL contexts.""" + + def data_to_temp_file(self, data: str) -> str: + """Path of a new owner-only temporary file holding ``data``.""" + + def is_memory_certificate(self, certificate: str) -> bool: + """Whether ``certificate`` is PEM data rather than a file path.""" + + def ssl_cert_chain( + self, + ssl_context: ssl.SSLContext, + cert_data: str | None, + key_data: str | None, + passphrase: str | None, + ) -> None: + """Load a certificate chain into ``ssl_context``.""" + + def ssl_verify_locations( + self, ssl_context: ssl.SSLContext, certdata: str | None + ) -> None: + """Load CA verification locations into ``ssl_context``.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/protocols.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/protocols.py new file mode 100644 index 00000000..97964ebb --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/protocols.py @@ -0,0 +1,152 @@ +"""Consumer-facing protocols: the API client, daemons and loggers. + +Extension SDKs and consumers type-hint against these protocols instead of a +concrete implementation. +""" + +from typing import Any, Callable, Protocol, runtime_checkable + +__all__ = ["AppLoggerProtocol", "BaseClient", "DaemonProtocol"] + + +@runtime_checkable +class AppLoggerProtocol(Protocol): + """Behavioral contract for SDK logging services. + + Any logger implementation used by SDK consumers must satisfy this + protocol. The built-in AppLogger class implements it. + + The __call__ method allows switching the logger name (factory pattern): + >>> log = AppLogger(logging.DEBUG) + >>> module_log = log("my_module") # returns self with new name + """ + + def __call__(self, name: str) -> "AppLoggerProtocol": + """Switch logger name and return self.""" + ... + + def debug(self, message: str, meta: Any = None) -> None: + """Log at DEBUG level.""" + ... + + def info(self, message: str, meta: Any = None) -> None: + """Log at INFO level.""" + ... + + def warning(self, message: str, meta: Any = None) -> None: + """Log at WARNING level.""" + ... + + def error(self, message: str, meta: Any = None) -> None: + """Log at ERROR level with traceback.""" + ... + + @staticmethod + def setup_logger_level(lib: str, log_level: int) -> None: + """Override log level for a third-party library.""" + ... + + +@runtime_checkable +class BaseClient(Protocol): + """Protocol defining the behavioral contract for an OpenAEV HTTP client. + + Any class implementing all methods and properties defined here + will pass isinstance(obj, BaseClient) checks at runtime. + + IMPORTANT: Properties are used (not bare attributes) to ensure + isinstance checks work on Python 3.11 where @runtime_checkable + only inspects methods, not data attributes. + """ + + @property + def per_page(self) -> int: + """Number of items per page requested from the API.""" + + @property + def pagination(self) -> str: + """Pagination mode requested from the API.""" + + @property + def order_by(self) -> str: + """Sort key requested from the API.""" + + @property + def url(self) -> str: + """Base URL of the OpenAEV API.""" + + @property + def headers(self) -> dict[str, str]: + """HTTP headers sent with every request.""" + + @property + def ssl_verify(self) -> bool | str: + """TLS verification flag or CA bundle path.""" + + @property + def timeout(self) -> int | float | None: + """Request timeout in seconds.""" + + @property + def tenant_id(self) -> str | None: + """Tenant scoping the API requests, if any.""" + + @property + def session(self) -> Any: + """The underlying HTTP session (e.g. requests.Session).""" + ... + + @property + def backend(self) -> Any: + """The HTTP backend implementation handling transport.""" + ... + + def http_head(self, path: str, **kwargs: Any) -> Any: + """Send a HEAD request.""" + + def http_get(self, path: str, **kwargs: Any) -> Any: + """Send a GET request.""" + + def http_post(self, path: str, **kwargs: Any) -> Any: + """Send a POST request.""" + + def http_patch(self, path: str, **kwargs: Any) -> Any: + """Send a PATCH request.""" + + def http_put(self, path: str, **kwargs: Any) -> Any: + """Send a PUT request.""" + + def http_delete(self, path: str, **kwargs: Any) -> Any: + """Send a DELETE request.""" + + def http_list(self, path: str, **kwargs: Any) -> Any: + """Send a paginated GET request.""" + + +@runtime_checkable +class DaemonProtocol(Protocol): + """Protocol defining the behavioral contract for a daemon runtime. + + Any class implementing all methods defined here will pass + isinstance(obj, DaemonProtocol) checks at runtime. + + The three methods capture the public lifecycle surface shared by + CollectorDaemon and injector daemon runtimes: + + - ``start()`` — run setup then enter the main loop. + - ``set_callback(callback)`` — configure the periodic callback. + - ``get_id()`` — return the daemon instance identifier. + + IMPORTANT: Only public methods are part of this contract. + Implementation details (_setup, _start_loop, _try_callback) + are excluded — they belong to the concrete daemon class. + """ + + def start(self) -> None: + """Run the daemon setup, then its main loop.""" + + def set_callback(self, callback: Callable[..., Any]) -> None: + """Configure the callback run by the main loop.""" + + def get_id(self) -> str | None: + """Return the daemon instance identifier.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/types.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/security_domains.py similarity index 100% rename from libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/security_domain/types.py rename to libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/security_domains.py diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/values.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/values.py new file mode 100644 index 00000000..75713bbd --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/values.py @@ -0,0 +1,98 @@ +"""Value objects shared across capabilities: encoded identifiers and +field-presence rules.""" + +import dataclasses +import urllib.parse +from typing import Any + +__all__ = ["EncodedId", "RequiredOptional"] + + +@dataclasses.dataclass(frozen=True) +class RequiredOptional: + """Immutable field-presence validator for API payloads. + + Validates that required fields are present, optional fields are allowed, + and exclusive fields follow mutual exclusion rules. + + Args: + required: Tuple of field names that must be present. + optional: Tuple of field names that may be present. + exclusive: Tuple of field names where exactly one must be present. + + Example: + >>> ro = RequiredOptional(required=("name", "type"), exclusive=("id", "key")) + >>> ro.validate_attrs(data={"name": "x", "type": "y", "id": "123"}) # OK + >>> ro.validate_attrs(data={"name": "x"}) # raises AttributeError + """ + + required: tuple[str, ...] = () + optional: tuple[str, ...] = () + exclusive: tuple[str, ...] = () + + def validate_attrs( + self, + *, + data: dict[str, Any], + excludes: list[str] | None = None, + ) -> None: + """Validate field presence against the configured rules. + + Args: + data: Dict of field names to values to validate. + excludes: Field names to skip during required-field checks. + + Raises: + AttributeError: If required fields are missing, or exclusive + field constraints are violated. + """ + if excludes is None: + excludes = [] + + if self.required: + required = [k for k in self.required if k not in excludes] + missing = [attr for attr in required if attr not in data] + if missing: + raise AttributeError(f"Missing attributes: {', '.join(missing)}") + + if self.exclusive: + exclusives = [attr for attr in data if attr in self.exclusive] + if len(exclusives) > 1: + raise AttributeError( + f"Provide only one of these attributes: {', '.join(exclusives)}" + ) + if not exclusives: + raise AttributeError( + f"Must provide one of these attributes: " + f"{', '.join(self.exclusive)}" + ) + + +class EncodedId(str): + """URL-safe encoded identifier. + + Wraps a string or integer value with URL percent-encoding applied. + Idempotent: encoding an already-encoded value returns it unchanged. + + Args: + value: The raw identifier to encode. Accepts str, int, or EncodedId. + + Raises: + TypeError: If value is not str, int, or EncodedId. + + Example: + >>> EncodedId("hello world") + 'hello%20world' + >>> EncodedId(42) + '42' + """ + + def __new__(cls, value: "str | int | EncodedId") -> "EncodedId": + if isinstance(value, EncodedId): + return value + + if not isinstance(value, (int, str)): + raise TypeError(f"Unsupported type received: {type(value)}") + if isinstance(value, str): + value = urllib.parse.quote(value, safe="") + return super().__new__(cls, value) diff --git a/pyoaev/apis/attack_pattern.py b/pyoaev/apis/attack_pattern.py index 2d96fde5..ffd035ad 100644 --- a/pyoaev/apis/attack_pattern.py +++ b/pyoaev/apis/attack_pattern.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List -from xtm_oaev_sdk.seam import common as exc +import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/collector.py b/pyoaev/apis/collector.py index a134bff4..c49cd450 100644 --- a/pyoaev/apis/collector.py +++ b/pyoaev/apis/collector.py @@ -1,7 +1,7 @@ from typing import Any, Dict +import xtm_oaev_sdk as exc from xtm_oaev_sdk import RequiredOptional -from xtm_oaev_sdk.seam import common as exc from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, GetMixin, ListMixin, UpdateMixin diff --git a/pyoaev/apis/cve.py b/pyoaev/apis/cve.py index 50a9d8fc..5bd0122f 100644 --- a/pyoaev/apis/cve.py +++ b/pyoaev/apis/cve.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from xtm_oaev_sdk.seam import common as exc +import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/document.py b/pyoaev/apis/document.py index 98c41225..4089a25a 100644 --- a/pyoaev/apis/document.py +++ b/pyoaev/apis/document.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from xtm_oaev_sdk.seam import common as exc +import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/endpoint.py b/pyoaev/apis/endpoint.py index 2a1b5452..f65a07b3 100644 --- a/pyoaev/apis/endpoint.py +++ b/pyoaev/apis/endpoint.py @@ -1,7 +1,7 @@ from typing import Any, Dict +import xtm_oaev_sdk as exc from xtm_oaev_sdk import RequiredOptional -from xtm_oaev_sdk.seam import common as exc from pyoaev.apis.inputs.search import SearchPaginationInput from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/inject.py b/pyoaev/apis/inject.py index 73a1abe9..bd5a376e 100644 --- a/pyoaev/apis/inject.py +++ b/pyoaev/apis/inject.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from xtm_oaev_sdk.seam import common as exc +import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/inject_expectation/inject_expectation.py b/pyoaev/apis/inject_expectation/inject_expectation.py index 9fede64d..27a1722a 100644 --- a/pyoaev/apis/inject_expectation/inject_expectation.py +++ b/pyoaev/apis/inject_expectation/inject_expectation.py @@ -1,12 +1,12 @@ from typing import Any, Dict, List +import xtm_oaev_sdk as exc from collectors_sdk import ( DetectionExpectation, ExpectationTypeEnum, PreventionExpectation, ) from xtm_oaev_sdk import RequiredOptional -from xtm_oaev_sdk.seam import common as exc from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import ListMixin, UpdateMixin diff --git a/pyoaev/apis/inject_expectation_trace.py b/pyoaev/apis/inject_expectation_trace.py index 2792839e..64587080 100644 --- a/pyoaev/apis/inject_expectation_trace.py +++ b/pyoaev/apis/inject_expectation_trace.py @@ -1,7 +1,7 @@ from typing import Any, Dict, List +import xtm_oaev_sdk as exc from xtm_oaev_sdk import RequiredOptional -from xtm_oaev_sdk.seam import common as exc from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin diff --git a/pyoaev/apis/injector_contract.py b/pyoaev/apis/injector_contract.py index 2147a289..7c53f13f 100644 --- a/pyoaev/apis/injector_contract.py +++ b/pyoaev/apis/injector_contract.py @@ -1,7 +1,7 @@ from typing import Any, Dict +import xtm_oaev_sdk as exc from xtm_oaev_sdk import RequiredOptional -from xtm_oaev_sdk.seam import common as exc from pyoaev.apis.inputs.search import InjectorContractSearchPaginationInput from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/kill_chain_phase.py b/pyoaev/apis/kill_chain_phase.py index 9a565051..5e7677c6 100644 --- a/pyoaev/apis/kill_chain_phase.py +++ b/pyoaev/apis/kill_chain_phase.py @@ -1,6 +1,6 @@ from typing import Any, Dict, List -from xtm_oaev_sdk.seam import common as exc +import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/payload.py b/pyoaev/apis/payload.py index cad3e645..7cb9f2f2 100644 --- a/pyoaev/apis/payload.py +++ b/pyoaev/apis/payload.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from xtm_oaev_sdk.seam import common as exc +import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/security_platform.py b/pyoaev/apis/security_platform.py index 2c22edd8..12052b97 100644 --- a/pyoaev/apis/security_platform.py +++ b/pyoaev/apis/security_platform.py @@ -1,7 +1,7 @@ from typing import Any, Dict +import xtm_oaev_sdk as exc from xtm_oaev_sdk import RequiredOptional -from xtm_oaev_sdk.seam import common as exc from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, GetMixin, ListMixin, UpdateMixin diff --git a/pyoaev/apis/signature.py b/pyoaev/apis/signature.py index 80dbbed6..c4196011 100644 --- a/pyoaev/apis/signature.py +++ b/pyoaev/apis/signature.py @@ -5,6 +5,7 @@ import time from typing import Any +import xtm_oaev_sdk as exc from pydantic import ValidationError from xtm_oaev_sdk import ( ExecutionDetails, @@ -12,7 +13,6 @@ SignatureOutputStructure, SignatureTransmissionError, ) -from xtm_oaev_sdk.seam import common as exc from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/tag.py b/pyoaev/apis/tag.py index 8ee34127..0d0c914d 100644 --- a/pyoaev/apis/tag.py +++ b/pyoaev/apis/tag.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from xtm_oaev_sdk.seam import common as exc +import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/apis/team.py b/pyoaev/apis/team.py index 82b9a00a..27ed4ace 100644 --- a/pyoaev/apis/team.py +++ b/pyoaev/apis/team.py @@ -1,7 +1,7 @@ from typing import Any, Dict +import xtm_oaev_sdk as exc from xtm_oaev_sdk import RequiredOptional -from xtm_oaev_sdk.seam import common as exc from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, ListMixin, UpdateMixin diff --git a/pyoaev/apis/user.py b/pyoaev/apis/user.py index e3fff082..a6628b44 100644 --- a/pyoaev/apis/user.py +++ b/pyoaev/apis/user.py @@ -1,7 +1,7 @@ from typing import Any, Dict +import xtm_oaev_sdk as exc from xtm_oaev_sdk import RequiredOptional -from xtm_oaev_sdk.seam import common as exc from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, ListMixin, UpdateMixin diff --git a/pyoaev/apis/vulnerability.py b/pyoaev/apis/vulnerability.py index 587001ac..433c444b 100644 --- a/pyoaev/apis/vulnerability.py +++ b/pyoaev/apis/vulnerability.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from xtm_oaev_sdk.seam import common as exc +import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject diff --git a/pyoaev/client.py b/pyoaev/client.py index 3f39144b..b02e3a58 100644 --- a/pyoaev/client.py +++ b/pyoaev/client.py @@ -3,8 +3,8 @@ from uuid import UUID import requests +import xtm_oaev_sdk as exceptions from xtm_oaev_sdk import copy_dict -from xtm_oaev_sdk.seam import common as exceptions from pyoaev import utils from pyoaev._version import __version__ # noqa: F401 diff --git a/pyoaev/mixins.py b/pyoaev/mixins.py index c07bea11..411fbf56 100644 --- a/pyoaev/mixins.py +++ b/pyoaev/mixins.py @@ -12,8 +12,8 @@ ) import requests +import xtm_oaev_sdk as exc from xtm_oaev_sdk import EncodedId -from xtm_oaev_sdk.seam import common as exc import pyoaev from pyoaev import base From 6725c1d090a91692867da3035bad4a2e279fc1c2 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 15:25:37 +0200 Subject: [PATCH 11/22] :recycle: refactor(sdk): enhancing port-adapter public boundaries for daemons --- libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 3 +- .../xtm_oaev_sdk/_core/daemons/__init__.py | 8 +- .../xtm_oaev_sdk/_core/daemons/heartbeat.py | 34 ++++++ .../xtm_oaev_sdk/_core/daemons/ping_alive.py | 42 ------- .../xtm_oaev_sdk/_core/daemons/runtime.py | 80 ++++++++++++ .../xtm_oaev_sdk/public/__init__.py | 2 +- .../base_daemon.py => public/daemons.py} | 114 +++++++++--------- .../xtm_oaev_sdk/seam/__init__.py | 11 +- .../xtm_oaev_sdk/seam/daemons/__init__.py | 16 ++- .../xtm_oaev_sdk/seam/daemons/ports.py | 62 ++++++++++ 10 files changed, 256 insertions(+), 116 deletions(-) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/heartbeat.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/ping_alive.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/runtime.py rename libs/xtm-oaev-sdk/xtm_oaev_sdk/{_core/daemons/base_daemon.py => public/daemons.py} (58%) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/ports.py diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py index 27f32458..b898c1ec 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -10,13 +10,14 @@ seam/common/ Shared errors, enums, value objects, protocols, ports seam/configuration/ Configuration contracts and ports (closed, versioned) seam/contracts/ Scenario contracts and ports (closed, versioned) - seam/daemons/ Daemon runtime (API client injected) + seam/daemons/ Daemon runtime ports (closed, versioned) seam/signatures/ Signatures inbound I/O seam (versioned) seam/port_io/ Port I/O wire schema (versioned, seam-owned) seam/deprecation.py PEP 562 deprecation primitive public/common Common adapters public/configuration Configuration adapters public/contracts Scenario contract adapters + public/daemons Daemon runtime adapters (API client injected) public/ User-facing re-exports Port/adapter layering, capability by capability (see ``seam``): diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py index 842d589c..6a5b1fc7 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/__init__.py @@ -1,6 +1,6 @@ -"""Daemon runtime shared by collectors and injectors.""" +"""Daemons core: implementations of the daemons seam ports.""" -from .base_daemon import BaseDaemon -from .ping_alive import PingAlive +from .heartbeat import Heartbeat +from .runtime import DaemonRuntime -__all__ = ["BaseDaemon", "PingAlive"] +__all__ = ["DaemonRuntime", "Heartbeat"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/heartbeat.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/heartbeat.py new file mode 100644 index 00000000..5e8a57c6 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/heartbeat.py @@ -0,0 +1,34 @@ +"""Heartbeat: keep a daemon registered on the platform.""" + +import threading +from typing import Any + +__all__ = ["Heartbeat"] + + +class Heartbeat: + """Implements ``HeartbeatPort``.""" + + def tenant_of(self, api: Any) -> Any: + return getattr(api, "tenant_id", None) + + def tenant_meta(self, tenant_id: Any) -> dict[str, Any]: + return {"tenant_id": str(tenant_id) if tenant_id else None} + + def ping( + self, + api: Any, + config: Any, + ping_type: str, + logger: Any, + exit_event: threading.Event, + ) -> None: + while not exit_event.is_set(): + try: + if ping_type == "injector": + api.injector.create(config, False) + else: + api.collector.create(config, False) + except Exception as err: # pylint: disable=broad-except + logger.error("Error pinging the API: " + str(err)) + exit_event.wait(40) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/ping_alive.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/ping_alive.py deleted file mode 100644 index 65f354e8..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/ping_alive.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Heartbeat thread keeping a daemon registered on the platform.""" - -import threading - -__all__ = ["PingAlive"] - - -class PingAlive(threading.Thread): - def __init__(self, api, config, logger, ping_type) -> None: - threading.Thread.__init__(self) - self.ping_type = ping_type - self.api = api - self.tenant_id = getattr(self.api, "tenant_id", None) - self.config = config - self.logger = logger - self.in_error = False - self.exit_event = threading.Event() - - def ping(self) -> None: - while not self.exit_event.is_set(): - try: - if self.ping_type == "injector": - self.api.injector.create(self.config, False) - else: - self.api.collector.create(self.config, False) - except Exception as err: # pylint: disable=broad-except - self.logger.error("Error pinging the API: " + str(err)) - self.exit_event.wait(40) - - def run(self) -> None: - self.logger.info( - "Starting PingAlive thread", - {"tenant_id": str(self.tenant_id) if self.tenant_id else None}, - ) - self.ping() - - def stop(self) -> None: - self.logger.info( - "Preparing PingAlive for clean shutdown", - {"tenant_id": str(self.tenant_id) if self.tenant_id else None}, - ) - self.exit_event.set() diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/runtime.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/runtime.py new file mode 100644 index 00000000..80968cb0 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/runtime.py @@ -0,0 +1,80 @@ +"""Daemon runtime: lifecycle shared by collector and injector daemons.""" + +import argparse +from inspect import signature +from types import FunctionType +from typing import Any, Callable + +from xtm_oaev_sdk.seam.common import BaseClient, OpenAEVError +from xtm_oaev_sdk.seam.configuration import ConfigurationProtocol + +__all__ = ["DaemonRuntime"] + + +class DaemonRuntime: + """Implements ``DaemonRuntimePort``.""" + + # compatibility layer: in order for older configs to still work, search for legacy names + def resolve_log_level(self, configuration: ConfigurationProtocol) -> str: + return ( + configuration.get("log_level") + or configuration.get("collector_log_level") + or configuration.get("injector_log_level") + or "info" + ) + + def resolve_name(self, configuration: ConfigurationProtocol) -> str: + return ( + configuration.get("name") + or configuration.get("collector_name") + or configuration.get("injector_name") + or "daemon" + ) + + def resolve_id(self, configuration: ConfigurationProtocol) -> Any: + return ( + configuration.get("id") + or configuration.get("collector_id") + or configuration.get("injector_id") + ) + + def invoke_callback( + self, callback: Callable[..., Any] | None, daemon: Any, logger: Any + ) -> None: + try: + # this is some black magic to allow injecting the collector daemon instance + # into an arbitrary callback that has a specific argument name + # this allow for avoiding subclassing the CollectorDaemon class just to provide the callback + # Example: + # + # def standalone_func(collector): + # collector.api.call_openaev() + # + # CollectorDaemon(config=, standalone_func).start() + if ( + isinstance(callback, FunctionType) + and "collector" in signature(callback).parameters + ): + callback(collector=daemon) + else: + callback() + except Exception as err: # pylint: disable=broad-except + logger.error(f"Error calling: {err}") + + def dump_config_schema_requested(self) -> bool: + parser = argparse.ArgumentParser(description="parse daemon options") + parser.add_argument("--dump-config-schema", action="store_true") + args = parser.parse_args() + return bool(args.dump_config_schema) + + def dump_config_schema(self, configuration: ConfigurationProtocol) -> None: + print(configuration.schema()) + + def ensure_callback(self, callback: Callable[..., Any] | None) -> None: + if callback is None: + raise OpenAEVError("This daemon has no configured callback.") + + def default_api_client(self) -> BaseClient: + raise OpenAEVError( + "No api_client was provided to the daemon; inject an OpenAEV API client." + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index f8b476e7..c910c9b7 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -103,7 +103,7 @@ ) # --- Daemons --- -from xtm_oaev_sdk.seam.daemons import BaseDaemon, PingAlive +from xtm_oaev_sdk.public.daemons import BaseDaemon, PingAlive # --- Seam / compatibility --- from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/daemons.py similarity index 58% rename from libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py rename to libs/xtm-oaev-sdk/xtm_oaev_sdk/public/daemons.py index 03773326..2a292756 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/daemons/base_daemon.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/daemons.py @@ -1,17 +1,22 @@ -"""Base daemon: the shared runtime of collector and injector daemons.""" +"""Daemon adapters: the base daemon runtime and its heartbeat. -import argparse +Each adapter speaks the daemons seam types and delegates to the core +implementation of its port. The API client is injected (``BaseClient``). +""" + +import threading from abc import ABC, abstractmethod -from inspect import signature -from types import FunctionType -from typing import TYPE_CHECKING -from xtm_oaev_sdk.seam.common import BaseClient, OpenAEVError +from xtm_oaev_sdk._core.daemons import DaemonRuntime, Heartbeat +from xtm_oaev_sdk.public.common import logger +from xtm_oaev_sdk.public.configuration import Configuration +from xtm_oaev_sdk.seam.common import BaseClient +from xtm_oaev_sdk.seam.daemons import DaemonRuntimePort, HeartbeatPort -if TYPE_CHECKING: # annotation only: the core never imports the public layer - from xtm_oaev_sdk.public.configuration import Configuration +__all__ = ["BaseDaemon", "PingAlive"] -__all__ = ["BaseDaemon"] +_runtime: DaemonRuntimePort = DaemonRuntime() +_heartbeat: HeartbeatPort = Heartbeat() class BaseDaemon(ABC): @@ -35,7 +40,7 @@ class BaseDaemon(ABC): def __init__( self, - configuration: "Configuration", + configuration: Configuration, callback: callable = None, logger=None, api_client: BaseClient | None = None, @@ -46,18 +51,8 @@ def __init__( # logging # compatibility layer: in order for older configs to still work, search for legacy names - actual_log_level = ( - self._configuration.get("log_level") - or self._configuration.get("collector_log_level") - or self._configuration.get("injector_log_level") - or "info" - ) - actual_log_name = ( - self._configuration.get("name") - or self._configuration.get("collector_name") - or self._configuration.get("injector_name") - or "daemon" - ) + actual_log_level = _runtime.resolve_log_level(self._configuration) + actual_log_name = _runtime.resolve_name(self._configuration) self.logger = logger or BaseDaemon.__get_default_logger( actual_log_level.upper(), actual_log_name, @@ -82,40 +77,18 @@ def _try_callback(self): it is immediately swallowed (but still logged) allowing the collector to keep running. This is useful for any transient issue (e.g. API endpoint down...). """ - try: - # this is some black magic to allow injecting the collector daemon instance - # into an arbitrary callback that has a specific argument name - # this allow for avoiding subclassing the CollectorDaemon class just to provide the callback - # Example: - # - # def standalone_func(collector): - # collector.api.call_openaev() - # - # CollectorDaemon(config=, standalone_func).start() - if ( - isinstance(self._callback, FunctionType) - and "collector" in signature(self._callback).parameters - ): - self._callback(collector=self) - else: - self._callback() - except Exception as err: # pylint: disable=broad-except - self.logger.error(f"Error calling: {err}") + _runtime.invoke_callback(self._callback, self, self.logger) def start(self): """Start the daemon. This will run the implementor's run-once setup method and follow-up with the main execution loop. Note that at this point, if there is no configured callback, the method will abort and kill the daemon. """ - parser = argparse.ArgumentParser(description="parse daemon options") - parser.add_argument("--dump-config-schema", action="store_true") - args = parser.parse_args() - if args.dump_config_schema: - print(self._configuration.schema()) + if _runtime.dump_config_schema_requested(): + _runtime.dump_config_schema(self._configuration) return - if self._callback is None: - raise OpenAEVError("This daemon has no configured callback.") + _runtime.ensure_callback(self._callback) self._setup() self._start_loop() @@ -129,11 +102,7 @@ def get_id(self): """Returns the daemon instance's ID contained in configuration. Configuration must define any of these keys: `id`, `collector_id`, `injector_id`. """ - return ( - self._configuration.get("id") - or self._configuration.get("collector_id") - or self._configuration.get("injector_id") - ) + return _runtime.resolve_id(self._configuration) def _create_default_api_client(self) -> BaseClient: """Builds the API client used when none was injected. The SDK has no @@ -141,13 +110,40 @@ def _create_default_api_client(self) -> BaseClient: construct one from the ``openaev_url``, ``openaev_token`` and ``openaev_tenant_id`` configuration keys). """ - raise OpenAEVError( - "No api_client was provided to the daemon; inject an OpenAEV API client." - ) + return _runtime.default_api_client() @classmethod def __get_default_logger(cls, log_level, name): - # transitional until the daemons capability is layered - from xtm_oaev_sdk.public.common import logger - return logger(log_level)(name) + + +class PingAlive(threading.Thread): + """Heartbeat thread keeping a daemon registered on the platform.""" + + def __init__(self, api, config, logger, ping_type) -> None: + threading.Thread.__init__(self) + self.ping_type = ping_type + self.api = api + self.tenant_id = _heartbeat.tenant_of(self.api) + self.config = config + self.logger = logger + self.in_error = False + self.exit_event = threading.Event() + + def ping(self) -> None: + _heartbeat.ping( + self.api, self.config, self.ping_type, self.logger, self.exit_event + ) + + def run(self) -> None: + self.logger.info( + "Starting PingAlive thread", _heartbeat.tenant_meta(self.tenant_id) + ) + self.ping() + + def stop(self) -> None: + self.logger.info( + "Preparing PingAlive for clean shutdown", + _heartbeat.tenant_meta(self.tenant_id), + ) + self.exit_event.set() diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py index b6551d5b..03f99a0f 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -13,8 +13,8 @@ definitions, field elements, enums, template variables), the builder protocol and the ports its public adapters delegate to; closed and versioned (``SEAM_VERSION``). -- **``daemons``** — the daemon runtime shared by collectors and injectors; - the API client is injected through the ``BaseClient`` protocol. +- **``daemons``** — the ports of the daemon runtime shared by collectors and + injectors (lifecycle, heartbeat); closed and versioned (``SEAM_VERSION``). - **``signatures``** — the signatures capability's inbound seam (``seam/signatures/seam.py``), versioned ``SEAM_VERSION``. - **``port_io``** — the port I/O wire schema crossing the SDK↔platform @@ -30,9 +30,10 @@ core. As long as the core keeps answering the seam, the public surface keeps working; a seam change is a contract change (``SEAM_VERSION`` bump, changelog). -``common``, ``configuration``, ``contracts``, ``seam/signatures/seam.py`` and -``seam/port_io/models.py`` are closed today; the other packages still re-export ``_core`` until their -capability is migrated. +``common``, ``configuration``, ``contracts``, ``daemons``, +``seam/signatures/seam.py`` and ``seam/port_io/models.py`` are closed today; +the other packages still re-export ``_core`` until their capability is +migrated. """ from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py index 1ec21077..4f2e43c0 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/__init__.py @@ -1,6 +1,14 @@ -"""Daemon runtime: the base daemon and its heartbeat, shared by the collector -and injector daemons. The API client is injected (``BaseClient``).""" +"""Daemons seam: the contract between the public daemon runtime and the core. -from xtm_oaev_sdk._core.daemons import BaseDaemon, PingAlive +The daemon consumer protocols (``DaemonProtocol``, ``BaseClient``) belong to +the common seam. Closed: it imports nothing from ``xtm_oaev_sdk._core`` nor +``xtm_oaev_sdk.public``. Any change to these contracts must bump +:data:`SEAM_VERSION`. +""" -__all__ = ["BaseDaemon", "PingAlive"] +from xtm_oaev_sdk.seam.daemons.ports import DaemonRuntimePort, HeartbeatPort + +#: Version of the daemons seam. Bump on any contract change. +SEAM_VERSION = "1.0.0" + +__all__ = ["SEAM_VERSION", "DaemonRuntimePort", "HeartbeatPort"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/ports.py new file mode 100644 index 00000000..fa79cb96 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/daemons/ports.py @@ -0,0 +1,62 @@ +"""Daemon ports: the behaviour the public daemon runtime expects from the core.""" + +import threading +from typing import Any, Callable, Protocol, runtime_checkable + +from xtm_oaev_sdk.seam.common import BaseClient +from xtm_oaev_sdk.seam.configuration import ConfigurationProtocol + +__all__ = ["DaemonRuntimePort", "HeartbeatPort"] + + +@runtime_checkable +class DaemonRuntimePort(Protocol): + """Runs the shared lifecycle of collector and injector daemons.""" + + def resolve_log_level(self, configuration: ConfigurationProtocol) -> str: + """Log level, from the generic or the legacy collector/injector keys.""" + + def resolve_name(self, configuration: ConfigurationProtocol) -> str: + """Daemon name, from the generic or the legacy collector/injector keys.""" + + def resolve_id(self, configuration: ConfigurationProtocol) -> Any: + """Daemon id, from ``id``, ``collector_id`` or ``injector_id``.""" + + def invoke_callback( + self, callback: Callable[..., Any] | None, daemon: Any, logger: Any + ) -> None: + """Call the callback (injecting the daemon as ``collector`` when asked + for), logging and swallowing any error.""" + + def dump_config_schema_requested(self) -> bool: + """Whether the daemon was started with ``--dump-config-schema``.""" + + def dump_config_schema(self, configuration: ConfigurationProtocol) -> None: + """Print the configuration JSON schema.""" + + def ensure_callback(self, callback: Callable[..., Any] | None) -> None: + """Raise when the daemon has no callback to run.""" + + def default_api_client(self) -> BaseClient: + """The API client used when none is injected: none, the SDK refuses.""" + + +@runtime_checkable +class HeartbeatPort(Protocol): + """Keeps a daemon registered on the platform.""" + + def tenant_of(self, api: Any) -> Any: + """Tenant the API client is scoped to, if any.""" + + def tenant_meta(self, tenant_id: Any) -> dict[str, Any]: + """Log metadata identifying the tenant.""" + + def ping( + self, + api: Any, + config: Any, + ping_type: str, + logger: Any, + exit_event: threading.Event, + ) -> None: + """Register the daemon every 40 seconds until ``exit_event`` is set.""" From 4da17393558e7b006335862c80da86c5fb777121 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 15:34:41 +0200 Subject: [PATCH 12/22] :recycle: refactor(sdk): enhancing port-adapter public boundaries for signatures --- .../seam/test_signatures_seam_schema_pin.py | 10 +- .../test_signature_manager_ip_resolution.py | 14 +- .../test_signature_manager_pre_execution.py | 4 +- libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 3 +- .../xtm_oaev_sdk/_core/signatures/__init__.py | 76 +--- .../_core/signatures/addresses.py | 60 +++ .../_core/signatures/ai_marker.py | 17 - .../xtm_oaev_sdk/_core/signatures/engine.py | 132 +++++++ .../xtm_oaev_sdk/_core/signatures/marker.py | 14 + .../xtm_oaev_sdk/_core/signatures/models.py | 129 ------- .../_core/signatures/signature_manager.py | 359 ------------------ .../_core/signatures/signature_match.py | 31 -- .../_core/signatures/signature_type.py | 54 --- .../xtm_oaev_sdk/_core/signatures/targets.py | 40 ++ .../xtm_oaev_sdk/_core/signatures/types.py | 65 ---- .../xtm_oaev_sdk/public/__init__.py | 40 +- .../xtm_oaev_sdk/public/signatures.py | 255 +++++++++++++ .../xtm_oaev_sdk/seam/__init__.py | 11 +- .../xtm_oaev_sdk/seam/signatures/__init__.py | 83 ++-- .../xtm_oaev_sdk/seam/signatures/models.py | 142 +++++++ .../xtm_oaev_sdk/seam/signatures/ports.py | 164 ++++++++ 21 files changed, 892 insertions(+), 811 deletions(-) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/addresses.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/ai_marker.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/engine.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/marker.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/models.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_type.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/targets.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/types.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/public/signatures.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/models.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/ports.py diff --git a/libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py b/libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py index 674262a7..6fa80d3a 100644 --- a/libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py +++ b/libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py @@ -87,15 +87,15 @@ def test_seam_imports_nothing_from_capability(): def test_reexports_preserve_identity(): - """The seam, the port's models module, and the public surface share one object.""" + """The seam module, the seam package, and the public surface share one object.""" import xtm_oaev_sdk as sdk - import xtm_oaev_sdk._core.signatures.models as models + import xtm_oaev_sdk.seam.signatures as seam_package for name, cls in SEAM_MODELS.items(): assert ( - getattr(models, name) is cls - ), f"{name}: models.py re-export broke identity" + getattr(seam_package, name) is cls + ), f"{name}: seam package re-export broke identity" assert ( getattr(sdk, name) is cls ), f"{name}: public surface re-export broke identity" - assert sdk.InjectorConfig is models.InjectorConfig + assert sdk.InjectorConfig is seam_package.InjectorConfig diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py index 4f488771..e1e7d42e 100644 --- a/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_ip_resolution.py @@ -36,11 +36,11 @@ def context(): def signature_manager_environment(context, monkeypatch, environment): if environment == "Docker": monkeypatch.setattr( - "xtm_oaev_sdk._core.signatures.signature_manager.socket.gethostbyname", + "socket.gethostbyname", lambda _: "172.17.0.2", ) monkeypatch.setattr( - "xtm_oaev_sdk._core.signatures.signature_manager.subprocess.run", + "subprocess.run", lambda *args, **kwargs: SimpleNamespace( returncode=0, stdout="172.17.0.2\n", @@ -48,11 +48,11 @@ def signature_manager_environment(context, monkeypatch, environment): ) elif environment == "Kubernetes": monkeypatch.setattr( - "xtm_oaev_sdk._core.signatures.signature_manager.socket.gethostbyname", + "socket.gethostbyname", lambda _: (_ for _ in ()).throw(OSError("socket fail")), ) monkeypatch.setattr( - "xtm_oaev_sdk._core.signatures.signature_manager.subprocess.run", + "subprocess.run", lambda *args, **kwargs: SimpleNamespace( returncode=0, stdout="10.244.0.8\n", @@ -60,7 +60,7 @@ def signature_manager_environment(context, monkeypatch, environment): ) else: monkeypatch.setattr( - "xtm_oaev_sdk._core.signatures.signature_manager.socket.gethostbyname", + "socket.gethostbyname", lambda _: "192.0.2.20", ) @@ -68,11 +68,11 @@ def signature_manager_environment(context, monkeypatch, environment): @given("all IP resolution strategies are mocked to fail") def all_ip_resolution_strategies_fail(context, monkeypatch): monkeypatch.setattr( - "xtm_oaev_sdk._core.signatures.signature_manager.socket.gethostbyname", + "socket.gethostbyname", lambda _: (_ for _ in ()).throw(OSError("socket fail")), ) monkeypatch.setattr( - "xtm_oaev_sdk._core.signatures.signature_manager.subprocess.run", + "subprocess.run", lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("command fail")), ) diff --git a/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py index 22f967f4..20743c33 100644 --- a/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py +++ b/libs/xtm-oaev-sdk/test/signatures/test_signature_manager_pre_execution.py @@ -211,7 +211,7 @@ def raw_mixed_target_list(value_1, value_2, value_3): ) def resolvable_container_ipv4(request): patcher = patch( - "xtm_oaev_sdk._core.signatures.signature_manager.SignatureManager.resolve_container_ip", + "xtm_oaev_sdk.SignatureManager.resolve_container_ip", return_value="172.17.0.2", ) patcher.start() @@ -227,7 +227,7 @@ def resolvable_container_ipv4(request): ) def resolvable_container_ipv4_explicit(request, source_ipv4): patcher = patch( - "xtm_oaev_sdk._core.signatures.signature_manager.SignatureManager.resolve_container_ip", + "xtm_oaev_sdk.SignatureManager.resolve_container_ip", return_value=source_ipv4, ) patcher.start() diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py index b898c1ec..95949819 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -11,13 +11,14 @@ seam/configuration/ Configuration contracts and ports (closed, versioned) seam/contracts/ Scenario contracts and ports (closed, versioned) seam/daemons/ Daemon runtime ports (closed, versioned) - seam/signatures/ Signatures inbound I/O seam (versioned) + seam/signatures/ Signature contracts, inbound I/O models and ports (closed) seam/port_io/ Port I/O wire schema (versioned, seam-owned) seam/deprecation.py PEP 562 deprecation primitive public/common Common adapters public/configuration Configuration adapters public/contracts Scenario contract adapters public/daemons Daemon runtime adapters (API client injected) + public/signatures Signature manager and helpers adapters public/ User-facing re-exports Port/adapter layering, capability by capability (see ``seam``): diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py index b4eed062..86b58811 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/__init__.py @@ -1,71 +1,13 @@ -"""Public surface for the signatures subpackage. +"""Signatures core: implementations of the signatures seam ports.""" -This module re-exports the canonical signature types, models, and manager -protocols used by the SDK engine signatures package. -""" - -from .ai_marker import build_marker -from .models import ( - CloudInjectorConfig, - ExecutionDetails, - ExecutionSignature, - ExpectationSignatureGroup, - ExtraSignatureData, - InjectorConfig, - NetworkInjectorConfig, - SignatureCallbackPayload, - SignatureOutputStructure, - SignaturePayload, - SignatureTarget, - SignatureValue, - TargetSignatures, - ToolErrorInfo, - ToolOutput, - ToolTimeoutInfo, - build_network_configs, -) -from .signature_manager import ( - SignatureManager, - SignatureManagerProtocol, - SignatureTransportProtocol, -) -from .signature_match import SignatureMatch -from .signature_type import SignatureType -from .types import ( - ExecutionStatus, - InjectExecutionActions, - MatchTypes, - SignatureExpectationType, - SignatureTypes, -) +from .addresses import ContainerAddressResolver +from .engine import SignatureEngine +from .marker import MarkerBuilder +from .targets import NetworkTargets __all__ = [ - "CloudInjectorConfig", - "ExecutionDetails", - "ExecutionSignature", - "ExecutionStatus", - "ExpectationSignatureGroup", - "ExtraSignatureData", - "InjectExecutionActions", - "InjectorConfig", - "MatchTypes", - "NetworkInjectorConfig", - "SignatureCallbackPayload", - "SignatureExpectationType", - "SignatureManager", - "SignatureManagerProtocol", - "SignatureMatch", - "SignatureOutputStructure", - "SignaturePayload", - "SignatureTarget", - "SignatureTransportProtocol", - "SignatureType", - "SignatureTypes", - "SignatureValue", - "TargetSignatures", - "ToolErrorInfo", - "ToolOutput", - "ToolTimeoutInfo", - "build_marker", - "build_network_configs", + "ContainerAddressResolver", + "MarkerBuilder", + "NetworkTargets", + "SignatureEngine", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/addresses.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/addresses.py new file mode 100644 index 00000000..bc8d8b80 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/addresses.py @@ -0,0 +1,60 @@ +"""Container addresses: the source IPs of network execution signatures.""" + +import logging +import os +import socket +import subprocess + +__all__ = ["ContainerAddressResolver"] + + +class ContainerAddressResolver: + """Implements ``ContainerAddressPort``.""" + + def resolve_ipv4(self, logger: logging.Logger) -> str | None: + env_ip = os.environ.get("CONTAINER_IP") + if env_ip: + return env_ip + + try: + ip = socket.gethostbyname(socket.gethostname()) + if ip and ip != "127.0.0.1": + return ip + except (socket.gaierror, OSError): + pass + + try: + result = subprocess.run( + ["hostname", "-i"], + capture_output=True, + text=True, + timeout=5, + check=False, + ) + if result.returncode == 0: + ip = result.stdout.strip().split()[0] + if ip and ip != "127.0.0.1": + return ip + except (OSError, RuntimeError, subprocess.TimeoutExpired): + pass + + logger.warning("Could not resolve container IP; returning 'unknown'") + return None + + def resolve_ipv6(self) -> str | None: + try: + infos = socket.getaddrinfo( + socket.gethostname(), None, socket.AF_INET6, socket.SOCK_STREAM + ) + for info in infos: + addr = info[4][0] + if ( + isinstance(addr, str) + and addr + and not addr.startswith("::1") + and not addr.startswith("fe80") + ): + return addr + except (socket.gaierror, OSError): + pass + return None diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/ai_marker.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/ai_marker.py deleted file mode 100644 index 67cd8638..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/ai_marker.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Deterministic per-inject canary marker shared by the AI red-team injector and the AI defense -collectors. - -The marker is derived purely from the inject (and optional agent) id, so the injector that sends the -attack and the collector that validates an AI defense response compute the same value independently, -without the platform having to store it. It is emitted by the injector (request header + in-prompt -token) and matched by collectors against guardrail / firewall logs. -""" - -import hashlib - -__all__ = ["build_marker"] - - -def build_marker(inject_id: str, agent_id: str = "") -> str: - seed = f"{inject_id}:{agent_id}".encode("utf-8") - return "oaev" + hashlib.sha256(seed).hexdigest()[:16] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/engine.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/engine.py new file mode 100644 index 00000000..5c3dca3b --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/engine.py @@ -0,0 +1,132 @@ +"""Signature engine: compile pre-execution signatures, merge tool output, +package the wire payload.""" + +from datetime import datetime, timezone +from typing import Any, Callable + +from pydantic import ValidationError +from xtm_oaev_sdk.seam.common import OpenAEVError +from xtm_oaev_sdk.seam.port_io import ( + ExecutionDetails, + ExpectationSignatureGroup, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, +) +from xtm_oaev_sdk.seam.signatures import ( + CloudInjectorConfig, + ExecutionSignature, + ExtraSignatureData, + InjectorConfig, + NetworkInjectorConfig, + ToolOutput, +) + +__all__ = ["SignatureEngine"] + + +class SignatureEngine: + """Implements ``SignatureEnginePort``.""" + + def now(self) -> datetime: + return datetime.now(timezone.utc) + + def validate_configs(self, configs: list[InjectorConfig]) -> None: + if not configs: + raise ValueError("build_execution_signatures requires at least one config") + first_type = type(configs[0]) + for c in configs: + if not isinstance(c, first_type): + raise ValueError( + "build_execution_signatures does not mix injector config types; " + f"got {sorted({type(c).__name__ for c in configs})}" + ) + + def format_start_time(self, now: datetime) -> str: + return now.strftime("%Y-%m-%dT%H:%M:%SZ") + + def source_context( + self, + config: InjectorConfig, + source_ipv4: Callable[[], str], + source_ipv6: Callable[[], str | None], + ) -> dict[str, Any]: + if isinstance(config, NetworkInjectorConfig): + return {"source_ipv4": source_ipv4(), "source_ipv6": source_ipv6()} + if isinstance(config, CloudInjectorConfig): + return {} + raise TypeError(f"unsupported injector config type: {type(config).__name__}") + + def build_signature( + self, config: InjectorConfig, start_time: str, source_context: dict[str, Any] + ) -> ExecutionSignature: + base: dict[str, Any] = {"start_time": start_time} + base.update(source_context) + base.update(config.model_dump(exclude_none=True)) + return ExecutionSignature(**base) + + def apply_tool_output( + self, + execution_details: ExecutionDetails, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + tool_output: dict[str, Any], + now: Callable[[], datetime], + ) -> None: + try: + tool_output = ToolOutput.model_validate(tool_output or {}) + except ValidationError as exc: + raise OpenAEVError(error_message=f"Invalid tool_output: {exc}") from exc + + now = now() + execution_details.post_execution_update(tool_output, now) + if isinstance(execution_signatures, list): + for exec_sig in execution_signatures: + exec_sig.post_execution_update(tool_output, now) + else: + execution_signatures.post_execution_update(tool_output, now) + + def build_payload( + self, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + targets_meta: Any | list[Any], + expectation_types: list[str], + extra_signatures: ExtraSignatureData, + ) -> dict[str, Any]: + if not isinstance(execution_signatures, list): + execution_signatures = [execution_signatures] + if isinstance(targets_meta, dict): + targets_meta = [targets_meta] * len(execution_signatures) + + targets = [] + for signature, target in zip(execution_signatures, targets_meta): + signature_values = [] + for expectation_type in expectation_types: + signature_data = signature.model_dump(exclude_none=True) + signature_data.update(extra_signatures.get_extra(expectation_type)) + values = [ + SignatureValue(signature_type=key, signature_value=value) + for key, value in signature_data.items() + ] + signature_values.append( + ExpectationSignatureGroup( + expectation_type=expectation_type, values=values + ) + ) + targets.append( + TargetSignatures( + signature_target=SignatureTarget( + agent=target.agent_id, + asset=target.asset_id, + asset_group=target.asset_group_id, + ), + signature_values=signature_values, + ) + ) + return SignaturePayload(targets=targets).model_dump() + + def output_structure(self, signatures: dict[str, Any]) -> SignatureOutputStructure: + return SignatureOutputStructure( + expectation_signatures=SignaturePayload(**signatures) + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/marker.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/marker.py new file mode 100644 index 00000000..f68ed684 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/marker.py @@ -0,0 +1,14 @@ +"""AI marker: deterministic per-inject canary shared by the AI red-team injector +and the AI defense collectors.""" + +import hashlib + +__all__ = ["MarkerBuilder"] + + +class MarkerBuilder: + """Implements ``MarkerPort``.""" + + def build_marker(self, inject_id: str, agent_id: str = "") -> str: + seed = f"{inject_id}:{agent_id}".encode("utf-8") + return "oaev" + hashlib.sha256(seed).hexdigest()[:16] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/models.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/models.py deleted file mode 100644 index 9d3756d1..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/models.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Pydantic schemas pinning every shape SignatureManager touches. - -The seam models live in the architectural seam layer and are re-exported -here so this module keeps its historical surface: - -- the inbound seam (the ``InjectorConfig`` family, ``ExecutionSignature``, - and the ``ToolOutput`` family) in - ``xtm_oaev_sdk.seam.signatures.seam`` (versioned ``SEAM_VERSION``); -- the observation / callback wire models (the port I/O contract) in - ``xtm_oaev_sdk.seam.port_io.models`` (versioned - ``PORT_IO_CONTRACT_VERSION``). - -Only ``ExtraSignatureData`` and the target-builder helpers are defined here -— they are the port's own shapes, internal to the capability. -""" - -import ipaddress -from typing import Any - -from pydantic import BaseModel, Field, JsonValue -from xtm_oaev_sdk.seam.port_io.models import ( - ExecutionDetails, - ExpectationSignatureGroup, - SignatureCallbackPayload, - SignatureOutputStructure, - SignaturePayload, - SignatureTarget, - SignatureValue, - TargetSignatures, -) -from xtm_oaev_sdk.seam.signatures.seam import ( - CloudInjectorConfig, - ExecutionSignature, - InjectorConfig, - NetworkInjectorConfig, - ToolErrorInfo, - ToolOutput, - ToolTimeoutInfo, -) - - -class ExtraSignatureData(BaseModel): - """Format for extra signatures added to the default signatures.""" - - detection: dict[str, JsonValue] = Field(default_factory=dict) - prevention: dict[str, JsonValue] = Field(default_factory=dict) - vulnerability: dict[str, JsonValue] = Field(default_factory=dict) - - def get_extra(self, expectation_type: str) -> dict[str, JsonValue]: - if expectation_type.lower() == "detection": - return self.detection - if expectation_type.lower() == "prevention": - return self.prevention - if expectation_type.lower() == "vulnerability": - return self.vulnerability - raise ValueError( - f"Expectation type should be one of the available parameters: {list(self.model_fields.keys())}" - ) - - -# --------------------------------------------------------------------------- -# Builders. Cheap helpers to turn raw injector input into typed configs. -# --------------------------------------------------------------------------- - - -def _classify_network_target(value: str) -> NetworkInjectorConfig: - """Decide whether ``value`` is an IPv4, IPv6, or hostname and wrap it.""" - try: - addr = ipaddress.ip_address(value) - except ValueError: - return NetworkInjectorConfig(target_hostname=value) - if isinstance(addr, ipaddress.IPv4Address): - return NetworkInjectorConfig(target_ipv4=value) - return NetworkInjectorConfig(target_ipv6=value) - - -def build_network_configs( - targets: list[str | dict[str, Any] | NetworkInjectorConfig], -) -> list[NetworkInjectorConfig]: - """Forge a list of `NetworkInjectorConfig` from a heterogeneous target list. - - Each item is one distinct asset. Accepted shapes: - - - `NetworkInjectorConfig`: passed through unchanged. - - `dict`: validated against :class:`NetworkInjectorConfig`. - - `str`: auto-classified into IPv4 / IPv6 / hostname. - - Args: - targets: Raw target list straight out of the injector. - - Returns: - One `NetworkInjectorConfig` per input target, order preserved. - - Raises: - TypeError: An item is not one of the accepted shapes. - ValidationError: A dict item fails the one-identity invariant. - """ - configs: list[NetworkInjectorConfig] = [] - for target in targets: - if isinstance(target, NetworkInjectorConfig): - configs.append(target) - elif isinstance(target, dict): - configs.append(NetworkInjectorConfig(**target)) - elif isinstance(target, str): - configs.append(_classify_network_target(target)) - else: - raise TypeError(f"unsupported network target type: {type(target).__name__}") - return configs - - -__all__ = [ - "SignatureValue", - "ExpectationSignatureGroup", - "ExtraSignatureData", - "SignatureTarget", - "TargetSignatures", - "SignaturePayload", - "SignatureOutputStructure", - "ExecutionDetails", - "SignatureCallbackPayload", - "ExecutionSignature", - "ToolErrorInfo", - "ToolTimeoutInfo", - "ToolOutput", - "NetworkInjectorConfig", - "CloudInjectorConfig", - "InjectorConfig", - "build_network_configs", -] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py deleted file mode 100644 index d1eae55e..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_manager.py +++ /dev/null @@ -1,359 +0,0 @@ -"""Signature lifecycle for OpenAEV injectors: compile pre, merge post, ship to backend.""" - -import logging -import os -import socket -import subprocess -from datetime import datetime, timezone -from typing import Any, Protocol, runtime_checkable - -from pydantic import ValidationError -from xtm_oaev_sdk._core.signatures.models import ( - CloudInjectorConfig, - ExecutionDetails, - ExecutionSignature, - ExpectationSignatureGroup, - ExtraSignatureData, - InjectorConfig, - NetworkInjectorConfig, - SignatureOutputStructure, - SignaturePayload, - SignatureTarget, - SignatureValue, - TargetSignatures, - ToolOutput, -) -from xtm_oaev_sdk.seam.common import OpenAEVError - -__all__ = [ - "SignatureManager", - "SignatureManagerProtocol", - "SignatureTransportProtocol", -] - - -@runtime_checkable -class SignatureTransportProtocol(Protocol): - """Transport contract for signature delivery. - - pyoaev implements this via its Signature API manager. - The SDK only knows this interface, never the concrete client. - """ - - def send_signatures( - self, - inject_id: str, - sig_output: SignatureOutputStructure, - execution_details: ExecutionDetails, - *, - max_payload_size: int, - logger: logging.Logger, - ) -> None: - """Transmit signatures to the platform callback endpoint.""" - ... - - -@runtime_checkable -class SignatureManagerProtocol(Protocol): - """Behavioral contract for the signature lifecycle manager. - - Covers the full compile → merge → transmit pipeline. - """ - - def build_execution_signatures( - self, - config: InjectorConfig | list[InjectorConfig], - ) -> ExecutionSignature | list[ExecutionSignature]: - """Build pre-execution signatures from injector configs.""" - ... - - def post_execution_updates( - self, - execution_details: ExecutionDetails, - execution_signatures: ExecutionSignature | list[ExecutionSignature], - tool_output: dict[str, Any], - ) -> None: - """Update execution metadata after tool completion.""" - ... - - def send_signatures( - self, - inject_id: str, - execution_details: ExecutionDetails, - signatures: dict[str, Any], - ) -> None: - """Ship signatures to the callback endpoint.""" - ... - - def resolve_container_ip(self) -> str: - """Resolve the container's primary IPv4 address.""" - ... - - -class SignatureManager: - """End-to-end signature pipeline: compile, merge, transmit. One class, three jobs.""" - - DEFAULT_MAX_PAYLOAD_SIZE = 5_242_880 # 5 MiB - - def __init__( - self, - transport: SignatureTransportProtocol, - logger: logging.Logger | None = None, - max_payload_size: int = DEFAULT_MAX_PAYLOAD_SIZE, - ) -> None: - """Initialize manager with a transport implementation for signature delivery.""" - self.transport = transport - self.logger = logger or logging.getLogger(__name__) - self.max_payload_size = max_payload_size - self._cached_ipv4: str | None = None - self._cached_ipv6: str | None = None - - def _utcnow(self) -> datetime: - """Current UTC time. Carved out so tests can pin the clock.""" - return datetime.now(timezone.utc) - - def build_execution_signatures( - self, - config: InjectorConfig | list[InjectorConfig], - ) -> ExecutionSignature | list[ExecutionSignature]: - """Build pre-execution signature dicts from one or more typed injector configs. - - The category is carried by the config type itself - (:class:`NetworkInjectorConfig`, :class:`CloudInjectorConfig`), - so no separate ``category`` flag is needed. - - Args: - config: A single injector config or a homogeneous list of them. - Multi-target injects must be expressed as a list. - - Returns: - One ExecutionSignature object when a single config is given, - otherwise a list of ExecutionSignature in input order. - - Raises: - ValueError: Empty list, or mixed config types in a single call. - TypeError: Unknown injector config type. - """ - configs = list(config) if isinstance(config, list) else [config] - if not configs: - raise ValueError("build_execution_signatures requires at least one config") - - first_type = type(configs[0]) - for c in configs: - if not isinstance(c, first_type): - raise ValueError( - "build_execution_signatures does not mix injector config types; " - f"got {sorted({type(c).__name__ for c in configs})}" - ) - - start_time = self._utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") - results = [self._build_one(cfg, start_time) for cfg in configs] - return results[0] if len(results) == 1 else results - - def _build_one(self, config: InjectorConfig, start_time: str) -> ExecutionSignature: - """Project a single injector config into a flat pre-execution signature dict. - - Common pipeline for every category: - 1. Seed the base dict with ``start_time`` and category-specific context - (network gets resolved source IPs; cloud add nothing). - 2. Layer the config's own fields on top. - 3. Run it through :class:`ExecutionSignature` for validation - and emit JSON-ready output stripped of ``None``\\s. - """ - base: dict[str, Any] = {"start_time": start_time} - base.update(self._source_context(config)) - base.update(config.model_dump(exclude_none=True)) - return ExecutionSignature(**base) - - def _source_context(self, config: InjectorConfig) -> dict[str, Any]: - """Return the source identity bits injected for the config's category. - - Only network signatures need the running container's source IPs; - cloud rows have no source identity to carry. - """ - if isinstance(config, NetworkInjectorConfig): - return { - "source_ipv4": self.resolve_container_ip(), - "source_ipv6": self._cached_ipv6, - } - if isinstance(config, CloudInjectorConfig): - return {} - raise TypeError(f"unsupported injector config type: {type(config).__name__}") - - def post_execution_updates( - self, - execution_details: ExecutionDetails, - execution_signatures: ExecutionSignature | list[ExecutionSignature], - tool_output: dict[str, Any], - ) -> None: - """ - Update both execution details and execution signatures according to tool output - """ - try: - tool_output = ToolOutput.model_validate(tool_output or {}) - except ValidationError as exc: - raise OpenAEVError( - error_message=f"Invalid tool_output: {exc}", - ) from exc - - now = self._utcnow() - - execution_details.post_execution_update(tool_output, now) - - if isinstance(execution_signatures, list): - for exec_sig in execution_signatures: - exec_sig.post_execution_update(tool_output, now) - else: - execution_signatures.post_execution_update(tool_output, now) - - @staticmethod - def build_payload( - execution_signatures: ExecutionSignature | list[ExecutionSignature], - targets_meta: Any | list[Any], - expectation_types: list[str], - extra_signatures: ExtraSignatureData = ExtraSignatureData(), - ) -> dict[str, Any]: - """Build the nested wire payload from flat post-execution signatures. - - Bridges the gap between compile_post_execution_signatures output (flat dicts) - and send_signatures input (nested wire format). - - Args: - execution_signatures: A single post-execution ExecutionSignature or a list (multi-targets). - targets_meta: Target metadata dict(s) with keys like agent, asset, asset_group. - expectation_types: The 1+ expectation type labels (e.g. ['DETECTION', 'PREVENTION']). - extra_signatures: Optional mapping of expectation types to additional signature fields that will be merged - into the base post_signatures. - - Returns: - A payload dict ready for send_signatures. - """ - if not isinstance(execution_signatures, list): - execution_signatures = [execution_signatures] - if isinstance(targets_meta, dict): - targets_meta = [targets_meta] * len(execution_signatures) - - targets = [] - for signature, target in zip(execution_signatures, targets_meta): - signature_values = [] - for expectation_type in expectation_types: - signature_data = signature.model_dump(exclude_none=True) - signature_data.update(extra_signatures.get_extra(expectation_type)) - values = [ - SignatureValue(signature_type=key, signature_value=value) - for key, value in signature_data.items() - ] - signature_values.append( - ExpectationSignatureGroup( - expectation_type=expectation_type, - values=values, - ) - ) - targets.append( - TargetSignatures( - signature_target=SignatureTarget( - agent=target.agent_id, - asset=target.asset_id, - asset_group=target.asset_group_id, - ), - signature_values=signature_values, - ) - ) - - return SignaturePayload(targets=targets).model_dump() - - def send_signatures( - self, - inject_id: str, - execution_details: ExecutionDetails, - signatures: dict[str, Any], - ) -> None: - """Ship signatures to the callback endpoint via the configured transport. - - Constructs typed ``SignatureOutputStructure`` and ``ExecutionDetails`` - models, then delegates transport (retry, envelope splitting, validation) - to the configured signature transport. - - Args: - inject_id: Inject UUID. - execution_details: execution-related metadata as an ExecutionDetails object - signatures: Full signatures dict with a ``targets`` list. - - Raises: - SignatureTransmissionError: Validation failed, 4xx hit, or retries exhausted. - """ - sig_output = SignatureOutputStructure( - expectation_signatures=SignaturePayload(**signatures) - ) - - self.transport.send_signatures( - inject_id, - sig_output, - execution_details, - max_payload_size=self.max_payload_size, - logger=self.logger, - ) - - def resolve_container_ip(self) -> str: - """Sniff the container's primary IPv4. Env var, hostname, then ``hostname -i``. - - Returns: - The IPv4 string, or ``'unknown'`` with a warning when all strategies fail. - """ - if self._cached_ipv4 and self._cached_ipv4 != "unknown": - return self._cached_ipv4 - - env_ip = os.environ.get("CONTAINER_IP") - if env_ip: - self._cached_ipv4 = env_ip - self._resolve_ipv6() - return env_ip - - try: - ip = socket.gethostbyname(socket.gethostname()) - if ip and ip != "127.0.0.1": - self._cached_ipv4 = ip - self._resolve_ipv6() - return ip - except (socket.gaierror, OSError): - pass - - try: - result = subprocess.run( - ["hostname", "-i"], - capture_output=True, - text=True, - timeout=5, - check=False, - ) - if result.returncode == 0: - ip = result.stdout.strip().split()[0] - if ip and ip != "127.0.0.1": - self._cached_ipv4 = ip - self._resolve_ipv6() - return ip - except (OSError, RuntimeError, subprocess.TimeoutExpired): - pass - - self.logger.warning("Could not resolve container IP; returning 'unknown'") - self._cached_ipv4 = "unknown" - return "unknown" - - def _resolve_ipv6(self) -> None: - """Best-effort IPv6 sniff. Silent on failure, no exceptions escape.""" - try: - infos = socket.getaddrinfo( - socket.gethostname(), None, socket.AF_INET6, socket.SOCK_STREAM - ) - for info in infos: - addr = info[4][0] - if ( - isinstance(addr, str) - and addr - and not addr.startswith("::1") - and not addr.startswith("fe80") - ): - self._cached_ipv6 = addr - return - except (socket.gaierror, OSError): - pass diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py deleted file mode 100644 index 9a7438ce..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_match.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Defines signature match result objects. - -This module provides the `SignatureMatch` class, which stores the selected -match type and its optional score. -""" - -from xtm_oaev_sdk._core.signatures.types import MatchTypes -from xtm_oaev_sdk.seam.common import OpenAEVError - -__all__ = ["SignatureMatch"] - - -class SignatureMatch: - """Represents a signature matching outcome. - - Args: - match_type: The matching strategy/result type. - match_score: Optional score produced by matching. - - Raises: - OpenAEVError: If `match_score` is not provided for non-simple match - types. - """ - - def __init__(self, match_type: MatchTypes, match_score: int | None): - if match_score is None and match_type != MatchTypes.MATCH_TYPE_SIMPLE: - raise OpenAEVError( - f"Match type {match_type} requires score to be set, found score = {match_score}" - ) - self.match_type = match_type - self.match_score = match_score diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_type.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_type.py deleted file mode 100644 index e6da710b..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/signature_type.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Signature type model and matching-structure builder. - -This module defines ``SignatureType``, which couples a signature label with a -matching policy and can generate helper structures for signature matching. -""" - -from typing import Any - -from xtm_oaev_sdk._core.signatures.signature_match import SignatureMatch -from xtm_oaev_sdk._core.signatures.types import MatchTypes, SignatureTypes - -__all__ = ["SignatureType"] - - -class SignatureType: - """Describe a signature type and its matching policy. - - Args: - label: Type specifier. - match_type: Matching policy used when trying to match this signature - type, for example fuzzy or simple. - match_score: Threshold score used when ``match_type`` is fuzzy. - """ - - def __init__( - self, - label: SignatureTypes, - match_type: MatchTypes = MatchTypes.MATCH_TYPE_SIMPLE, - match_score: int | None = None, - ) -> None: - self.label = label - self.match_policy = SignatureMatch(match_type, match_score) - - def make_struct_for_matching(self, data: Any) -> dict[str, Any]: - """Format matching metadata for helper-based signature matching. - - Args: - data: Arbitrary data, most often a string or numeric primitive. - - Returns: - A dictionary with matching specifiers containing: - - ``type``: Matching type string. - - ``data``: Provided data payload. - - ``score``: Optional score threshold when configured. - """ - struct: dict[str, Any] = { - "type": self.match_policy.match_type.value, - "data": data, - } - - if self.match_policy.match_score is not None: - struct["score"] = self.match_policy.match_score - - return struct diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/targets.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/targets.py new file mode 100644 index 00000000..e46f2850 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/targets.py @@ -0,0 +1,40 @@ +"""Network targets: raw injector targets into network injector configs.""" + +import ipaddress +from typing import Any + +from xtm_oaev_sdk.seam.signatures import NetworkInjectorConfig + +__all__ = ["NetworkTargets"] + + +class NetworkTargets: + """Implements ``NetworkTargetsPort``.""" + + def build_network_configs( + self, + targets: list[str | dict[str, Any] | NetworkInjectorConfig], + ) -> list[NetworkInjectorConfig]: + configs: list[NetworkInjectorConfig] = [] + for target in targets: + if isinstance(target, NetworkInjectorConfig): + configs.append(target) + elif isinstance(target, dict): + configs.append(NetworkInjectorConfig(**target)) + elif isinstance(target, str): + configs.append(self._classify_network_target(target)) + else: + raise TypeError( + f"unsupported network target type: {type(target).__name__}" + ) + return configs + + def _classify_network_target(self, value: str) -> NetworkInjectorConfig: + """Decide whether ``value`` is an IPv4, IPv6, or hostname and wrap it.""" + try: + addr = ipaddress.ip_address(value) + except ValueError: + return NetworkInjectorConfig(target_hostname=value) + if isinstance(addr, ipaddress.IPv4Address): + return NetworkInjectorConfig(target_ipv4=value) + return NetworkInjectorConfig(target_ipv6=value) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/types.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/types.py deleted file mode 100644 index 99d07bcb..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/signatures/types.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Enumerations used by the signature engine. - -The wire vocabulary — the three enums that appear in the port I/O JSON -schema (``SignatureExpectationType``, ``ExecutionStatus``, -``InjectExecutionActions``) — is owned by the seam -(``xtm_oaev_sdk.seam.port_io.models``) and re-exported here so internal -import paths stay stable. The matching-engine vocabulary (``MatchTypes``, -``SignatureTypes``) never crosses the wire and stays defined here. -""" - -from enum import Enum - -from xtm_oaev_sdk.seam.port_io.models import ( - ExecutionStatus, - InjectExecutionActions, - SignatureExpectationType, -) - -__all__ = [ - "SignatureExpectationType", - "ExecutionStatus", - "InjectExecutionActions", - "MatchTypes", - "SignatureTypes", -] - - -class MatchTypes(str, Enum): - """Supported match algorithm types for signature evaluation.""" - - MATCH_TYPE_FUZZY = "fuzzy" - MATCH_TYPE_SIMPLE = "simple" - - -class SignatureTypes(str, Enum): - """Supported signature field identifiers.""" - - SIG_TYPE_PROCESS_NAME = "process_name" - SIG_TYPE_PARENT_PROCESS_NAME = "parent_process_name" - SIG_TYPE_COMMAND_LINE = "command_line" - SIG_TYPE_IPV4_ADDRESS = "ipv4_address" - SIG_TYPE_IPV6_ADDRESS = "ipv6_address" - SIG_TYPE_SOURCE_IPV4_ADDRESS = "source_ipv4_address" - SIG_TYPE_SOURCE_IPV6_ADDRESS = "source_ipv6_address" - SIG_TYPE_TARGET_IPV4_ADDRESS = "target_ipv4_address" - SIG_TYPE_TARGET_IPV6_ADDRESS = "target_ipv6_address" - SIG_TYPE_HOSTNAME = "hostname" - SIG_TYPE_TARGET_HOSTNAME_ADDRESS = "target_hostname_address" - SIG_TYPE_START_DATE = "start_date" - SIG_TYPE_END_DATE = "end_date" - SIG_TYPE_CLOUD_PROVIDER = "cloud_provider" - SIG_TYPE_CLOUD_ACCOUNT_ID = "cloud_account_id" - SIG_TYPE_CLOUD_REGION = "cloud_region" - SIG_TYPE_TARGET_SERVICE = "target_service" - SIG_TYPE_QUERY = "query" - SIG_TYPE_URL_HASH = "url_hash" - SIG_TYPE_FILE_NAME = "file_name" - SIG_TYPE_FILE_HASH = "file_hash" - SIG_TYPE_SOURCE_EMAIL = "source_email" - SIG_TYPE_TARGET_EMAIL = "target_email" - SIG_TYPE_EMAIL_CUSTOM_HEADER = "email_custom_header" - # AI adversarial validation: correlate AI defense (LLM firewall / guardrail) events back to a - # specific AI inject execution. - SIG_TYPE_AI_REQUEST_MARKER = "ai_request_marker" - SIG_TYPE_AI_TARGET_ENDPOINT = "ai_target_endpoint" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index c910c9b7..9f1f3466 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -105,27 +105,8 @@ # --- Daemons --- from xtm_oaev_sdk.public.daemons import BaseDaemon, PingAlive -# --- Seam / compatibility --- -from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr - -# --- Port I/O contract (versioned wire schema) --- -from xtm_oaev_sdk.seam.port_io import ( - PORT_IO_CONTRACT_VERSION, - ExecutionDetails, - ExecutionStatus, - ExpectationSignatureGroup, - InjectExecutionActions, - SignatureCallbackPayload, - SignatureExpectationType, - SignatureOutputStructure, - SignaturePayload, - SignatureTarget, - SignatureValue, - TargetSignatures, -) - # --- Signatures --- -from xtm_oaev_sdk.seam.signatures import ( +from xtm_oaev_sdk.public.signatures import ( CloudInjectorConfig, ExecutionSignature, ExtraSignatureData, @@ -145,6 +126,25 @@ build_network_configs, ) +# --- Seam / compatibility --- +from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr + +# --- Port I/O contract (versioned wire schema) --- +from xtm_oaev_sdk.seam.port_io import ( + PORT_IO_CONTRACT_VERSION, + ExecutionDetails, + ExecutionStatus, + ExpectationSignatureGroup, + InjectExecutionActions, + SignatureCallbackPayload, + SignatureExpectationType, + SignatureOutputStructure, + SignaturePayload, + SignatureTarget, + SignatureValue, + TargetSignatures, +) + __all__ = [ "__version__", # Common: errors diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/signatures.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/signatures.py new file mode 100644 index 00000000..2028e7fc --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/signatures.py @@ -0,0 +1,255 @@ +"""Signature adapters: the signature manager and the target/marker helpers. + +Each adapter speaks the signatures seam types and delegates to the core +implementation of its port; signatures are shipped through an injected +``SignatureTransportProtocol``. +""" + +import logging +from datetime import datetime +from typing import Any + +from xtm_oaev_sdk._core.signatures import ( + ContainerAddressResolver, + MarkerBuilder, + NetworkTargets, + SignatureEngine, +) +from xtm_oaev_sdk.seam.port_io import ExecutionDetails +from xtm_oaev_sdk.seam.signatures import ( + CloudInjectorConfig, + ContainerAddressPort, + ExecutionSignature, + ExtraSignatureData, + InjectorConfig, + MarkerPort, + MatchTypes, + NetworkInjectorConfig, + NetworkTargetsPort, + SignatureEnginePort, + SignatureManagerProtocol, + SignatureMatch, + SignatureTransportProtocol, + SignatureType, + SignatureTypes, + ToolErrorInfo, + ToolOutput, + ToolTimeoutInfo, +) + +__all__ = [ + "CloudInjectorConfig", + "ExecutionSignature", + "ExtraSignatureData", + "InjectorConfig", + "MatchTypes", + "NetworkInjectorConfig", + "SignatureManager", + "SignatureManagerProtocol", + "SignatureMatch", + "SignatureTransportProtocol", + "SignatureType", + "SignatureTypes", + "ToolErrorInfo", + "ToolOutput", + "ToolTimeoutInfo", + "build_marker", + "build_network_configs", +] + +_engine: SignatureEnginePort = SignatureEngine() +_addresses: ContainerAddressPort = ContainerAddressResolver() +_targets: NetworkTargetsPort = NetworkTargets() +_marker: MarkerPort = MarkerBuilder() + + +class SignatureManager: + """End-to-end signature pipeline: compile, merge, transmit. One class, three jobs.""" + + DEFAULT_MAX_PAYLOAD_SIZE = 5_242_880 # 5 MiB + + def __init__( + self, + transport: SignatureTransportProtocol, + logger: logging.Logger | None = None, + max_payload_size: int = DEFAULT_MAX_PAYLOAD_SIZE, + ) -> None: + """Initialize manager with a transport implementation for signature delivery.""" + self.transport = transport + self.logger = logger or logging.getLogger(__name__) + self.max_payload_size = max_payload_size + self._cached_ipv4: str | None = None + self._cached_ipv6: str | None = None + + def _utcnow(self) -> datetime: + """Current UTC time. Carved out so tests can pin the clock.""" + return _engine.now() + + def build_execution_signatures( + self, + config: InjectorConfig | list[InjectorConfig], + ) -> ExecutionSignature | list[ExecutionSignature]: + """Build pre-execution signature dicts from one or more typed injector configs. + + The category is carried by the config type itself + (:class:`NetworkInjectorConfig`, :class:`CloudInjectorConfig`), + so no separate ``category`` flag is needed. + + Args: + config: A single injector config or a homogeneous list of them. + Multi-target injects must be expressed as a list. + + Returns: + One ExecutionSignature object when a single config is given, + otherwise a list of ExecutionSignature in input order. + + Raises: + ValueError: Empty list, or mixed config types in a single call. + TypeError: Unknown injector config type. + """ + configs = list(config) if isinstance(config, list) else [config] + _engine.validate_configs(configs) + start_time = _engine.format_start_time(self._utcnow()) + results = [self._build_one(cfg, start_time) for cfg in configs] + return results[0] if len(results) == 1 else results + + def _build_one(self, config: InjectorConfig, start_time: str) -> ExecutionSignature: + """Project a single injector config into a flat pre-execution signature dict. + + Common pipeline for every category: + 1. Seed the base dict with ``start_time`` and category-specific context + (network gets resolved source IPs; cloud add nothing). + 2. Layer the config's own fields on top. + 3. Run it through :class:`ExecutionSignature` for validation + and emit JSON-ready output stripped of ``None``\\s. + """ + return _engine.build_signature(config, start_time, self._source_context(config)) + + def _source_context(self, config: InjectorConfig) -> dict[str, Any]: + """Return the source identity bits injected for the config's category. + + Only network signatures need the running container's source IPs; + cloud rows have no source identity to carry. + """ + return _engine.source_context( + config, self.resolve_container_ip, lambda: self._cached_ipv6 + ) + + def post_execution_updates( + self, + execution_details: ExecutionDetails, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + tool_output: dict[str, Any], + ) -> None: + """ + Update both execution details and execution signatures according to tool output + """ + _engine.apply_tool_output( + execution_details, execution_signatures, tool_output, self._utcnow + ) + + @staticmethod + def build_payload( + execution_signatures: ExecutionSignature | list[ExecutionSignature], + targets_meta: Any | list[Any], + expectation_types: list[str], + extra_signatures: ExtraSignatureData = ExtraSignatureData(), + ) -> dict[str, Any]: + """Build the nested wire payload from flat post-execution signatures. + + Bridges the gap between compile_post_execution_signatures output (flat dicts) + and send_signatures input (nested wire format). + + Args: + execution_signatures: A single post-execution ExecutionSignature or a list (multi-targets). + targets_meta: Target metadata dict(s) with keys like agent, asset, asset_group. + expectation_types: The 1+ expectation type labels (e.g. ['DETECTION', 'PREVENTION']). + extra_signatures: Optional mapping of expectation types to additional signature fields that will be merged + into the base post_signatures. + + Returns: + A payload dict ready for send_signatures. + """ + return _engine.build_payload( + execution_signatures, targets_meta, expectation_types, extra_signatures + ) + + def send_signatures( + self, + inject_id: str, + execution_details: ExecutionDetails, + signatures: dict[str, Any], + ) -> None: + """Ship signatures to the callback endpoint via the configured transport. + + Constructs typed ``SignatureOutputStructure`` and ``ExecutionDetails`` + models, then delegates transport (retry, envelope splitting, validation) + to the configured signature transport. + + Args: + inject_id: Inject UUID. + execution_details: execution-related metadata as an ExecutionDetails object + signatures: Full signatures dict with a ``targets`` list. + + Raises: + SignatureTransmissionError: Validation failed, 4xx hit, or retries exhausted. + """ + sig_output = _engine.output_structure(signatures) + self.transport.send_signatures( + inject_id, + sig_output, + execution_details, + max_payload_size=self.max_payload_size, + logger=self.logger, + ) + + def resolve_container_ip(self) -> str: + """Sniff the container's primary IPv4. Env var, hostname, then ``hostname -i``. + + Returns: + The IPv4 string, or ``'unknown'`` with a warning when all strategies fail. + """ + if self._cached_ipv4 and self._cached_ipv4 != "unknown": + return self._cached_ipv4 + + ip = _addresses.resolve_ipv4(self.logger) + if ip is None: + self._cached_ipv4 = "unknown" + return "unknown" + self._cached_ipv4 = ip + self._resolve_ipv6() + return ip + + def _resolve_ipv6(self) -> None: + """Best-effort IPv6 sniff. Silent on failure, no exceptions escape.""" + addr = _addresses.resolve_ipv6() + if addr: + self._cached_ipv6 = addr + + +def build_network_configs( + targets: list[str | dict[str, Any] | NetworkInjectorConfig], +) -> list[NetworkInjectorConfig]: + """Forge a list of `NetworkInjectorConfig` from a heterogeneous target list. + + Each item is one distinct asset. Accepted shapes: + + - `NetworkInjectorConfig`: passed through unchanged. + - `dict`: validated against :class:`NetworkInjectorConfig`. + - `str`: auto-classified into IPv4 / IPv6 / hostname. + + Args: + targets: Raw target list straight out of the injector. + + Returns: + One `NetworkInjectorConfig` per input target, order preserved. + + Raises: + TypeError: An item is not one of the accepted shapes. + ValidationError: A dict item fails the one-identity invariant. + """ + return _targets.build_network_configs(targets) + + +def build_marker(inject_id: str, agent_id: str = "") -> str: + return _marker.build_marker(inject_id, agent_id) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py index 03f99a0f..ef5ace38 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -15,8 +15,9 @@ versioned (``SEAM_VERSION``). - **``daemons``** — the ports of the daemon runtime shared by collectors and injectors (lifecycle, heartbeat); closed and versioned (``SEAM_VERSION``). -- **``signatures``** — the signatures capability's inbound seam - (``seam/signatures/seam.py``), versioned ``SEAM_VERSION``. +- **``signatures``** — the matching vocabulary and policies, the signature + ports, and the inbound I/O models (``seam/signatures/seam.py``, versioned + ``SEAM_VERSION``). - **``port_io``** — the port I/O wire schema crossing the SDK↔platform boundary; the seam **owns** the models (``seam/port_io/models.py``), versioned ``PORT_IO_CONTRACT_VERSION``. @@ -30,10 +31,8 @@ core. As long as the core keeps answering the seam, the public surface keeps working; a seam change is a contract change (``SEAM_VERSION`` bump, changelog). -``common``, ``configuration``, ``contracts``, ``daemons``, -``seam/signatures/seam.py`` and ``seam/port_io/models.py`` are closed today; -the other packages still re-export ``_core`` until their capability is -migrated. +``common``, ``configuration``, ``contracts``, ``daemons``, ``signatures`` and +``port_io`` are closed; only ``deprecation`` remains to be layered. """ from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py index 2704e300..cafa5e92 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/__init__.py @@ -1,71 +1,58 @@ -"""Signature feature data models. +"""Signatures seam: the contract between the public adapters and the core. -The inbound seam (the versioned ``InjectorConfig`` / ``ExecutionSignature`` / -``ToolOutput`` I/O models) is declared in :mod:`xtm_oaev_sdk.seam.signatures.seam`; -everything else in this package is the port's interface, re-export surface, -and the concrete matching-engine layer. +The inbound I/O models (``seam.py``, versioned :data:`SEAM_VERSION` and +schema-pinned), the matching vocabulary and policies, and the signature ports. +The wire models live in :mod:`xtm_oaev_sdk.seam.port_io`. Closed: it imports +nothing from ``xtm_oaev_sdk._core`` nor ``xtm_oaev_sdk.public``. """ -from xtm_oaev_sdk._core.signatures import ( - CloudInjectorConfig, - ExecutionDetails, - ExecutionSignature, - ExecutionStatus, - ExpectationSignatureGroup, +from xtm_oaev_sdk.seam.signatures.models import ( ExtraSignatureData, - InjectExecutionActions, - InjectorConfig, MatchTypes, - NetworkInjectorConfig, - SignatureCallbackPayload, - SignatureExpectationType, - SignatureManager, - SignatureManagerProtocol, SignatureMatch, - SignatureOutputStructure, - SignaturePayload, - SignatureTarget, - SignatureTransportProtocol, SignatureType, SignatureTypes, - SignatureValue, - TargetSignatures, +) +from xtm_oaev_sdk.seam.signatures.ports import ( + ContainerAddressPort, + MarkerPort, + NetworkTargetsPort, + SignatureEnginePort, + SignatureManagerProtocol, + SignatureTransportProtocol, +) +from xtm_oaev_sdk.seam.signatures.seam import ( + SEAM_VERSION, + CloudInjectorConfig, + ExecutionSignature, + InjectorConfig, + NetworkInjectorConfig, ToolErrorInfo, ToolOutput, ToolTimeoutInfo, - build_marker, - build_network_configs, ) -from xtm_oaev_sdk.seam.signatures.seam import SEAM_VERSION __all__ = [ "SEAM_VERSION", + # Inbound I/O models (seam.py) "CloudInjectorConfig", - "ExecutionDetails", "ExecutionSignature", - "ExecutionStatus", - "ExpectationSignatureGroup", - "ExtraSignatureData", - "InjectExecutionActions", "InjectorConfig", - "MatchTypes", "NetworkInjectorConfig", - "SignatureCallbackPayload", - "SignatureExpectationType", - "SignatureManager", - "SignatureManagerProtocol", - "SignatureMatch", - "SignatureOutputStructure", - "SignaturePayload", - "SignatureTarget", - "SignatureTransportProtocol", - "SignatureType", - "SignatureTypes", - "SignatureValue", - "TargetSignatures", "ToolErrorInfo", "ToolOutput", "ToolTimeoutInfo", - "build_marker", - "build_network_configs", + # Data contracts + "ExtraSignatureData", + "MatchTypes", + "SignatureMatch", + "SignatureType", + "SignatureTypes", + # Ports + "ContainerAddressPort", + "MarkerPort", + "NetworkTargetsPort", + "SignatureEnginePort", + "SignatureManagerProtocol", + "SignatureTransportProtocol", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/models.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/models.py new file mode 100644 index 00000000..03a62b2d --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/models.py @@ -0,0 +1,142 @@ +"""Signature data contracts: matching vocabulary, matching policies and the +extra signatures merged into the default ones. + +The inbound I/O models live in :mod:`xtm_oaev_sdk.seam.signatures.seam` and the +wire models in :mod:`xtm_oaev_sdk.seam.port_io`. +""" + +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field, JsonValue +from xtm_oaev_sdk.seam.common import OpenAEVError + +__all__ = [ + "ExtraSignatureData", + "MatchTypes", + "SignatureMatch", + "SignatureType", + "SignatureTypes", +] + + +class MatchTypes(str, Enum): + """Supported match algorithm types for signature evaluation.""" + + MATCH_TYPE_FUZZY = "fuzzy" + MATCH_TYPE_SIMPLE = "simple" + + +class SignatureTypes(str, Enum): + """Supported signature field identifiers.""" + + SIG_TYPE_PROCESS_NAME = "process_name" + SIG_TYPE_PARENT_PROCESS_NAME = "parent_process_name" + SIG_TYPE_COMMAND_LINE = "command_line" + SIG_TYPE_IPV4_ADDRESS = "ipv4_address" + SIG_TYPE_IPV6_ADDRESS = "ipv6_address" + SIG_TYPE_SOURCE_IPV4_ADDRESS = "source_ipv4_address" + SIG_TYPE_SOURCE_IPV6_ADDRESS = "source_ipv6_address" + SIG_TYPE_TARGET_IPV4_ADDRESS = "target_ipv4_address" + SIG_TYPE_TARGET_IPV6_ADDRESS = "target_ipv6_address" + SIG_TYPE_HOSTNAME = "hostname" + SIG_TYPE_TARGET_HOSTNAME_ADDRESS = "target_hostname_address" + SIG_TYPE_START_DATE = "start_date" + SIG_TYPE_END_DATE = "end_date" + SIG_TYPE_CLOUD_PROVIDER = "cloud_provider" + SIG_TYPE_CLOUD_ACCOUNT_ID = "cloud_account_id" + SIG_TYPE_CLOUD_REGION = "cloud_region" + SIG_TYPE_TARGET_SERVICE = "target_service" + SIG_TYPE_QUERY = "query" + SIG_TYPE_URL_HASH = "url_hash" + SIG_TYPE_FILE_NAME = "file_name" + SIG_TYPE_FILE_HASH = "file_hash" + SIG_TYPE_SOURCE_EMAIL = "source_email" + SIG_TYPE_TARGET_EMAIL = "target_email" + SIG_TYPE_EMAIL_CUSTOM_HEADER = "email_custom_header" + # AI adversarial validation: correlate AI defense (LLM firewall / guardrail) events back to a + # specific AI inject execution. + SIG_TYPE_AI_REQUEST_MARKER = "ai_request_marker" + SIG_TYPE_AI_TARGET_ENDPOINT = "ai_target_endpoint" + + +class SignatureMatch: + """Represents a signature matching outcome. + + Args: + match_type: The matching strategy/result type. + match_score: Optional score produced by matching. + + Raises: + OpenAEVError: If `match_score` is not provided for non-simple match + types. + """ + + def __init__(self, match_type: MatchTypes, match_score: int | None): + if match_score is None and match_type != MatchTypes.MATCH_TYPE_SIMPLE: + raise OpenAEVError( + f"Match type {match_type} requires score to be set, found score = {match_score}" + ) + self.match_type = match_type + self.match_score = match_score + + +class SignatureType: + """Describe a signature type and its matching policy. + + Args: + label: Type specifier. + match_type: Matching policy used when trying to match this signature + type, for example fuzzy or simple. + match_score: Threshold score used when ``match_type`` is fuzzy. + """ + + def __init__( + self, + label: SignatureTypes, + match_type: MatchTypes = MatchTypes.MATCH_TYPE_SIMPLE, + match_score: int | None = None, + ) -> None: + self.label = label + self.match_policy = SignatureMatch(match_type, match_score) + + def make_struct_for_matching(self, data: Any) -> dict[str, Any]: + """Format matching metadata for helper-based signature matching. + + Args: + data: Arbitrary data, most often a string or numeric primitive. + + Returns: + A dictionary with matching specifiers containing: + - ``type``: Matching type string. + - ``data``: Provided data payload. + - ``score``: Optional score threshold when configured. + """ + struct: dict[str, Any] = { + "type": self.match_policy.match_type.value, + "data": data, + } + + if self.match_policy.match_score is not None: + struct["score"] = self.match_policy.match_score + + return struct + + +class ExtraSignatureData(BaseModel): + """Format for extra signatures added to the default signatures.""" + + detection: dict[str, JsonValue] = Field(default_factory=dict) + prevention: dict[str, JsonValue] = Field(default_factory=dict) + vulnerability: dict[str, JsonValue] = Field(default_factory=dict) + + def get_extra(self, expectation_type: str) -> dict[str, JsonValue]: + if expectation_type.lower() == "detection": + return self.detection + if expectation_type.lower() == "prevention": + return self.prevention + if expectation_type.lower() == "vulnerability": + return self.vulnerability + raise ValueError( + f"Expectation type should be one of the available parameters: {list(self.model_fields.keys())}" + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/ports.py new file mode 100644 index 00000000..f4966707 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/ports.py @@ -0,0 +1,164 @@ +"""Signature ports. + +``SignatureTransportProtocol`` is the driven port the signature manager ships +signatures through (implemented by API clients); ``SignatureManagerProtocol`` +is the consumer-facing contract of signature managers. The other protocols +are the ports the public adapters delegate to; the core implements them. +""" + +import logging +from datetime import datetime +from typing import Any, Callable, Protocol, runtime_checkable + +from xtm_oaev_sdk.seam.port_io import ExecutionDetails, SignatureOutputStructure +from xtm_oaev_sdk.seam.signatures.models import ExtraSignatureData +from xtm_oaev_sdk.seam.signatures.seam import ( + ExecutionSignature, + InjectorConfig, + NetworkInjectorConfig, +) + +__all__ = [ + "ContainerAddressPort", + "MarkerPort", + "NetworkTargetsPort", + "SignatureEnginePort", + "SignatureManagerProtocol", + "SignatureTransportProtocol", +] + + +@runtime_checkable +class SignatureTransportProtocol(Protocol): + """Transport contract for signature delivery. + + pyoaev implements this via its Signature API manager. + The SDK only knows this interface, never the concrete client. + """ + + def send_signatures( + self, + inject_id: str, + sig_output: SignatureOutputStructure, + execution_details: ExecutionDetails, + *, + max_payload_size: int, + logger: logging.Logger, + ) -> None: + """Transmit signatures to the platform callback endpoint.""" + ... + + +@runtime_checkable +class SignatureManagerProtocol(Protocol): + """Behavioral contract for the signature lifecycle manager. + + Covers the full compile → merge → transmit pipeline. + """ + + def build_execution_signatures( + self, + config: InjectorConfig | list[InjectorConfig], + ) -> ExecutionSignature | list[ExecutionSignature]: + """Build pre-execution signatures from injector configs.""" + ... + + def post_execution_updates( + self, + execution_details: ExecutionDetails, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + tool_output: dict[str, Any], + ) -> None: + """Update execution metadata after tool completion.""" + ... + + def send_signatures( + self, + inject_id: str, + execution_details: ExecutionDetails, + signatures: dict[str, Any], + ) -> None: + """Ship signatures to the callback endpoint.""" + ... + + def resolve_container_ip(self) -> str: + """Resolve the container's primary IPv4 address.""" + ... + + +@runtime_checkable +class SignatureEnginePort(Protocol): + """Compiles, merges and packages execution signatures.""" + + def now(self) -> datetime: + """Current UTC time.""" + + def validate_configs(self, configs: list[InjectorConfig]) -> None: + """Raise unless there is at least one config, all of the same type.""" + + def format_start_time(self, now: datetime) -> str: + """Start time of the execution signatures.""" + + def source_context( + self, + config: InjectorConfig, + source_ipv4: Callable[[], str], + source_ipv6: Callable[[], str | None], + ) -> dict[str, Any]: + """Source identity fields of a config's signature.""" + + def build_signature( + self, config: InjectorConfig, start_time: str, source_context: dict[str, Any] + ) -> ExecutionSignature: + """Pre-execution signature of one config.""" + + def apply_tool_output( + self, + execution_details: ExecutionDetails, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + tool_output: dict[str, Any], + now: Callable[[], datetime], + ) -> None: + """Merge the validated tool output into details and signatures, at the + time ``now`` returns once the output is valid.""" + + def build_payload( + self, + execution_signatures: ExecutionSignature | list[ExecutionSignature], + targets_meta: Any | list[Any], + expectation_types: list[str], + extra_signatures: ExtraSignatureData, + ) -> dict[str, Any]: + """Wire payload of the signatures, per target and expectation type.""" + + def output_structure(self, signatures: dict[str, Any]) -> SignatureOutputStructure: + """Structure shipped through the transport.""" + + +@runtime_checkable +class ContainerAddressPort(Protocol): + """Resolves the addresses of the container running the injector.""" + + def resolve_ipv4(self, logger: logging.Logger) -> str | None: + """Container IPv4 (env, socket, ``hostname -i``), None when unresolved.""" + + def resolve_ipv6(self) -> str | None: + """Container global IPv6, None when there is none.""" + + +@runtime_checkable +class NetworkTargetsPort(Protocol): + """Turns raw injector targets into network injector configs.""" + + def build_network_configs( + self, targets: list[str | dict[str, Any] | NetworkInjectorConfig] + ) -> list[NetworkInjectorConfig]: + """One config per target, order preserved.""" + + +@runtime_checkable +class MarkerPort(Protocol): + """Derives the per-inject canary marker of AI injects.""" + + def build_marker(self, inject_id: str, agent_id: str = "") -> str: + """Deterministic marker of an inject (and agent).""" From de38ada5a4e47e3c61e8da37238ad81a505b2dc0 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 15:40:58 +0200 Subject: [PATCH 13/22] :recycle: refactor(sdk): enhancing port-adapter public boundaries for deprecation --- libs/xtm-oaev-sdk/test/test_deprecation.py | 2 - libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py | 6 +-- .../_core/deprecation/__init__.py | 5 ++ .../xtm_oaev_sdk/_core/deprecation/shims.py | 41 +++++++++++++++ .../xtm_oaev_sdk/public/__init__.py | 6 +-- .../{seam => public}/deprecation.py | 51 +++++++------------ .../xtm_oaev_sdk/seam/__init__.py | 13 ++--- .../xtm_oaev_sdk/seam/deprecation/__init__.py | 14 +++++ .../xtm_oaev_sdk/seam/deprecation/ports.py | 23 +++++++++ 9 files changed, 111 insertions(+), 50 deletions(-) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/shims.py rename libs/xtm-oaev-sdk/xtm_oaev_sdk/{seam => public}/deprecation.py (64%) create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/__init__.py create mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/ports.py diff --git a/libs/xtm-oaev-sdk/test/test_deprecation.py b/libs/xtm-oaev-sdk/test/test_deprecation.py index 4216acd3..52da97ca 100644 --- a/libs/xtm-oaev-sdk/test/test_deprecation.py +++ b/libs/xtm-oaev-sdk/test/test_deprecation.py @@ -126,10 +126,8 @@ def test_mapping_is_copied_not_referenced() -> None: # --- 8. the primitive is a public, documented export --- def test_public_export_and_all() -> None: import xtm_oaev_sdk as sdk - from xtm_oaev_sdk.seam import make_deprecated_getattr as from_seam assert sdk.make_deprecated_getattr is make_deprecated_getattr - assert from_seam is make_deprecated_getattr assert "make_deprecated_getattr" in sdk.__all__ diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py index 95949819..3c399d68 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/__init__.py @@ -6,6 +6,7 @@ _core/configuration/ Configuration feature implementation _core/contracts/ OpenAEV scenarios feature implementation _core/daemons/ Daemon runtime (base daemon, heartbeat) + _core/deprecation/ Deprecation shims (PEP 562 __getattr__) _core/signatures/ Signature feature implementation seam/common/ Shared errors, enums, value objects, protocols, ports seam/configuration/ Configuration contracts and ports (closed, versioned) @@ -13,12 +14,13 @@ seam/daemons/ Daemon runtime ports (closed, versioned) seam/signatures/ Signature contracts, inbound I/O models and ports (closed) seam/port_io/ Port I/O wire schema (versioned, seam-owned) - seam/deprecation.py PEP 562 deprecation primitive + seam/deprecation/ Deprecation port (closed, versioned) public/common Common adapters public/configuration Configuration adapters public/contracts Scenario contract adapters public/daemons Daemon runtime adapters (API client injected) public/signatures Signature manager and helpers adapters + public/deprecation PEP 562 deprecation primitive adapter public/ User-facing re-exports Port/adapter layering, capability by capability (see ``seam``): @@ -29,8 +31,6 @@ - ``public`` exposes adapters that speak seam types and delegate to ``_core``, plus the seam contracts themselves — never a core object. -Capabilities not migrated yet still re-export ``_core`` through their seam -package. """ __version__ = "0.1.0" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/__init__.py new file mode 100644 index 00000000..8d899be1 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/__init__.py @@ -0,0 +1,5 @@ +"""Deprecation core: implementation of the deprecation seam port.""" + +from .shims import DeprecationShims + +__all__ = ["DeprecationShims"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/shims.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/shims.py new file mode 100644 index 00000000..39eaaa6d --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/deprecation/shims.py @@ -0,0 +1,41 @@ +"""Deprecation shims: PEP 562 ``__getattr__`` serving retired names.""" + +import warnings +from collections.abc import Mapping +from typing import Any, Callable + +__all__ = ["DeprecationShims"] + + +class DeprecationShims: + """Implements ``DeprecationPort``.""" + + def resolve_mapping( + self, mapping: Mapping[str, tuple[str, Any]] + ) -> dict[str, tuple[str, Any]]: + resolved: dict[str, tuple[str, Any]] = {} + for key, value in mapping.items(): + if not isinstance(key, str) or not key: + raise ValueError( + f"deprecated-name mapping keys must be non-empty strings, got {key!r}" + ) + if not isinstance(value, tuple) or len(value) != 2: + raise ValueError( + f"mapping entry for {key!r} must be (message, target), got {value!r}" + ) + resolved[key] = (value[0], value[1]) + return resolved + + def build_getattr( + self, resolved: dict[str, tuple[str, Any]], module_name: str | None + ) -> Callable[[str], Any]: + def __getattr__(name: str) -> Any: + entry = resolved.get(name) + if entry is None: + where = module_name if module_name is not None else "this module" + raise AttributeError(f"{where} has no attribute {name!r}") from None + message, target = entry + warnings.warn(message, DeprecationWarning, stacklevel=2) + return target + + return __getattr__ diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index 9f1f3466..991ef464 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -105,6 +105,9 @@ # --- Daemons --- from xtm_oaev_sdk.public.daemons import BaseDaemon, PingAlive +# --- Seam / compatibility --- +from xtm_oaev_sdk.public.deprecation import make_deprecated_getattr + # --- Signatures --- from xtm_oaev_sdk.public.signatures import ( CloudInjectorConfig, @@ -126,9 +129,6 @@ build_network_configs, ) -# --- Seam / compatibility --- -from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr - # --- Port I/O contract (versioned wire schema) --- from xtm_oaev_sdk.seam.port_io import ( PORT_IO_CONTRACT_VERSION, diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/deprecation.py similarity index 64% rename from libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation.py rename to libs/xtm-oaev-sdk/xtm_oaev_sdk/public/deprecation.py index 625fd9f2..36f3150e 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/deprecation.py @@ -8,14 +8,15 @@ ``(deprecation_message, replacement)`` and installs a module-level ``__getattr__`` built by :func:`make_deprecated_getattr`:: - from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr + from xtm_oaev_sdk import make_deprecated_getattr - _DEPRECATED: dict[str, tuple[str, object]] = { - "OldName": ("Use 'from xtm_oaev_sdk import NewName' instead.", NewName), - } + __getattr__ = make_deprecated_getattr( # PEP 562 + {"OldName": ("Use 'from xtm_oaev_sdk import NewName' instead.", NewName)}, + module_name=__name__, + ) - def __getattr__(name: str): # PEP 562 - return make_deprecated_getattr(_DEPRECATED, module_name=__name__)(name) +Assign the factory's result directly: a ``def __getattr__`` wrapper adds a +frame, and the warning would then point at the shim instead of the caller. Why a factory, and why it returns the object directly: @@ -32,19 +33,20 @@ def __getattr__(name: str): # PEP 562 ``AttributeError`` (naming the module), so a typo or a not-yet-retired name is never silently swallowed by the deprecation layer. -This module is a *pure compatibility primitive*: it imports nothing from the -SDK's capabilities, so the dependency direction of the seam holds — the seam -is imported by consumers, never the other way round. +This adapter speaks the deprecation seam types and delegates to the core +implementation of its port. """ -from __future__ import annotations - -import warnings from collections.abc import Mapping as _Mapping from typing import Any +from xtm_oaev_sdk._core.deprecation import DeprecationShims +from xtm_oaev_sdk.seam.deprecation import DeprecationPort + __all__ = ["make_deprecated_getattr"] +_deprecation: DeprecationPort = DeprecationShims() + def make_deprecated_getattr( mapping: _Mapping[str, tuple[str, Any]], @@ -78,25 +80,6 @@ def make_deprecated_getattr( raises ``ValueError`` at call time — i.e. at import time of the module that installs the shim — rather than at first access. """ - resolved: dict[str, tuple[str, Any]] = {} - for key, value in mapping.items(): - if not isinstance(key, str) or not key: - raise ValueError( - f"deprecated-name mapping keys must be non-empty strings, got {key!r}" - ) - if not isinstance(value, tuple) or len(value) != 2: - raise ValueError( - f"mapping entry for {key!r} must be (message, target), got {value!r}" - ) - resolved[key] = (value[0], value[1]) - - def __getattr__(name: str) -> Any: - entry = resolved.get(name) - if entry is None: - where = module_name if module_name is not None else "this module" - raise AttributeError(f"{where} has no attribute {name!r}") from None - message, target = entry - warnings.warn(message, DeprecationWarning, stacklevel=2) - return target - - return __getattr__ + return _deprecation.build_getattr( + _deprecation.resolve_mapping(mapping), module_name + ) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py index ef5ace38..93083f68 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/__init__.py @@ -21,9 +21,9 @@ - **``port_io``** — the port I/O wire schema crossing the SDK↔platform boundary; the seam **owns** the models (``seam/port_io/models.py``), versioned ``PORT_IO_CONTRACT_VERSION``. -- **``deprecation``** — the PEP 562 deprecation primitive - (:func:`make_deprecated_getattr`) for retiring public names without - breaking importers. +- **``deprecation``** — the port behind the PEP 562 deprecation primitive + (``make_deprecated_getattr``) that retires public names without breaking + importers; closed and versioned (``SEAM_VERSION``). A seam is **closed**: it imports only the standard library, third-party data libraries and sibling seam modules — never ``_core`` nor ``public``. The core @@ -31,10 +31,7 @@ core. As long as the core keeps answering the seam, the public surface keeps working; a seam change is a contract change (``SEAM_VERSION`` bump, changelog). -``common``, ``configuration``, ``contracts``, ``daemons``, ``signatures`` and -``port_io`` are closed; only ``deprecation`` remains to be layered. +Every seam package is closed. """ -from xtm_oaev_sdk.seam.deprecation import make_deprecated_getattr - -__all__ = ["make_deprecated_getattr"] +__all__: list[str] = [] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/__init__.py new file mode 100644 index 00000000..e25c9c67 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/__init__.py @@ -0,0 +1,14 @@ +"""Deprecation seam: the contract between the public deprecation adapter and +the core. + +Closed: it imports nothing from ``xtm_oaev_sdk._core`` nor +``xtm_oaev_sdk.public``. Any change to these contracts must bump +:data:`SEAM_VERSION`. +""" + +from xtm_oaev_sdk.seam.deprecation.ports import DeprecatedNames, DeprecationPort + +#: Version of the deprecation seam. Bump on any contract change. +SEAM_VERSION = "1.0.0" + +__all__ = ["SEAM_VERSION", "DeprecatedNames", "DeprecationPort"] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/ports.py new file mode 100644 index 00000000..2e149d67 --- /dev/null +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/deprecation/ports.py @@ -0,0 +1,23 @@ +"""Deprecation port: serve retired public names without breaking importers.""" + +from collections.abc import Mapping +from typing import Any, Callable, Protocol, runtime_checkable + +__all__ = ["DeprecatedNames", "DeprecationPort"] + +DeprecatedNames = Mapping[str, tuple[str, Any]] +"""``{old_name: (deprecation_message, replacement)}``.""" + + +@runtime_checkable +class DeprecationPort(Protocol): + """Builds PEP 562 module ``__getattr__`` functions for retired names.""" + + def resolve_mapping(self, mapping: DeprecatedNames) -> dict[str, tuple[str, Any]]: + """Validated copy of the mapping; ValueError on a malformed entry.""" + + def build_getattr( + self, resolved: dict[str, tuple[str, Any]], module_name: str | None + ) -> Callable[[str], Any]: + """``__getattr__`` warning (at the caller's line) and returning the + replacement of a retired name, raising AttributeError otherwise.""" From aefca5960410e0c06484101bc03e2a6286cb985c Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 16:03:48 +0200 Subject: [PATCH 14/22] :wrench: chore(pyoaev): read the legacy _is_base64_encoded from the collectors-sdk public layer --- pyoaev/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyoaev/helpers.py b/pyoaev/helpers.py index a10a666a..5df41803 100644 --- a/pyoaev/helpers.py +++ b/pyoaev/helpers.py @@ -11,7 +11,7 @@ import collectors_sdk as _collectors_sdk import injectors_sdk as _injectors_sdk import xtm_oaev_sdk as _sdk -from collectors_sdk.seam import detection as _detection +from collectors_sdk.public import detection as _detection from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr from pyoaev import OpenAEV From 74819435ff143c4000024d0bc5a9605afc1155d6 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 21:48:37 +0200 Subject: [PATCH 15/22] :memo: docs(sdk): adding the migration guide, contributing guide, changelog and TDRs --- libs/xtm-oaev-sdk/CHANGELOG.md | 37 +++ libs/xtm-oaev-sdk/CLAUDE.md | 92 ++++++ libs/xtm-oaev-sdk/CONTRIBUTING.md | 177 ++++++++++ libs/xtm-oaev-sdk/README.md | 104 ++++++ libs/xtm-oaev-sdk/docs/migration_guide.md | 305 ++++++++++++++++++ libs/xtm-oaev-sdk/docs/seam_strategy.md | 239 ++++++++++++++ libs/xtm-oaev-sdk/docs/shim_strategy.md | 189 +++++++++++ libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md | 51 +++ ...adapters-keep-the-user-facing-shape.tdr.md | 71 ++++ ...iding-names-get-a-capability-prefix.tdr.md | 68 ++++ ...rocess-seams-pin-their-json-schemas.tdr.md | 74 +++++ .../tdr/every-behaviour-behind-a-port.tdr.md | 70 ++++ .../tdr/every-sdk-module-imports-alone.tdr.md | 65 ++++ .../tdr/import-from-the-package-root.tdr.md | 70 ++++ .../tdr/importing-pyoaev-stays-silent.tdr.md | 71 ++++ ...mes-served-by-pep562-module-getattr.tdr.md | 82 +++++ ...tch-targets-are-part-of-the-surface.tdr.md | 70 ++++ ...cy-surface-guarded-from-the-release.tdr.md | 68 ++++ ...aking-change-for-legacy-pyoaev-code.tdr.md | 79 +++++ ...ss-bound-callables-to-keep-dispatch.tdr.md | 77 +++++ .../tdr/public-class-extends-seam-data.tdr.md | 79 +++++ .../public-seam-core-dependency-rule.tdr.md | 73 +++++ ...ends-on-the-sdks-during-deprecation.tdr.md | 72 +++++ .../docs/tdr/sdks-never-import-pyoaev.tdr.md | 79 +++++ ...nges-are-versioned-contract-changes.tdr.md | 70 ++++ ...-contracts-and-their-own-invariants.tdr.md | 69 ++++ ...ims-return-the-identical-sdk-object.tdr.md | 80 +++++ ...m-oaev-sdk-stays-connector-agnostic.tdr.md | 67 ++++ libs/xtm-oaev-sdk/pyproject.toml | 2 +- pyproject.toml | 2 +- 30 files changed, 2650 insertions(+), 2 deletions(-) create mode 100644 libs/xtm-oaev-sdk/CHANGELOG.md create mode 100644 libs/xtm-oaev-sdk/CLAUDE.md create mode 100644 libs/xtm-oaev-sdk/CONTRIBUTING.md create mode 100644 libs/xtm-oaev-sdk/docs/migration_guide.md create mode 100644 libs/xtm-oaev-sdk/docs/seam_strategy.md create mode 100644 libs/xtm-oaev-sdk/docs/shim_strategy.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/adapters-keep-the-user-facing-shape.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/colliding-names-get-a-capability-prefix.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/every-sdk-module-imports-alone.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/import-from-the-package-root.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/importing-pyoaev-stays-silent.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/legacy-names-served-by-pep562-module-getattr.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/legacy-patch-targets-are-part-of-the-surface.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/legacy-surface-guarded-from-the-release.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/no-breaking-change-for-legacy-pyoaev-code.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/pass-bound-callables-to-keep-dispatch.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/public-class-extends-seam-data.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/public-seam-core-dependency-rule.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/sdks-never-import-pyoaev.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/shims-return-the-identical-sdk-object.tdr.md create mode 100644 libs/xtm-oaev-sdk/docs/tdr/xtm-oaev-sdk-stays-connector-agnostic.tdr.md diff --git a/libs/xtm-oaev-sdk/CHANGELOG.md b/libs/xtm-oaev-sdk/CHANGELOG.md new file mode 100644 index 00000000..0adb3f01 --- /dev/null +++ b/libs/xtm-oaev-sdk/CHANGELOG.md @@ -0,0 +1,37 @@ +# Changelog + +Every notable change to xtm-oaev-sdk lands here. Every seam change too, with the new +version of its seam +([seam-changes-are-versioned-contract-changes](docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md)). + +## [0.1.0] - Unreleased + +First version. The shared, client-free code of `pyoaev` 3.260923.0 moves out and into +this SDK. The former `pyoaev` paths keep working, each one firing a +`DeprecationWarning` ([migration guide](docs/migration_guide.md)). + +### Added + +- `common`: the `OpenAEVError` hierarchy, `on_http_error`, `AppLogger`, logging setup, + JSON encoding, SSL helpers, asset and security-domain enums, and the `BaseClient`, + `DaemonProtocol` and `AppLoggerProtocol` protocols. Seam `1.0.0`. +- `configuration`: `Configuration`, `ConfigurationHint`, `SettingsLoader`, the settings + models, sources and schema generator. Seam `1.0.0`. +- `contracts`: `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, + the contract elements and enums. Seam `1.0.0`. +- `signatures`: `SignatureManager`, `build_network_configs`, `build_marker`, the + matching types and the inbound I/O models. Seam `1.0.0`, JSON schemas pinned. +- `port_io`: the wire models sent to the platform. `PORT_IO_CONTRACT_VERSION` `1.0.0`, + JSON schemas pinned. +- `daemons`: `BaseDaemon` and `PingAlive`. Seam `1.0.0`. +- `deprecation` (new, not from `pyoaev`): `make_deprecated_getattr`, the PEP 562 + deprecation primitive. Seam `1.0.0`. + +### Changed (compared with pyoaev) + +- `ExpectationType` splits in two: `SignatureExpectationType` and + `ContractExpectationType`. +- `BaseDaemon` and `SignatureManager` stop building the API client. Pass + `api_client=` / `transport=` yourself. +- `requires-python` moves from `>=3.11` to `>=3.12`, the version `typing.override` + actually needs. Released `pyoaev` already failed to import on 3.11. diff --git a/libs/xtm-oaev-sdk/CLAUDE.md b/libs/xtm-oaev-sdk/CLAUDE.md new file mode 100644 index 00000000..35bf735a --- /dev/null +++ b/libs/xtm-oaev-sdk/CLAUDE.md @@ -0,0 +1,92 @@ +# xtm-oaev-sdk: agent instructions + +Applies to work under `libs/xtm-oaev-sdk/`, and to the `pyoaev/` deprecation shims when +code moves out of `pyoaev`. Paths below are relative to the repository root. + +xtm-oaev-sdk is the shared, client-free runtime of the OpenAEV connectors +(configuration, contracts, signatures, daemons, errors). `pyoaev` is the HTTP client +and serves the former names through deprecation shims. collectors-sdk and injectors-sdk +build on this SDK from their own repositories. + +## Hard rules + +1. NEVER break legacy `pyoaev` code. Every public name of the latest released `pyoaev` + (recorded in `test/fixtures/legacy_surface.json`) MUST keep resolving from its legacy path, returning the SDK object and emitting a + `DeprecationWarning` that names the new import. Why: connectors pin `pyoaev`, import + its old paths and patch them in their tests. +2. Respect the layering of every capability: + - `seam//` imports neither `public/` nor `_core/`; + - `_core//` NEVER imports `public/` (type-only imports go under `TYPE_CHECKING`); + - `public/.py` imports `_core/` ONLY to assign module-level defaults typed by + their port (`_resolver: ConfigurationResolverPort = ConfigurationResolver(...)`); + - every exported name is an adapter defined in `public/` or a contract defined in + `seam/`, NEVER a `_core` object. + + Why: the core can then be rewritten freely while the public API keeps working. +3. Every piece of behaviour goes through a seam `Protocol` port, one `_core` + implementation and a `public` adapter, one-liners included. No exceptions. +4. A public adapter MUST keep the signatures, docstrings, attributes, overridable hooks + and framework base class of the code it replaces. When the old code called a method on + `self`, pass that bound method to the core as a callable, so overrides and test + patches keep applying. +5. NEVER import `pyoaev`, `pika`, `thefuzz` or `requests` in the SDK. The API client is + injected (`BaseClient`, `SignatureTransportProtocol`). +6. Users import from the package root (`from xtm_oaev_sdk import X`). A new public name + goes in its `public/.py` `__all__` and in `public/__init__.py`. +7. A seam change (field, type, enum member, error, port signature) is a contract change: + bump that capability's `SEAM_VERSION` (`PORT_IO_CONTRACT_VERSION` for `seam/port_io`) + and add an entry to `libs/xtm-oaev-sdk/CHANGELOG.md`. For `seam/signatures/seam.py` or + `seam/port_io`, also update the expected version in the matching + `test/seam/test_*_schema_pin.py` and regenerate the pinned schemas in the same change: + `python libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py` and + `python libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py`. NEVER regenerate them + to make a failing pin pass without a deliberate seam change; follow + `libs/xtm-oaev-sdk/CONTRIBUTING.md`, "Pinned seam schemas". +8. NEVER regenerate `test/fixtures/legacy_surface.json` from the working tree: it records + the released `pyoaev`. NEVER rewrite the legacy tests moved under + `libs/xtm-oaev-sdk/test//shim/`. + +## Scope + +Only make the changes directly requested. Do not add capabilities, abstractions, files, +dependencies or refactors beyond the task. + +## Stop and ask before + +- changing a seam contract or a `SEAM_VERSION`; +- renaming or removing a public name, or removing a `pyoaev` shim; +- adding or changing a dependency in any `pyproject.toml` or in `libs/sdk-requirements.txt`; +- editing `test/fixtures/legacy_surface.json`, the golden schema fixtures under + `libs/xtm-oaev-sdk/test/seam/fixtures/`, or CI configuration; +- deleting any file. + +## Done when + +Run from the repository root, and report the real output: + +```bash +isort --check-only . && black --check . && flake8 --ignore=E,W --extend-exclude=.venv . +python -m pytest libs/xtm-oaev-sdk/test test/test_legacy_surface.py test/test_sdk_split_shims.py +python -m pytest +``` + +All three pass. If a check fails, say which one and why. NEVER report a check as passing +without running it. + +## Conventions + +- Tests live in `libs/xtm-oaev-sdk/test//`, exercise behaviour through the + public API, and are written as `_given_*` / `_when_*` / `_then_*` helpers with a + one-line `"""It ..."""` docstring. Copy the style of the neighbouring tests. +- Adding a capability: follow the checklist at the end of `libs/xtm-oaev-sdk/docs/seam_strategy.md`. +- Recording a decision: add `libs/xtm-oaev-sdk/docs/tdr/.tdr.md` with the header of + an existing record and `status: draft`, plus a row in `docs/tdr/TDR_ToC.md`. +- Commit messages, when asked for: gitmoji + Conventional Commits with the `sdk` scope, + e.g. `:recycle: refactor(sdk): enhancing port-adapter public boundaries for configuration`. + +## Read when relevant + +- `libs/xtm-oaev-sdk/docs/seam_strategy.md`: layering patterns and capability map. +- `libs/xtm-oaev-sdk/docs/shim_strategy.md`: moving code out of `pyoaev` without breaking it. +- `libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md`: the decision behind each rule above. +- `libs/xtm-oaev-sdk/CONTRIBUTING.md`: setup and repository conventions. diff --git a/libs/xtm-oaev-sdk/CONTRIBUTING.md b/libs/xtm-oaev-sdk/CONTRIBUTING.md new file mode 100644 index 00000000..227c2725 --- /dev/null +++ b/libs/xtm-oaev-sdk/CONTRIBUTING.md @@ -0,0 +1,177 @@ +# Contributing to xtm-oaev-sdk + +This guide is about what's specific to the SDKs. Everything repository-wide +(Conventional Commits titles with an issue reference, signed commits, labels, coverage) +lives in the repository's [CONTRIBUTING.md](../../CONTRIBUTING.md). `collectors-sdk` +and `injectors-sdk` run on this guide too, plus a short one of their own. + +## Set up + +From the repository root, on Python 3.12 or later, since the code uses `typing.override`: + +```bash +python -m venv .venv && . .venv/bin/activate +pip install -e ./libs/xtm-oaev-sdk -r libs/sdk-requirements.txt -e '.[dev]' +``` + +`libs/sdk-requirements.txt` pulls `collectors-sdk` and `injectors-sdk` from git, +because `pyoaev` depends on them. + +## Run the checks + +```bash +isort --check-only . +black --check . +flake8 --ignore=E,W --extend-exclude=.venv . +python -m pytest # pyoaev and xtm-oaev-sdk +python -m pytest libs/xtm-oaev-sdk/test # the SDK alone +``` + +CI runs the exact same four tools. Locally, `pre-commit run --files ` takes +care of isort, black and the file hygiene hooks. + +## Where code goes + +Every capability is cut in three: `public/` (adapters), `seam/` (contracts), `_core/` +(implementations). [docs/seam_strategy.md](docs/seam_strategy.md) lays out the layering +and the patterns that hold it together. The rules, each one nailed down as a TDR in +[docs/tdr](docs/tdr/TDR_ToC.md): + +- the seam imports neither `public` nor `_core`, `_core` never imports `public`, and + `public` touches `_core` only to wire default implementations + ([public-seam-core-dependency-rule](docs/tdr/public-seam-core-dependency-rule.tdr.md)); +- every piece of behaviour goes behind a port, one-liners included + ([every-behaviour-behind-a-port](docs/tdr/every-behaviour-behind-a-port.tdr.md)); +- adapters keep the signatures, attributes, hooks and base classes of what they replace + ([adapters-keep-the-user-facing-shape](docs/tdr/adapters-keep-the-user-facing-shape.tdr.md)); +- users import from the package root, nowhere else + ([import-from-the-package-root](docs/tdr/import-from-the-package-root.tdr.md)); +- the SDK never imports `pyoaev`, `pika`, `thefuzz` or `requests` + ([sdks-never-import-pyoaev](docs/tdr/sdks-never-import-pyoaev.tdr.md), + [xtm-oaev-sdk-stays-connector-agnostic](docs/tdr/xtm-oaev-sdk-stays-connector-agnostic.tdr.md)). + +`test/test_boundaries.py` catches the forbidden imports. + +Adding a capability? The checklist sits at the end of +[docs/seam_strategy.md](docs/seam_strategy.md). Follow it. + +## Changing a seam + +A seam change is a contract change. Treat it like one +([seam-changes-are-versioned-contract-changes](docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md)): + +1. bump the capability's `SEAM_VERSION` (`PORT_IO_CONTRACT_VERSION` for `seam/port_io`); +2. for `seam/signatures/seam.py` or `seam/port_io`, update the pinned schemas in the same + change, not in a follow-up (see [Pinned seam schemas](#pinned-seam-schemas-testseam)); +3. add an entry to [CHANGELOG.md](CHANGELOG.md); +4. if a public name changes or disappears, the old one keeps working through a + deprecation shim (`make_deprecated_getattr`) first. + +## Pinned seam schemas (`test/seam`) + +Some seams carry models that leave the process: the inbound signature I/O in +`seam/signatures/seam.py` (injector configs, execution signatures, tool output) and the +wire payloads in `seam/port_io`. Another process reads them. Change their schema and the +other side breaks while every local test stays happily green. `libs/xtm-oaev-sdk/test/seam/` +pins those schemas: touch one and a test goes red, forcing a deliberate decision +([cross-process-seams-pin-their-json-schemas](docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md)). + +### What is in there + +| File | Role | +|---|---| +| `fixtures/signatures_seam_schemas.json`, `fixtures/port_io_schemas.json` | The golden files: one SHA-256 per model, computed from its `model_json_schema()` serialized with sorted keys. Hashes only. The schemas themselves are not in there. | +| `test_signatures_seam_schema_pin.py`, `test_port_io_schema_pin.py` | Recompute every hash and hold it against the golden file. On top of that, they assert the seam version, make sure the seam module imports nothing from the SDK internals, and (signatures) that the package root re-exports the very same objects. | +| `generate_signatures_seam_golden.py`, `generate_port_io_golden.py` | Overwrite the golden file with the hashes of the current models. No check. No confirmation. | + +The other seams carry a `SEAM_VERSION` and no pinned schema. Their models never leave +the process, so there is no other side to break. + +### When it fails + +The moment `model_json_schema()` changes for a pinned model, for any reason: + +- a field is added, removed, renamed or retyped, or its default changes; +- a `Field` description changes, or the docstring of a model or enum. Yes, the + docstring: it becomes the schema `description`; +- a pydantic upgrade renders the schema differently, even though no model changed. + +### Updating it on purpose + +From the repository root: + +1. make the model change; +2. bump the version: `SEAM_VERSION` in `seam/signatures/seam.py` or + `PORT_IO_CONTRACT_VERSION` in `seam/port_io/models.py`, plus the expected value in + `test_seam_version_is_pinned` or `test_port_io_contract_version_is_pinned`; +3. regenerate the golden file: + + ```bash + python libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py + python libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py + ``` + +4. run `python -m pytest libs/xtm-oaev-sdk/test/seam`; +5. commit the model, the version, the test and the golden file together, with a + [CHANGELOG.md](CHANGELOG.md) entry. + +Pinning a new model? Add it to the model dictionary of the test *and* of its generator. +Each keeps its own copy, and forgetting one is the trap. + +### Seeing what changed + +The golden files only hold hashes, so a failure tells you *that* a schema moved, never +*how*. To see it, dump the schema on your branch and on the base commit (a +`git worktree` does the job), then diff the two: + +```bash +python -c "import json; from xtm_oaev_sdk.seam.port_io import ExecutionDetails as M; print(json.dumps(M.model_json_schema(), indent=2, sort_keys=True))" +``` + +NEVER regenerate a golden file just to turn a red test green, and NEVER edit a hash by +hand. That silences the one alarm this directory exists for. If a hash moved without any +model change (a pydantic upgrade, say), compare the dumped schemas and ask before deciding +whether the wire contract really changed. + +## Moving code out of pyoaev + +Legacy code keeps working. Non-negotiable +([no-breaking-change-for-legacy-pyoaev-code](docs/tdr/no-breaking-change-for-legacy-pyoaev-code.tdr.md)): + +- serve the old path with a shim, by the rules of [docs/shim_strategy.md](docs/shim_strategy.md); +- move the old tests verbatim into `libs/xtm-oaev-sdk/test//shim/`; +- `test/test_legacy_surface.py` stays green. Never regenerate its fixture from the + working tree: it records the released `pyoaev`, and that is the whole point. + +## Tests + +Tests live in `libs/xtm-oaev-sdk/test//` and go through the public API to +hit behaviour. Write them as `_given_` / `_when_` / `_then_` helpers, topped with a +one-line "It ..." docstring: + +```python +def test_configuration_resolves_with_override_priority(tmp_path, monkeypatch) -> None: + """It prioritizes override over env, file, and default.""" + module = _given_public_module() + ... + _when_set_configuration_value(configuration, "target", "override-value") +``` + +## Commits + +Repository conventions first. On top of them, the SDK history prefixes a gitmoji and +uses the `sdk` scope, one capability or concern per commit: + +``` +:recycle: refactor(sdk): enhancing port-adapter public boundaries for configuration +``` + +## Documentation + +- [README.md](README.md): the intent, and a quickstart; +- [docs/shim_strategy.md](docs/shim_strategy.md) and + [docs/seam_strategy.md](docs/seam_strategy.md): the how and the why; +- [docs/migration_guide.md](docs/migration_guide.md): getting connectors onto the SDKs; +- [docs/tdr](docs/tdr/TDR_ToC.md): the technical decision records. A new decision gets + its own `.tdr.md`, same header as the others, `draft` status, and a line in + `TDR_ToC.md`. diff --git a/libs/xtm-oaev-sdk/README.md b/libs/xtm-oaev-sdk/README.md index e69de29b..5b963e86 100644 --- a/libs/xtm-oaev-sdk/README.md +++ b/libs/xtm-oaev-sdk/README.md @@ -0,0 +1,104 @@ +# xtm-oaev-sdk + +The **common SDK** of the OpenAEV connector ecosystem. Everything collectors and +injectors share lives here, and none of it cares which HTTP client you run. + +`pyoaev` used to be a monolith. One package carried the OpenAEV API client *and* the +configuration system, the contract builder, the signature pipeline, the daemon +runtime, the error hierarchy and a drawer of assorted helpers. This SDK cuts the +shared, client-free half out of it. The stack now reads: + +``` +xtm-oaev-sdk shared vocabulary and runtime (this package) +├── collectors-sdk collector runtime (OpenAEV-Platform/collectors, libs/) +├── injectors-sdk injector runtime (OpenAEV-Platform/injectors, libs/) +└── pyoaev the OpenAEV HTTP client, plus deprecation shims of its former code +``` + +The SDK never depends on `pyoaev`. When it has to talk to the platform, the API +client gets **injected** (`BaseClient` protocol); `pyoaev.OpenAEV` is just one +implementation of it. + +## What's inside + +| Capability | Public API | +|---|---| +| common | `OpenAEVError` and its subclasses, `on_http_error`, `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, `EncodedId`, `RequiredOptional`, `copy_dict`, `remove_none_from_dict`, SSL helpers, asset and security-domain enums, `BaseClient` / `DaemonProtocol` / `AppLoggerProtocol` | +| configuration | `Configuration`, `ConfigurationHint`, `SettingsLoader`, `BaseConfigModel`, `ConfigLoaderOAEV`, `ConfigLoaderCollector`, `ConnectorConfigSchemaGenerator`, sources, serializer types | +| contracts | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, every contract element (`ContractText`, `ContractSelect`, …) and enum | +| signatures | `SignatureManager`, `build_network_configs`, `build_marker`, matching types, the inbound I/O models and the port I/O wire models (`PORT_IO_CONTRACT_VERSION`) | +| daemons | `BaseDaemon`, `PingAlive` | +| deprecation | `make_deprecated_getattr` (PEP 562 deprecation primitive) | + +Import from the package root. Always. That is the stable API: + +```python +from xtm_oaev_sdk import Configuration, ContractBuilder, SignatureManager +``` + +## Quickstart + +```python +from xtm_oaev_sdk import ( + Configuration, + ContractBuilder, + ContractCheckbox, + ContractText, + SignatureManager, + build_network_configs, +) + +# Hint-based configuration: override, then env var, then config file, then default +config = Configuration({"log_level": {"env": "LOG_LEVEL", "default": "info"}}) +config.get("log_level") # 'info' + +# Contract fields, fluent builder +fields = ( + ContractBuilder() + .mandatory(ContractText(key="target", label="Target")) + .optional(ContractCheckbox(key="verbose", label="Verbose")) + .build_fields() +) + +# Signatures: the manager ships them through an injected transport +# (pyoaev's `OpenAEV().signature` is one) +manager = SignatureManager(transport=api.signature) +signatures = manager.build_execution_signatures(build_network_configs(["10.0.0.1"])) +``` + +## Architecture + +Every capability is cut into three layers, the same way across all three SDKs: + +``` +public/.py adapters: what users import; speak seam types, delegate to _core +seam// contracts: data models, enums, errors, Protocol ports (closed, versioned) +_core// implementations of the seam ports +``` + +`public` → `seam` ← `_core`. The seam imports neither of the other two, the core +never touches `public`, and `public` never hands a core object to users. Keep the +core answering the seam and the public API keeps working. Touch the seam and you +have changed a contract: `SEAM_VERSION` bump, changelog entry. The full wiring is in +[docs/seam_strategy.md](docs/seam_strategy.md). + +## Migrating from pyoaev + +Every former `pyoaev` path still works, and each one fires a `DeprecationWarning` +pointing at the new import. How that holds together: +[docs/shim_strategy.md](docs/shim_strategy.md). + +```python +from pyoaev.configuration import Configuration # DeprecationWarning, still works +from xtm_oaev_sdk import Configuration # new home +``` + +## Development + +```bash +pip install -e libs/xtm-oaev-sdk +python -m pytest libs/xtm-oaev-sdk/test +``` + +Python: 3.12 or later. The code uses `typing.override`, exactly as pyoaev already did. +Runtime dependencies: pydantic, pydantic-settings, PyYAML, python-json-logger. diff --git a/libs/xtm-oaev-sdk/docs/migration_guide.md b/libs/xtm-oaev-sdk/docs/migration_guide.md new file mode 100644 index 00000000..8a6616ac --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/migration_guide.md @@ -0,0 +1,305 @@ +# Migrating from pyoaev to the SDKs + +This guide is for you if your collector or injector still pulls shared code out of +`pyoaev`: configuration, contracts, signatures, daemons, helpers, errors. That code has +moved into three SDKs, and `pyoaev` is back to its real job: the OpenAEV API client. + +## Nothing breaks today + +Every former `pyoaev` path still works. Touch one and it fires a `DeprecationWarning` +naming the new import, then hands you the same object as before. The classes that used +to build the API client are the exception: they come back as a thin subclass that still builds the API client for you. +Migrate at your own pace, one import at a time. The legacy paths only disappear in a +future `pyoaev` major release. The machinery is laid out in +[shim_strategy.md](shim_strategy.md). + +## 1. Pick your dependencies + +| You write | Depend on | +|---|---| +| a collector | `collectors-sdk` and `pyoaev` | +| an injector | `injectors-sdk` and `pyoaev` | +| shared code only (configuration, contracts, signatures) | `xtm-oaev-sdk` | + +`collectors-sdk` and `injectors-sdk` drag `xtm-oaev-sdk` in on their own. Keep `pyoaev` +around for one thing: the API client (`OpenAEV`). + +```toml +dependencies = [ + "pyoaev (>=…)", + "collectors-sdk (>=0.1.0,<0.2.0)", # or injectors-sdk +] +``` + +Until the SDKs land on PyPI, pull them straight from their repositories, exactly like +`pyoaev`'s CI does in `libs/sdk-requirements.txt`: + +``` +xtm-oaev-sdk @ git+https://github.com/OpenAEV-Platform/client-python.git@#subdirectory=libs/xtm-oaev-sdk +collectors-sdk @ git+https://github.com/OpenAEV-Platform/collectors.git@#subdirectory=libs/collectors-sdk +injectors-sdk @ git+https://github.com/OpenAEV-Platform/injectors.git@#subdirectory=libs/injectors-sdk +``` + +## 2. Find the legacy imports + +Make the deprecation warnings fatal. Every one of them names its own replacement: + +```bash +python -m pytest -W error::DeprecationWarning +python -W error::DeprecationWarning -c "from pyoaev.helpers import OpenAEVInjectorHelper" +# DeprecationWarning: Use 'from injectors_sdk import OpenAEVInjectorHelper' and pass api_client=OpenAEV(...) instead. +``` + +Importing `pyoaev` itself stays silent. Only the moved names speak up. + +## 3. Change the imports + +Import from the package root (`xtm_oaev_sdk`, `collectors_sdk`, `injectors_sdk`). +Always. Every name of `pyoaev` that moved is in the tables below, next to +every legacy module that exposed it. + +### To xtm-oaev-sdk + +| Legacy name | Legacy modules | New import | +|---|---|---| +| `AppLogger` | `pyoaev.utils` | `from xtm_oaev_sdk import AppLogger` | +| `AssetCategory` | `pyoaev`, `pyoaev.asset_types` | `from xtm_oaev_sdk import AssetCategory` | +| `AssetCriticality` | `pyoaev`, `pyoaev.asset_types` | `from xtm_oaev_sdk import AssetCriticality` | +| `AssetSubCategory` | `pyoaev`, `pyoaev.asset_types` | `from xtm_oaev_sdk import AssetSubCategory` | +| `BaseConfigModel` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import BaseConfigModel` | +| `BaseDaemon` | `pyoaev.daemons`, `pyoaev.daemons.base_daemon`, `pyoaev.daemons.collector_daemon` | `from xtm_oaev_sdk import BaseDaemon` | +| `build_marker` | `pyoaev.signatures.ai_marker` | `from xtm_oaev_sdk import build_marker` | +| `build_network_configs` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models` | `from xtm_oaev_sdk import build_network_configs` | +| `CloudInjectorConfig` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import CloudInjectorConfig` | +| `CloudProvider` | `pyoaev`, `pyoaev.asset_types` | `from xtm_oaev_sdk import CloudProvider` | +| `ConfigLoaderCollector` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import ConfigLoaderCollector` | +| `ConfigLoaderOAEV` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import ConfigLoaderOAEV` | +| `Configuration` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.configuration`, `pyoaev.daemons.base_daemon`, `pyoaev.daemons.collector_daemon`, `pyoaev.helpers` | `from xtm_oaev_sdk import Configuration` | +| `ConfigurationError` | `pyoaev`, `pyoaev.exceptions`, `pyoaev.helpers` | `from xtm_oaev_sdk import ConfigurationError` | +| `ConfigurationHint` | `pyoaev.configuration.configuration` | `from xtm_oaev_sdk import ConfigurationHint` | +| `ConnectorConfigSchemaGenerator` | `pyoaev.configuration.configuration`, `pyoaev.configuration.connector_config_schema_generator` | `from xtm_oaev_sdk import ConnectorConfigSchemaGenerator` | +| `Contract` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import Contract` | +| `ContractAiTarget` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractAiTarget` | +| `ContractAsset` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractAsset` | +| `ContractAssetGroup` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractAssetGroup` | +| `ContractAttachment` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractAttachment` | +| `ContractBuilder` | `pyoaev`, `pyoaev.contracts`, `pyoaev.contracts.contract_builder` | `from xtm_oaev_sdk import ContractBuilder` | +| `ContractCardinality` | `pyoaev.contracts.contract_config`, `pyoaev.contracts.contract_utils`, `pyoaev.contracts.variable_helper` | `from xtm_oaev_sdk import ContractCardinality` | +| `ContractCardinalityElement` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractCardinalityElement` | +| `ContractCheckbox` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractCheckbox` | +| `ContractConfig` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractConfig` | +| `ContractElement` | `pyoaev.contracts.contract_builder`, `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractElement` | +| `ContractExpectations` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractExpectations` | +| `ContractFieldKey` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractFieldKey` | +| `ContractFieldType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractFieldType` | +| `ContractOutputElement` | `pyoaev.contracts.contract_builder`, `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractOutputElement` | +| `ContractOutputType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractOutputType` | +| `ContractPayload` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractPayload` | +| `ContractSelect` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractSelect` | +| `ContractTeam` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractTeam` | +| `ContractText` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractText` | +| `ContractTextArea` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractTextArea` | +| `ContractTuple` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractTuple` | +| `ContractVariable` | `pyoaev.contracts.contract_config`, `pyoaev.contracts.contract_utils`, `pyoaev.contracts.variable_helper` | `from xtm_oaev_sdk import ContractVariable` | +| `copy_dict` | `pyoaev.utils` | `from xtm_oaev_sdk import copy_dict` | +| `CustomJsonFormatter` | `pyoaev.utils` | `from xtm_oaev_sdk import CustomJsonFormatter` | +| `data_to_temp_file` | `pyoaev.helpers` | `from xtm_oaev_sdk import data_to_temp_file` | +| `DictionarySource` | `pyoaev.configuration.configuration`, `pyoaev.configuration.sources` | `from xtm_oaev_sdk import DictionarySource` | +| `Domain` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import Domain` | +| `EncodedId` | `pyoaev.utils` | `from xtm_oaev_sdk import EncodedId` | +| `EnhancedJSONEncoder` | `pyoaev.utils` | `from xtm_oaev_sdk import EnhancedJSONEncoder` | +| `EnvironmentSource` | `pyoaev.configuration.configuration`, `pyoaev.configuration.sources` | `from xtm_oaev_sdk import EnvironmentSource` | +| `ExecutionDetails` | `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ExecutionDetails` | +| `ExecutionSignature` | `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ExecutionSignature` | +| `ExecutionStatus` | `pyoaev.signatures.models`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import ExecutionStatus` | +| `Expectation` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import Expectation` | +| `ExpectationSignatureGroup` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ExpectationSignatureGroup` | +| `ExpectationType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractExpectationType` | +| `ExpectationType` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import SignatureExpectationType` | +| `ExtraSignatureData` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ExtraSignatureData` | +| `FALSY` | `pyoaev.helpers` | `from xtm_oaev_sdk import is_falsy` | +| `InjectExecutionActions` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import InjectExecutionActions` | +| `InjectorConfig` | `pyoaev.signatures`, `pyoaev.signatures.models` | `from xtm_oaev_sdk import InjectorConfig` | +| `is_falsy` | `pyoaev.configuration.configuration` | `from xtm_oaev_sdk import is_falsy` | +| `is_memory_certificate` | `pyoaev.helpers` | `from xtm_oaev_sdk import is_memory_certificate` | +| `is_truthy` | `pyoaev.configuration.configuration` | `from xtm_oaev_sdk import is_truthy` | +| `LinkedFieldModel` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import LinkedFieldModel` | +| `logger` | `pyoaev.daemons.base_daemon`, `pyoaev.utils` | `from xtm_oaev_sdk import logger` | +| `MatchTypes` | `pyoaev`, `pyoaev.apis.inject_expectation.model.expectation`, `pyoaev.signatures`, `pyoaev.signatures.signature_match`, `pyoaev.signatures.signature_type`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import MatchTypes` | +| `NetworkInjectorConfig` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import NetworkInjectorConfig` | +| `on_http_error` | `pyoaev.exceptions` | `from xtm_oaev_sdk import on_http_error` | +| `OpenAEVAuthenticationError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVAuthenticationError` | +| `OpenAEVCreateError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVCreateError` | +| `OpenAEVDeleteError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVDeleteError` | +| `OpenAEVError` | `pyoaev.daemons.base_daemon`, `pyoaev.exceptions`, `pyoaev.signatures.signature_manager`, `pyoaev.signatures.signature_match` | `from xtm_oaev_sdk import OpenAEVError` | +| `OpenAEVGetError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVGetError` | +| `OpenAEVHeadError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVHeadError` | +| `OpenAEVHttpError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVHttpError` | +| `OpenAEVListError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVListError` | +| `OpenAEVParsingError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVParsingError` | +| `OpenAEVUpdateError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVUpdateError` | +| `PingAlive` | `pyoaev.daemons.collector_daemon`, `pyoaev.helpers`, `pyoaev.utils` | `from xtm_oaev_sdk import PingAlive` | +| `prepare_contracts` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import prepare_contracts` | +| `PrimitiveType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import PrimitiveType` | +| `RedirectError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import RedirectError` | +| `remove_none_from_dict` | `pyoaev.utils` | `from xtm_oaev_sdk import remove_none_from_dict` | +| `RequiredOptional` | `pyoaev.utils` | `from xtm_oaev_sdk import RequiredOptional` | +| `SecurityDomains` | `pyoaev.security_domain.types` | `from xtm_oaev_sdk import SecurityDomains` | +| `SecurityPlatformType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import SecurityPlatformType` | +| `SettingsLoader` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import SettingsLoader` | +| `setup_logging_config` | `pyoaev.utils` | `from xtm_oaev_sdk import setup_logging_config` | +| `SignatureCallbackPayload` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models` | `from xtm_oaev_sdk import SignatureCallbackPayload` | +| `SignatureManager` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import SignatureManager` | +| `SignatureMatch` | `pyoaev.signatures.signature_match`, `pyoaev.signatures.signature_type` | `from xtm_oaev_sdk import SignatureMatch` | +| `SignatureOutputStructure` | `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import SignatureOutputStructure` | +| `SignaturePayload` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import SignaturePayload` | +| `SignatureTarget` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import SignatureTarget` | +| `SignatureTransmissionError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import SignatureTransmissionError` | +| `SignatureType` | `pyoaev.apis.inject_expectation.model.expectation`, `pyoaev.signatures.signature_type` | `from xtm_oaev_sdk import SignatureType` | +| `SignatureTypes` | `pyoaev`, `pyoaev.apis.inject_expectation.model.expectation`, `pyoaev.signatures`, `pyoaev.signatures.signature_type`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import SignatureTypes` | +| `SignatureValue` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import SignatureValue` | +| `ssl_cert_chain` | `pyoaev.helpers` | `from xtm_oaev_sdk import ssl_cert_chain` | +| `ssl_verify_locations` | `pyoaev.helpers` | `from xtm_oaev_sdk import ssl_verify_locations` | +| `SupportedLanguage` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import SupportedLanguage` | +| `TargetSignatures` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import TargetSignatures` | +| `ToolErrorInfo` | `pyoaev.signatures.models` | `from xtm_oaev_sdk import ToolErrorInfo` | +| `ToolOutput` | `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ToolOutput` | +| `ToolTimeoutInfo` | `pyoaev.signatures.models` | `from xtm_oaev_sdk import ToolTimeoutInfo` | +| `TRUTHY` | `pyoaev.helpers` | `from xtm_oaev_sdk import is_truthy` | +| `VariableHelper` | `pyoaev.contracts.contract_config`, `pyoaev.contracts.variable_helper` | `from xtm_oaev_sdk import VariableHelper` | +| `VariableType` | `pyoaev.contracts.contract_utils`, `pyoaev.contracts.variable_helper` | `from xtm_oaev_sdk import VariableType` | + +### To collectors-sdk + +| Legacy name | Legacy modules | New import | +|---|---|---| +| `CollectorDaemon` | `pyoaev.daemons`, `pyoaev.daemons.collector_daemon`, `pyoaev.helpers` | `from collectors_sdk import CollectorDaemon` | +| `DEFAULT_PERIOD_SECONDS` | `pyoaev.daemons.collector_daemon` | `from collectors_sdk import DEFAULT_PERIOD_SECONDS` | +| `DEFAULT_PLATFORM_TAG_COLOR` | `pyoaev.daemons.collector_daemon` | `from collectors_sdk import DEFAULT_PLATFORM_TAG_COLOR` | +| `DetectionExpectation` | `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import DetectionExpectation` | +| `Expectation` | `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import Expectation` | +| `ExpectationSignature` | `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import ExpectationSignature` | +| `ExpectationTypeEnum` | `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import ExpectationTypeEnum` | +| `OpenAEVCollectorHelper` | `pyoaev.helpers` | `from collectors_sdk import CollectorDaemon` | +| `OpenAEVDetectionHelper` | `pyoaev.helpers` | `from collectors_sdk import OpenAEVDetectionHelper` | +| `PreventionExpectation` | `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import PreventionExpectation` | + +### To injectors-sdk + +| Legacy name | Legacy modules | New import | +|---|---|---| +| `create_mq_ssl_context` | `pyoaev.helpers` | `from injectors_sdk import create_mq_ssl_context` | +| `ListenQueue` | `pyoaev.helpers` | `from injectors_sdk import ListenQueue` | +| `OpenAEVConfigHelper` | `pyoaev.helpers` | `from injectors_sdk import OpenAEVConfigHelper` | +| `OpenAEVInjectorHelper` | `pyoaev.helpers` | `from injectors_sdk import OpenAEVInjectorHelper` | + +### Without replacement + +| Legacy name | Legacy modules | New import | +|---|---|---| +| `CHALLENGES_URI` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `COMCHECK` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `EXERCISE` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `LESSONS_URI` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `PLAYER_URI` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `SCOREBOARD_URI` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `TEAMS` | `pyoaev.contracts.variable_helper` | no replacement (see below) | +| `USER` | `pyoaev.contracts.variable_helper` | no replacement (see below) | + +## 4. Inject the API client + +The SDKs never build the API client. You build it, and you hand it over. Four legacy +classes used to build it themselves from the configuration (`openaev_url`, +`openaev_token`, `openaev_tenant_id`), and their legacy paths still do. + +Collector: + +```python +# before +from pyoaev.daemons import CollectorDaemon + +CollectorDaemon(configuration=configuration, callback=collect, collector_type="openaev_my_collector") + +# after +from pyoaev import OpenAEV +from collectors_sdk import CollectorDaemon + +api = OpenAEV( + url=configuration.get("openaev_url"), + token=configuration.get("openaev_token"), + tenant_id=configuration.get("openaev_tenant_id"), +) +CollectorDaemon( + configuration=configuration, + callback=collect, + api_client=api, + collector_type="openaev_my_collector", +) +``` + +`BaseDaemon` (`pyoaev.daemons` → `xtm_oaev_sdk`) plays by the same rule: same +`api_client=`. + +Injector: + +```python +# before +from pyoaev.helpers import OpenAEVInjectorHelper + +helper = OpenAEVInjectorHelper(config, icon) + +# after +from pyoaev import OpenAEV +from injectors_sdk import OpenAEVInjectorHelper + +api = OpenAEV( + url=config.get_conf("openaev_url"), + token=config.get_conf("openaev_token"), + tenant_id=config.get_conf("openaev_tenant_id"), +) +helper = OpenAEVInjectorHelper(config, icon, api_client=api) +``` + +Signature manager: + +```python +# before +from pyoaev.signatures import SignatureManager + +manager = SignatureManager(client, logger) + +# after +from xtm_oaev_sdk import SignatureManager + +manager = SignatureManager(transport=client.signature, logger=logger) +``` + +## Renamed names + +| Legacy | New | +|---|---| +| `pyoaev.signatures.ExpectationType` | `xtm_oaev_sdk.SignatureExpectationType` | +| `pyoaev.contracts.contract_config.ExpectationType` | `xtm_oaev_sdk.ContractExpectationType` | +| `pyoaev.helpers.TRUTHY` / `FALSY` | `xtm_oaev_sdk.is_truthy` / `is_falsy` | +| `pyoaev.helpers.OpenAEVCollectorHelper` (already deprecated) | `collectors_sdk.CollectorDaemon` | + +The constants of `pyoaev.contracts.variable_helper` (`USER`, `EXERCISE`, `TEAMS`, +`COMCHECK` and the `*_URI` ones) get no public replacement. Reach for the built-in +variables of `xtm_oaev_sdk.VariableHelper` instead. + +## What stays in pyoaev + +The API client, and everything wired around it: `OpenAEV`, `pyoaev.apis` (including +`inputs.search`), `pyoaev.backends`, `pyoaev.base`, `pyoaev.client`, `pyoaev.mixins`, +and `pyoaev.utils.get_content_type` / `response_content`. Nothing to migrate there. + +## Things you may notice + +- In tests, the legacy patch targets (`pyoaev.daemons.base_daemon.OpenAEV`, + `pyoaev.helpers.OpenAEV`, `pyoaev.utils.PingAlive.start`…) keep biting as long as + your code runs through the legacy paths. Once it imports from the SDKs, patch the + names it actually looks up now. +- `from import X` on a legacy package (for example `from pyoaev.signatures + import SignatureManager`) fires the warning twice. Plain modules fire it once. +- An `AppLogger` built without a name, and a `SignatureManager` built without a logger, + now log under the SDK module name, not the former `pyoaev` one. diff --git a/libs/xtm-oaev-sdk/docs/seam_strategy.md b/libs/xtm-oaev-sdk/docs/seam_strategy.md new file mode 100644 index 00000000..61583fef --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/seam_strategy.md @@ -0,0 +1,239 @@ +# Seam strategy: port/adapter layering of the SDKs + +`xtm-oaev-sdk`, `collectors-sdk` and `injectors-sdk` share one build. Every +capability is cut into a **public** adapter layer, a **seam** of contracts, and a +**core** implementation. This document covers the intent, the rules, the patterns +that applied them without touching behaviour, and how to add a capability. + +## Intent + +| Layer | Role | +|---|---| +| `public/.py` | What users import: ready-to-use adapters (classes, functions) plus the seam contracts users need. Adapters speak seam types and hand the work to the core. | +| `seam//` | The **contract** between public and core: data models, enums, errors, value objects, and `Protocol` ports, plus `SEAM_VERSION`. | +| `_core//` | The code behind the public API: one implementation per seam port. | + +As long as the core keeps answering the seam, the public layer keeps working, and +so does every consumer downstream of it. The core can be rewritten at will. A change +*to the seam* is another matter: that is a contract change. It bumps `SEAM_VERSION`, +earns a changelog entry, and needs a deprecation once it reaches the public surface. + +## Rules + +``` +public ──► seam ◄── _core + └───────────────────────► (wiring only: default implementations) +``` + +1. **The seam is closed.** It imports only the standard library, third-party + libraries (pydantic…), other packages' *public* API (e.g. `collectors_sdk` seams + import `xtm_oaev_sdk`) and sibling seam modules. Never `_core`. Never `public`. +2. **The core implements the seam.** It imports seams, other core modules and + third-party libraries. Never `public`, not even lazily; type-only imports go under + `TYPE_CHECKING`. +3. **Public never hands out a core object.** Every exported name is either an adapter + defined in `public/` or a contract defined in `seam/`. Public touches `_core` for + one reason only, picking the default implementation of each port: + + ```python + _truthiness: TruthinessPort = Truthiness() + _sources: ConfigurationSourcesPort = ConfigurationSources() + _resolver: ConfigurationResolverPort = ConfigurationResolver(_sources, _truthiness) + ``` + +4. **Users import from the package root** (`from xtm_oaev_sdk import X`). The layers + are internal wiring. + +## What goes where + +### In the seam: data contracts + +Pydantic models, dataclasses, enums, exceptions, type aliases and constants. In +short, whatever crosses the boundary. They may carry **their own invariants**, logic +that only reads or derives from their own data, because that logic is part of what +the data *is*: + +- validators and derived fields (`ConfigurationHint`, `ExpectationData`'s null + signatures coerced to `[]`); +- `__post_init__` invariants of the contract elements (derived `type`, forced + cardinality, predefined expectations); +- value objects: `EncodedId` (URL encoding), `RequiredOptional.validate_attrs`, + `SignatureMatch` (score invariant), `SignatureType.make_struct_for_matching`; +- exceptions, including `OpenAEVError`'s extraction of a readable message from its + own response body: the core raises them and users catch them, so they have to live + where both sides can reach them. + +Anything that does I/O, orchestrates, or leans on another capability is +**behaviour**, and it goes behind a port. + +### Behind a port: everything else, however small + +Every piece of behaviour gets a `Protocol` port in the seam, an implementation in +the core and an adapter in public, **even a one-liner**. `is_truthy` gets its +`TruthinessPort`, `Truthiness` and public `is_truthy`, same treatment as +`SignatureManager`. Uniformity is the whole point: no "too small to need a seam" +exception left to argue about. + +A seam holds two kinds of protocols: + +- **Consumer protocols**: what users type against or implement: `BaseClient`, + `DaemonProtocol`, `AppLoggerProtocol`, `ConfigurationProtocol`, + `ContractBuilderProtocol`, `SignatureManagerProtocol`, and the driven port + `SignatureTransportProtocol` (implemented by `pyoaev`'s signature API). +- **Internal ports**: what a public adapter needs from the core: + `ConfigurationResolverPort`, `SignatureEnginePort`, `CollectorRegistrationPort`… + +## Patterns used to adapt without changing behaviour + +The layering went onto existing code with behaviour held identical: public surface, +wire payloads and connector test suites, compared before and after. Pulling that off +took a handful of recurring patterns. + +### Data + behaviour classes: split, the public class extends the seam data + +When a user-facing class mixes data and behaviour (`Contract`, `Expectation`), the +seam keeps the data (`ContractData`, `ExpectationData`) and the public class extends +it, bolting the behaviour on through ports. Users keep constructing the public class; +the core only ever sees the seam type. + +```python +@dataclass +class Contract(ContractData): + variables: list[ContractVariable] = field( + default_factory=lambda: _variables.default_variables() + ) + + def to_contract_add_input(self, source_id: str) -> dict[str, Any]: + return _serializer.to_add_input(self, source_id) +``` + +### Framework subclasses stay framework subclasses + +Adapters that users hand to a framework keep their base class, override only the +hook, and delegate the logic: `EnhancedJSONEncoder` (`json.JSONEncoder.default`), +`CustomJsonFormatter` (pythonjsonlogger `add_fields`), `ConnectorConfigSchemaGenerator` +(pydantic `GenerateJsonSchema`), `SettingsLoader` (pydantic-settings +`settings_customise_sources`), `PingAlive` / `ListenQueue` (`threading.Thread`). + +### Stateful adapters keep their attributes and hooks + +`BaseDaemon`, `CollectorDaemon`, `SignatureManager`, `AppLogger`, +`OpenAEVInjectorHelper` keep their attributes (`_configuration`, `api`, `logger`, +`local_logger`, IP caches…) and their overridable hooks (`_setup`, `_start_loop`, +`_try_callback`, `_create_default_api_client`, `_utcnow`…). Subclasses and tests +lean on them. The adapter holds the state; the core holds the logic. + +### Pass callables to keep dispatch and patch points + +Where the original code called back into its own object, the adapter passes that +callback down to the core, so overrides and patches still land: + +```python +def _source_context(self, config): + return _engine.source_context( + config, self.resolve_container_ip, lambda: self._cached_ipv6 + ) +``` + +`SignatureManager.post_execution_updates` passes `self._utcnow` (read only after the +tool output validates, as before); `Expectation.match_alert` passes +`self.match_fuzzy` / `self.match_simple`; `OpenAEVDetectionHelper` passes +`self.match_alert_element_fuzzy` / `self._decode_value`; `CollectorDaemon` passes +`self._try_callback` to the scheduler. A dependency the core cannot import travels +the same way: `Configuration.schema()` hands the public +`ConnectorConfigSchemaGenerator` to the core resolver. + +### A state protocol when the core drives a stateful loop + +`ListenQueue`'s RabbitMQ loop rewrites public attributes (`pika_connection`, +`channel`, `thread`…). The seam declares a `QueueListenerProtocol` describing that +state; the core runs the loop (connect, consume, recover, process, stop) against it, +and the adapter's methods shrink to one-line delegations: + +```python +def run(self) -> None: + _consumer.run(self) +``` + +## Versioning + +- Every capability seam exposes `SEAM_VERSION`. `port_io`, a pure + wire schema, carries `PORT_IO_CONTRACT_VERSION` instead. +- The seams that cross a process boundary pin their JSON schemas with golden hashes: + `seam/signatures/seam.py` (inbound I/O: injector configs, execution + signatures, tool output) and `seam/port_io` (`PORT_IO_CONTRACT_VERSION`: the wire + payloads sent to the platform). See `test/seam/`. +- Policy: a change to a seam (field, type, enum member, error, port signature) bumps + its version and lands in the changelog. If it reaches the public surface, the old + name goes through a deprecation shim first (see [shim_strategy.md](shim_strategy.md)). + +## Walkthrough: the configuration capability + +```python +# seam/configuration/ports.py — the contract (closed) +@runtime_checkable +class TruthinessPort(Protocol): + def is_truthy(self, value: str) -> bool: ... + def is_falsy(self, value: str) -> bool: ... + +# _core/configuration/truthiness.py — the implementation (imports the seam only) +class Truthiness: + def is_truthy(self, value: str) -> bool: + return value.lower() in ["yes", "true"] + +# public/configuration.py — the adapter (what users import) +_truthiness: TruthinessPort = Truthiness() + +def is_truthy(value: str) -> bool: + return _truthiness.is_truthy(value) +``` + +The same capability carries the heavier cases too. `Configuration` keeps its +constructor, `get`, `set` and `schema`, and delegates resolution to +`ConfigurationResolverPort`. `SettingsLoader` stays a `BaseSettings` and delegates +source selection to `SettingsSourcesPort`. The settings models (`BaseConfigModel`, +`ConfigLoaderOAEV`…) are data contracts in the seam. + +## Capability map + +### xtm-oaev-sdk + +| Capability | Seam: contracts | Seam: ports | Core | Public adapters | +|---|---|---|---|---| +| common | the error hierarchy, the asset and security-domain enums, `EncodedId`, `RequiredOptional`, `BaseClient`, `DaemonProtocol`, `AppLoggerProtocol` | `HttpErrorPort`, `JsonEncodingPort`, `LoggingPort`, `DictToolsPort`, `CertificatesPort` | `HttpErrorTranslator`, `JsonEncoding`, `LoggingSetup`, `DictTools`, `Certificates` | `on_http_error`, `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, `copy_dict`, `remove_none_from_dict`, the SSL helpers | +| configuration | `ConfigurationHint`, settings models, serializer types, `CONFIGURATION_TYPES`, `ConfigurationProtocol` | `ConfigurationResolverPort`, `ConfigurationSourcesPort`, `ConnectorSchemaPort`, `SettingsSourcesPort`, `TruthinessPort` | `ConfigurationResolver`, `ConfigurationSources`, `ConnectorSchemaTransformer`, `SettingsSourceSelector`, `Truthiness` | `Configuration`, `EnvironmentSource`, `DictionarySource`, `ConnectorConfigSchemaGenerator`, `SettingsLoader`, `is_truthy`, `is_falsy` | +| contracts | `ContractData`, contract elements and enums, `ContractVariable`, `Expectation`, `ContractBuilderProtocol` | `ContractAssemblyPort`, `ContractVariablesPort`, `ContractSerializerPort` | `ContractAssembly`, `BuiltinContractVariables`, `ContractSerializer` | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts` | +| signatures | inbound I/O models, matching types and policies, `ExtraSignatureData`, `SignatureTransportProtocol`, `SignatureManagerProtocol` | `SignatureEnginePort`, `ContainerAddressPort`, `NetworkTargetsPort`, `MarkerPort` | `SignatureEngine`, `ContainerAddressResolver`, `NetworkTargets`, `MarkerBuilder` | `SignatureManager`, `build_network_configs`, `build_marker` | +| port_io | wire models and enums, `PORT_IO_CONTRACT_VERSION` | (none: pure wire schema) | | re-exported as is | +| daemons | (consumer protocols live in common) | `DaemonRuntimePort`, `HeartbeatPort` | `DaemonRuntime`, `Heartbeat` | `BaseDaemon`, `PingAlive` | +| deprecation | `DeprecatedNames` | `DeprecationPort` | `DeprecationShims` | `make_deprecated_getattr` | + +### collectors-sdk + +| Capability | Seam: contracts | Seam: ports | Core | Public adapters | +|---|---|---|---|---| +| daemons | `DEFAULT_PERIOD_SECONDS`, `DEFAULT_PLATFORM_TAG_COLOR` | `CollectorRegistrationPort`, `CollectorSchedulePort` | `CollectorRegistration`, `CollectorSchedule` | `CollectorDaemon` | +| detection | | `AlertMatchingPort` | `AlertMatcher` | `OpenAEVDetectionHelper` | +| expectations | `ExpectationData`, `ExpectationSignature`, `ExpectationTypeEnum` | `ExpectationMatchingPort`, `ExpectationUpdatePort` | `ExpectationMatcher`, `ExpectationUpdates` | `Expectation`, `DetectionExpectation`, `PreventionExpectation` | + +### injectors-sdk + +| Capability | Seam: contracts | Seam: ports | Core | Public adapters | +|---|---|---|---|---| +| helpers | `QueueListenerProtocol` (listener state) | `ConfigLookupPort`, `InjectorRegistrationPort`, `QueueConsumerPort`, `MqSslPort` | `ConfigLookup`, `InjectorRegistration`, `QueueConsumer`, `MqSsl` | `OpenAEVConfigHelper`, `OpenAEVInjectorHelper`, `ListenQueue`, `create_mq_ssl_context` | + +## Adding a capability + +1. **Seam** (`seam//`): the data contracts in `models.py`, one `Protocol` + per piece of behaviour the public API needs in `ports.py`, `SEAM_VERSION = "1.0.0"` + in `__init__.py`. Import nothing from `_core` or `public`. +2. **Core** (`_core//`): one class per port, implementing it with the + exact method signatures. Import the seam, never `public`. +3. **Public** (`public/.py`): module-level default implementations typed + by their port, adapters carrying the user-facing signatures and delegating to them, + and the seam contracts users need. Set `__all__`. +4. **Root**: import the module in `public/__init__.py` and extend `__all__`. +5. **Tests**: behaviour through the public API; core implementations through their + port when useful. +6. If the capability replaces a `pyoaev` name, add its deprecation shim in `pyoaev` + (see [shim_strategy.md](shim_strategy.md)). diff --git a/libs/xtm-oaev-sdk/docs/shim_strategy.md b/libs/xtm-oaev-sdk/docs/shim_strategy.md new file mode 100644 index 00000000..93651e89 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/shim_strategy.md @@ -0,0 +1,189 @@ +# Shim strategy: splitting pyoaev without breaking anyone + +`pyoaev` used to carry everything: the OpenAEV API client, and on top of it +configuration, contracts, signatures, the daemon runtime, errors, helpers. That +shared code now lives in three SDKs (`xtm-oaev-sdk`, `collectors-sdk`, +`injectors-sdk`), and `pyoaev` is back to its actual job: being the API client. + +One rule governs the split: **no breaking change for existing code.** Every +collector and injector written against `pyoaev` keeps running untouched until the +shims are deliberately pulled. This document lays out how that holds, and why it is +built this way. + +## The contract + +For every name a released `pyoaev` exposed: + +1. **The legacy path still resolves**: `from pyoaev.configuration import Configuration`, + `pyoaev.helpers.OpenAEVInjectorHelper`, `from pyoaev import AssetCategory`… +2. **It resolves to the SDK object**, and to the *identical* object whenever possible, + so `is`, `isinstance`, subclassing, pickling-by-reference and introspection behave + exactly as before. +3. **Access fires a `DeprecationWarning`** naming the new import, pinned to the + **consumer's line**, so the default warning filters actually show it. +4. **Importing `pyoaev` stays silent**: only touching a moved name warns. +5. **Patch targets keep landing**: what connectors' tests patch (`pyoaev.utils.PingAlive.start`, + `pyoaev.daemons.collector_daemon.CollectorDaemon._setup`, + `pyoaev.daemons.base_daemon.OpenAEV`, `pyoaev.signatures.signature_manager.socket.gethostbyname`…) + still hits the code that actually runs. + +## Dependency direction + +``` +xtm-oaev-sdk ← collectors-sdk + ↑ ← injectors-sdk + └──────────────┴────────── pyoaev (API client + shims) +``` + +- The SDKs **never** import `pyoaev`. When they need the platform, the API client + gets injected (`xtm_oaev_sdk.BaseClient` protocol; `pyoaev.OpenAEV` satisfies it). +- `pyoaev` **hard-depends** on the three SDKs for the whole deprecation window. That + is what keeps a connector that only lists `pyoaev` alive. An optional extra + (`pyoaev[collectors]`) would have broken exactly those connectors. +- The direct dependencies `pyoaev` no longer imports (`pika`, `thefuzz`, `PyYAML`, + `pydantic-settings`, `python-json-logger`) are gone from its `pyproject.toml`. + They still arrive transitively through the SDKs, so a `pyoaev` install pulls the + same third-party packages as before. + +## The mechanism: PEP 562 module `__getattr__` + +Every shim module serves its moved names lazily, through a module-level +`__getattr__` built by the SDK primitive `make_deprecated_getattr`: + +```python +"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`.""" + +import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + +__all__ = [ # noqa: F822 (names are served by the PEP 562 __getattr__ below) + "DictionarySource", + "EnvironmentSource", +] + +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) +``` + +`make_deprecated_getattr(mapping, module_name=...)` validates the +`{old_name: (message, replacement)}` mapping once, then hands back a `__getattr__` +that warns (`DeprecationWarning`, `stacklevel=2`) and returns the replacement **as +is**. Unknown names still raise `AttributeError`. Typos never get swallowed. + +### Why not the alternatives + +- **`typing_extensions.deprecated(...)(cls)`** does not wrap the + class. It **patches the original class in place** (`__new__` / + `__init_subclass__`). The moment anything imported `pyoaev`, users of + `xtm_oaev_sdk.Configuration` *also* caught "use `xtm_oaev_sdk`" warnings. Rejected. +- **A wrapper class or proxy** breaks identity: `isinstance` against the SDK class, + metaclasses and signature introspection all shift. Rejected. +- **A plain re-export** (`from xtm_oaev_sdk import X`) keeps identity but never + warns, so nobody would ever migrate. Rejected. +- **Optional dependencies or lazy imports of the SDKs** would make legacy imports + blow up for connectors that only list `pyoaev`. Rejected (see dependency direction). + +## The kinds of shims + +| Kind | Example | How it is served | +|---|---|---| +| **Moved, identical** (the vast majority) | `pyoaev.configuration.Configuration` → `xtm_oaev_sdk.Configuration` | the SDK object itself | +| **Renamed** | `pyoaev.signatures.ExpectationType` → `xtm_oaev_sdk.SignatureExpectationType`; `pyoaev.contracts.contract_config.ExpectationType` → `xtm_oaev_sdk.ContractExpectationType` | the SDK object; the message names the new name | +| **Client-building adapter** | `pyoaev.daemons.BaseDaemon`, `pyoaev.daemons.CollectorDaemon`, `pyoaev.signatures.SignatureManager`, `pyoaev.helpers.OpenAEVInjectorHelper` | a thin subclass of the SDK class (see below) | +| **Value without public replacement** | `pyoaev.contracts.variable_helper.USER`…, `pyoaev.helpers.TRUTHY` / `FALSY` | the literal value, with a message pointing at the right API | +| **Private legacy helper** | `pyoaev.helpers._is_base64_encoded` | the collectors-sdk implementation, "private and will be removed" | +| **Legacy-only glue** | `pyoaev.helpers.OpenAEVCollectorHelper` (already deprecated) | kept in `pyoaev` on top of the collectors-sdk daemon, warns on access | +| **Re-exported names** | `pyoaev.daemons.base_daemon.Configuration`, `pyoaev.apis.inject_expectation.model.expectation.SignatureTypes`… | the SDK object (names a module merely imported are part of its surface too) | + +### Client-building adapters + +The SDKs take the API client as an argument (`api_client=` / `transport=`). The +legacy constructors built `OpenAEV(...)` themselves. So those legacy names resolve to +a thin subclass of the SDK class whose only job is to restore that behaviour: + +| Legacy name | Subclass of | What it adds | +|---|---|---| +| `pyoaev.daemons.BaseDaemon` | `xtm_oaev_sdk.BaseDaemon` | `_create_default_api_client()` builds `OpenAEV` from `openaev_url` / `openaev_token` / `openaev_tenant_id` | +| `pyoaev.daemons.CollectorDaemon` | `collectors_sdk.CollectorDaemon` + the class above | same hook; still an instance of the legacy `BaseDaemon` | +| `pyoaev.signatures.SignatureManager` | `xtm_oaev_sdk.SignatureManager` | `SignatureManager(client, logger)` uses `client.signature` as transport | +| `pyoaev.helpers.OpenAEVInjectorHelper` | `injectors_sdk.OpenAEVInjectorHelper` | `_create_default_api_client(config)` builds `OpenAEV` from the config helper | + +`isinstance(legacy_object, )` holds. Each adapter is **one object**, +whatever path reaches it (`pyoaev.daemons.CollectorDaemon is +pyoaev.daemons.collector_daemon.CollectorDaemon is pyoaev.helpers.CollectorDaemon`), +so patching a method on any of those paths hits the class connectors instantiate. + +## Rules learned the hard way + +- **Assign `__getattr__` directly.** A `def __getattr__(name): return + _deprecated_getattr(name)` wrapper slips in an extra frame. The warning then gets + pinned to the shim file, and Python's default filters (which only show + `DeprecationWarning` for `__main__`) hide it from scripts. +- **Never star-import a shim module.** `from pyoaev.signatures import *` walks + `__all__`, trips every lazy name and makes `import pyoaev` warn. Instead, the + `pyoaev` package root binds the shim submodules and serves their names through its + own `__getattr__` (`pyoaev.__all__` is unchanged, so `from pyoaev import *` keeps + exporting the same names, and warns for the moved ones). +- **pyoaev's own code imports from the SDKs**, never from its shims. Importing any + `pyoaev` module, `pyoaev.apis` included, stays silent. +- **Keep patch surfaces.** `pyoaev.signatures.signature_manager` still imports + `socket` and `subprocess` (tests patch `...signature_manager.socket.gethostbyname`, + which reaches the shared stdlib module the SDK uses). The client-building adapters + call `OpenAEV` through their module global, so `patch("pyoaev.daemons.base_daemon.OpenAEV")` + and `patch("pyoaev.helpers.OpenAEV")` still bite. +- **`__all__` lists lazily served names**: flake8 flags F822 outside `__init__.py`, + so the line carries `# noqa: F822` with the reason. +- **Known quirk**: `from import X` on a shim *package* warns twice + (importlib's `_handle_fromlist` probes with `hasattr` before the real lookup). + Plain modules warn once. Harmless, and baked into PEP 562. + +## Where the shims are + +| pyoaev module | Serves | +|---|---| +| `pyoaev/__init__.py` | the moved names it exported (asset enums, exceptions, configuration, contracts, signatures) | +| `pyoaev/configuration/*` | xtm-oaev-sdk configuration | +| `pyoaev/contracts/*` | xtm-oaev-sdk contracts (`ExpectationType` renamed, variable constants) | +| `pyoaev/signatures/*` | xtm-oaev-sdk signatures (`ExpectationType` renamed, `SignatureManager` adapter) | +| `pyoaev/exceptions.py`, `asset_types.py`, `security_domain/types.py` | xtm-oaev-sdk common | +| `pyoaev/utils.py` | xtm-oaev-sdk common and `PingAlive`; keeps the HTTP helpers `get_content_type`, `response_content` | +| `pyoaev/daemons/*` | `BaseDaemon` / `CollectorDaemon` adapters | +| `pyoaev/helpers.py` | collectors-sdk detection, injectors-sdk helpers (`OpenAEVInjectorHelper` adapter), xtm SSL helpers, `OpenAEVCollectorHelper` | +| `pyoaev/apis/inject_expectation/model/*` | collectors-sdk expectations (the API manager builds them) | + +## How it is tested + +| Test | Guards | +|---|---| +| `libs/xtm-oaev-sdk/test//shim/test_shim_contract.py` | identity, warning text, pinning to the consumer's line, silent imports, unknown names, adapters (client construction, `isinstance`) | +| `libs/xtm-oaev-sdk/test//shim/test_*.py` | the original pyoaev tests, moved **verbatim**: they still drive the legacy paths | +| `test/test_sdk_split_shims.py` | the collectors-sdk / injectors-sdk shims, adapters and connector patch targets | +| `test/test_legacy_surface.py` | every public name of the **released** pyoaev, recorded in `test/fixtures/legacy_surface.json` by `test/generate_legacy_surface.py` from the release, not the working tree) still resolves | + +Beyond the suites, the split went through the real collectors|injectors own test suites +run once against the released `pyoaev` and once +against the new stack. Identical results. + +## Observable differences (not API breaks) + +- Legacy imports warn (that is the whole point of the shims). +- The client-building adapters are subclasses, not the identical SDK classes. +- An `AppLogger` built without a name, and a `SignatureManager` built without a + logger, log under the SDK module name instead of the former pyoaev one. + +## Removing the shims + +In a major release of `pyoaev`: + +1. delete the shim modules and the `__getattr__` of `pyoaev/__init__.py`; +2. delete the client-building adapters and `OpenAEVCollectorHelper`; +3. drop the `xtm-oaev-sdk` / `collectors-sdk` / `injectors-sdk` dependencies only the + shims need (keep `xtm-oaev-sdk` if the client still leans on its errors and + transport models); +4. delete `test/test_legacy_surface.py` and the `shim/` test folders; +5. announce it: every removed name has been warning since the split, new import included. diff --git a/libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md b/libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md new file mode 100644 index 00000000..968d31fe --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md @@ -0,0 +1,51 @@ +# Technical Decision Records + +The technical decision records of the SDK split. Each one is a rule that future work on +xtm-oaev-sdk, collectors-sdk, injectors-sdk and the `pyoaev` shims is expected to respect. + +## Compatibility with pyoaev + +| TDR | Status | Rule | +|---|---|---| +| [Legacy pyoaev code keeps working until a pyoaev major release](no-breaking-change-for-legacy-pyoaev-code.tdr.md) | draft | Every public name of the latest released pyoaev must keep resolving from its legacy path, with a DeprecationWarning naming its new import, until a pyoaev major release removes the shims. | +| [The SDKs never depend on pyoaev; the API client is injected](sdks-never-import-pyoaev.tdr.md) | draft | xtm-oaev-sdk, collectors-sdk and injectors-sdk must never import pyoaev; anything that talks to the platform receives its API client or transport as an argument typed by a seam protocol. | +| [pyoaev hard-depends on the SDKs that host its former code](pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md) | draft | While pyoaev ships deprecation shims, it must declare a hard dependency on every SDK those shims serve, never an optional extra. | +| [Legacy names are served by a directly assigned PEP 562 __getattr__](legacy-names-served-by-pep562-module-getattr.tdr.md) | draft | A moved name must be served from its legacy module by a module-level __getattr__ built with make_deprecated_getattr and assigned directly, never wrapped in a def. | +| [Shims return the identical SDK object; adapters only where constructors differ](shims-return-the-identical-sdk-object.tdr.md) | draft | A shim must return the SDK object itself; a legacy name may resolve to a thin subclass only to restore legacy constructor behaviour, and that subclass must be one single object whatever legacy path reaches it. | +| [Importing pyoaev never warns; only touching a moved name does](importing-pyoaev-stays-silent.tdr.md) | draft | Importing any pyoaev module must not emit a DeprecationWarning; only accessing a moved name may warn. | +| [Legacy patch targets are part of the compatibility surface](legacy-patch-targets-are-part-of-the-surface.tdr.md) | draft | Module attributes that connectors patch must stay reachable at their legacy path and still reach the code that actually runs. | +| [The legacy surface is guarded against the released pyoaev](legacy-surface-guarded-from-the-release.tdr.md) | draft | The legacy-surface guard must be generated from the released pyoaev package, and the pre-split tests of moved code must be kept verbatim against their legacy paths. | + +## Layering + +| TDR | Status | Rule | +|---|---|---| +| [public and _core depend on the seam; the seam depends on neither](public-seam-core-dependency-rule.tdr.md) | draft | Within each SDK capability, the seam imports neither public nor _core, _core never imports public, and public imports _core only to wire the default implementation of each port. | +| [Every piece of behaviour goes behind a port, even a one-liner](every-behaviour-behind-a-port.tdr.md) | draft | Every piece of behaviour exposed by an SDK goes through a seam Protocol port, one _core implementation and a public adapter, whatever its size. | +| [The seam holds data contracts and only their intrinsic invariants](seam-holds-data-contracts-and-their-own-invariants.tdr.md) | draft | A seam may contain data contracts and logic that only reads or derives from their own data; anything doing I/O, orchestrating, or needing another capability goes behind a port. | +| [Data-and-behaviour classes split: the public class extends the seam data](public-class-extends-seam-data.tdr.md) | draft | When a user-facing class mixes data and behaviour, the seam holds the data class and the public class extends it, adding behaviour through ports; the core only knows the seam type. | +| [Public adapters keep the shape of the code they replace](adapters-keep-the-user-facing-shape.tdr.md) | draft | A public adapter must keep the signatures, docstrings, attributes, overridable hooks and framework base class of the code it replaces; only the logic moves to the core. | +| [Adapters pass their own methods to the core to keep dispatch and patch points](pass-bound-callables-to-keep-dispatch.tdr.md) | draft | When the original code dispatched through its own object, the adapter must pass the bound method or a callable to the core instead of letting the core reimplement the call. | + +## Contracts and versioning + +| TDR | Status | Rule | +|---|---|---| +| [A seam change is a versioned contract change](seam-changes-are-versioned-contract-changes.tdr.md) | draft | Any change to a seam contract (field, type, enum member, error, port signature) must bump that seam's version and be recorded in the changelog, and a change reaching the public surface must go through a deprecation shim first. | +| [Seams crossing a process boundary pin their JSON schemas](cross-process-seams-pin-their-json-schemas.tdr.md) | draft | A seam whose models cross a process boundary must pin their JSON schemas with golden hashes, regenerated only together with a version bump. | + +## Public API, tests and checks + +| TDR | Status | Rule | +|---|---|---| +| [Consumers import from the package root only](import-from-the-package-root.tdr.md) | draft | Consumers must import SDK names from the package root (xtm_oaev_sdk, collectors_sdk, injectors_sdk); public, seam and _core module paths are internal and may move. | +| [Colliding moved names are renamed with their capability prefix](colliding-names-get-a-capability-prefix.tdr.md) | draft | A moved name that collides with another capability's name in the flat root namespace must be renamed with its capability as prefix, and its legacy shim must name the new name. | +| [xtm-oaev-sdk stays agnostic of collectors, injectors and transports](xtm-oaev-sdk-stays-connector-agnostic.tdr.md) | draft | xtm-oaev-sdk must not contain collector- or injector-specific runtime, nor import pika, thefuzz or requests; those belong to collectors-sdk, injectors-sdk or pyoaev. | +| [Every SDK module imports on its own in a fresh interpreter](every-sdk-module-imports-alone.tdr.md) | draft | Every SDK module must import successfully as the first import of a fresh interpreter running with -W error. | + +## Adding a TDR + +Create `.tdr.md`, the slug naming the rule. Same header as the other records, +`status: draft`, then a row in this table. A TDR moves to `accepted` once validated. The +day it stops governing, it goes `superseded` (with `superseded_by` / `supersedes` set on +both records) or `revoked`. diff --git a/libs/xtm-oaev-sdk/docs/tdr/adapters-keep-the-user-facing-shape.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/adapters-keep-the-user-facing-shape.tdr.md new file mode 100644 index 00000000..57f517e1 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/adapters-keep-the-user-facing-shape.tdr.md @@ -0,0 +1,71 @@ +# Public adapters keep the shape of the code they replace +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +The classes you use from the SDK look and behave exactly like before: same names, same +arguments, same methods to override. Only the work they do moved, to a place behind +them. Think of swapping a car's engine without touching the steering wheel or the +pedals. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The layering went onto existing code with behaviour held identical. Users subclass SDK +> classes, hand them to frameworks, read their attributes and override their hooks. A +> facade would snap inheritance. + +--- + +## **The TDR** +| **A public adapter must keep the signatures, docstrings, attributes, overridable hooks and framework base class of the code it replaces; only the logic moves to the core.** + + + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- Framework subclasses stay subclasses and override only the hook: `EnhancedJSONEncoder` (`json.JSONEncoder.default`), `CustomJsonFormatter` (`add_fields`), `ConnectorConfigSchemaGenerator` (`GenerateJsonSchema`), `SettingsLoader` (`settings_customise_sources`), `PingAlive` / `ListenQueue` (`threading.Thread`). +- Stateful adapters keep attributes (`_configuration`, `api`, `logger`, `local_logger`…) and hooks (`_setup`, `_start_loop`, `_try_callback`, `_create_default_api_client`, `_utcnow`…). +- The public surface was compared before and after the refactors. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| A facade in front of classes users subclass (`BaseDaemon`, `CollectorDaemon`, `BaseConfigModel`) | Would break inheritance | The adapter is the class; the logic behind it moves | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- an SDK major version is allowed to change its public shape; +- a hook is formally deprecated. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- Adapters hold the state. The core holds the logic. + +--- + +## **Resources** +- [Seam strategy, patterns](../seam_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/colliding-names-get-a-capability-prefix.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/colliding-names-get-a-capability-prefix.tdr.md new file mode 100644 index 00000000..4420c5cf --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/colliding-names-get-a-capability-prefix.tdr.md @@ -0,0 +1,68 @@ +# Colliding moved names are renamed with their capability prefix +--- +scope: [xtm-oaev-sdk, public-api, shims] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +You import everything in the SDK from one single place, so two things can't share a +name. pyoaev had two different things both called `ExpectationType`. They became +`SignatureExpectationType` and `ContractExpectationType`. The old names still work, and +they tell you the new one. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The SDK root namespace is flat (TDR [import-from-the-package-root](import-from-the-package-root.tdr.md)). `pyoaev` had two different +> `ExpectationType`, one in `signatures` and one in `contracts.contract_config`. Two +> classes, one name, one flat root: they could not both keep it. + +--- + +## **The TDR** +| **A moved name that collides with another capability's name in the flat root namespace must be renamed with its capability as prefix, and its legacy shim must name the new name.** + +> The rule generalises the two existing cases. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `pyoaev.signatures.ExpectationType` → `xtm_oaev_sdk.SignatureExpectationType` +- `pyoaev.contracts.contract_config.ExpectationType` → `xtm_oaev_sdk.ContractExpectationType` + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- public sub-namespaces are introduced; +- a collision appears where a capability prefix does not disambiguate. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- No material lesson beyond the TDR itself. + +--- + +## **Resources** +- [Shim strategy, kinds of shims](../shim_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md new file mode 100644 index 00000000..4f141e16 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md @@ -0,0 +1,74 @@ +# Seams crossing a process boundary pin their JSON schemas +--- +scope: [xtm-oaev-sdk, signatures, port_io, testing, versioning] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 21:39:05 +--- + +## ELI5 + +Some of our data gets read by another program, like the platform. Change the shape of +that data by accident and the other program breaks, while our own tests notice nothing. +So we keep a fingerprint of each shape. Change the shape, a test fails, and you have to +choose, on purpose, to bump the version and update the fingerprint. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The inbound signature I/O (injector configs, execution signatures, tool output) and the +> port I/O wire payloads leave the process. Let a schema drift silently and the other side +> breaks while every local test stays green. + +--- + +## **The TDR** +| **A seam whose models cross a process boundary must pin their JSON schemas with golden hashes, regenerated only together with a version bump.** + +> Seams whose models never leave the process carry a `SEAM_VERSION` and no pinned schema. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `test/seam/test_signatures_seam_schema_pin.py` and `test/seam/test_port_io_schema_pin.py` hash each model's `model_json_schema()`, serialized with sorted keys, and hold it against the golden hashes in `test/seam/fixtures/`. +- The same tests also assert the seam version, make sure the seam module stays closed, and (signatures) that the package root re-exports the seam's own objects. +- To change the contract: change the model, bump the version and its expected value in the version test, run `test/seam/generate_*_golden.py`, and commit all of it together, in one go. The step-by-step procedure lives in `CONTRIBUTING.md`, "Pinned seam schemas". + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the schemas are published and checked by the platform side; +- a CI contract tool replaces the in-repository pins; +- dependency upgrades keep moving the hashes while the wire format stays exactly the same. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- These two pins predate the call to keep architecture checks out of the SDK tests. +- The golden files store hashes, not schemas. A failure tells you *that* a schema moved, never *how*: dumping `model_json_schema()` on both commits and diffing is the only way to see it. +- The hashed schema includes the docstrings of models and enums. Reword one and the pin trips, as surely as if you had retyped a field. +- Each pin test and its generator keep their own copy of the model list. Pin a new model and you edit both. + +--- + +## **Resources** +- [Schema pin tests](../../test/seam/) +- [Contributing: pinned seam schemas](../../CONTRIBUTING.md#pinned-seam-schemas-testseam) diff --git a/libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md new file mode 100644 index 00000000..7ee3fe6b --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md @@ -0,0 +1,70 @@ +# Every piece of behaviour goes behind a port, even a one-liner +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, architecture] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Every piece of logic, even a one-line function, follows the same three-part pattern: an +agreement (the port), the code that does the job, and the public entry point people +call. No exception for small things. That way nobody wastes time arguing about which +rule applies. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Give a layering rule size-based exceptions and every new helper turns into a debate +> about whether it deserves a seam. The call was uniformity: "we want unicity everywhere +> and not some exception". + +--- + +## **The TDR** +| **Every piece of behaviour exposed by an SDK goes through a seam Protocol port, one _core implementation and a public adapter, whatever its size.** + +> Data contracts are not behaviour; they live in the seam directly (TDR [seam-holds-data-contracts-and-their-own-invariants](seam-holds-data-contracts-and-their-own-invariants.tdr.md)). + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +`is_truthy` gets `TruthinessPort`, `Truthiness` and the public `is_truthy`: exactly the +treatment `SignatureManager` gets. `copy_dict` goes through `DictToolsPort` / `DictTools`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Lesson | +| --- | --- | +| Reserve ports for real I/O boundaries; leave pure helpers such as `copy_dict` without a seam | Uniformity beats per-case judgement | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the boilerplate measurably slows contributions or reviews; +- a generator or base class can provide the port/adapter pair, changing what "one implementation per port" costs. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- The price is more files for trivial helpers. + +--- + +## **Resources** +- [Seam strategy, behind a port](../seam_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/every-sdk-module-imports-alone.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/every-sdk-module-imports-alone.tdr.md new file mode 100644 index 00000000..9e16227c --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/every-sdk-module-imports-alone.tdr.md @@ -0,0 +1,65 @@ +# Every SDK module imports on its own in a fresh interpreter +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, testing] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Every file of the SDK has to work as the very first thing you import, in a fresh Python, +with warnings turned into errors. That flushes out hidden loops between files, the kind +that only work because the tests happen to import things in a lucky order. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Three layers per capability, plus module-level default wiring: an import cycle can hide +> behind a lucky import order in the test session. + +--- + +## **The TDR** +| **Every SDK module must import successfully as the first import of a fresh interpreter running with -W error.** + + + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +`test_every_sdk_module_imports_first_in_a_fresh_interpreter` (in each SDK's +`test/test_boundaries.py`) runs `python -W error -c "import "` for every module. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- a CI tool checks import cycles statically and the subprocess cost becomes a problem. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- No material lesson beyond the TDR itself. + +--- + +## **Resources** +- [Boundaries test](../../test/test_boundaries.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/import-from-the-package-root.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/import-from-the-package-root.tdr.md new file mode 100644 index 00000000..ee866067 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/import-from-the-package-root.tdr.md @@ -0,0 +1,70 @@ +# Consumers import from the package root only +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, public-api] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Always import from the top of the SDK: `from xtm_oaev_sdk import X`. The folders inside +(`public`, `seam`, `_core`) are the SDK's internal plumbing. What's in there can move at +any time, without asking you. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The layers are internal wiring. Refactors moved names between `public`, `seam` and +> `_core` modules (out of `seam/configuration/__init__.py`, for one) and the package root +> never flinched. + +--- + +## **The TDR** +| **Consumers must import SDK names from the package root (xtm_oaev_sdk, collectors_sdk, injectors_sdk); public, seam and _core module paths are internal and may move.** + +> The package root's `__all__` is the stable API. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +from xtm_oaev_sdk import Configuration, ContractBuilder, SignatureManager +``` + +Every legacy shim message points to the root import: `Use 'from xtm_oaev_sdk import X' instead.` + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the root namespace grows too large to stay flat and sub-namespaces become public; +- a consumer needs a seam type that the root does not export. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- A surface tool that treats every module as public reports internal moves as removals. + +--- + +## **Resources** +- [README](../../README.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/importing-pyoaev-stays-silent.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/importing-pyoaev-stays-silent.tdr.md new file mode 100644 index 00000000..6fcbdce5 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/importing-pyoaev-stays-silent.tdr.md @@ -0,0 +1,71 @@ +# Importing pyoaev never warns; only touching a moved name does +--- +scope: [pyoaev, shims, deprecation] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Writing `import pyoaev` must never show a deprecation warning. A warning only shows up +when you actually use a name that moved to the SDKs. Otherwise it would warn people +about code they never touched, and crash programs that treat warnings as errors. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> A warning on a bare `import pyoaev` hits code that never touched a moved name, and it +> crashes anything running under `-W error`. + +--- + +## **The TDR** +| **Importing any pyoaev module must not emit a DeprecationWarning; only accessing a moved name may warn.** + +> `pyoaev`'s own code imports from the SDKs, never from its shims. And no module +> star-imports a shim module. Ever. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- The package root binds the shim submodules and serves their names through its own `__getattr__`; `pyoaev.__all__` is unchanged. +- `test_importing_pyoaev_modules_is_silent` imports every `pyoaev` module under `-W error`. +- `test_sdk_objects_stay_silent_once_pyoaev_is_loaded` checks that SDK users get no warning. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| Star-import shim modules from `pyoaev/__init__` | `import *` walks `__all__` and trips every lazy name: `import pyoaev` warned, and crashed under `-W error` | Serve package-level names through the root `__getattr__` | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the shims are removed; +- a moved name has to be eagerly bound at import time for a technical reason. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- `from pyoaev import *` still warns for the moved names in `pyoaev.__all__`. That is the signal doing its job, not a leak. + +--- + +## **Resources** +- [Shim contract tests](../../test/common/shim/test_shim_contract.py) +- [collectors/injectors shim tests](../../../../test/test_sdk_split_shims.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/legacy-names-served-by-pep562-module-getattr.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/legacy-names-served-by-pep562-module-getattr.tdr.md new file mode 100644 index 00000000..2f4135db --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/legacy-names-served-by-pep562-module-getattr.tdr.md @@ -0,0 +1,82 @@ +# Legacy names are served by a directly assigned PEP 562 __getattr__ +--- +scope: [pyoaev, deprecation, shims] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +When old code asks pyoaev for a name that moved, a small lookup function in the old +module catches the request. It shows a warning pointing to the new import, then hands +back the real SDK object. It's plugged in directly, so the warning points at your line +of code, not at pyoaev's. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Legacy access has three jobs: warn, so people migrate; return the SDK object, so nothing +> else moves; and leave SDK users alone. + +--- + +## **The TDR** +| **A moved name must be served from its legacy module by a module-level __getattr__ built with make_deprecated_getattr and assigned directly, never wrapped in a def.** + +> The returned `__getattr__` warns with `DeprecationWarning` and `stacklevel=2`, which pins +> the warning on the consumer's line. Unknown names still raise `AttributeError`. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +__getattr__ = _make_deprecated_getattr( + { + name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) + for name in __all__ + }, + module_name=__name__, +) +``` + +`__all__` lists the lazily served names, flagged with `# noqa: F822`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| `typing_extensions.deprecated(...)(cls)` | Patches the original class in place: the moment anything imported `pyoaev`, `xtm_oaev_sdk.Configuration()` warned too | Never mutate the SDK object to deprecate its legacy name | +| `def __getattr__(name): return _deprecated_getattr(name)` | The extra frame pins the warning on the shim file, and the default filters hide it from scripts | Assign the built `__getattr__` directly | +| Wrapper class or proxy | Breaks identity: `isinstance`, metaclasses, signature introspection | See TDR [shims-return-the-identical-sdk-object](shims-return-the-identical-sdk-object.tdr.md) | +| Plain re-export | Keeps identity, never warns. Nobody ever migrates | The warning is the migration signal | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the shims are removed. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- `from import X` on a shim package warns twice (importlib's `_handle_fromlist` probes with `hasattr`); plain modules warn once. Accepted: that one is baked into PEP 562. + +--- + +## **Resources** +- [Shim strategy, mechanism](../shim_strategy.md) +- [Deprecation adapter](../../xtm_oaev_sdk/public/deprecation.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/legacy-patch-targets-are-part-of-the-surface.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/legacy-patch-targets-are-part-of-the-surface.tdr.md new file mode 100644 index 00000000..c1dd05e8 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/legacy-patch-targets-are-part-of-the-surface.tdr.md @@ -0,0 +1,70 @@ +# Legacy patch targets are part of the compatibility surface +--- +scope: [pyoaev, shims, testing, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +The connectors' tests swap some pyoaev pieces for fakes: they "patch" them. Those pieces +have to stay where the tests expect them, and swapping them has to keep affecting the +code that really runs. Otherwise those tests break, with no import error to tell you +why. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Connector test suites patch `pyoaev` module attributes. Before a single line moved, +> every `pyoaev` patch target used by the connectors' tests was listed. Move the code +> without keeping them and those suites break, without a single import error. + +--- + +## **The TDR** +| **Module attributes that connectors patch must stay reachable at their legacy path and still reach the code that actually runs.** + + + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `pyoaev.signatures.signature_manager` still imports `socket` and `subprocess`: patching `...signature_manager.socket.gethostbyname` reaches the shared stdlib module the SDK uses. +- The client-building adapters call `OpenAEV` through their module global, so `patch("pyoaev.daemons.base_daemon.OpenAEV")` and `patch("pyoaev.helpers.OpenAEV")` apply. +- `pyoaev.utils.PingAlive.start` and `pyoaev.daemons.collector_daemon.CollectorDaemon._setup` land on the class the connectors instantiate (TDR [shims-return-the-identical-sdk-object](shims-return-the-identical-sdk-object.tdr.md)). + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored: the patch targets counted as surface from day one. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the connectors stop patching `pyoaev` paths; +- the shims are removed. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- A patch target can be a stdlib module re-bound in a legacy module. Keep the import line and it keeps working; that is all it takes. + +--- + +## **Resources** +- [collectors/injectors shim tests](../../../../test/test_sdk_split_shims.py) +- [Shim strategy, rules learned the hard way](../shim_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/legacy-surface-guarded-from-the-release.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/legacy-surface-guarded-from-the-release.tdr.md new file mode 100644 index 00000000..5b2dc520 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/legacy-surface-guarded-from-the-release.tdr.md @@ -0,0 +1,68 @@ +# The legacy surface is guarded against the released pyoaev +--- +scope: [pyoaev, testing, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +We keep a list of every public name in the pyoaev people actually installed, and a test +checks they all still work. That list comes from the real release, not from our work in +progress. Otherwise it would happily accept a name we lost by mistake. The old tests of +the moved code stay exactly as they were. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Generate the guard from the working tree and it blesses whatever the split produced, +> lost names included. The reference has to be what users actually installed. + +--- + +## **The TDR** +| **The legacy-surface guard must be generated from the released pyoaev package, and the pre-split tests of moved code must be kept verbatim against their legacy paths.** + +> New behaviour tests target the SDKs. The moved `pyoaev` tests live under +> `libs/xtm-oaev-sdk/test//shim/`, still hammering the legacy imports. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `test/generate_legacy_surface.py` records `test/fixtures/legacy_surface.json` from `pyoaev`. +- `test/test_legacy_surface.py` checks every one still resolves. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the shims are removed. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- Moved verbatim, the old tests become a second guard: same assertions, still driving the legacy paths. + +--- + +## **Resources** +- [Legacy surface guard](../../../../test/test_legacy_surface.py) +- [Legacy surface generator](../../../../test/generate_legacy_surface.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/no-breaking-change-for-legacy-pyoaev-code.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/no-breaking-change-for-legacy-pyoaev-code.tdr.md new file mode 100644 index 00000000..7aa51332 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/no-breaking-change-for-legacy-pyoaev-code.tdr.md @@ -0,0 +1,79 @@ +# Legacy pyoaev code keeps working until a pyoaev major release +--- +scope: [pyoaev, xtm-oaev-sdk, collectors-sdk, injectors-sdk, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +pyoaev used to hold everything, and its shared code moved into the SDKs. Old code that +imports from pyoaev must keep working, with nothing more than a warning telling where +each thing went. That holds until a future major pyoaev release removes the old paths. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> `pyoaev` was a monolith: the OpenAEV API client, with configuration, contracts, +> signatures, the daemon runtime, errors and helpers piled on top. That shared code now +> lives in `xtm-oaev-sdk`, `collectors-sdk` and `injectors-sdk`. +> +> The collectors and injectors in the connector repositories import the former `pyoaev` +> paths, and their test suites patch them. The split had to land without dragging every +> connector into a migration on the same day: "no breaking change for old code". + +--- + +## **The TDR** +| **Every public name of the latest released pyoaev must keep resolving from its legacy path, with a DeprecationWarning naming its new import, until a pyoaev major release removes the shims.** + +> That covers names, the behaviour of legacy constructors (see TDR [shims-return-the-identical-sdk-object](shims-return-the-identical-sdk-object.tdr.md)) and the patch +> targets connectors lean on (see TDR [legacy-patch-targets-are-part-of-the-surface](legacy-patch-targets-are-part-of-the-surface.tdr.md)). What moves into the SDKs is the released +> `pyoaev` behaviour. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `from pyoaev.configuration import Configuration` warns and returns `xtm_oaev_sdk.Configuration`. +- `test/test_legacy_surface.py` checks that the names in modules of `pyoaev` still resolve (TDR [legacy-surface-guarded-from-the-release](legacy-surface-guarded-from-the-release.tdr.md)). +- The real collectors|injectors own suites ran against the released `pyoaev`, then against the new stack. Identical results. +- The removal steps for the next major release are already written down in `docs/shim_strategy.md`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- a `pyoaev` major release is planned: the shims are removed and this TDR is revoked or superseded; +- every collector|injector imports from the SDKs and nobody consumes the legacy paths anymore; +- a deprecation policy fixes a time- or release-bound window instead of "until the next major". + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- The legacy surface runs wider than `__all__`: collectors also use names a module merely imported (`pyoaev.daemons.base_daemon.Configuration`), and patch targets. +- A few observable differences are not API breaks. They were accepted, then documented: warnings on legacy access, SDK module names for unnamed loggers, thin subclasses for the client-building adapters. + +--- + +## **Resources** +- [Shim strategy](../shim_strategy.md) +- [Legacy surface guard](../../../../test/test_legacy_surface.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/pass-bound-callables-to-keep-dispatch.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/pass-bound-callables-to-keep-dispatch.tdr.md new file mode 100644 index 00000000..87767310 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/pass-bound-callables-to-keep-dispatch.tdr.md @@ -0,0 +1,77 @@ +# Adapters pass their own methods to the core to keep dispatch and patch points +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Some SDK classes used to call their own methods, and that logic moved to the core. So +now the class hands its own method to the core, instead of letting the core redo the +call. Override that method or fake it, and it still counts. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Logic moves to the core. If the core then reimplemented the calls that used to go +> through `self`, subclass overrides and test patches would be bypassed without a sound. + +--- + +## **The TDR** +| **When the original code dispatched through its own object, the adapter must pass the bound method or a callable to the core instead of letting the core reimplement the call.** + +> Same deal for a dependency the core may not import: the adapter passes it down. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +def _source_context(self, config): + return _engine.source_context( + config, self.resolve_container_ip, lambda: self._cached_ipv6 + ) +``` + +- `SignatureManager.post_execution_updates` passes `self._utcnow`. +- `Expectation.match_alert` passes `self.match_fuzzy` / `self.match_simple`. +- `OpenAEVDetectionHelper` passes `self.match_alert_element_fuzzy` / `self._decode_value`. +- `CollectorDaemon` passes `self._try_callback` to the scheduler. +- `Configuration.schema()` passes the public `ConnectorConfigSchemaGenerator` to the core resolver. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the hooks are formally deprecated and no longer need to stay overridable; +- the core gets an explicit strategy object replacing the callables. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- Call order is part of the behaviour: `_utcnow` is still read only after the tool output validates. + +--- + +## **Resources** +- [Seam strategy, pass callables](../seam_strategy.md) +- [Signatures adapter](../../xtm_oaev_sdk/public/signatures.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/public-class-extends-seam-data.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/public-class-extends-seam-data.tdr.md new file mode 100644 index 00000000..0c10066b --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/public-class-extends-seam-data.tdr.md @@ -0,0 +1,79 @@ +# Data-and-behaviour classes split: the public class extends the seam data +--- +scope: [xtm-oaev-sdk, collectors-sdk, architecture] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Some classes, like `Contract`, hold both data and actions. We cut them in two: the plain +data lives in the seam (the agreement layer), and the public class adds the actions on +top. You still create `Contract(...)` exactly as before. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> `Contract` and `Expectation` are what users construct, and both mix data with behaviour +> (default variables, serialization, matching, updates). The seam cannot depend on +> behaviour. Users must keep constructing the same class. + +--- + +## **The TDR** +| **When a user-facing class mixes data and behaviour, the seam holds the data class and the public class extends it, adding behaviour through ports; the core only knows the seam type.** + + + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +@dataclass +class Contract(ContractData): + variables: list[ContractVariable] = field( + default_factory=lambda: _variables.default_variables() + ) + + def to_contract_add_input(self, source_id: str) -> dict[str, Any]: + return _serializer.to_add_input(self, source_id) +``` + +collectors-sdk plays the same move with `ExpectationData` / `Expectation`. After the +contracts split, the JSON of `to_contract_add_input()`, `to_contract_update_input()` and +`prepare_contracts()` came out byte-identical to the previous commit. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the data and behaviour can be separated in the user-facing API itself (a major version); +- the core needs behaviour from the public subclass. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: + +--- + +## **Resources** +- [Seam strategy, patterns](../seam_strategy.md) +- [Contracts adapter](../../xtm_oaev_sdk/public/contracts.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/public-seam-core-dependency-rule.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/public-seam-core-dependency-rule.tdr.md new file mode 100644 index 00000000..f3e735b3 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/public-seam-core-dependency-rule.tdr.md @@ -0,0 +1,73 @@ +# public and _core depend on the seam; the seam depends on neither +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, architecture] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Each part of an SDK has three layers: `public` (what users touch), `_core` (the engine) +and the seam (the agreement between the two). Both sides lean on the agreement, never on +each other. The one exception: `public` chooses which engine to plug in. So the engine +can be rebuilt freely, as long as it keeps the agreement. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The SDKs were designed in three layers: `public` is what users get, `_core` is the code +> behind it, and the seam is the contract between the two, so that "as long as `_core` +> answers it, `public` stays usable with no breaking change". + +--- + +## **The TDR** +| **Within each SDK capability, the seam imports neither public nor _core, _core never imports public, and public imports _core only to wire the default implementation of each port.** + +> Public never hands out a core object. Every exported name is either an adapter defined +> in `public/` or a contract defined in `seam/`. The seam may import the standard library, +> third-party libraries, other packages' public API and sibling seam modules, and that is +> the whole list. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +_truthiness: TruthinessPort = Truthiness() +_sources: ConfigurationSourcesPort = ConfigurationSources() +_resolver: ConfigurationResolverPort = ConfigurationResolver(_sources, _truthiness) +``` + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- users need to swap implementations at runtime beyond the module-level defaults; + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: + +--- + +## **Resources** +- [Seam strategy, rules](../seam_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md new file mode 100644 index 00000000..a47af668 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md @@ -0,0 +1,72 @@ +# pyoaev hard-depends on the SDKs that host its former code +--- +scope: [pyoaev, packaging, dependencies, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +Old connectors only install pyoaev, but what they use lives in the SDKs now. So +installing pyoaev always installs the SDKs too, not only when you ask for an option. +That's what keeps old connectors running while the old paths still exist. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> A connector that lists only `pyoaev` still imports `CollectorDaemon`, +> `OpenAEVInjectorHelper` and friends from their legacy paths. Those names live in the +> SDKs now. For TDR [no-breaking-change-for-legacy-pyoaev-code](no-breaking-change-for-legacy-pyoaev-code.tdr.md) to hold, `pyoaev` has to carry them in. + +--- + +## **The TDR** +| **While pyoaev ships deprecation shims, it must declare a hard dependency on every SDK those shims serve, never an optional extra.** + +> Third-party packages `pyoaev` no longer imports itself (`pika`, `thefuzz`, `PyYAML`, +> `pydantic-settings`, `python-json-logger`) are gone from its own declaration. They still +> ride in transitively through the SDKs, so an install pulls the same packages as before. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `pyproject.toml` requires `xtm-oaev-sdk`, `collectors-sdk` and `injectors-sdk` (`>=0.1.0,<0.2.0`). +- Until the SDKs are published, CI pulls collectors-sdk and injectors-sdk straight from git through `libs/sdk-requirements.txt`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| Optional extras (`pyoaev[collectors]`, `pyoaev[injectors]`) with lazily loaded SDKs | Proposed to spare every `pyoaev` install both SDKs; legacy imports would then blow up for connectors listing only `pyoaev` | Rejected: no breaking change wins | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the shims are removed: `collectors-sdk` and `injectors-sdk` can be dropped (keep `xtm-oaev-sdk` if the client still uses its errors and transport models); +- an SDK gains heavy dependencies that `pyoaev` users should not pay for. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- The install footprint did not grow. What moved out of `pyoaev` comes straight back in through the SDKs. + +--- + +## **Resources** +- [pyproject.toml](../../../../pyproject.toml) +- [SDK requirements for CI](../../../../libs/sdk-requirements.txt) diff --git a/libs/xtm-oaev-sdk/docs/tdr/sdks-never-import-pyoaev.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/sdks-never-import-pyoaev.tdr.md new file mode 100644 index 00000000..64137eeb --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/sdks-never-import-pyoaev.tdr.md @@ -0,0 +1,79 @@ +# The SDKs never depend on pyoaev; the API client is injected +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, dependencies] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +The SDKs never use pyoaev. When they need to talk to the OpenAEV platform, you hand them +the client (`api_client=` or `transport=`). pyoaev already depends on the SDKs: the +other way around would make a loop, and chain the SDKs to one specific client. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> `pyoaev` depends on the SDKs (TDR [pyoaev-hard-depends-on-the-sdks-during-deprecation](pyoaev-hard-depends-on-the-sdks-during-deprecation.tdr.md)) and is meant to end up as the OpenAEV HTTP +> client, and only that. The SDKs carry the shared, client-free runtime. One import back +> into `pyoaev` and you get a circular package dependency, with that runtime welded to a +> single client. + +--- + +## **The TDR** +| **xtm-oaev-sdk, collectors-sdk and injectors-sdk must never import pyoaev; anything that talks to the platform receives its API client or transport as an argument typed by a seam protocol.** + +> The injected types are `BaseClient` (seam `common`) and `SignatureTransportProtocol` +> (seam `signatures`). `pyoaev.OpenAEV` and its signature API satisfy them. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +```python +SignatureManager(transport=api.signature) +CollectorDaemon(configuration=configuration, callback=collect, api_client=api, collector_type="...") +OpenAEVInjectorHelper(config, icon, api_client=api) +``` + +- No injected client? `BaseDaemon` and `OpenAEVInjectorHelper` fall back on the overridable `_create_default_api_client` hook, and the SDK default raises rather than build one. +- `pyoaev` brings the legacy self-building constructors back by overriding that hook (TDR [shims-return-the-identical-sdk-object](shims-return-the-identical-sdk-object.tdr.md)). +- `test/test_boundaries.py` in each SDK fails the moment a source imports `pyoaev` (or `requests`). + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| Keep the legacy constructors that built `OpenAEV(...)` themselves | That drags `pyoaev` into the SDK | Client construction lives in `pyoaev`'s client-building adapters, and only there | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the API client itself moves into an SDK; +- an SDK needs a client capability that cannot be expressed as a protocol without importing `pyoaev` types. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- Pulling client construction out took a hook (`_create_default_api_client`): that is how `pyoaev` restores the old constructors without the SDK ever knowing. + +--- + +## **Resources** +- [xtm-oaev-sdk boundaries test](../../test/test_boundaries.py) +- [Shim strategy, dependency direction](../shim_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md new file mode 100644 index 00000000..29c37ee5 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md @@ -0,0 +1,70 @@ +# A seam change is a versioned contract change +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, versioning] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +The seam is the agreement between the public part and the engine. Changing the engine is +free. Changing the agreement changes things for users, so it gets a new version number +and a changelog line. And if users can see the change, the old way keeps working, with a +warning, first. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The whole point of the seam: the public API survives any core rewrite. Touch the seam +> itself and you have changed the contract, and unlike a core change, that has to be +> announced. + +--- + +## **The TDR** +| **Any change to a seam contract (field, type, enum member, error, port signature) must bump that seam's version and be recorded in the changelog, and a change reaching the public surface must go through a deprecation shim first.** + +> Every capability seam exposes `SEAM_VERSION`; `port_io` is versioned by +> `PORT_IO_CONTRACT_VERSION`. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- every capability seam of the three SDKs defines `SEAM_VERSION`. +- `seam/port_io/models.py` defines `PORT_IO_CONTRACT_VERSION`. +- Deprecations use `make_deprecated_getattr` (TDR [legacy-names-served-by-pep562-module-getattr](legacy-names-served-by-pep562-module-getattr.tdr.md)). + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- seam versions get derived automatically by tooling; +- the release process ties seam versions to package versions. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- No changelog file exists in the SDKs yet. The first seam change gets to create it. + +--- + +## **Resources** +- [Seam strategy, versioning](../seam_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md new file mode 100644 index 00000000..4bc69753 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md @@ -0,0 +1,69 @@ +# The seam holds data contracts and only their intrinsic invariants +--- +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, architecture] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +The seam only holds data: models, lists of allowed values, errors, and small rules that +check or compute something from that data alone. Anything that talks to the outside +world, runs the show, or needs another part of the SDK goes behind a port instead. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> The seam must stay closed (TDR [public-seam-core-dependency-rule](public-seam-core-dependency-rule.tdr.md)), yet the data crossing it carries rules of its +> own: validators, derived fields, errors that extract their own message. Those rules are +> part of what the data is. I/O and orchestration are not. + +--- + +## **The TDR** +| **A seam may contain data contracts and logic that only reads or derives from their own data; anything doing I/O, orchestrating, or needing another capability goes behind a port.** + +> Data contracts: pydantic models, dataclasses, enums, exceptions, type aliases, constants. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- validators and derived fields: `ConfigurationHint`, `ExpectationData`'s null signatures coerced to `[]`; +- `__post_init__` invariants of the contract elements; +- value objects: `EncodedId`, `RequiredOptional.validate_attrs`, `SignatureMatch`, `SignatureType.make_struct_for_matching`; +- exceptions, including `OpenAEVError`'s message extraction: the core raises them and users catch them, so both sides must reach them. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- an invariant needs external state or another capability: it must move behind a port; +- the seams are generated from an external schema that cannot carry logic. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- Exceptions belong in the seam: both sides of the boundary need them. + +--- + +## **Resources** +- [Seam strategy, what goes where](../seam_strategy.md) diff --git a/libs/xtm-oaev-sdk/docs/tdr/shims-return-the-identical-sdk-object.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/shims-return-the-identical-sdk-object.tdr.md new file mode 100644 index 00000000..d844f7ce --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/shims-return-the-identical-sdk-object.tdr.md @@ -0,0 +1,80 @@ +# Shims return the identical SDK object; adapters only where constructors differ +--- +scope: [pyoaev, shims, compatibility] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +When old code asks pyoaev for a moved name, it gets the very same object the SDK +provides. Not a copy, not a look-alike, so checks like `isinstance` and `is` keep +working. The only exceptions are the classes that used to build the API client +themselves: they come back as a thin subclass that still does it. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Legacy code compares with `is`, checks `isinstance`, subclasses, pickles by reference +> and patches. Hand it anything but the SDK object itself and one of those can snap. Some +> legacy constructors, though, built the `OpenAEV` client themselves, and the SDKs no +> longer do that (TDR [sdks-never-import-pyoaev](sdks-never-import-pyoaev.tdr.md)). + +--- + +## **The TDR** +| **A shim must return the SDK object itself; a legacy name may resolve to a thin subclass only to restore legacy constructor behaviour, and that subclass must be one single object whatever legacy path reaches it.** + +> A legacy value with no public replacement (`variable_helper.USER`…, `helpers.TRUTHY` / +> `FALSY`) is served as the literal value, with a message pointing at the right API. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +| Legacy name | Subclass of | Adds | +|---|---|---| +| `pyoaev.daemons.BaseDaemon` | `xtm_oaev_sdk.BaseDaemon` | `_create_default_api_client()` builds `OpenAEV` | +| `pyoaev.daemons.CollectorDaemon` | `collectors_sdk.CollectorDaemon` + the legacy `BaseDaemon` | same hook | +| `pyoaev.signatures.SignatureManager` | `xtm_oaev_sdk.SignatureManager` | `SignatureManager(client, logger)` sends through `client.signature` | +| `pyoaev.helpers.OpenAEVInjectorHelper` | `injectors_sdk.OpenAEVInjectorHelper` | `_create_default_api_client(config)` builds `OpenAEV` | + +`pyoaev.daemons.CollectorDaemon is pyoaev.daemons.collector_daemon.CollectorDaemon is pyoaev.helpers.CollectorDaemon`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +| Approach | Result | Lesson | +| --- | --- | --- | +| Wrapper class or proxy for every legacy name | Breaks identity, `isinstance` against the SDK class, metaclasses and introspection | Return the object; subclass only where behaviour must be restored | + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the SDK classes gain a client factory that makes the adapters unnecessary; +- the shims are removed. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- The legacy `CollectorDaemon` has to inherit from the legacy `BaseDaemon` adapter too. Skip that and `isinstance(daemon, pyoaev.daemons.BaseDaemon)` fails. + +--- + +## **Resources** +- [Shim strategy, client-building adapters](../shim_strategy.md) +- [Shim contract tests](../../test/daemons/shim/test_shim_contract.py) +- [collectors/injectors shim tests](../../../../test/test_sdk_split_shims.py) diff --git a/libs/xtm-oaev-sdk/docs/tdr/xtm-oaev-sdk-stays-connector-agnostic.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/xtm-oaev-sdk-stays-connector-agnostic.tdr.md new file mode 100644 index 00000000..3429f778 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/xtm-oaev-sdk-stays-connector-agnostic.tdr.md @@ -0,0 +1,67 @@ +# xtm-oaev-sdk stays agnostic of collectors, injectors and transports +--- +scope: [xtm-oaev-sdk, dependencies, architecture] +status: draft +superseded_by: +supersedes: +Aliases: +Creation Date: 2026/10/14 16:25:31 +Modification Date: 2026/10/14 16:25:31 +--- + +## ELI5 + +xtm-oaev-sdk is the shared base of both collectors and injectors. Nothing in it may +serve only one of them, and it stays away from libraries that belong elsewhere: RabbitMQ +and fuzzy matching go with injectors-sdk and collectors-sdk, HTTP requests with pyoaev. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> xtm-oaev-sdk is "the common sdk" under collectors-sdk and injectors-sdk. Put collector- +> or injector-specific code in it and you force it on the other connector type. + +--- + +## **The TDR** +| **xtm-oaev-sdk must not contain collector- or injector-specific runtime, nor import pika, thefuzz or requests; those belong to collectors-sdk, injectors-sdk or pyoaev.** + + + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +`test/test_boundaries.py` fails if an SDK source imports `pyoaev`, `requests`, `pika` or +`thefuzz`, or defines `ListenQueue`, `OpenAEVConfigHelper`, `OpenAEVCollectorHelper`, +`OpenAEVInjectorHelper`, `OpenAEVDetectionHelper` or `create_mq_ssl_context`. + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +No alternative was explored for this decision. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- collectors and injectors converge on a shared transport; +- a third connector type needs part of a type-specific runtime. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- No material lesson beyond the TDR itself. + +--- + +## **Resources** +- [Boundaries test](../../test/test_boundaries.py) diff --git a/libs/xtm-oaev-sdk/pyproject.toml b/libs/xtm-oaev-sdk/pyproject.toml index 99f745f2..1731914c 100644 --- a/libs/xtm-oaev-sdk/pyproject.toml +++ b/libs/xtm-oaev-sdk/pyproject.toml @@ -6,7 +6,7 @@ authors = [ {name = "Filigran", email = "contact@filigran.io"} ] license = { text = "Apache-2.0" } -requires-python = ">=3.11" +requires-python = ">=3.12" classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", diff --git a/pyproject.toml b/pyproject.toml index deb46a72..e8004912 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ maintainers = [ ] license = { text = "Apache-2.0" } readme = "README.md" -requires-python = ">=3.11" +requires-python = ">=3.12" classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From 4125af891675739c6b60601565a80318f0b53594 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 22:24:52 +0200 Subject: [PATCH 16/22] :memo: docs(sdk): documenting the credential references migrated from pyoaev.credential --- libs/xtm-oaev-sdk/CHANGELOG.md | 6 ++++-- libs/xtm-oaev-sdk/README.md | 2 +- libs/xtm-oaev-sdk/docs/migration_guide.md | 3 +++ libs/xtm-oaev-sdk/docs/seam_strategy.md | 2 +- libs/xtm-oaev-sdk/docs/shim_strategy.md | 1 + test/fixtures/legacy_surface.json | 16 +++++++++++++++- 6 files changed, 25 insertions(+), 5 deletions(-) diff --git a/libs/xtm-oaev-sdk/CHANGELOG.md b/libs/xtm-oaev-sdk/CHANGELOG.md index 0adb3f01..447409f8 100644 --- a/libs/xtm-oaev-sdk/CHANGELOG.md +++ b/libs/xtm-oaev-sdk/CHANGELOG.md @@ -6,7 +6,7 @@ version of its seam ## [0.1.0] - Unreleased -First version. The shared, client-free code of `pyoaev` 3.260923.0 moves out and into +First version. The shared, client-free code of `pyoaev` 3.261014.1 moves out and into this SDK. The former `pyoaev` paths keep working, each one firing a `DeprecationWarning` ([migration guide](docs/migration_guide.md)). @@ -18,7 +18,9 @@ this SDK. The former `pyoaev` paths keep working, each one firing a - `configuration`: `Configuration`, `ConfigurationHint`, `SettingsLoader`, the settings models, sources and schema generator. Seam `1.0.0`. - `contracts`: `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, - the contract elements and enums. Seam `1.0.0`. + the contract elements and enums, and the credential references (`CredentialType`, + `ContractReferencedCredential`, `build_single_referenced_credential_element`, from + `pyoaev.credential`). Seam `1.0.0`. - `signatures`: `SignatureManager`, `build_network_configs`, `build_marker`, the matching types and the inbound I/O models. Seam `1.0.0`, JSON schemas pinned. - `port_io`: the wire models sent to the platform. `PORT_IO_CONTRACT_VERSION` `1.0.0`, diff --git a/libs/xtm-oaev-sdk/README.md b/libs/xtm-oaev-sdk/README.md index 5b963e86..1f1dce73 100644 --- a/libs/xtm-oaev-sdk/README.md +++ b/libs/xtm-oaev-sdk/README.md @@ -25,7 +25,7 @@ implementation of it. |---|---| | common | `OpenAEVError` and its subclasses, `on_http_error`, `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, `EncodedId`, `RequiredOptional`, `copy_dict`, `remove_none_from_dict`, SSL helpers, asset and security-domain enums, `BaseClient` / `DaemonProtocol` / `AppLoggerProtocol` | | configuration | `Configuration`, `ConfigurationHint`, `SettingsLoader`, `BaseConfigModel`, `ConfigLoaderOAEV`, `ConfigLoaderCollector`, `ConnectorConfigSchemaGenerator`, sources, serializer types | -| contracts | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, every contract element (`ContractText`, `ContractSelect`, …) and enum | +| contracts | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, every contract element (`ContractText`, `ContractSelect`, …) and enum, credential references (`CredentialType`, `ContractReferencedCredential`, `build_single_referenced_credential_element`) | | signatures | `SignatureManager`, `build_network_configs`, `build_marker`, matching types, the inbound I/O models and the port I/O wire models (`PORT_IO_CONTRACT_VERSION`) | | daemons | `BaseDaemon`, `PingAlive` | | deprecation | `make_deprecated_getattr` (PEP 562 deprecation primitive) | diff --git a/libs/xtm-oaev-sdk/docs/migration_guide.md b/libs/xtm-oaev-sdk/docs/migration_guide.md index 8a6616ac..46d4310d 100644 --- a/libs/xtm-oaev-sdk/docs/migration_guide.md +++ b/libs/xtm-oaev-sdk/docs/migration_guide.md @@ -70,6 +70,7 @@ every legacy module that exposed it. | `BaseDaemon` | `pyoaev.daemons`, `pyoaev.daemons.base_daemon`, `pyoaev.daemons.collector_daemon` | `from xtm_oaev_sdk import BaseDaemon` | | `build_marker` | `pyoaev.signatures.ai_marker` | `from xtm_oaev_sdk import build_marker` | | `build_network_configs` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models` | `from xtm_oaev_sdk import build_network_configs` | +| `build_single_referenced_credential_element` | `pyoaev.credential`, `pyoaev.credential.utils` | `from xtm_oaev_sdk import build_single_referenced_credential_element` | | `CloudInjectorConfig` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import CloudInjectorConfig` | | `CloudProvider` | `pyoaev`, `pyoaev.asset_types` | `from xtm_oaev_sdk import CloudProvider` | | `ConfigLoaderCollector` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import ConfigLoaderCollector` | @@ -95,6 +96,7 @@ every legacy module that exposed it. | `ContractOutputElement` | `pyoaev.contracts.contract_builder`, `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractOutputElement` | | `ContractOutputType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractOutputType` | | `ContractPayload` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractPayload` | +| `ContractReferencedCredential` | `pyoaev.contracts.contract_config`, `pyoaev.credential.utils` | `from xtm_oaev_sdk import ContractReferencedCredential` | | `ContractSelect` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractSelect` | | `ContractTeam` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractTeam` | | `ContractText` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractText` | @@ -102,6 +104,7 @@ every legacy module that exposed it. | `ContractTuple` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractTuple` | | `ContractVariable` | `pyoaev.contracts.contract_config`, `pyoaev.contracts.contract_utils`, `pyoaev.contracts.variable_helper` | `from xtm_oaev_sdk import ContractVariable` | | `copy_dict` | `pyoaev.utils` | `from xtm_oaev_sdk import copy_dict` | +| `CredentialType` | `pyoaev.contracts.contract_config`, `pyoaev.credential`, `pyoaev.credential.types`, `pyoaev.credential.utils` | `from xtm_oaev_sdk import CredentialType` | | `CustomJsonFormatter` | `pyoaev.utils` | `from xtm_oaev_sdk import CustomJsonFormatter` | | `data_to_temp_file` | `pyoaev.helpers` | `from xtm_oaev_sdk import data_to_temp_file` | | `DictionarySource` | `pyoaev.configuration.configuration`, `pyoaev.configuration.sources` | `from xtm_oaev_sdk import DictionarySource` | diff --git a/libs/xtm-oaev-sdk/docs/seam_strategy.md b/libs/xtm-oaev-sdk/docs/seam_strategy.md index 61583fef..2a433484 100644 --- a/libs/xtm-oaev-sdk/docs/seam_strategy.md +++ b/libs/xtm-oaev-sdk/docs/seam_strategy.md @@ -202,7 +202,7 @@ source selection to `SettingsSourcesPort`. The settings models (`BaseConfigModel |---|---|---|---|---| | common | the error hierarchy, the asset and security-domain enums, `EncodedId`, `RequiredOptional`, `BaseClient`, `DaemonProtocol`, `AppLoggerProtocol` | `HttpErrorPort`, `JsonEncodingPort`, `LoggingPort`, `DictToolsPort`, `CertificatesPort` | `HttpErrorTranslator`, `JsonEncoding`, `LoggingSetup`, `DictTools`, `Certificates` | `on_http_error`, `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, `copy_dict`, `remove_none_from_dict`, the SSL helpers | | configuration | `ConfigurationHint`, settings models, serializer types, `CONFIGURATION_TYPES`, `ConfigurationProtocol` | `ConfigurationResolverPort`, `ConfigurationSourcesPort`, `ConnectorSchemaPort`, `SettingsSourcesPort`, `TruthinessPort` | `ConfigurationResolver`, `ConfigurationSources`, `ConnectorSchemaTransformer`, `SettingsSourceSelector`, `Truthiness` | `Configuration`, `EnvironmentSource`, `DictionarySource`, `ConnectorConfigSchemaGenerator`, `SettingsLoader`, `is_truthy`, `is_falsy` | -| contracts | `ContractData`, contract elements and enums, `ContractVariable`, `Expectation`, `ContractBuilderProtocol` | `ContractAssemblyPort`, `ContractVariablesPort`, `ContractSerializerPort` | `ContractAssembly`, `BuiltinContractVariables`, `ContractSerializer` | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts` | +| contracts | `ContractData`, contract elements and enums, `ContractVariable`, `Expectation`, `CredentialType`, `ContractBuilderProtocol` | `ContractAssemblyPort`, `ContractVariablesPort`, `ContractSerializerPort`, `ContractCredentialsPort` | `ContractAssembly`, `BuiltinContractVariables`, `ContractSerializer`, `CredentialReferences` | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, `build_single_referenced_credential_element` | | signatures | inbound I/O models, matching types and policies, `ExtraSignatureData`, `SignatureTransportProtocol`, `SignatureManagerProtocol` | `SignatureEnginePort`, `ContainerAddressPort`, `NetworkTargetsPort`, `MarkerPort` | `SignatureEngine`, `ContainerAddressResolver`, `NetworkTargets`, `MarkerBuilder` | `SignatureManager`, `build_network_configs`, `build_marker` | | port_io | wire models and enums, `PORT_IO_CONTRACT_VERSION` | (none: pure wire schema) | | re-exported as is | | daemons | (consumer protocols live in common) | `DaemonRuntimePort`, `HeartbeatPort` | `DaemonRuntime`, `Heartbeat` | `BaseDaemon`, `PingAlive` | diff --git a/libs/xtm-oaev-sdk/docs/shim_strategy.md b/libs/xtm-oaev-sdk/docs/shim_strategy.md index 93651e89..57f6d65e 100644 --- a/libs/xtm-oaev-sdk/docs/shim_strategy.md +++ b/libs/xtm-oaev-sdk/docs/shim_strategy.md @@ -149,6 +149,7 @@ so patching a method on any of those paths hits the class connectors instantiate | `pyoaev/__init__.py` | the moved names it exported (asset enums, exceptions, configuration, contracts, signatures) | | `pyoaev/configuration/*` | xtm-oaev-sdk configuration | | `pyoaev/contracts/*` | xtm-oaev-sdk contracts (`ExpectationType` renamed, variable constants) | +| `pyoaev/credential/*` | xtm-oaev-sdk contracts (credential references) | | `pyoaev/signatures/*` | xtm-oaev-sdk signatures (`ExpectationType` renamed, `SignatureManager` adapter) | | `pyoaev/exceptions.py`, `asset_types.py`, `security_domain/types.py` | xtm-oaev-sdk common | | `pyoaev/utils.py` | xtm-oaev-sdk common and `PingAlive`; keeps the HTTP helpers `get_content_type`, `response_content` | diff --git a/test/fixtures/legacy_surface.json b/test/fixtures/legacy_surface.json index 44b4bbfa..3a0a26dd 100644 --- a/test/fixtures/legacy_surface.json +++ b/test/fixtures/legacy_surface.json @@ -444,12 +444,14 @@ "ContractOutputElement", "ContractOutputType", "ContractPayload", + "ContractReferencedCredential", "ContractSelect", "ContractTeam", "ContractText", "ContractTextArea", "ContractTuple", "ContractVariable", + "CredentialType", "Domain", "Expectation", "ExpectationType", @@ -479,6 +481,18 @@ "VariableHelper", "VariableType" ], + "pyoaev.credential": [ + "CredentialType", + "build_single_referenced_credential_element" + ], + "pyoaev.credential.types": [ + "CredentialType" + ], + "pyoaev.credential.utils": [ + "ContractReferencedCredential", + "CredentialType", + "build_single_referenced_credential_element" + ], "pyoaev.daemons": [ "BaseDaemon", "CollectorDaemon" @@ -639,5 +653,5 @@ "setup_logging_config" ] }, - "pyoaev_version": "3.260923.0" + "pyoaev_version": "3.261014.1" } From 9f35088dcdad7f11dc4e9af11f2fa72e70c51270 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 22:43:10 +0200 Subject: [PATCH 17/22] :recycle: refactor(pyoaev): bring the REST client helpers back from xtm-oaev-sdk --- libs/xtm-oaev-sdk/CHANGELOG.md | 4 +- libs/xtm-oaev-sdk/README.md | 2 +- libs/xtm-oaev-sdk/docs/migration_guide.md | 9 +- libs/xtm-oaev-sdk/docs/seam_strategy.md | 5 +- libs/xtm-oaev-sdk/docs/shim_strategy.md | 4 +- ...-contracts-and-their-own-invariants.tdr.md | 2 +- .../test/common/shim/test_shim_contract.py | 15 +- .../test/common/shim/test_utils.py | 36 ---- libs/xtm-oaev-sdk/test/common/test_utils.py | 124 ----------- .../xtm_oaev_sdk/_core/common/__init__.py | 2 - .../xtm_oaev_sdk/_core/common/http_errors.py | 23 -- .../xtm_oaev_sdk/public/__init__.py | 6 - .../xtm_oaev_sdk/public/common.py | 42 +--- .../xtm_oaev_sdk/seam/common/__init__.py | 6 - .../xtm_oaev_sdk/seam/common/errors.py | 6 +- .../xtm_oaev_sdk/seam/common/ports.py | 17 +- .../xtm_oaev_sdk/seam/common/values.py | 98 --------- pyoaev/apis/ai_target.py | 3 +- pyoaev/apis/attack_pattern.py | 3 +- pyoaev/apis/collector.py | 5 +- pyoaev/apis/cve.py | 3 +- pyoaev/apis/document.py | 5 +- pyoaev/apis/endpoint.py | 9 +- pyoaev/apis/inject.py | 5 +- .../inject_expectation/inject_expectation.py | 15 +- pyoaev/apis/inject_expectation_trace.py | 5 +- pyoaev/apis/injector.py | 3 +- pyoaev/apis/injector_contract.py | 5 +- pyoaev/apis/kill_chain_phase.py | 3 +- pyoaev/apis/payload.py | 5 +- pyoaev/apis/security_platform.py | 5 +- pyoaev/apis/signature.py | 3 +- pyoaev/apis/tag.py | 3 +- pyoaev/apis/team.py | 5 +- pyoaev/apis/user.py | 5 +- pyoaev/apis/vulnerability.py | 3 +- pyoaev/base.py | 4 +- pyoaev/exceptions.py | 32 ++- pyoaev/mixins.py | 17 +- pyoaev/utils.py | 57 ++++- test/test_rest_client_helpers.py | 201 ++++++++++++++++++ test/test_utils.py | 36 ++++ 42 files changed, 406 insertions(+), 435 deletions(-) delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/http_errors.py delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/values.py create mode 100644 test/test_rest_client_helpers.py diff --git a/libs/xtm-oaev-sdk/CHANGELOG.md b/libs/xtm-oaev-sdk/CHANGELOG.md index 447409f8..f2d0ba02 100644 --- a/libs/xtm-oaev-sdk/CHANGELOG.md +++ b/libs/xtm-oaev-sdk/CHANGELOG.md @@ -12,8 +12,8 @@ this SDK. The former `pyoaev` paths keep working, each one firing a ### Added -- `common`: the `OpenAEVError` hierarchy, `on_http_error`, `AppLogger`, logging setup, - JSON encoding, SSL helpers, asset and security-domain enums, and the `BaseClient`, +- `common`: the `OpenAEVError` hierarchy, `AppLogger`, logging setup, JSON encoding, + SSL helpers, asset and security-domain enums, and the `BaseClient`, `DaemonProtocol` and `AppLoggerProtocol` protocols. Seam `1.0.0`. - `configuration`: `Configuration`, `ConfigurationHint`, `SettingsLoader`, the settings models, sources and schema generator. Seam `1.0.0`. diff --git a/libs/xtm-oaev-sdk/README.md b/libs/xtm-oaev-sdk/README.md index 1f1dce73..c2399219 100644 --- a/libs/xtm-oaev-sdk/README.md +++ b/libs/xtm-oaev-sdk/README.md @@ -23,7 +23,7 @@ implementation of it. | Capability | Public API | |---|---| -| common | `OpenAEVError` and its subclasses, `on_http_error`, `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, `EncodedId`, `RequiredOptional`, `copy_dict`, `remove_none_from_dict`, SSL helpers, asset and security-domain enums, `BaseClient` / `DaemonProtocol` / `AppLoggerProtocol` | +| common | `OpenAEVError` and its subclasses, `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, `copy_dict`, `remove_none_from_dict`, SSL helpers, asset and security-domain enums, `BaseClient` / `DaemonProtocol` / `AppLoggerProtocol` | | configuration | `Configuration`, `ConfigurationHint`, `SettingsLoader`, `BaseConfigModel`, `ConfigLoaderOAEV`, `ConfigLoaderCollector`, `ConnectorConfigSchemaGenerator`, sources, serializer types | | contracts | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, every contract element (`ContractText`, `ContractSelect`, …) and enum, credential references (`CredentialType`, `ContractReferencedCredential`, `build_single_referenced_credential_element`) | | signatures | `SignatureManager`, `build_network_configs`, `build_marker`, matching types, the inbound I/O models and the port I/O wire models (`PORT_IO_CONTRACT_VERSION`) | diff --git a/libs/xtm-oaev-sdk/docs/migration_guide.md b/libs/xtm-oaev-sdk/docs/migration_guide.md index 46d4310d..d4502b41 100644 --- a/libs/xtm-oaev-sdk/docs/migration_guide.md +++ b/libs/xtm-oaev-sdk/docs/migration_guide.md @@ -109,7 +109,6 @@ every legacy module that exposed it. | `data_to_temp_file` | `pyoaev.helpers` | `from xtm_oaev_sdk import data_to_temp_file` | | `DictionarySource` | `pyoaev.configuration.configuration`, `pyoaev.configuration.sources` | `from xtm_oaev_sdk import DictionarySource` | | `Domain` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import Domain` | -| `EncodedId` | `pyoaev.utils` | `from xtm_oaev_sdk import EncodedId` | | `EnhancedJSONEncoder` | `pyoaev.utils` | `from xtm_oaev_sdk import EnhancedJSONEncoder` | | `EnvironmentSource` | `pyoaev.configuration.configuration`, `pyoaev.configuration.sources` | `from xtm_oaev_sdk import EnvironmentSource` | | `ExecutionDetails` | `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import ExecutionDetails` | @@ -130,7 +129,6 @@ every legacy module that exposed it. | `logger` | `pyoaev.daemons.base_daemon`, `pyoaev.utils` | `from xtm_oaev_sdk import logger` | | `MatchTypes` | `pyoaev`, `pyoaev.apis.inject_expectation.model.expectation`, `pyoaev.signatures`, `pyoaev.signatures.signature_match`, `pyoaev.signatures.signature_type`, `pyoaev.signatures.types` | `from xtm_oaev_sdk import MatchTypes` | | `NetworkInjectorConfig` | `pyoaev`, `pyoaev.signatures`, `pyoaev.signatures.models`, `pyoaev.signatures.signature_manager` | `from xtm_oaev_sdk import NetworkInjectorConfig` | -| `on_http_error` | `pyoaev.exceptions` | `from xtm_oaev_sdk import on_http_error` | | `OpenAEVAuthenticationError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVAuthenticationError` | | `OpenAEVCreateError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVCreateError` | | `OpenAEVDeleteError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import OpenAEVDeleteError` | @@ -146,7 +144,6 @@ every legacy module that exposed it. | `PrimitiveType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import PrimitiveType` | | `RedirectError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import RedirectError` | | `remove_none_from_dict` | `pyoaev.utils` | `from xtm_oaev_sdk import remove_none_from_dict` | -| `RequiredOptional` | `pyoaev.utils` | `from xtm_oaev_sdk import RequiredOptional` | | `SecurityDomains` | `pyoaev.security_domain.types` | `from xtm_oaev_sdk import SecurityDomains` | | `SecurityPlatformType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import SecurityPlatformType` | | `SettingsLoader` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import SettingsLoader` | @@ -293,8 +290,10 @@ variables of `xtm_oaev_sdk.VariableHelper` instead. ## What stays in pyoaev The API client, and everything wired around it: `OpenAEV`, `pyoaev.apis` (including -`inputs.search`), `pyoaev.backends`, `pyoaev.base`, `pyoaev.client`, `pyoaev.mixins`, -and `pyoaev.utils.get_content_type` / `response_content`. Nothing to migrate there. +`inputs.search`), `pyoaev.backends`, `pyoaev.base`, `pyoaev.client`, `pyoaev.mixins`, and +the REST client helpers `pyoaev.utils.get_content_type` / `response_content` / +`EncodedId` / `RequiredOptional`, plus `pyoaev.exceptions.on_http_error`. Nothing to +migrate there. ## Things you may notice diff --git a/libs/xtm-oaev-sdk/docs/seam_strategy.md b/libs/xtm-oaev-sdk/docs/seam_strategy.md index 2a433484..e0e2e4f6 100644 --- a/libs/xtm-oaev-sdk/docs/seam_strategy.md +++ b/libs/xtm-oaev-sdk/docs/seam_strategy.md @@ -57,8 +57,7 @@ the data *is*: signatures coerced to `[]`); - `__post_init__` invariants of the contract elements (derived `type`, forced cardinality, predefined expectations); -- value objects: `EncodedId` (URL encoding), `RequiredOptional.validate_attrs`, - `SignatureMatch` (score invariant), `SignatureType.make_struct_for_matching`; +- value objects: `SignatureMatch` (score invariant), `SignatureType.make_struct_for_matching`; - exceptions, including `OpenAEVError`'s extraction of a readable message from its own response body: the core raises them and users catch them, so they have to live where both sides can reach them. @@ -200,7 +199,7 @@ source selection to `SettingsSourcesPort`. The settings models (`BaseConfigModel | Capability | Seam: contracts | Seam: ports | Core | Public adapters | |---|---|---|---|---| -| common | the error hierarchy, the asset and security-domain enums, `EncodedId`, `RequiredOptional`, `BaseClient`, `DaemonProtocol`, `AppLoggerProtocol` | `HttpErrorPort`, `JsonEncodingPort`, `LoggingPort`, `DictToolsPort`, `CertificatesPort` | `HttpErrorTranslator`, `JsonEncoding`, `LoggingSetup`, `DictTools`, `Certificates` | `on_http_error`, `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, `copy_dict`, `remove_none_from_dict`, the SSL helpers | +| common | the error hierarchy, the asset and security-domain enums, `BaseClient`, `DaemonProtocol`, `AppLoggerProtocol` | `JsonEncodingPort`, `LoggingPort`, `DictToolsPort`, `CertificatesPort` | `JsonEncoding`, `LoggingSetup`, `DictTools`, `Certificates` | `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, `copy_dict`, `remove_none_from_dict`, the SSL helpers | | configuration | `ConfigurationHint`, settings models, serializer types, `CONFIGURATION_TYPES`, `ConfigurationProtocol` | `ConfigurationResolverPort`, `ConfigurationSourcesPort`, `ConnectorSchemaPort`, `SettingsSourcesPort`, `TruthinessPort` | `ConfigurationResolver`, `ConfigurationSources`, `ConnectorSchemaTransformer`, `SettingsSourceSelector`, `Truthiness` | `Configuration`, `EnvironmentSource`, `DictionarySource`, `ConnectorConfigSchemaGenerator`, `SettingsLoader`, `is_truthy`, `is_falsy` | | contracts | `ContractData`, contract elements and enums, `ContractVariable`, `Expectation`, `CredentialType`, `ContractBuilderProtocol` | `ContractAssemblyPort`, `ContractVariablesPort`, `ContractSerializerPort`, `ContractCredentialsPort` | `ContractAssembly`, `BuiltinContractVariables`, `ContractSerializer`, `CredentialReferences` | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, `build_single_referenced_credential_element` | | signatures | inbound I/O models, matching types and policies, `ExtraSignatureData`, `SignatureTransportProtocol`, `SignatureManagerProtocol` | `SignatureEnginePort`, `ContainerAddressPort`, `NetworkTargetsPort`, `MarkerPort` | `SignatureEngine`, `ContainerAddressResolver`, `NetworkTargets`, `MarkerBuilder` | `SignatureManager`, `build_network_configs`, `build_marker` | diff --git a/libs/xtm-oaev-sdk/docs/shim_strategy.md b/libs/xtm-oaev-sdk/docs/shim_strategy.md index 57f6d65e..4c7972a8 100644 --- a/libs/xtm-oaev-sdk/docs/shim_strategy.md +++ b/libs/xtm-oaev-sdk/docs/shim_strategy.md @@ -151,8 +151,8 @@ so patching a method on any of those paths hits the class connectors instantiate | `pyoaev/contracts/*` | xtm-oaev-sdk contracts (`ExpectationType` renamed, variable constants) | | `pyoaev/credential/*` | xtm-oaev-sdk contracts (credential references) | | `pyoaev/signatures/*` | xtm-oaev-sdk signatures (`ExpectationType` renamed, `SignatureManager` adapter) | -| `pyoaev/exceptions.py`, `asset_types.py`, `security_domain/types.py` | xtm-oaev-sdk common | -| `pyoaev/utils.py` | xtm-oaev-sdk common and `PingAlive`; keeps the HTTP helpers `get_content_type`, `response_content` | +| `pyoaev/exceptions.py`, `asset_types.py`, `security_domain/types.py` | xtm-oaev-sdk common; `exceptions.py` keeps the REST client decorator `on_http_error` | +| `pyoaev/utils.py` | xtm-oaev-sdk common and `PingAlive`; keeps the REST client helpers `get_content_type`, `response_content`, `EncodedId`, `RequiredOptional` | | `pyoaev/daemons/*` | `BaseDaemon` / `CollectorDaemon` adapters | | `pyoaev/helpers.py` | collectors-sdk detection, injectors-sdk helpers (`OpenAEVInjectorHelper` adapter), xtm SSL helpers, `OpenAEVCollectorHelper` | | `pyoaev/apis/inject_expectation/model/*` | collectors-sdk expectations (the API manager builds them) | diff --git a/libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md index 4bc69753..e38d026d 100644 --- a/libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md +++ b/libs/xtm-oaev-sdk/docs/tdr/seam-holds-data-contracts-and-their-own-invariants.tdr.md @@ -38,7 +38,7 @@ world, runs the show, or needs another part of the SDK goes behind a port instea - validators and derived fields: `ConfigurationHint`, `ExpectationData`'s null signatures coerced to `[]`; - `__post_init__` invariants of the contract elements; -- value objects: `EncodedId`, `RequiredOptional.validate_attrs`, `SignatureMatch`, `SignatureType.make_struct_for_matching`; +- value objects: `SignatureMatch`, `SignatureType.make_struct_for_matching`; - exceptions, including `OpenAEVError`'s message extraction: the core raises them and users catch them, so both sides must reach them. --- diff --git a/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py index 96ab33a5..015b5c1e 100644 --- a/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py +++ b/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py @@ -26,14 +26,11 @@ "OpenAEVUpdateError", "RedirectError", "SignatureTransmissionError", - "on_http_error", ] _UTILS = [ "AppLogger", "CustomJsonFormatter", - "EncodedId", "EnhancedJSONEncoder", - "RequiredOptional", "copy_dict", "logger", "remove_none_from_dict", @@ -59,11 +56,7 @@ + [("pyoaev.asset_types", name) for name in _ASSET_TYPES] + [("pyoaev.security_domain.types", "SecurityDomains")] + [("pyoaev", name) for name in _ASSET_TYPES] - + [ - ("pyoaev", name) - for name in _ERRORS - if name not in ("OpenAEVError", "on_http_error") - ] + + [("pyoaev", name) for name in _ERRORS if name != "OpenAEVError"] ) @@ -110,7 +103,7 @@ def test_pyoaev_all_still_lists_legacy_exceptions() -> None: import pyoaev for name in _ERRORS: - if name not in ("OpenAEVError", "on_http_error"): + if name != "OpenAEVError": assert name in pyoaev.__all__ for name in _ASSET_TYPES: assert name in pyoaev.__all__ @@ -133,8 +126,8 @@ def test_sdk_objects_stay_silent_once_pyoaev_is_loaded() -> None: script = ( "import pyoaev, pyoaev.exceptions, pyoaev.utils, pyoaev.helpers\n" "import xtm_oaev_sdk as sdk\n" - "sdk.EncodedId('a b')\n" - "sdk.RequiredOptional(required=('a',)).validate_attrs(data={'a': 1})\n" + "sdk.copy_dict(src={'a': {'b': 1}}, dest={})\n" + "sdk.remove_none_from_dict({'a': None})\n" "sdk.AssetCategory('WEB_APPLICATION')\n" "try:\n" " raise sdk.OpenAEVHttpError('boom', 500)\n" diff --git a/libs/xtm-oaev-sdk/test/common/shim/test_utils.py b/libs/xtm-oaev-sdk/test/common/shim/test_utils.py index 3bc1dc74..5a90347c 100644 --- a/libs/xtm-oaev-sdk/test/common/shim/test_utils.py +++ b/libs/xtm-oaev-sdk/test/common/shim/test_utils.py @@ -2,7 +2,6 @@ import json import logging import unittest -from typing import Any, cast from unittest.mock import MagicMock, patch from pythonjsonlogger.json import JsonFormatter @@ -30,18 +29,6 @@ def test_copy_dict_flattens_nested_dict(self): def test_remove_none_from_dict(self): self.assertEqual(module.remove_none_from_dict({"a": 1, "b": None}), {"a": 1}) - def test_encoded_id_from_existing_instance_returns_same_object(self): - encoded_id = module.EncodedId("with space") - self.assertIs(module.EncodedId(encoded_id), encoded_id) - - def test_encoded_id_encodes_string_and_keeps_int(self): - self.assertEqual(module.EncodedId("a/b c"), "a%2Fb%20c") - self.assertEqual(module.EncodedId(42), "42") - - def test_encoded_id_rejects_unsupported_type(self): - with self.assertRaises(TypeError): - module.EncodedId(cast(Any, ["bad"])) - def test_enhanced_json_encoder_serializes_not_dataclasses(self): data = {"test": "this is a test", "other": 3} classic_json_dump = json.dumps(data) @@ -58,29 +45,6 @@ def test_enhanced_json_encoder_serializes_dataclasses(self): '{"value": 3}', ) - def test_required_optional_required_attribute_missing(self): - rules = module.RequiredOptional(required=("name",)) - with self.assertRaises(AttributeError): - rules.validate_attrs(data={}) - - def test_required_optional_excludes_required_attribute(self): - rules = module.RequiredOptional(required=("name",)) - rules.validate_attrs(data={}, excludes=["name"]) - - def test_required_optional_exclusive_allows_only_one_key(self): - rules = module.RequiredOptional(exclusive=("a", "b")) - with self.assertRaises(AttributeError): - rules.validate_attrs(data={"a": 1, "b": 2}) - - def test_required_optional_exclusive_requires_one_key(self): - rules = module.RequiredOptional(exclusive=("a", "b")) - with self.assertRaises(AttributeError): - rules.validate_attrs(data={}) - - def test_required_optional_exclusive_with_single_key_is_valid(self): - rules = module.RequiredOptional(exclusive=("a", "b")) - rules.validate_attrs(data={"a": 1}) - def test_custom_json_formatter_add_fields_sets_timestamp_and_level(self): formatter = module.CustomJsonFormatter("%(message)s") record = logging.LogRecord( diff --git a/libs/xtm-oaev-sdk/test/common/test_utils.py b/libs/xtm-oaev-sdk/test/common/test_utils.py index 49691327..a0ffb99a 100644 --- a/libs/xtm-oaev-sdk/test/common/test_utils.py +++ b/libs/xtm-oaev-sdk/test/common/test_utils.py @@ -54,15 +54,6 @@ def _given_app_logger_with_recording_logger() -> tuple[Any, _RecordingLogger]: return app_logger, recorder -def _given_required_optional(**kwargs: Any) -> Any: - required_optional_cls = _given_public_symbol("RequiredOptional") - return required_optional_cls(**kwargs) - - -def _given_exclusive_spec() -> Any: - return _given_required_optional(exclusive=("id", "key")) - - def _given_json_encoder() -> Any: encoder_cls = _given_public_symbol("EnhancedJSONEncoder") return encoder_cls() @@ -100,39 +91,6 @@ def _when_encode_unsupported_type_raises(encoder: Any, value: Any) -> Exception: return exc_info.value -def _when_validate_attrs_raises( - validator: Any, data: dict[str, Any], excludes: list[str] | None = None -) -> Exception: - with pytest.raises(AttributeError) as exc_info: - validator.validate_attrs(data=data, excludes=excludes) - return exc_info.value - - -def _when_validate_attrs( - validator: Any, data: dict[str, Any], excludes: list[str] | None = None -) -> None: - validator.validate_attrs(data=data, excludes=excludes) - - -def _when_set_required_optional_field_raises( - instance: Any, field_name: str, value: Any -) -> Exception: - with pytest.raises(dataclasses.FrozenInstanceError) as exc_info: - setattr(instance, field_name, value) - return exc_info.value - - -def _when_encode_id(value: Any) -> Any: - encoded_id_cls = _given_public_symbol("EncodedId") - return encoded_id_cls(value) - - -def _when_encode_invalid_id_raises(value: Any) -> Exception: - with pytest.raises(TypeError) as exc_info: - _when_encode_id(value) - return exc_info.value - - def _when_copy_dict(src: dict[str, Any], dest: dict[str, Any]) -> None: copy_dict_fn = _given_public_symbol("copy_dict") copy_dict_fn(src=src, dest=dest) @@ -190,23 +148,10 @@ def _then_encoded_object_equals(encoded: Any, expected: Any) -> None: assert encoded == expected -def _then_contains_text(exc: Exception, text: str) -> None: - assert text in str(exc) - - -def _then_error_is_frozen(exc: Exception) -> None: - assert isinstance(exc, dataclasses.FrozenInstanceError) - - def _then_type_error_for_unsupported_type(exc: Exception) -> None: assert "not json serializable" in str(exc).lower() -def _then_idempotent_reencode(value: Any) -> None: - encoded_id_cls = _given_public_symbol("EncodedId") - assert encoded_id_cls(encoded_id_cls(value)) == encoded_id_cls(value) - - def _then_dict_equals(actual: dict[str, Any], expected: dict[str, Any]) -> None: assert actual == expected @@ -270,75 +215,6 @@ def test_enhanced_json_encoder_raises_type_error_for_unsupported_types() -> None _then_type_error_for_unsupported_type(exc) -def test_required_optional_is_frozen() -> None: - """Verify RequiredOptional is immutable and rejects field assignment.""" - required_optional = _given_required_optional(required=("name",)) - exc = _when_set_required_optional_field_raises( - required_optional, "required", ("id",) - ) - _then_error_is_frozen(exc) - - -def test_required_optional_validate_attrs_missing_required_raises() -> None: - """Verify RequiredOptional.validate_attrs raises for missing required fields.""" - required_optional = _given_required_optional(required=("name", "type")) - exc = _when_validate_attrs_raises(required_optional, {"name": "sdk"}) - _then_contains_text(exc, "Missing attributes") - - -@pytest.mark.parametrize( - ("data", "expected_text"), - [ - ({}, "Must provide one of these attributes"), - ({"id": "1", "key": "2"}, "Provide only one of these attributes"), - ], -) -def test_required_optional_validate_attrs_exclusive_violations_raise( - data: dict[str, Any], expected_text: str -) -> None: - """Verify RequiredOptional.validate_attrs raises for exclusive rule violations.""" - required_optional = _given_exclusive_spec() - exc = _when_validate_attrs_raises(required_optional, data) - _then_contains_text(exc, expected_text) - - -def test_required_optional_validate_attrs_excludes_skip_required_check() -> None: - """Verify RequiredOptional.validate_attrs excludes bypass required checks.""" - required_optional = _given_required_optional(required=("name", "type")) - _when_validate_attrs(required_optional, {"name": "sdk"}, excludes=["type"]) - - -def test_required_optional_validate_attrs_valid_data_passes() -> None: - """Verify RequiredOptional.validate_attrs accepts valid required/exclusive data.""" - required_optional = _given_required_optional( - required=("name",), exclusive=("id", "key") - ) - _when_validate_attrs(required_optional, {"name": "sdk", "id": "123"}) - - -def test_encoded_id_url_encodes_strings() -> None: - """Verify EncodedId percent-encodes unsafe string characters.""" - encoded = _when_encode_id("hello world") - _then_encoded_object_equals(encoded, "hello%20world") - - -def test_encoded_id_is_idempotent_when_reencoded() -> None: - """Verify EncodedId(EncodedId(x)) yields the same encoded value.""" - _then_idempotent_reencode("hello world") - - -def test_encoded_id_rejects_non_str_non_int_values() -> None: - """Verify EncodedId raises TypeError for unsupported value types.""" - exc = _when_encode_invalid_id_raises(3.14) - _then_contains_text(exc, "Unsupported type received") - - -def test_encoded_id_accepts_int_and_returns_string_representation() -> None: - """Verify EncodedId passes integer values through as string content.""" - encoded = _when_encode_id(42) - _then_encoded_object_equals(encoded, "42") - - def test_copy_dict_flattens_nested_dict_keys() -> None: """Verify copy_dict flattens nested dict keys to bracket notation.""" destination: dict[str, Any] = {} diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py index adff0d9e..5c927d9a 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py @@ -3,13 +3,11 @@ from .certificates import Certificates from .dicts import DictTools from .encoding import JsonEncoding -from .http_errors import HttpErrorTranslator from .logging_setup import LoggingSetup __all__ = [ "Certificates", "DictTools", - "HttpErrorTranslator", "JsonEncoding", "LoggingSetup", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/http_errors.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/http_errors.py deleted file mode 100644 index be4d46c4..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/http_errors.py +++ /dev/null @@ -1,23 +0,0 @@ -"""HTTP error translation: generic HTTP errors into operation-specific ones.""" - -from typing import Any, Callable - -from xtm_oaev_sdk.seam.common import OpenAEVHttpError - -__all__ = ["HttpErrorTranslator"] - - -class HttpErrorTranslator: - """Implements ``HttpErrorPort``.""" - - def call( - self, - function: Callable[..., Any], - error: type[Exception], - *args: Any, - **kwargs: Any, - ) -> Any: - try: - return function(*args, **kwargs) - except OpenAEVHttpError as e: - raise error(e.error_message, e.response_code, e.response_body) from e diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index 991ef464..daaa9056 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -14,7 +14,6 @@ ConfigurationError, CustomJsonFormatter, DaemonProtocol, - EncodedId, EnhancedJSONEncoder, OpenAEVAuthenticationError, OpenAEVCreateError, @@ -27,14 +26,12 @@ OpenAEVParsingError, OpenAEVUpdateError, RedirectError, - RequiredOptional, SecurityDomains, SignatureTransmissionError, copy_dict, data_to_temp_file, is_memory_certificate, logger, - on_http_error, remove_none_from_dict, setup_logging_config, ssl_cert_chain, @@ -161,14 +158,11 @@ "OpenAEVUpdateError", "RedirectError", "SignatureTransmissionError", - "on_http_error", # Common: utils "AppLogger", "AppLoggerProtocol", "CustomJsonFormatter", - "EncodedId", "EnhancedJSONEncoder", - "RequiredOptional", "copy_dict", "logger", "remove_none_from_dict", diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py index 8e3ededa..52767b21 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py @@ -6,17 +6,15 @@ implementation of its port. """ -import functools import json import logging import ssl -from typing import Any, Callable, Type, TypeVar, cast +from typing import Any from pythonjsonlogger.json import JsonFormatter from xtm_oaev_sdk._core.common import ( Certificates, DictTools, - HttpErrorTranslator, JsonEncoding, LoggingSetup, ) @@ -31,8 +29,6 @@ ConfigurationError, DaemonProtocol, DictToolsPort, - EncodedId, - HttpErrorPort, JsonEncodingPort, LoggingPort, OpenAEVAuthenticationError, @@ -46,7 +42,6 @@ OpenAEVParsingError, OpenAEVUpdateError, RedirectError, - RequiredOptional, SecurityDomains, SignatureTransmissionError, ) @@ -62,7 +57,6 @@ "ConfigurationError", "CustomJsonFormatter", "DaemonProtocol", - "EncodedId", "EnhancedJSONEncoder", "OpenAEVAuthenticationError", "OpenAEVCreateError", @@ -75,57 +69,23 @@ "OpenAEVParsingError", "OpenAEVUpdateError", "RedirectError", - "RequiredOptional", "SecurityDomains", "SignatureTransmissionError", "copy_dict", "data_to_temp_file", "is_memory_certificate", "logger", - "on_http_error", "remove_none_from_dict", "setup_logging_config", "ssl_cert_chain", "ssl_verify_locations", ] -_http_errors: HttpErrorPort = HttpErrorTranslator() _encoding: JsonEncodingPort = JsonEncoding() _logging: LoggingPort = LoggingSetup() _dicts: DictToolsPort = DictTools() _certificates: CertificatesPort = Certificates() -_F = TypeVar("_F", bound=Callable[..., Any]) - - -def on_http_error(error: Type[Exception]) -> Callable[[_F], _F]: - """Decorator that wraps OpenAEVHttpError into a caller-specified type. - - Use on API methods to convert generic HTTP errors into operation-specific - exception types while preserving error context. - - Args: - error: The exception class to raise instead of OpenAEVHttpError. - Must accept (error_message, response_code, response_body) args. - - Returns: - A decorator that wraps the target function. - - Example: - >>> @on_http_error(OpenAEVCreateError) - ... def create_entity(self, data): - ... return self._post("/entities", data) - """ - - def wrap(f: _F) -> _F: - @functools.wraps(f) - def wrapped_f(*args: Any, **kwargs: Any) -> Any: - return _http_errors.call(f, error, *args, **kwargs) - - return cast(_F, wrapped_f) - - return wrap - class EnhancedJSONEncoder(json.JSONEncoder): """JSON encoder that serializes ``@dataclass`` instances via diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py index 7457caf2..6d798bc9 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py @@ -30,7 +30,6 @@ from xtm_oaev_sdk.seam.common.ports import ( CertificatesPort, DictToolsPort, - HttpErrorPort, JsonEncodingPort, LoggingPort, ) @@ -40,7 +39,6 @@ DaemonProtocol, ) from xtm_oaev_sdk.seam.common.security_domains import SecurityDomains -from xtm_oaev_sdk.seam.common.values import EncodedId, RequiredOptional #: Version of the common seam. Bump on any contract change. SEAM_VERSION = "1.0.0" @@ -68,9 +66,6 @@ "CloudProvider", # Enums (security domains) "SecurityDomains", - # Value objects - "EncodedId", - "RequiredOptional", # Consumer protocols "AppLoggerProtocol", "BaseClient", @@ -78,7 +73,6 @@ # Ports "CertificatesPort", "DictToolsPort", - "HttpErrorPort", "JsonEncodingPort", "LoggingPort", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/errors.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/errors.py index 8cca2995..13d9f49e 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/errors.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/errors.py @@ -8,7 +8,7 @@ OpenAEVError (base) ├── OpenAEVAuthenticationError — credential/token failures ├── OpenAEVHttpError — generic HTTP transport errors - │ └── (wrapped by on_http_error decorator into caller-specific types) + │ └── (wrapped by the HTTP client, e.g. pyoaev's on_http_error, into caller-specific types) ├── OpenAEVParsingError — response deserialization failures ├── RedirectError — unexpected HTTP redirects ├── OpenAEVHeadError — HEAD request failures @@ -198,8 +198,8 @@ class OpenAEVAuthenticationError(OpenAEVError): class OpenAEVHttpError(OpenAEVError): """Generic HTTP transport error. - Raised on unexpected HTTP responses. Use the ``on_http_error`` - decorator to automatically wrap these into operation-specific types. + Raised on unexpected HTTP responses. HTTP clients (e.g. pyoaev's + ``on_http_error`` decorator) wrap these into operation-specific types. """ pass diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py index aa150713..82576fbd 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py @@ -2,31 +2,16 @@ import logging import ssl -from typing import Any, Callable, Protocol, runtime_checkable +from typing import Any, Protocol, runtime_checkable __all__ = [ "CertificatesPort", "DictToolsPort", - "HttpErrorPort", "JsonEncodingPort", "LoggingPort", ] -@runtime_checkable -class HttpErrorPort(Protocol): - """Converts generic HTTP errors into operation-specific ones.""" - - def call( - self, - function: Callable[..., Any], - error: type[Exception], - *args: Any, - **kwargs: Any, - ) -> Any: - """Result of ``function``, raising ``error`` instead of an HTTP error.""" - - @runtime_checkable class JsonEncodingPort(Protocol): """Encodes the objects the standard JSON encoder does not support.""" diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/values.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/values.py deleted file mode 100644 index 75713bbd..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/values.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Value objects shared across capabilities: encoded identifiers and -field-presence rules.""" - -import dataclasses -import urllib.parse -from typing import Any - -__all__ = ["EncodedId", "RequiredOptional"] - - -@dataclasses.dataclass(frozen=True) -class RequiredOptional: - """Immutable field-presence validator for API payloads. - - Validates that required fields are present, optional fields are allowed, - and exclusive fields follow mutual exclusion rules. - - Args: - required: Tuple of field names that must be present. - optional: Tuple of field names that may be present. - exclusive: Tuple of field names where exactly one must be present. - - Example: - >>> ro = RequiredOptional(required=("name", "type"), exclusive=("id", "key")) - >>> ro.validate_attrs(data={"name": "x", "type": "y", "id": "123"}) # OK - >>> ro.validate_attrs(data={"name": "x"}) # raises AttributeError - """ - - required: tuple[str, ...] = () - optional: tuple[str, ...] = () - exclusive: tuple[str, ...] = () - - def validate_attrs( - self, - *, - data: dict[str, Any], - excludes: list[str] | None = None, - ) -> None: - """Validate field presence against the configured rules. - - Args: - data: Dict of field names to values to validate. - excludes: Field names to skip during required-field checks. - - Raises: - AttributeError: If required fields are missing, or exclusive - field constraints are violated. - """ - if excludes is None: - excludes = [] - - if self.required: - required = [k for k in self.required if k not in excludes] - missing = [attr for attr in required if attr not in data] - if missing: - raise AttributeError(f"Missing attributes: {', '.join(missing)}") - - if self.exclusive: - exclusives = [attr for attr in data if attr in self.exclusive] - if len(exclusives) > 1: - raise AttributeError( - f"Provide only one of these attributes: {', '.join(exclusives)}" - ) - if not exclusives: - raise AttributeError( - f"Must provide one of these attributes: " - f"{', '.join(self.exclusive)}" - ) - - -class EncodedId(str): - """URL-safe encoded identifier. - - Wraps a string or integer value with URL percent-encoding applied. - Idempotent: encoding an already-encoded value returns it unchanged. - - Args: - value: The raw identifier to encode. Accepts str, int, or EncodedId. - - Raises: - TypeError: If value is not str, int, or EncodedId. - - Example: - >>> EncodedId("hello world") - 'hello%20world' - >>> EncodedId(42) - '42' - """ - - def __new__(cls, value: "str | int | EncodedId") -> "EncodedId": - if isinstance(value, EncodedId): - return value - - if not isinstance(value, (int, str)): - raise TypeError(f"Unsupported type received: {type(value)}") - if isinstance(value, str): - value = urllib.parse.quote(value, safe="") - return super().__new__(cls, value) diff --git a/pyoaev/apis/ai_target.py b/pyoaev/apis/ai_target.py index 56ae892b..7ec68eb9 100644 --- a/pyoaev/apis/ai_target.py +++ b/pyoaev/apis/ai_target.py @@ -1,7 +1,6 @@ -from xtm_oaev_sdk import RequiredOptional - from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, DeleteMixin, GetMixin, ListMixin, UpdateMixin +from pyoaev.utils import RequiredOptional class AiTarget(RESTObject): diff --git a/pyoaev/apis/attack_pattern.py b/pyoaev/apis/attack_pattern.py index ffd035ad..bf74d1c4 100644 --- a/pyoaev/apis/attack_pattern.py +++ b/pyoaev/apis/attack_pattern.py @@ -3,6 +3,7 @@ import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class AttackPattern(RESTObject): @@ -13,7 +14,7 @@ class AttackPatternManager(RESTManager): _path = "/attack_patterns" _obj_cls = AttackPattern - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert( self, attack_patterns: List[Dict[str, Any]], diff --git a/pyoaev/apis/collector.py b/pyoaev/apis/collector.py index c49cd450..8673f380 100644 --- a/pyoaev/apis/collector.py +++ b/pyoaev/apis/collector.py @@ -1,10 +1,11 @@ from typing import Any, Dict import xtm_oaev_sdk as exc -from xtm_oaev_sdk import RequiredOptional from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin, GetMixin, ListMixin, UpdateMixin +from pyoaev.utils import RequiredOptional class Collector(RESTObject): @@ -29,7 +30,7 @@ class CollectorManager(GetMixin, ListMixin, CreateMixin, UpdateMixin, RESTManage ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def get(self, collector_id: str, **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/" + collector_id result = self.openaev.http_get(path, **kwargs) diff --git a/pyoaev/apis/cve.py b/pyoaev/apis/cve.py index 5bd0122f..1604d40e 100644 --- a/pyoaev/apis/cve.py +++ b/pyoaev/apis/cve.py @@ -3,6 +3,7 @@ import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Cve(RESTObject): @@ -12,7 +13,7 @@ class Cve(RESTObject): class CveManager(RESTManager): _path = "/cves" - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/bulk" result = self.openaev.http_post(path, post_data=data, **kwargs) diff --git a/pyoaev/apis/document.py b/pyoaev/apis/document.py index 4089a25a..c1b70d93 100644 --- a/pyoaev/apis/document.py +++ b/pyoaev/apis/document.py @@ -3,6 +3,7 @@ import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Document(RESTObject): @@ -13,13 +14,13 @@ class DocumentManager(RESTManager): _path = "/documents" _obj_cls = Document - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def download(self, document_id: str, **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/" + document_id + "/file" result = self.openaev.http_get(path, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert( self, document: Dict[str, Any], file: tuple, **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/endpoint.py b/pyoaev/apis/endpoint.py index f65a07b3..f4354384 100644 --- a/pyoaev/apis/endpoint.py +++ b/pyoaev/apis/endpoint.py @@ -1,10 +1,11 @@ from typing import Any, Dict import xtm_oaev_sdk as exc -from xtm_oaev_sdk import RequiredOptional from pyoaev.apis.inputs.search import SearchPaginationInput from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error +from pyoaev.utils import RequiredOptional class Endpoint(RESTObject): @@ -41,19 +42,19 @@ class EndpointManager(RESTManager): ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def get(self, asset_id: str, **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/" + asset_id result = self.openaev.http_get(path, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, endpoint: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/agentless/upsert" result = self.openaev.http_post(path, post_data=endpoint, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def searchTargets( self, input: SearchPaginationInput, **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/inject.py b/pyoaev/apis/inject.py index bd5a376e..2a6c7b3b 100644 --- a/pyoaev/apis/inject.py +++ b/pyoaev/apis/inject.py @@ -3,6 +3,7 @@ import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Inject(RESTObject): @@ -13,7 +14,7 @@ class InjectManager(RESTManager): _path = "/injects" _obj_cls = Inject - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def execution_callback( self, inject_id: str, data: Dict[str, Any], **kwargs: Any ) -> Dict[str, Any]: @@ -21,7 +22,7 @@ def execution_callback( result = self.openaev.http_post(path, post_data=data, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def execution_reception( self, inject_id: str, data: Dict[str, Any], **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/inject_expectation/inject_expectation.py b/pyoaev/apis/inject_expectation/inject_expectation.py index 27a1722a..8fd63c85 100644 --- a/pyoaev/apis/inject_expectation/inject_expectation.py +++ b/pyoaev/apis/inject_expectation/inject_expectation.py @@ -6,10 +6,11 @@ ExpectationTypeEnum, PreventionExpectation, ) -from xtm_oaev_sdk import RequiredOptional from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import ListMixin, UpdateMixin +from pyoaev.utils import RequiredOptional class InjectExpectation(RESTObject): @@ -21,7 +22,7 @@ class InjectExpectationManager(ListMixin, UpdateMixin, RESTManager): _obj_cls = InjectExpectation _update_attrs = RequiredOptional(required=("collector_id", "result", "is_success")) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def expectations_assets_for_source( self, source_id: str, expiration_time: int = None, **kwargs: Any ) -> Dict[str, Any]: @@ -41,7 +42,7 @@ def expectations_assets_for_source( ) return result - @exc.on_http_error(exc.OpenAEVListError) + @on_http_error(exc.OpenAEVListError) def ai_expectations_for_source( self, source_id: str, **kwargs: Any ) -> List[Dict[str, Any]]: @@ -115,7 +116,7 @@ def expectations_models_for_source(self, source_id: str, **kwargs: Any): ) return expectations - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def prevention_expectations_for_source( self, source_id: str, **kwargs: Any ) -> Dict[str, Any]: @@ -123,7 +124,7 @@ def prevention_expectations_for_source( result = self.openaev.http_get(path, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def detection_expectations_for_source( self, source_id: str, expiration_time: int = None, **kwargs: Any ) -> Dict[str, Any]: @@ -137,7 +138,7 @@ def detection_expectations_for_source( ) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def update( self, inject_expectation_id: str, @@ -148,7 +149,7 @@ def update( result = self.openaev.http_put(path, post_data=inject_expectation, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def bulk_update( self, inject_expectation_input_by_id: Dict[str, Dict[str, Any]], diff --git a/pyoaev/apis/inject_expectation_trace.py b/pyoaev/apis/inject_expectation_trace.py index 64587080..7474e619 100644 --- a/pyoaev/apis/inject_expectation_trace.py +++ b/pyoaev/apis/inject_expectation_trace.py @@ -1,10 +1,11 @@ from typing import Any, Dict, List import xtm_oaev_sdk as exc -from xtm_oaev_sdk import RequiredOptional from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin +from pyoaev.utils import RequiredOptional class InjectExpectationTrace(RESTObject): @@ -24,7 +25,7 @@ class InjectExpectationTraceManager(CreateMixin, RESTManager): ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def bulk_create( self, payload: Dict[str, List[Dict[str, str]]], **kwargs: Any ) -> dict[str, Any]: diff --git a/pyoaev/apis/injector.py b/pyoaev/apis/injector.py index f760dc4b..c6ea0d8d 100644 --- a/pyoaev/apis/injector.py +++ b/pyoaev/apis/injector.py @@ -1,7 +1,6 @@ -from xtm_oaev_sdk import RequiredOptional - from pyoaev.base import RESTManager, RESTObject from pyoaev.mixins import CreateMixin, GetMixin, ListMixin, UpdateMixin +from pyoaev.utils import RequiredOptional class Injector(RESTObject): diff --git a/pyoaev/apis/injector_contract.py b/pyoaev/apis/injector_contract.py index 7c53f13f..d4e42db4 100644 --- a/pyoaev/apis/injector_contract.py +++ b/pyoaev/apis/injector_contract.py @@ -1,11 +1,12 @@ from typing import Any, Dict import xtm_oaev_sdk as exc -from xtm_oaev_sdk import RequiredOptional from pyoaev.apis.inputs.search import InjectorContractSearchPaginationInput from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin, DeleteMixin, UpdateMixin +from pyoaev.utils import RequiredOptional class InjectorContract(RESTObject): @@ -47,7 +48,7 @@ class InjectorContractManager(CreateMixin, UpdateMixin, DeleteMixin, RESTManager ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def search( self, input: InjectorContractSearchPaginationInput, **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/kill_chain_phase.py b/pyoaev/apis/kill_chain_phase.py index 5e7677c6..8ba373f4 100644 --- a/pyoaev/apis/kill_chain_phase.py +++ b/pyoaev/apis/kill_chain_phase.py @@ -3,6 +3,7 @@ import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class KillChainPhase(RESTObject): @@ -13,7 +14,7 @@ class KillChainPhaseManager(RESTManager): _path = "/kill_chain_phases" _obj_cls = KillChainPhase - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert( self, kill_chain_phases: List[Dict[str, Any]], **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/payload.py b/pyoaev/apis/payload.py index 7cb9f2f2..0f3bf57a 100644 --- a/pyoaev/apis/payload.py +++ b/pyoaev/apis/payload.py @@ -3,6 +3,7 @@ import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Payload(RESTObject): @@ -13,13 +14,13 @@ class PayloadManager(RESTManager): _path = "/payloads" _obj_cls = Payload - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, payload: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/upsert" result = self.openaev.http_post(path, post_data=payload, **kwargs) return result - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def deprecate( self, payloads_processed: Dict[str, Any], **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/security_platform.py b/pyoaev/apis/security_platform.py index 12052b97..c7ad83dc 100644 --- a/pyoaev/apis/security_platform.py +++ b/pyoaev/apis/security_platform.py @@ -1,10 +1,11 @@ from typing import Any, Dict import xtm_oaev_sdk as exc -from xtm_oaev_sdk import RequiredOptional from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin, GetMixin, ListMixin, UpdateMixin +from pyoaev.utils import RequiredOptional class SecurityPlatform(RESTObject): @@ -25,7 +26,7 @@ class SecurityPlatformManager( ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert( self, security_platform: Dict[str, Any], **kwargs: Any ) -> Dict[str, Any]: diff --git a/pyoaev/apis/signature.py b/pyoaev/apis/signature.py index c4196011..fdd89c36 100644 --- a/pyoaev/apis/signature.py +++ b/pyoaev/apis/signature.py @@ -15,6 +15,7 @@ ) from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Signature(RESTObject): @@ -207,7 +208,7 @@ def _build_envelope( SignatureCallbackPayload.model_validate(envelope) return envelope - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def callback( self, inject_id: str, data: dict[str, Any], **kwargs: Any ) -> dict[str, Any]: diff --git a/pyoaev/apis/tag.py b/pyoaev/apis/tag.py index 0d0c914d..72941f15 100644 --- a/pyoaev/apis/tag.py +++ b/pyoaev/apis/tag.py @@ -3,6 +3,7 @@ import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Tag(RESTObject): @@ -13,7 +14,7 @@ class TagManager(RESTManager): _path = "/tags" _obj_cls = Tag - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/upsert" result = self.openaev.http_post(path, post_data=data, **kwargs) diff --git a/pyoaev/apis/team.py b/pyoaev/apis/team.py index 27ed4ace..947be9d7 100644 --- a/pyoaev/apis/team.py +++ b/pyoaev/apis/team.py @@ -1,10 +1,11 @@ from typing import Any, Dict import xtm_oaev_sdk as exc -from xtm_oaev_sdk import RequiredOptional from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin, ListMixin, UpdateMixin +from pyoaev.utils import RequiredOptional class Team(RESTObject): @@ -19,7 +20,7 @@ class TeamManager(CreateMixin, ListMixin, UpdateMixin, RESTManager): optional=("team_description", "team_organization", "team_tags"), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, team: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/upsert" result = self.openaev.http_post(path, post_data=team, **kwargs) diff --git a/pyoaev/apis/user.py b/pyoaev/apis/user.py index a6628b44..29055913 100644 --- a/pyoaev/apis/user.py +++ b/pyoaev/apis/user.py @@ -1,10 +1,11 @@ from typing import Any, Dict import xtm_oaev_sdk as exc -from xtm_oaev_sdk import RequiredOptional from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error from pyoaev.mixins import CreateMixin, ListMixin, UpdateMixin +from pyoaev.utils import RequiredOptional class User(RESTObject): @@ -25,7 +26,7 @@ class UserManager(CreateMixin, ListMixin, UpdateMixin, RESTManager): ), ) - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, user: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/upsert" result = self.openaev.http_post(path, post_data=user, **kwargs) diff --git a/pyoaev/apis/vulnerability.py b/pyoaev/apis/vulnerability.py index 433c444b..73d6bd48 100644 --- a/pyoaev/apis/vulnerability.py +++ b/pyoaev/apis/vulnerability.py @@ -3,6 +3,7 @@ import xtm_oaev_sdk as exc from pyoaev.base import RESTManager, RESTObject +from pyoaev.exceptions import on_http_error class Vulnerability(RESTObject): @@ -12,7 +13,7 @@ class Vulnerability(RESTObject): class VulnerabilityManager(RESTManager): _path = "/vulnerabilities" - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def upsert(self, data: Dict[str, Any], **kwargs: Any) -> Dict[str, Any]: path = f"{self.path}/bulk" result = self.openaev.http_post(path, post_data=data, **kwargs) diff --git a/pyoaev/base.py b/pyoaev/base.py index 8576eed5..3cb970fa 100644 --- a/pyoaev/base.py +++ b/pyoaev/base.py @@ -15,7 +15,9 @@ get_type_hints, ) -from xtm_oaev_sdk import EncodedId, OpenAEVParsingError +from xtm_oaev_sdk import OpenAEVParsingError + +from pyoaev.utils import EncodedId from .client import OpenAEV, OpenAEVList diff --git a/pyoaev/exceptions.py b/pyoaev/exceptions.py index b92a2a4d..5f00376b 100644 --- a/pyoaev/exceptions.py +++ b/pyoaev/exceptions.py @@ -1,10 +1,15 @@ -"""DEPRECATED: moved to :mod:`xtm_oaev_sdk`. +"""DEPRECATED: the error types moved to :mod:`xtm_oaev_sdk`. -Backward-compatibility shim: every name below resolves to the *identical* +Backward-compatibility shim: every error name below resolves to the *identical* xtm_oaev_sdk object and emits a :class:`DeprecationWarning` on access. +``on_http_error`` is a REST client helper and stays defined here. """ +import functools +from typing import Any, Callable, Type, TypeVar, cast + import xtm_oaev_sdk as _sdk +from xtm_oaev_sdk import OpenAEVHttpError as _OpenAEVHttpError from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr # Unchanged public export list of the legacy module (also extended into @@ -25,7 +30,28 @@ "SignatureTransmissionError", ] -_MOVED = [*__all__, "OpenAEVError", "on_http_error"] +_MOVED = [*__all__, "OpenAEVError"] + +# For an explanation of how these type-hints work see: +# https://mypy.readthedocs.io/en/stable/generics.html#declaring-decorators +# +# The goal here is that functions which get decorated will retain their types. +__F = TypeVar("__F", bound=Callable[..., Any]) + + +def on_http_error(error: Type[Exception]) -> Callable[[__F], __F]: + def wrap(f: __F) -> __F: + @functools.wraps(f) + def wrapped_f(*args: Any, **kwargs: Any) -> Any: + try: + return f(*args, **kwargs) + except _OpenAEVHttpError as e: + raise error(e.error_message, e.response_code, e.response_body) from e + + return cast(__F, wrapped_f) + + return wrap + __getattr__ = _make_deprecated_getattr( { diff --git a/pyoaev/mixins.py b/pyoaev/mixins.py index 411fbf56..dd98938a 100644 --- a/pyoaev/mixins.py +++ b/pyoaev/mixins.py @@ -13,10 +13,11 @@ import requests import xtm_oaev_sdk as exc -from xtm_oaev_sdk import EncodedId import pyoaev from pyoaev import base +from pyoaev.exceptions import on_http_error +from pyoaev.utils import EncodedId __all__ = [ "GetMixin", @@ -33,7 +34,7 @@ class HeadMixin(_RestManagerBase): - @exc.on_http_error(exc.OpenAEVHeadError) + @on_http_error(exc.OpenAEVHeadError) def head( self, id: Optional[Union[str, int]] = None, **kwargs: Any ) -> "requests.structures.CaseInsensitiveDict[Any]": @@ -57,7 +58,7 @@ class GetMixin(HeadMixin, _RestManagerBase): _path: Optional[str] openaev: pyoaev.OpenAEV - @exc.on_http_error(exc.OpenAEVGetError) + @on_http_error(exc.OpenAEVGetError) def get(self, id: Union[str, int], **kwargs: Any) -> base.RESTObject: if isinstance(id, str): id = EncodedId(id) @@ -80,7 +81,7 @@ class GetWithoutIdMixin(HeadMixin, _RestManagerBase): _path: Optional[str] openaev: pyoaev.OpenAEV - @exc.on_http_error(exc.OpenAEVGetError) + @on_http_error(exc.OpenAEVGetError) def get(self, **kwargs: Any) -> base.RESTObject: if TYPE_CHECKING: assert self.path is not None @@ -101,7 +102,7 @@ class ListMixin(HeadMixin, _RestManagerBase): _path: Optional[str] openaev: pyoaev.OpenAEV - @exc.on_http_error(exc.OpenAEVListError) + @on_http_error(exc.OpenAEVListError) def list(self, **kwargs: Any) -> Union[base.RESTObjectList, List[base.RESTObject]]: if self.openaev.per_page: @@ -159,7 +160,7 @@ def _get_update_method( http_method = self.openaev.http_put return http_method - @exc.on_http_error(exc.OpenAEVUpdateError) + @on_http_error(exc.OpenAEVUpdateError) def update( self, id: Optional[Union[str, int]] = None, @@ -193,7 +194,7 @@ class CreateMixin(_RestManagerBase): _path: Optional[str] openaev: pyoaev.OpenAEV - @exc.on_http_error(exc.OpenAEVCreateError) + @on_http_error(exc.OpenAEVCreateError) def create( self, data: Optional[Dict[str, Any]] = None, icon: tuple = None, **kwargs: Any ) -> base.RESTObject: @@ -227,7 +228,7 @@ class DeleteMixin(_RestManagerBase): _path: Optional[str] openaev: pyoaev.OpenAEV - @exc.on_http_error(exc.OpenAEVDeleteError) + @on_http_error(exc.OpenAEVDeleteError) def delete( self, id: Optional[Union[str, int]] = None, **kwargs: Any ) -> requests.Response: diff --git a/pyoaev/utils.py b/pyoaev/utils.py index 9fe4586a..42cc1dc8 100644 --- a/pyoaev/utils.py +++ b/pyoaev/utils.py @@ -2,11 +2,14 @@ The shared utilities moved to :mod:`xtm_oaev_sdk`; their legacy names here resolve to the *identical* SDK objects and emit a :class:`DeprecationWarning` -on access. +on access. ``EncodedId`` and ``RequiredOptional`` are REST client helpers and +stay defined here. """ +import dataclasses import email.message -from typing import Any, Callable, Iterator, Optional, Union +import urllib.parse +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union import requests import xtm_oaev_sdk as _sdk @@ -15,9 +18,7 @@ _MOVED = [ "AppLogger", "CustomJsonFormatter", - "EncodedId", "EnhancedJSONEncoder", - "RequiredOptional", "copy_dict", "logger", "remove_none_from_dict", @@ -26,6 +27,8 @@ ] __all__ = [ # noqa: F822 (moved names are served by the PEP 562 __getattr__ below) + "EncodedId", + "RequiredOptional", "get_content_type", "response_content", *_MOVED, @@ -67,6 +70,52 @@ def response_content( return None +class EncodedId(str): + def __new__(cls, value: Union[str, int, "EncodedId"]) -> "EncodedId": + if isinstance(value, EncodedId): + return value + + if not isinstance(value, (int, str)): + raise TypeError(f"Unsupported type received: {type(value)}") + if isinstance(value, str): + value = urllib.parse.quote(value, safe="") + return super().__new__(cls, value) + + +@dataclasses.dataclass(frozen=True) +class RequiredOptional: + required: Tuple[str, ...] = () + optional: Tuple[str, ...] = () + exclusive: Tuple[str, ...] = () + + def validate_attrs( + self, + *, + data: Dict[str, Any], + excludes: Optional[List[str]] = None, + ) -> None: + if excludes is None: + excludes = [] + + if self.required: + required = [k for k in self.required if k not in excludes] + missing = [attr for attr in required if attr not in data] + if missing: + raise AttributeError(f"Missing attributes: {', '.join(missing)}") + + if self.exclusive: + exclusives = [attr for attr in data if attr in self.exclusive] + if len(exclusives) > 1: + raise AttributeError( + f"Provide only one of these attributes: {', '.join(exclusives)}" + ) + if not exclusives: + raise AttributeError( + f"Must provide one of these attributes: " + f"{', '.join(self.exclusive)}" + ) + + __getattr__ = _make_deprecated_getattr( { name: (f"Use 'from xtm_oaev_sdk import {name}' instead.", getattr(_sdk, name)) diff --git a/test/test_rest_client_helpers.py b/test/test_rest_client_helpers.py new file mode 100644 index 00000000..42f2770e --- /dev/null +++ b/test/test_rest_client_helpers.py @@ -0,0 +1,201 @@ +"""REST client helpers owned by pyoaev: RequiredOptional, EncodedId, on_http_error.""" + +import dataclasses +import subprocess +import sys +from typing import Any + +import pytest +import xtm_oaev_sdk as sdk + +from pyoaev.exceptions import on_http_error +from pyoaev.utils import EncodedId, RequiredOptional + + +def _given_required_optional(**kwargs: Any) -> RequiredOptional: + return RequiredOptional(**kwargs) + + +def _given_exclusive_spec() -> RequiredOptional: + return _given_required_optional(exclusive=("id", "key")) + + +def _given_function_raising(error: Exception): + def function() -> None: + raise error + + return function + + +def _when_set_required_optional_field_raises( + instance: RequiredOptional, field_name: str, value: Any +) -> Exception: + with pytest.raises(dataclasses.FrozenInstanceError) as exc_info: + setattr(instance, field_name, value) + return exc_info.value + + +def _when_validate_attrs( + instance: RequiredOptional, data: dict[str, Any], excludes: list[str] | None = None +) -> None: + instance.validate_attrs(data=data, excludes=excludes) + + +def _when_validate_attrs_raises( + instance: RequiredOptional, data: dict[str, Any] +) -> Exception: + with pytest.raises(AttributeError) as exc_info: + _when_validate_attrs(instance, data) + return exc_info.value + + +def _when_encode_id(value: Any) -> EncodedId: + return EncodedId(value) + + +def _when_encode_invalid_id_raises(value: Any) -> Exception: + with pytest.raises(TypeError) as exc_info: + _when_encode_id(value) + return exc_info.value + + +def _when_call_decorated_raises(function, error: type[Exception]) -> Exception: + with pytest.raises(error) as exc_info: + on_http_error(error)(function)() + return exc_info.value + + +def _when_run(script: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, "-W", "error::DeprecationWarning", "-c", script], + capture_output=True, + text=True, + ) + + +def _then_contains_text(exc: Exception, text: str) -> None: + assert text in str(exc) + + +def _then_error_is_frozen(exc: Exception) -> None: + assert isinstance(exc, dataclasses.FrozenInstanceError) + + +def _then_encoded_object_equals(encoded: EncodedId, expected: str) -> None: + assert isinstance(encoded, EncodedId) + assert encoded == expected + + +def test_required_optional_is_frozen() -> None: + """It rejects field assignment on RequiredOptional.""" + required_optional = _given_required_optional(required=("name",)) + exc = _when_set_required_optional_field_raises( + required_optional, "required", ("id",) + ) + _then_error_is_frozen(exc) + + +def test_required_optional_validate_attrs_missing_required_raises() -> None: + """It raises when a required attribute is missing.""" + required_optional = _given_required_optional(required=("name", "type")) + exc = _when_validate_attrs_raises(required_optional, {"name": "sdk"}) + _then_contains_text(exc, "Missing attributes") + + +@pytest.mark.parametrize( + ("data", "expected_text"), + [ + ({}, "Must provide one of these attributes"), + ({"id": "1", "key": "2"}, "Provide only one of these attributes"), + ], +) +def test_required_optional_validate_attrs_exclusive_violations_raise( + data: dict[str, Any], expected_text: str +) -> None: + """It raises when exclusive attributes are both missing or both present.""" + required_optional = _given_exclusive_spec() + exc = _when_validate_attrs_raises(required_optional, data) + _then_contains_text(exc, expected_text) + + +def test_required_optional_validate_attrs_excludes_skip_required_check() -> None: + """It skips the required check for excluded attributes.""" + required_optional = _given_required_optional(required=("name", "type")) + _when_validate_attrs(required_optional, {"name": "sdk"}, excludes=["type"]) + + +def test_required_optional_validate_attrs_valid_data_passes() -> None: + """It accepts data satisfying the required and exclusive rules.""" + required_optional = _given_required_optional( + required=("name",), exclusive=("id", "key") + ) + _when_validate_attrs(required_optional, {"name": "sdk", "id": "123"}) + + +def test_encoded_id_url_encodes_strings() -> None: + """It percent-encodes unsafe string characters.""" + encoded = _when_encode_id("hello world") + _then_encoded_object_equals(encoded, "hello%20world") + + +def test_encoded_id_is_idempotent_when_reencoded() -> None: + """It returns the same value when re-encoding an EncodedId.""" + encoded = _when_encode_id("hello world") + _then_encoded_object_equals(_when_encode_id(encoded), "hello%20world") + + +def test_encoded_id_rejects_non_str_non_int_values() -> None: + """It raises TypeError for unsupported value types.""" + exc = _when_encode_invalid_id_raises(3.14) + _then_contains_text(exc, "Unsupported type received") + + +def test_encoded_id_accepts_int_and_returns_string_representation() -> None: + """It passes integer values through as their string form.""" + encoded = _when_encode_id(42) + _then_encoded_object_equals(encoded, "42") + + +def test_on_http_error_wraps_http_errors_into_the_given_type() -> None: + """It re-raises an HTTP error as the given error, keeping its details.""" + http_error = sdk.OpenAEVHttpError("boom", 404, b"missing") + exc = _when_call_decorated_raises( + _given_function_raising(http_error), sdk.OpenAEVGetError + ) + assert (exc.error_message, exc.response_code, exc.response_body) == ( + "boom", + 404, + b"missing", + ) + assert exc.__cause__ is http_error + + +def test_on_http_error_lets_other_errors_through() -> None: + """It does not touch errors that are not HTTP errors.""" + exc = _when_call_decorated_raises(_given_function_raising(KeyError("k")), KeyError) + _then_contains_text(exc, "k") + + +def test_on_http_error_returns_the_wrapped_result_and_metadata() -> None: + """It returns the decorated function's result and keeps its name.""" + + def answer() -> int: + return 42 + + decorated = on_http_error(sdk.OpenAEVGetError)(answer) + assert decorated() == 42 + assert decorated.__name__ == "answer" + + +def test_rest_client_helpers_are_pyoaev_own_and_silent() -> None: + """It serves the REST client helpers from pyoaev itself, without any warning.""" + result = _when_run( + "import pyoaev.utils, pyoaev.exceptions, xtm_oaev_sdk as sdk\n" + "assert pyoaev.utils.EncodedId.__module__ == 'pyoaev.utils'\n" + "assert pyoaev.utils.RequiredOptional.__module__ == 'pyoaev.utils'\n" + "assert pyoaev.exceptions.on_http_error.__module__ == 'pyoaev.exceptions'\n" + "assert not hasattr(sdk, 'EncodedId')\n" + "assert not hasattr(sdk, 'RequiredOptional')\n" + "assert not hasattr(sdk, 'on_http_error')\n" + ) + assert result.returncode == 0, result.stderr diff --git a/test/test_utils.py b/test/test_utils.py index d1d1d090..610c0a3d 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,4 +1,5 @@ import unittest +from typing import Any, cast from unittest.mock import MagicMock, patch import pyoaev.utils as module @@ -72,6 +73,41 @@ def test_response_content_streamed_defaults_to_stdout_stream(self): mock_print.assert_called_once_with(b"visible") + def test_encoded_id_from_existing_instance_returns_same_object(self): + encoded_id = module.EncodedId("with space") + self.assertIs(module.EncodedId(encoded_id), encoded_id) + + def test_encoded_id_encodes_string_and_keeps_int(self): + self.assertEqual(module.EncodedId("a/b c"), "a%2Fb%20c") + self.assertEqual(module.EncodedId(42), "42") + + def test_encoded_id_rejects_unsupported_type(self): + with self.assertRaises(TypeError): + module.EncodedId(cast(Any, ["bad"])) + + def test_required_optional_required_attribute_missing(self): + rules = module.RequiredOptional(required=("name",)) + with self.assertRaises(AttributeError): + rules.validate_attrs(data={}) + + def test_required_optional_excludes_required_attribute(self): + rules = module.RequiredOptional(required=("name",)) + rules.validate_attrs(data={}, excludes=["name"]) + + def test_required_optional_exclusive_allows_only_one_key(self): + rules = module.RequiredOptional(exclusive=("a", "b")) + with self.assertRaises(AttributeError): + rules.validate_attrs(data={"a": 1, "b": 2}) + + def test_required_optional_exclusive_requires_one_key(self): + rules = module.RequiredOptional(exclusive=("a", "b")) + with self.assertRaises(AttributeError): + rules.validate_attrs(data={}) + + def test_required_optional_exclusive_with_single_key_is_valid(self): + rules = module.RequiredOptional(exclusive=("a", "b")) + rules.validate_attrs(data={"a": 1}) + if __name__ == "__main__": unittest.main() From 11135ec2fbddcb59930b2337c3093a18857d02b5 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 22:56:50 +0200 Subject: [PATCH 18/22] :recycle: refactor(pyoaev): bring copy_dict and remove_none_from_dict back from xtm-oaev-sdk --- libs/xtm-oaev-sdk/README.md | 2 +- libs/xtm-oaev-sdk/docs/migration_guide.md | 6 +-- libs/xtm-oaev-sdk/docs/seam_strategy.md | 2 +- libs/xtm-oaev-sdk/docs/shim_strategy.md | 2 +- .../tdr/every-behaviour-behind-a-port.tdr.md | 2 +- .../test/common/shim/test_shim_contract.py | 5 +- .../test/common/shim/test_utils.py | 11 ----- libs/xtm-oaev-sdk/test/common/test_utils.py | 40 --------------- .../xtm_oaev_sdk/_core/common/__init__.py | 2 - .../xtm_oaev_sdk/_core/common/dicts.py | 20 -------- .../xtm_oaev_sdk/public/__init__.py | 4 -- .../xtm_oaev_sdk/public/common.py | 46 +---------------- .../xtm_oaev_sdk/seam/common/__init__.py | 2 - .../xtm_oaev_sdk/seam/common/ports.py | 12 ----- pyoaev/client.py | 2 +- pyoaev/utils.py | 25 ++++++++-- test/test_rest_client_helpers.py | 49 ++++++++++++++++++- test/test_utils.py | 11 +++++ 18 files changed, 88 insertions(+), 155 deletions(-) delete mode 100644 libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/dicts.py diff --git a/libs/xtm-oaev-sdk/README.md b/libs/xtm-oaev-sdk/README.md index c2399219..964131f0 100644 --- a/libs/xtm-oaev-sdk/README.md +++ b/libs/xtm-oaev-sdk/README.md @@ -23,7 +23,7 @@ implementation of it. | Capability | Public API | |---|---| -| common | `OpenAEVError` and its subclasses, `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, `copy_dict`, `remove_none_from_dict`, SSL helpers, asset and security-domain enums, `BaseClient` / `DaemonProtocol` / `AppLoggerProtocol` | +| common | `OpenAEVError` and its subclasses, `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, SSL helpers, asset and security-domain enums, `BaseClient` / `DaemonProtocol` / `AppLoggerProtocol` | | configuration | `Configuration`, `ConfigurationHint`, `SettingsLoader`, `BaseConfigModel`, `ConfigLoaderOAEV`, `ConfigLoaderCollector`, `ConnectorConfigSchemaGenerator`, sources, serializer types | | contracts | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, every contract element (`ContractText`, `ContractSelect`, …) and enum, credential references (`CredentialType`, `ContractReferencedCredential`, `build_single_referenced_credential_element`) | | signatures | `SignatureManager`, `build_network_configs`, `build_marker`, matching types, the inbound I/O models and the port I/O wire models (`PORT_IO_CONTRACT_VERSION`) | diff --git a/libs/xtm-oaev-sdk/docs/migration_guide.md b/libs/xtm-oaev-sdk/docs/migration_guide.md index d4502b41..5424e848 100644 --- a/libs/xtm-oaev-sdk/docs/migration_guide.md +++ b/libs/xtm-oaev-sdk/docs/migration_guide.md @@ -103,7 +103,6 @@ every legacy module that exposed it. | `ContractTextArea` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractTextArea` | | `ContractTuple` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import ContractTuple` | | `ContractVariable` | `pyoaev.contracts.contract_config`, `pyoaev.contracts.contract_utils`, `pyoaev.contracts.variable_helper` | `from xtm_oaev_sdk import ContractVariable` | -| `copy_dict` | `pyoaev.utils` | `from xtm_oaev_sdk import copy_dict` | | `CredentialType` | `pyoaev.contracts.contract_config`, `pyoaev.credential`, `pyoaev.credential.types`, `pyoaev.credential.utils` | `from xtm_oaev_sdk import CredentialType` | | `CustomJsonFormatter` | `pyoaev.utils` | `from xtm_oaev_sdk import CustomJsonFormatter` | | `data_to_temp_file` | `pyoaev.helpers` | `from xtm_oaev_sdk import data_to_temp_file` | @@ -143,7 +142,6 @@ every legacy module that exposed it. | `prepare_contracts` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import prepare_contracts` | | `PrimitiveType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import PrimitiveType` | | `RedirectError` | `pyoaev`, `pyoaev.exceptions` | `from xtm_oaev_sdk import RedirectError` | -| `remove_none_from_dict` | `pyoaev.utils` | `from xtm_oaev_sdk import remove_none_from_dict` | | `SecurityDomains` | `pyoaev.security_domain.types` | `from xtm_oaev_sdk import SecurityDomains` | | `SecurityPlatformType` | `pyoaev.contracts.contract_config` | `from xtm_oaev_sdk import SecurityPlatformType` | | `SettingsLoader` | `pyoaev`, `pyoaev.configuration`, `pyoaev.configuration.settings_loader` | `from xtm_oaev_sdk import SettingsLoader` | @@ -292,8 +290,8 @@ variables of `xtm_oaev_sdk.VariableHelper` instead. The API client, and everything wired around it: `OpenAEV`, `pyoaev.apis` (including `inputs.search`), `pyoaev.backends`, `pyoaev.base`, `pyoaev.client`, `pyoaev.mixins`, and the REST client helpers `pyoaev.utils.get_content_type` / `response_content` / -`EncodedId` / `RequiredOptional`, plus `pyoaev.exceptions.on_http_error`. Nothing to -migrate there. +`EncodedId` / `RequiredOptional` / `copy_dict` / `remove_none_from_dict`, plus +`pyoaev.exceptions.on_http_error`. Nothing to migrate there. ## Things you may notice diff --git a/libs/xtm-oaev-sdk/docs/seam_strategy.md b/libs/xtm-oaev-sdk/docs/seam_strategy.md index e0e2e4f6..f90f4d48 100644 --- a/libs/xtm-oaev-sdk/docs/seam_strategy.md +++ b/libs/xtm-oaev-sdk/docs/seam_strategy.md @@ -199,7 +199,7 @@ source selection to `SettingsSourcesPort`. The settings models (`BaseConfigModel | Capability | Seam: contracts | Seam: ports | Core | Public adapters | |---|---|---|---|---| -| common | the error hierarchy, the asset and security-domain enums, `BaseClient`, `DaemonProtocol`, `AppLoggerProtocol` | `JsonEncodingPort`, `LoggingPort`, `DictToolsPort`, `CertificatesPort` | `JsonEncoding`, `LoggingSetup`, `DictTools`, `Certificates` | `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, `copy_dict`, `remove_none_from_dict`, the SSL helpers | +| common | the error hierarchy, the asset and security-domain enums, `BaseClient`, `DaemonProtocol`, `AppLoggerProtocol` | `JsonEncodingPort`, `LoggingPort`, `CertificatesPort` | `JsonEncoding`, `LoggingSetup`, `Certificates` | `AppLogger`, `logger`, `setup_logging_config`, `CustomJsonFormatter`, `EnhancedJSONEncoder`, the SSL helpers | | configuration | `ConfigurationHint`, settings models, serializer types, `CONFIGURATION_TYPES`, `ConfigurationProtocol` | `ConfigurationResolverPort`, `ConfigurationSourcesPort`, `ConnectorSchemaPort`, `SettingsSourcesPort`, `TruthinessPort` | `ConfigurationResolver`, `ConfigurationSources`, `ConnectorSchemaTransformer`, `SettingsSourceSelector`, `Truthiness` | `Configuration`, `EnvironmentSource`, `DictionarySource`, `ConnectorConfigSchemaGenerator`, `SettingsLoader`, `is_truthy`, `is_falsy` | | contracts | `ContractData`, contract elements and enums, `ContractVariable`, `Expectation`, `CredentialType`, `ContractBuilderProtocol` | `ContractAssemblyPort`, `ContractVariablesPort`, `ContractSerializerPort`, `ContractCredentialsPort` | `ContractAssembly`, `BuiltinContractVariables`, `ContractSerializer`, `CredentialReferences` | `Contract`, `ContractBuilder`, `VariableHelper`, `prepare_contracts`, `build_single_referenced_credential_element` | | signatures | inbound I/O models, matching types and policies, `ExtraSignatureData`, `SignatureTransportProtocol`, `SignatureManagerProtocol` | `SignatureEnginePort`, `ContainerAddressPort`, `NetworkTargetsPort`, `MarkerPort` | `SignatureEngine`, `ContainerAddressResolver`, `NetworkTargets`, `MarkerBuilder` | `SignatureManager`, `build_network_configs`, `build_marker` | diff --git a/libs/xtm-oaev-sdk/docs/shim_strategy.md b/libs/xtm-oaev-sdk/docs/shim_strategy.md index 4c7972a8..fd272b02 100644 --- a/libs/xtm-oaev-sdk/docs/shim_strategy.md +++ b/libs/xtm-oaev-sdk/docs/shim_strategy.md @@ -152,7 +152,7 @@ so patching a method on any of those paths hits the class connectors instantiate | `pyoaev/credential/*` | xtm-oaev-sdk contracts (credential references) | | `pyoaev/signatures/*` | xtm-oaev-sdk signatures (`ExpectationType` renamed, `SignatureManager` adapter) | | `pyoaev/exceptions.py`, `asset_types.py`, `security_domain/types.py` | xtm-oaev-sdk common; `exceptions.py` keeps the REST client decorator `on_http_error` | -| `pyoaev/utils.py` | xtm-oaev-sdk common and `PingAlive`; keeps the REST client helpers `get_content_type`, `response_content`, `EncodedId`, `RequiredOptional` | +| `pyoaev/utils.py` | xtm-oaev-sdk common and `PingAlive`; keeps the REST client helpers `get_content_type`, `response_content`, `EncodedId`, `RequiredOptional`, `copy_dict`, `remove_none_from_dict` | | `pyoaev/daemons/*` | `BaseDaemon` / `CollectorDaemon` adapters | | `pyoaev/helpers.py` | collectors-sdk detection, injectors-sdk helpers (`OpenAEVInjectorHelper` adapter), xtm SSL helpers, `OpenAEVCollectorHelper` | | `pyoaev/apis/inject_expectation/model/*` | collectors-sdk expectations (the API manager builds them) | diff --git a/libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md index 7ee3fe6b..7e9c3713 100644 --- a/libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md +++ b/libs/xtm-oaev-sdk/docs/tdr/every-behaviour-behind-a-port.tdr.md @@ -38,7 +38,7 @@ rule applies. | *How this TDR manifests in the real world.* `is_truthy` gets `TruthinessPort`, `Truthiness` and the public `is_truthy`: exactly the -treatment `SignatureManager` gets. `copy_dict` goes through `DictToolsPort` / `DictTools`. +treatment `SignatureManager` gets. `build_marker` goes through `MarkerPort` / `MarkerBuilder`. --- diff --git a/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py b/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py index 015b5c1e..116a8644 100644 --- a/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py +++ b/libs/xtm-oaev-sdk/test/common/shim/test_shim_contract.py @@ -31,9 +31,7 @@ "AppLogger", "CustomJsonFormatter", "EnhancedJSONEncoder", - "copy_dict", "logger", - "remove_none_from_dict", "setup_logging_config", ] _SSL = [ @@ -126,8 +124,7 @@ def test_sdk_objects_stay_silent_once_pyoaev_is_loaded() -> None: script = ( "import pyoaev, pyoaev.exceptions, pyoaev.utils, pyoaev.helpers\n" "import xtm_oaev_sdk as sdk\n" - "sdk.copy_dict(src={'a': {'b': 1}}, dest={})\n" - "sdk.remove_none_from_dict({'a': None})\n" + "sdk.EnhancedJSONEncoder().encode({'a': 1})\n" "sdk.AssetCategory('WEB_APPLICATION')\n" "try:\n" " raise sdk.OpenAEVHttpError('boom', 500)\n" diff --git a/libs/xtm-oaev-sdk/test/common/shim/test_utils.py b/libs/xtm-oaev-sdk/test/common/shim/test_utils.py index 5a90347c..7c00bfe1 100644 --- a/libs/xtm-oaev-sdk/test/common/shim/test_utils.py +++ b/libs/xtm-oaev-sdk/test/common/shim/test_utils.py @@ -18,17 +18,6 @@ class TestUtils(unittest.TestCase): def test_custom_json_formatter_inherits_non_deprecated_formatter(self): self.assertTrue(issubclass(module.CustomJsonFormatter, JsonFormatter)) - def test_copy_dict_flattens_nested_dict(self): - destination = {} - module.copy_dict( - src={"a": 1, "meta": {"x": "y", "n": 2}}, - dest=destination, - ) - self.assertEqual(destination, {"a": 1, "meta[x]": "y", "meta[n]": 2}) - - def test_remove_none_from_dict(self): - self.assertEqual(module.remove_none_from_dict({"a": 1, "b": None}), {"a": 1}) - def test_enhanced_json_encoder_serializes_not_dataclasses(self): data = {"test": "this is a test", "other": 3} classic_json_dump = json.dumps(data) diff --git a/libs/xtm-oaev-sdk/test/common/test_utils.py b/libs/xtm-oaev-sdk/test/common/test_utils.py index a0ffb99a..df2c7909 100644 --- a/libs/xtm-oaev-sdk/test/common/test_utils.py +++ b/libs/xtm-oaev-sdk/test/common/test_utils.py @@ -91,16 +91,6 @@ def _when_encode_unsupported_type_raises(encoder: Any, value: Any) -> Exception: return exc_info.value -def _when_copy_dict(src: dict[str, Any], dest: dict[str, Any]) -> None: - copy_dict_fn = _given_public_symbol("copy_dict") - copy_dict_fn(src=src, dest=dest) - - -def _when_remove_none_from_dict(data: dict[str, Any]) -> dict[str, Any]: - remove_none_fn = _given_public_symbol("remove_none_from_dict") - return remove_none_fn(data) - - def _when_call_logger(level: int, json_logging: bool) -> Any: logger_fn = _given_public_symbol("logger") with warnings.catch_warnings(): @@ -152,10 +142,6 @@ def _then_type_error_for_unsupported_type(exc: Exception) -> None: assert "not json serializable" in str(exc).lower() -def _then_dict_equals(actual: dict[str, Any], expected: dict[str, Any]) -> None: - assert actual == expected - - def _then_is_app_logger_instance(instance: Any) -> None: app_logger_cls = _given_public_symbol("AppLogger") assert isinstance(instance, app_logger_cls) @@ -215,32 +201,6 @@ def test_enhanced_json_encoder_raises_type_error_for_unsupported_types() -> None _then_type_error_for_unsupported_type(exc) -def test_copy_dict_flattens_nested_dict_keys() -> None: - """Verify copy_dict flattens nested dict keys to bracket notation.""" - destination: dict[str, Any] = {} - _when_copy_dict(src={"filters": {"type": "malware"}}, dest=destination) - _then_dict_equals(destination, {"filters[type]": "malware"}) - - -def test_copy_dict_copies_non_dict_values_directly() -> None: - """Verify copy_dict copies non-dict values without transformation.""" - destination: dict[str, Any] = {} - _when_copy_dict(src={"name": "sdk", "count": 2}, dest=destination) - _then_dict_equals(destination, {"name": "sdk", "count": 2}) - - -def test_remove_none_from_dict_strips_none_valued_keys() -> None: - """Verify remove_none_from_dict removes entries with None values.""" - result = _when_remove_none_from_dict({"a": 1, "b": None, "c": "x"}) - _then_dict_equals(result, {"a": 1, "c": "x"}) - - -def test_remove_none_from_dict_keeps_non_none_values() -> None: - """Verify remove_none_from_dict keeps entries with non-None values.""" - result = _when_remove_none_from_dict({"a": 0, "b": False, "c": ""}) - _then_dict_equals(result, {"a": 0, "b": False, "c": ""}) - - def test_logger_function_returns_app_logger() -> None: """Verify logger() returns an AppLogger without warning.""" logger_instance = _when_call_logger(logging.INFO, json_logging=False) diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py index 5c927d9a..fcc3faa4 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/__init__.py @@ -1,13 +1,11 @@ """Common core: implementations of the common seam ports.""" from .certificates import Certificates -from .dicts import DictTools from .encoding import JsonEncoding from .logging_setup import LoggingSetup __all__ = [ "Certificates", - "DictTools", "JsonEncoding", "LoggingSetup", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/dicts.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/dicts.py deleted file mode 100644 index 12e87a0c..00000000 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/_core/common/dicts.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Dict tools: reshape the dictionaries exchanged with the platform.""" - -from typing import Any - -__all__ = ["DictTools"] - - -class DictTools: - """Implements ``DictToolsPort``.""" - - def copy_dict(self, *, src: dict[str, Any], dest: dict[str, Any]) -> None: - for k, v in src.items(): - if isinstance(v, dict): - for dict_k, dict_v in v.items(): - dest[f"{k}[{dict_k}]"] = dict_v - else: - dest[k] = v - - def remove_none_from_dict(self, data: dict[str, Any]) -> dict[str, Any]: - return {k: v for k, v in data.items() if v is not None} diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py index daaa9056..416842aa 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/__init__.py @@ -28,11 +28,9 @@ RedirectError, SecurityDomains, SignatureTransmissionError, - copy_dict, data_to_temp_file, is_memory_certificate, logger, - remove_none_from_dict, setup_logging_config, ssl_cert_chain, ssl_verify_locations, @@ -163,9 +161,7 @@ "AppLoggerProtocol", "CustomJsonFormatter", "EnhancedJSONEncoder", - "copy_dict", "logger", - "remove_none_from_dict", "setup_logging_config", # Common: protocols "BaseClient", diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py index 52767b21..7d2df976 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/public/common.py @@ -12,12 +12,7 @@ from typing import Any from pythonjsonlogger.json import JsonFormatter -from xtm_oaev_sdk._core.common import ( - Certificates, - DictTools, - JsonEncoding, - LoggingSetup, -) +from xtm_oaev_sdk._core.common import Certificates, JsonEncoding, LoggingSetup from xtm_oaev_sdk.seam.common import ( AppLoggerProtocol, AssetCategory, @@ -28,7 +23,6 @@ CloudProvider, ConfigurationError, DaemonProtocol, - DictToolsPort, JsonEncodingPort, LoggingPort, OpenAEVAuthenticationError, @@ -71,11 +65,9 @@ "RedirectError", "SecurityDomains", "SignatureTransmissionError", - "copy_dict", "data_to_temp_file", "is_memory_certificate", "logger", - "remove_none_from_dict", "setup_logging_config", "ssl_cert_chain", "ssl_verify_locations", @@ -83,7 +75,6 @@ _encoding: JsonEncodingPort = JsonEncoding() _logging: LoggingPort = LoggingSetup() -_dicts: DictToolsPort = DictTools() _certificates: CertificatesPort = Certificates() @@ -117,41 +108,6 @@ def default(self, o: Any) -> Any: return _encoding.encode_default(o) -def remove_none_from_dict(data: dict[str, Any]) -> dict[str, Any]: - """Remove all keys with None values from a dict. - - Args: - data: Source dictionary. - - Returns: - New dictionary with None-valued entries removed. - - Example: - >>> remove_none_from_dict({"a": 1, "b": None, "c": "x"}) - {'a': 1, 'c': 'x'} - """ - return _dicts.remove_none_from_dict(data) - - -def copy_dict(*, src: dict[str, Any], dest: dict[str, Any]) -> None: - """Flatten nested dict values into bracket-notation keys. - - For nested dicts, expands ``{"key": {"sub": "val"}}`` into - ``dest["key[sub]"] = "val"``. Non-dict values copy directly. - - Args: - src: Source dictionary to copy from. - dest: Destination dictionary to copy into (mutated in place). - - Example: - >>> dest = {} - >>> copy_dict(src={"filters": {"type": "malware"}}, dest=dest) - >>> dest - {'filters[type]': 'malware'} - """ - _dicts.copy_dict(src=src, dest=dest) - - class CustomJsonFormatter(JsonFormatter): """JSON log formatter with ISO timestamp and uppercase level. diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py index 6d798bc9..91ddf9d3 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/__init__.py @@ -29,7 +29,6 @@ ) from xtm_oaev_sdk.seam.common.ports import ( CertificatesPort, - DictToolsPort, JsonEncodingPort, LoggingPort, ) @@ -72,7 +71,6 @@ "DaemonProtocol", # Ports "CertificatesPort", - "DictToolsPort", "JsonEncodingPort", "LoggingPort", ] diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py index 82576fbd..5ff10fe9 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/common/ports.py @@ -6,7 +6,6 @@ __all__ = [ "CertificatesPort", - "DictToolsPort", "JsonEncodingPort", "LoggingPort", ] @@ -47,17 +46,6 @@ def complete_record( """Add the timestamp and uppercase level fields of a JSON record.""" -@runtime_checkable -class DictToolsPort(Protocol): - """Reshapes the dictionaries exchanged with the platform.""" - - def copy_dict(self, *, src: dict[str, Any], dest: dict[str, Any]) -> None: - """Copy ``src`` into ``dest``, flattening nested dicts as ``key[sub]``.""" - - def remove_none_from_dict(self, data: dict[str, Any]) -> dict[str, Any]: - """New dict without the None-valued entries.""" - - @runtime_checkable class CertificatesPort(Protocol): """Loads certificates, files or in-memory PEM data, into SSL contexts.""" diff --git a/pyoaev/client.py b/pyoaev/client.py index b02e3a58..5c1b3dd2 100644 --- a/pyoaev/client.py +++ b/pyoaev/client.py @@ -4,10 +4,10 @@ import requests import xtm_oaev_sdk as exceptions -from xtm_oaev_sdk import copy_dict from pyoaev import utils from pyoaev._version import __version__ # noqa: F401 +from pyoaev.utils import copy_dict REDIRECT_MSG = ( "pyoaev detected a {status_code} ({reason!r}) redirection. You must update " diff --git a/pyoaev/utils.py b/pyoaev/utils.py index 42cc1dc8..a9177928 100644 --- a/pyoaev/utils.py +++ b/pyoaev/utils.py @@ -2,8 +2,8 @@ The shared utilities moved to :mod:`xtm_oaev_sdk`; their legacy names here resolve to the *identical* SDK objects and emit a :class:`DeprecationWarning` -on access. ``EncodedId`` and ``RequiredOptional`` are REST client helpers and -stay defined here. +on access. ``EncodedId``, ``RequiredOptional``, ``copy_dict`` and +``remove_none_from_dict`` are REST client helpers and stay defined here. """ import dataclasses @@ -19,9 +19,7 @@ "AppLogger", "CustomJsonFormatter", "EnhancedJSONEncoder", - "copy_dict", "logger", - "remove_none_from_dict", "setup_logging_config", "PingAlive", ] @@ -29,7 +27,9 @@ __all__ = [ # noqa: F822 (moved names are served by the PEP 562 __getattr__ below) "EncodedId", "RequiredOptional", + "copy_dict", "get_content_type", + "remove_none_from_dict", "response_content", *_MOVED, ] @@ -82,6 +82,23 @@ def __new__(cls, value: Union[str, int, "EncodedId"]) -> "EncodedId": return super().__new__(cls, value) +def copy_dict( + *, + src: Dict[str, Any], + dest: Dict[str, Any], +) -> None: + for k, v in src.items(): + if isinstance(v, dict): + for dict_k, dict_v in v.items(): + dest[f"{k}[{dict_k}]"] = dict_v + else: + dest[k] = v + + +def remove_none_from_dict(data: Dict[str, Any]) -> Dict[str, Any]: + return {k: v for k, v in data.items() if v is not None} + + @dataclasses.dataclass(frozen=True) class RequiredOptional: required: Tuple[str, ...] = () diff --git a/test/test_rest_client_helpers.py b/test/test_rest_client_helpers.py index 42f2770e..c9775674 100644 --- a/test/test_rest_client_helpers.py +++ b/test/test_rest_client_helpers.py @@ -1,4 +1,7 @@ -"""REST client helpers owned by pyoaev: RequiredOptional, EncodedId, on_http_error.""" +"""REST client helpers owned by pyoaev. + +RequiredOptional, EncodedId, copy_dict, remove_none_from_dict and on_http_error. +""" import dataclasses import subprocess @@ -9,7 +12,7 @@ import xtm_oaev_sdk as sdk from pyoaev.exceptions import on_http_error -from pyoaev.utils import EncodedId, RequiredOptional +from pyoaev.utils import EncodedId, RequiredOptional, copy_dict, remove_none_from_dict def _given_required_optional(**kwargs: Any) -> RequiredOptional: @@ -59,6 +62,14 @@ def _when_encode_invalid_id_raises(value: Any) -> Exception: return exc_info.value +def _when_copy_dict(src: dict[str, Any], dest: dict[str, Any]) -> None: + copy_dict(src=src, dest=dest) + + +def _when_remove_none_from_dict(data: dict[str, Any]) -> dict[str, Any]: + return remove_none_from_dict(data) + + def _when_call_decorated_raises(function, error: type[Exception]) -> Exception: with pytest.raises(error) as exc_info: on_http_error(error)(function)() @@ -81,6 +92,10 @@ def _then_error_is_frozen(exc: Exception) -> None: assert isinstance(exc, dataclasses.FrozenInstanceError) +def _then_dict_equals(actual: dict[str, Any], expected: dict[str, Any]) -> None: + assert actual == expected + + def _then_encoded_object_equals(encoded: EncodedId, expected: str) -> None: assert isinstance(encoded, EncodedId) assert encoded == expected @@ -156,6 +171,32 @@ def test_encoded_id_accepts_int_and_returns_string_representation() -> None: _then_encoded_object_equals(encoded, "42") +def test_copy_dict_flattens_nested_dict_keys() -> None: + """It flattens nested dict keys to bracket notation.""" + destination: dict[str, Any] = {} + _when_copy_dict(src={"filters": {"type": "malware"}}, dest=destination) + _then_dict_equals(destination, {"filters[type]": "malware"}) + + +def test_copy_dict_copies_non_dict_values_directly() -> None: + """It copies non-dict values without transformation.""" + destination: dict[str, Any] = {} + _when_copy_dict(src={"name": "sdk", "count": 2}, dest=destination) + _then_dict_equals(destination, {"name": "sdk", "count": 2}) + + +def test_remove_none_from_dict_strips_none_valued_keys() -> None: + """It removes the entries whose value is None.""" + result = _when_remove_none_from_dict({"a": 1, "b": None, "c": "x"}) + _then_dict_equals(result, {"a": 1, "c": "x"}) + + +def test_remove_none_from_dict_keeps_non_none_values() -> None: + """It keeps falsy values that are not None.""" + result = _when_remove_none_from_dict({"a": 0, "b": False, "c": ""}) + _then_dict_equals(result, {"a": 0, "b": False, "c": ""}) + + def test_on_http_error_wraps_http_errors_into_the_given_type() -> None: """It re-raises an HTTP error as the given error, keeping its details.""" http_error = sdk.OpenAEVHttpError("boom", 404, b"missing") @@ -193,9 +234,13 @@ def test_rest_client_helpers_are_pyoaev_own_and_silent() -> None: "import pyoaev.utils, pyoaev.exceptions, xtm_oaev_sdk as sdk\n" "assert pyoaev.utils.EncodedId.__module__ == 'pyoaev.utils'\n" "assert pyoaev.utils.RequiredOptional.__module__ == 'pyoaev.utils'\n" + "assert pyoaev.utils.copy_dict.__module__ == 'pyoaev.utils'\n" + "assert pyoaev.utils.remove_none_from_dict.__module__ == 'pyoaev.utils'\n" "assert pyoaev.exceptions.on_http_error.__module__ == 'pyoaev.exceptions'\n" "assert not hasattr(sdk, 'EncodedId')\n" "assert not hasattr(sdk, 'RequiredOptional')\n" "assert not hasattr(sdk, 'on_http_error')\n" + "assert not hasattr(sdk, 'copy_dict')\n" + "assert not hasattr(sdk, 'remove_none_from_dict')\n" ) assert result.returncode == 0, result.stderr diff --git a/test/test_utils.py b/test/test_utils.py index 610c0a3d..3f290703 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -108,6 +108,17 @@ def test_required_optional_exclusive_with_single_key_is_valid(self): rules = module.RequiredOptional(exclusive=("a", "b")) rules.validate_attrs(data={"a": 1}) + def test_copy_dict_flattens_nested_dict(self): + destination = {} + module.copy_dict( + src={"a": 1, "meta": {"x": "y", "n": 2}}, + dest=destination, + ) + self.assertEqual(destination, {"a": 1, "meta[x]": "y", "meta[n]": 2}) + + def test_remove_none_from_dict(self): + self.assertEqual(module.remove_none_from_dict({"a": 1, "b": None}), {"a": 1}) + if __name__ == "__main__": unittest.main() From e818566bda8e9e1aa21a4cf66eecbec710df813b Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 23:14:12 +0200 Subject: [PATCH 19/22] :recycle: refactor(pyoaev): delegate the expectation models to collectors-sdk --- libs/xtm-oaev-sdk/docs/migration_guide.md | 24 ++++++- libs/xtm-oaev-sdk/docs/seam_strategy.md | 2 +- libs/xtm-oaev-sdk/docs/shim_strategy.md | 5 +- pyoaev/apis/__init__.py | 23 ++++++- pyoaev/apis/inject_expectation/__init__.py | 20 ++++++ .../inject_expectation/inject_expectation.py | 63 ++++++++++--------- test/test_sdk_split_shims.py | 55 +++++++++++++++- 7 files changed, 151 insertions(+), 41 deletions(-) diff --git a/libs/xtm-oaev-sdk/docs/migration_guide.md b/libs/xtm-oaev-sdk/docs/migration_guide.md index 5424e848..9c84d83f 100644 --- a/libs/xtm-oaev-sdk/docs/migration_guide.md +++ b/libs/xtm-oaev-sdk/docs/migration_guide.md @@ -174,13 +174,13 @@ every legacy module that exposed it. | `CollectorDaemon` | `pyoaev.daemons`, `pyoaev.daemons.collector_daemon`, `pyoaev.helpers` | `from collectors_sdk import CollectorDaemon` | | `DEFAULT_PERIOD_SECONDS` | `pyoaev.daemons.collector_daemon` | `from collectors_sdk import DEFAULT_PERIOD_SECONDS` | | `DEFAULT_PLATFORM_TAG_COLOR` | `pyoaev.daemons.collector_daemon` | `from collectors_sdk import DEFAULT_PLATFORM_TAG_COLOR` | -| `DetectionExpectation` | `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import DetectionExpectation` | +| `DetectionExpectation` | `pyoaev.apis`, `pyoaev.apis.inject_expectation`, `pyoaev.apis.inject_expectation.inject_expectation`, `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import DetectionExpectation` | | `Expectation` | `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import Expectation` | | `ExpectationSignature` | `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import ExpectationSignature` | -| `ExpectationTypeEnum` | `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import ExpectationTypeEnum` | +| `ExpectationTypeEnum` | `pyoaev.apis`, `pyoaev.apis.inject_expectation`, `pyoaev.apis.inject_expectation.inject_expectation`, `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import ExpectationTypeEnum` | | `OpenAEVCollectorHelper` | `pyoaev.helpers` | `from collectors_sdk import CollectorDaemon` | | `OpenAEVDetectionHelper` | `pyoaev.helpers` | `from collectors_sdk import OpenAEVDetectionHelper` | -| `PreventionExpectation` | `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import PreventionExpectation` | +| `PreventionExpectation` | `pyoaev.apis`, `pyoaev.apis.inject_expectation`, `pyoaev.apis.inject_expectation.inject_expectation`, `pyoaev.apis.inject_expectation.model`, `pyoaev.apis.inject_expectation.model.expectation` | `from collectors_sdk import PreventionExpectation` | ### To injectors-sdk @@ -272,6 +272,24 @@ from xtm_oaev_sdk import SignatureManager manager = SignatureManager(transport=client.signature, logger=logger) ``` +Collector expectations: + +```python +# before +for expectation in api.inject_expectation.expectations_models_for_source(source_id=source_id): + ... + +# after +from collectors_sdk import expectations_models_for_source + +for expectation in expectations_models_for_source(api.inject_expectation, source_id=source_id): + ... +``` + +The client fetches the expectation data. collectors-sdk turns it into +`DetectionExpectation` / `PreventionExpectation` models. The former method still works, +firing a `DeprecationWarning`. + ## Renamed names | Legacy | New | diff --git a/libs/xtm-oaev-sdk/docs/seam_strategy.md b/libs/xtm-oaev-sdk/docs/seam_strategy.md index f90f4d48..610afcbc 100644 --- a/libs/xtm-oaev-sdk/docs/seam_strategy.md +++ b/libs/xtm-oaev-sdk/docs/seam_strategy.md @@ -213,7 +213,7 @@ source selection to `SettingsSourcesPort`. The settings models (`BaseConfigModel |---|---|---|---|---| | daemons | `DEFAULT_PERIOD_SECONDS`, `DEFAULT_PLATFORM_TAG_COLOR` | `CollectorRegistrationPort`, `CollectorSchedulePort` | `CollectorRegistration`, `CollectorSchedule` | `CollectorDaemon` | | detection | | `AlertMatchingPort` | `AlertMatcher` | `OpenAEVDetectionHelper` | -| expectations | `ExpectationData`, `ExpectationSignature`, `ExpectationTypeEnum` | `ExpectationMatchingPort`, `ExpectationUpdatePort` | `ExpectationMatcher`, `ExpectationUpdates` | `Expectation`, `DetectionExpectation`, `PreventionExpectation` | +| expectations | `ExpectationData`, `ExpectationSignature`, `ExpectationTypeEnum`, `ExpectationApiProtocol` | `ExpectationLoadingPort`, `ExpectationMatchingPort`, `ExpectationUpdatePort` | `ExpectationLoader`, `ExpectationMatcher`, `ExpectationUpdates` | `Expectation`, `DetectionExpectation`, `PreventionExpectation`, `expectations_models_for_source` | ### injectors-sdk diff --git a/libs/xtm-oaev-sdk/docs/shim_strategy.md b/libs/xtm-oaev-sdk/docs/shim_strategy.md index fd272b02..300c87f7 100644 --- a/libs/xtm-oaev-sdk/docs/shim_strategy.md +++ b/libs/xtm-oaev-sdk/docs/shim_strategy.md @@ -155,7 +155,7 @@ so patching a method on any of those paths hits the class connectors instantiate | `pyoaev/utils.py` | xtm-oaev-sdk common and `PingAlive`; keeps the REST client helpers `get_content_type`, `response_content`, `EncodedId`, `RequiredOptional`, `copy_dict`, `remove_none_from_dict` | | `pyoaev/daemons/*` | `BaseDaemon` / `CollectorDaemon` adapters | | `pyoaev/helpers.py` | collectors-sdk detection, injectors-sdk helpers (`OpenAEVInjectorHelper` adapter), xtm SSL helpers, `OpenAEVCollectorHelper` | -| `pyoaev/apis/inject_expectation/model/*` | collectors-sdk expectations (the API manager builds them) | +| `pyoaev/apis/__init__.py`, `apis/inject_expectation/*` | collectors-sdk expectations; `InjectExpectationManager.expectations_models_for_source` delegates to `collectors_sdk.expectations_models_for_source`, with a warning | ## How it is tested @@ -182,7 +182,8 @@ against the new stack. Identical results. In a major release of `pyoaev`: 1. delete the shim modules and the `__getattr__` of `pyoaev/__init__.py`; -2. delete the client-building adapters and `OpenAEVCollectorHelper`; +2. delete the client-building adapters, `OpenAEVCollectorHelper` and + `InjectExpectationManager.expectations_models_for_source`; 3. drop the `xtm-oaev-sdk` / `collectors-sdk` / `injectors-sdk` dependencies only the shims need (keep `xtm-oaev-sdk` if the client still leans on its errors and transport models); diff --git a/pyoaev/apis/__init__.py b/pyoaev/apis/__init__.py index 67bea1d6..d8cd3342 100644 --- a/pyoaev/apis/__init__.py +++ b/pyoaev/apis/__init__.py @@ -1,3 +1,6 @@ +import collectors_sdk as _collectors_sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + from .ai_target import * # noqa: F401,F403 from .attack_pattern import * # noqa: F401,F403 from .collector import * # noqa: F401,F403 @@ -20,4 +23,22 @@ from .user import * # noqa: F401,F403 from .vulnerability import * # noqa: F401,F403 -__all__ = [name for name in dir() if not name.startswith("_")] +# The expectation models moved to collectors_sdk; their legacy names stay in +# ``__all__`` and are served by the PEP 562 __getattr__ below, with a warning. +_MOVED_MODELS = ["DetectionExpectation", "ExpectationTypeEnum", "PreventionExpectation"] + +__all__ = [ # noqa: F822 (moved names are served by the PEP 562 __getattr__ below) + *[name for name in dir() if not name.startswith("_")], + *_MOVED_MODELS, +] + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in _MOVED_MODELS + }, + module_name=__name__, +) diff --git a/pyoaev/apis/inject_expectation/__init__.py b/pyoaev/apis/inject_expectation/__init__.py index 2bbeaa66..8f08ea7e 100644 --- a/pyoaev/apis/inject_expectation/__init__.py +++ b/pyoaev/apis/inject_expectation/__init__.py @@ -1,2 +1,22 @@ +import collectors_sdk as _collectors_sdk +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr + from . import model # noqa: F401 (legacy: exposed as pyoaev.apis.model) from .inject_expectation import * # noqa: F401,F403 + +# The expectation models moved to collectors_sdk; their legacy names here +# resolve to the identical objects and emit a DeprecationWarning on access. +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in ( + "DetectionExpectation", + "ExpectationTypeEnum", + "PreventionExpectation", + ) + }, + module_name=__name__, +) diff --git a/pyoaev/apis/inject_expectation/inject_expectation.py b/pyoaev/apis/inject_expectation/inject_expectation.py index 8fd63c85..4193fa13 100644 --- a/pyoaev/apis/inject_expectation/inject_expectation.py +++ b/pyoaev/apis/inject_expectation/inject_expectation.py @@ -1,11 +1,9 @@ +import warnings as _warnings from typing import Any, Dict, List +import collectors_sdk as _collectors_sdk import xtm_oaev_sdk as exc -from collectors_sdk import ( - DetectionExpectation, - ExpectationTypeEnum, - PreventionExpectation, -) +from xtm_oaev_sdk import make_deprecated_getattr as _make_deprecated_getattr from pyoaev.base import RESTManager, RESTObject from pyoaev.exceptions import on_http_error @@ -88,33 +86,20 @@ def expectations_models_for_source(self, source_id: str, **kwargs: Any): :return: a list of expectation objects :rtype: list[DetectionExpectation|PreventionExpectation] + + .. deprecated:: + Building the expectation models is the collector runtime's job: use + ``collectors_sdk.expectations_models_for_source(api.inject_expectation, + source_id)`` instead. """ - # TODO: we should implement a more clever mechanism to obtain - # specialised Expectation instances rather than just if/elseing - # through this list of possibilities. - expectations = [] - for expectation_dict in self.expectations_assets_for_source( - source_id=source_id, **kwargs - ): - if ( - expectation_dict["inject_expectation_type"] - == ExpectationTypeEnum.Detection.value - ): - expectations.append( - DetectionExpectation(**expectation_dict, api_client=self) - ) - elif ( - expectation_dict["inject_expectation_type"] - == ExpectationTypeEnum.Prevention.value - ): - expectations.append( - PreventionExpectation(**expectation_dict, api_client=self) - ) - else: - expectations.append( - PreventionExpectation(**expectation_dict, api_client=self) - ) - return expectations + _warnings.warn( + "expectations_models_for_source is deprecated. Use 'from collectors_sdk " + "import expectations_models_for_source' and call " + "expectations_models_for_source(api.inject_expectation, source_id) instead.", + DeprecationWarning, + stacklevel=2, + ) + return _collectors_sdk.expectations_models_for_source(self, source_id, **kwargs) @on_http_error(exc.OpenAEVUpdateError) def prevention_expectations_for_source( @@ -159,3 +144,19 @@ def bulk_update( self.openaev.http_put( path, post_data={"inputs": inject_expectation_input_by_id}, **kwargs ) + + +# The expectation models moved to collectors_sdk; their legacy names here +# resolve to the identical objects and emit a DeprecationWarning on access. +_MOVED_MODELS = ["DetectionExpectation", "ExpectationTypeEnum", "PreventionExpectation"] + +__getattr__ = _make_deprecated_getattr( + { + name: ( + f"Use 'from collectors_sdk import {name}' instead.", + getattr(_collectors_sdk, name), + ) + for name in _MOVED_MODELS + }, + module_name=__name__, +) diff --git a/test/test_sdk_split_shims.py b/test/test_sdk_split_shims.py index d5e1633a..06747c8c 100644 --- a/test/test_sdk_split_shims.py +++ b/test/test_sdk_split_shims.py @@ -38,6 +38,18 @@ "ExpectationSignature", collectors_sdk, ), + *[ + (module, name, collectors_sdk) + for module in ( + "pyoaev.apis.inject_expectation", + "pyoaev.apis.inject_expectation.inject_expectation", + ) + for name in ( + "DetectionExpectation", + "ExpectationTypeEnum", + "PreventionExpectation", + ) + ], ] @@ -162,8 +174,7 @@ def test_legacy_collector_helper_still_resolves() -> None: assert callable(legacy) -def test_expectation_manager_builds_sdk_models() -> None: - """It hands out collectors-sdk expectation models.""" +def _given_expectation_manager(): from pyoaev.apis import InjectExpectationManager manager = InjectExpectationManager(MagicMock()) @@ -176,10 +187,48 @@ def test_expectation_manager_builds_sdk_models() -> None: } ] ) + return manager + + +def _when_call_legacy_models_for_source(manager, **kwargs): + namespace: dict = {"manager": manager, "kwargs": kwargs} + source = "_result = manager.expectations_models_for_source(**kwargs)" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exec(compile(source, _CONSUMER, "exec"), namespace) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + return namespace["_result"], deprecations + - (model,) = manager.expectations_models_for_source(source_id="collector") +def test_expectation_manager_builds_sdk_models() -> None: + """It still hands out collectors-sdk expectation models, with a warning.""" + manager = _given_expectation_manager() + + (model,), deprecations = _when_call_legacy_models_for_source( + manager, source_id="collector" + ) assert isinstance(model, collectors_sdk.DetectionExpectation) + _then_warns_at_consumer(deprecations) + assert "from collectors_sdk import expectations_models_for_source" in str( + deprecations[0].message + ) + + +def test_expectation_manager_delegates_to_collectors_sdk() -> None: + """It builds nothing itself: collectors-sdk builds the models from the manager.""" + manager = _given_expectation_manager() + sentinel = [object()] + + with patch.object( + collectors_sdk, "expectations_models_for_source", return_value=sentinel + ) as builder: + result, _ = _when_call_legacy_models_for_source( + manager, source_id="collector", expiration_time=5 + ) + + builder.assert_called_once_with(manager, "collector", expiration_time=5) + assert result is sentinel def test_importing_pyoaev_modules_is_silent() -> None: From b68d1d63fc2e4e183ef9ce20383e389133cdee81 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Wed, 14 Oct 2026 23:24:49 +0200 Subject: [PATCH 20/22] :package: build(pyoaev): package only pyoaev in the wheel --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index e8004912..12b5ab72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ build-backend = "setuptools.build_meta" include-package-data = true [tool.setuptools.packages.find] +include = ["pyoaev*"] [tool.setuptools.dynamic] version = { attr = "pyoaev.__version__" } From f5e7ce3e8d315792ad840ca573d4822f49a558cd Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Thu, 15 Oct 2026 08:46:20 +0200 Subject: [PATCH 21/22] :white_check_mark: test(sdk): fingerprinting every seam object in one dynamic golden --- libs/xtm-oaev-sdk/CHANGELOG.md | 5 +- libs/xtm-oaev-sdk/CLAUDE.md | 15 +- libs/xtm-oaev-sdk/CONTRIBUTING.md | 70 ++- libs/xtm-oaev-sdk/docs/seam_strategy.md | 7 +- libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md | 3 +- ...rocess-seams-pin-their-json-schemas.tdr.md | 8 +- .../every-seam-object-is-fingerprinted.tdr.md | 78 +++ .../test/seam/fixtures/port_io_schemas.json | 9 - .../test/seam/fixtures/seam_golden.json | 462 ++++++++++++++++++ .../fixtures/signatures_seam_schemas.json | 8 - .../test/seam/generate_port_io_golden.py | 47 -- .../test/seam/generate_seam_golden.py | 187 +++++++ .../seam/generate_signatures_seam_golden.py | 45 -- .../test/seam/test_port_io_schema_pin.py | 97 ---- .../test/seam/test_seam_closed.py | 36 ++ .../test/seam/test_seam_golden.py | 66 +++ .../seam/test_signatures_seam_schema_pin.py | 101 ---- .../xtm_oaev_sdk/seam/port_io/__init__.py | 4 +- .../xtm_oaev_sdk/seam/port_io/models.py | 6 +- .../xtm_oaev_sdk/seam/signatures/seam.py | 5 +- 20 files changed, 884 insertions(+), 375 deletions(-) create mode 100644 libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md delete mode 100644 libs/xtm-oaev-sdk/test/seam/fixtures/port_io_schemas.json create mode 100644 libs/xtm-oaev-sdk/test/seam/fixtures/seam_golden.json delete mode 100644 libs/xtm-oaev-sdk/test/seam/fixtures/signatures_seam_schemas.json delete mode 100644 libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py create mode 100644 libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py delete mode 100644 libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py delete mode 100644 libs/xtm-oaev-sdk/test/seam/test_port_io_schema_pin.py create mode 100644 libs/xtm-oaev-sdk/test/seam/test_seam_closed.py create mode 100644 libs/xtm-oaev-sdk/test/seam/test_seam_golden.py delete mode 100644 libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py diff --git a/libs/xtm-oaev-sdk/CHANGELOG.md b/libs/xtm-oaev-sdk/CHANGELOG.md index f2d0ba02..abf7aa5d 100644 --- a/libs/xtm-oaev-sdk/CHANGELOG.md +++ b/libs/xtm-oaev-sdk/CHANGELOG.md @@ -22,9 +22,8 @@ this SDK. The former `pyoaev` paths keep working, each one firing a `ContractReferencedCredential`, `build_single_referenced_credential_element`, from `pyoaev.credential`). Seam `1.0.0`. - `signatures`: `SignatureManager`, `build_network_configs`, `build_marker`, the - matching types and the inbound I/O models. Seam `1.0.0`, JSON schemas pinned. -- `port_io`: the wire models sent to the platform. `PORT_IO_CONTRACT_VERSION` `1.0.0`, - JSON schemas pinned. + matching types and the inbound I/O models. Seam `1.0.0`. +- `port_io`: the wire models sent to the platform. `PORT_IO_CONTRACT_VERSION` `1.0.0`. - `daemons`: `BaseDaemon` and `PingAlive`. Seam `1.0.0`. - `deprecation` (new, not from `pyoaev`): `make_deprecated_getattr`, the PEP 562 deprecation primitive. Seam `1.0.0`. diff --git a/libs/xtm-oaev-sdk/CLAUDE.md b/libs/xtm-oaev-sdk/CLAUDE.md index 35bf735a..c5444944 100644 --- a/libs/xtm-oaev-sdk/CLAUDE.md +++ b/libs/xtm-oaev-sdk/CLAUDE.md @@ -35,13 +35,10 @@ build on this SDK from their own repositories. goes in its `public/.py` `__all__` and in `public/__init__.py`. 7. A seam change (field, type, enum member, error, port signature) is a contract change: bump that capability's `SEAM_VERSION` (`PORT_IO_CONTRACT_VERSION` for `seam/port_io`) - and add an entry to `libs/xtm-oaev-sdk/CHANGELOG.md`. For `seam/signatures/seam.py` or - `seam/port_io`, also update the expected version in the matching - `test/seam/test_*_schema_pin.py` and regenerate the pinned schemas in the same change: - `python libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py` and - `python libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py`. NEVER regenerate them - to make a failing pin pass without a deliberate seam change; follow - `libs/xtm-oaev-sdk/CONTRIBUTING.md`, "Pinned seam schemas". + and add an entry to `libs/xtm-oaev-sdk/CHANGELOG.md`. Then regenerate the seam golden + in the same change: `python libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py`. + NEVER regenerate it to make a failing `test/seam/test_seam_golden.py` pass without a + deliberate seam change; follow `libs/xtm-oaev-sdk/CONTRIBUTING.md`, "Seam golden". 8. NEVER regenerate `test/fixtures/legacy_surface.json` from the working tree: it records the released `pyoaev`. NEVER rewrite the legacy tests moved under `libs/xtm-oaev-sdk/test//shim/`. @@ -56,8 +53,8 @@ dependencies or refactors beyond the task. - changing a seam contract or a `SEAM_VERSION`; - renaming or removing a public name, or removing a `pyoaev` shim; - adding or changing a dependency in any `pyproject.toml` or in `libs/sdk-requirements.txt`; -- editing `test/fixtures/legacy_surface.json`, the golden schema fixtures under - `libs/xtm-oaev-sdk/test/seam/fixtures/`, or CI configuration; +- editing `test/fixtures/legacy_surface.json`, the seam golden + `libs/xtm-oaev-sdk/test/seam/fixtures/seam_golden.json`, or CI configuration; - deleting any file. ## Done when diff --git a/libs/xtm-oaev-sdk/CONTRIBUTING.md b/libs/xtm-oaev-sdk/CONTRIBUTING.md index 227c2725..d4fbdd7e 100644 --- a/libs/xtm-oaev-sdk/CONTRIBUTING.md +++ b/libs/xtm-oaev-sdk/CONTRIBUTING.md @@ -61,77 +61,69 @@ A seam change is a contract change. Treat it like one ([seam-changes-are-versioned-contract-changes](docs/tdr/seam-changes-are-versioned-contract-changes.tdr.md)): 1. bump the capability's `SEAM_VERSION` (`PORT_IO_CONTRACT_VERSION` for `seam/port_io`); -2. for `seam/signatures/seam.py` or `seam/port_io`, update the pinned schemas in the same - change, not in a follow-up (see [Pinned seam schemas](#pinned-seam-schemas-testseam)); +2. regenerate the seam golden in the same change, not in a follow-up (see + [Seam golden](#seam-golden-testseam)); 3. add an entry to [CHANGELOG.md](CHANGELOG.md); 4. if a public name changes or disappears, the old one keeps working through a deprecation shim (`make_deprecated_getattr`) first. -## Pinned seam schemas (`test/seam`) +## Seam golden (`test/seam`) -Some seams carry models that leave the process: the inbound signature I/O in -`seam/signatures/seam.py` (injector configs, execution signatures, tool output) and the -wire payloads in `seam/port_io`. Another process reads them. Change their schema and the -other side breaks while every local test stays happily green. `libs/xtm-oaev-sdk/test/seam/` -pins those schemas: touch one and a test goes red, forcing a deliberate decision -([cross-process-seams-pin-their-json-schemas](docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md)). +The seam is the contract between the public API and the core, and part of it rides the +wire: payloads sent to the platform, data coming back, the connector manifest. Every +change to it must be deliberate. `libs/xtm-oaev-sdk/test/seam/` fingerprints every +public object of every seam module, so any change fails a test until someone signs off +on it +([every-seam-object-is-fingerprinted](docs/tdr/every-seam-object-is-fingerprinted.tdr.md)). ### What is in there | File | Role | |---|---| -| `fixtures/signatures_seam_schemas.json`, `fixtures/port_io_schemas.json` | The golden files: one SHA-256 per model, computed from its `model_json_schema()` serialized with sorted keys. Hashes only. The schemas themselves are not in there. | -| `test_signatures_seam_schema_pin.py`, `test_port_io_schema_pin.py` | Recompute every hash and hold it against the golden file. On top of that, they assert the seam version, make sure the seam module imports nothing from the SDK internals, and (signatures) that the package root re-exports the very same objects. | -| `generate_signatures_seam_golden.py`, `generate_port_io_golden.py` | Overwrite the golden file with the hashes of the current models. No check. No confirmation. | +| `generate_seam_golden.py` | Finds the seam on its own: every `*.py` file under `xtm_oaev_sdk/seam/`, every public top-level definition (class, function, constant, type alias). It fingerprints each one, then rewrites the golden: changed fingerprints updated, new objects added, deleted objects removed. It prints what moved, and warns when a capability changed but its version did not. | +| `fixtures/seam_golden.json` | The golden: one entry per object, `".": {"kind", "sha256"}`. | +| `test_seam_golden.py` | Recomputes the fingerprints and fails on any object added, removed or changed. It also checks that the package root re-exports the seam's own objects. | +| `test_seam_closed.py` | Checks that `port_io` imports only the signatures seam, and that the signatures seam imports nothing from the SDK. | -The other seams carry a `SEAM_VERSION` and no pinned schema. Their models never leave -the process, so there is no other side to break. +No list to maintain. The next run picks up any new seam object. -### When it fails +### What a fingerprint covers -The moment `model_json_schema()` changes for a pinned model, for any reason: - -- a field is added, removed, renamed or retyped, or its default changes; -- a `Field` description changes, or the docstring of a model or enum. Yes, the - docstring: it becomes the schema `description`; -- a pydantic upgrade renders the schema differently, even though no model changed. +The sha256 of the definition's normalized AST. Formatting, comments and docstrings do +not count. So it moves with fields, defaults, enum members, method signatures, +validators and constant values. Pydantic models add their validation and serialization +JSON schemas on top, model and field descriptions included. Those reach the wire, so +rewording a pydantic model's docstring is a change. ### Updating it on purpose From the repository root: -1. make the model change; -2. bump the version: `SEAM_VERSION` in `seam/signatures/seam.py` or - `PORT_IO_CONTRACT_VERSION` in `seam/port_io/models.py`, plus the expected value in - `test_seam_version_is_pinned` or `test_port_io_contract_version_is_pinned`; -3. regenerate the golden file: +1. make the seam change; +2. bump the capability's `SEAM_VERSION` (`PORT_IO_CONTRACT_VERSION` for `seam/port_io`); +3. regenerate the golden: ```bash - python libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py - python libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py + python libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py ``` 4. run `python -m pytest libs/xtm-oaev-sdk/test/seam`; -5. commit the model, the version, the test and the golden file together, with a +5. commit the seam change, the version bump and the golden in one go, with a [CHANGELOG.md](CHANGELOG.md) entry. -Pinning a new model? Add it to the model dictionary of the test *and* of its generator. -Each keeps its own copy, and forgetting one is the trap. - ### Seeing what changed -The golden files only hold hashes, so a failure tells you *that* a schema moved, never -*how*. To see it, dump the schema on your branch and on the base commit (a -`git worktree` does the job), then diff the two: +A failure names every object that moved, as `.`. `git diff` that +definition: the code diff is the change. A pydantic model can move with no code change +at all (a pydantic upgrade, a referenced type). Then compare its schema on both commits: ```bash python -c "import json; from xtm_oaev_sdk.seam.port_io import ExecutionDetails as M; print(json.dumps(M.model_json_schema(), indent=2, sort_keys=True))" ``` -NEVER regenerate a golden file just to turn a red test green, and NEVER edit a hash by -hand. That silences the one alarm this directory exists for. If a hash moved without any -model change (a pydantic upgrade, say), compare the dumped schemas and ask before deciding -whether the wire contract really changed. +NEVER regenerate the golden just to turn a red test green. NEVER edit a hash by hand. If +a fingerprint moved with no seam change, find out why before you decide whether the +contract really changed. ## Moving code out of pyoaev diff --git a/libs/xtm-oaev-sdk/docs/seam_strategy.md b/libs/xtm-oaev-sdk/docs/seam_strategy.md index 610afcbc..97925511 100644 --- a/libs/xtm-oaev-sdk/docs/seam_strategy.md +++ b/libs/xtm-oaev-sdk/docs/seam_strategy.md @@ -158,10 +158,9 @@ def run(self) -> None: - Every capability seam exposes `SEAM_VERSION`. `port_io`, a pure wire schema, carries `PORT_IO_CONTRACT_VERSION` instead. -- The seams that cross a process boundary pin their JSON schemas with golden hashes: - `seam/signatures/seam.py` (inbound I/O: injector configs, execution - signatures, tool output) and `seam/port_io` (`PORT_IO_CONTRACT_VERSION`: the wire - payloads sent to the platform). See `test/seam/`. +- Every public seam object is fingerprinted in a golden: any seam change fails a test + until someone regenerates the golden on purpose, version bump included. See `test/seam/` and + [every-seam-object-is-fingerprinted](tdr/every-seam-object-is-fingerprinted.tdr.md). - Policy: a change to a seam (field, type, enum member, error, port signature) bumps its version and lands in the changelog. If it reaches the public surface, the old name goes through a deprecation shim first (see [shim_strategy.md](shim_strategy.md)). diff --git a/libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md b/libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md index 968d31fe..dae0cb36 100644 --- a/libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md +++ b/libs/xtm-oaev-sdk/docs/tdr/TDR_ToC.md @@ -32,7 +32,8 @@ xtm-oaev-sdk, collectors-sdk, injectors-sdk and the `pyoaev` shims is expected t | TDR | Status | Rule | |---|---|---| | [A seam change is a versioned contract change](seam-changes-are-versioned-contract-changes.tdr.md) | draft | Any change to a seam contract (field, type, enum member, error, port signature) must bump that seam's version and be recorded in the changelog, and a change reaching the public surface must go through a deprecation shim first. | -| [Seams crossing a process boundary pin their JSON schemas](cross-process-seams-pin-their-json-schemas.tdr.md) | draft | A seam whose models cross a process boundary must pin their JSON schemas with golden hashes, regenerated only together with a version bump. | +| [Every seam object is fingerprinted](every-seam-object-is-fingerprinted.tdr.md) | draft | Every public top-level object of every seam module must be fingerprinted in one golden, discovered dynamically, and the golden is regenerated only together with a deliberate seam change and its version bump. | +| [Seams crossing a process boundary pin their JSON schemas](cross-process-seams-pin-their-json-schemas.tdr.md) | superseded by [every-seam-object-is-fingerprinted](every-seam-object-is-fingerprinted.tdr.md) | A seam whose models cross a process boundary must pin their JSON schemas with golden hashes, regenerated only together with a version bump. | ## Public API, tests and checks diff --git a/libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md index 4f141e16..7a5e4ec8 100644 --- a/libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md +++ b/libs/xtm-oaev-sdk/docs/tdr/cross-process-seams-pin-their-json-schemas.tdr.md @@ -1,12 +1,12 @@ # Seams crossing a process boundary pin their JSON schemas --- scope: [xtm-oaev-sdk, signatures, port_io, testing, versioning] -status: draft -superseded_by: +status: superseded +superseded_by: every-seam-object-is-fingerprinted supersedes: Aliases: Creation Date: 2026/10/14 16:25:31 -Modification Date: 2026/10/14 21:39:05 +Modification Date: 2026/10/15 08:36:35 --- ## ELI5 @@ -71,4 +71,4 @@ What we learned while forging this TDR: ## **Resources** - [Schema pin tests](../../test/seam/) -- [Contributing: pinned seam schemas](../../CONTRIBUTING.md#pinned-seam-schemas-testseam) +- [Contributing: seam golden, which replaced these pins](../../CONTRIBUTING.md#seam-golden-testseam) diff --git a/libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md new file mode 100644 index 00000000..1325fe81 --- /dev/null +++ b/libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md @@ -0,0 +1,78 @@ +# Every seam object is fingerprinted +--- +scope: [xtm-oaev-sdk, seam, testing, versioning] +status: draft +superseded_by: +supersedes: cross-process-seams-pin-their-json-schemas +Aliases: +Creation Date: 2026/10/15 08:36:35 +Modification Date: 2026/10/15 08:36:35 +--- + +## ELI5 + +The seam is the promise between the parts of the SDK, and part of that promise travels +to the platform. Every piece of it gets a fingerprint. Change a piece, even by accident, +and its fingerprint changes: a test goes red. Then you choose, on purpose, to bump the +version and save the new fingerprint. + +--- + +## **Why This Exists** +| *The forcing reality that demanded this TDR.* + +> Pinning the JSON schemas of two seam modules left the rest of the seam wide open: +> ports, enums living elsewhere, constants, functions. And each pin kept its own model +> list, which nobody remembered to extend. A seam change could walk right through with +> every test green. + +--- + +## **The TDR** +| **Every public top-level object of every seam module must be fingerprinted in one golden, discovered dynamically, and the golden is regenerated only together with a deliberate seam change and its version bump.** + +> No hand-kept list. The next run picks up any new seam module or object. + +--- + +## **In Practice** +| *How this TDR manifests in the real world.* + +- `test/seam/generate_seam_golden.py` parses every `*.py` file under `xtm_oaev_sdk/seam/` and fingerprints each public class, function, constant and type alias: the sha256 of its normalized AST, minus formatting, comments and docstrings. Pydantic models get their validation and serialization JSON schemas folded in too. +- It rewrites `test/seam/fixtures/seam_golden.json`: changed fingerprints updated, new objects added, deleted ones removed. It prints what moved, and warns when a capability changed but its version did not. +- `test/seam/test_seam_golden.py` recomputes the fingerprints and fails on anything added, removed or changed, and names every object. It also checks that the package root re-exports the seam's own objects. +- `test/seam/test_seam_closed.py` keeps the two closed-import checks: `port_io` imports only the signatures seam, and the signatures seam imports nothing from the SDK. +- The step-by-step procedure lives in `CONTRIBUTING.md`, "Seam golden". + +--- + +## **Alternatives Explored** +| *The paths we tried and why they failed.* + +- **Keep the JSON schema pins, add more models to their lists.** Rejected: they only see pydantic models, and someone still has to keep the lists by hand. +- **Fingerprint `inspect.getsource()`.** Rejected: a comment or a reformat would trip it. That is noise, not a contract change. + +--- + +## **Mutation Trigger** +| *When this TDR should be reconsidered.* + +This TDR becomes questionable if: +- the seam contracts are published and checked by the platform side; +- a CI contract tool replaces the in-repository golden; +- Python or pydantic upgrades keep moving fingerprints while no contract changed. + +## **Hansei** +| *Scars, surprises, and extracted wisdom.* + +What we learned while forging this TDR: +- The AST drops formatting and comments for free. Docstrings do not go quietly: strip them by hand. Skip the AST's empty fields too, or the text shifts between Python versions. +- A pydantic model's JSON schema carries its docstring and field descriptions. Those reach the wire, so rewording one counts as a change. +- The golden stores hashes, not definitions. A failure tells you which object moved; `git diff` on it tells you how. + +--- + +## **Resources** +- [Seam golden tests](../../test/seam/) +- [Contributing: seam golden](../../CONTRIBUTING.md#seam-golden-testseam) +- [Superseded: seams crossing a process boundary pin their JSON schemas](cross-process-seams-pin-their-json-schemas.tdr.md) diff --git a/libs/xtm-oaev-sdk/test/seam/fixtures/port_io_schemas.json b/libs/xtm-oaev-sdk/test/seam/fixtures/port_io_schemas.json deleted file mode 100644 index a3e010fb..00000000 --- a/libs/xtm-oaev-sdk/test/seam/fixtures/port_io_schemas.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "ExecutionDetails": "4cc109aef2950c555a6d0dce2e637e3ff709b86588a08fe0e459c5eb5328f939", - "ExpectationSignatureGroup": "8258ae33685d9826dc3b090186c6fa76935353a0fabd12ca9b6974184df251ff", - "SignatureOutputStructure": "4291aed5c4a354327c70db42534cd90d38bfbf7b39cc241cacf946bb0b70c066", - "SignaturePayload": "5f7451f4640c22c3f51cc1dab1fe0e65ca85d31f67349b2c42151111980d1467", - "SignatureTarget": "52a8848d8b60e2f68aaa8769f303ec886beb3b77a9088b98ad783690512e9ea9", - "SignatureValue": "dd21d79b3127f06390e5e17120bd6b662c11b04bb45f7db9c8c7848ca33d3d95", - "TargetSignatures": "490debc49e8fd5352b6afa91fe018935b17c8a4497c35fcc35ac1ebaf79242df" -} diff --git a/libs/xtm-oaev-sdk/test/seam/fixtures/seam_golden.json b/libs/xtm-oaev-sdk/test/seam/fixtures/seam_golden.json new file mode 100644 index 00000000..8ff3ea3e --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/fixtures/seam_golden.json @@ -0,0 +1,462 @@ +{ + "xtm_oaev_sdk.seam.common.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.common.asset_types.AssetCategory": { + "kind": "class", + "sha256": "cb67f3d2a86d93fbb49632b69aac7774a1e31e3db05caf133dd91b5690d0c038" + }, + "xtm_oaev_sdk.seam.common.asset_types.AssetCriticality": { + "kind": "class", + "sha256": "d3dea966125525447d49a427bc2d367bdcc5aec55fc56b991e587796f5fe4da5" + }, + "xtm_oaev_sdk.seam.common.asset_types.AssetSubCategory": { + "kind": "class", + "sha256": "231d692d0185a13b417635bc42a83a1b7ee5a9cbf459e9fedc34327c400db920" + }, + "xtm_oaev_sdk.seam.common.asset_types.CloudProvider": { + "kind": "class", + "sha256": "c2a4fbf4b7164d34f0b8a2bf7504d1dcbc802e1e0313e9759d758b068fe5e6d6" + }, + "xtm_oaev_sdk.seam.common.errors.ConfigurationError": { + "kind": "class", + "sha256": "125c52aa906fe42d596f504d38ada75f770e485fffaac96d30acbca1b048737f" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVAuthenticationError": { + "kind": "class", + "sha256": "b4d32e039bd713e2e07514a31e70985e8be126b15f95f5d354fd2c2747c10942" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVCreateError": { + "kind": "class", + "sha256": "dd175bda1f009adb5aa6f34dc89e7cf93deac5ded93bf8602bedb8ad85c779e1" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVDeleteError": { + "kind": "class", + "sha256": "8a47c256d3a07a25812a7fe3779593ae805be299fbebd46dfe598d060953039e" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVError": { + "kind": "class", + "sha256": "f3c868a0b346592a84a66b71427732e5ff010efd7d9b2ef11f777da0914aded5" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVGetError": { + "kind": "class", + "sha256": "7464137d3c19d05ab07fc37335458d5921bbb1aad37f509ddcb1b11b7386ede7" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVHeadError": { + "kind": "class", + "sha256": "6c8d650e59878e26c39db48bbb81d62aba25dcfa7ddd6220bd2313dbcad61894" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVHttpError": { + "kind": "class", + "sha256": "5e161b4cacac55feef064b5f83697684906ca2b374d7e7444c1b1513eeabef50" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVListError": { + "kind": "class", + "sha256": "76f81f2f44676b5fc8b8e2f09a51cd0101ed422ef0fa282694e974446bd5c293" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVParsingError": { + "kind": "class", + "sha256": "3ed6ea9145afa9ff953356e88a51fe31e7ec07ec3f3e0a04101ccac3892c7f5c" + }, + "xtm_oaev_sdk.seam.common.errors.OpenAEVUpdateError": { + "kind": "class", + "sha256": "87d0e9ab82f0ddabf054815332454a3e58e09e431b78f4f13efaf1e70d6e0c0c" + }, + "xtm_oaev_sdk.seam.common.errors.RedirectError": { + "kind": "class", + "sha256": "2d2529e12ab4453fd29a4600c66583dc3490715f61039b6bb89b9a3455407d58" + }, + "xtm_oaev_sdk.seam.common.errors.SignatureTransmissionError": { + "kind": "class", + "sha256": "9543fea4fe7a04030a363ec7539fc05b63327b5608af454aa55f4ee722e1be6c" + }, + "xtm_oaev_sdk.seam.common.ports.CertificatesPort": { + "kind": "class", + "sha256": "99ac2da214d3b0882f5d47d9d4212edf5965efc5006fd4b1758c3a8242686885" + }, + "xtm_oaev_sdk.seam.common.ports.JsonEncodingPort": { + "kind": "class", + "sha256": "c5592e4eb2c586c13e1a8ae633b1d18557c895bc2e0067e5d1957200e7ee0f35" + }, + "xtm_oaev_sdk.seam.common.ports.LoggingPort": { + "kind": "class", + "sha256": "31d0542014237114be994ae7c2b9023e2dd65a2c9c3758fc7fb0c638e23b19a2" + }, + "xtm_oaev_sdk.seam.common.protocols.AppLoggerProtocol": { + "kind": "class", + "sha256": "98410e8645499cef7ecf8d51fbf81613e1cdb9bb2ac19e345512c38d22eef273" + }, + "xtm_oaev_sdk.seam.common.protocols.BaseClient": { + "kind": "class", + "sha256": "9727fa9446d2be988fa26c983aaf23f65128fd86a6b8fd742ee2fd4c03b99120" + }, + "xtm_oaev_sdk.seam.common.protocols.DaemonProtocol": { + "kind": "class", + "sha256": "6e04f94e2ba40fa1b883e3cc8350d55f35c201fdfa5daf7a3ea8c17c8e66cab9" + }, + "xtm_oaev_sdk.seam.common.security_domains.SecurityDomains": { + "kind": "class", + "sha256": "8a27be9c4fd908e1cdab182e559ef8d4689410a405d11b424d3fb0f23995db30" + }, + "xtm_oaev_sdk.seam.configuration.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.configuration.models.BaseConfigModel": { + "kind": "pydantic model", + "sha256": "2744c173dd64c929eba8ba559e138e56cc76a5638c3d74705f3924ef156d651f" + }, + "xtm_oaev_sdk.seam.configuration.models.CONFIGURATION_TYPES": { + "kind": "assignment", + "sha256": "3a094919b672c6be37a56a9d698b39f75c372815e9e6703c32ee799aec21175c" + }, + "xtm_oaev_sdk.seam.configuration.models.ConfigLoaderCollector": { + "kind": "pydantic model", + "sha256": "b791f4b65fc00537c117e0664a894867e8f34b4a8abec4f8eabea3a16e062431" + }, + "xtm_oaev_sdk.seam.configuration.models.ConfigLoaderOAEV": { + "kind": "pydantic model", + "sha256": "066ed8c10d20abfe63042ec2be39c0cae23f9a15088681dc48d9a8a003925a62" + }, + "xtm_oaev_sdk.seam.configuration.models.ConfigurationHint": { + "kind": "pydantic model", + "sha256": "59e06102d55627f0c3e41c04116cbaa5f9a0269df750be9a56c9ed1121660c1d" + }, + "xtm_oaev_sdk.seam.configuration.models.HttpUrlToString": { + "kind": "assignment", + "sha256": "7ce246816b036a5b93ecc244dfddec85adf28f3d53562c92052225071323dc96" + }, + "xtm_oaev_sdk.seam.configuration.models.LogLevelToLower": { + "kind": "assignment", + "sha256": "969e62cd77ce3f1aa7df05a3d2b8dd5c4a9a74dcd6bb5b31bcac5ff4386348d7" + }, + "xtm_oaev_sdk.seam.configuration.models.TimedeltaInSeconds": { + "kind": "assignment", + "sha256": "fdc6122dbc50a989af4baed5b798f25143eeb2c9d4c7e0d6b5189079d19e5ef4" + }, + "xtm_oaev_sdk.seam.configuration.ports.ConfigurationProtocol": { + "kind": "class", + "sha256": "43a6d1c58de96eeb8a268e62983d1c0d04c3ec5ca60bc7db3cf6440f1b5e2ef5" + }, + "xtm_oaev_sdk.seam.configuration.ports.ConfigurationResolverPort": { + "kind": "class", + "sha256": "c36adb803d996956a4a88fefcd3468f9a2e67dec149eb3170cff4d6e5532c8f9" + }, + "xtm_oaev_sdk.seam.configuration.ports.ConfigurationSourcesPort": { + "kind": "class", + "sha256": "7951e9dfa860e4c8d2553c9ca55cf46d59ce41435aa64aa8c0773245251ca161" + }, + "xtm_oaev_sdk.seam.configuration.ports.ConnectorSchemaPort": { + "kind": "class", + "sha256": "9ba35717945700beeaa45407c628283098b2ed983858af8a9ffefa5ad78cce8e" + }, + "xtm_oaev_sdk.seam.configuration.ports.SettingsSourcesPort": { + "kind": "class", + "sha256": "67681b5861788fdcf12585371293b814ef5a32418dbee64f28934ef919c24ecf" + }, + "xtm_oaev_sdk.seam.configuration.ports.TruthinessPort": { + "kind": "class", + "sha256": "3c5238b25e5c46bbde80ec4843a624776521e65e323ec018eb41531a8ef63535" + }, + "xtm_oaev_sdk.seam.contracts.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractAiTarget": { + "kind": "class", + "sha256": "bec76a7237ac27571146a586dfc1867275978925020cb495e92c487bfb3dbc82" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractAsset": { + "kind": "class", + "sha256": "4118fa0f218b9fa9ad13cb46f54cd42708826b9b8597c681f1f423846781aae0" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractAssetGroup": { + "kind": "class", + "sha256": "e68cdc9fd005c8d71e3399b8dd0cdfb82f4d06e32f6a6dbcb46971266942bbb7" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractAttachment": { + "kind": "class", + "sha256": "cb454a9aef9ac29dffc69eca75a577fce9160222f273a21ced23431e80ed62e9" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractCardinality": { + "kind": "class", + "sha256": "2c81f89d9e843a1a0171f3a4bd3646595d4ce398e5bd9a9361aa4d9bf94e17e7" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractCardinalityElement": { + "kind": "class", + "sha256": "1c6592f1e5664ad6ead85276e6987dd081896f59ae86ced5e6ab8ce1220987ee" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractCheckbox": { + "kind": "class", + "sha256": "9dd31fc0a96c97e790fface784a338b58682e02d8f9cbba9a32a44f33fc1f4d2" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractConfig": { + "kind": "class", + "sha256": "0186912bf55cc89902df1a42041c2b57e3b033cdfe79f7bf7ce4e043b7f35281" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractData": { + "kind": "class", + "sha256": "8abb76a14b43660f67ecad3a47ab6026b3785bfa832d8b6c5969ad209d024112" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractElement": { + "kind": "class", + "sha256": "43fa42de3e2c120636e4f4ae800952f11d0a75b2bf7f9db4b3afb129609031b6" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractExpectationType": { + "kind": "class", + "sha256": "cdbf56a38b1b4cff63229ad6fdb5939b07ffbc7e292112fbcb77206f173b3dc3" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractExpectations": { + "kind": "class", + "sha256": "81f1e6aa56166766600d43854360c6b565ba88c7b38c1c7c9aba1c55a440a768" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractFieldKey": { + "kind": "class", + "sha256": "3aa5cdebc64b0e09fa18950aa7c1bff2f1fad4cbfd4ac106a200cf01a453c7ae" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractFieldType": { + "kind": "class", + "sha256": "3efc706f2a68aa7fb006df594a5708ee018e9d615bbd572d96f7a64f65c0fbb7" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractOutputElement": { + "kind": "class", + "sha256": "903e0d46c2f0ccfbe550f0cd067d212f67905f208940fc06f8638ee4854d5a97" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractOutputType": { + "kind": "class", + "sha256": "52604db256eb23b5d7dc0a564a5e41f7e4b3e764ad89af0d97162471ca8be3d2" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractPayload": { + "kind": "class", + "sha256": "a424a20ef31a74cdce9c3b3b585b0d46a408c1dabdc4d70a53e88a5a830dab63" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractReferencedCredential": { + "kind": "class", + "sha256": "cfadf2858d353ad60b2ab963ed8dbc98054dec687862cbaf70efe3acf3f00ea2" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractSelect": { + "kind": "class", + "sha256": "1ff539b29b70b4090f175b11e7cf42f5476643a32937531301c27b8af2eba3ee" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractTeam": { + "kind": "class", + "sha256": "47b63fd6ca940482b964b0ecedb182de60a249a5fbf9310642745f7b1148b214" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractText": { + "kind": "class", + "sha256": "9e8170048f9340f1e3178ffe60e428c8574bca27148357856dbe5303fc89df15" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractTextArea": { + "kind": "class", + "sha256": "dc9d182716bf85bad565b2b7d0a9025e2869701f6e7dfb14b83009685771967e" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractTuple": { + "kind": "class", + "sha256": "a89fdb4ccf80bbbf87b19e3d330978bb995657d82e02c72fa814f57bfaa7eed6" + }, + "xtm_oaev_sdk.seam.contracts.models.ContractVariable": { + "kind": "class", + "sha256": "46522f5a0c908d7376b8ffa389d87113a972848467711a46cdc4dc5d6262d49e" + }, + "xtm_oaev_sdk.seam.contracts.models.CredentialType": { + "kind": "class", + "sha256": "7e29de5fbd972fab40d3525275d4c93fbb54347869c1a5efe24b6737f2719890" + }, + "xtm_oaev_sdk.seam.contracts.models.Domain": { + "kind": "class", + "sha256": "26fd50e1160ff864f4ac1404a08e1594a99c5df39d3f482c0c6492de3a08f223" + }, + "xtm_oaev_sdk.seam.contracts.models.Expectation": { + "kind": "class", + "sha256": "8de1930e714539c664d53e0fee6e37ee8c2af6fe72502181db9c0cc6bd75046d" + }, + "xtm_oaev_sdk.seam.contracts.models.LinkedFieldModel": { + "kind": "class", + "sha256": "73c3eec7ba6d4182212b9d9e09cee3b11cc437b3a8d8997b84c54fad8731d234" + }, + "xtm_oaev_sdk.seam.contracts.models.PrimitiveType": { + "kind": "class", + "sha256": "741b00f6883baea168f1ea19c705eba1e1e9f001e02a95a3570c471eafe2c020" + }, + "xtm_oaev_sdk.seam.contracts.models.SecurityPlatformType": { + "kind": "class", + "sha256": "8d9bdf6573fa6c32cae8784b83bef9cd19532db2238ef45cc65b6ad9078d5988" + }, + "xtm_oaev_sdk.seam.contracts.models.SupportedLanguage": { + "kind": "class", + "sha256": "a367cd2ecbe05dc4c118940ab719dabf0128dfacd8fbf18ec162bcd3b4677592" + }, + "xtm_oaev_sdk.seam.contracts.models.VariableType": { + "kind": "class", + "sha256": "22dc85d597bba70a70742f392b5d621514da4d23662f116b00f33808c8262593" + }, + "xtm_oaev_sdk.seam.contracts.ports.ContractAssemblyPort": { + "kind": "class", + "sha256": "0bd7a273151c8726d905138f48a45239d9d86e776cc82686465607a6a0d6fe5e" + }, + "xtm_oaev_sdk.seam.contracts.ports.ContractBuilderProtocol": { + "kind": "class", + "sha256": "c64343786d3ce2d454cb8966b36ec7a3afecc359ef954191fa9ca5ec22af76ff" + }, + "xtm_oaev_sdk.seam.contracts.ports.ContractCredentialsPort": { + "kind": "class", + "sha256": "d2320f0e0d1a601d423d7478a7106758061a83a99644bebfa91edd7f197e1687" + }, + "xtm_oaev_sdk.seam.contracts.ports.ContractSerializerPort": { + "kind": "class", + "sha256": "0366d67e6b581aac3fdc5f6dfeb4e7eba65a21da98a1fc89a0add1c192cb3327" + }, + "xtm_oaev_sdk.seam.contracts.ports.ContractVariablesPort": { + "kind": "class", + "sha256": "a3845ad0c166456174e25d299cff4a16cae2117ce13bd10174fb1c843f183886" + }, + "xtm_oaev_sdk.seam.daemons.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.daemons.ports.DaemonRuntimePort": { + "kind": "class", + "sha256": "057f096c283a09b2cb560b9d39de886df4f237c20c438ce02acab991523c2e2f" + }, + "xtm_oaev_sdk.seam.daemons.ports.HeartbeatPort": { + "kind": "class", + "sha256": "1e42dda89a11a88eead8163c170c27d5136996820d4d1fc3464bd6dfed7fc242" + }, + "xtm_oaev_sdk.seam.deprecation.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.deprecation.ports.DeprecatedNames": { + "kind": "assignment", + "sha256": "0fa67de53de219b64454162f3fea7f07bed0504929cc8520bba25a5a7794d0b0" + }, + "xtm_oaev_sdk.seam.deprecation.ports.DeprecationPort": { + "kind": "class", + "sha256": "c0631fd4c1bfa509bce632c1c815865d504293e100b8798981df36567bad5c3b" + }, + "xtm_oaev_sdk.seam.port_io.models.ExecutionDetails": { + "kind": "pydantic model", + "sha256": "4d9546d2033c3d68bc31a688481bc580fb876fa1f984495bd8d1ac4bb8c6ce15" + }, + "xtm_oaev_sdk.seam.port_io.models.ExecutionStatus": { + "kind": "class", + "sha256": "1ac92fe06592d1ef55703df64941baf70b75c646d138e4f7698abe02bba50af0" + }, + "xtm_oaev_sdk.seam.port_io.models.ExpectationSignatureGroup": { + "kind": "pydantic model", + "sha256": "1349c78c33892854ef77030364dd310f1dd98ebf23688b97267a046bd5926a8c" + }, + "xtm_oaev_sdk.seam.port_io.models.InjectExecutionActions": { + "kind": "class", + "sha256": "45eaf82367b11a87617ec5563c1109b2ec9050fd5c58f70a7c5875c4352f9eb1" + }, + "xtm_oaev_sdk.seam.port_io.models.PORT_IO_CONTRACT_VERSION": { + "kind": "assignment", + "sha256": "4b3aa3a5b079ff1222ce2bb723750e882555bc2c857097f589066dcae2398d0a" + }, + "xtm_oaev_sdk.seam.port_io.models.SignatureCallbackPayload": { + "kind": "pydantic model", + "sha256": "c9bc7d84b064b595fd1cb7000d43d47246cb1026b89b2607e4870443e17dde61" + }, + "xtm_oaev_sdk.seam.port_io.models.SignatureExpectationType": { + "kind": "class", + "sha256": "d66c56b5404fbb512cb3d29ff52dcef0cb0a984fdfab1492f59185854c8117e3" + }, + "xtm_oaev_sdk.seam.port_io.models.SignatureOutputStructure": { + "kind": "pydantic model", + "sha256": "81f15a75c0b933604f13462c1721c2698e46acfdd170e3d629516f7f68946f5d" + }, + "xtm_oaev_sdk.seam.port_io.models.SignaturePayload": { + "kind": "pydantic model", + "sha256": "88f1171c10351454a081797da8dd3331d4fcda745b1b88654d7a2fbb50c41eab" + }, + "xtm_oaev_sdk.seam.port_io.models.SignatureTarget": { + "kind": "pydantic model", + "sha256": "f1d7d41f96daaaf60fe0a4fe6ae31e702c69f22080e096e867a632299ce033a5" + }, + "xtm_oaev_sdk.seam.port_io.models.SignatureValue": { + "kind": "pydantic model", + "sha256": "84f20eb4416d5279034c1c56108f5c28d52d2a9842e8d14cbde288f084f73cc1" + }, + "xtm_oaev_sdk.seam.port_io.models.TargetSignatures": { + "kind": "pydantic model", + "sha256": "59c1618a317ff3f4390d3c0531e815168dce910204987062c9e044c6466b908d" + }, + "xtm_oaev_sdk.seam.signatures.models.ExtraSignatureData": { + "kind": "pydantic model", + "sha256": "b69dd6b7476302eff126622e5d662fc297ca964c0484e6f948589cd1450ae4c5" + }, + "xtm_oaev_sdk.seam.signatures.models.MatchTypes": { + "kind": "class", + "sha256": "99eb73ffc33a3163857a2fa3d5ca00d8c56184ea98984fedf7a173fc197eb306" + }, + "xtm_oaev_sdk.seam.signatures.models.SignatureMatch": { + "kind": "class", + "sha256": "99908bfec243c4fb233ffeca62108abb0929ad8f0fbac1ce8eb434779fa0fc16" + }, + "xtm_oaev_sdk.seam.signatures.models.SignatureType": { + "kind": "class", + "sha256": "1eb1fba1a1470c6a2dd8d453ce6b318e2f9db20627c1ba4f6d36ef3c4266605a" + }, + "xtm_oaev_sdk.seam.signatures.models.SignatureTypes": { + "kind": "class", + "sha256": "46b9692d876bb8344229523d0fa65e46f4c59d521ebc24c47994a39a950756ae" + }, + "xtm_oaev_sdk.seam.signatures.ports.ContainerAddressPort": { + "kind": "class", + "sha256": "7fc3b66bf6cbf6aa3ee01c2d6273acabd35a5e1fe75d542709a91c22fb6f8f07" + }, + "xtm_oaev_sdk.seam.signatures.ports.MarkerPort": { + "kind": "class", + "sha256": "a6b4deafa934f4cf37987788f84b17df4dd69cffc8a9e7cf356682d735188ef1" + }, + "xtm_oaev_sdk.seam.signatures.ports.NetworkTargetsPort": { + "kind": "class", + "sha256": "9b60ef67d3ecd1370ad20d52a31a0b500bada8ed098e9d863249e421f3f77f74" + }, + "xtm_oaev_sdk.seam.signatures.ports.SignatureEnginePort": { + "kind": "class", + "sha256": "fdd1998986921f3c8d4b40d1a08f4897c10cee4b39b0d7186e0044b4841ea166" + }, + "xtm_oaev_sdk.seam.signatures.ports.SignatureManagerProtocol": { + "kind": "class", + "sha256": "7254b834dd9f952a5c248eb3039f7ccc9f88ec92e8279b211bf5968b14bfe513" + }, + "xtm_oaev_sdk.seam.signatures.ports.SignatureTransportProtocol": { + "kind": "class", + "sha256": "0bb363c3e37df931c589bbeb5b5febdb40910e35768c39b2d5e8a039829e8bb8" + }, + "xtm_oaev_sdk.seam.signatures.seam.CloudInjectorConfig": { + "kind": "pydantic model", + "sha256": "6dfe3660acd0b99ec29b1898a88501ba75fc83d11e6c967ac130a77d6950a980" + }, + "xtm_oaev_sdk.seam.signatures.seam.ExecutionSignature": { + "kind": "pydantic model", + "sha256": "75fcceb51cf8dd3c0c3e2b7eb65f8b0644b08ec6e7b8d012d5624250600553ed" + }, + "xtm_oaev_sdk.seam.signatures.seam.InjectorConfig": { + "kind": "assignment", + "sha256": "dd77ee2c6e42ebacd37e4177bc5d10df3b0321f279ef57b3dfd7a31fd934f100" + }, + "xtm_oaev_sdk.seam.signatures.seam.NetworkInjectorConfig": { + "kind": "pydantic model", + "sha256": "ceafda27b94d3b0043d34965dc5a927f8b8a1b833d0b2269140a04dbac797d89" + }, + "xtm_oaev_sdk.seam.signatures.seam.SEAM_VERSION": { + "kind": "assignment", + "sha256": "c26dc544bb2976ca7c5a7465be69719edcc7893657a494ef17b0285e7e2a37db" + }, + "xtm_oaev_sdk.seam.signatures.seam.ToolErrorInfo": { + "kind": "pydantic model", + "sha256": "27a3d760a76e74e0a556293741de45032384c3df107c1b33504c2b0003018f53" + }, + "xtm_oaev_sdk.seam.signatures.seam.ToolOutput": { + "kind": "pydantic model", + "sha256": "25d4fc1fa9f7e6bfc358a1104fd7e0fb620a232d70a1c3d8fcc65e6a79c88c9d" + }, + "xtm_oaev_sdk.seam.signatures.seam.ToolTimeoutInfo": { + "kind": "pydantic model", + "sha256": "57f73a9a8003e3907ef8e81120a31c2d2a7efb99d85c0effee699b133b05a44a" + } +} diff --git a/libs/xtm-oaev-sdk/test/seam/fixtures/signatures_seam_schemas.json b/libs/xtm-oaev-sdk/test/seam/fixtures/signatures_seam_schemas.json deleted file mode 100644 index d4202b8e..00000000 --- a/libs/xtm-oaev-sdk/test/seam/fixtures/signatures_seam_schemas.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "ToolErrorInfo": "5c7a5427eec8e163b614c69c5a3b4bd0e17bd5e483f2ca1820965561601fa413", - "ToolTimeoutInfo": "6a6da8f7280971640ff0ae8ae50f844c241e36bfb35a73a682e255e6758a36b2", - "ToolOutput": "9269914b152c6c42bd77110a483595b70fe3811a50cd0b8880770b7b9b686161", - "ExecutionSignature": "e9655dffea0daee5ad1c242f4d43138235ae7f4f55eb357b70752614fa68daf0", - "NetworkInjectorConfig": "68a8f930e3e606bf95efed0c4c386332714e0c8b540918d440ef58b4f36caf0d", - "CloudInjectorConfig": "908a129f91d32fd6e1f553c445ac92ec07c2ebeb065f51a5e955c1d78fce6ad9" -} diff --git a/libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py b/libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py deleted file mode 100644 index 31df41d2..00000000 --- a/libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Regenerate the golden schema hashes for the port I/O wire contract. - -Run ``python libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py``. The -golden file pins the JSON schema of every wire model; it is only regenerated -on an intentional, version-bumped contract change. -""" - -import hashlib -import json -from pathlib import Path - -from xtm_oaev_sdk.seam.port_io import ( - ExecutionDetails, - ExpectationSignatureGroup, - SignatureOutputStructure, - SignaturePayload, - SignatureTarget, - SignatureValue, - TargetSignatures, -) - -GOLDEN_PATH = Path(__file__).parent / "fixtures" / "port_io_schemas.json" - -CONTRACT_MODELS = { - "ExecutionDetails": ExecutionDetails, - "ExpectationSignatureGroup": ExpectationSignatureGroup, - "SignatureOutputStructure": SignatureOutputStructure, - "SignaturePayload": SignaturePayload, - "SignatureTarget": SignatureTarget, - "SignatureValue": SignatureValue, - "TargetSignatures": TargetSignatures, -} - - -def main() -> None: - out: dict[str, str] = {} - for name, cls in CONTRACT_MODELS.items(): - canonical = json.dumps(cls.model_json_schema(), sort_keys=True) - out[name] = hashlib.sha256(canonical.encode()).hexdigest() - with GOLDEN_PATH.open("w") as f: - json.dump(out, f, indent=2) - f.write("\n") - print(f"wrote {GOLDEN_PATH}") - - -if __name__ == "__main__": - main() diff --git a/libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py b/libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py new file mode 100644 index 00000000..9c330bf1 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py @@ -0,0 +1,187 @@ +"""Fingerprint every public object of the xtm-oaev-sdk seam and update the golden. + +Discovery is dynamic: every ``*.py`` file under ``xtm_oaev_sdk/seam`` is parsed, and +every public top-level definition (class, function, assignment) gets a sha256 of: + +- its normalized AST: formatting, comments and docstrings are ignored, so the hash + moves only when the definition itself changes (fields, defaults, enum members, + signatures, validators, constant values...); +- for pydantic models, their validation and serialization JSON schemas too, which + also catch what reaches the wire through the types they reference. + +Run from the repository root after a deliberate seam change, together with the +``SEAM_VERSION`` bump of the capability (``PORT_IO_CONTRACT_VERSION`` for port_io):: + + python libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py + +It rewrites ``fixtures/seam_golden.json``: changed fingerprints are updated, new +objects added, and objects that no longer exist removed. ``test_seam_golden.py`` +fails on any difference between the golden and the code. +""" + +import ast +import copy +import hashlib +import importlib +import json +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import xtm_oaev_sdk.seam + +SEAM_ROOT = Path(xtm_oaev_sdk.seam.__file__).parent +PACKAGE_ROOT = SEAM_ROOT.parent.parent +GOLDEN_PATH = Path(__file__).parent / "fixtures" / "seam_golden.json" + +_IGNORED_AST_FIELDS = {"ctx", "type_comment"} +_DOCSTRING_OWNERS = (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef) + + +def _module_name(path: Path) -> str: + parts = path.relative_to(PACKAGE_ROOT).with_suffix("").parts + return ".".join(parts[:-1] if parts[-1] == "__init__" else parts) + + +def _defined_names(node: ast.stmt) -> list[str]: + if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + return [node.name] + if isinstance(node, ast.Assign): + return [t.id for t in node.targets if isinstance(t, ast.Name)] + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + return [node.target.id] + if isinstance(node, ast.TypeAlias): + return [node.name.id] + return [] + + +def _without_docstrings(node: ast.stmt) -> ast.stmt: + node = copy.deepcopy(node) + for child in ast.walk(node): + if isinstance(child, _DOCSTRING_OWNERS) and child.body: + first = child.body[0] + if ( + isinstance(first, ast.Expr) + and isinstance(first.value, ast.Constant) + and isinstance(first.value.value, str) + ): + child.body = child.body[1:] + return node + + +def _canonical_ast(node: Any) -> str: + """Python-version-stable text of an AST: no positions, no empty fields.""" + if isinstance(node, ast.AST): + fields = [] + for field in node._fields: + if field in _IGNORED_AST_FIELDS: + continue + value = getattr(node, field, None) + if value is None or value == []: + continue + fields.append(f"{field}={_canonical_ast(value)}") + return f"{type(node).__name__}({', '.join(fields)})" + if isinstance(node, list): + return "[" + ", ".join(_canonical_ast(item) for item in node) + "]" + return repr(node) + + +def _pydantic_schemas(obj: Any) -> str | None: + from pydantic import BaseModel + + if not (isinstance(obj, type) and issubclass(obj, BaseModel)): + return None + schemas = { + mode: obj.model_json_schema(mode=mode) + for mode in ("validation", "serialization") + } + return json.dumps(schemas, sort_keys=True) + + +def iter_seam_definitions() -> Iterator[tuple[str, str, ast.stmt, Any]]: + """Every public top-level definition of the seam: (module, name, node, object).""" + for path in sorted(SEAM_ROOT.rglob("*.py")): + module_name = _module_name(path) + module = importlib.import_module(module_name) + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + for name in _defined_names(node): + if not name.startswith("_"): + yield module_name, name, node, getattr(module, name) + + +def _kind(node: ast.stmt, obj: Any) -> str: + if _pydantic_schemas(obj) is not None: + return "pydantic model" + if isinstance(node, ast.ClassDef): + return "class" + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + return "function" + return "assignment" + + +def compute_seam_fingerprints() -> dict[str, dict[str, str]]: + """``{".": {"kind": ..., "sha256": ...}}`` for the whole seam.""" + fingerprints = {} + for module_name, name, node, obj in iter_seam_definitions(): + text = "ast:" + _canonical_ast(_without_docstrings(node)) + schemas = _pydantic_schemas(obj) + if schemas is not None: + text += "\nschemas:" + schemas + fingerprints[f"{module_name}.{name}"] = { + "kind": _kind(node, obj), + "sha256": hashlib.sha256(text.encode()).hexdigest(), + } + return dict(sorted(fingerprints.items())) + + +def diff_fingerprints( + golden: dict[str, dict[str, str]], current: dict[str, dict[str, str]] +) -> dict[str, list[str]]: + """Seam objects added, removed and changed since the golden.""" + return { + "added": sorted(current.keys() - golden.keys()), + "removed": sorted(golden.keys() - current.keys()), + "changed": sorted( + key + for key in current.keys() & golden.keys() + if current[key]["sha256"] != golden[key]["sha256"] + ), + } + + +def _capability(key: str) -> str: + return key.split(".")[2] + + +def _unversioned_capabilities(diff: dict[str, list[str]]) -> list[str]: + touched = {_capability(key) for keys in diff.values() for key in keys} + bumped = { + _capability(key) + for keys in diff.values() + for key in keys + if key.rsplit(".", 1)[1].endswith("VERSION") + } + return sorted(touched - bumped) + + +def main() -> int: + golden = json.loads(GOLDEN_PATH.read_text()) if GOLDEN_PATH.exists() else {} + current = compute_seam_fingerprints() + diff = diff_fingerprints(golden, current) + for change, keys in diff.items(): + for key in keys: + print(f"{change:8} {key}") + for capability in _unversioned_capabilities(diff): + print( + f"WARNING the {capability} seam changed but its version did not: " + "bump it if this is a contract change." + ) + GOLDEN_PATH.write_text(json.dumps(current, indent=2) + "\n") + print(f"wrote {len(current)} fingerprints to {GOLDEN_PATH}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py b/libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py deleted file mode 100644 index 572ade4d..00000000 --- a/libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Regenerate the golden schema hashes for the signatures inbound seam. - -Run ``python libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py``. The golden file pins the -JSON schema of every seam model; it is only regenerated on an -intentional, version-bumped contract change. -""" - -import hashlib -import json -from pathlib import Path - -from xtm_oaev_sdk.seam.signatures.seam import ( - CloudInjectorConfig, - ExecutionSignature, - NetworkInjectorConfig, - ToolErrorInfo, - ToolOutput, - ToolTimeoutInfo, -) - -GOLDEN_PATH = Path(__file__).parent / "fixtures" / "signatures_seam_schemas.json" - -SEAM_MODELS = { - "ToolErrorInfo": ToolErrorInfo, - "ToolTimeoutInfo": ToolTimeoutInfo, - "ToolOutput": ToolOutput, - "ExecutionSignature": ExecutionSignature, - "NetworkInjectorConfig": NetworkInjectorConfig, - "CloudInjectorConfig": CloudInjectorConfig, -} - - -def main() -> None: - out: dict[str, str] = {} - for name, cls in SEAM_MODELS.items(): - canonical = json.dumps(cls.model_json_schema(), sort_keys=True) - out[name] = hashlib.sha256(canonical.encode()).hexdigest() - with GOLDEN_PATH.open("w") as f: - json.dump(out, f, indent=2) - f.write("\n") - print(f"wrote {GOLDEN_PATH}") - - -if __name__ == "__main__": - main() diff --git a/libs/xtm-oaev-sdk/test/seam/test_port_io_schema_pin.py b/libs/xtm-oaev-sdk/test/seam/test_port_io_schema_pin.py deleted file mode 100644 index a5931ee9..00000000 --- a/libs/xtm-oaev-sdk/test/seam/test_port_io_schema_pin.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Schema-pinning guard for the port I/O data contract. - -Hashes each contract model's ``model_json_schema()`` and compares against -the committed golden file. Any drift in the wire schema fails this test. - -To intentionally update the contract: - 1. Bump ``PORT_IO_CONTRACT_VERSION`` in ``seam/port_io/models.py``. - 2. Regenerate the golden file: - ``python libs/xtm-oaev-sdk/test/seam/generate_port_io_golden.py`` - 3. Commit both changes together. -""" - -import ast -import hashlib -import json -from pathlib import Path - -from xtm_oaev_sdk.seam.port_io import ( - PORT_IO_CONTRACT_VERSION, - ExecutionDetails, - ExpectationSignatureGroup, - SignatureOutputStructure, - SignaturePayload, - SignatureTarget, - SignatureValue, - TargetSignatures, -) -from xtm_oaev_sdk.seam.port_io import models as port_io_models - -GOLDEN_PATH = Path(__file__).parent / "fixtures" / "port_io_schemas.json" - -CONTRACT_MODELS = { - "ExecutionDetails": ExecutionDetails, - "ExpectationSignatureGroup": ExpectationSignatureGroup, - "SignatureOutputStructure": SignatureOutputStructure, - "SignaturePayload": SignaturePayload, - "SignatureTarget": SignatureTarget, - "SignatureValue": SignatureValue, - "TargetSignatures": TargetSignatures, -} - - -def _schema_hash(model_cls) -> str: - """SHA-256 of the canonical JSON Schema for a pydantic model class.""" - schema = model_cls.model_json_schema() - canonical = json.dumps(schema, sort_keys=True) - return hashlib.sha256(canonical.encode()).hexdigest() - - -def _load_golden() -> dict[str, str]: - with GOLDEN_PATH.open() as f: - return json.load(f) - - -def test_port_io_contract_version_is_pinned(): - """The version constant must be a semver string, not a placeholder.""" - assert PORT_IO_CONTRACT_VERSION == "1.0.0" - - -def test_port_io_wire_schema_matches_golden(): - """Every contract model's JSON Schema must match the golden file. - - This is the anti-breaking guard: if a field is added, removed, renamed, - or retyped in any of the 7 wire models, this test fails. - """ - golden = _load_golden() - - # Guard against accidental golden file truncation. - assert len(golden) == len(CONTRACT_MODELS), ( - f"Golden file has {len(golden)} entries; expected {len(CONTRACT_MODELS)}. " - f"Missing: {set(CONTRACT_MODELS) - set(golden)}" - ) - - for name, cls in sorted(CONTRACT_MODELS.items()): - current = _schema_hash(cls) - expected = golden[name] - assert current == expected, ( - f"Schema drift detected in {name} (port I/O contract " - f"v{PORT_IO_CONTRACT_VERSION}).\n" - f" expected: {expected}\n" - f" actual: {current}\n" - f"If intentional, bump PORT_IO_CONTRACT_VERSION and regenerate " - f"the golden file." - ) - - -def test_port_io_models_import_only_the_signatures_seam(): - """The wire contract is closed: no import of the capability internals.""" - source = Path(port_io_models.__file__).read_text() - for node in ast.walk(ast.parse(source)): - if isinstance(node, ast.ImportFrom) and (node.module or "").startswith( - "xtm_oaev_sdk" - ): - assert node.module == "xtm_oaev_sdk.seam.signatures.seam", node.module - elif isinstance(node, ast.Import): - for alias in node.names: - assert not alias.name.startswith("xtm_oaev_sdk"), alias.name diff --git a/libs/xtm-oaev-sdk/test/seam/test_seam_closed.py b/libs/xtm-oaev-sdk/test/seam/test_seam_closed.py new file mode 100644 index 00000000..29f5e502 --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/test_seam_closed.py @@ -0,0 +1,36 @@ +"""The two wire-facing seam modules stay closed to the SDK internals.""" + +import ast +from pathlib import Path + +from xtm_oaev_sdk.seam.port_io import models as port_io_models +from xtm_oaev_sdk.seam.signatures import seam as seam_module + + +def test_port_io_models_import_only_the_signatures_seam(): + """The wire contract is closed: no import of the capability internals.""" + source = Path(port_io_models.__file__).read_text() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.ImportFrom) and (node.module or "").startswith( + "xtm_oaev_sdk" + ): + assert node.module == "xtm_oaev_sdk.seam.signatures.seam", node.module + elif isinstance(node, ast.Import): + for alias in node.names: + assert not alias.name.startswith("xtm_oaev_sdk"), alias.name + + +def test_seam_imports_nothing_from_capability(): + """The seam is pure data: no imports from xtm_oaev_sdk itself.""" + source = Path(seam_module.__file__).read_text() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + for alias in node.names: + assert not alias.name.startswith( + "xtm_oaev_sdk" + ), f"seam imports capability module {alias.name!r} — pure-data violation" + elif isinstance(node, ast.ImportFrom): + module = node.module or "" + assert not module.startswith( + "xtm_oaev_sdk" + ), f"seam imports from capability module {module!r} — pure-data violation" diff --git a/libs/xtm-oaev-sdk/test/seam/test_seam_golden.py b/libs/xtm-oaev-sdk/test/seam/test_seam_golden.py new file mode 100644 index 00000000..f1ab096a --- /dev/null +++ b/libs/xtm-oaev-sdk/test/seam/test_seam_golden.py @@ -0,0 +1,66 @@ +"""The seam golden: every public seam object is fingerprinted, and any drift fails. + +The fingerprints come from ``generate_seam_golden.py``, which discovers the seam +dynamically (every ``*.py`` file, every public top-level definition). On a failure, +``git diff`` the objects listed: they are exactly what changed. If the change is +deliberate, bump the capability's ``SEAM_VERSION`` (``PORT_IO_CONTRACT_VERSION`` for +port_io), then run, from the repository root:: + + python libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py +""" + +import importlib.util +import inspect +import json +from pathlib import Path +from types import ModuleType + +import xtm_oaev_sdk + +_GENERATOR = Path(__file__).parent / "generate_seam_golden.py" + + +def _given_generator() -> ModuleType: + spec = importlib.util.spec_from_file_location("generate_seam_golden", _GENERATOR) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _given_golden(generator: ModuleType) -> dict[str, dict[str, str]]: + return json.loads(generator.GOLDEN_PATH.read_text()) + + +def _when_fingerprinting_the_seam(generator: ModuleType) -> dict[str, dict[str, str]]: + return generator.compute_seam_fingerprints() + + +def _then_no_drift(diff: dict[str, list[str]]) -> None: + drift = [f"{change}: {key}" for change, keys in diff.items() for key in keys] + assert not drift, ( + "The seam drifted from its golden:\n " + + "\n ".join(drift) + + "\nIf deliberate: bump the capability's version, then run " + "`python libs/xtm-oaev-sdk/test/seam/generate_seam_golden.py`." + ) + + +def test_seam_matches_its_golden() -> None: + """It fails on any seam object added, removed or changed since the golden.""" + generator = _given_generator() + golden = _given_golden(generator) + current = _when_fingerprinting_the_seam(generator) + _then_no_drift(generator.diff_fingerprints(golden, current)) + + +def test_root_reexports_are_the_seam_objects() -> None: + """It hands out, from the package root, the very objects the seam defines.""" + generator = _given_generator() + broken = [ + f"{module}.{name}" + for module, name, _, obj in generator.iter_seam_definitions() + if (inspect.isclass(obj) or inspect.isfunction(obj)) + and hasattr(xtm_oaev_sdk, name) + and getattr(xtm_oaev_sdk, name) is not obj + ] + assert not broken, f"Root re-exports are not the seam objects: {broken}" diff --git a/libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py b/libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py deleted file mode 100644 index 6fa80d3a..00000000 --- a/libs/xtm-oaev-sdk/test/seam/test_signatures_seam_schema_pin.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Schema-pinning guard for the signatures inbound seam. - -Hashes each seam model's ``model_json_schema()`` and compares against -the committed golden file. Any drift in the seam schema fails this test. - -The seam is the frozen boundary both sides of the capability share, so -this guard also enforces the dependency rule (the seam imports nothing -from the capability) and the identity contract (every re-export layer -shares the exact same objects as the seam). - -To intentionally update the seam: - 1. Bump ``SEAM_VERSION`` in ``seam/signatures/seam.py``. - 2. Regenerate the golden file: - ``python libs/xtm-oaev-sdk/test/seam/generate_signatures_seam_golden.py`` - 3. Commit both changes together. -""" - -import ast -import hashlib -import json -from pathlib import Path - -from xtm_oaev_sdk.seam.signatures import ( - SEAM_VERSION, - CloudInjectorConfig, - ExecutionSignature, - NetworkInjectorConfig, - ToolErrorInfo, - ToolOutput, - ToolTimeoutInfo, -) -from xtm_oaev_sdk.seam.signatures import seam as seam_module - -GOLDEN_PATH = Path(__file__).parent / "fixtures" / "signatures_seam_schemas.json" - -SEAM_MODELS = { - "ToolErrorInfo": ToolErrorInfo, - "ToolTimeoutInfo": ToolTimeoutInfo, - "ToolOutput": ToolOutput, - "ExecutionSignature": ExecutionSignature, - "NetworkInjectorConfig": NetworkInjectorConfig, - "CloudInjectorConfig": CloudInjectorConfig, -} - - -def _schema_hash(model_cls) -> str: - """SHA-256 of the canonical JSON Schema for a pydantic model class.""" - schema = model_cls.model_json_schema() - canonical = json.dumps(schema, sort_keys=True) - return hashlib.sha256(canonical.encode()).hexdigest() - - -def test_seam_version_is_pinned(): - """The version constant must be the pinned semver string.""" - assert SEAM_VERSION == "1.0.0" - - -def test_seam_schema_is_pinned(): - """Every seam model's JSON schema must match the committed golden file.""" - with GOLDEN_PATH.open() as f: - golden = json.load(f) - for name, cls in SEAM_MODELS.items(): - assert name in golden, f"{name} missing from golden file" - actual = _schema_hash(cls) - assert actual == golden[name], ( - f"{name}: seam schema drift detected\n" - f" golden {golden[name]}\n" - f" actual {actual}\n" - "If this was intentional, bump SEAM_VERSION and regenerate the golden file." - ) - - -def test_seam_imports_nothing_from_capability(): - """The seam is pure data: no imports from xtm_oaev_sdk itself.""" - source = Path(seam_module.__file__).read_text() - for node in ast.walk(ast.parse(source)): - if isinstance(node, ast.Import): - for alias in node.names: - assert not alias.name.startswith( - "xtm_oaev_sdk" - ), f"seam imports capability module {alias.name!r} — pure-data violation" - elif isinstance(node, ast.ImportFrom): - module = node.module or "" - assert not module.startswith( - "xtm_oaev_sdk" - ), f"seam imports from capability module {module!r} — pure-data violation" - - -def test_reexports_preserve_identity(): - """The seam module, the seam package, and the public surface share one object.""" - import xtm_oaev_sdk as sdk - import xtm_oaev_sdk.seam.signatures as seam_package - - for name, cls in SEAM_MODELS.items(): - assert ( - getattr(seam_package, name) is cls - ), f"{name}: seam package re-export broke identity" - assert ( - getattr(sdk, name) is cls - ), f"{name}: public surface re-export broke identity" - assert sdk.InjectorConfig is seam_package.InjectorConfig diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py index fc6375a9..7bbfdb9b 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/__init__.py @@ -15,8 +15,8 @@ Any modification to the JSON schema of a model in this module is a breaking change for consumers and must bump :data:`PORT_IO_CONTRACT_VERSION`. -The schema-pinning guard (``tests/test_port_io_schema_pin.py``) hashes each -model's ``model_json_schema()`` against a golden file and fails on drift. +The seam golden (``test/seam/test_seam_golden.py``) fingerprints every public +object of the seam, this module included, and fails on drift. """ from xtm_oaev_sdk.seam.port_io.models import ( diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py index 5b641ef9..2bf0b685 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/port_io/models.py @@ -1,6 +1,6 @@ """The port I/O wire schema — the seam-owned models crossing the SDK↔platform boundary. -These eight pydantic models — together with the three wire-vocabulary enums +These pydantic models — together with the three wire-vocabulary enums they serialise (``SignatureExpectationType``, ``ExecutionStatus``, ``InjectExecutionActions``) — are the exact JSON shapes transmitted between the SDK's transport port (``SignatureTransportProtocol``) and the @@ -15,8 +15,8 @@ Any modification to the JSON schema of a model in this module is a breaking change for consumers and must bump :data:`PORT_IO_CONTRACT_VERSION`. -The schema-pinning guard (``tests/test_port_io_schema_pin.py``) hashes each -model's ``model_json_schema()`` against a golden file and fails on drift. +The seam golden (``test/seam/test_seam_golden.py``) fingerprints every public +object of the seam, this module included, and fails on drift. The module is **closed**: at runtime it imports only the standard library and pydantic. ``ToolOutput`` (from the signatures inbound seam, diff --git a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py index 13219309..6571d876 100644 --- a/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py +++ b/libs/xtm-oaev-sdk/xtm_oaev_sdk/seam/signatures/seam.py @@ -16,9 +16,8 @@ Any modification to the JSON schema of a model in this module is a breaking change for consumers and must bump :data:`SEAM_VERSION`. -The schema-pinning guard (``tests/test_signatures_seam_schema_pin.py``) -hashes each model's ``model_json_schema()`` against a golden file and -fails on drift. +The seam golden (``test/seam/test_seam_golden.py``) fingerprints every public +object of the seam, this module included, and fails on drift. """ from datetime import datetime, timezone From 71c3b566e4dc30767be95b693f75718a7a537a43 Mon Sep 17 00:00:00 2001 From: Christophe Melchior Date: Thu, 15 Oct 2026 08:50:23 +0200 Subject: [PATCH 22/22] :memo: docs(sdk): extending the seam golden TDR to collectors-sdk and injectors-sdk --- libs/xtm-oaev-sdk/CONTRIBUTING.md | 3 +++ .../docs/tdr/every-seam-object-is-fingerprinted.tdr.md | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/libs/xtm-oaev-sdk/CONTRIBUTING.md b/libs/xtm-oaev-sdk/CONTRIBUTING.md index d4fbdd7e..c270bf9d 100644 --- a/libs/xtm-oaev-sdk/CONTRIBUTING.md +++ b/libs/xtm-oaev-sdk/CONTRIBUTING.md @@ -87,6 +87,9 @@ on it No list to maintain. The next run picks up any new seam object. +collectors-sdk and injectors-sdk carry the same golden under their own `test/seam/`. There, +run the generator from `libs/`: `python test/seam/generate_seam_golden.py`. + ### What a fingerprint covers The sha256 of the definition's normalized AST. Formatting, comments and docstrings do diff --git a/libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md b/libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md index 1325fe81..9d420154 100644 --- a/libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md +++ b/libs/xtm-oaev-sdk/docs/tdr/every-seam-object-is-fingerprinted.tdr.md @@ -1,12 +1,12 @@ # Every seam object is fingerprinted --- -scope: [xtm-oaev-sdk, seam, testing, versioning] +scope: [xtm-oaev-sdk, collectors-sdk, injectors-sdk, seam, testing, versioning] status: draft superseded_by: supersedes: cross-process-seams-pin-their-json-schemas Aliases: Creation Date: 2026/10/15 08:36:35 -Modification Date: 2026/10/15 08:36:35 +Modification Date: 2026/10/15 08:49:12 --- ## ELI5 @@ -42,6 +42,7 @@ version and save the new fingerprint. - It rewrites `test/seam/fixtures/seam_golden.json`: changed fingerprints updated, new objects added, deleted ones removed. It prints what moved, and warns when a capability changed but its version did not. - `test/seam/test_seam_golden.py` recomputes the fingerprints and fails on anything added, removed or changed, and names every object. It also checks that the package root re-exports the seam's own objects. - `test/seam/test_seam_closed.py` keeps the two closed-import checks: `port_io` imports only the signatures seam, and the signatures seam imports nothing from the SDK. +- collectors-sdk and injectors-sdk carry the same generator and golden test under their own `test/seam/`, run from their `libs/` directory. - The step-by-step procedure lives in `CONTRIBUTING.md`, "Seam golden". ---