Feat: credential and auth model (tokens, proxy, delagation)

docs/user/reference/api.md

@@ -11,6 +11,22 @@ Anything else, including every name starting with `_`, is internal.
 
 ::: intercede.models
 
+## Configuration
+
+::: intercede.config
+
+## Registry
+
+::: intercede.registry
+
+## Credentials
+
+::: intercede.auth.models
+
+::: intercede.auth.protocols
+
+::: intercede.auth.providers
+
 ## Exceptions
 
 ::: intercede.exceptions

mkdocs.yml

@@ -17,6 +17,12 @@ plugins:
             show_source: false
             members_order: source
             merge_init_into_class: true
+            inherited_members: true
+            # Field tables for the configuration models, read from the models
+            # themselves, so the documentation cannot drift from the code.
+            extensions:
+              - griffe_pydantic:
+                  schema: false
 
 validation:
   links:

pyproject.toml

@@ -37,6 +37,7 @@ Repository = "https://github.com/DIRACGrid/intercede"
 testing = [
   "pytest",
   "pytest-asyncio",
+  "pytest-cov",
   "pytest-xdist",
   "pyyaml",
   "ty",
@@ -59,6 +60,19 @@ markers = [
   "destructive_fetch: asserts one-shot/destructive output-retrieval semantics",
 ]
 
+[tool.coverage.run]
+branch = true
+patch = ["subprocess"]  # also measure the tests that start a fresh interpreter
+source = ["intercede"]
+
+[tool.coverage.report]
+show_missing = true
+skip_covered = true
+exclude_also = [
+  "if TYPE_CHECKING:",
+  "^\\s*\\.\\.\\.$",  # protocol method bodies
+]
+
 [tool.mypy]
 strict = true
 plugins = ["pydantic.mypy"]
@@ -100,6 +114,9 @@ ignore = [
   "FLY002",
 ]
 
+[tool.ruff.lint.pydocstyle]
+convention = "google"
+
 [tool.ruff.lint.isort]
 required-imports = ["from __future__ import annotations"]
 
@@ -144,6 +161,7 @@ intercede = { path = ".", editable = true }
 mkdocs = "<2"
 mkdocs-material = "*"
 mkdocstrings = { version = "*", extras = ["python"] }
+griffe-pydantic = "*"
 [tool.pixi.feature.docs.tasks]
 docs = { cmd = "mkdocs build --strict", description = "Build the documentation" }
 

src/intercede/__init__.py

@@ -5,10 +5,21 @@
 from importlib.metadata import PackageNotFoundError
 from importlib.metadata import version as get_version
 
+from intercede import registry
+from intercede.config import (
+    ARCConfig,
+    BackendConfig,
+    BatchLocalConfig,
+    BatchSSHConfig,
+    CloudConfig,
+    HTCondorCEConfig,
+    validate_config,
+)
 from intercede.exceptions import (
     AuthenticationError,
     ConfigurationError,
     InterCEdeError,
+    RegistryError,
     ResourceUnavailableError,
     SpecificationRejectedError,
 )
@@ -47,6 +58,7 @@
 
 __all__ = [
     "__version__",
+    "registry",
     # protocols
     "Diagnosable",
     "JobBackend",
@@ -71,10 +83,19 @@
     "StatusMap",
     "Submission",
     "SubmissionSpec",
+    # configuration
+    "ARCConfig",
+    "BackendConfig",
+    "BatchLocalConfig",
+    "BatchSSHConfig",
+    "CloudConfig",
+    "HTCondorCEConfig",
+    "validate_config",
     # exceptions
     "AuthenticationError",
     "ConfigurationError",
     "InterCEdeError",
+    "RegistryError",
     "ResourceUnavailableError",
     "SpecificationRejectedError",
 ]

src/intercede/_backends/config.py

@@ -0,0 +1,18 @@
+"""Settings of the in-memory fake, which no public configuration reaches.
+
+Every other backend is built from its public configuration model in
+``intercede.config``.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from intercede.models import _Model
+
+
+class FakeSettings(_Model):
+    """The in-memory fake, for the test suites."""
+
+    kind: Literal["fake"] = "fake"
+    resource: str = "fake://local"

src/intercede/_backends/credentials.py

@@ -0,0 +1,158 @@
+"""What a backend asks its credential provider for, when, and what it accepts back.
+
+``credential_requirements`` turns a configuration into what to ask for. This is
+also the one place freshness is decided. A backend keeps the set it was
+last given, privately, and before each operation calls ``refresh_if_stale``:
+the provider is asked only when that set is missing a required kind or one of
+its credentials is about to expire. Where credentials come from, and how they
+are renewed, stays the consumer's business, inside its provider.
+"""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
+
+from intercede.auth.models import (
+    BearerToken,
+    Credential,
+    CredentialRequirements,
+    CredentialSet,
+    SSHPassword,
+    SSHPrivateKey,
+    X509Proxy,
+)
+from intercede.auth.protocols import CredentialProvider
+from intercede.config import (
+    ARCConfig,
+    BackendConfig,
+    BatchSSHConfig,
+    HTCondorCEConfig,
+)
+from intercede.exceptions import AuthenticationError, InterCEdeError
+
+# What each configured auth mode asks the credential provider for.
+# "agent" is absent on purpose: an SSH agent holds the key, so nothing is asked.
+AUTH_KINDS: dict[str, type[Credential]] = {
+    "token": BearerToken,
+    "proxy": X509Proxy,
+    "key": SSHPrivateKey,
+    "password": SSHPassword,
+}
+
+# Ask again this long before a credential expires, so it cannot run out in the
+# middle of an operation. A library default; whether a provider should be able
+# to change it is still open.
+DEFAULT_MARGIN = timedelta(minutes=5)
+
+
+def credential_requirements(
+    config: BackendConfig, scopes: frozenset[str] = frozenset()
+) -> CredentialRequirements | None:
+    """Say what a backend must ask its provider for.
+
+    Args:
+        config: The backend's validated configuration.
+        scopes: The scopes of the operation about to run, for token-based
+            backends; ignored by the others.
+
+    Returns:
+        What to ask for, or None when the backend needs no credential: it runs
+        on this host, or an SSH agent holds the key.
+    """
+    if isinstance(config, ARCConfig | HTCondorCEConfig):
+        # The endpoint is the resource, so a token is minted for this CE's audience.
+        return CredentialRequirements(
+            resource=config.endpoint,
+            kinds=frozenset(AUTH_KINDS[mode] for mode in config.auth),
+            scopes=scopes,
+        )
+    if isinstance(config, BatchSSHConfig) and config.auth != "agent":
+        # The resource is the login, ``user@host:port``: a provider finds the key
+        # or password by it, and the same host can take different users.
+        return CredentialRequirements(
+            resource=f"{config.user}@{config.host}:{config.port}",
+            kinds=frozenset({AUTH_KINDS[config.auth]}),
+        )
+    return None
+
+
+def is_stale(
+    credentials: CredentialSet | None,
+    requirements: CredentialRequirements,
+    *,
+    margin: timedelta = DEFAULT_MARGIN,
+    now: datetime | None = None,
+) -> bool:
+    """Tell whether ``credentials`` cannot serve ``requirements`` for ``margin`` more.
+
+    Stale when there are no credentials yet, when a required kind is missing,
+    or when a required credential expires within ``margin``. A credential that
+    never expires is never stale.
+    """
+    if credentials is None:
+        return True
+    moment = now or datetime.now(UTC)
+    for kind in requirements.kinds:
+        credential = credentials.get(kind)
+        if credential is None:
+            return True
+        if (
+            credential.expires_at is not None
+            and credential.expires_at - margin <= moment
+        ):
+            return True
+    return False
+
+
+async def refresh_if_stale(
+    current: CredentialSet | None,
+    requirements: CredentialRequirements,
+    provider: CredentialProvider,
+    *,
+    margin: timedelta = DEFAULT_MARGIN,
+    now: datetime | None = None,
+) -> CredentialSet:
+    """Return ``current`` while it is fresh, otherwise a new set from ``provider``.
+
+    Raises:
+        AuthenticationError: The provider failed, left out a required kind, or
+            returned a credential that has already expired. Always a failure of
+            the whole operation, never a per-job outcome.
+
+    """
+    if not is_stale(current, requirements, margin=margin, now=now):
+        assert current is not None  # is_stale() is True for None
+        return current
+
+    try:
+        fresh = await provider.get(requirements)
+    except InterCEdeError:
+        raise  # already typed, e.g. an AuthenticationError from the provider
+    except Exception as exc:
+        raise AuthenticationError(
+            f"the credential provider failed for {requirements.resource}: {exc}"
+        ) from exc
+
+    if not isinstance(fresh, CredentialSet):
+        raise AuthenticationError(
+            f"the credential provider returned a {type(fresh).__name__} "
+            f"for {requirements.resource}, not a CredentialSet"
+        )
+    missing = []
+    for kind in requirements.kinds:
+        if fresh.get(kind) is None:
+            missing.append(kind.__name__)
+    if missing:
+        raise AuthenticationError(
+            # Sorted: kinds is a set, whose order changes from run to run.
+            f"the credential provider returned no {', '.join(sorted(missing))} "
+            f"for {requirements.resource}"
+        )
+    # Expiring within the margin is accepted (it is asked again next time);
+    # already expired is not, since it cannot authenticate anything.
+    if is_stale(fresh, requirements, margin=timedelta(0), now=now):
+        raise AuthenticationError(
+            f"the credential provider returned expired credentials "
+            f"for {requirements.resource}"
+        )
+    return fresh

src/intercede/_backends/fake.py

@@ -6,12 +6,14 @@
 
 from __future__ import annotations
 
-from collections.abc import Collection, Mapping, Sequence
+from collections.abc import Callable, Collection, Mapping, Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
 from types import TracebackType
 from typing import Self
 
+from intercede._backends.config import FakeSettings
+from intercede.auth.protocols import CredentialProvider
 from intercede.exceptions import (
     InterCEdeError,
     ResourceUnavailableError,
@@ -92,6 +94,22 @@ def __init__(
         self._next_id = 0
         self._closed = False
 
+    @classmethod
+    def from_settings(
+        cls, settings: FakeSettings, credentials: CredentialProvider
+    ) -> Self:
+        """Build the fake from its settings; how the registry constructs it.
+
+        Args:
+            settings: The fake's settings: only the resource it reports.
+            credentials: Never asked: nothing leaves the process, so the fake
+                needs no credentials.
+
+        Returns:
+            A new fake, not yet opened.
+        """
+        return cls(resource=settings.resource)
+
     # -- lifecycle ---------------------------------------------------------
 
     async def __aenter__(self) -> Self:
@@ -252,10 +270,8 @@ def _check_supported(self, spec: SubmissionSpec) -> None:
 
         if not self.resource_staging:
             self._check_can_fetch(spec.inputs)
-            if spec.outputs is not None:
+            if spec.outputs is not None:  # no outputs means nothing to upload
                 self._check_can_upload(spec.outputs.members)
-            else:
-                raise SpecificationRejectedError(f"tag: {spec.tag} contains no outputs")
 
     def _check_container_support(self, spec: SubmissionSpec) -> None:
         if spec.container is not None and not self.containers:
@@ -368,3 +384,6 @@ def _write_job_files(
 _kill: type[Killable] = FakeBackend
 _purge: type[Purgeable] = FakeBackend
 _diag: type[Diagnosable] = FakeBackend
+_built_by_registry: Callable[[FakeSettings, CredentialProvider], JobBackend] = (
+    FakeBackend.from_settings
+)