refactor of testing suite

.github/workflows/ci.yml

@@ -35,7 +35,7 @@ jobs:
           --password postgres \
           --database-name test_db \
           --test-only false
-        uv run omop-config databases add cdm test_cdm_db --connection pg_test --cdm-schema public
+        uv run omop-config databases add cdm test_cdm_db --connection pg_test --cdm-schema gw_primary --vocab-schema gw_vocab
         uv run omop-config databases add generic test_vector_db --connection pg_test --schema-name embeddings
         uv run omop-config vector-stores add test_embeddings \
           --backend-type pgvector \
@@ -44,3 +44,49 @@ jobs:
           --cdm-db test_cdm_db \
           --vector-store-name test_embeddings
         uv run python -m tests.support.seed_postgres_fixture
+  test-evidence:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@v6
+      - name: Run hermetic suite with coverage and skip reasons
+        run: |
+          uv sync --frozen --all-extras --dev
+          export GROUNDWORKERS_REQUIRE_FORMAT_TESTS=1
+          echo "GROUNDWORKERS_REQUIRE_FORMAT_TESTS=1" >> "$GITHUB_ENV"
+          COVERAGE_FILE="$RUNNER_TEMP/groundworkers.coverage" uv run pytest -q -ra \
+            --cov=groundworkers --cov-branch \
+            --cov-report=xml:"$RUNNER_TEMP/coverage.xml" \
+            --junitxml="$RUNNER_TEMP/pytest.xml"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: groundworkers-test-evidence
+          path: |
+            ${{ runner.temp }}/coverage.xml
+            ${{ runner.temp }}/pytest.xml
+          if-no-files-found: warn
+  wheel-smoke:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: astral-sh/setup-uv@v6
+      - name: Build and smoke-test base and optional wheel installs
+        run: |
+          uv build --wheel
+          wheel="$(find dist -maxdepth 1 -name '*.whl' -print -quit)"
+          wheel="$(pwd)/$wheel"
+          uv venv "$RUNNER_TEMP/gw-wheel-base" --python 3.12
+          uv pip install --python "$RUNNER_TEMP/gw-wheel-base/bin/python" "$wheel"
+          mkdir -p "$RUNNER_TEMP/gw-wheel-run"
+          cd "$RUNNER_TEMP/gw-wheel-run"
+          "$RUNNER_TEMP/gw-wheel-base/bin/python" "$GITHUB_WORKSPACE/tests/support/wheel_smoke.py" base
+          uv venv "$RUNNER_TEMP/gw-wheel-extras" --python 3.12
+          uv pip install --python "$RUNNER_TEMP/gw-wheel-extras/bin/python" "$wheel[all_source,tui]" httpx
+          "$RUNNER_TEMP/gw-wheel-extras/bin/python" "$GITHUB_WORKSPACE/tests/support/wheel_smoke.py" extras

docs/usage/testing.md

@@ -1,34 +1,45 @@
 # Testing
 
+## Test lanes
+
+Groundworkers uses distinct lanes because an isolated unit run cannot prove the database, optional parser, plugin-entry-point, or built-wheel contracts.
+
+| Lane | What it proves | CI behavior |
+|---|---|---|
+| Unit and service | Deterministic policy, adapter boundaries, source planning, transport behavior, and controlled provider responses | Runs without a live database or model service |
+| Optional dependency | Real XLSX, DOCX, and PDF parsing plus missing-extra guidance | Required format tests must pass in the all-extras CI lane; missing-extra behavior is simulated |
+| PostgreSQL integration | Seeded graph retrieval, real pgvector storage and reader behavior, model selection, filters, and embedding-tier grounding | Required: every required_integration case must be collected and pass |
+| Wheel smoke | Base and optional-extra installations, package entry-point discovery, CLI startup, and successful MCP/REST planning operations | Builds once and runs outside the checkout |
+| Optional live provider | Real configured model-provider connectivity | Opt-in developer smoke only; never substitutes for seeded embedding tests |
+
 ## Running tests
 
-```bash
-# all tests (unit + integration; integration tests are skipped without a live DB)
-pytest
+~~~bash
+# Hermetic suite, including real optional-format parsers
+uv run pytest -q -ra
+
+# Unit tests only
+uv run pytest tests/unit/
 
-# unit tests only (no database required)
-pytest tests/unit/
+# Required PostgreSQL cases; provide a seeded test stack first
+GROUNDWORKERS_CONFIG_PATH=/path/to/stack.toml \
+GROUNDWORKERS_REQUIRE_INTEGRATION=1 \
+GROUNDWORKERS_TEST_DATABASE_URL=postgresql+psycopg://... \
+uv run pytest -m required_integration -q -ra
 
-# single file
-pytest tests/unit/test_concept_tools.py
-```
+# Coverage for local review
+COVERAGE_FILE=/tmp/groundworkers.coverage uv run pytest -q -ra \
+  --cov=groundworkers --cov-branch --cov-report=term-missing
+~~~
 
-## Test configuration
+The required integration gate derives its cases from the `required_integration` marker, so every marked case is checked for collection and a passing result. Missing configuration, unavailable adapters, and absent seeded rows fail in that lane. The all-extras CI job applies the same execution check to `required_format` cases, so a missing parser cannot silently turn the format contract into a skip. Without the corresponding `GROUNDWORKERS_REQUIRE_*` flag, PostgreSQL and optional-format fixtures remain opt-in and skip with a reason.
 
-Integration tests that query the OMOP database require a `.env` file at the repo root (or an environment-variable equivalent):
+## Configuration and optional provider smoke
 
-```dotenv
-OA_CONFIG_PATH=/path/to/stack.toml
-```
+Set GROUNDWORKERS_CONFIG_PATH (or GROUNDWORKERS_CONFIG) to the explicit Groundworkers stack file for integration tests. OA_CONFIG_PATH selects the oa-configurator default stack, but does not by itself opt the graph integration tests into a required run. PostgreSQL CI sets both paths and seeds a compact fixture before pytest starts.
 
-Unit tests mock the adapter layer and run without a live database.
+The external-provider smoke tests are separate from seeded deterministic embedding tests. To run them locally, set GROUNDWORKERS_RUN_EMBEDDING_INTEGRATION=1, GROUNDWORKERS_CONFIG_PATH, and GROUNDWORKERS_TEST_CONCEPT_ID; they require a configured provider and a concept with stored vectors.
 
-## What is tested
+## Coverage and skip evidence
 
-| Area | Scope |
-|---|---|
-| Concept tools | Deterministic concept lookup, hierarchy, relationship, path, neighborhood, classified-edge, and standard-mapping tools: correct responses, `NOT_FOUND` errors, `INVALID_INPUT` guards, and depth clamping |
-| Resolver tools | `concept_ground`: empty query rejection, result ordering by score, match_kind on exact hits |
-| Embedding tools | `embedding_index_status`, `embedding_neighbours`, `embedding_search`, `embedding_search_batch`, `embedding_encode`: correct responses, clamped limits, aligned batch rows, error propagation |
-| Server registry | `create_server` with and without each adapter; system tools always present; adapter-conditional tool registration |
-| Config validation | oa-configurator stack validation, Groundworkers package fields, and extra field rejection |
+CI publishes coverage.xml and pytest.xml from the hermetic all-extras run. The JUnit report carries named skip reasons. PostgreSQL CI separately enforces the required test list, so optional local skips cannot satisfy that gate. Coverage percentages are review evidence, not a global pass threshold. Prioritize unexecuted behavior at database, provider, transport, and packaging boundaries.

pyproject.toml

@@ -112,5 +112,8 @@ include = ["src"]
 testpaths = ["tests"]
 markers = [
   "integration: requires a live backend or database",
+  "required_integration: deterministic integration contract required by CI",
+  "required_format: optional document-format contract required in all-extras CI",
+  "optional_integration: environment-dependent integration smoke test",
   "requires_client: requires an embedding API client",
 ]

src/groundworkers/plugins.py

@@ -20,7 +20,6 @@
 from importlib.metadata import entry_points
 from typing import TYPE_CHECKING, Any, ClassVar, Protocol, runtime_checkable
 
-from groundskeeping.contracts import CommandPlan
 from oa_configurator import (  # type: ignore[import-untyped]
     PackageConfigBase,
     ResolvedDatabase,
@@ -36,6 +35,7 @@
         ConfigWorkflowSpec,
         MutationOperation,
     )
+    from groundskeeping.contracts import CommandPlan
 
     from groundworkers.services.grounding import ConceptGroundingService
     from groundworkers.services.mapping import MappingService

src/groundworkers/services/source_planning/assisted.py

@@ -9,7 +9,7 @@
 
 from dataclasses import replace
 
-from pydantic import BaseModel, Field, ValidationError
+from pydantic import BaseModel, ConfigDict, Field, ValidationError
 
 from groundworkers.adapters.llm import LLMAdapter
 from groundworkers.base.errors import GroundworkersError
@@ -31,6 +31,8 @@
 
 
 class AssistedColumnDecision(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
     header: str
     role: str | None = None
     confidence: float = Field(ge=0.0, le=1.0)
@@ -40,6 +42,8 @@ class AssistedColumnDecision(BaseModel):
 
 
 class AssistedClassificationResult(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
     decisions: list[AssistedColumnDecision] = Field(default_factory=list)
 
 

tests/conftest.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 import sys
 from pathlib import Path
 
@@ -8,6 +9,13 @@
 ROOT = Path(__file__).resolve().parents[1]
 SRC = ROOT / "src"
 
+_REQUIRED_TEST_MARKERS = {
+    "GROUNDWORKERS_REQUIRE_INTEGRATION": "required_integration",
+    "GROUNDWORKERS_REQUIRE_FORMAT_TESTS": "required_format",
+}
+_required_test_ids: dict[str, set[str]] = {}
+_required_test_outcomes: dict[str, dict[str, str]] = {}
+
 if str(SRC) not in sys.path:
     sys.path.insert(0, str(SRC))
 
@@ -31,3 +39,76 @@ def _isolated_env_file(tmp_path_factory, monkeypatch):
     # it leaves every later test looking like it was launched with a broken
     # OA_CONFIG_PATH -- which makes the setup page open the location wizard.
     monkeypatch.setattr(_env, "_rejected", None)
+
+
+def _missing_required_tests(expected: set[str], selected: set[str]) -> list[str]:
+    return sorted(expected - selected)
+
+
+def _required_test_failures(
+    expected: set[str], outcomes: dict[str, str]
+) -> list[str]:
+    return sorted(nodeid for nodeid in expected if outcomes.get(nodeid) != "passed")
+
+
+@pytest.hookimpl(hookwrapper=True, tryfirst=True)
+def pytest_collection_modifyitems(session: pytest.Session, items: list[pytest.Item]):
+    enabled = {
+        marker
+        for variable, marker in _REQUIRED_TEST_MARKERS.items()
+        if os.getenv(variable) == "1"
+    }
+    expected = {
+        marker: {item.nodeid for item in items if item.get_closest_marker(marker)}
+        for marker in enabled
+    }
+    yield
+    selected_items = {item.nodeid for item in session.items}
+    _required_test_ids.clear()
+    _required_test_ids.update(expected)
+    for marker in enabled:
+        _required_test_outcomes[marker] = {}
+    session.config._required_test_ids = expected
+    session.config._required_test_collection_errors = {
+        marker: _missing_required_tests(ids, selected_items)
+        for marker, ids in expected.items()
+    }
+    session.config._required_test_outcomes = {
+        marker: _required_test_outcomes.setdefault(marker, {}) for marker in enabled
+    }
+
+
+def pytest_runtest_logreport(report: pytest.TestReport) -> None:
+    if report.when not in {"setup", "call", "teardown"}:
+        return
+    for marker, outcomes in _required_test_outcomes.items():
+        if report.when == "call" or report.outcome != "passed":
+            if report.nodeid in _required_test_ids.get(marker, set()):
+                outcomes[report.nodeid] = report.outcome
+
+
+def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
+    required = getattr(session.config, "_required_test_ids", {})
+    collection_errors = getattr(session.config, "_required_test_collection_errors", {})
+    outcomes = getattr(session.config, "_required_test_outcomes", {})
+    problems = {
+        marker: sorted(
+            set(collection_errors.get(marker, []))
+            | set(_required_test_failures(ids, outcomes.get(marker, {})))
+        )
+        for marker, ids in required.items()
+    }
+    for marker, ids in required.items():
+        if not ids:
+            problems[marker].append(f"no tests collected with {marker} marker")
+    failures = {marker: entries for marker, entries in problems.items() if entries}
+    if not failures:
+        return
+    reporter = session.config.pluginmanager.get_plugin("terminalreporter")
+    if reporter is not None:
+        for marker, entries in failures.items():
+            reporter.write_sep("!", f"required {marker} execution gate failed")
+            for nodeid in entries:
+                outcome = outcomes.get(marker, {}).get(nodeid, "not executed or deselected")
+                reporter.write_line(f"{nodeid}: {outcome}")
+    session.exitstatus = pytest.ExitCode.TESTS_FAILED

tests/integration/test_omop_emb_adapter.py

@@ -42,7 +42,7 @@ def _load_integration_adapter():
     return adapter
 
 
-@pytest.mark.integration
+@pytest.mark.optional_integration
 def test_index_status_returns_at_least_one_model():
     adapter = _load_integration_adapter()
 
@@ -59,7 +59,7 @@ def _get_test_concept_id() -> int:
     return int(value)
 
 
-@pytest.mark.integration
+@pytest.mark.optional_integration
 def test_get_neighbours_returns_sorted_results():
     adapter = _load_integration_adapter()
     status = adapter.index_status()
@@ -71,11 +71,12 @@ def test_get_neighbours_returns_sorted_results():
         model_name=status["models"][0]["model_name"],
     )
 
+    assert result["results"], "configured embedding smoke test returned no neighbours"
     similarities = [item["similarity"] for item in result["results"]]
     assert similarities == sorted(similarities, reverse=True)
 
 
-@pytest.mark.integration
+@pytest.mark.optional_integration
 def test_get_neighbours_excludes_query_concept():
     adapter = _load_integration_adapter()
     status = adapter.index_status()
@@ -87,10 +88,11 @@ def test_get_neighbours_excludes_query_concept():
         model_name=status["models"][0]["model_name"],
     )
 
+    assert result["results"], "configured embedding smoke test returned no neighbours"
     assert all(item["concept_id"] != concept_id for item in result["results"])
 
 
-@pytest.mark.integration
+@pytest.mark.optional_integration
 def test_get_neighbours_unknown_concept_raises_not_found():
     adapter = _load_integration_adapter()
     status = adapter.index_status()