Version-tag release deploys; faster integration suite

.github/workflows/ci.yml

@@ -7,6 +7,12 @@ on:
 
 jobs:
   lint:
+    # A release (release-builder's PR, and its chore(release) commit on master)
+    # only bumps the version and CHANGELOG: nothing to test.
+    if: >-
+      !(github.event_name == 'pull_request' && github.head_ref == 'release'
+        && github.event.pull_request.user.login == 'github-actions[bot]')
+      && !(github.event_name == 'push' && startsWith(github.event.head_commit.message, 'chore(release):'))
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -24,6 +30,12 @@ jobs:
         run: script/lint
 
   test:
+    # A release (release-builder's PR, and its chore(release) commit on master)
+    # only bumps the version and CHANGELOG: nothing to test.
+    if: >-
+      !(github.event_name == 'pull_request' && github.head_ref == 'release'
+        && github.event.pull_request.user.login == 'github-actions[bot]')
+      && !(github.event_name == 'push' && startsWith(github.event.head_commit.message, 'chore(release):'))
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -42,9 +54,22 @@ jobs:
 
   # Real containers: a local registry, stub services, and an sshd, driven by
   # flow-deploy as a subprocess. Kept out of the unit job's coverage gate.
+  # Sharded across runners (pytest-split, balanced by the recorded
+  # tests/integration/.test_durations); locally, run it with -n auto instead.
   integration:
+    # A release (release-builder's PR, and its chore(release) commit on master)
+    # only bumps the version and CHANGELOG: nothing to test.
+    if: >-
+      !(github.event_name == 'pull_request' && github.head_ref == 'release'
+        && github.event.pull_request.user.login == 'github-actions[bot]')
+      && !(github.event_name == 'push' && startsWith(github.event.head_commit.message, 'chore(release):'))
+    name: integration (${{ matrix.group }}/4)
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1, 2, 3, 4]
     steps:
       - uses: actions/checkout@v4
 
@@ -58,4 +83,8 @@ jobs:
           poetry install --with test
 
       - name: Integration tests
-        run: poetry run pytest -m integration --no-cov -p no:randomly
+        run: >-
+          poetry run pytest -m integration --no-cov -p no:randomly
+          --splits 4 --group ${{ matrix.group }}
+          --splitting-algorithm least_duration
+          --durations-path tests/integration/.test_durations

.github/workflows/release.yml

@@ -65,7 +65,13 @@ jobs:
             dist/flow-deploy-linux-glibc
 
   build_and_deploy:
-    if: inputs.force_deploy || needs.release.outputs.created-tag
+    # build_release is skipped on a force_deploy run that cut no release; a
+    # plain `if:` implies success(), which would skip this job with it.
+    if: >-
+      !cancelled()
+      && needs.release.result == 'success'
+      && (needs.build_release.result == 'success' || needs.build_release.result == 'skipped')
+      && (inputs.force_deploy || needs.release.outputs.created-tag)
     name: Build & Deploy
     runs-on: ubuntu-latest
     needs: [release, build_release]
@@ -83,12 +89,30 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      # Deploy by the release's version tag (the git tag build_release made,
+      # v<version>), or by the commit SHA on a force_deploy run with no release.
+      - id: deploy_tag
+        name: Choose the deploy tag
+        env:
+          CREATED_TAG: ${{ needs.release.outputs.created-tag }}
+          VERSION_TAG: v${{ needs.release.outputs.current-version }}
+        run: |
+          if [ -n "$CREATED_TAG" ]; then
+            echo "tag=$VERSION_TAG" >> "$GITHUB_OUTPUT"
+          else
+            echo "tag=$GITHUB_SHA" >> "$GITHUB_OUTPUT"
+          fi
+
+      # The image is pushed under every name a deploy or a revert may use:
+      # the version tag (releases only), and the short and full SHA.
       - id: meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.IMAGE }}
           tags: |
+            type=raw,value=v${{ needs.release.outputs.current-version }},enable=${{ needs.release.outputs.created-tag != '' }}
             type=sha,prefix=
+            type=sha,format=long,prefix=
 
       - uses: docker/build-push-action@v6
         with:
@@ -98,7 +122,7 @@ jobs:
 
       - uses: flowcanon/deploy-action@master
         with:
-          tag: ${{ steps.meta.outputs.version }}
+          tag: ${{ steps.deploy_tag.outputs.tag }}
           ssh-key: ${{ secrets.DEPLOY_SSH_KEY }}
           host: ${{ vars.HOST_NAME }}
           user: ${{ vars.HOST_USER }}

pyproject.toml

@@ -23,6 +23,8 @@ autoflake = "^2.3"
 pytest = "^8.0"
 pytest-cov = "^6.0"
 pytest-randomly = "^3.16"
+pytest-xdist = "^3.6"
+pytest-split = "^0.10"
 
 [tool.poetry.scripts]
 flow-deploy = "flow_deploy.cli:main"
@@ -41,7 +43,7 @@ line_length = 100
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
-# Integration tests (real Docker) run separately: pytest -m integration --no-cov
+# Integration tests (real Docker) run separately, in parallel: pytest -m integration --no-cov -n auto
 addopts = "-v --tb=short --cov=flow_deploy --cov-report=term-missing -m 'not integration'"
 markers = ["integration: needs a real Docker daemon (tests/integration)"]
 

tests/integration/.test_durations

@@ -0,0 +1,35 @@
+{
+    "tests/integration/test_ci_cancel.py::test_with_a_forced_pty_the_remote_deploy_gets_sighup": 7.867385687306523,
+    "tests/integration/test_ci_cancel.py::test_without_a_pty_the_remote_deploy_gets_no_signal_and_runs_on[SIGKILL]": 8.759982107207179,
+    "tests/integration/test_ci_cancel.py::test_without_a_pty_the_remote_deploy_gets_no_signal_and_runs_on[SIGTERM]": 7.4407060500234365,
+    "tests/integration/test_deploys.py::test_a_clean_deploy_warns_about_nothing": 8.96708051301539,
+    "tests/integration/test_deploys.py::test_a_revert_waits_for_restored_services_and_exits_3_if_unhealthy": 11.525916080921888,
+    "tests/integration/test_deploys.py::test_annotated_version_tag_deploy": 9.19166273996234,
+    "tests/integration/test_deploys.py::test_another_project_with_other_services_in_the_directory_is_fine": 10.586324241012335,
+    "tests/integration/test_deploys.py::test_bad_tag_fails_at_pull_before_anything_stops": 3.823063874617219,
+    "tests/integration/test_deploys.py::test_containers_of_another_project_name_here_are_refused": 5.549123642966151,
+    "tests/integration/test_deploys.py::test_crash_looping_drain_first_worker_is_drained": 11.324078911915421,
+    "tests/integration/test_deploys.py::test_digest_pinned_worker_is_restored_by_the_previous_commit": 11.196486052125692,
+    "tests/integration/test_deploys.py::test_discovery_works_without_the_apps_env": 0.28856269642710686,
+    "tests/integration/test_deploys.py::test_drain_first_service_fails_its_healthcheck_in_phase_3": 16.68825369887054,
+    "tests/integration/test_deploys.py::test_drain_that_hits_its_timeout_is_killed_and_the_deploy_continues": 12.823152102530003,
+    "tests/integration/test_deploys.py::test_every_deploy_leaves_a_host_side_log": 23.279017873108387,
+    "tests/integration/test_deploys.py::test_failed_deploy_restores_the_version_that_was_running[annotated]": 23.906655395403504,
+    "tests/integration/test_deploys.py::test_failed_deploy_restores_the_version_that_was_running[lightweight]": 24.169602466747165,
+    "tests/integration/test_deploys.py::test_first_deploy_from_latest_restores_by_the_previous_commit": 17.861694214865565,
+    "tests/integration/test_deploys.py::test_first_deploy_of_a_new_drain_first_service": 8.649966936558485,
+    "tests/integration/test_deploys.py::test_flow_deploy_never_leaves_the_wrapper_to_find_deploy_tag": 10.473964996635914,
+    "tests/integration/test_deploys.py::test_happy_path": 13.902889674529433,
+    "tests/integration/test_deploys.py::test_lock_held_by_a_live_process_exits_2_and_changes_nothing": 3.1837950069457293,
+    "tests/integration/test_deploys.py::test_manual_compose_keeps_the_deployed_version": 10.220468347892165,
+    "tests/integration/test_deploys.py::test_manual_compose_refuses_when_nothing_tells_the_tag": 0.3125146795064211,
+    "tests/integration/test_deploys.py::test_moved_tag_deploys_its_new_commit_and_image": 15.277251683175564,
+    "tests/integration/test_deploys.py::test_rolling_cutover_that_has_to_kill_warns": 7.621987188234925,
+    "tests/integration/test_deploys.py::test_rolling_healthcheck_that_passes_late_still_deploys": 13.103993121534586,
+    "tests/integration/test_deploys.py::test_rolling_only_project_keeps_its_old_container_on_a_failed_healthcheck": 6.816878102719784,
+    "tests/integration/test_deploys.py::test_rolling_service_fails_its_healthcheck": 11.677399668842554,
+    "tests/integration/test_deploys.py::test_sha_deploy": 9.240587916225195,
+    "tests/integration/test_deploys.py::test_signal_mid_drain_reverts[SIGINT-to-process-group]": 16.096034040674567,
+    "tests/integration/test_deploys.py::test_signal_mid_drain_reverts[SIGINT]": 16.133966751396656,
+    "tests/integration/test_deploys.py::test_signal_mid_drain_reverts[SIGTERM]": 16.09249211475253
+}

tests/integration/conftest.py

@@ -5,10 +5,13 @@
 compose project of stub services pulled from a local registry. flow-deploy
 runs as a real subprocess, so tests can signal it, against real Docker.
 
-Run with:  poetry run pytest -m integration --no-cov
+Run with:  poetry run pytest -m integration --no-cov -n auto
+(-n auto runs the tests in parallel, one per core; drop it to run them in order.)
 Skipped automatically when Docker isn't available.
 """
 
+import contextlib
+import fcntl
 import json
 import os
 import shutil
@@ -55,9 +58,11 @@ def docker():
         pytest.skip("Docker Compose is too old: `docker compose up --wait` is required")
 
 
-@pytest.fixture(scope="session")
-def registry(docker):
-    """A local registry:2 with the stub image pushed under each STUB_TAGS tag."""
+def _start_registry() -> tuple[str, str]:
+    """Start a local registry:2 and push the stub image under each STUB_TAGS tag.
+
+    Returns (host, container name).
+    """
     name = f"flowdeploy-it-registry-{uuid.uuid4().hex[:8]}"
     run(["docker", "run", "-d", "--name", name, "-p", "127.0.0.1::5000", "registry:2"])
     try:
@@ -75,9 +80,57 @@ def registry(docker):
         # :latest starts out as v1, as a host first brought up by hand would be.
         run(["docker", "tag", f"{host}/stub:v1", f"{host}/stub:latest"])
         run(["docker", "push", "-q", f"{host}/stub:latest"])
-        yield host
-    finally:
+    except BaseException:
         run(["docker", "rm", "-f", name], check=False)
+        raise
+    return host, name
+
+
+@contextlib.contextmanager
+def _locked(path: Path):
+    with open(path, "a") as f:
+        fcntl.flock(f, fcntl.LOCK_EX)
+        yield
+
+
+@pytest.fixture(scope="session")
+def registry(docker, request, tmp_path_factory):
+    """A local registry with the stub images, shared by every test.
+
+    Under pytest-xdist (`-n auto`) every worker shares one: the first to need
+    it starts it, and the last to finish removes it. Tests only push tags
+    unique to them (per-stack commit SHAs, pid-stamped names), so sharing is
+    safe.
+    """
+    if not hasattr(request.config, "workerinput"):  # not under xdist
+        host, name = _start_registry()
+        try:
+            yield host
+        finally:
+            run(["docker", "rm", "-f", name], check=False)
+        return
+
+    shared = tmp_path_factory.getbasetemp().parent  # the same for every worker
+    state = shared / "registry.json"
+    with _locked(shared / "registry.lock"):
+        if state.exists():
+            data = json.loads(state.read_text())
+        else:
+            host, name = _start_registry()
+            data = {"host": host, "name": name, "users": 0}
+        data["users"] += 1
+        state.write_text(json.dumps(data))
+    try:
+        yield data["host"]
+    finally:
+        with _locked(shared / "registry.lock"):
+            data = json.loads(state.read_text())
+            data["users"] -= 1
+            if data["users"] == 0:
+                run(["docker", "rm", "-f", data["name"]], check=False)
+                state.unlink()
+            else:
+                state.write_text(json.dumps(data))
 
 
 def service(
@@ -137,7 +190,9 @@ def __init__(self, root: Path, registry: str):
         self.repo = root / "repo"
         run(["git", "init", "-q", "--bare", str(self.origin)])
         run(["git", "init", "-q", "-b", "main", str(self.repo)])
-        self.git("config", "user.email", "[email protected]")
+        # A per-stack author makes every commit SHA unique, so stacks pushing
+        # images under their SHAs can't collide in the shared registry.
+        self.git("config", "user.email", f"it-{self.project}@example.com")
         self.git("config", "user.name", "it")
         self.git("config", "commit.gpgsign", "false")
         self.git("remote", "add", "origin", str(self.origin))
@@ -165,7 +220,19 @@ def commit(self, tag, compose, annotated=False, force=False):
         """Commit ``compose`` as docker-compose.yml, tag it ``tag`` (unless None), push.
 
         ``force`` moves an existing tag, here and on origin, as a re-release does.
+
+        Every service gets ``network_mode: none``: the stubs never talk to each
+        other (health checks run inside each container), and without it each
+        stack's default network uses up one of Docker's address pools, which
+        run out at a few dozen, well short of running in parallel.
         """
+        compose = {
+            **compose,
+            "services": {
+                name: {"network_mode": "none", **svc}
+                for name, svc in compose.get("services", {}).items()
+            },
+        }
         (self.repo / "docker-compose.yml").write_text(yaml.safe_dump(compose, sort_keys=False))
         self.git("add", "docker-compose.yml")
         self.git("commit", "-q", "--allow-empty", "-m", tag or "untagged")

tests/integration/test_ci_cancel.py

@@ -21,7 +21,7 @@
 
 from tests.integration.conftest import IMAGES, SRC, run
 
-PROBE_SECONDS = 6
+PROBE_SECONDS = 4  # killed ~1s in; long enough to tell running on from dead
 
 
 @pytest.fixture(scope="module")