From 45c9e778f200adfe214b887190110861be76ab9b Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 5 Nov 2026 11:23:15 -0500 Subject: [PATCH 1/3] ci: restore complete claims JUnit checks Signed-off-by: abrichr --- .github/ci-tools/pyproject.toml | 1 + .github/ci-tools/uv.lock | 78 +++++- .github/workflows/packages.yml | 56 ++++- packages/flow/docs/VERIFICATION.md | 4 +- packages/flow/scripts/validate_claims.py | 9 +- packages/flow/tests/nightly_only.txt | 2 - scripts/ci/merge_flow_junit.py | 91 +++++++ tests/test_flow_claims_ci.py | 304 +++++++++++++++++++++++ 8 files changed, 530 insertions(+), 15 deletions(-) create mode 100644 scripts/ci/merge_flow_junit.py create mode 100644 tests/test_flow_claims_ci.py diff --git a/.github/ci-tools/pyproject.toml b/.github/ci-tools/pyproject.toml index b142fc37fc..3c657f7c3f 100644 --- a/.github/ci-tools/pyproject.toml +++ b/.github/ci-tools/pyproject.toml @@ -16,6 +16,7 @@ test = ["pytest>=9.1.1", "packaging>=25.0"] build = ["build>=1.6.1", "pip>=25.3"] pip = ["pip>=25.3"] lint = ["ruff==0.17.0"] +claims = ["pyyaml>=6.0"] release = ["build>=1.6.1", "twine>=7.0.0"] # The in-guest Windows agent's screen and input modules, for the hosted # Windows replay (desktop-replay.yml), which runs Python 3.12. CI only: Flow diff --git a/.github/ci-tools/uv.lock b/.github/ci-tools/uv.lock index 1b15787978..b6937bd057 100644 --- a/.github/ci-tools/uv.lock +++ b/.github/ci-tools/uv.lock @@ -1,9 +1,9 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.10" resolution-markers = [ "python_full_version >= '3.12' and sys_platform == 'win32'", - "(python_full_version >= '3.11' and sys_platform != 'win32') or (python_full_version == '3.11.*' and sys_platform == 'win32')", + "python_full_version == '3.11.*' or (python_full_version >= '3.12' and sys_platform != 'win32')", "python_full_version < '3.11'", ] @@ -383,7 +383,7 @@ name = "importlib-metadata" version = "10.1.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "(python_full_version >= '3.11' and sys_platform != 'win32') or (python_full_version == '3.11.*' and sys_platform == 'win32')", + "python_full_version == '3.11.*' or (python_full_version >= '3.12' and sys_platform != 'win32')", ] dependencies = [ { name = "zipp" }, @@ -507,7 +507,7 @@ version = "11.2.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.12' and sys_platform == 'win32'", - "(python_full_version >= '3.11' and sys_platform != 'win32') or (python_full_version == '3.11.*' and sys_platform == 'win32')", + "python_full_version == '3.11.*' or (python_full_version >= '3.12' and sys_platform != 'win32')", ] sdist = { url = "https://files.pythonhosted.org/packages/52/b8/c63995bebbb9ac9c059350e0fb8c82c2420c85aeca7d34e40aecf3df9281/more_itertools-11.2.1.tar.gz", hash = "sha256:cbf08fd0af284dc69718b9b76a8e6203df624d70209ea511e4219d350c856f63", size = 154133, upload-time = "2026-10-14T18:50:30.407Z" } wheels = [ @@ -576,6 +576,9 @@ build = [ { name = "build" }, { name = "pip" }, ] +claims = [ + { name = "pyyaml" }, +] lint = [ { name = "ruff" }, ] @@ -602,6 +605,7 @@ build = [ { name = "build", specifier = ">=1.6.1" }, { name = "pip", specifier = ">=25.3" }, ] +claims = [{ name = "pyyaml", specifier = ">=6.0" }] lint = [{ name = "ruff", specifier = "==0.17.0" }] pip = [{ name = "pip", specifier = ">=25.3" }] release = [ @@ -674,7 +678,7 @@ name = "pycparser" version = "3.12" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "(python_full_version >= '3.11' and sys_platform != 'win32') or (python_full_version == '3.11.*' and sys_platform == 'win32')", + "python_full_version == '3.11.*' or (python_full_version >= '3.12' and sys_platform != 'win32')", ] sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } wheels = [ @@ -771,6 +775,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, ] +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + [[package]] name = "readme-renderer" version = "46.0" diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.yml index 48e9ba0828..b36e6e232a 100644 --- a/.github/workflows/packages.yml +++ b/.github/workflows/packages.yml @@ -114,6 +114,7 @@ jobs: timeout-minutes: 5 outputs: flow: ${{ steps.classify.outputs.flow }} + flow_unit_shards: ${{ steps.classify.outputs.flow_unit_shards }} types: ${{ steps.classify.outputs.types }} agent: ${{ steps.classify.outputs.agent }} capture: ${{ steps.classify.outputs.capture }} @@ -136,6 +137,9 @@ jobs: GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha }} run: | set -euo pipefail + # Both the matrix and the claims consumer use this exact count. + if [ "$EVENT_NAME" = pull_request ]; then unit_shards=3; else unit_shards=2; fi + echo "flow_unit_shards=$unit_shards" >> "$GITHUB_OUTPUT" all() { for pkg in flow types agent capture privacy desktop tray program_graph weight; do echo "$pkg=true" >> "$GITHUB_OUTPUT"; done exit 0 @@ -439,7 +443,7 @@ jobs: # together: a pull request's fast lane takes three shards, about five # minutes each, to fit `ci` in ten minutes. The full suite on main and # nightly takes two. - include: ${{ fromJSON(github.event_name == 'pull_request' && '[{"shard":1,"of":3},{"shard":2,"of":3},{"shard":3,"of":3}]' || '[{"shard":1,"of":2},{"shard":2,"of":2}]') }} + include: ${{ fromJSON(needs.scope.outputs.flow_unit_shards == '3' && '[{"shard":1,"of":3},{"shard":2,"of":3},{"shard":3,"of":3}]' || '[{"shard":1,"of":2},{"shard":2,"of":2}]') }} defaults: run: working-directory: packages/flow @@ -528,7 +532,16 @@ jobs: set -euo pipefail mkdir -p runs mapfile -t files < "$RUNNER_TEMP/shard.txt" - pytest -n logical --dist loadfile -v --durations=20 --basetemp=runs/ci "${files[@]}" + pytest -n logical --dist loadfile -v --durations=20 --basetemp=runs/ci \ + --junitxml=runs/flow-unit-shard-${{ matrix.shard }}-of-${{ matrix.of }}.xml "${files[@]}" + - name: Retain this shard's claims evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: flow-unit-junit-${{ matrix.shard }}-of-${{ matrix.of }} + path: packages/flow/runs/flow-unit-shard-${{ matrix.shard }}-of-${{ matrix.of }}.xml + if-no-files-found: error + retention-days: 7 flow-critical: name: flow core critical (golden VERIFIED, zero model calls) @@ -621,6 +634,11 @@ jobs: sys.exit(1 if bad else 0) PY + - name: Validate passing browser claim evidence + run: >- + python scripts/validate_claims.py --check --ci-job e2e-browser + --junit runs/critical-e2e-junit.xml + # The agent-framework adapters (openadapt_flow.integrations.agents): each # framework's own agent loop with a scripted model, writing through a real # HTTP app, from the lock with every framework extra (CrewAI pinned beside @@ -1528,3 +1546,37 @@ jobs: print(f"failed: {bad}") sys.exit(1 if bad else 0) PY + # Unit evidence is complete only after every current-run shard passed. + # The merge queue deliberately rechecks critical tests, not unit shards. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group' + with: + persist-credentials: false + - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group' + with: + python-version: "3.12" + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group' + with: + enable-cache: true + cache-dependency-glob: .github/ci-tools/uv.lock + cache-suffix: flow-claims + - name: Install the locked claims validator dependency + if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group' + run: python scripts/ci/locked_install.py --lock .github/ci-tools:claims + - name: Download this run's unit claim evidence + if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: flow-unit-junit-* + path: runs/flow-unit-junit + merge-multiple: false + - name: Require every unit shard and validate all supported unit evidence + if: needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group' + run: | + set -euo pipefail + python scripts/ci/merge_flow_junit.py --artifacts-dir runs/flow-unit-junit \ + --shards '${{ needs.scope.outputs.flow_unit_shards }}' --out runs/flow-unit-junit.xml + cd packages/flow + python scripts/validate_claims.py --check --ci-job test --junit ../../runs/flow-unit-junit.xml diff --git a/packages/flow/docs/VERIFICATION.md b/packages/flow/docs/VERIFICATION.md index 2ff8922f48..11dbf24b17 100644 --- a/packages/flow/docs/VERIFICATION.md +++ b/packages/flow/docs/VERIFICATION.md @@ -3,9 +3,9 @@ > GENERATED by `scripts/validate_claims.py --report` from `claims/` (one file per claim). Do not edit by hand — edit the registry and regenerate. - Generated at: **committed registry state (regenerate: scripts/validate_claims.py --report)** -- JUnit pass check: **not embedded in this generated registry view** (required CI jobs enforce pass evidence) +- JUnit pass check: **not embedded in this generated registry view** (see the current CI run for file-level pass evidence) - Structure gate: `python scripts/validate_claims.py --check --structure-only` (a claim whose tier outranks its strongest backing evidence fails CI). -- Pass gates: required `test` and `e2e-browser` jobs supply their own JUnit files; an absent, all-skipped, or failed supported evidence file fails that required job. +- File-level pass check: the validator requires each supported backing file to appear in its job's JUnit result, with at least one passing case and no failed or errored cases. This generated report does not itself prove a CI run passed. - Scoped validating refreshes name each selected evidence file; each selected file must pass on its declared substrate. Unselected validating evidence is not represented as checked. **What this validator does and does not do.** It makes each public capability claim a *function* of automated evidence: a `supported` claim must be backed by a test file that has a real passing case in its required default CI job; a `validating` claim must be grounded in a REAL opt-in / infra-gated proof or a field test, and is never presented as supported. These are evidence tiers, not product lifecycle states. The validator does not replace workflow- and deployment-specific acceptance: application controls, identity rules, effect oracles, and live transport conditions remain bound to their counted evidence. diff --git a/packages/flow/scripts/validate_claims.py b/packages/flow/scripts/validate_claims.py index 5a87802408..bc7911a55e 100644 --- a/packages/flow/scripts/validate_claims.py +++ b/packages/flow/scripts/validate_claims.py @@ -635,7 +635,7 @@ def render_markdown( else: lines.append( "- JUnit pass check: **not embedded in this generated registry view** " - "(required CI jobs enforce pass evidence)" + "(see the current CI run for file-level pass evidence)" ) lines.append( "- Structure gate: `python scripts/validate_claims.py --check " @@ -643,9 +643,10 @@ def render_markdown( "evidence fails CI)." ) lines.append( - "- Pass gates: required `test` and `e2e-browser` jobs supply their own " - "JUnit files; an absent, all-skipped, or failed supported evidence file " - "fails that required job." + "- File-level pass check: the validator requires each supported backing " + "file to appear in its job's JUnit result, with at least one passing " + "case and no failed or errored cases. This generated report does not " + "itself prove a CI run passed." ) lines.append( "- Scoped validating refreshes name each selected evidence file; each " diff --git a/packages/flow/tests/nightly_only.txt b/packages/flow/tests/nightly_only.txt index 95b959ba0c..1a1c2ba038 100644 --- a/packages/flow/tests/nightly_only.txt +++ b/packages/flow/tests/nightly_only.txt @@ -18,8 +18,6 @@ tests/test_desktop_check_it_cli_mockmed.py # 126s tests/test_effectbench_task_pack.py # 34s tests/test_ap_invoice_benchmark.py # 12s tests/test_o2c_recon_benchmark.py # 9s -tests/test_ui_readback_benchmark.py # 24s -tests/test_silent_wrong_action_benchmark.py # 31s tests/test_agent_teach_branch.py::test_two_demonstrations_become_one_approved_branch # 5s tests/test_agent_teach_repair.py::test_dismiss_without_a_dialog_box_stays_unarmed # 8s diff --git a/scripts/ci/merge_flow_junit.py b/scripts/ci/merge_flow_junit.py new file mode 100644 index 0000000000..20eaeb055a --- /dev/null +++ b/scripts/ci/merge_flow_junit.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Merge a complete set of Flow unit-shard JUnit artifacts without losing cases.""" + +from __future__ import annotations + +import argparse +import importlib.util +import sys +import xml.etree.ElementTree as ET +from pathlib import Path +from types import ModuleType + +ROOT = Path(__file__).resolve().parents[2] +FLOW = ROOT / "packages" / "flow" + + +def _validator() -> ModuleType: + name = "_flow_claims_validator" + spec = importlib.util.spec_from_file_location( + name, FLOW / "scripts" / "validate_claims.py" + ) + if spec is None or spec.loader is None: + raise ValueError("cannot load the claims validator") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +def merge_reports( + artifacts: Path, shards: int, out: Path, *, repo_root: Path = FLOW +) -> None: + """Require every named shard report, then preserve all its test suites.""" + if shards < 1: + raise ValueError("the shard count must be positive") + expected = { + f"flow-unit-junit-{index}-of-{shards}": ( + f"flow-unit-shard-{index}-of-{shards}.xml" + ) + for index in range(1, shards + 1) + } + if not artifacts.is_dir(): + raise ValueError("the unit JUnit artifact directory is missing") + actual = {path.name for path in artifacts.iterdir()} + if actual != set(expected): + raise ValueError( + f"unit JUnit artifact set differs: missing={sorted(set(expected) - actual)}, " + f"unexpected={sorted(actual - set(expected))}" + ) + if out.exists(): + raise ValueError("the merged JUnit output already exists") + validator = _validator() + merged = ET.Element("testsuites") + for name, filename in sorted(expected.items()): + directory = artifacts / name + if not directory.is_dir() or {p.name for p in directory.iterdir()} != { + filename + }: + raise ValueError(f"unit JUnit artifact {name} needs exactly {filename}") + report = directory / filename + # Reuse the existing fail-closed parser: every artifact must contain + # repository test cases, not merely well-formed or nonempty XML. + validator.parse_junit(report, repo_root=repo_root) + document = ET.parse(report).getroot() + if document.tag == "testsuite": + merged.append(document) + elif document.tag == "testsuites": + merged.extend(document) + else: + raise ValueError(f"unit JUnit artifact {name} has an invalid root") + out.parent.mkdir(parents=True, exist_ok=True) + ET.ElementTree(merged).write(out, encoding="utf-8", xml_declaration=True) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--artifacts-dir", type=Path, required=True) + parser.add_argument("--shards", type=int, required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args(argv) + try: + merge_reports(args.artifacts_dir, args.shards, args.out) + except (OSError, ValueError, ET.ParseError) as exc: + print(f"Flow claims evidence FAILED: {exc}", file=sys.stderr) + return 1 + print(f"Merged all {args.shards} Flow unit JUnit reports into {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_flow_claims_ci.py b/tests/test_flow_claims_ci.py new file mode 100644 index 0000000000..2d785818a3 --- /dev/null +++ b/tests/test_flow_claims_ci.py @@ -0,0 +1,304 @@ +"""Required Flow claims consume complete current-run evidence and keep failures.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[1] +FLOW = ROOT / "packages" / "flow" +SPEC = importlib.util.spec_from_file_location( + "merge_flow_junit", ROOT / "scripts" / "ci" / "merge_flow_junit.py" +) +assert SPEC and SPEC.loader +merge = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(merge) +validator = merge._validator() + + +def _report(artifacts: Path, index: int, shards: int, cases: str) -> Path: + directory = artifacts / f"flow-unit-junit-{index}-of-{shards}" + directory.mkdir(parents=True) + path = directory / f"flow-unit-shard-{index}-of-{shards}.xml" + path.write_text(f"{cases}") + return path + + +def _case(module: str, detail: str = "", name: str = "example") -> str: + return f'{detail}' + + +def _registry(path: Path) -> Path: + path.write_text( + yaml.safe_dump( + { + "claims": [ + { + "id": "transport-proof", + "claim": "synthetic transport control", + "tier": "supported", + "evidence": [ + {"path": "tests/test_replayer.py", "proves": "control"}, + { + "path": "tests/test_effect_verifier.py", + "proves": "control", + }, + ], + } + ] + } + ) + ) + return path + + +def _check(out: Path, registry: Path) -> int: + return validator.main( + [ + "--check", + "--ci-job", + "test", + "--registry", + str(registry), + "--junit", + str(out), + ] + ) + + +@pytest.mark.parametrize("shards", [2, 3]) +def test_complete_partial_shards_pass_the_unchanged_full_scope_validator( + tmp_path: Path, shards: int +) -> None: + artifacts, out = tmp_path / "artifacts", tmp_path / "merged.xml" + for index in range(1, shards + 1): + module = "test_replayer" if index == 1 else "test_effect_verifier" + _report(artifacts, index, shards, _case(module)) + merge.merge_reports(artifacts, shards, out) + assert _check(out, _registry(tmp_path / "claims.yaml")) == 0 + + +@pytest.mark.parametrize( + ("first", "second"), + [ + (_case("test_replayer"), _case("test_replayer")), # required file absent + (_case("test_replayer"), _case("test_effect_verifier", "")), + ( + _case("test_replayer", "") + _case("test_effect_verifier"), + _case("test_replayer"), + ), + ( + _case("test_replayer"), + _case("test_replayer", "") + _case("test_effect_verifier"), + ), + ], +) +def test_aggregation_never_hides_absent_skipped_failed_or_errored_evidence( + tmp_path: Path, first: str, second: str +) -> None: + artifacts, out = tmp_path / "artifacts", tmp_path / "merged.xml" + _report(artifacts, 1, 2, first) + _report(artifacts, 2, 2, second) + merge.merge_reports(artifacts, 2, out) + assert _check(out, _registry(tmp_path / "claims.yaml")) == 1 + + +def test_optional_skip_does_not_overwrite_the_existing_file_pass_rule( + tmp_path: Path, +) -> None: + artifacts, out = tmp_path / "artifacts", tmp_path / "merged.xml" + _report(artifacts, 1, 2, _case("test_replayer", "", "optional")) + _report(artifacts, 2, 2, _case("test_replayer") + _case("test_effect_verifier")) + merge.merge_reports(artifacts, 2, out) + assert _check(out, _registry(tmp_path / "claims.yaml")) == 0 + + +@pytest.mark.parametrize("fault", ["missing", "extra", "wrong-count", "duplicate"]) +def test_artifact_set_must_match_every_expected_shard( + tmp_path: Path, fault: str +) -> None: + artifacts, out = tmp_path / "artifacts", tmp_path / "merged.xml" + _report(artifacts, 1, 2, _case("test_replayer")) + if fault != "missing": + _report(artifacts, 2, 2, _case("test_effect_verifier")) + if fault == "extra": + _report(artifacts, 3, 3, _case("test_replayer")) + if fault == "duplicate": + directory = artifacts / "flow-unit-junit-1-of-2" + (directory / "duplicate.xml").write_text("") + with pytest.raises(ValueError): + merge.merge_reports(artifacts, 3 if fault == "wrong-count" else 2, out) + assert not out.exists() + + +@pytest.mark.parametrize( + "content", + [ + "", + '', + ], +) +def test_bad_report_cannot_be_covered_by_another_shards_good_report( + tmp_path: Path, content: str +) -> None: + artifacts, out = tmp_path / "artifacts", tmp_path / "merged.xml" + _report(artifacts, 1, 2, _case("test_replayer") + _case("test_effect_verifier")) + report = _report(artifacts, 2, 2, _case("test_replayer")) + report.write_text(content) + assert ( + merge.main( + ["--artifacts-dir", str(artifacts), "--shards", "2", "--out", str(out)] + ) + == 1 + ) + assert not out.exists() + + +def test_missing_download_or_stale_output_is_never_a_fallback(tmp_path: Path) -> None: + artifacts, out = tmp_path / "missing", tmp_path / "merged.xml" + with pytest.raises(ValueError): + merge.merge_reports(artifacts, 2, out) + for index in (1, 2): + _report(artifacts, index, 2, _case("test_replayer")) + out.write_text("old report") + with pytest.raises(ValueError, match="already exists"): + merge.merge_reports(artifacts, 2, out) + assert out.read_text() == "old report" + + +def _workflow() -> dict: + return yaml.safe_load((ROOT / ".github/workflows/packages.yml").read_text()) + + +def test_producer_and_consumer_share_the_shard_count_and_current_run_artifacts() -> ( + None +): + jobs = _workflow()["jobs"] + classify = jobs["scope"]["steps"][-1]["run"] + assert classify.index('echo "flow_unit_shards=') < classify.index("all() {") + assert ( + 'if [ "$EVENT_NAME" = pull_request ]; then unit_shards=3; else unit_shards=2; fi' + in classify + ) + unit = jobs["flow-unit"] + assert ( + "needs.scope.outputs.flow_unit_shards" in unit["strategy"]["matrix"]["include"] + ) + test = next(s for s in unit["steps"] if s.get("name") == "Test") + assert ( + "--junitxml=runs/flow-unit-shard-${{ matrix.shard }}-of-${{ matrix.of }}.xml" + in test["run"] + ) + upload = unit["steps"][-1] + assert upload["if"] == "always()" + assert upload["with"]["if-no-files-found"] == "error" + assert ( + upload["with"]["name"] + == "flow-unit-junit-${{ matrix.shard }}-of-${{ matrix.of }}" + ) + gate = jobs["packages-gate"] + download = next( + s for s in gate["steps"] if "download-artifact@" in s.get("uses", "") + ) + assert download["with"] == { + "pattern": "flow-unit-junit-*", + "path": "runs/flow-unit-junit", + "merge-multiple": False, + } + # No run-id, repository, or token override: the action reads this run only. + for step in gate["steps"][2:]: + assert ( + step["if"] + == "needs.scope.outputs.flow == 'true' && github.event_name != 'merge_group'" + ) + assert not step.get("continue-on-error") + consumer = gate["steps"][-1]["run"] + assert consumer.startswith("set -euo pipefail\n") + assert "--shards '${{ needs.scope.outputs.flow_unit_shards }}'" in consumer + assert "--check --ci-job test --junit ../../runs/flow-unit-junit.xml" in consumer + assert "--evidence-path" not in consumer and "--structure-only" not in consumer + assert "merge_group" in unit["if"] + + +def test_existing_required_status_and_critical_checks_still_precede_claims() -> None: + jobs = _workflow()["jobs"] + gate = jobs["packages-gate"] + status = gate["steps"][1] + assert "RESULTS" in status["env"] + assert 'if result not in ("success", "skipped"):' in status["run"] + assert 'bad[name] = "skipped although selected"' in status["run"] + assert 'not_in_queue = {"flow-unit"}' in status["run"] + critical = jobs["flow-critical"]["steps"] + assert "skipped" in critical[-2]["run"] + assert "tests == 0 or skipped" in critical[-2]["run"] + assert "--check --ci-job e2e-browser" in critical[-1]["run"] + assert "--junit runs/critical-e2e-junit.xml" in critical[-1]["run"] + assert "--evidence-path" not in critical[-1]["run"] + + +@pytest.mark.parametrize( + ("event", "flow_selected", "producer_result", "expected"), + [ + ("pull_request", True, "success", 0), + ("pull_request", True, "failure", 1), + ("pull_request", True, "cancelled", 1), + ("pull_request", True, "skipped", 1), + ("merge_group", True, "skipped", 0), + ("push", True, "skipped", 1), + ("pull_request", False, "skipped", 0), + ], +) +def test_the_actual_required_status_gate_rejects_failed_or_missing_producers( + event: str, flow_selected: bool, producer_result: str, expected: int +) -> None: + gate = _workflow()["jobs"]["packages-gate"]["steps"][1]["run"] + script = "\n".join(gate.splitlines()[1:-1]) + needs = { + "scope": {"result": "success", "outputs": {"flow": str(flow_selected).lower()}}, + "flow-unit": {"result": producer_result}, + "flow-critical": {"result": "success"}, + } + result = subprocess.run( + [sys.executable, "-c", script], + env={**os.environ, "RESULTS": json.dumps(needs), "EVENT_NAME": event}, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == expected, result.stdout + result.stderr + + +def test_supported_backing_files_cannot_be_wholly_deselected_in_the_fast_lane() -> None: + excluded = { + line.split("#", 1)[0].strip() + for line in (FLOW / "tests/nightly_only.txt").read_text().splitlines() + } + for claim in validator.load_registry()["claims"]: + if claim["tier"] != "supported": + continue + for evidence in claim["evidence"]: + if evidence["path"].startswith("tests/"): + assert evidence["path"] not in excluded + + +def test_workflow_uses_only_the_small_locked_yaml_group() -> None: + groups = (ROOT / ".github/ci-tools/pyproject.toml").read_text() + assert 'claims = ["pyyaml>=6.0"]' in groups + steps = _workflow()["jobs"]["packages-gate"]["steps"] + install = next( + s + for s in steps + if s.get("name") == "Install the locked claims validator dependency" + ) + assert ( + install["run"] + == "python scripts/ci/locked_install.py --lock .github/ci-tools:claims" + ) From 4bf9d242a5b7eec2ebb4e52d2ff9579bc6611601 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 5 Nov 2026 11:39:36 -0500 Subject: [PATCH 2/3] test: wait for owned WPF row hit targets Signed-off-by: abrichr --- .../flow/tests/test_native_app_startup.py | 97 ++++++++++++++++++- .../tests/test_native_census_real_apps.py | 74 +++++++++++--- 2 files changed, 158 insertions(+), 13 deletions(-) diff --git a/packages/flow/tests/test_native_app_startup.py b/packages/flow/tests/test_native_app_startup.py index 47e5574993..a7fbb6ff42 100644 --- a/packages/flow/tests/test_native_app_startup.py +++ b/packages/flow/tests/test_native_app_startup.py @@ -255,14 +255,31 @@ def test_wpf_fixture_uses_signalled_handle_and_owned_window( from tests import test_native_census_real_apps as native calls = [] - items = [SimpleNamespace(ControlTypeName="ListItemControl") for _ in range(2)] + items = [] + points = {} + for index in range(2): + rect = SimpleNamespace( + left=10, top=20 * index, right=100, bottom=20 * index + 10 + ) + cell = SimpleNamespace(BoundingRectangle=rect) + item = SimpleNamespace( + ControlTypeName="ListItemControl", + ProcessId=1234, + GetChildren=lambda cell=cell: [cell], + ) + items.append(item) + points[(55, 20 * index + 5)] = item listing = SimpleNamespace(Exists=lambda *args: True, GetChildren=lambda: items) window = SimpleNamespace( ProcessId=1234 if identity != "other_pid" else 999, Name=native.TITLE if identity != "other_title" else "other window", ListControl=lambda **kwargs: listing, ) - auto = SimpleNamespace(ControlFromHandle=lambda hwnd: calls.append(hwnd) or window) + auto = SimpleNamespace( + ControlFromHandle=lambda hwnd: calls.append(hwnd) or window, + ControlFromPoint=lambda x, y: points[(x, y)], + ControlsAreSame=lambda left, right: left is right, + ) class Owned(Process): def terminate(self): @@ -296,3 +313,79 @@ def spawn(argv, stdout, stderr): finally: fixture.close() assert proc.poll() == -15 + + +@pytest.mark.parametrize("fault", ["empty", "missing", "other_pid", "other_row"]) +def test_wpf_points_are_not_ready_when_the_physical_target_is_unproven(fault): + from tests import test_native_census_real_apps as native + + rect = SimpleNamespace( + left=10, top=20, right=10 if fault == "empty" else 100, bottom=30 + ) + item = SimpleNamespace( + GetChildren=lambda: [SimpleNamespace(BoundingRectangle=rect)] + ) + hit = SimpleNamespace( + ProcessId=999 if fault == "other_pid" else 1234, + ControlTypeName="TextControl", + GetParentControl=lambda: None, + ) + auto = SimpleNamespace( + ControlFromPoint=lambda *point: None if fault == "missing" else hit, + ControlsAreSame=lambda left, right: left is right, + ) + ready, detail = native._wpf_points_ready(auto, [item], 1234) + assert ready is False + assert "row=0" in detail + + +def test_wpf_points_accept_the_owned_row_ancestor_only(): + from tests import test_native_census_real_apps as native + + rect = SimpleNamespace(left=10, top=20, right=100, bottom=30) + item = SimpleNamespace( + GetChildren=lambda: [SimpleNamespace(BoundingRectangle=rect)] + ) + hit = SimpleNamespace( + ProcessId=1234, + ControlTypeName="TextControl", + GetParentControl=lambda: item, + ) + auto = SimpleNamespace( + ControlFromPoint=lambda *point: hit, + ControlsAreSame=lambda left, right: left is right, + ) + assert native._wpf_points_ready(auto, [item], 1234)[0] is True + + +def test_wpf_readiness_waits_for_the_point_before_returning_items(fixture_state): + from tests import test_native_census_real_apps as native + + clock, _, ready_path, _, _ = fixture_state + _signal(ready_path) + rect = SimpleNamespace(left=10, top=20, right=100, bottom=30) + item = SimpleNamespace( + GetChildren=lambda: [SimpleNamespace(BoundingRectangle=rect)] + ) + outside = SimpleNamespace( + ProcessId=999, ControlTypeName="PaneControl", GetParentControl=lambda: None + ) + inside = SimpleNamespace( + ProcessId=1234, ControlTypeName="TextControl", GetParentControl=lambda: item + ) + hits = iter([outside, inside, inside]) + auto = SimpleNamespace( + ControlFromPoint=lambda *point: next(hits), + ControlsAreSame=lambda left, right: left is right, + ) + + def find_items(hwnd): + assert hwnd == 5678 + return ( + [item, item] + if native._wpf_points_ready(auto, [item, item], 1234)[0] + else None + ) + + assert _wait(fixture_state, find_items) == [item, item] + assert 0.1 <= clock.now < 0.2 diff --git a/packages/flow/tests/test_native_census_real_apps.py b/packages/flow/tests/test_native_census_real_apps.py index e82397c336..5e07750073 100644 --- a/packages/flow/tests/test_native_census_real_apps.py +++ b/packages/flow/tests/test_native_census_real_apps.py @@ -28,7 +28,11 @@ from openadapt_flow.runtime import identity_template as itmpl from openadapt_flow.runtime.identity_census import CensusDecision, decide -from tests._native_app_startup import stop_owned_process, wait_for_rendered_items +from tests._native_app_startup import ( + NativeAppStartupError, + stop_owned_process, + wait_for_rendered_items, +) pytestmark = pytest.mark.timeout(180) @@ -112,6 +116,42 @@ def _wait(predicate, timeout_s: float = 30.0) -> Any: """.replace("__TITLE__", TITLE) +def _wpf_points_ready(auto, items: list[Any], pid: int) -> tuple[bool, str]: + """Require the points used by the tests to hit their owned rendered rows.""" + for index, item in enumerate(items): + try: + children = item.GetChildren() + if not children: + return False, f"row={index} has no first cell" + rect = children[0].BoundingRectangle + bounds = (rect.left, rect.top, rect.right, rect.bottom) + if rect.right <= rect.left or rect.bottom <= rect.top: + return False, f"row={index} has empty cell bounds={bounds}" + point = ((rect.left + rect.right) // 2, (rect.top + rect.bottom) // 2) + hit = auto.ControlFromPoint(*point) + if hit is None: + return False, f"row={index} bounds={bounds} point={point} has no hit" + hit_pid, hit_type = hit.ProcessId, hit.ControlTypeName + detail = ( + f"row={index} bounds={bounds} point={point} " + f"hit_pid={hit_pid} hit_type={hit_type}" + ) + if hit_pid != pid: + return False, detail + " is outside the owned process" + current = hit + for _ in range(16): + if auto.ControlsAreSame(current, item): + break + current = current.GetParentControl() + if current is None: + return False, detail + " is outside the expected row" + else: + return False, detail + " ancestry did not reach the expected row" + except Exception as exc: + return False, f"row={index} point discovery failed ({type(exc).__name__})" + return True, f"all {len(items)} first-cell points hit their owned rows" + + @pytest.fixture def wpf_worklist(tmp_path: Path): if sys.platform != "win32" or os.environ.get("OPENADAPT_TEST_WINDOWS_UIA") != "1": @@ -153,8 +193,10 @@ def start(rows: list[dict[str, Any]]) -> list[Any]: ) procs.append(proc) owned.callback(stop_owned_process, proc) + last_point_state = "point discovery not reached" def items_from_owned_window(hwnd: int) -> Optional[list[Any]]: + nonlocal last_point_state window = auto.ControlFromHandle(hwnd) if window is None: return None @@ -165,23 +207,33 @@ def items_from_owned_window(hwnd: int) -> Optional[list[Any]]: listing = window.ListControl(searchDepth=3) if not listing.Exists(0, 0): return None - return [ + items = [ child for child in listing.GetChildren() if child.ControlTypeName == "ListItemControl" ] + if len(items) != len(rows): + last_point_state = f"UIA exposes {len(items)} of {len(rows)} rows" + return None + ready, last_point_state = _wpf_points_ready(auto, items, proc.pid) + return items if ready else None # One startup budget inside the existing 180s test limit leaves # 60s for the unchanged census assertions and owned cleanup. - return wait_for_rendered_items( - proc, - ready, - len(rows), - items_from_owned_window, - stdout_path, - stderr_path, - timeout_s=120.0, - ) + try: + return wait_for_rendered_items( + proc, + ready, + len(rows), + items_from_owned_window, + stdout_path, + stderr_path, + timeout_s=120.0, + ) + except NativeAppStartupError as exc: + raise NativeAppStartupError( + f"{exc}; point_readiness={last_point_state}" + ) from exc yield auto, start From 4c0df82886d772c0c05e568b2ed1f7ce5b06eed6 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 5 Nov 2026 12:26:06 -0500 Subject: [PATCH 3/3] test: isolate tutorial artifacts before running the fixture Signed-off-by: abrichr --- packages/flow/tests/e2e/test_tutorial_break_it_e2e.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/flow/tests/e2e/test_tutorial_break_it_e2e.py b/packages/flow/tests/e2e/test_tutorial_break_it_e2e.py index 3d9aed648a..d6d2baba46 100644 --- a/packages/flow/tests/e2e/test_tutorial_break_it_e2e.py +++ b/packages/flow/tests/e2e/test_tutorial_break_it_e2e.py @@ -32,9 +32,9 @@ from openadapt_flow.tutorial import TutorialResult, run_tutorial -@pytest.fixture(scope="module") +@pytest.fixture def break_it_path(tmp_path_factory: pytest.TempPathFactory) -> TutorialResult: - """Run the tutorial once with ``break_it=True`` for the whole module.""" + """Run after the function-scoped synthetic at-rest isolation is active.""" return run_tutorial(tmp_path_factory.mktemp("break-it"), break_it=True)