From 7025c577416366217a2cba9cfc37ccdf105198fc Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Tue, 13 Oct 2026 07:47:51 -0500 Subject: [PATCH 1/2] ci(health): bump pyscn to 1.32.4 and read its snake_case report - Pin pyscn==1.32.4 (was 1.29.0) in the [health] extra. - One pyscn run now writes both the JSON and the HTML report (--json --html), instead of analyzing the package twice. - The sanity step reads the 1.30.0+ snake_case keys (system.summary.project_root, total_dependencies, total_modules) and also reports resolved_imports and unresolved_imports, warning when any import is unresolved as well as when the root is not the checkout. - Regenerate .pyscn-known-clones.json with 1.32.4. Its default clone grouping (complete_linkage, was connected) and min_lines (8, was 10) changed, so the accepted groups differ. - Note the new Linux ARM64 wheel. The scores are not comparable across the bump: on main, 1.29.0 reports 77/100 and 1.32.4 reports 87/100 for the same tree, with the same root and no unresolved imports. Closes #415. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HBt3Ee7eneKtdKQWq2T3YK --- .github/workflows/code-health.yml | 36 +++++++------- .pyscn-known-clones.json | 80 ++++++++++++++----------------- CONTRIBUTING.md | 4 +- pyproject.toml | 7 +-- 4 files changed, 60 insertions(+), 67 deletions(-) diff --git a/.github/workflows/code-health.yml b/.github/workflows/code-health.yml index c04b9b597..8e9a7ab3e 100644 --- a/.github/workflows/code-health.yml +++ b/.github/workflows/code-health.yml @@ -43,20 +43,19 @@ jobs: pip install -e .[metrics,health] - name: Structural analysis (pyscn) - # One output format per invocation, so run it twice: JSON to read the - # numbers, HTML to read the report. Never fails the job -- a non-zero - # exit here means "findings", which are the point. + # One run writes both the JSON report (the numbers) and the HTML report. + # Never fails the job -- a non-zero exit here means "findings", which + # are the point. continue-on-error: true run: | - pyscn analyze --json --no-open dataretrieval 2>&1 | tee pyscn-summary.txt - pyscn analyze --html --no-open dataretrieval >/dev/null + pyscn analyze --json --html --no-open dataretrieval 2>&1 | tee pyscn-summary.txt - name: Check the analysis actually resolved the package - # pyscn infers a project root, and when it guesses wrong it silently - # resolves only a fraction of the imports -- which *raises* the score, - # because most of what it grades is dependency-derived. A degraded run - # therefore looks like an improved one. Record the resolved edge count - # next to the score so that is visible rather than flattering. + # pyscn infers a project root. If it chose the wrong one, it would + # resolve only part of the imports, which *raises* the score, because + # most of what it grades is dependency-derived. Record the root and the + # unresolved import count next to the score so a degraded run is + # visible rather than flattering. if: always() continue-on-error: true run: | @@ -65,13 +64,16 @@ jobs: reports = sorted(glob.glob(".pyscn/reports/*.json")) if not reports: print("no pyscn JSON report found"); raise SystemExit - s = json.load(open(reports[-1]))["system"]["Summary"] - root, deps = s["ProjectRoot"], s["TotalDependencies"] - print(f"modules={s['TotalModules']} resolved_dependencies={deps} root={root}") - if os.path.realpath(root) != os.path.realpath(os.getcwd()): - print(f"WARNING: project root {root!r} is not the checkout; " - "import resolution is probably degraded and the scores " - "above are not comparable to previous runs.") + s = json.load(open(reports[-1]))["system"]["summary"] + root, unresolved = s["project_root"], s["unresolved_imports"] + print(f"modules={s['total_modules']} " + f"dependencies={s['total_dependencies']} " + f"resolved_imports={s['resolved_imports']} " + f"unresolved_imports={unresolved} root={root}") + if os.path.realpath(root) != os.path.realpath(os.getcwd()) or unresolved: + print(f"WARNING: project root {root!r} or {unresolved} unresolved " + "imports; import resolution is probably degraded and the " + "scores above are not comparable to previous runs.") PY cat pyscn-sanity.txt diff --git a/.pyscn-known-clones.json b/.pyscn-known-clones.json index 41448e734..5aaee671f 100644 --- a/.pyscn-known-clones.json +++ b/.pyscn-known-clones.json @@ -1,9 +1,9 @@ { "_comment": [ "Accepted pyscn clone groups. Purpose: make a NEW clone group a visible event.", - "The duplication sub-score is dominated by these five accepted families, so it", - "barely moves when real duplication is added -- this file is what the weekly", - "Code Health workflow diffs against instead.", + "The duplication sub-score is dominated by these accepted families, so it barely", + "moves when real duplication is added -- compare a weekly Code Health report", + "against this file instead.", "", "Groups are keyed by the SET OF FUNCTION NAMES they contain, not by line numbers,", "so ordinary edits above a getter do not churn this file. Regenerate only when", @@ -11,86 +11,76 @@ "", "These are NOT a backlog. Collapsing them means **kwargs and the loss of the typed", "public surface tests/contracts/public_api_test.py freezes.", - "Verified 2026-08-09: at similarity_threshold 0.55 (default 0.65) the package still", - "reports exactly these five groups, so no latent clone sits below the bar.", - "Verified 2026-08-09: stripping docstrings makes duplication far WORSE (60 -> 0),", - "so suppressing them is not an option -- they are what keeps similarity below 0.86." + "Measured with pyscn 1.29.0 on 2026-08-09: at similarity_threshold 0.55 (default 0.65) the", + "package reported the same groups, and stripping docstrings made duplication far WORSE", + "(60 -> 0), so suppressing them is not an option.", + "Regenerated with pyscn 1.32.4 on 2026-10-13. Its default grouping (complete_linkage,", + "was connected) and min_lines (8, was 10) differ from 1.29.0, so it no longer reports", + "the NGWMN, NWIS, and WQP getter families and reports two small validator pairs instead." ], - "pyscn_version": "1.29.0", + "pyscn_version": "1.32.4", "default_thresholds": { "similarity_threshold": 0.65, - "min_lines": 10, + "min_lines": 8, "min_nodes": 20 }, "observed": { - "total_fragments": 283, - "cloned_fragments": 25, + "total_fragments": 351, + "cloned_fragments": 13, "groups": 5 }, "groups": [ { "members": [ - "dataretrieval/ngwmn.py::get_lithology", - "dataretrieval/ngwmn.py::get_providers", - "dataretrieval/ngwmn.py::get_well_construction" + "dataretrieval/_validation.py::reject_together", + "dataretrieval/_validation.py::require_any_of" ], - "similarity": 0.831, + "similarity": 0.85, + "clone_type": 4, + "rationale": "Shared argument checks (AGENTS.md, Error messages): each returns early when the arguments are valid and otherwise raises one message from the same parts. Each check owns its wording, so a common helper would only move the message apart from the check that states it." + }, + { + "members": [ + "dataretrieval/nwdc.py::_validate_county", + "dataretrieval/nwdc.py::_validate_huc" + ], + "similarity": 0.85, "clone_type": 2, - "rationale": "NGWMN collection getters over the OGC facade, one per collection." + "rationale": "NWDC selector validators: each strips a code, checks its digits and length, and raises a message naming that selector's own form and examples." }, { "members": [ - "dataretrieval/nwis.py::get_dv", - "dataretrieval/nwis.py::get_iv" + "dataretrieval/waterdata/time_series.py::get_stats_date_range", + "dataretrieval/waterdata/time_series.py::get_stats_por" ], "similarity": 0.85, "clone_type": 2, - "rationale": "Deprecated NWIS getters sharing the waterservices call shape; bodies already delegate to _get_json_values." + "rationale": "Water Data collection-family getters: ~30 explicitly typed parameters and a worked-example docstring around a thin body." }, { "members": [ - "dataretrieval/waterdata/measurements.py::get_channel", "dataretrieval/waterdata/measurements.py::get_field_measurements", "dataretrieval/waterdata/measurements.py::get_peaks", - "dataretrieval/waterdata/metadata.py::get_field_measurements_metadata", - "dataretrieval/waterdata/metadata.py::get_monitoring_locations", - "dataretrieval/waterdata/metadata.py::get_time_series_metadata", - "dataretrieval/waterdata/time_series.py::get_continuous", "dataretrieval/waterdata/time_series.py::get_daily", "dataretrieval/waterdata/time_series.py::get_latest_continuous", "dataretrieval/waterdata/time_series.py::get_latest_daily" ], - "similarity": 0.805, + "similarity": 0.842, "clone_type": 2, "rationale": "Water Data collection-family getters: ~30 explicitly typed parameters and a worked-example docstring around a thin body." }, { "members": [ - "dataretrieval/waterdata/time_series.py::get_stats_date_range", - "dataretrieval/waterdata/time_series.py::get_stats_por" + "dataretrieval/waterdata/measurements.py::get_channel", + "dataretrieval/waterdata/metadata.py::get_monitoring_locations" ], - "similarity": 0.85, - "clone_type": 2, + "similarity": 0.672, + "clone_type": 4, "rationale": "Water Data collection-family getters: ~30 explicitly typed parameters and a worked-example docstring around a thin body." - }, - { - "members": [ - "dataretrieval/wqp.py::what_activities", - "dataretrieval/wqp.py::what_activity_metrics", - "dataretrieval/wqp.py::what_detection_limits", - "dataretrieval/wqp.py::what_habitat_metrics", - "dataretrieval/wqp.py::what_organizations", - "dataretrieval/wqp.py::what_project_weights", - "dataretrieval/wqp.py::what_projects", - "dataretrieval/wqp.py::what_sites" - ], - "similarity": 0.85, - "clone_type": 2, - "rationale": "WQP what_* wrappers: one shared query per service endpoint, differing only by the service path segment and the column each documents." } ], "sub_threshold_accepted": { - "_comment": "Families that are NOT reported at default thresholds (they sit under min_lines=10 / min_nodes=20) but were found by a deliberate scan at min_nodes=8 on 2026-08-09. Recorded so they are not rediscovered and re-litigated. They are NOT counted in `observed` above.", + "_comment": "Families that are NOT reported at default thresholds (they sit under min_lines=10 / min_nodes=20) but were found by a deliberate scan at min_nodes=8 on 2026-08-09 (pyscn 1.29.0). Recorded so they are not rediscovered and re-litigated. They are NOT counted in `observed` above.", "groups": [ { "members": [ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 015c2b942..03b594c17 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -218,9 +218,9 @@ including for an agent working on this repo, which gets a whole-package structural overview from one command: ```bash -pip install -e '.[health]' # wheels: macOS ARM64, Linux x86-64, Windows x86-64 +pip install -e '.[health]' # wheels: macOS ARM64, Linux x86-64/ARM64, Windows x86-64 # (no source distribution or other platform wheels) -pyscn analyze dataretrieval # HTML report, or --json for the numbers +pyscn analyze dataretrieval # HTML report; --json --html writes both ``` Read its findings as suggestions, not conclusions. Its clone detector flags this diff --git a/pyproject.toml b/pyproject.toml index 86cab0260..f4d8c6ac0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,10 +69,11 @@ metrics = [ "import-linter==2.13", ] # Advisory weekly deep sweep -- see .github/workflows/code-health.yml. Pyscn has -# wheels only for macOS ARM64, Linux x86-64, and Windows x86-64, with no sdist, -# so keep it separate from the extra used by merge gates and contributors. +# wheels only for macOS ARM64, Linux x86-64 and ARM64, and Windows x86-64, with +# no sdist, so keep it separate from the extra used by merge gates and +# contributors. health = [ - "pyscn==1.29.0", + "pyscn==1.32.4", ] test = [ "pytest > 5.0.0", From a3cd7fce5727dd1bef6df3fc590550b649810936 Mon Sep 17 00:00:00 2001 From: thodson-usgs Date: Tue, 13 Oct 2026 08:03:27 -0500 Subject: [PATCH 2/2] ci(health): detect a degraded pyscn run by module count; note glibc 2.34 Review fixes for the pyscn 1.32.4 bump. - The sanity step no longer relies on unresolved_imports: under a wrong root pyscn counts the package's own imports as external, so the count stays 0 (checked by analyzing a copy under another root: 15 modules, 5 dependencies, 0 unresolved). It now warns separately when the root is not the checkout and when pyscn analyzed fewer than 90% of the package's .py files. - Missing report keys are named in the summary instead of raising a KeyError to stderr, so a future key rename shows up rather than silently dropping the sanity line. - The 1.32.4 Linux wheels need glibc 2.34 (1.29.0's needed 2.17), and there is no sdist; pyproject.toml and CONTRIBUTING.md say so. - .pyscn-known-clones.json: the 2026-08-09 docstring measurements are marked as pyscn 1.29.0 only, since 1.32.4 skips docstrings; the Water Data rationales no longer cite docstrings; the marginal type-4 pair (0.672) gets its own rationale; default_thresholds records group_mode, group_threshold, skip_docstrings, and clone_types; and the sub-threshold note says it was a 1.29.0 scan. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01HBt3Ee7eneKtdKQWq2T3YK --- .github/workflows/code-health.yml | 39 ++++++++++++++++++++----------- .pyscn-known-clones.json | 25 +++++++++++++------- CONTRIBUTING.md | 5 ++-- pyproject.toml | 6 ++--- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/.github/workflows/code-health.yml b/.github/workflows/code-health.yml index 8e9a7ab3e..0a4904387 100644 --- a/.github/workflows/code-health.yml +++ b/.github/workflows/code-health.yml @@ -52,28 +52,39 @@ jobs: - name: Check the analysis actually resolved the package # pyscn infers a project root. If it chose the wrong one, it would - # resolve only part of the imports, which *raises* the score, because - # most of what it grades is dependency-derived. Record the root and the - # unresolved import count next to the score so a degraded run is - # visible rather than flattering. + # resolve only part of the package, which *raises* the score, because + # most of what it grades is dependency-derived. Its unresolved-import + # count does not show this: under a wrong root the package's own + # imports count as external. So compare the root with the checkout and + # the module count with the files on disk, and print each warning on + # its own. Missing report keys are named rather than raising, so a + # renamed key shows in the summary instead of dropping this line. if: always() continue-on-error: true run: | - python - <<'PY' > pyscn-sanity.txt + python - <<'PY' > pyscn-sanity.txt 2>&1 import glob, json, os reports = sorted(glob.glob(".pyscn/reports/*.json")) if not reports: print("no pyscn JSON report found"); raise SystemExit - s = json.load(open(reports[-1]))["system"]["summary"] - root, unresolved = s["project_root"], s["unresolved_imports"] - print(f"modules={s['total_modules']} " - f"dependencies={s['total_dependencies']} " - f"resolved_imports={s['resolved_imports']} " - f"unresolved_imports={unresolved} root={root}") - if os.path.realpath(root) != os.path.realpath(os.getcwd()) or unresolved: - print(f"WARNING: project root {root!r} or {unresolved} unresolved " - "imports; import resolution is probably degraded and the " + s = json.load(open(reports[-1])).get("system", {}).get("summary", {}) + keys = ("project_root", "total_modules", "total_dependencies") + missing = [k for k in keys if k not in s] + if missing: + print(f"WARNING: the pyscn report has no system.summary {missing}; " + "update this check for the pinned pyscn version.") + raise SystemExit + root, modules = s["project_root"], s["total_modules"] + files = len(glob.glob("dataretrieval/**/*.py", recursive=True)) + print(f"modules={modules} files={files} " + f"dependencies={s['total_dependencies']} root={root}") + if os.path.realpath(root) != os.path.realpath(os.getcwd()): + print(f"WARNING: project root {root!r} is not the checkout; the " "scores above are not comparable to previous runs.") + if modules < 0.9 * files: + print(f"WARNING: pyscn analyzed {modules} modules of {files} files; " + "import resolution is probably degraded and the scores " + "above are not comparable to previous runs.") PY cat pyscn-sanity.txt diff --git a/.pyscn-known-clones.json b/.pyscn-known-clones.json index 5aaee671f..10cc38b05 100644 --- a/.pyscn-known-clones.json +++ b/.pyscn-known-clones.json @@ -11,9 +11,10 @@ "", "These are NOT a backlog. Collapsing them means **kwargs and the loss of the typed", "public surface tests/contracts/public_api_test.py freezes.", - "Measured with pyscn 1.29.0 on 2026-08-09: at similarity_threshold 0.55 (default 0.65) the", - "package reported the same groups, and stripping docstrings made duplication far WORSE", - "(60 -> 0), so suppressing them is not an option.", + "Measured with pyscn 1.29.0 on 2026-08-09, which compared docstrings: at", + "similarity_threshold 0.55 (default 0.65) it reported the same groups, and stripping", + "docstrings made duplication far worse (60 -> 0). pyscn 1.32.4 skips docstrings by", + "default (skip_docstrings below), so neither measurement applies to the pinned version.", "Regenerated with pyscn 1.32.4 on 2026-10-13. Its default grouping (complete_linkage,", "was connected) and min_lines (8, was 10) differ from 1.29.0, so it no longer reports", "the NGWMN, NWIS, and WQP getter families and reports two small validator pairs instead." @@ -22,7 +23,15 @@ "default_thresholds": { "similarity_threshold": 0.65, "min_lines": 8, - "min_nodes": 20 + "min_nodes": 20, + "group_mode": "complete_linkage", + "group_threshold": 0.65, + "skip_docstrings": true, + "clone_types": [ + 1, + 2, + 4 + ] }, "observed": { "total_fragments": 351, @@ -55,7 +64,7 @@ ], "similarity": 0.85, "clone_type": 2, - "rationale": "Water Data collection-family getters: ~30 explicitly typed parameters and a worked-example docstring around a thin body." + "rationale": "Water Data collection-family getters: ~30 explicitly typed parameters around a thin body. pyscn 1.32.4 skips docstrings, so the similarity is the parameter lists and bodies." }, { "members": [ @@ -67,7 +76,7 @@ ], "similarity": 0.842, "clone_type": 2, - "rationale": "Water Data collection-family getters: ~30 explicitly typed parameters and a worked-example docstring around a thin body." + "rationale": "Water Data collection-family getters: ~30 explicitly typed parameters around a thin body. pyscn 1.32.4 skips docstrings, so the similarity is the parameter lists and bodies." }, { "members": [ @@ -76,11 +85,11 @@ ], "similarity": 0.672, "clone_type": 4, - "rationale": "Water Data collection-family getters: ~30 explicitly typed parameters and a worked-example docstring around a thin body." + "rationale": "Two Water Data getters matched as a semantic (type-4) clone at 0.672, just above the 0.65 threshold. Expect this pair to appear and disappear between runs as either getter is edited; that churn is not a new clone." } ], "sub_threshold_accepted": { - "_comment": "Families that are NOT reported at default thresholds (they sit under min_lines=10 / min_nodes=20) but were found by a deliberate scan at min_nodes=8 on 2026-08-09 (pyscn 1.29.0). Recorded so they are not rediscovered and re-litigated. They are NOT counted in `observed` above.", + "_comment": "Families that pyscn 1.29.0 did NOT report at its defaults (they sat under its min_lines=10 / min_nodes=20) but that a deliberate scan at min_nodes=8 found on 2026-08-09. Not re-scanned with 1.32.4, whose min_lines is 8. Recorded so they are not rediscovered and re-litigated. They are NOT counted in `observed` above.", "groups": [ { "members": [ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 03b594c17..d1047fc8c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -218,8 +218,9 @@ including for an agent working on this repo, which gets a whole-package structural overview from one command: ```bash -pip install -e '.[health]' # wheels: macOS ARM64, Linux x86-64/ARM64, Windows x86-64 - # (no source distribution or other platform wheels) +pip install -e '.[health]' # wheels: macOS ARM64, Linux x86-64/ARM64 with + # glibc 2.34+, Windows x86-64 (no source distribution + # or other platform wheels) pyscn analyze dataretrieval # HTML report; --json --html writes both ``` diff --git a/pyproject.toml b/pyproject.toml index f4d8c6ac0..45e0e0a4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,9 +69,9 @@ metrics = [ "import-linter==2.13", ] # Advisory weekly deep sweep -- see .github/workflows/code-health.yml. Pyscn has -# wheels only for macOS ARM64, Linux x86-64 and ARM64, and Windows x86-64, with -# no sdist, so keep it separate from the extra used by merge gates and -# contributors. +# wheels only for macOS ARM64, Linux x86-64 and ARM64 (glibc 2.34 or later), and +# Windows x86-64, with no sdist, so keep it separate from the extra used by +# merge gates and contributors. health = [ "pyscn==1.32.4", ]