From 70f3fccbb90dcd743667b8fd05546c0a69179854 Mon Sep 17 00:00:00 2001 From: Jim Lin Date: Sat, 17 Oct 2026 18:11:30 +0800 Subject: [PATCH 1/2] feat(sbom): re-judge OpenStack's python packages by the OpenStack Security Advisories grype matches OpenStack's own packages in the service venvs by GitHub's advisories, which can name only the newest series' fix where an OpenStack Security Advisory fixed every maintained one: OSSA-2026-007 fixed keystone in 25.0.1, 26.1.1, 27.0.1 and 28.0.1, its GitHub advisory says < 28.0.1, and a 27.1.0 that carries the fix is reported. recheckvulns.py gains a third pass, --ossa. It keeps a checkout of openstack/ossa from opendev.org, updated between scans like the kernel CNA's (OSSA_GIT points at a mirror). A python match whose CVE an advisory names for that package is judged by the advisory's own affected versions. Outside every range it moves to ignoredMatches, with the advisory as the reason; inside one it stays, with that range's fix. A match is cleared only when every advisory naming its CVE for the package gives ranges: the oldest advisories' prose versions leave it as grype has it. The YAML is read without a YAML library, since the rest of the script needs none, and versions are compared in PEP 440 order. The kernel pass's clone-or-update is now shared with the new pass, and projsbom.mk passes --ossa with an ossa-advisories checkout beside kernel-vulns. Verified with scripts/tests/test_recheckvulns.sh, 89 checks, 25 of them new: ranges per series, an affected version keeping its series' fix, product names in any case, quoted and annotated CVE ids, prose versions, an advisory updated between scans, an unreachable checkout, and no fetch without --ossa or a python match. test_projsbom.sh passes its 12 checks with the new argument. Over the jim build's 3.2.0 report, with the opendev checkout, the pass found 17 matches an advisory names for their package and moved 5 to ignoredMatches, each version past its series' fix in the advisory's own ranges; the other 12 are named only by the oldest advisories, whose versions are prose. Refs bigstack-oss/cubecos#578 Signed-off-by: Jim Lin Co-authored-by: Eandalf Co-authored-by: Claude Opus 5.5 (1M context) --- make/projsbom.mk | 16 ++- scripts/recheckvulns.py | 207 ++++++++++++++++++++++++++--- scripts/tests/test_recheckvulns.sh | 70 +++++++++- 3 files changed, 268 insertions(+), 25 deletions(-) diff --git a/make/projsbom.mk b/make/projsbom.mk index e13d362..5bec323 100644 --- a/make/projsbom.mk +++ b/make/projsbom.mk @@ -64,10 +64,12 @@ sbom: $(PROJ_SBOM) # The SBOM, the package-level SBOM derived from it and the vulnerability report ship like any # other deliverable: projsign.mk covers them with the signed SHA256SUMS manifest, and attests # the package-level SBOM to each image (it says why that one, not the full SBOM). Before the -# report ships, its kernel matches are re-judged by the kernel CNA's records and re-rated, and -# its matches on python packages from RHEL's rpms by Red Hat's records and the rpms' changelogs -# (recheckvulns.py says why); kernel-vulns is its checkout of the CNA's records, the SBOM names -# the rpms, and the scanned rootfs holds their changelogs. Before the scan, a Go binary whose own +# report ships, its kernel matches are re-judged by the kernel CNA's records and re-rated, its +# matches on python packages from RHEL's rpms by Red Hat's records and the rpms' changelogs, and +# its matches on OpenStack's own python packages by the OpenStack Security Advisories +# (recheckvulns.py says why); kernel-vulns and ossa-advisories are its checkouts of the CNA's +# records and of the advisories, the SBOM names the rpms, and the scanned rootfs holds their +# changelogs. Before the scan, a Go binary whose own # version syft could only guess, or not at all, takes the version of the rpm that ships it # (fixgoversions.py says when), so the SBOM that is scanned and ships names the real one. $(PROJ_SBOM): syft-fs-cubecos.cdx.json $(HEX_SCRIPTSDIR)/makesbompackages $(HEX_SCRIPTSDIR)/recheckvulns.py $(HEX_SCRIPTSDIR)/fixgoversions.py $(PROJ_VEX) @@ -76,7 +78,7 @@ $(PROJ_SBOM): syft-fs-cubecos.cdx.json $(HEX_SCRIPTSDIR)/makesbompackages $(HEX_ $(call RUN_CMD_TIMED, grype sbom:$< $(foreach v,$(PROJ_VEX),--vex $(v)) --output=json > $(PROJ_SHIPDIR)/$$(readlink $(PROJ_RELEASE))_vuln.json," SCAN vuln") $(call RUN_CMD_TIMED, cp -f $< $(PROJ_SHIPDIR)/$$(readlink $(PROJ_RELEASE))_sbom.json," COPY $<") $(call RUN_CMD_TIMED, $(SHELL) $(HEX_SCRIPTSDIR)/makesbompackages $< $(PROJ_SHIPDIR)/$$(readlink $(PROJ_RELEASE))_sbom_packages.json," GEN sbom_packages.json") - $(call RUN_CMD_TIMED, python3 $(HEX_SCRIPTSDIR)/recheckvulns.py $(PROJ_SHIPDIR)/$$(readlink $(PROJ_RELEASE))_vuln.json kernel-vulns --sbom $< --root $(PROJ_BASE_ROOTFS)," RECHECK vuln") + $(call RUN_CMD_TIMED, python3 $(HEX_SCRIPTSDIR)/recheckvulns.py $(PROJ_SHIPDIR)/$$(readlink $(PROJ_RELEASE))_vuln.json kernel-vulns --sbom $< --root $(PROJ_BASE_ROOTFS) --ossa ossa-advisories," RECHECK vuln") $(call RUN_CMD_TIMED, ln -sf $(PROJ_SHIPDIR)/$$(readlink $(PROJ_RELEASE))_sbom.json $@," GEN $@") syft-fs-cubecos.cdx.json: $(PROJ_BASE_ROOTFS) @@ -85,5 +87,5 @@ syft-fs-cubecos.cdx.json: $(PROJ_BASE_ROOTFS) PKGCLEAN += $(PROJ_SBOM) syft-fs-cubecos.cdx.json # Downloads: kept by `make clean`, so the next build reuses them once their digest checks out, and -# fetches only what changed in kernel-vulns -DISTCLEAN += $(notdir $(SYFT_RPM) $(GRYPE_RPM)) kernel-vulns +# fetches only what changed in kernel-vulns and ossa-advisories +DISTCLEAN += $(notdir $(SYFT_RPM) $(GRYPE_RPM)) kernel-vulns ossa-advisories diff --git a/scripts/recheckvulns.py b/scripts/recheckvulns.py index c5029f2..c039161 100755 --- a/scripts/recheckvulns.py +++ b/scripts/recheckvulns.py @@ -5,7 +5,7 @@ # requires-python = ">=3.9" # dependencies = [] # /// -"""Re-judge grype's kernel matches by the kernel CNA, and RHEL python rpm matches by Red Hat. +"""Re-judge grype's kernel matches by the kernel CNA, RHEL rpm matches by Red Hat, OpenStack's by its OSSAs. grype judges a kernel by data that does not describe it. The vmlinuz is matched by NVD's CPE ranges, which often stop at the mainline fix and leave out the stable backports (CVE-2026-43197: @@ -46,6 +46,17 @@ by CentOS or Red Hat with a Stream or RHEL dist tag, so a SIG (el9s) or EPEL build, which carries none of RHEL's backports, keeps grype's verdict; kernel rpms are left to the kernel pass. A match that stays is rated by Red Hat where Red Hat rates it. + +OpenStack's own packages, in the service venvs, grype matches by GitHub's advisories, and those can +name only the newest series' fix where an OpenStack Security Advisory fixed every maintained +series: OSSA-2026-007 fixed keystone in 25.0.1, 26.1.1, 27.0.1 and 28.0.1, its GitHub advisory says +"< 28.0.1", and a 27.1.0 that carries the fix is reported. Given a checkout of the advisories +(--ossa, ossa.git), a python match whose CVE an advisory names for that package is judged by the +advisory's own affected versions: outside every range it moves to "ignoredMatches", inside one it +stays with that range's fix. A match is cleared only when every advisory naming its CVE for the +package gives ranges: one whose versions are prose (the oldest: "All versions", "Folsom, +Grizzly") leaves it as grype has it. The checkout is kept like the CNA's, and the report's +descriptor records its commit; OSSA_GIT in the environment points at a mirror. """ import argparse @@ -66,6 +77,7 @@ REDHAT_SECDATA = os.environ.get( 'REDHAT_SECDATA', 'https://access.redhat.com/hydra/rest/securitydata' ) +OSSA_GIT = os.environ.get('OSSA_GIT', 'https://opendev.org/openstack/ossa.git') RH_SEVERITY = {'Critical': 'Critical', 'Important': 'High', 'Moderate': 'Medium', 'Low': 'Low'} RHEL = 'Red Hat Enterprise Linux 9' DISTRO_PUBLISHERS = {'CentOS', 'Red Hat, Inc.'} @@ -121,24 +133,19 @@ def judge(cna, ver): return 'affected', None -def fetch_records(vulns): - """Clone or update the checkout of the CNA's records; return its commit and date.""" +def fetch_git(url, path, files): + """Clone or update a checkout of url's master, keeping only files; return its commit and date.""" # git.kernel.org serves no partial clones, so a first clone takes the whole tree (~1 minute); - # the checkout keeps only the JSON records, and later scans fetch just what changed. - git = ['git', '-C', vulns] - if os.path.isdir(os.path.join(vulns, '.git')): - subprocess.run( - git + ['fetch', '-q', '--depth', '1', KERNEL_VULNS_GIT, 'master'], check=True - ) + # the checkout keeps only the records, and later scans fetch just what changed. + git = ['git', '-C', path] + if os.path.isdir(os.path.join(path, '.git')): + subprocess.run(git + ['fetch', '-q', '--depth', '1', url, 'master'], check=True) subprocess.run(git + ['reset', '-q', '--hard', 'FETCH_HEAD'], check=True) else: subprocess.run( - ['git', 'clone', '-q', '--depth', '1', '--no-checkout', KERNEL_VULNS_GIT, vulns], - check=True, - ) - subprocess.run( - git + ['sparse-checkout', 'set', '--no-cone', '/cve/published/*/*.json'], check=True + ['git', 'clone', '-q', '--depth', '1', '--no-checkout', url, path], check=True ) + subprocess.run(git + ['sparse-checkout', 'set', '--no-cone', files], check=True) subprocess.run(git + ['checkout', '-q'], check=True) out = subprocess.run( git + ['log', '-1', '--format=%H %cI'], @@ -294,7 +301,7 @@ def is_kernel(a): ) try: - commit, date = fetch_records(vulns) + commit, date = fetch_git(KERNEL_VULNS_GIT, vulns, '/cve/published/*/*.json') except subprocess.CalledProcessError: sys.exit(f'recheckvulns: cannot fetch the kernel CNA records from {KERNEL_VULNS_GIT}') @@ -490,6 +497,167 @@ def recheck_distro(rep, sbom, root): ) +def pyname(name): + return re.sub(r'[-_.]+', '-', name).lower() + + +def pyver(v): + """PEP 440 order, enough for OpenStack's versions: 1.0.dev1 < 1.0rc1 < 1.0 < 1.0.post1.""" + m = re.match( + r'v?(\d+(?:\.\d+)*)(?:[-_.]?(a|b|c|rc|alpha|beta|pre|preview)[-_.]?(\d*))?' + r'(?:[-_.]?(post|rev|r)[-_.]?(\d*))?(?:[-_.]?dev[-_.]?(\d*))?', + v.lower(), + ) + if not m: + return None + rel, pre, pren, post, postn, dev = m.groups() + nums = [int(x) for x in rel.split('.')] + nums += [0] * (10 - len(nums)) + rank = {'a': 0, 'alpha': 0, 'b': 1, 'beta': 1}.get(pre, 2) + if pre: + pk = (0, rank, int(pren or 0)) + elif dev is not None and post is None: + pk = (-1, 0, 0) + else: + pk = (1, 0, 0) + return ( + tuple(nums), + pk, + int(postn or 0) if post else -1, + int(dev or 0) if dev is not None else float('inf'), + ) + + +OSSA_COND = r'(<=|>=|==|!=|<|>)\s*(\d[\w.]*)' + + +def ossa_ranges(spec): + """An advisory's affected versions as ranges, each a list of (operator, version); None for + prose ("All versions", "Folsom, Grizzly") or anything else that is not purely ranges.""" + ranges = [] + for seg in spec.split(','): + conds = re.findall(OSSA_COND, seg) + if not conds or re.sub(OSSA_COND, '', seg).strip(): + return None + ranges.append(conds) + return ranges + + +def in_range(ver, conds): + ops = { + '<': lambda a, b: a < b, + '<=': lambda a, b: a <= b, + '>': lambda a, b: a > b, + '>=': lambda a, b: a >= b, + '==': lambda a, b: a == b, + '!=': lambda a, b: a != b, + } + return all(ops[op](ver, pyver(x)) for op, x in conds) + + +def section(text, name): + """A top-level key's block of an advisory's YAML, as text.""" + m = re.search(rf'^{name}:[^\n]*\n((?:[ \t-].*\n?|\n)*)', text, re.M) + return m.group(1) if m else '' + + +def ossa_index(path): + """{CVE: [(advisory, product, affected versions)]} from a checkout of ossa.git. The YAML is + read for its two blocks only, so no YAML library is needed: affected-products, whose items + name a product and its versions, and vulnerabilities, whose items name the CVEs.""" + index = {} + folder = os.path.join(path, 'ossa') + for f in sorted(os.listdir(folder)): + if not re.match(r'OSSA-\d+-\d+\.yaml$', f): + continue + with open(os.path.join(folder, f)) as fh: + text = fh.read() + cves = sorted(set(re.findall(r'CVE-\d{4}-\d+', section(text, 'vulnerabilities')))) + items, item = [], {} + for line in section(text, 'affected-products').splitlines(): + m = re.match(r'\s*(-\s+)?(product|version):\s*(.*?)\s*$', line) + if not m: + continue + if m.group(1) and item: + items.append(item) + item = {} + item[m.group(2)] = m.group(3).strip('\'"') + if item: + items.append(item) + for it in items: + if 'product' in it and 'version' in it: + for cve in cves: + index.setdefault(cve, []).append( + (f[:-5], pyname(it['product']), it['version']) + ) + return index + + +def recheck_ossa(rep, path): + """Re-judge matches on OpenStack's python packages by the OSSAs in place; a summary, or None + if the report has no python match to judge.""" + if not any(m['artifact']['type'] == 'python' for m in rep['matches']): + return None + try: + commit, date = fetch_git(OSSA_GIT, path, '/ossa/*.yaml') + except subprocess.CalledProcessError: + sys.exit(f'recheckvulns: cannot fetch the OpenStack security advisories from {OSSA_GIT}') + index = ossa_index(path) + + ignored, judged, kept = [], 0, 0 + for m in rep['matches']: + vul, art = m['vulnerability'], m['artifact'] + if art['type'] != 'python': + continue + ids = [vul['id']] + [x.get('id', '') for x in m.get('relatedVulnerabilities') or []] + cve = next((i for i in ids if i in index), None) + ver = pyver(art['version']) + said = [x for x in index.get(cve, []) if x[1] == pyname(art['name'])] + if not said or ver is None: + continue + judged += 1 + ranges = [(oid, spec, ossa_ranges(spec)) for oid, _, spec in said] + hit = [ + (oid, conds) + for oid, _, rs in ranges + if rs + for conds in rs + if in_range(ver, conds) + ] + if hit: + fix = next((x for op, x in hit[0][1] if op in ('<', '<=')), None) + vul['fix'] = ( + {'versions': [fix], 'state': 'fixed'} + if fix + else {'versions': [], 'state': 'not-fixed'} + ) + kept += 1 + elif all(rs for _, _, rs in ranges): + oid, spec, _ = ranges[0] + ignore( + m, + f'OSSA: {oid} affects {art["name"]} {spec}, not {art["version"]} ' + f'(ossa.git {commit[:12]})', + ) + ignored.append(m) + else: + kept += 1 + + drop = {id(m) for m in ignored} + rep['matches'] = [m for m in rep['matches'] if id(m) not in drop] + rep['ignoredMatches'] = (rep.get('ignoredMatches') or []) + ignored + rep.setdefault('descriptor', {})['ossaRecheck'] = { + 'advisories': {'repository': OSSA_GIT, 'commit': commit, 'date': date}, + 'judged': judged, + 'ignored': len(ignored), + 'kept': kept, + } + return ( + f'OpenStack python matches: {judged} named by an OSSA, {len(ignored)} ignored by it, ' + f'{kept} kept' + ) + + def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument('report', help="grype's JSON report, rewritten in place") @@ -500,6 +668,11 @@ def main(): ) ap.add_argument('--sbom', help='the CycloneDX SBOM the report was scanned from') ap.add_argument('--root', help="the rootfs the SBOM was made from, for its rpms' changelogs") + ap.add_argument( + '--ossa', + help='checkout of the OpenStack security advisories (ossa.git): cloned when missing, ' + "brought up to date otherwise; with it, OpenStack's python packages are re-judged", + ) args = ap.parse_args() with open(args.report) as f: @@ -508,9 +681,11 @@ def main(): done = [recheck_kernel(rep, args.vulns)] if args.sbom: done.append(recheck_distro(rep, args.sbom, args.root)) + if args.ossa: + done.append(recheck_ossa(rep, args.ossa)) done = [x for x in done if x] if not done: - print('no kernel image in the report: nothing to re-check') + print('no kernel image or python package in the report: nothing to re-check') return with open(args.report + '.tmp', 'w') as f: diff --git a/scripts/tests/test_recheckvulns.sh b/scripts/tests/test_recheckvulns.sh index a17be5d..e290146 100755 --- a/scripts/tests/test_recheckvulns.sh +++ b/scripts/tests/test_recheckvulns.sh @@ -1,8 +1,9 @@ #!/bin/bash # # Unit test for scripts/recheckvulns.py: which kernel matches the kernel CNA's records move to -# ignoredMatches, the fix and severity of the ones that stay, and that nothing else is touched. -# Self-contained: vulns.git is a local repo and Red Hat's security data a file:// directory. +# ignoredMatches, the fix and severity of the ones that stay, and that nothing else is touched; +# likewise for RHEL python rpms by Red Hat's records and OpenStack's packages by the OSSAs. +# Self-contained: vulns.git and ossa.git are local repos, Red Hat's security data a file:// directory. # # Needs bash, git and python3. # Run: bash test_recheckvulns.sh @@ -195,6 +196,71 @@ REDHAT_SECDATA=http://127.0.0.1:9 KERNEL_VULNS_GIT="file://$T/nowhere" run "$T/v chk "python pass: Red Hat unreachable fails" "$RC" "1" chk " report untouched" "$(cmp -s "$T/before.json" "$T/vuln.json" && echo same)" "same" +# OpenStack's python packages (--ossa): the advisories, in the shape ossa.git keeps them +mkdir -p "$T/ossa/ossa" +adv(){ printf 'date: 2099-01-01\n\nid: %s\n\ntitle: t\n\naffected-products:\n%b\nvulnerabilities:\n%b\n' "$1" "$2" "$3" > "$T/ossa/ossa/$1.yaml"; } +adv OSSA-2099-001 " - product: keystone\n version: '>=8.0.0 <25.0.1, >=26.0.0 <26.1.1, >=27.0.0 <27.0.1, >=28.0.0 <28.0.1'\n" " - cve-id: CVE-2026-3001\n" +adv OSSA-2099-002 " - product: Nova\n version: '>=30.0.0 <30.2.1, >=31.0.0 <31.1.1'\n" " - cve-id: 'CVE-2026-3002'\n" +adv OSSA-2099-003 " - product: glance\n version: All versions\n" " - cve-id: CVE-2026-3003\n" +adv OSSA-2099-004 " - product: oslo.messaging\n version: '<16.1.3'\n - product: python-keystoneclient\n version: '<5.6.1'\n" " - cve-id: CVE-2026-3004 (client to proxy)\n" +echo 'not an advisory' > "$T/ossa/README.rst" +git -C "$T/ossa" init -q -b master && git -C "$T/ossa" add -A && + git -C "$T/ossa" -c user.name=t -c user.email=t@t -c commit.gpgsign=false commit -q -m one +VENV=/opt/openstack-epoxy/lib/python3.12/site-packages +osreport(){ cat > "$T/vuln.json" <=8.0.0 <25.0.1, >=26.0.0 <26.1.1, >=27.0.0 <27.0.1, >=28.0.0 <28.0.1, not 27.1.0" +chk " a release candidate of a series it never affected" "$(owhy GHSA-k003)" "OSSA: OSSA-2099-001 affects keystone >=8.0.0 <25.0.1, >=26.0.0 <26.1.1, >=27.0.0 <27.0.1, >=28.0.0 <28.0.1, not 29.0.0.0rc1" +chk " inside a range: its series' fix" "$(ofix GHSA-k002)" "['27.0.1']" +chk " a capitalized product, a quoted CVE" "$(ofix GHSA-n001)" "['31.1.1']" +chk " the series' fix release itself" "$(owhy GHSA-n002)" "OSSA: OSSA-2099-002 affects nova >=30.0.0 <30.2.1, >=31.0.0 <31.1.1, not 31.1.1" +chk " prose versions judge nothing" "$(ofix GHSA-g001)" "['9.9']" +chk " a dotted package name, a CVE with a note" "$(ofix GHSA-o001)" "['16.1.3']" +chk " the advisory's second product" "$(owhy GHSA-o002)" "OSSA: OSSA-2099-004 affects python-keystoneclient <5.6.1, not 5.6.1" +chk " a CVE named for another package: untouched" "$(ofix GHSA-x001)" "['9.9']" +chk " descriptor" "$(q "r['descriptor']['ossaRecheck']['judged'], r['descriptor']['ossaRecheck']['ignored'], r['descriptor']['ossaRecheck']['kept'], r['descriptor']['ossaRecheck']['advisories']['commit'] == '$(git -C "$T/ossa" rev-parse HEAD)'")" "8 4 4 True" +chk " summary" "$OUT" "OpenStack python matches: 8 named by an OSSA, 4 ignored by it, 4 kept" +chk " checkout holds only the advisories" "$(cd "$T/ossa-checkout" && find . -path ./.git -prune -o -type f -print | grep -vc '^\./ossa/OSSA-.*\.yaml$')" "0" + +# The next scan fetches what changed: the advisory now says keystone 27.0.0 is not affected +adv OSSA-2099-001 " - product: keystone\n version: '>=8.0.0 <25.0.1, >=26.0.0 <26.1.1, >=28.0.0 <28.0.1'\n" " - cve-id: CVE-2026-3001\n" +git -C "$T/ossa" -c user.name=t -c user.email=t@t -c commit.gpgsign=false commit -q -am two +osreport; KERNEL_VULNS_GIT="file://$T/nowhere" OSSA_GIT="file://$T/ossa" run "$T/vuln.json" "$T/checkout5" --ossa "$T/ossa-checkout" +chk " second run updates" "$(owhy GHSA-k002)" "OSSA: OSSA-2099-001 affects keystone >=8.0.0 <25.0.1, >=26.0.0 <26.1.1, >=28.0.0 <28.0.1, not 27.0.0" + +osreport; cp "$T/vuln.json" "$T/before.json" +KERNEL_VULNS_GIT="file://$T/nowhere" OSSA_GIT="file://$T/nowhere" run "$T/vuln.json" "$T/checkout5" --ossa "$T/ossa-checkout2" +chk "ossa.git unreachable fails" "$RC" "1" +chk " and says so" "$(grep -c "cannot fetch the OpenStack security advisories from file://$T/nowhere" <<<"$OUT")" "1" +chk " report untouched" "$(cmp -s "$T/before.json" "$T/vuln.json" && echo same)" "same" +KERNEL_VULNS_GIT="file://$T/nowhere" OSSA_GIT="file://$T/nowhere" run "$T/vuln.json" "$T/checkout5" +chk "without --ossa: nothing fetched, report untouched" "$RC $(cmp -s "$T/before.json" "$T/vuln.json" && echo same)" "0 same" +echo '{"matches": []}' > "$T/vuln.json" +KERNEL_VULNS_GIT="file://$T/nowhere" OSSA_GIT="file://$T/nowhere" run "$T/vuln.json" "$T/checkout5" --ossa "$T/ossa-checkout3" +chk "no python match: no fetch" "$RC $([ -e "$T/ossa-checkout3" ] && echo yes || echo no)" "0 no" + +pv(){ python3 -B -c "import importlib.util as u; s = u.spec_from_file_location('r', '$SCRIPT'); m = u.module_from_spec(s); s.loader.exec_module(m); print(m.pyver('$1') < m.pyver('$2'))"; } +chk "pyver rc before its release" "$(pv 29.0.0.0rc1 29.0.0)" "True" +chk "pyver dev before a pre-release" "$(pv 1.0.dev1 1.0a1)" "True" +chk "pyver release before post" "$(pv 1.0 1.0.post1)" "True" +chk "pyver numeric, not text" "$(pv 27.9.0 27.10.0)" "True" +chk "pyver trailing zeros equal" "$(pv 27.1 27.1.0) $(pv 27.1.0 27.1)" "False False" + vc(){ python3 -B -c "import importlib.util as u; s = u.spec_from_file_location('r', '$SCRIPT'); m = u.module_from_spec(s); s.loader.exec_module(m); print(m.vercmp('$1', '$2'))"; } chk "vercmp newer base release" "$(vc 8.el9 6.el9_7.1)" "1" chk "vercmp z-stream rebuild sorts above" "$(vc 8.el9 8.el9_8)" "-1" From 58f18975b9ade78e6527d668422d9d4eeaafea11 Mon Sep 17 00:00:00 2001 From: Jim Lin Date: Sat, 17 Oct 2026 18:22:06 +0800 Subject: [PATCH 2/2] feat(sbom): compare versions in OpenStack's release order in the OSSA re-check OpenStack versioned its releases YYYY.N until Liberty (2015.2) and semantically since, so every semantic version came after every YYYY.N one; compared as numbers, nova 31.3.1 sorts below 2014.1.4. GitHub's advisories state the oldest flaws' fixes as YYYY.N releases, so grype reports current releases for flaws fixed in 2012 to 2015, and an advisory whose own ranges are written in YYYY.N releases took them in too. The OSSA pass now compares in OpenStack's order, an advisory's ranges included. It also moves a match to ignoredMatches when the package is a project the advisories name, every fix grype names is a YYYY.N release, and the installed version is semantic. The oldest advisories give their versions as prose, so this rule is what judges them. It also reads a range that a few advisories split at its comma as one range: '<=2014.2.3, >=2015.1.0, <=2015.1.1' and '>=25.0.0, <26.1.0' mean bounded ranges, not everything from the lower bound up. A lower bound alone, followed by an upper bound alone, is one range, as in all five advisories that write it so. The descriptor counts what each rule ignored. Verified with scripts/tests/test_recheckvulns.sh, 104 checks, 15 of them new: a prose advisory's YYYY.N fix, an OpenStack project no advisory names the CVE for, mixed fixes and a calendar-versioned package left alone, an installed YYYY.N release, advisory ranges in YYYY.N releases, a split range read whole and still taking in its own versions, and the version orders. Over the jim build's 3.2.0 report with the opendev checkout, the pass now clears all 17 matches an advisory names for their package, 7 by its ranges and 10 by a YYYY.N fix, where the previous commit cleared 5. Refs bigstack-oss/cubecos#578 Signed-off-by: Jim Lin Co-authored-by: Eandalf Co-authored-by: Claude Opus 5.5 (1M context) --- scripts/recheckvulns.py | 106 ++++++++++++++++++++--------- scripts/tests/test_recheckvulns.sh | 32 ++++++++- 2 files changed, 103 insertions(+), 35 deletions(-) diff --git a/scripts/recheckvulns.py b/scripts/recheckvulns.py index c039161..1e0e610 100755 --- a/scripts/recheckvulns.py +++ b/scripts/recheckvulns.py @@ -55,7 +55,13 @@ advisory's own affected versions: outside every range it moves to "ignoredMatches", inside one it stays with that range's fix. A match is cleared only when every advisory naming its CVE for the package gives ranges: one whose versions are prose (the oldest: "All versions", "Folsom, -Grizzly") leaves it as grype has it. The checkout is kept like the CNA's, and the report's +Grizzly") does not clear it. Those oldest flaws have a second tell. OpenStack versioned its +releases YYYY.N until Liberty (2015.2) and semantically since, so every semantic version came +after every YYYY.N one; but GitHub's advisories state those fixes as YYYY.N releases, which a +semantic version compares below: nova 31.3.1 is reported for a flaw fixed in 2014.1.4. The pass +compares in that order, an advisory's ranges included, and a match on a project the advisories +name, whose every fix grype names is a YYYY.N release and whose version is semantic, moves to +"ignoredMatches" too. The checkout is kept like the CNA's, and the report's descriptor records its commit; OSSA_GIT in the environment points at a mirror. """ @@ -528,18 +534,39 @@ def pyver(v): ) +def osver(v): + """An OpenStack project's version order: its YYYY.N releases (until Liberty, 2015.2) all came + before its semantic versions (12.0.0 onward), so 31.3.1 sorts after 2014.2.3, not below it.""" + p = pyver(v) + return None if p is None else (0 if p[0][0] >= 2000 else 1, p) + + OSSA_COND = r'(<=|>=|==|!=|<|>)\s*(\d[\w.]*)' def ossa_ranges(spec): """An advisory's affected versions as ranges, each a list of (operator, version); None for prose ("All versions", "Folsom, Grizzly") or anything else that is not purely ranges.""" - ranges = [] + segs = [] for seg in spec.split(','): conds = re.findall(OSSA_COND, seg) if not conds or re.sub(OSSA_COND, '', seg).strip(): return None - ranges.append(conds) + segs.append(conds) + # A few advisories split one range at its comma: "<=2014.2.3, >=2015.1.0, <=2015.1.1" and + # ">=25.0.0, <26.1.0" mean 2015.1.0 to 2015.1.1 and 25.0.0 to 26.1.0, not everything from + # 2015.1.0 or 25.0.0 up. A lower bound alone, followed by an upper bound alone, is one range. + ranges = [] + for conds in segs: + prev = ranges[-1] if ranges else None + if ( + prev + and all(op in ('>', '>=') for op, _ in prev) + and all(op in ('<', '<=') for op, _ in conds) + ): + prev.extend(conds) + else: + ranges.append(list(conds)) return ranges @@ -552,7 +579,7 @@ def in_range(ver, conds): '==': lambda a, b: a == b, '!=': lambda a, b: a != b, } - return all(ops[op](ver, pyver(x)) for op, x in conds) + return all(ops[op](ver, osver(x)) for op, x in conds) def section(text, name): @@ -593,6 +620,14 @@ def ossa_index(path): return index +DATE_VERSION = re.compile(r'20\d\d\.\d+') + + +def date_based(fixes): + """True if every version is one of OpenStack's old YYYY.N releases (2014.1.4, 2015.1.0).""" + return bool(fixes) and all(DATE_VERSION.match(x) for x in fixes) + + def recheck_ossa(rep, path): """Re-judge matches on OpenStack's python packages by the OSSAs in place; a summary, or None if the report has no python match to judge.""" @@ -603,27 +638,22 @@ def recheck_ossa(rep, path): except subprocess.CalledProcessError: sys.exit(f'recheckvulns: cannot fetch the OpenStack security advisories from {OSSA_GIT}') index = ossa_index(path) + projects = {p for said in index.values() for _, p, _ in said} - ignored, judged, kept = [], 0, 0 + ignored = {'advisory': [], 'date-based fix': []} + named, kept = 0, 0 for m in rep['matches']: vul, art = m['vulnerability'], m['artifact'] - if art['type'] != 'python': + name, ver = pyname(art['name']), osver(art['version']) + if art['type'] != 'python' or name not in projects or ver is None: continue ids = [vul['id']] + [x.get('id', '') for x in m.get('relatedVulnerabilities') or []] cve = next((i for i in ids if i in index), None) - ver = pyver(art['version']) - said = [x for x in index.get(cve, []) if x[1] == pyname(art['name'])] - if not said or ver is None: - continue - judged += 1 + said = [x for x in index.get(cve, []) if x[1] == name] ranges = [(oid, spec, ossa_ranges(spec)) for oid, _, spec in said] - hit = [ - (oid, conds) - for oid, _, rs in ranges - if rs - for conds in rs - if in_range(ver, conds) - ] + hit = [(oid, conds) for oid, _, rs in ranges if rs for conds in rs if in_range(ver, conds)] + if said: + named += 1 if hit: fix = next((x for op, x in hit[0][1] if op in ('<', '<=')), None) vul['fix'] = ( @@ -632,29 +662,41 @@ def recheck_ossa(rep, path): else {'versions': [], 'state': 'not-fixed'} ) kept += 1 - elif all(rs for _, _, rs in ranges): + continue + if said and all(rs for _, _, rs in ranges): oid, spec, _ = ranges[0] - ignore( - m, - f'OSSA: {oid} affects {art["name"]} {spec}, not {art["version"]} ' - f'(ossa.git {commit[:12]})', - ) - ignored.append(m) + why, by = f'OSSA: {oid} affects {art["name"]} {spec}, not {art["version"]}', 'advisory' else: - kept += 1 + # An advisory from before Liberty can give its versions as prose, and GitHub's states + # the fix as one of the project's old YYYY.N releases, which a semantic version + # compares below (31.3.1 < 2014.1.4). The project's semantic versions all came after. + fixes = (vul.get('fix') or {}).get('versions') or [] + if not (date_based(fixes) and ver[0] == 1): + kept += bool(said) + continue + why = ( + f'OSSA: {art["name"]} is an OpenStack project, and its fix {", ".join(fixes)} ' + f'is a YYYY.N release, all of which came before its semantic versions such as ' + f'{art["version"]}' + ) + by = 'date-based fix' + ignore(m, f'{why} (ossa.git {commit[:12]})') + ignored[by].append(m) - drop = {id(m) for m in ignored} + drop = {id(m) for ms in ignored.values() for m in ms} rep['matches'] = [m for m in rep['matches'] if id(m) not in drop] - rep['ignoredMatches'] = (rep.get('ignoredMatches') or []) + ignored + rep['ignoredMatches'] = ( + (rep.get('ignoredMatches') or []) + ignored['advisory'] + ignored['date-based fix'] + ) rep.setdefault('descriptor', {})['ossaRecheck'] = { 'advisories': {'repository': OSSA_GIT, 'commit': commit, 'date': date}, - 'judged': judged, - 'ignored': len(ignored), + 'named': named, + 'ignored': {by: len(ms) for by, ms in ignored.items()}, 'kept': kept, } return ( - f'OpenStack python matches: {judged} named by an OSSA, {len(ignored)} ignored by it, ' - f'{kept} kept' + f'OpenStack python matches: {named} named by an OSSA; {len(ignored["advisory"])} ignored ' + f'by the advisory, {len(ignored["date-based fix"])} by a date-based fix; {kept} kept' ) diff --git a/scripts/tests/test_recheckvulns.sh b/scripts/tests/test_recheckvulns.sh index e290146..84fafbc 100755 --- a/scripts/tests/test_recheckvulns.sh +++ b/scripts/tests/test_recheckvulns.sh @@ -203,10 +203,14 @@ adv OSSA-2099-001 " - product: keystone\n version: '>=8.0.0 <25.0.1, >=26.0. adv OSSA-2099-002 " - product: Nova\n version: '>=30.0.0 <30.2.1, >=31.0.0 <31.1.1'\n" " - cve-id: 'CVE-2026-3002'\n" adv OSSA-2099-003 " - product: glance\n version: All versions\n" " - cve-id: CVE-2026-3003\n" adv OSSA-2099-004 " - product: oslo.messaging\n version: '<16.1.3'\n - product: python-keystoneclient\n version: '<5.6.1'\n" " - cve-id: CVE-2026-3004 (client to proxy)\n" +adv OSSA-2099-005 " - product: nova\n version: '<2014.2.3, >=2015.1.0 <2015.1.2'\n" " - cve-id: CVE-2026-3010\n" +adv OSSA-2099-006 " - product: glance\n version: '<=2014.2.3, >=2015.1.0, <=2015.1.1'\n" " - cve-id: CVE-2026-3011\n" echo 'not an advisory' > "$T/ossa/README.rst" git -C "$T/ossa" init -q -b master && git -C "$T/ossa" add -A && git -C "$T/ossa" -c user.name=t -c user.email=t@t -c commit.gpgsign=false commit -q -m one VENV=/opt/openstack-epoxy/lib/python3.12/site-packages +# A match whose fix grype states as given: +pd(){ p "$1" "$2" "$3" "$4" $VENV | sed "s/\"versions\": \[\"9.9\"\]/\"versions\": $5/"; } osreport(){ cat > "$T/vuln.json" < "$T/vuln.json" <=8.0.0 <25.0.1, >=26.0.0 <26.1.1, >=27.0.0 <27.0.1, >=28.0.0 <28.0.1, not 27.1.0" chk " a release candidate of a series it never affected" "$(owhy GHSA-k003)" "OSSA: OSSA-2099-001 affects keystone >=8.0.0 <25.0.1, >=26.0.0 <26.1.1, >=27.0.0 <27.0.1, >=28.0.0 <28.0.1, not 29.0.0.0rc1" chk " inside a range: its series' fix" "$(ofix GHSA-k002)" "['27.0.1']" @@ -233,8 +242,18 @@ chk " prose versions judge nothing" "$(ofix GHSA-g001)" "['9.9']" chk " a dotted package name, a CVE with a note" "$(ofix GHSA-o001)" "['16.1.3']" chk " the advisory's second product" "$(owhy GHSA-o002)" "OSSA: OSSA-2099-004 affects python-keystoneclient <5.6.1, not 5.6.1" chk " a CVE named for another package: untouched" "$(ofix GHSA-x001)" "['9.9']" -chk " descriptor" "$(q "r['descriptor']['ossaRecheck']['judged'], r['descriptor']['ossaRecheck']['ignored'], r['descriptor']['ossaRecheck']['kept'], r['descriptor']['ossaRecheck']['advisories']['commit'] == '$(git -C "$T/ossa" rev-parse HEAD)'")" "8 4 4 True" -chk " summary" "$OUT" "OpenStack python matches: 8 named by an OSSA, 4 ignored by it, 4 kept" +chk " a prose advisory, a fix from the YYYY.N releases" "$(owhy GHSA-d001)" "OSSA: glance is an OpenStack project, and its fix 2014.2.3 is a YYYY.N release, all of which came before its semantic versions such as 30.2.0" +chk " no advisory names it, the project is OpenStack's" "$(owhy GHSA-d002)" "OSSA: nova is an OpenStack project, and its fix 2014.1.4, 2013.2.4 is a YYYY.N release, all of which came before its semantic versions such as 31.3.1" +chk " a fix that is not all YYYY.N: untouched" "$(ofix GHSA-d003)" "['2014.1.4', '28.0.1']" +chk " an installed YYYY.N release: untouched" "$(ofix GHSA-d004)" "['2014.1.4']" +chk " a calendar-versioned package that is not OpenStack's: untouched" "$(ofix GHSA-d005)" "['2024.7.4']" +chk " an advisory's range decides first" "$(ofix GHSA-d006)" "['27.0.1']" +chk " ranges in YYYY.N releases do not take in a semantic version" "$(owhy GHSA-e001)" "OSSA: OSSA-2099-005 affects nova <2014.2.3, >=2015.1.0 <2015.1.2, not 31.3.1" +chk " but do take in an older YYYY.N one" "$(ofix GHSA-e002)" "['2014.2.3']" +chk " a range split at its comma is read whole" "$(owhy GHSA-e003)" "OSSA: OSSA-2099-006 affects glance <=2014.2.3, >=2015.1.0, <=2015.1.1, not 30.2.0" +chk " and still takes in its own versions" "$(ofix GHSA-e004)" "['2015.1.1']" +chk " descriptor" "$(q "r['descriptor']['ossaRecheck']['named'], r['descriptor']['ossaRecheck']['ignored'], r['descriptor']['ossaRecheck']['kept'], r['descriptor']['ossaRecheck']['advisories']['commit'] == '$(git -C "$T/ossa" rev-parse HEAD)'")" "14 {'advisory': 6, 'date-based fix': 2} 7 True" +chk " summary" "$OUT" "OpenStack python matches: 14 named by an OSSA; 6 ignored by the advisory, 2 by a date-based fix; 7 kept" chk " checkout holds only the advisories" "$(cd "$T/ossa-checkout" && find . -path ./.git -prune -o -type f -print | grep -vc '^\./ossa/OSSA-.*\.yaml$')" "0" # The next scan fetches what changed: the advisory now says keystone 27.0.0 is not affected @@ -260,6 +279,13 @@ chk "pyver dev before a pre-release" "$(pv 1.0.dev1 1.0a1)" "True" chk "pyver release before post" "$(pv 1.0 1.0.post1)" "True" chk "pyver numeric, not text" "$(pv 27.9.0 27.10.0)" "True" chk "pyver trailing zeros equal" "$(pv 27.1 27.1.0) $(pv 27.1.0 27.1)" "False False" +or(){ python3 -B -c "import importlib.util as u; s = u.spec_from_file_location('r', '$SCRIPT'); m = u.module_from_spec(s); s.loader.exec_module(m); print(m.ossa_ranges('$1'))"; } +chk "ossa_ranges merges a split range" "$(or '<24.1.3, >=25.0.0 <26.1.1, >=27.0.0, <29.0.1')" "[[('<', '24.1.3')], [('>=', '25.0.0'), ('<', '26.1.1')], [('>=', '27.0.0'), ('<', '29.0.1')]]" +chk "ossa_ranges keeps a trailing lower bound open" "$(or '<24.1.3, >=27.0.0')" "[[('<', '24.1.3')], [('>=', '27.0.0')]]" +chk "ossa_ranges: prose is None" "$(or 'Folsom, Grizzly') $(or 'All versions')" "None None" +ov(){ python3 -B -c "import importlib.util as u; s = u.spec_from_file_location('r', '$SCRIPT'); m = u.module_from_spec(s); s.loader.exec_module(m); print(m.osver('$1') < m.osver('$2'))"; } +chk "osver YYYY.N before semantic" "$(ov 2015.1.2 12.0.0) $(ov 31.3.1 2014.2.3)" "True False" +chk "osver within an era, as pyver" "$(ov 2014.1.4 2014.2.3) $(ov 27.0.1 27.1.0)" "True True" vc(){ python3 -B -c "import importlib.util as u; s = u.spec_from_file_location('r', '$SCRIPT'); m = u.module_from_spec(s); s.loader.exec_module(m); print(m.vercmp('$1', '$2'))"; } chk "vercmp newer base release" "$(vc 8.el9 6.el9_7.1)" "1"