From 656b9cdc5a28586778f60ea1d05e93bf0c48e12a Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 8 Oct 2026 00:27:55 +0200 Subject: [PATCH 1/2] fix(frontend): reject syntax nested too deep instead of crashing the scan (#233) --- docs/usage.md | 5 +- src/coretrace_python/frontend/ast_adapter.py | 26 ++++++- tests/test_deep_nesting.py | 78 ++++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) create mode 100644 tests/test_deep_nesting.py diff --git a/docs/usage.md b/docs/usage.md index ec318ad..88dc00f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -84,7 +84,10 @@ get_asgi_application()` in `asgi.py` or `email = forms.EmailField()` in a form c calls the analyzer sees, and a module-level `os.system(input())` is a finding in ``. A function using syntax outside the supported subset is reported as an `unsupported-syntax` note and the other functions are still analysed; a file Python -itself cannot parse is reported as a `syntax-error`. The coverage line and the JSON +itself cannot parse is reported as a `syntax-error`, and so is a file whose syntax is +nested more than 500 levels deep (a chain of hundreds of `+` or chained calls, which only +generated code writes), which the analyzer does not follow; the other files are still +analysed. The coverage line and the JSON report's per-file detail tell "no findings" from "nothing analysed". Taint follows values between functions and across files through function summaries, diff --git a/src/coretrace_python/frontend/ast_adapter.py b/src/coretrace_python/frontend/ast_adapter.py index dbf3522..8d9e041 100644 --- a/src/coretrace_python/frontend/ast_adapter.py +++ b/src/coretrace_python/frontend/ast_adapter.py @@ -13,6 +13,13 @@ class HIRBuildError(Exception): """A source-located failure to represent parsed syntax as PyHIR.""" +# The frontend and the analyses after it recurse once per level of nesting: syntax nested +# deeper, which Python parses in left-nested chains (``"a" + "b" + …``, ``x.f().g()…``) +# and only generated code reaches, would exhaust Python's recursion limit (#233). The +# bound leaves the later stages room below that limit, whatever the call stack. +MAX_NESTING = 500 + + _BINARY_OPERATORS = { ast.Add: "add", ast.Sub: "sub", @@ -449,4 +456,21 @@ def module(self, tree: ast.Module) -> nodes.Module: def build_module(source: SourceFile, tree: ast.Module) -> nodes.Module: - return AstHIRBuilder(source).module(tree) + builder = AstHIRBuilder(source) + too_deep = _nested_too_deep(tree) + if too_deep is not None: + builder.fail(too_deep, f"syntax nested more than {MAX_NESTING} levels deep, beyond what the analyzer follows") + return builder.module(tree) + + +def _nested_too_deep(tree: ast.AST) -> ast.AST | None: + """The first node nested more than ``MAX_NESTING`` levels below ``tree``, found + without recursion; None when there is none.""" + + pending: list[tuple[ast.AST, int]] = [(tree, 0)] + while pending: + node, depth = pending.pop() + if depth > MAX_NESTING: + return node + pending.extend((child, depth + 1) for child in ast.iter_child_nodes(node)) + return None diff --git a/tests/test_deep_nesting.py b/tests/test_deep_nesting.py new file mode 100644 index 0000000..ea51cae --- /dev/null +++ b/tests/test_deep_nesting.py @@ -0,0 +1,78 @@ +"""Acceptance tests for issue #233: syntax nested deeper than the analyzer follows. + +The frontend and the analyses after it recurse once per level of nesting. Python parses +left-nested chains of any length (``"a" + "b" + …``, ``a.b.b…``, ``x.strip().strip()…``), +so the frontend rejects, before building PyHIR, a file whose syntax is nested deeper than +``MAX_NESTING`` levels, as it rejects one it cannot represent: a ``syntax-error`` note +for that file, and the rest of the project analysed. Syntax within the bound is analysed +as before. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from coretrace_python.cli import main +from coretrace_python.frontend import HIRBuildError, build_hir +from coretrace_python.frontend.ast_adapter import MAX_NESTING +from coretrace_python.source import SourceManager + + +def chain(terms: int) -> str: + return "X = " + " + ".join(['"ab"'] * terms) + "\n" + + +@pytest.mark.parametrize( + "text", + [ + chain(3000), + "X = a" + ".b" * 3000 + "\n", + "import sys\nX = sys.argv[1]" + ".strip()" * 600 + "\n", + ], +) +def test_a_file_nested_too_deep_is_rejected_and_the_project_is_analysed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], text: str +) -> None: + root = tmp_path / "proj" + root.mkdir() + (root / "deep.py").write_text(text, encoding="utf-8") + (root / "run.py").write_text("eval(input())\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + + assert main(["--check", "proj", "--format", "json"]) == 1 + captured = capsys.readouterr() + report = json.loads(captured.out) + + assert "Traceback" not in captured.err + assert {(f["location"]["path"], f["rule_id"]) for f in report["findings"]} >= { + ("deep.py", "syntax-error"), + ("run.py", "dangerous-eval"), + } + (note,) = [f for f in report["findings"] if f["rule_id"] == "syntax-error"] + assert f"nested more than {MAX_NESTING} levels" in note["message"] + + +def test_the_rejection_names_where_the_nesting_goes_too_deep() -> None: + with pytest.raises(HIRBuildError, match=rf"^a\.py:1:\d+: .*nested more than {MAX_NESTING}"): + build_hir(SourceManager().add_source("a.py", chain(3000))) + + +def test_syntax_within_the_bound_is_analysed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + root = tmp_path / "proj" + root.mkdir() + terms = MAX_NESTING - 10 + (root / "run.py").write_text( + "import os\nimport sys\n\nX = sys.argv[1]" + ' + "ab"' * terms + "\nos.system(X)\n", + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + + assert main(["--check", "proj", "--format", "json"]) == 1 + report = json.loads(capsys.readouterr().out) + + assert [f["rule_id"] for f in report["findings"]] == ["command-injection"] From cc37509f55587e30f620667d592ca1963065e9d0 Mon Sep 17 00:00:00 2001 From: Hugo Date: Thu, 8 Oct 2026 01:09:30 +0200 Subject: [PATCH 2/2] fix(frontend): bound nesting at 200 levels and report parser recursion on Python 3.11 (#233) --- docs/usage.md | 2 +- src/coretrace_python/frontend/ast_adapter.py | 8 ++++--- src/coretrace_python/frontend/parser.py | 3 +++ tests/test_deep_nesting.py | 24 ++++++++++++++++---- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 88dc00f..d0da908 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -85,7 +85,7 @@ calls the analyzer sees, and a module-level `os.system(input())` is a finding in A function using syntax outside the supported subset is reported as an `unsupported-syntax` note and the other functions are still analysed; a file Python itself cannot parse is reported as a `syntax-error`, and so is a file whose syntax is -nested more than 500 levels deep (a chain of hundreds of `+` or chained calls, which only +nested more than 200 levels deep (a chain of hundreds of `+` or chained calls, which only generated code writes), which the analyzer does not follow; the other files are still analysed. The coverage line and the JSON report's per-file detail tell "no findings" from "nothing analysed". diff --git a/src/coretrace_python/frontend/ast_adapter.py b/src/coretrace_python/frontend/ast_adapter.py index 8d9e041..7ed63cb 100644 --- a/src/coretrace_python/frontend/ast_adapter.py +++ b/src/coretrace_python/frontend/ast_adapter.py @@ -15,9 +15,11 @@ class HIRBuildError(Exception): # The frontend and the analyses after it recurse once per level of nesting: syntax nested # deeper, which Python parses in left-nested chains (``"a" + "b" + …``, ``x.f().g()…``) -# and only generated code reaches, would exhaust Python's recursion limit (#233). The -# bound leaves the later stages room below that limit, whatever the call stack. -MAX_NESTING = 500 +# and only generated code reaches, would exhaust Python's recursion limit (#233). Under +# the default limit the whole analysis follows about 480 levels on Python 3.11 and 970 +# on 3.12 and later; the bound, CPython's own for nested parentheses, leaves twice that +# room, whatever the call stack. +MAX_NESTING = 200 _BINARY_OPERATORS = { diff --git a/src/coretrace_python/frontend/parser.py b/src/coretrace_python/frontend/parser.py index 370ecc2..6469a78 100644 --- a/src/coretrace_python/frontend/parser.py +++ b/src/coretrace_python/frontend/parser.py @@ -17,6 +17,9 @@ def parse_source(source: str, filename: str = "") -> ast.Module: column = error.offset or 0 message = error.msg or "invalid syntax" raise ParseError(f"{filename}:{line}:{column}: {message}") from error + except RecursionError as error: + # Python 3.11 builds the AST of a long left-nested chain recursively (#233). + raise ParseError(f"{filename}:1:1: syntax nested too deeply for Python's parser") from error def parse_source_file(source: SourceFile) -> ast.Module: diff --git a/tests/test_deep_nesting.py b/tests/test_deep_nesting.py index ea51cae..bf44e75 100644 --- a/tests/test_deep_nesting.py +++ b/tests/test_deep_nesting.py @@ -4,19 +4,21 @@ left-nested chains of any length (``"a" + "b" + …``, ``a.b.b…``, ``x.strip().strip()…``), so the frontend rejects, before building PyHIR, a file whose syntax is nested deeper than ``MAX_NESTING`` levels, as it rejects one it cannot represent: a ``syntax-error`` note -for that file, and the rest of the project analysed. Syntax within the bound is analysed -as before. +for that file, and the rest of the project analysed. Python 3.11 builds the AST of such +a chain recursively too: its parser's ``RecursionError`` is a parse error of the file. +Syntax within the bound is analysed as before. """ from __future__ import annotations +import ast import json from pathlib import Path import pytest from coretrace_python.cli import main -from coretrace_python.frontend import HIRBuildError, build_hir +from coretrace_python.frontend import HIRBuildError, ParseError, build_hir from coretrace_python.frontend.ast_adapter import MAX_NESTING from coretrace_python.source import SourceManager @@ -52,12 +54,24 @@ def test_a_file_nested_too_deep_is_rejected_and_the_project_is_analysed( ("run.py", "dangerous-eval"), } (note,) = [f for f in report["findings"] if f["rule_id"] == "syntax-error"] - assert f"nested more than {MAX_NESTING} levels" in note["message"] + assert "nested" in note["message"] def test_the_rejection_names_where_the_nesting_goes_too_deep() -> None: with pytest.raises(HIRBuildError, match=rf"^a\.py:1:\d+: .*nested more than {MAX_NESTING}"): - build_hir(SourceManager().add_source("a.py", chain(3000))) + build_hir(SourceManager().add_source("a.py", chain(MAX_NESTING + 50))) + + +def test_a_parser_that_recurses_too_deep_reports_a_parse_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def recursing(*args: object, **kwargs: object) -> ast.Module: + raise RecursionError("maximum recursion depth exceeded during ast construction") + + monkeypatch.setattr(ast, "parse", recursing) + + with pytest.raises(ParseError, match=r"^a\.py:1:1: syntax nested too deeply"): + build_hir(SourceManager().add_source("a.py", "X = 1\n")) def test_syntax_within_the_bound_is_analysed(