diff --git a/docs/usage.md b/docs/usage.md index ec318ad..d0da908 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -84,7 +84,10 @@ get_asgi_application()` in `asgi.py` or `email = forms.EmailField()` in a form c calls the analyzer sees, and a module-level `os.system(input())` is a finding in ``. A function using syntax outside the supported subset is reported as an `unsupported-syntax` note and the other functions are still analysed; a file Python -itself cannot parse is reported as a `syntax-error`. The coverage line and the JSON +itself cannot parse is reported as a `syntax-error`, and so is a file whose syntax is +nested more than 200 levels deep (a chain of hundreds of `+` or chained calls, which only +generated code writes), which the analyzer does not follow; the other files are still +analysed. The coverage line and the JSON report's per-file detail tell "no findings" from "nothing analysed". Taint follows values between functions and across files through function summaries, diff --git a/src/coretrace_python/frontend/ast_adapter.py b/src/coretrace_python/frontend/ast_adapter.py index dbf3522..7ed63cb 100644 --- a/src/coretrace_python/frontend/ast_adapter.py +++ b/src/coretrace_python/frontend/ast_adapter.py @@ -13,6 +13,15 @@ class HIRBuildError(Exception): """A source-located failure to represent parsed syntax as PyHIR.""" +# The frontend and the analyses after it recurse once per level of nesting: syntax nested +# deeper, which Python parses in left-nested chains (``"a" + "b" + …``, ``x.f().g()…``) +# and only generated code reaches, would exhaust Python's recursion limit (#233). Under +# the default limit the whole analysis follows about 480 levels on Python 3.11 and 970 +# on 3.12 and later; the bound, CPython's own for nested parentheses, leaves twice that +# room, whatever the call stack. +MAX_NESTING = 200 + + _BINARY_OPERATORS = { ast.Add: "add", ast.Sub: "sub", @@ -449,4 +458,21 @@ def module(self, tree: ast.Module) -> nodes.Module: def build_module(source: SourceFile, tree: ast.Module) -> nodes.Module: - return AstHIRBuilder(source).module(tree) + builder = AstHIRBuilder(source) + too_deep = _nested_too_deep(tree) + if too_deep is not None: + builder.fail(too_deep, f"syntax nested more than {MAX_NESTING} levels deep, beyond what the analyzer follows") + return builder.module(tree) + + +def _nested_too_deep(tree: ast.AST) -> ast.AST | None: + """The first node nested more than ``MAX_NESTING`` levels below ``tree``, found + without recursion; None when there is none.""" + + pending: list[tuple[ast.AST, int]] = [(tree, 0)] + while pending: + node, depth = pending.pop() + if depth > MAX_NESTING: + return node + pending.extend((child, depth + 1) for child in ast.iter_child_nodes(node)) + return None diff --git a/src/coretrace_python/frontend/parser.py b/src/coretrace_python/frontend/parser.py index 370ecc2..6469a78 100644 --- a/src/coretrace_python/frontend/parser.py +++ b/src/coretrace_python/frontend/parser.py @@ -17,6 +17,9 @@ def parse_source(source: str, filename: str = "") -> ast.Module: column = error.offset or 0 message = error.msg or "invalid syntax" raise ParseError(f"{filename}:{line}:{column}: {message}") from error + except RecursionError as error: + # Python 3.11 builds the AST of a long left-nested chain recursively (#233). + raise ParseError(f"{filename}:1:1: syntax nested too deeply for Python's parser") from error def parse_source_file(source: SourceFile) -> ast.Module: diff --git a/tests/test_deep_nesting.py b/tests/test_deep_nesting.py new file mode 100644 index 0000000..bf44e75 --- /dev/null +++ b/tests/test_deep_nesting.py @@ -0,0 +1,92 @@ +"""Acceptance tests for issue #233: syntax nested deeper than the analyzer follows. + +The frontend and the analyses after it recurse once per level of nesting. Python parses +left-nested chains of any length (``"a" + "b" + …``, ``a.b.b…``, ``x.strip().strip()…``), +so the frontend rejects, before building PyHIR, a file whose syntax is nested deeper than +``MAX_NESTING`` levels, as it rejects one it cannot represent: a ``syntax-error`` note +for that file, and the rest of the project analysed. Python 3.11 builds the AST of such +a chain recursively too: its parser's ``RecursionError`` is a parse error of the file. +Syntax within the bound is analysed as before. +""" + +from __future__ import annotations + +import ast +import json +from pathlib import Path + +import pytest + +from coretrace_python.cli import main +from coretrace_python.frontend import HIRBuildError, ParseError, build_hir +from coretrace_python.frontend.ast_adapter import MAX_NESTING +from coretrace_python.source import SourceManager + + +def chain(terms: int) -> str: + return "X = " + " + ".join(['"ab"'] * terms) + "\n" + + +@pytest.mark.parametrize( + "text", + [ + chain(3000), + "X = a" + ".b" * 3000 + "\n", + "import sys\nX = sys.argv[1]" + ".strip()" * 600 + "\n", + ], +) +def test_a_file_nested_too_deep_is_rejected_and_the_project_is_analysed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], text: str +) -> None: + root = tmp_path / "proj" + root.mkdir() + (root / "deep.py").write_text(text, encoding="utf-8") + (root / "run.py").write_text("eval(input())\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + + assert main(["--check", "proj", "--format", "json"]) == 1 + captured = capsys.readouterr() + report = json.loads(captured.out) + + assert "Traceback" not in captured.err + assert {(f["location"]["path"], f["rule_id"]) for f in report["findings"]} >= { + ("deep.py", "syntax-error"), + ("run.py", "dangerous-eval"), + } + (note,) = [f for f in report["findings"] if f["rule_id"] == "syntax-error"] + assert "nested" in note["message"] + + +def test_the_rejection_names_where_the_nesting_goes_too_deep() -> None: + with pytest.raises(HIRBuildError, match=rf"^a\.py:1:\d+: .*nested more than {MAX_NESTING}"): + build_hir(SourceManager().add_source("a.py", chain(MAX_NESTING + 50))) + + +def test_a_parser_that_recurses_too_deep_reports_a_parse_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def recursing(*args: object, **kwargs: object) -> ast.Module: + raise RecursionError("maximum recursion depth exceeded during ast construction") + + monkeypatch.setattr(ast, "parse", recursing) + + with pytest.raises(ParseError, match=r"^a\.py:1:1: syntax nested too deeply"): + build_hir(SourceManager().add_source("a.py", "X = 1\n")) + + +def test_syntax_within_the_bound_is_analysed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + root = tmp_path / "proj" + root.mkdir() + terms = MAX_NESTING - 10 + (root / "run.py").write_text( + "import os\nimport sys\n\nX = sys.argv[1]" + ' + "ab"' * terms + "\nos.system(X)\n", + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + + assert main(["--check", "proj", "--format", "json"]) == 1 + report = json.loads(capsys.readouterr().out) + + assert [f["rule_id"] for f in report["findings"]] == ["command-injection"]