diff --git a/docs/codeql/codeql-overview/system-requirements.rst b/docs/codeql/codeql-overview/system-requirements.rst index 9f477a9b39bf..1fb75319bb40 100644 --- a/docs/codeql/codeql-overview/system-requirements.rst +++ b/docs/codeql/codeql-overview/system-requirements.rst @@ -30,7 +30,7 @@ For extraction of compiled languages on Windows: For TypeScript extraction on all platforms: -- Node.js 14 or higher must be installed and available on the ``PATH`` as ``node``. +- Node.js 16.20.0 or higher must be installed and available on the ``PATH`` as ``node``. For Python extraction: diff --git a/javascript/BUILD.bazel b/javascript/BUILD.bazel index 85b2024c283c..8f4a001265ae 100644 --- a/javascript/BUILD.bazel +++ b/javascript/BUILD.bazel @@ -33,5 +33,9 @@ codeql_pack( "//javascript/extractor:tools-extractor", "//javascript/resources", ], - zips = {"//javascript/extractor/lib/typescript": "tools"}, + arch_overrides = ["tools/typescript-parser-wrapper/node_modules/@typescript"], + zips = { + "//javascript/extractor/lib/typescript": "tools", + "//javascript/extractor/lib/typescript:native": "tools/typescript-parser-wrapper/node_modules/@typescript", + }, ) diff --git a/javascript/extractor/lib/typescript/BUILD.bazel b/javascript/extractor/lib/typescript/BUILD.bazel index 28003162673e..4d7c6edfd54f 100644 --- a/javascript/extractor/lib/typescript/BUILD.bazel +++ b/javascript/extractor/lib/typescript/BUILD.bazel @@ -1,12 +1,20 @@ -# Builds a zip file of the compiled typscript-parser-wrapper and its dependencies. +load("@rules_python//python:defs.bzl", "py_test") +load("//misc/bazel:os.bzl", "codeql_platform_select") +load("//misc/bazel:pkg.bzl", "codeql_pack", "codeql_pack_group") + +# Keep the JavaScript API common, but ship native executables in the arch package. genrule( - name = "typescript", + name = "build", srcs = [ "tsconfig.json", "package.json", "package-lock.json", + "package-runtime.mjs", ] + glob(["src/**"]), - outs = ["javascript.zip"], + outs = [ + "javascript.zip", + "native.zip", + ], cmd = "\n".join([ # the original working directory is not preserved anywhere, but needs to be accessible, as # all paths are relative to this @@ -25,20 +33,38 @@ genrule( "rm -rf $(RULEDIR)/inputs", "cp -L -R $$(dirname $(execpath package.json)) $(RULEDIR)/inputs", "cd $(RULEDIR)/inputs", - "$$NPM install", + "$$NPM ci", "$$NPM run build", "rm -rf node_modules", - # Install again with only runtime deps - "$$NPM install --prod", + "DISTRIBUTION=", + ]) + codeql_platform_select( + linux64 = "linux64", + linux_arm64 = "linux-arm64", + osx64 = "osx64", + win64 = "win64", + ) + "\n" + "\n".join([ + "$$NPM ci --omit=dev --omit=optional --ignore-scripts", + # Optional dependencies are host-filtered even with --force. Install the + # selected packages explicitly, retaining the lockfile's versions/integrities. + "$$NPM install --omit=dev --ignore-scripts --no-save --force $$(node package-runtime.mjs --packages $$DISTRIBUTION)", + "node package-runtime.mjs $$DISTRIBUTION", "mv node_modules build/", + # Node needs the ESM package boundary in the installed wrapper too. + "cp package.json build/", "mkdir -p typescript-parser-wrapper", "mv build/* typescript-parser-wrapper", - "OUT=$$BAZEL_ROOT/$@", + "OUT=$$BAZEL_ROOT/$(execpath javascript.zip)", + "NATIVE_OUT=$$BAZEL_ROOT/$(execpath native.zip)", "case $$OSTYPE in", - " cygwin|msys|win32) OUT=$$(cygpath -w $$OUT);;", + " cygwin|msys|win32) OUT=$$(cygpath -w $$OUT); NATIVE_OUT=$$(cygpath -w $$NATIVE_OUT);;", "esac", "", - "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT $$(find typescript-parser-wrapper -name '*' -print)", + # Use response files to stay below Windows' command-line length limit. + "find typescript-parser-wrapper -print > wrapper-files", + "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT @wrapper-files", + "cd native", + "find typescript-* -print > native-files", + "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$NATIVE_OUT @native-files", "rm -rf $$TEMP", ]), tools = [ @@ -48,3 +74,59 @@ genrule( ], visibility = ["//visibility:public"], ) + +filegroup( + name = "typescript", + srcs = ["javascript.zip"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "native", + srcs = ["native.zip"], + visibility = ["//visibility:public"], +) + +codeql_pack( + name = "package-test-pack", + srcs = [], + arch_overrides = ["tools/typescript-parser-wrapper/node_modules/@typescript"], + install_dest = None, + pack_prefix = "", + zips = { + ":native": "tools/typescript-parser-wrapper/node_modules/@typescript", + ":typescript": "tools", + }, +) + +codeql_pack_group( + name = "package-test-distribution", + srcs = [":package-test-pack"], + skip_installer = True, +) + +py_test( + name = "package-test", + size = "small", + srcs = ["test/package_test.py"], + data = [ + "package-runtime.mjs", + ":native", + ":package-test-distribution-arch-zip", + ":package-test-distribution-common-zip", + ":typescript", + "//misc/ripunzip", + "@nodejs//:node_bin", + ], + env = { + "NODE_BIN": "$(rlocationpath @nodejs//:node_bin)", + "PACKAGE_RUNTIME": "$(rlocationpath package-runtime.mjs)", + "RIPUNZIP": "$(rlocationpath //misc/ripunzip)", + "TS_ARCH_ZIP": "$(rlocationpath :package-test-distribution-arch-zip)", + "TS_COMMON_ZIP": "$(rlocationpath :package-test-distribution-common-zip)", + "TS_NATIVE_ZIP": "$(rlocationpath :native)", + "TS_WRAPPER_ZIP": "$(rlocationpath :typescript)", + }, + main = "test/package_test.py", + deps = ["@rules_python//python/runfiles"], +) diff --git a/javascript/extractor/lib/typescript/README.md b/javascript/extractor/lib/typescript/README.md index 998cec250aaa..792dad81cf23 100644 --- a/javascript/extractor/lib/typescript/README.md +++ b/javascript/extractor/lib/typescript/README.md @@ -16,3 +16,234 @@ To debug the parser script: 2. Open VSCode and choose "Debug: Attach to Node process" from the command palette, and choose the process that looks like the parser-wrapper. + +Building and running +-------------------- + +The wrapper uses the exact `typescript@7.1.0-dev.20261015.1` package for both +compilation/type checking and the native runtime. Its integrity and those of all +platform-specific optional dependencies are recorded in `package-lock.json`. + +```sh +npm ci +npm run build +npm test +node build/main.js --version +``` + +Node.js 16.20.0 is the minimum supported runtime, matching the pinned TypeScript +package's engine requirement. The existing Bazel toolchain uses Node.js 18.15.0 +and does not need an upgrade. Build scripts invoke TypeScript's +`lib/tsc.js` launcher explicitly because these Node.js versions cannot load its +extensionless ESM `bin/tsc` entry point. The installed wrapper must include its +`package.json` to retain the ESM package boundary. + +The existing TSLint rules require the legacy JavaScript compiler API, not the +native TypeScript 7 API. If compatible TSLint tooling is not already installed, +run it separately without replacing the wrapper's build/runtime dependency: + +```sh +npm exec --yes --package=tslint@6.1.3 --package=typescript@5.9.2 -- tslint --project . +``` + +`SEMMLE_TYPESCRIPT_HOME` still overrides the runtime dependency. It must point to +the installed TypeScript package directory (containing `package.json`), with the +native `typescript/sync` and `typescript/ast` entry points and matching executable. +A legacy 5.x module or a directory from which the bundled package merely happens +to be resolvable is not a valid override. + +Distribution packaging +---------------------- + +The Bazel build produces two archives: `:typescript` (`javascript.zip`) contains +the compiled ESM wrapper and JavaScript API; `:native` (`native.zip`) contains the +matching native packages for the target CodeQL distribution: + +| Distribution | Native packages | +| ------------- | ----------------------------------------------------------------------------- | +| `linux64` | `@typescript/typescript-linux-x64` | +| `linux-arm64` | `@typescript/typescript-linux-arm64` | +| `osx64` | `@typescript/typescript-darwin-arm64` and `@typescript/typescript-darwin-x64` | +| `win64` | `@typescript/typescript-win32-x64` | + +The build installs the selected packages explicitly with `--force --no-save`, +using the pinned versions and integrities in the lockfile. Merely passing +`--force` to `npm ci` still filters optional dependencies by the build host. +`package-runtime.mjs` verifies the native package versions and marks executables +as `0755` before ZIP creation. The common CodeQL archive contains the wrapper; +the architecture archive installs native packages into +`tools/typescript-parser-wrapper/node_modules/@typescript`. Both archives must +be installed, with executable permissions preserved. No separate TypeScript +installation is needed. + +Run the installed-package smoke test with: + +```sh +bazel test //javascript/extractor/lib/typescript:package-test +``` + +It checks all distribution selections, mismatched runtime rejection, common/arch +ZIP separation and executable metadata, then installs the merged archives with +the production `ripunzip` and parses TS, TSX, and malformed source through the +installed JSON protocol. It runs on the host platform and uses Bazel's Node.js +runtime. The Java extractor test suite also installs both ZIPs and restores +executable permissions using their central-directory metadata. + +All four distribution runtime ZIPs have been cross-assembled and inspected. +Installed parsing has been verified with Node.js 18.15.0 on macOS arm64 and on +macOS x64 through Rosetta. CI integration tests have also passed on Linux x64, +Linux arm64, and Windows x64, including Windows 2022. + +Native lifecycle and memory +--------------------------- + +A single native API/process is created lazily and reused across files and +prepare-files batches. Each file still owns a separate program: AST projection +and JSON serialization finish before the program is disposed, including on +failure. Program disposal releases its snapshot and reference-counted source-file +caches without restarting the compiler. Reset closes the API, and the next parse +creates a new one. Quit, EOF, failure, and completion of single-file mode also +close the API. + +`SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD` remains a positive megabyte budget, defaulting +to 1000 when the Java extractor does not supply it. At native-process creation, +the wrapper subtracts Node's current RSS (not just its V8 heap) from that budget +and passes the remaining bytes, with a minimum of one, as the child's +`GOMEMLIMIT`. The parent's original `GOMEMLIMIT` is restored after spawning. +This is a Go garbage-collector soft limit, not a hard combined RSS ceiling: +Node allocations during extraction, native non-Go allocations, and oversized +individual files can exceed the budget. The child's budget is fixed until the +API is closed; later changes to Node RSS do not resize it. Node RSS exceeding +the budget at native-process creation produces a warning on stderr. + +Native stderr is inherited separately from the JSON stdout protocol. Syntax +diagnostics remain successful AST responses; native, I/O, and protocol failures +are reported on stderr and terminate the wrapper with a nonzero exit status. +Reset, quit, EOF, and failure invalidate queued prefetch work; quit also closes +stdin without requiring the sender to close its pipe. + +AST projection and regression tests +----------------------------------- + +The wrapper targets the native `typescript/sync` and `typescript/ast` entry +points, using the same pinned dependency in the production ESM build. +`npm test` builds the production wrapper with the pinned compiler and runs +`test/wrapper.test.mjs` and `test/lifecycle.test.mjs` against its compiled output, +copied with its ESM package boundary into temporary installations. No separate +probe installation or legacy compiler is needed for these tests. + +AST nodes are explicitly projected into plain objects before their native +program is disposed. Syntax-kind and node-flag metadata are sent unchanged. +The Java converter consumes native fields directly, including `postfixToken`, +`defaultType`, and import attributes, and recognizes the native `EndOfFile` +spelling. Namespace classification and variable declaration kinds are derived +in Java without synthetic flags or properties. + +The tests cover AST projection, metadata, UTF-16 source positions, syntax +diagnostics, TSX, repeated parses, native process reuse and disposal, the child memory +budget, reset/quit/EOF, cancelled prefetch, runtime overrides, and fatal native +failures. Run them with Node.js 16.20.0 to verify the minimum supported runtime. + +Compatibility and regression coverage +------------------------------------ + +Native interface `extends` and class `implements` elements are type references. +The Java converter handles these using its existing type conversion, preserving +generic arguments and qualified names without rewriting their node kinds. + +The native parser still produces ASTs for legacy `assert { ... }` import +assertions, but adds diagnostic 2880. The wrapper excludes only that diagnostic +to preserve previously accepted syntax; genuine syntax diagnostics are unchanged. +Import-attribute names and values are explicitly serialized. Their absence +from the old wrapper's JSON was a bug: the old Java converter expected `key`/`value`, +but the old filter exposed neither correctly. Java now reads the native `name`/`value` +fields. The corresponding TRAP expectation was regenerated; existing QL +expectations did not need changes. + +Rescanning preserves token text and offsets while retaining the original +pre-rescan token kinds on the wire. In particular, regex and template-tail tokens +keep the classifications expected by existing TRAP tests. + +Source-phase imports (`import source wasm from "./module.wasm"`) are accepted. +Their declaration, imported path, and variable bindings use the existing import +representation; the schema does not yet distinguish the source phase. The new +`TypeScript/SourceImports` QL test covers these bindings. + +Validation on October 19, 2026 passed the pinned-compiler build/type check, +existing TSLint rules, parser compatibility and lifecycle tests, +500 Java tests, 68 TypeScript QL tests, the syntax-error query test, both extraction +diagnostic integration tests, and the installed-package smoke test. The +internal-error integration test deliberately terminates the native child rather +than relying on the old parser's stack overflow. + +All 19 production-output regression tests also pass on Node.js 16.20.0, the +minimum supported runtime. + +For QL tests, install the newly built extractor and explicitly select it; otherwise +the CLI may silently exercise an older installed extractor: + +```sh +# From the repository root, with the Java extractor build dependencies available: +bazel run //javascript:install +codeql resolve languages --search-path=javascript/extractor-pack:. --format=json +codeql test run -j4 --search-path=javascript/extractor-pack:. \ + javascript/ql/test/library-tests/TypeScript \ + javascript/ql/test/query-tests/LanguageFeatures/SyntaxError +``` + +Performance comparison +---------------------- + +The current shared-API implementation was compared with refreshed `main` at +`fb0ba8c8a8a` on October 21, 2026. Both ran on macOS arm64 with Node.js 18.15.0; +`main` was built and run with TypeScript 5.9.2, and this branch used the pinned +7.1 native compiler. These are medians of five fresh-process runs after one +warm-up per variant and workload, alternating variant order: + +| Workload | Main time | Branch time | Main combined RSS | Branch combined RSS | +| --- | --- | --- | --- | --- | +| 104 TypeScript fixtures (33,779 bytes) | 0.081 s | 0.120 s | 105.5 MiB | 110.4 MiB | +| Five parses of TypeScript's 588,128-byte declaration file | 0.886 s | 0.992 s | 263.7 MiB | 247.2 MiB | + +The branch takes 47.3% more time on the fixtures and 11.9% more on the declaration +workload. Sampled combined RSS is 4.7% higher and 6.3% lower, respectively. +Both variants parse identical inputs. Timing starts after the metadata handshake +and includes prepare-files, AST extraction, serialization, and protocol responses, +including the branch's first native-process startup. It excludes Java conversion +and database import. RSS is the maximum sampled sum of Node and its native +descendants, sampled at a nominal 5 ms interval; short-lived peaks may be missed. +Current `main` includes parser cleanup absent from the older baseline below. + +### Historical and API-lifetime comparisons + +The initial implementation created a native process per file. On macOS arm64 +with Node.js 18.15.0, a comparison against the pre-migration wrapper from +`1971c368e1d` compiled with TypeScript 5.9.2 gave these medians over three runs: + +| Workload | 5.9.2 time | 7.1 time | 5.9.2 combined RSS | 7.1 combined RSS | +| --------------------------------------------------------- | ---------- | -------- | ------------------ | ---------------- | +| 103 TypeScript fixtures (33,589 bytes) | 0.237 s | 1.349 s | 124.1 MiB | 113.9 MiB | +| Five parses of TypeScript's 588,128-byte declaration file | 0.991 s | 1.116 s | 269.3 MiB | 249.1 MiB | + +Timing covers prepare-files, extraction, serialization, and protocol responses +after the metadata handshake, not Java conversion or database import. Combined +RSS is the maximum sampled sum of Node and its native descendants, sampled every +5 ms; short-lived peaks may be missed. The per-file native process costs about +5.7x on tiny files and 1.13x on the declaration workload, while measured memory is +slightly lower. + +A follow-up comparison on October 21, 2026, using the same platform and Node.js +version, measured the per-file and shared-API lifetimes directly (medians of three +runs): + +| Workload | Per-file API time | Shared API time | Per-file combined RSS | Shared combined RSS | +| --- | --- | --- | --- | --- | +| 104 TypeScript fixtures (33,779 bytes) | 1.017 s | 0.118 s | 115.8 MiB | 109.4 MiB | +| Five parses of the same declaration file | 1.044 s | 0.985 s | 250.2 MiB | 260.2 MiB | + +AST responses were byte-identical between lifetimes. A longer shared-API run on +Node.js 26.7.0 processed 50,000 distinct generated TypeScript files: native RSS +was 27.9 MiB after 1,000 files and 30.4 MiB after 50,000, while combined RSS +stabilized around 176 MiB. Another run completed 500 parses of the declaration +file with peak native RSS of 56.5 MiB. These workloads showed no unbounded +per-file retention, so the wrapper reuses the API without a fixed batch limit. diff --git a/javascript/extractor/lib/typescript/package-lock.json b/javascript/extractor/lib/typescript/package-lock.json index 68923719ff8d..72ccac5fe8c2 100644 --- a/javascript/extractor/lib/typescript/package-lock.json +++ b/javascript/extractor/lib/typescript/package-lock.json @@ -6,10 +6,13 @@ "": { "name": "typescript-parser-wrapper", "dependencies": { - "typescript": "5.9" + "typescript": "7.1.0-dev.20261015.1" }, "devDependencies": { "@types/node": "^24.3.1" + }, + "engines": { + "node": ">=16.20.0" } }, "node_modules/@types/node": { @@ -21,16 +24,137 @@ "undici-types": "~7.10.0" } }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-cIBrgsPrcPJfnyEP1gnonNoBcV6WZAxpWdrOO6F60eDhwknkHPxcTmdwuFA9ar07fFzyfp7XhKRrqz7OlrSQnA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-TkxvjvYuMDSzEmeI3g6XzLPKb6FBlonWu9FkmLMeXMNAxWRBjJmXdyKEBhDM3EASDBXAZL68X2sEim1l7e9r4g==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-8hnNa5ptGrsXCl5bs7ijxVQsuOcaCGxeDOAWqFdCNIRDQ9Z69NNrmizH650gPnZQCcNyxkFpK7QmoGSv9jsHSg==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-LaWnTy/FahvhOdo8w2ugYkDAmbcBuY6Zm8jNKYtqghx8bGeH0Mj+dX4533++tzPxAZoRnCE0WdGoBfzpDlLS8Q==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-9hRSH3i84yqjpVZtapYH/GAvRxTnh71Bui6EQDWkvrBZ6wvFleEgsyEg9RXxQDedXNMMHwKRGSYT7NvRymuytw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-hUdo1fnWfYsC4Az+1uBySKCaLBvrfWEH+jALFFvY5fROBU7cqoobVRQmOweJpiLsUQDqrHXAD3v0rnCEhCKI9w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-SQephnLxrS46fwPsM2K1LGbD8jHT4H4KF+bBoGnaSoEMLgLulAdrhY7A7lWUI3OMRuwawjfHxsnSV4TjeKEhcw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, "node_modules/typescript": { - "version": "5.9.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.2.tgz", - "integrity": "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A==", + "version": "7.1.0-dev.20261015.1", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.1.0-dev.20261015.1.tgz", + "integrity": "sha512-IonOVC0CPC24brjf3BOPYSrU8CioR/p15jRRUnx6M6zywu4tcDw0vcIKE1dR9XpOEpArd15xZ5rIFj9wjAyOkg==", + "license": "Apache-2.0", "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" + "tsc": "bin/tsc" }, "engines": { - "node": ">=14.17" + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-darwin-arm64": "7.1.0-dev.20261015.1", + "@typescript/typescript-darwin-x64": "7.1.0-dev.20261015.1", + "@typescript/typescript-linux-arm": "7.1.0-dev.20261015.1", + "@typescript/typescript-linux-arm64": "7.1.0-dev.20261015.1", + "@typescript/typescript-linux-x64": "7.1.0-dev.20261015.1", + "@typescript/typescript-win32-arm64": "7.1.0-dev.20261015.1", + "@typescript/typescript-win32-x64": "7.1.0-dev.20261015.1" } }, "node_modules/undici-types": { diff --git a/javascript/extractor/lib/typescript/package-runtime.mjs b/javascript/extractor/lib/typescript/package-runtime.mjs new file mode 100644 index 000000000000..159f39ba3cf6 --- /dev/null +++ b/javascript/extractor/lib/typescript/package-runtime.mjs @@ -0,0 +1,57 @@ +import { + chmodSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + statSync, +} from "node:fs"; +import { join } from "node:path"; + +const platforms = { + linux64: ["linux-x64"], + "linux-arm64": ["linux-arm64"], + osx64: ["darwin-arm64", "darwin-x64"], + win64: ["win32-x64"], +}; +const packagesOnly = process.argv[2] === "--packages"; +const distribution = process.argv[packagesOnly ? 3 : 2]; +if (!Object.hasOwn(platforms, distribution)) { + throw new Error("Unsupported CodeQL distribution: " + distribution); +} +const readJson = (filename) => JSON.parse(readFileSync(filename, "utf8")); +const typescript = readJson("node_modules/typescript/package.json"); +if (packagesOnly) { + console.log( + platforms[distribution] + .map( + (platform) => + "@typescript/typescript-" + platform + "@" + typescript.version, + ) + .join(" "), + ); + process.exit(0); +} +mkdirSync("native"); +for (const platform of platforms[distribution]) { + const name = "typescript-" + platform; + const source = join("node_modules", "@typescript", name); + const manifest = readJson(join(source, "package.json")); + if ( + manifest.name !== "@typescript/" + name || + manifest.version !== typescript.version || + typescript.optionalDependencies[manifest.name] !== manifest.version + ) { + throw new Error( + "Native runtime does not match TypeScript: " + manifest.name, + ); + } + const executableName = platform.startsWith("win32-") ? "tsc.exe" : "tsc"; + const executable = join(source, "lib", executableName); + if (!statSync(executable).isFile()) { + throw new Error("Native executable is not a file: " + executable); + } + chmodSync(executable, 0o755); + renameSync(source, join("native", name)); +} +rmSync("node_modules/@typescript", { recursive: true }); diff --git a/javascript/extractor/lib/typescript/package.json b/javascript/extractor/lib/typescript/package.json index f23f197e870b..cb2617408f63 100644 --- a/javascript/extractor/lib/typescript/package.json +++ b/javascript/extractor/lib/typescript/package.json @@ -1,15 +1,20 @@ { "name": "typescript-parser-wrapper", "private": true, + "type": "module", + "engines": { + "node": ">=16.20.0" + }, "dependencies": { - "typescript": "5.9" + "typescript": "7.1.0-dev.20261015.1" }, "scripts": { - "build": "tsc --project tsconfig.json", - "check": "tsc --noEmit --project . && tslint --project .", + "build": "node node_modules/typescript/lib/tsc.js --project tsconfig.json", + "check": "node node_modules/typescript/lib/tsc.js --noEmit --project . && tslint --project .", + "test": "npm run build && node --test test/lifecycle.test.mjs test/wrapper.test.mjs", "lint": "tslint --project .", "lint-fix": "tslint --project . --fix", - "watch": "tsc -p . -w --sourceMap" + "watch": "node node_modules/typescript/lib/tsc.js -p . -w --sourceMap" }, "devDependencies": { "@types/node": "^24.3.1" diff --git a/javascript/extractor/lib/typescript/src/ast_extractor.ts b/javascript/extractor/lib/typescript/src/ast_extractor.ts index a39395f12498..e1413c2c5067 100644 --- a/javascript/extractor/lib/typescript/src/ast_extractor.ts +++ b/javascript/extractor/lib/typescript/src/ast_extractor.ts @@ -1,143 +1,194 @@ -import * as ts from "./typescript"; +import * as ts from "./typescript.js"; -/** - * Interface that exposes some internal properties we rely on, as well as - * some properties we add ourselves. - */ -export interface AugmentedSourceFile extends ts.SourceFile { - parseDiagnostics?: any[]; - /** Internal property that we expose as a workaround. */ - redirectInfo?: object | null; - $tokens?: Token[]; - $lineStarts?: ReadonlyArray; +export type JsonValue = string | number | boolean | JsonObject | JsonValue[]; +export interface JsonObject { + [property: string]: JsonValue; } -export interface AugmentedNode extends ts.Node { - $pos?: any; - $end?: any; - $declarationKind?: string; -} - -export type AugmentedPos = number; +/** Child and scalar properties consumed by the Java AST converter. */ +const astProperties = [ + "argument", + "argumentExpression", + "arguments", + "assertsModifier", + "asteriskToken", + "attributes", + "block", + "body", + "caseBlock", + "catchClause", + "checkType", + "children", + "clauses", + "closingElement", + "closingFragment", + "condition", + "constraint", + "declarationList", + "declarations", + "defaultType", + "dotDotDotToken", + "elements", + "elementType", + "elseStatement", + "exclamationToken", + "exportClause", + "expression", + "exprName", + "extendsType", + "falseType", + "finallyBlock", + "head", + "heritageClauses", + "importClause", + "incrementor", + "indexType", + "initializer", + "keywordToken", + "label", + "left", + "literal", + "members", + "modifiers", + "moduleReference", + "moduleSpecifier", + "name", + "namedBindings", + "objectType", + "openingElement", + "openingFragment", + "operand", + "operator", + "operatorToken", + "parameterName", + "parameters", + "phaseModifier", + "postfixToken", + "properties", + "propertyName", + "qualifier", + "questionDotToken", + "questionToken", + "right", + "statement", + "statements", + "tag", + "tagName", + "template", + "templateSpans", + "text", + "thenStatement", + "token", + "trueType", + "tryBlock", + "type", + "typeArguments", + "typeName", + "typeParameter", + "typeParameters", + "types", + "value", + "variableDeclaration", + "whenFalse", + "whenTrue", +]; -export interface Token { - kind: ts.SyntaxKind; - tokenPos: AugmentedPos; - text: string; -} +const skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g; -let skipWhiteSpace = /(?:\s|\/\/.*|\/\*[^]*?\*\/)*/g; +export function extractAst(ast: ts.SourceFile, diagnostics: readonly ts.Diagnostic[]): JsonObject { + const code = ast.text; + // The native parser still builds assertion ASTs, but now diagnoses this formerly accepted syntax. + const parseDiagnostics = diagnostics.filter(diagnostic => diagnostic.code !== 2880); -/** - * Invokes the given callback for every AST node in the given tree. - */ -function forEachNode(ast: ts.Node, callback: (node: ts.Node) => void) { - function visit(node: ts.Node) { - ts.forEachChild(node, visit); - callback(node); + function augmentPos(pos: number): number { + skipWhiteSpace.lastIndex = pos; + return pos + skipWhiteSpace.exec(code)![0].length; } - visit(ast); -} - -export function augmentAst(ast: AugmentedSourceFile, code: string) { - ast.$lineStarts = ast.getLineStarts(); - /** - * Converts a numeric offset to the value expected by the Java counterpart of the extractor. - */ - function augmentPos(pos: number, shouldSkipWhitespace?: boolean): AugmentedPos { - // skip over leading spaces/comments - if (shouldSkipWhitespace) { - skipWhiteSpace.lastIndex = pos; - pos += skipWhiteSpace.exec(code)[0].length; + function projectValue(value: unknown): JsonValue { + if (Array.isArray(value)) return Array.from(value, projectValue); + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + return value; } - return pos; - } - - // Find the position of all tokens where the scanner requires parse-tree information. - // At these offsets, a call to `scanner.reScanX` is required. - type RescanEvent = () => void; - let reScanEvents: RescanEvent[] = []; - let reScanEventPos: number[] = []; - let scanner = ts.createScanner(ts.ScriptTarget.ES2015, false, 1, code); - let reScanSlashToken = scanner.reScanSlashToken.bind(scanner); - let reScanTemplateToken = scanner.reScanTemplateToken.bind(scanner); - let reScanGreaterToken = scanner.reScanGreaterToken.bind(scanner); - if (!ast.parseDiagnostics || ast.parseDiagnostics.length === 0) { - forEachNode(ast, node => { - if (ts.isRegularExpressionLiteral(node)) { - reScanEventPos.push(node.getStart(ast, false)); - reScanEvents.push(reScanSlashToken); - } - if (ts.isTemplateMiddle(node) || ts.isTemplateTail(node)) { - reScanEventPos.push(node.getStart(ast, false)); - reScanEvents.push(reScanTemplateToken); - } - if (ts.isBinaryExpression(node)) { - let operator = node.operatorToken; - switch (operator.kind) { - case ts.SyntaxKind.GreaterThanEqualsToken: - case ts.SyntaxKind.GreaterThanGreaterThanEqualsToken: - case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: - case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanToken: - case ts.SyntaxKind.GreaterThanGreaterThanToken: - reScanEventPos.push(operator.getStart(ast, false)); - reScanEvents.push(reScanGreaterToken); - break; - } - } - }); + if (value != null && typeof value === "object" && "forEachChild" in value) { + return projectNode(value as ts.Node); + } + throw new Error("Unexpected TypeScript AST property value"); } - reScanEventPos.push(Infinity); + function projectNode(node: ts.Node): JsonObject { + const result: JsonObject = { + kind: node.kind, + flags: node.flags, + $pos: augmentPos(node.pos), + $end: node.end, + }; + const properties = node as ts.Node & { [property: string]: unknown }; + for (const property of astProperties) { + const value = properties[property]; + if (value != null) result[property] = projectValue(value); + } - // add tokens and comments to the AST - ast.$tokens = []; - let rescanEventIndex = 0; - let nextRescanPosition = reScanEventPos[0]; - let tk; - do { - tk = scanner.scan(); - if (scanner.getTokenPos() === nextRescanPosition) { - let callback = reScanEvents[rescanEventIndex]; - callback(); - ++rescanEventIndex; - nextRescanPosition = reScanEventPos[rescanEventIndex]; + // Native nodes share boolean getters even on kinds where they are not meaningful. + if (ts.isExportAssignment(node)) result.isExportEquals = node.isExportEquals; + if (ts.isExportDeclaration(node) || ts.isImportEqualsDeclaration(node) + || ts.isImportSpecifier(node) || ts.isExportSpecifier(node)) { + result.isTypeOnly = node.isTypeOnly; } - ast.$tokens.push({ - kind: tk, - tokenPos: augmentPos(scanner.getTokenPos()), - text: scanner.getTokenText(), - }); - } while (tk !== ts.SyntaxKind.EndOfFileToken); + if (ts.isImportTypeNode(node)) result.isTypeOf = node.isTypeOf; - if (ast.parseDiagnostics) { - ast.parseDiagnostics.forEach(d => { - delete d.file; - d.$pos = augmentPos(d.start); - }); + return result; } - visitAstNode(ast); - function visitAstNode(node: AugmentedNode) { - ts.forEachChild(node, visitAstNode); + const result = projectNode(ast); + result.$lineStarts = Array.from(ast.getLineStarts()); + result.$tokens = extractTokens(ast, parseDiagnostics.length === 0); + result.parseDiagnostics = parseDiagnostics.map(diagnostic => ({ + messageText: diagnostic.text, + $pos: diagnostic.pos, + })); + return result; +} - // fill in line/column info - if ("pos" in node) { - node.$pos = augmentPos(node.pos, true); - } - if ("end" in node) { - node.$end = augmentPos(node.end); - } +function extractTokens(ast: ts.SourceFile, rescan: boolean): JsonObject[] { + const scanner = ts.createScanner(false, ts.LanguageVariant.JSX, ast.text); + const rescans = new Map ts.SyntaxKind>(); + if (rescan) visit(ast); - if (ts.isVariableDeclarationList(node as any)) { - if (node.flags & ts.NodeFlags.Let) { - node.$declarationKind = "let"; - } else if (node.flags & ts.NodeFlags.Const) { - node.$declarationKind = "const"; - } else { - node.$declarationKind = "var"; + function visit(node: ts.Node): void { + node.forEachChild(visit); + if (ts.isRegularExpressionLiteral(node)) { + rescans.set(node.getStart(ast), () => scanner.reScanSlashToken()); + } else if (ts.isTemplateMiddle(node) || ts.isTemplateTail(node)) { + const template = node.parent.parent; + const tagged = ts.isTemplateExpression(template) && ts.isTaggedTemplateExpression(template.parent); + rescans.set(node.getStart(ast), () => scanner.reScanTemplateToken(tagged)); + } else if (ts.isBinaryExpression(node)) { + const operator = node.operatorToken; + switch (operator.kind) { + case ts.SyntaxKind.GreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanEqualsToken: + case ts.SyntaxKind.GreaterThanGreaterThanGreaterThanToken: + case ts.SyntaxKind.GreaterThanGreaterThanToken: + rescans.set(operator.getStart(ast), () => scanner.reScanGreaterToken()); + break; } } } + + const tokens: JsonObject[] = []; + let kind: ts.SyntaxKind; + do { + kind = scanner.scan(); + const callback = rescans.get(scanner.getTokenStart()); + // Keep the pre-rescan kind, as in the original Java-facing token protocol. + if (callback != null) callback(); + tokens.push({ + kind, + tokenPos: scanner.getTokenStart(), + text: scanner.getTokenText(), + }); + } while (kind !== ts.SyntaxKind.EndOfFile); + return tokens; } diff --git a/javascript/extractor/lib/typescript/src/main.ts b/javascript/extractor/lib/typescript/src/main.ts index 3d6b10561f3b..487cdb38f64a 100644 --- a/javascript/extractor/lib/typescript/src/main.ts +++ b/javascript/extractor/lib/typescript/src/main.ts @@ -1,42 +1,13 @@ /** - * @fileoverview Wrapper for TypeScript parser, inspired by - * [typescript-eslint-parser](https://github.com/eslint/typescript-eslint-parser). - * - * The wrapper can be invoked with a single argument, which is interpreted as a file name. - * The file is parsed and the AST dumped to stdout as described below. - * - * When run without arguments, the wrapper enters server mode, expecting JSON-encoded requests - * on `stdin`, one per line. - * - * Each request should have a `command` property, which should either be `parse` or `quit`. - * - * A `quit` request causes the wrapper to terminate. - * - * A `parse` request should have a `filename` property, which should be the path of a - * TypeScript file. - * - * In response to a `parse` request, the wrapper invokes the TypeScript parser to parse - * the given file, and dumps a JSON representation of the resulting AST to stdout, - * with the following adjustments: - * - * - the `kind` and `operator` properties have symbolic string values instead of numeric - * values (the latter change between versions of the TypeScript compiler, the former - * don't); - * - position information is given in terms of line/column pairs instead of offsets; - * - declarations have a `declarationKind` property indicating whether they are `var`, - * `let` or `const` declarations; - * - `parent` pointers are omitted (since they cannot be serialised to JSON); - * - tokens and comments are added to the AST root node in a `tokens` property. + * Wrapper for the TypeScript parser. Accepts newline-delimited JSON commands on stdin + * and writes ASTs and parser metadata to stdout, or parses a single command-line file. */ -"use strict"; - import * as pathlib from "path"; import * as readline from "readline"; -import * as ts from "./typescript"; -import * as ast_extractor from "./ast_extractor"; +import * as ts from "./typescript.js"; +import { extractAst } from "./ast_extractor.js"; -// Remove limit on stack trace depth. Error.stackTraceLimit = Infinity; interface ParseCommand { @@ -58,339 +29,195 @@ interface GetMetadataCommand { } type Command = ParseCommand | ResetCommand | QuitCommand | PrepareFilesCommand | GetMetadataCommand; -/** The state to be shared between commands. */ class State { - /** List of files that have been requested. */ public pendingFiles: string[] = []; public pendingFileIndex = 0; - - /** Next response to be delivered. */ - public pendingResponse: string = null; + public pendingResponse: string | null = null; } let state = new State(); - -/** - * Debugging method for finding cycles in the TypeScript AST. Should not be used in production. - * - * If cycles are found, the whitelist in `astProperties` is too permissive. - */ -// tslint:disable-next-line:no-unused-variable -function checkCycle(root: any) { - let path: string[] = []; - function visit(obj: any): boolean { - if (obj == null || typeof obj !== "object") return false; - if (obj.$cycle_visiting) { - return true; - } - obj.$cycle_visiting = true; - for (let k in obj) { - if (!obj.hasOwnProperty(k)) continue; - // Ignore numeric and whitelisted properties. - if (+k !== +k && !astPropertySet.has(k)) continue; - if (k === "$cycle_visiting") continue; - let cycle = visit(obj[k]); - if (cycle) { - path.push(k); - return true; - } - } - obj.$cycle_visiting = undefined; - return false; +let stopped = false; +let readLineInterface: readline.Interface | undefined; +let reportedMemoryPressure = false; + +// The API instance used to interact with the TypeScript compiler. Should be +// accessed through `getAPI` and `closeAPI`. +let api: ts.API | undefined; + +function createAPI(): ts.API { + const threshold = Number(process.env.SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD ?? 1000) * 1000000; + if (!Number.isSafeInteger(threshold) || threshold <= 0) { + throw new Error("SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD must be a positive number of megabytes"); + } + const rss = process.memoryUsage().rss; + if (rss >= threshold && !reportedMemoryPressure) { + console.warn("Node.js RSS exceeds the TypeScript memory budget; limiting native compiler memory"); } - visit(root); - if (path.length > 0) { - path.reverse(); - console.log(JSON.stringify({ type: "error", message: "Cycle = " + path.join(".") })); + reportedMemoryPressure = rss >= threshold; + + // The API inherits the environment when it spawns its native process. + const previousLimit = process.env.GOMEMLIMIT; + process.env.GOMEMLIMIT = String(Math.max(1, threshold - rss)); + try { + return new ts.API(); + } finally { + if (previousLimit == null) delete process.env.GOMEMLIMIT; + else process.env.GOMEMLIMIT = previousLimit; } } -/** Property names to extract from the TypeScript AST. */ -const astProperties: string[] = [ - "$declarationKind", - "$end", - "$lineStarts", - "$overloadIndex", - "$pos", - "$tokens", - "argument", - "argumentExpression", - "arguments", - "assertsModifier", - "asteriskToken", - "attributes", - "block", - "body", - "caseBlock", - "catchClause", - "checkType", - "children", - "clauses", - "closingElement", - "closingFragment", - "condition", - "constraint", - "constructor", - "declarationList", - "declarations", - "default", - "delete", - "dotDotDotToken", - "elements", - "elementType", - "elementTypes", - "elseStatement", - "escapedText", - "exclamationToken", - "exportClause", - "expression", - "exprName", - "extendsType", - "falseType", - "finallyBlock", - "flags", - "head", - "heritageClauses", - "importClause", - "incrementor", - "indexType", - "init", - "initializer", - "isExportEquals", - "isTypeOf", - "isTypeOnly", - "keywordToken", - "kind", - "label", - "left", - "literal", - "members", - "messageText", - "modifiers", - "moduleReference", - "moduleSpecifier", - "name", - "namedBindings", - "objectType", - "openingElement", - "openingFragment", - "operand", - "operator", - "operatorToken", - "parameterName", - "parameters", - "parseDiagnostics", - "phaseModifier", - "properties", - "propertyName", - "qualifier", - "questionDotToken", - "questionToken", - "right", - "selfClosing", - "statement", - "statements", - "tag", - "tagName", - "template", - "templateSpans", - "text", - "thenStatement", - "token", - "tokenPos", - "trueType", - "tryBlock", - "type", - "typeArguments", - "typeName", - "typeParameter", - "typeParameters", - "types", - "variableDeclaration", - "whenFalse", - "whenTrue", -]; - -/** Property names used in a parse command response, in addition to the AST itself. */ -const astMetaProperties: string[] = [ - "ast", - "type", -]; - -/** Property names to extract in an AST response. */ -const astPropertySet = new Set([...astProperties, ...astMetaProperties]); +function getAPI(): ts.API { + return api ??= createAPI(); +} -/** - * Converts (part of) an AST to a JSON string, ignoring properties we're not interested in. - */ -function stringifyAST(obj: any) { - return JSON.stringify(obj, (k, v) => { - // Filter out properties that aren't numeric, empty, or whitelisted. - // Note `k` is the empty string for the root object, which is also covered by +k === +k. - return (+k === +k || astPropertySet.has(k)) ? v : undefined; - }); +function closeAPI() { + const previous = api; + api = undefined; + previous?.close(); } function extractFile(filename: string): string { - let ast = getAstForFile(filename); - - return stringifyAST({ - type: "ast", - ast, + const program = getAPI().createProgram([pathlib.resolve(filename)], { + experimentalDecorators: true, + jsx: ts.JsxEmit.Preserve, + noResolve: true, + noLib: true, }); + try { + const ast = program.getSourceFile(pathlib.resolve(filename)); + if (ast == null) { + throw new Error("TypeScript did not return a source file for " + filename); + } + return JSON.stringify({ + type: "ast", + ast: extractAst(ast, program.getSyntacticDiagnostics()), + }); + } finally { + program.dispose(); + } } function prepareNextFile() { if (state.pendingResponse != null) return; if (state.pendingFileIndex < state.pendingFiles.length) { - let nextFilename = state.pendingFiles[state.pendingFileIndex]; - state.pendingResponse = extractFile(nextFilename); - } -} - -function handleParseCommand(command: ParseCommand, checkPending = true) { - let filename = command.filename; - let expectedFilename = state.pendingFiles[state.pendingFileIndex]; - if (expectedFilename !== filename && checkPending) { - // File was requested out of order. This happens in rare cases because the Java process decided against extracting it, - // for example because it was too large. Just recover and accept that some work was wasted. - state.pendingResponse = null; - state.pendingFileIndex = state.pendingFiles.indexOf(filename); + state.pendingResponse = extractFile(state.pendingFiles[state.pendingFileIndex]); } - ++state.pendingFileIndex; - let response = state.pendingResponse || extractFile(command.filename); - state.pendingResponse = null; - process.stdout.write(response + "\n", () => { - // Start working on the next file as soon as the old one is flushed. - // Note that if we didn't wait for flushing, this would block the I/O - // loop and delay flushing. - prepareNextFile(); - }); -} - -/** - * Returns true if the given source file has an actual AST, as opposed to - * a SourceFile that is a "redirect" to another SourceFile. - * - * The TypeScript API should not expose such redirecting SourceFiles, - * but we sometimes get them anyway. - */ -function isExtractableSourceFile(ast: ast_extractor.AugmentedSourceFile): boolean { - return ast.redirectInfo == null; } -/** - * Gets the AST and source code by parsing the given file. - */ -function getAstForFile(filename: string): ts.SourceFile { - let { ast, code } = parseSingleFile(filename); - if (ast != null && isExtractableSourceFile(ast)) { - ast_extractor.augmentAst(ast, code); +function fail(error: unknown) { + stopped = true; + state = new State(); + console.error(error); + process.exitCode = 1; + try { + closeAPI(); + } catch (closeError) { + console.error(closeError); + } finally { + readLineInterface?.close(); + process.stdin.destroy(); } - return ast; } -function parseSingleFile(filename: string): { ast: ts.SourceFile, code: string } { - let code = ts.sys.readFile(filename); - - // create a compiler host that only allows access to `filename` - let compilerHost: ts.CompilerHost = { - fileExists() { return true; }, - getCanonicalFileName() { return filename; }, - getCurrentDirectory() { return ""; }, - getDefaultLibFileName() { return "lib.d.ts"; }, - getNewLine() { return "\n"; }, - getSourceFile() { - return ts.createSourceFile(filename, code, ts.ScriptTarget.Latest, true); - }, - readFile() { return null; }, - useCaseSensitiveFileNames() { return true; }, - writeFile() { return null; }, - getDirectories() { return []; }, - }; - - // parse `filename` with minimial checking - let compilerOptions: ts.CompilerOptions = { - experimentalDecorators: true, - jsx: ts.JsxEmit.Preserve, - noResolve: true, +function afterResponse(expectedState: State) { + return (error?: Error | null) => { + if (error != null) { + fail(error); + return; + } + if (stopped || state !== expectedState) return; + try { + prepareNextFile(); + } catch (failure) { + fail(failure); + } }; - - let program = ts.createProgram([filename], compilerOptions, compilerHost); - - let ast = program.getSourceFile(filename); - - return { ast, code }; -} - -function handleResetCommand(command: ResetCommand) { - reset(); - console.log(JSON.stringify({ - type: "reset-done", - })); } -function handlePrepareFilesCommand(command: PrepareFilesCommand) { - state.pendingFiles = command.filenames; - state.pendingFileIndex = 0; +function handleParseCommand(command: ParseCommand) { + const filename = command.filename; + const expectedFilename = state.pendingFiles[state.pendingFileIndex]; + if (expectedFilename !== filename) { + // Java may skip a prepared file, for example because it is too large. + state.pendingResponse = null; + state.pendingFileIndex = state.pendingFiles.indexOf(filename); + } + ++state.pendingFileIndex; + const response = state.pendingResponse || extractFile(filename); state.pendingResponse = null; - process.stdout.write('{"type":"ok"}\n', () => { - prepareNextFile(); - }); -} - -function handleGetMetadataCommand(command: GetMetadataCommand) { - console.log(JSON.stringify({ - type: "metadata", - syntaxKinds: ts.SyntaxKind, - nodeFlags: ts.NodeFlags, - })); -} - -function reset() { - state = new State(); + process.stdout.write(response + "\n", afterResponse(state)); } function runReadLineInterface() { - reset(); - let rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + const rl = readline.createInterface({ input: process.stdin, output: process.stdout }); + readLineInterface = rl; + rl.on("close", () => { + stopped = true; + state = new State(); + process.stdin.destroy(); + try { + closeAPI(); + } catch (error) { + fail(error); + } + }); + rl.on("error", fail); rl.on("line", (line: string) => { - let req: Command = JSON.parse(line); - switch (req.command) { - case "parse": - handleParseCommand(req); - break; - case "prepare-files": - handlePrepareFilesCommand(req); - break; - case "reset": - handleResetCommand(req); - break; - case "get-metadata": - handleGetMetadataCommand(req); - break; - case "quit": - rl.close(); - break; - default: - throw new Error("Unknown command " + (req as any).command + "."); + if (stopped) return; + try { + const req: Command = JSON.parse(line); + switch (req.command) { + case "parse": + handleParseCommand(req); + break; + case "prepare-files": + state = new State(); + state.pendingFiles = req.filenames; + process.stdout.write('{"type":"ok"}\n', afterResponse(state)); + break; + case "reset": + state = new State(); + closeAPI(); + console.log('{"type":"reset-done"}'); + break; + case "get-metadata": + console.log(JSON.stringify({ + type: "metadata", + syntaxKinds: ts.SyntaxKind, + nodeFlags: ts.NodeFlags, + })); + break; + case "quit": + rl.close(); + break; + default: + throw new Error("Unknown command " + (req as { command: string }).command + "."); + } + } catch (error) { + fail(error); } }); } -// Parse command-line arguments. -if (process.argv.length > 2) { - let argument = process.argv[2]; - if (argument === "--version") { - console.log("parser-wrapper with TypeScript " + ts.version); - } else if (pathlib.extname(argument) === ".ts" || pathlib.extname(argument) === ".tsx") { - handleParseCommand({ - command: "parse", - filename: argument, - }, false); +process.stdout.on("error", fail); + +try { + if (process.argv.length > 2) { + const argument = process.argv[2]; + if (argument === "--version") { + console.log("parser-wrapper with TypeScript " + ts.version); + } else if (pathlib.extname(argument) === ".ts" || pathlib.extname(argument) === ".tsx") { + try { + process.stdout.write(extractFile(argument) + "\n"); + } finally { + closeAPI(); + } + } else { + console.error("Unrecognized file or flag: " + argument); + process.exitCode = 1; + } } else { - console.error("Unrecognized file or flag: " + argument); + runReadLineInterface(); } - process.exit(0); -} else { - runReadLineInterface(); +} catch (error) { + fail(error); } diff --git a/javascript/extractor/lib/typescript/src/typescript.d.ts b/javascript/extractor/lib/typescript/src/typescript.d.ts index 769a318e7249..45b416b59de8 100644 --- a/javascript/extractor/lib/typescript/src/typescript.d.ts +++ b/javascript/extractor/lib/typescript/src/typescript.d.ts @@ -1,2 +1,12 @@ -import * as ts from "typescript"; -export = ts; +export { + SyntaxKind, NodeFlags, LanguageVariant, createScanner, + isRegularExpressionLiteral, isTemplateMiddle, isTemplateTail, isTemplateExpression, + isTaggedTemplateExpression, isBinaryExpression, + isExportAssignment, isExportDeclaration, isImportEqualsDeclaration, isImportSpecifier, + isExportSpecifier, isImportTypeNode, +} from "typescript/ast"; +export type { Node, SourceFile } from "typescript/ast"; +export { API, JsxEmit } from "typescript/sync"; +export type { Diagnostic } from "typescript/sync"; + +export const version: string; diff --git a/javascript/extractor/lib/typescript/src/typescript.js b/javascript/extractor/lib/typescript/src/typescript.js index f469e2e098dd..0f4df34d8242 100644 --- a/javascript/extractor/lib/typescript/src/typescript.js +++ b/javascript/extractor/lib/typescript/src/typescript.js @@ -1,7 +1,24 @@ -let overridePath = process.env['SEMMLE_TYPESCRIPT_HOME']; +import { createRequire } from "node:module"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; -if (overridePath != null) { - module.exports = require(overridePath); -} else { - module.exports = require('typescript'); +const overridePath = process.env["SEMMLE_TYPESCRIPT_HOME"]; +const require = createRequire(overridePath == null + ? import.meta.url + : resolve(overridePath, "package.json")); +if (overridePath != null + && require.resolve("typescript/package.json") !== require.resolve("./package.json")) { + throw new Error("SEMMLE_TYPESCRIPT_HOME must point to an installed TypeScript package with the native 7.1 API"); } +const ast = await import(pathToFileURL(require.resolve("typescript/ast")).href); +const sync = await import(pathToFileURL(require.resolve("typescript/sync")).href); + +export const { API, JsxEmit } = sync; +export const { + SyntaxKind, NodeFlags, LanguageVariant, createScanner, + isRegularExpressionLiteral, isTemplateMiddle, isTemplateTail, isTemplateExpression, + isTaggedTemplateExpression, isBinaryExpression, + isExportAssignment, isExportDeclaration, isImportEqualsDeclaration, isImportSpecifier, + isExportSpecifier, isImportTypeNode, +} = ast; +export const version = require("typescript").version; diff --git a/javascript/extractor/lib/typescript/test/lifecycle.test.mjs b/javascript/extractor/lib/typescript/test/lifecycle.test.mjs new file mode 100644 index 000000000000..f7e5620dced6 --- /dev/null +++ b/javascript/extractor/lib/typescript/test/lifecycle.test.mjs @@ -0,0 +1,254 @@ +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { after, test } from "node:test"; +import { setTimeout } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const directory = mkdtempSync(join(tmpdir(), "codeql-typescript-lifecycle-")); +const entryPoint = join(directory, "main.js"); +for (const name of ["main.js", "ast_extractor.js", "typescript.js"]) { + copyFileSync(join(root, "build", name), join(directory, name)); +} +copyFileSync(join(root, "package.json"), join(directory, "package.json")); +symlinkSync(join(root, "node_modules"), join(directory, "node_modules"), "dir"); +const filename = join(directory, "input.ts"); +writeFileSync(filename, "const value = 1;"); +const preload = join(directory, "track-native.cjs"); +writeFileSync(preload, ` +const cp = require("node:child_process"); +const fs = require("node:fs"); +const spawn = cp.spawn; +cp.spawn = function (...args) { + const child = spawn(...args); + fs.appendFileSync(process.env.NATIVE_LOG, JSON.stringify({ + pid: child.pid, + memoryLimit: process.env.GOMEMLIMIT, + stdio: args[2].stdio, + }) + "\\n"); + if (process.env.KILL_NATIVE) child.kill("SIGKILL"); + return child; +}; +`); +after(() => rmSync(directory, { recursive: true, force: true })); + +function options(name, env = {}) { + const environment = { ...process.env }; + delete environment.SEMMLE_TYPESCRIPT_HOME; + return { + encoding: "utf8", + timeout: 10000, + env: { + ...environment, + ...env, + NATIVE_LOG: join(directory, name + ".log"), + }, + }; +} + +function run(name, input, env = {}, args = []) { + const opts = options(name, env); + const result = spawnSync(process.execPath, ["--require", preload, entryPoint, ...args], { + ...opts, input, + }); + assert.ifError(result.error); + assert.equal(result.signal, null, result.stderr); + return result; +} + +function nativeProcesses(name) { + return readFileSync(join(directory, name + ".log"), "utf8").trim().split("\n").map(JSON.parse); +} + +async function assertTerminated(pid) { + for (let i = 0; i < 100; ++i) { + try { + process.kill(pid, 0); + } catch (error) { + assert.equal(error.code, "ESRCH"); + return; + } + await setTimeout(10); + } + assert.fail(`Native process ${pid} survived disposal`); +} + +test("compiled wrapper preserves heritage, assertions and tokens, and accepts source-phase imports", () => { + const input = join(directory, "compatibility.ts"); + writeFileSync(input, ` + interface I extends A, N.B {} + class C implements I {} + import data from "data" assert { type: "json" }; + import source wasm from "./module.wasm"; + const expression = /data/g; + const template = \`value:\${data}\`; + `); + const result = run("compatibility", [ + { command: "get-metadata" }, + { command: "parse", filename: input }, + { command: "quit" }, + ].map(command => JSON.stringify(command) + "\n").join("")); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stderr, ""); + const [metadata, { ast }] = result.stdout.trim().split("\n").map(JSON.parse); + const kinds = metadata.syntaxKinds; + assert.deepEqual(ast.parseDiagnostics, []); + const heritage = ast.statements[0].heritageClauses[0].types; + assert.equal(heritage[0].kind, kinds.TypeReference); + assert.equal(heritage[0].typeName.text, "A"); + assert.equal(heritage[1].typeName.right.text, "B"); + assert.equal(heritage[1].typeArguments[0].kind, kinds.StringKeyword); + assert.equal(ast.statements[2].attributes.attributes[0].name.text, "type"); + assert.equal(ast.statements[2].attributes.attributes[0].value.text, "json"); + assert.equal(ast.statements[3].importClause.phaseModifier, kinds.SourceKeyword); + assert.equal(ast.statements[3].importClause.name.text, "wasm"); + assert(ast.$tokens.some(token => token.text === "/data/g" && token.kind === kinds.SlashToken)); + assert(ast.$tokens.some(token => token.text === "}`" && token.kind === kinds.CloseBraceToken)); +}); + +test("compiled ESM wrapper reuses its budgeted native process until reset or quit", { timeout: 15000 }, async () => { + const name = "repeated"; + const opts = options(name, { SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD: "1000", GOMEMLIMIT: "900MiB" }); + const child = spawn(process.execPath, ["--require", preload, entryPoint], opts); + let stderr = ""; + child.stderr.setEncoding("utf8").on("data", data => { stderr += data; }); + const lines = createInterface({ input: child.stdout })[Symbol.asyncIterator](); + const exited = new Promise((accept, reject) => { + child.on("error", reject); + child.on("close", code => accept(code)); + }); + async function request(command) { + child.stdin.write(JSON.stringify(command) + "\n"); + const line = await lines.next(); + assert.equal(line.done, false, stderr); + return JSON.parse(line.value); + } + try { + for (let session = 0; session < 2; ++session) { + for (let batch = 0; batch < 2; ++batch) { + assert.deepEqual(await request({ command: "prepare-files", filenames: Array(5).fill(filename) }), + { type: "ok" }); + for (let i = 0; i < 5; ++i) { + const response = await request({ command: "parse", filename }); + assert.equal(response.type, "ast"); + assert.deepEqual(response.ast.parseDiagnostics, []); + const processes = nativeProcesses(name); + assert.equal(processes.length, session + 1); + const native = processes[session]; + assert(Number(native.memoryLimit) > 0); + assert(Number(native.memoryLimit) < 1000000000); + assert.deepEqual(native.stdio, process.platform === "win32" + ? ["ignore", "ignore", "inherit"] : ["pipe", "pipe", "inherit"]); + process.kill(native.pid, 0); + } + } + if (session === 0) { + assert.deepEqual(await request({ command: "reset" }), { type: "reset-done" }); + await assertTerminated(nativeProcesses(name)[session].pid); + } + } + // Quit must terminate even when the sender leaves stdin open. + child.stdin.write('{"command":"quit"}\n'); + assert.equal(await exited, 0, stderr); + assert.equal(stderr, ""); + await assertTerminated(nativeProcesses(name)[1].pid); + } finally { + child.stdin.destroy(); + child.kill(); + await exited; + } +}); + +test("metadata and reset do not start a native process", () => { + const result = run("lazy", [ + { command: "get-metadata" }, + { command: "reset" }, + { command: "quit" }, + ].map(command => JSON.stringify(command) + "\n").join("")); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stderr, ""); + assert.deepEqual(result.stdout.trim().split("\n").map(line => JSON.parse(line).type), ["metadata", "reset-done"]); + assert.equal(existsSync(options("lazy").env.NATIVE_LOG), false); +}); + +test("reset and quit invalidate queued prefetch callbacks", () => { + const missing = join(directory, "missing.ts"); + const result = run("cancelled", [ + { command: "prepare-files", filenames: [missing] }, + { command: "reset" }, + { command: "prepare-files", filenames: [missing] }, + { command: "quit" }, + { command: "parse", filename: missing }, + ].map(command => JSON.stringify(command) + "\n").join("")); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stderr, ""); + assert.deepEqual(result.stdout.trim().split("\n").map(JSON.parse), [ + { type: "ok" }, { type: "reset-done" }, { type: "ok" }, + ]); +}); + +test("single-file mode and EOF flush responses and close native processes", async () => { + for (const [name, input, args] of [ + ["single", "", [filename]], + ["eof", JSON.stringify({ command: "parse", filename }) + "\n", []], + ]) { + const result = run(name, input, {}, args); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).type, "ast"); + assert.equal(result.stderr, ""); + await assertTerminated(nativeProcesses(name)[0].pid); + } +}); + +test("native failures are fatal, close the process, and never become JSON diagnostics", async () => { + const result = run("killed", "", { KILL_NATIVE: "1" }, [filename]); + assert.equal(result.status, 1); + assert.equal(result.stdout, ""); + assert.match(result.stderr, /Error|EOF|EPIPE|ECONNRESET/); + await assertTerminated(nativeProcesses("killed")[0].pid); +}); + +test("protocol and extraction failures close an already running native process", async () => { + for (const [name, input, message] of [ + ["active-json", "{\n", /SyntaxError/], + ["active-command", '{"command":"invalid"}\n', /Unknown command/], + ["active-missing", JSON.stringify({ command: "parse", filename: join(directory, "missing.ts") }) + "\n", + /did not return a source file/], + ]) { + const result = run(name, JSON.stringify({ command: "parse", filename }) + "\n" + input); + assert.equal(result.status, 1, name); + assert.equal(JSON.parse(result.stdout).type, "ast"); + assert.match(result.stderr, message); + const processes = nativeProcesses(name); + assert.equal(processes.length, 1); + await assertTerminated(processes[0].pid); + } +}); + +test("invalid commands, files, memory settings, and runtime overrides fail on stderr", () => { + for (const [name, input, env, args, message] of [ + ["json", "{\n", {}, [], /SyntaxError/], + ["command", '{"command":"invalid"}\n', {}, [], /Unknown command/], + ["missing", "", {}, [join(directory, "missing.ts")], /did not return a source file/], + ["memory", "", { SEMMLE_TYPESCRIPT_MEMORY_THRESHOLD: "invalid" }, [filename], /MEMORY_THRESHOLD/], + ["override", "", { SEMMLE_TYPESCRIPT_HOME: directory }, [filename], /SEMMLE_TYPESCRIPT_HOME/], + ]) { + const result = run(name, input, env, args); + assert.equal(result.status, 1, name); + assert.equal(result.stdout, "", name); + assert.match(result.stderr, message); + } +}); + +test("SEMMLE_TYPESCRIPT_HOME selects the installed native package", () => { + const result = run("override-valid", "", { + SEMMLE_TYPESCRIPT_HOME: join(root, "node_modules", "typescript"), + }, [filename]); + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).type, "ast"); + assert.equal(result.stderr, ""); +}); diff --git a/javascript/extractor/lib/typescript/test/package_test.py b/javascript/extractor/lib/typescript/test/package_test.py new file mode 100644 index 000000000000..b0feefcd94b7 --- /dev/null +++ b/javascript/extractor/lib/typescript/test/package_test.py @@ -0,0 +1,216 @@ +import json +import os +from pathlib import Path +import stat +import subprocess +import tempfile +import unittest +import zipfile + +from python.runfiles import runfiles + + +PLATFORMS = { + "linux64": ["linux-x64"], + "linux-arm64": ["linux-arm64"], + "osx64": ["darwin-arm64", "darwin-x64"], + "win64": ["win32-x64"], +} + + +class PackageTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + files = runfiles.Create() + cls.node = files.Rlocation(os.environ["NODE_BIN"]) + cls.ripunzip = files.Rlocation(os.environ["RIPUNZIP"]) + cls.script = files.Rlocation(os.environ["PACKAGE_RUNTIME"]) + cls.wrapper_zip = files.Rlocation(os.environ["TS_WRAPPER_ZIP"]) + cls.native_zip = files.Rlocation(os.environ["TS_NATIVE_ZIP"]) + cls.common_zip = files.Rlocation(os.environ["TS_COMMON_ZIP"]) + cls.arch_zip = files.Rlocation(os.environ["TS_ARCH_ZIP"]) + with zipfile.ZipFile(cls.wrapper_zip) as archive: + cls.version = json.loads( + archive.read("typescript-parser-wrapper/package.json") + )["dependencies"]["typescript"] + + def stage(self, directory, distribution, mismatch=False): + modules = directory / "node_modules" + typescript = modules / "typescript" + typescript.mkdir(parents=True) + dependencies = {} + for platform in {p for ps in PLATFORMS.values() for p in ps}: + name = "typescript-" + platform + package = modules / "@typescript" / name + (package / "lib").mkdir(parents=True) + manifest = { + "name": "@typescript/" + name, + "version": "invalid" if mismatch else self.version, + } + (package / "package.json").write_text(json.dumps(manifest)) + executable = package / "lib" / ( + "tsc.exe" if platform.startswith("win32-") else "tsc" + ) + executable.write_text("native executable fixture") + executable.chmod(0o644) + dependencies[manifest["name"]] = self.version + (typescript / "package.json").write_text( + json.dumps({"version": self.version, "optionalDependencies": dependencies}) + ) + packages = subprocess.run( + [self.node, self.script, "--packages", distribution], + cwd=directory, + capture_output=True, + text=True, + timeout=30, + ) + if distribution in PLATFORMS: + self.assertEqual(packages.returncode, 0, packages.stderr) + self.assertEqual( + packages.stdout.split(), + [ + "@typescript/typescript-" + platform + "@" + self.version + for platform in PLATFORMS[distribution] + ], + ) + else: + self.assertNotEqual(packages.returncode, 0) + self.assertIn("Unsupported CodeQL distribution", packages.stderr) + return subprocess.run( + [self.node, self.script, distribution], + cwd=directory, + capture_output=True, + text=True, + timeout=30, + ) + + def test_distribution_selection_is_independent_of_host(self): + for distribution, platforms in PLATFORMS.items(): + with ( + self.subTest(distribution=distribution), + tempfile.TemporaryDirectory() as temp, + ): + directory = Path(temp) + result = self.stage(directory, distribution) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "") + self.assertEqual( + sorted(p.name for p in (directory / "native").iterdir()), + sorted("typescript-" + p for p in platforms), + ) + self.assertFalse((directory / "node_modules/@typescript").exists()) + if os.name != "nt": + for executable in (directory / "native").glob("*/lib/tsc"): + self.assertEqual(stat.S_IMODE(executable.stat().st_mode), 0o755) + + def test_invalid_distribution_and_mismatched_runtime_fail(self): + for distribution, mismatch, message in [ + ("unsupported", False, "Unsupported CodeQL distribution"), + ("osx64", True, "Native runtime does not match TypeScript"), + ]: + with ( + self.subTest(distribution=distribution), + tempfile.TemporaryDirectory() as temp, + ): + result = self.stage(Path(temp), distribution, mismatch) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + self.assertIn(message, result.stderr) + + def extract(self, filename, destination): + destination.mkdir(parents=True, exist_ok=True) + result = subprocess.run( + [self.ripunzip, "unzip-file", filename, "-d", str(destination)], + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_installed_archives_parse_without_external_typescript(self): + host = subprocess.check_output( + [self.node, "-p", "process.platform + '-' + process.arch"], text=True + ).strip() + platforms = next(ps for ps in PLATFORMS.values() if host in ps) + with zipfile.ZipFile(self.wrapper_zip) as archive: + self.assertFalse(any("/@typescript/" in n for n in archive.namelist())) + with zipfile.ZipFile(self.native_zip) as archive: + names = archive.namelist() + self.assertEqual( + sorted({n.removeprefix("./").split("/")[0] for n in names}), + sorted("typescript-" + p for p in platforms), + ) + for platform in platforms: + prefix = "typescript-" + platform + manifest = json.loads(archive.read(prefix + "/package.json")) + self.assertEqual(manifest["version"], self.version) + binary = prefix + "/lib/" + ( + "tsc.exe" if platform.startswith("win32-") else "tsc" + ) + self.assertGreater(archive.getinfo(binary).file_size, 0) + if not platform.startswith("win32-"): + self.assertEqual( + archive.getinfo(binary).external_attr >> 16 & 0o777, 0o755 + ) + + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + with zipfile.ZipFile(self.common_zip) as archive: + self.assertFalse(any("/@typescript/" in n for n in archive.namelist())) + self.assertIn( + "tools/typescript-parser-wrapper/main.js", archive.namelist() + ) + with zipfile.ZipFile(self.arch_zip) as archive: + self.assertEqual( + sorted(n for n in archive.namelist() if not n.endswith("/")), + sorted( + "tools/typescript-parser-wrapper/node_modules/@typescript/" + n + for n in names + if not n.endswith("/") + ), + ) + self.extract(self.common_zip, directory) + self.extract(self.arch_zip, directory) + wrapper = directory / "tools/typescript-parser-wrapper" + environment = dict(os.environ) + environment.pop("SEMMLE_TYPESCRIPT_HOME", None) + environment.pop("NODE_PATH", None) + commands = [] + for name, source in [ + ("valid.ts", "const value: number = 1;"), + ("valid.tsx", "const value =
Hello
;"), + ("invalid.ts", "const value = ;"), + ]: + filename = directory / name + filename.write_text(source) + commands.extend( + [ + {"command": "parse", "filename": str(filename)}, + {"command": "reset"}, + ] + ) + commands.append({"command": "quit"}) + result = subprocess.run( + [self.node, str(wrapper / "main.js")], + input="".join(json.dumps(command) + "\n" for command in commands), + cwd=directory, + env=environment, + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, "") + responses = [json.loads(line) for line in result.stdout.splitlines()] + self.assertEqual(len(responses), 6) + for response in responses[::2]: + self.assertEqual(response["type"], "ast") + for response in responses[1::2]: + self.assertEqual(response, {"type": "reset-done"}) + self.assertEqual(responses[0]["ast"]["parseDiagnostics"], []) + self.assertEqual(responses[2]["ast"]["parseDiagnostics"], []) + self.assertTrue(responses[4]["ast"]["parseDiagnostics"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/javascript/extractor/lib/typescript/test/wrapper.test.mjs b/javascript/extractor/lib/typescript/test/wrapper.test.mjs new file mode 100644 index 000000000000..0a68da90b2c9 --- /dev/null +++ b/javascript/extractor/lib/typescript/test/wrapper.test.mjs @@ -0,0 +1,282 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawn } from "node:child_process"; +import { copyFileSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { createInterface } from "node:readline"; +import { after, test } from "node:test"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { API, JsxEmit } from "typescript/sync"; +import { SyntaxKind, NodeFlags } from "typescript/ast"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const directory = mkdtempSync(join(tmpdir(), "codeql-typescript-wrapper-")); +const entryPoint = join(directory, "main.js"); +copyFileSync(join(root, "package.json"), join(directory, "package.json")); +symlinkSync(join(root, "node_modules"), join(directory, "node_modules"), "dir"); +for (const name of ["main.js", "ast_extractor.js", "typescript.js"]) { + copyFileSync(join(root, "build", name), join(directory, name)); +} +const { extractAst } = await import(pathToFileURL(join(directory, "ast_extractor.js"))); +const ts = await import(pathToFileURL(join(directory, "typescript.js"))); +const api = new API(); + +after(() => { + try { + api.close(); + } finally { + rmSync(directory, { recursive: true }); + } +}); + +function extract(text, extension = ".ts") { + const filename = join(directory, "input" + extension); + writeFileSync(filename, text); + const program = api.createProgram([filename], { + noResolve: true, noLib: true, jsx: JsxEmit.Preserve, experimentalDecorators: true, + }); + try { + return JSON.parse(JSON.stringify(extractAst(program.getSourceFile(filename), program.getSyntacticDiagnostics()))); + } finally { + program.dispose(); + api.clearSourceFileCache(); + } +} + +function declaration(ast, index = 0) { + return ast.statements[index].declarationList.declarations[0]; +} + +test("wire metadata uses native syntax names and flags without compatibility aliases", () => { + assert.deepEqual(ts.SyntaxKind, SyntaxKind); + assert.deepEqual(ts.NodeFlags, NodeFlags); + assert(!("EndOfFileToken" in ts.SyntaxKind)); + assert(!("GlobalAugmentation" in ts.NodeFlags)); +}); + +test("AST getters are projected without native implementation details or unrelated booleans", () => { + const text = "// comment\r\nexport const \\u0061 = \"\u{1f600}\";\r\nlet b = 2; var c = 3;"; + const ast = extract(text); + const first = declaration(ast); + assert.equal(first.name.text, "a"); + assert.equal(first.name.$pos, text.indexOf("\\u0061")); + assert.equal(first.initializer.text, "\u{1f600}"); + assert.equal(first.initializer.$end, text.indexOf('";') + 1); + assert.equal(ast.statements[0].modifiers[0].kind, SyntaxKind.ExportKeyword); + assert.deepEqual(ast.$lineStarts, [0, text.indexOf("export"), text.indexOf("let")]); + assert.deepEqual(ast.statements.map(statement => statement.declarationList.flags & NodeFlags.BlockScoped), + [NodeFlags.Const, NodeFlags.Let, NodeFlags.None]); + assert.deepEqual(ast.parseDiagnostics, []); + const json = JSON.stringify(ast); + for (const field of ["parent", "view", "sourceFile", "isTypeOnly", "isTypeOf", "isExportEquals"]) { + assert(!json.includes(`"${field}":`), field); + } +}); + +test("syntax error messages and UTF-16 offsets retain the Java-facing response", () => { + const text = 'const emoji = "\u{1f600}";\r\nconst broken = ;\r\n'; + const ast = extract(text); + assert.deepEqual(ast.parseDiagnostics, [{ + messageText: "Expression expected.", $pos: text.lastIndexOf(";"), + }]); + assert.deepEqual(ast.$lineStarts, [0, text.indexOf("const broken"), text.length]); +}); + +test("optional members, definite assignment, type defaults, and decorators use native fields", () => { + const ast = extract(` + type T = U; + interface I { p?: string; m?(): void } + @sealed export class C { + @field p!: string; + @method m(@parameter publicArg?: number) {} + } + `); + assert.deepEqual(ast.parseDiagnostics, []); + assert.equal(ast.statements[0].typeParameters[0].defaultType.kind, SyntaxKind.NumberKeyword); + const [property, method] = ast.statements[1].members; + assert.equal(property.postfixToken.kind, SyntaxKind.QuestionToken); + assert.equal(method.postfixToken.kind, SyntaxKind.QuestionToken); + const cls = ast.statements[2]; + assert.deepEqual(cls.modifiers.map(modifier => modifier.kind), [SyntaxKind.Decorator, SyntaxKind.ExportKeyword]); + assert.equal(cls.members[0].postfixToken.kind, SyntaxKind.ExclamationToken); + assert.equal(cls.members[0].modifiers[0].kind, SyntaxKind.Decorator); + assert.equal(cls.members[1].parameters[0].modifiers[0].kind, SyntaxKind.Decorator); + assert.equal(cls.members[1].parameters[0].questionToken.kind, SyntaxKind.QuestionToken); +}); + +test("global augmentations and dotted namespaces need no synthetic flags", () => { + const ast = extract(` + declare /* a */ global { interface Window {} } + namespace global {} + module global {} + namespace A.B.C {} + namespace A { namespace B {} } + declare module "external" {} + `); + assert.deepEqual(ast.parseDiagnostics, []); + const [global, namedGlobal, moduleGlobal, dotted, nested, external] = ast.statements; + assert.equal(global.name.text, "global"); + assert.equal(namedGlobal.name.text, "global"); + assert.equal(moduleGlobal.name.text, "global"); + assert.equal(dotted.body.kind, SyntaxKind.ModuleDeclaration); + assert.equal(dotted.body.body.kind, SyntaxKind.ModuleDeclaration); + assert.equal(dotted.body.flags, NodeFlags.None); + assert.equal(dotted.body.body.flags, NodeFlags.None); + assert.equal(nested.body.statements[0].flags, NodeFlags.None); + assert.equal(external.name.text, "external"); +}); + +test("import attributes and type-only phases use native property names", () => { + const ast = extract(` + import type { X } from "types"; + import { type Y, Z } from "types"; + import data from "data" with { type: "json" }; + export { data } from "data" with { type: "json" }; + type T = typeof import("data", { with: { type: "json" } }); + export type { X }; + `); + assert.deepEqual(ast.parseDiagnostics, []); + assert.equal(ast.statements[0].importClause.phaseModifier, SyntaxKind.TypeKeyword); + assert.equal(ast.statements[1].importClause.namedBindings.elements[0].isTypeOnly, true); + assert.equal(ast.statements[1].importClause.namedBindings.elements[1].isTypeOnly, false); + for (const node of [ast.statements[2], ast.statements[3], ast.statements[4].type]) { + assert.equal(node.attributes.attributes[0].name.text, "type"); + assert.equal(node.attributes.attributes[0].value.text, "json"); + } + assert.equal(ast.statements[4].type.isTypeOf, true); + assert.equal(ast.statements[5].isTypeOnly, true); +}); + +test("regex, nested templates, tagged templates, type templates and operators are rescanned", () => { + const text = 'const x = `a${/a+/gi.test(s) ? tag`b${v}c` : `d${w}e`}f`; ' + + 'type T = `a${string}b`; const y = a >= b >> c >>> d; a >>= b; a >>>= b;'; + const ast = extract(text); + assert.deepEqual(ast.parseDiagnostics, []); + const rescannedTexts = ["/a+/gi", "}c`", "}e`", "}f`", "}b`", ">=", ">>", ">>>", ">>=", ">>>="]; + const tokens = ast.$tokens.filter(token => rescannedTexts.includes(token.text)); + assert.deepEqual(tokens.map(token => token.text), ["/a+/gi", "}c`", "}e`", "}f`", "}b`", ">=", ">>", ">>>", ">>=", ">>>="]); + assert.deepEqual(tokens.map(token => token.kind), [ + SyntaxKind.SlashToken, + ...Array(4).fill(SyntaxKind.CloseBraceToken), + ...Array(5).fill(SyntaxKind.GreaterThanToken), + ]); + for (const token of tokens) { + assert.equal(text.slice(token.tokenPos, token.tokenPos + token.text.length), token.text); + } + assert.equal(ast.$tokens.at(-1).kind, SyntaxKind.EndOfFile); + assert.equal(ast.$tokens.at(-1).tokenPos, text.length); +}); + +test("interface heritage preserves native type references", () => { + const ast = extract(` + interface I extends A, N.B {} + class C extends Base implements A, N.B {} + `); + assert.deepEqual(ast.parseDiagnostics, []); + const [iface, cls] = ast.statements; + for (const heritage of [iface.heritageClauses[0], cls.heritageClauses[1]]) { + assert.equal(heritage.types[0].kind, SyntaxKind.TypeReference); + assert.equal(heritage.types[0].typeName.text, "A"); + assert.equal(heritage.types[1].typeName.right.text, "B"); + assert.equal(heritage.types[1].typeArguments[0].kind, SyntaxKind.StringKeyword); + assert(!("expression" in heritage.types[1])); + } + assert.equal(cls.heritageClauses[0].types[0].expression.text, "Base"); +}); + +test("legacy import assertions retain attributes without hiding genuine syntax errors", () => { + const source = ` + import data from "data" assert { type: "json" }; + export { data } from "data" assert { type: "json" }; + const pattern = /data/g; + `; + const ast = extract(source); + assert.deepEqual(ast.parseDiagnostics, []); + for (const node of ast.statements.slice(0, 2)) { + assert.equal(node.attributes.attributes[0].name.text, "type"); + assert.equal(node.attributes.attributes[0].value.text, "json"); + } + assert(ast.$tokens.some(token => token.kind === SyntaxKind.SlashToken && token.text === "/data/g")); + const invalid = extract(source + "const broken = ;"); + assert.equal(invalid.parseDiagnostics.length, 1); + assert.equal(invalid.parseDiagnostics[0].messageText, "Expression expected."); + assert.equal(invalid.parseDiagnostics[0].$pos, (source + "const broken = ;").lastIndexOf(";")); +}); + +test("TSX children, attributes, and positions survive projection", () => { + const ast = extract('const view = <>
{value}
;', ".tsx"); + assert.deepEqual(ast.parseDiagnostics, []); + const fragment = declaration(ast).initializer; + assert.equal(fragment.kind, SyntaxKind.JsxFragment); + assert.equal(fragment.children[0].openingElement.attributes.properties[0].name.text, "title"); + assert.equal(fragment.children[0].children[0].expression.text, "value"); + assert.equal(fragment.children[1].kind, SyntaxKind.JsxSelfClosingElement); + assert.equal(fragment.children[1].attributes.properties[0].expression.text, "props"); +}); + +test("the actual wrapper handles metadata, prepared files, out-of-order parsing, reset and quit", async () => { + const first = join(directory, "first.ts"); + const second = join(directory, "second.ts"); + writeFileSync(first, "const first = 1;"); + writeFileSync(second, "let second = 2;"); + const child = spawn(process.execPath, [entryPoint], { stdio: ["pipe", "pipe", "pipe"] }); + let stderr = ""; + const lines = createInterface({ input: child.stdout })[Symbol.asyncIterator](); + child.stderr.setEncoding("utf8").on("data", data => { stderr += data; }); + const exited = new Promise((accept, reject) => { + child.on("error", reject); + child.on("close", (code, signal) => { + try { + assert.equal(signal, null); + assert.equal(code, 0, stderr); + accept(); + } catch (error) { + reject(error); + } + }); + }); + async function request(command) { + child.stdin.write(JSON.stringify(command) + "\n"); + const line = await lines.next(); + assert.equal(line.done, false, stderr); + return JSON.parse(line.value); + } + try { + const metadata = await request({ command: "get-metadata" }); + assert.equal(metadata.type, "metadata"); + assert.deepEqual(metadata.syntaxKinds, SyntaxKind); + assert.deepEqual(metadata.nodeFlags, NodeFlags); + assert.deepEqual(await request({ command: "prepare-files", filenames: [first, second] }), { type: "ok" }); + assert.equal(declaration((await request({ command: "parse", filename: first })).ast).name.text, "first"); + assert.equal(declaration((await request({ command: "parse", filename: second })).ast).name.text, "second"); + await request({ command: "prepare-files", filenames: [first, second] }); + assert.equal(declaration((await request({ command: "parse", filename: second })).ast).name.text, "second"); + assert.deepEqual(await request({ command: "reset" }), { type: "reset-done" }); + assert.equal(declaration((await request({ command: "parse", filename: first })).ast).name.text, "first"); + child.stdin.end('{"command":"quit"}\n'); + await exited; + assert.equal(stderr, ""); + } finally { + child.stdin.destroy(); + child.kill(); + await exited; + } +}); + +test("single-file invocation flushes JSON, supports the runtime override, and reports missing files", () => { + const filename = join(directory, "command-line.ts"); + writeFileSync(filename, "const value = 1;"); + const options = { + encoding: "utf8", + env: { ...process.env, SEMMLE_TYPESCRIPT_HOME: join(root, "node_modules", "typescript") }, + }; + const response = JSON.parse(execFileSync(process.execPath, [entryPoint, filename], options)); + assert.equal(declaration(response.ast).name.text, "value"); + assert.throws(() => execFileSync(process.execPath, [entryPoint, resolve(directory, "missing.ts")], { + ...options, stdio: ["ignore", "pipe", "pipe"], + }), error => { + assert.equal(error.stdout, ""); + assert.match(error.stderr, /TypeScript did not return a source file/); + return true; + }); +}); diff --git a/javascript/extractor/lib/typescript/tsconfig.json b/javascript/extractor/lib/typescript/tsconfig.json index 81038684f405..e5581708195b 100644 --- a/javascript/extractor/lib/typescript/tsconfig.json +++ b/javascript/extractor/lib/typescript/tsconfig.json @@ -1,12 +1,13 @@ { "compilerOptions": { - "module": "commonjs", + "module": "nodenext", "noImplicitAny": true, "removeComments": true, "sourceMap": false, - "baseUrl": "./", - "lib": ["es2016"], - "target": "es5", + "lib": ["es2022", "esnext.disposable"], + "types": ["node"], + "target": "es2022", + "rootDir": "src", "outDir": "build", "allowJs": true, }, diff --git a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptASTConverter.java b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptASTConverter.java index 0c013f906b2b..898e2e1eac02 100644 --- a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptASTConverter.java +++ b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptASTConverter.java @@ -261,7 +261,7 @@ private void extractTokensAndComments( SourceLocation loc = new SourceLocation(text, start, end); String kind = getKind(token); switch (kind) { - case "EndOfFileToken": + case "EndOfFile": tokens.add(new Token(loc, Token.Type.EOF)); break; case "SingleLineCommentTrivia": @@ -927,22 +927,6 @@ private Node convertCatchClause(JsonObject node, SourceLocation loc) throws Pars return new CatchClause(loc, pattern, null, convertChild(node, "block")); } - private List convertSuperInterfaceClause(JsonArray supers) throws ParseError { - List result = new ArrayList<>(); - for (JsonElement elt : supers) { - JsonObject superType = elt.getAsJsonObject(); - ITypeExpression objectType = convertChildAsType(superType, "expression"); - if (objectType == null) continue; - List typeArguments = convertChildrenAsTypes(superType, "typeArguments"); - if (typeArguments.isEmpty()) { - result.add(objectType); - } else { - result.add(new GenericTypeExpr(getSourceLocation(superType), objectType, typeArguments)); - } - } - return result; - } - private Node convertClass(JsonObject node, String kind, SourceLocation loc) throws ParseError { Identifier id = convertChild(node, "name"); List typeParameters = convertChildrenNotNull(node, "typeParameters"); @@ -957,7 +941,7 @@ private Node convertClass(JsonObject node, String kind, SourceLocation loc) thro superClass = (Expression) convertNode(supers.get(0).getAsJsonObject()); } } else { - superInterfaces = convertSuperInterfaceClause(supers); + superInterfaces = convertChildrenAsTypes(heritageClause, "types"); } afterHead = heritageClause.get("$end").getAsInt(); } @@ -1562,7 +1546,7 @@ private Node convertInterfaceDeclaration(JsonObject node, SourceLocation loc) th for (JsonElement elt : getChildIterable(node, "heritageClauses")) { JsonObject heritageClause = elt.getAsJsonObject(); if (heritageClause.get("token").getAsInt() == syntaxKindExtends) { - superInterfaces = convertSuperInterfaceClause(heritageClause.get("types").getAsJsonArray()); + superInterfaces = convertChildrenAsTypes(heritageClause, "types"); break; } } @@ -1762,6 +1746,10 @@ private Node convertNamespaceDeclaration(JsonObject node, SourceLocation loc) th Node nameNode = convertChild(node, "name"); List body; Statement b = convertChild(node, "body"); + if (hasKind(node.get("body"), "ModuleDeclaration")) { + // The inner declarations in `namespace A.B` are implicitly exported. + b = new ExportNamedDeclaration(b.getLoc(), b, new ArrayList<>(), null, null); + } if (b instanceof BlockStatement) { body = ((BlockStatement) b).getBody(); } else { @@ -1774,11 +1762,11 @@ private Node convertNamespaceDeclaration(JsonObject node, SourceLocation loc) th attachSymbolInformation(decl, node); return decl; } - if (hasFlag(node, "GlobalAugmentation")) { + Identifier name = (Identifier) nameNode; + if (isGlobalAugmentation(node, loc, name)) { // Declaration of form: declare global {...} return new GlobalAugmentationDeclaration(loc, body); } - Identifier name = (Identifier) nameNode; boolean isInstantiated = false; for (Statement stmt : body) { isInstantiated = isInstantiated || isInstantiatingNamespaceMember(stmt); @@ -1787,13 +1775,18 @@ private Node convertNamespaceDeclaration(JsonObject node, SourceLocation loc) th NamespaceDeclaration decl = new NamespaceDeclaration(loc, name, body, isInstantiated, hasDeclareKeyword); attachSymbolInformation(decl, node); - if (hasFlag(node, "NestedNamespace")) { - // In a nested namespace declaration `namespace A.B`, the nested namespace `B` - // is implicitly exported. - return new ExportNamedDeclaration(loc, decl, new ArrayList<>(), null, null); - } else { - return fixExports(loc, decl); + return fixExports(loc, decl); + } + + private boolean isGlobalAugmentation(JsonObject node, SourceLocation loc, Identifier name) { + if (!name.getName().equals("global")) return false; + // There is no `module` or `namespace` keyword between the modifiers and `global`. + int start = loc.getStart().getOffset(); + for (JsonElement modifier : getModifiers(node)) { + start = modifier.getAsJsonObject().get("$end").getAsInt(); } + return source.substring(start, name.getLoc().getStart().getOffset()) + .matches("(?U)(?:\\uFEFF|" + WHITESPACE_CHAR + ")*"); } private boolean isInstantiatingNamespaceMember(Statement node) { @@ -2123,10 +2116,10 @@ private Node convertPropertyDeclaration(JsonObject node, String kind, SourceLoca if (kind.equals("PropertySignature")) { flags |= DeclarationFlags.abstract_; } - if (node.get("questionToken") != null) { + if (hasKind(node.get("postfixToken"), "QuestionToken")) { flags |= DeclarationFlags.optional; } - if (node.get("exclamationToken") != null) { + if (hasKind(node.get("postfixToken"), "ExclamationToken")) { flags |= DeclarationFlags.definiteAssignmentAssertion; } if (hasModifier(node, "DeclareKeyword")) { @@ -2323,7 +2316,7 @@ private Node convertTypeAssertionExpression(JsonObject node, SourceLocation loc) private Node convertImportAttributes(JsonObject node, SourceLocation loc) throws ParseError { List properties = new ArrayList<>(); - for (INode child : convertChildren(node, "elements")) { + for (INode child : convertChildren(node, "attributes")) { properties.add((Property) child); } // Adjust location to skip over the `with` or `assert` keyword. @@ -2336,7 +2329,7 @@ private Node convertImportAttributes(JsonObject node, SourceLocation loc) throws private Node convertImportAttribute(JsonObject node, SourceLocation loc) throws ParseError { return new Property( - loc, convertChild(node, "key"), convertChild(node, "value"), "init", false, false); + loc, convertChild(node, "name"), convertChild(node, "value"), "init", false, false); } private Node convertSatisfiesExpression(JsonObject node, SourceLocation loc) throws ParseError { @@ -2371,7 +2364,7 @@ private Node convertTypeParameter(JsonObject node, SourceLocation loc) throws Pa loc, convertChild(node, "name"), convertChildAsType(node, "constraint"), - convertChildAsType(node, "default")); + convertChildAsType(node, "defaultType")); } private Node convertTypePredicate(JsonObject node, SourceLocation loc) throws ParseError { @@ -2720,6 +2713,9 @@ private String getDeclarationKind(JsonObject declarationList) { if (hasFlag(declarationList, "Using")) { return "using"; } - return declarationList.get("$declarationKind").getAsString(); + if (hasFlag(declarationList, "Let")) { + return "let"; + } + return hasFlag(declarationList, "Const") ? "const" : "var"; } } diff --git a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java index 09f05b2d9e0b..946e2200b46e 100644 --- a/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java +++ b/javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java @@ -369,12 +369,7 @@ private JsonObject talkToParserWrapper(JsonObject request) { throw getExceptionFromMalformedResponse(response, e); } } catch (IOException e) { - throw new CatastrophicError( - "Could not communicate with TypeScript parser wrapper " - + "(command: ." - + parserWrapperCommand - + ").", - e); + throw getExceptionFromMalformedResponse(response, e); } } @@ -401,6 +396,14 @@ private RuntimeException getExceptionFromMalformedResponse(String response, Exce } catch (InterruptedException e1) { Exceptions.ignore(e, "This is for diagnostic purposes only."); } + if (e instanceof IOException) { + return new CatastrophicError( + "Could not communicate with TypeScript parser wrapper " + + "(command: ." + + parserWrapperCommand + + ").", + e); + } if (response == null) { return new CatastrophicError("No response from TypeScript parser wrapper", e); } diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java index 4061f1a8651b..b179eb2d1134 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java +++ b/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java @@ -2,13 +2,14 @@ import com.google.devtools.build.runfiles.Runfiles; import com.semmle.util.process.Env; -import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipFile; import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -20,6 +21,7 @@ NodeJSDetectorTests.class, ES2015DetectorTests.class, TrapTests.class, + TypeScriptTests.class, ObjectRestSpreadTests.class, ClassPropertiesTests.class, FunctionSentTests.class, @@ -36,20 +38,10 @@ public static void setUp() throws Exception { Runfiles.Preloaded runfiles = Runfiles.preload(); String nodePath = runfiles.unmapped().rlocation(System.getenv("NODE_BIN")); String tsWrapperZip = runfiles.unmapped().rlocation(System.getenv("TS_WRAPPER_ZIP")); + String tsNativeZip = runfiles.unmapped().rlocation(System.getenv("TS_NATIVE_ZIP")); Path tempDir = Files.createTempDirectory("ts-wrapper"); - // extract the ts-wrapper.zip to tempDir: - try (ZipInputStream zis = new ZipInputStream(new FileInputStream(tsWrapperZip))) { - ZipEntry entry = zis.getNextEntry(); - while (entry != null) { - Path entryPath = tempDir.resolve(entry.getName()); - if (entry.isDirectory()) { - Files.createDirectories(entryPath); - } else { - Files.copy(zis, entryPath); - } - entry = zis.getNextEntry(); - } - } + extractZip(tsWrapperZip, tempDir); + extractZip(tsNativeZip, tempDir.resolve("typescript-parser-wrapper/node_modules/@typescript")); Path tsWrapper = tempDir.resolve("typescript-parser-wrapper/main.js"); if (!Files.exists(tsWrapper)) { throw new RuntimeException("Could not find ts-wrapper at " + tsWrapper); @@ -60,4 +52,31 @@ public static void setUp() throws Exception { Env.systemEnv().pushEnvironmentContext(envUpdate); } + + private static void extractZip(String filename, Path destination) throws IOException { + // Unix modes are in the central directory, which ZipInputStream does not expose. + try (ZipFile zip = new ZipFile(Path.of(filename).toFile())) { + var entries = zip.getEntries(); + while (entries.hasMoreElements()) { + ZipArchiveEntry entry = entries.nextElement(); + Path entryPath = destination.resolve(entry.getName()).normalize(); + if (!entryPath.startsWith(destination)) { + throw new IOException("ZIP entry is outside its destination: " + entry.getName()); + } + if (entry.isDirectory()) { + Files.createDirectories(entryPath); + } else { + Files.createDirectories(entryPath.getParent()); + try (InputStream input = zip.getInputStream(entry)) { + Files.copy(input, entryPath); + } + if ((entry.getUnixMode() & 0111) != 0 + && entryPath.getFileSystem().supportedFileAttributeViews().contains("posix") + && !entryPath.toFile().setExecutable(true, false)) { + throw new IOException("Could not make ZIP entry executable: " + entryPath); + } + } + } + } + } } diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel b/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel index 2d6a5d01c660..faecec7cb010 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel +++ b/javascript/extractor/test/com/semmle/js/extractor/test/BUILD.bazel @@ -5,15 +5,18 @@ java_test( srcs = glob(["**/*.java"]), data = [ "//javascript/extractor/lib/typescript", + "//javascript/extractor/lib/typescript:native", "//javascript/extractor/parser-tests", "//javascript/extractor/tests", "@nodejs//:node_bin", ], env = { "NODE_BIN": "$(rlocationpath @nodejs//:node_bin)", + "TS_NATIVE_ZIP": "$(rlocationpath //javascript/extractor/lib/typescript:native)", "TS_WRAPPER_ZIP": "$(rlocationpath //javascript/extractor/lib/typescript)", }, test_class = "com.semmle.js.extractor.test.AllTests", + runtime_deps = ["@semmle_code//resources/lib/java:commons-io"], # To use `replaceExpectedOutput` you need to uncomment the following line # (to be allowed to override the .trap files on disk) # tags = ["no-sandbox"], diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/TypeScriptTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/TypeScriptTests.java new file mode 100644 index 000000000000..1bc36f2fcc61 --- /dev/null +++ b/javascript/extractor/test/com/semmle/js/extractor/test/TypeScriptTests.java @@ -0,0 +1,99 @@ +package com.semmle.js.extractor.test; + +import com.semmle.js.ast.ClassDeclaration; +import com.semmle.js.ast.DeclarationFlags; +import com.semmle.js.ast.ExportNamedDeclaration; +import com.semmle.js.ast.Program; +import com.semmle.js.ast.Token; +import com.semmle.js.ast.VariableDeclaration; +import com.semmle.js.extractor.ExtractionMetrics; +import com.semmle.js.parser.JSParser.Result; +import com.semmle.ts.ast.ExternalModuleDeclaration; +import com.semmle.ts.ast.GlobalAugmentationDeclaration; +import com.semmle.ts.ast.InterfaceDeclaration; +import com.semmle.ts.ast.NamespaceDeclaration; +import com.semmle.ts.extractor.TypeScriptParser; +import java.io.File; +import java.nio.file.Files; +import org.junit.After; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class TypeScriptTests { + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + private final TypeScriptParser parser = new TypeScriptParser(); + + @After + public void tearDown() { + parser.killProcess(); + } + + private Program parse(String source) throws Exception { + File file = folder.newFile("input.ts"); + Files.writeString(file.toPath(), source); + Result result = parser.parse(file, source, new ExtractionMetrics()); + Assert.assertTrue(result.getErrors().toString(), result.getErrors().isEmpty()); + Assert.assertEquals(Token.Type.EOF, result.getTokens().get(result.getTokens().size() - 1).getType()); + return (Program) result.getAST(); + } + + @Test + public void namespaceClassifications() throws Exception { + Program program = + parse( + "declare /* a */ global { interface Window {} }\n" + + "declare\u00a0/* b */\ufeffglobal {}\n" + + "namespace global {}\n" + + "module global {}\n" + + "namespace A.B.C {}\n" + + "namespace X { namespace Y {} }\n" + + "declare module \"external\" {}"); + Assert.assertTrue(program.getBody().get(0) instanceof GlobalAugmentationDeclaration); + Assert.assertTrue(program.getBody().get(1) instanceof GlobalAugmentationDeclaration); + Assert.assertTrue(program.getBody().get(2) instanceof NamespaceDeclaration); + Assert.assertTrue(program.getBody().get(3) instanceof NamespaceDeclaration); + NamespaceDeclaration a = (NamespaceDeclaration) program.getBody().get(4); + ExportNamedDeclaration exportedB = (ExportNamedDeclaration) a.getBody().get(0); + NamespaceDeclaration b = (NamespaceDeclaration) exportedB.getDeclaration(); + ExportNamedDeclaration exportedC = (ExportNamedDeclaration) b.getBody().get(0); + Assert.assertEquals("B", b.getName().getName()); + Assert.assertEquals( + "C", ((NamespaceDeclaration) exportedC.getDeclaration()).getName().getName()); + NamespaceDeclaration x = (NamespaceDeclaration) program.getBody().get(5); + Assert.assertTrue(x.getBody().get(0) instanceof NamespaceDeclaration); + Assert.assertTrue(program.getBody().get(6) instanceof ExternalModuleDeclaration); + } + + @Test + public void memberPostfixTokensAndTypeDefaults() throws Exception { + Program program = + parse( + "interface I { p?: string; m?(): void; required: T }\n" + + "class C { p!: string; optional?: number; required: number }"); + InterfaceDeclaration iface = (InterfaceDeclaration) program.getBody().get(0); + Assert.assertEquals("number", iface.getTypeParameters().get(0).getDefault().getLoc().getSource()); + Assert.assertTrue(DeclarationFlags.isOptional(iface.getBody().get(0).getFlags())); + Assert.assertTrue(DeclarationFlags.isAbstract(iface.getBody().get(1).getFlags())); + Assert.assertFalse(DeclarationFlags.isOptional(iface.getBody().get(2).getFlags())); + ClassDeclaration cls = (ClassDeclaration) program.getBody().get(1); + var members = cls.getClassDef().getBody().getBody(); + Assert.assertTrue(DeclarationFlags.hasDefiniteAssignmentAssertion(members.get(0).getFlags())); + Assert.assertFalse(DeclarationFlags.isOptional(members.get(0).getFlags())); + Assert.assertTrue(DeclarationFlags.isOptional(members.get(1).getFlags())); + Assert.assertFalse(DeclarationFlags.hasDefiniteAssignmentAssertion(members.get(1).getFlags())); + Assert.assertFalse(DeclarationFlags.isOptional(members.get(2).getFlags())); + } + + @Test + public void variableDeclarationKinds() throws Exception { + Program program = + parse("var a; let b; const c = 1; using d = resource(); await using e = resource();"); + String[] kinds = {"var", "let", "const", "using", "using"}; + for (int i = 0; i < kinds.length; ++i) { + Assert.assertEquals(kinds[i], ((VariableDeclaration) program.getBody().get(i)).getKind()); + } + } +} diff --git a/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap b/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap index 193a2afeb185..91161d974584 100644 --- a/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap +++ b/javascript/extractor/tests/ts/output/trap/import-attributes.ts.trap @@ -1311,837 +1311,1089 @@ properties(#20503,#20501,0,0,"type: ""json""") locations_default(#20504,#10000,1,24,1,35) hasLocation(#20503,#20504) #20505=* -stmts(#20505,27,#20001,1,"import ... son"" };") -hasLocation(#20505,#20009) -stmt_containers(#20505,#20001) +exprs(#20505,0,#20503,0,"type") +hasLocation(#20505,#20063) +enclosing_stmt(#20505,#20497) +expr_containers(#20505,#20001) +literals("type","type",#20505) #20506=* -exprs(#20506,4,#20505,-1,"""module""") -hasLocation(#20506,#20083) -enclosing_stmt(#20506,#20505) +exprs(#20506,4,#20503,1,"""json""") +hasLocation(#20506,#20067) +enclosing_stmt(#20506,#20497) expr_containers(#20506,#20001) -literals("module","""module""",#20506) +literals("json","""json""",#20506) #20507=* -regexpterm(#20507,14,#20506,0,"module") -#20508=@"loc,{#10000},2,22,2,27" -locations_default(#20508,#10000,2,22,2,27) +regexpterm(#20507,14,#20506,0,"json") +#20508=@"loc,{#10000},1,31,1,34" +locations_default(#20508,#10000,1,31,1,34) hasLocation(#20507,#20508) -regexp_const_value(#20507,"module") +regexp_const_value(#20507,"json") #20509=* -exprs(#20509,8,#20505,-10,"{ type: ""json"" }") -#20510=@"loc,{#10000},2,35,2,50" -locations_default(#20510,#10000,2,35,2,50) -hasLocation(#20509,#20510) -enclosing_stmt(#20509,#20505) -expr_containers(#20509,#20001) +stmts(#20509,27,#20001,1,"import ... son"" };") +hasLocation(#20509,#20009) +stmt_containers(#20509,#20001) +#20510=* +exprs(#20510,4,#20509,-1,"""module""") +hasLocation(#20510,#20083) +enclosing_stmt(#20510,#20509) +expr_containers(#20510,#20001) +literals("module","""module""",#20510) #20511=* -properties(#20511,#20509,0,0,"type: ""json""") -#20512=@"loc,{#10000},2,37,2,48" -locations_default(#20512,#10000,2,37,2,48) +regexpterm(#20511,14,#20510,0,"module") +#20512=@"loc,{#10000},2,22,2,27" +locations_default(#20512,#10000,2,22,2,27) hasLocation(#20511,#20512) +regexp_const_value(#20511,"module") #20513=* -exprs(#20513,85,#20505,0,"* as v1") -#20514=@"loc,{#10000},2,8,2,14" -locations_default(#20514,#10000,2,8,2,14) +exprs(#20513,8,#20509,-10,"{ type: ""json"" }") +#20514=@"loc,{#10000},2,35,2,50" +locations_default(#20514,#10000,2,35,2,50) hasLocation(#20513,#20514) -enclosing_stmt(#20513,#20505) +enclosing_stmt(#20513,#20509) expr_containers(#20513,#20001) #20515=* -exprs(#20515,78,#20513,1,"v1") -hasLocation(#20515,#20079) -enclosing_stmt(#20515,#20505) -expr_containers(#20515,#20001) -literals("v1","v1",#20515) -decl(#20515,#20486) -typedecl(#20515,#20489) -namespacedecl(#20515,#20492) -#20516=* -stmts(#20516,27,#20001,2,"import ... son"" };") -hasLocation(#20516,#20011) -stmt_containers(#20516,#20001) +properties(#20515,#20513,0,0,"type: ""json""") +#20516=@"loc,{#10000},2,37,2,48" +locations_default(#20516,#10000,2,37,2,48) +hasLocation(#20515,#20516) #20517=* -exprs(#20517,4,#20516,-1,"""module""") -hasLocation(#20517,#20109) -enclosing_stmt(#20517,#20516) +exprs(#20517,0,#20515,0,"type") +hasLocation(#20517,#20089) +enclosing_stmt(#20517,#20509) expr_containers(#20517,#20001) -literals("module","""module""",#20517) +literals("type","type",#20517) #20518=* -regexpterm(#20518,14,#20517,0,"module") -#20519=@"loc,{#10000},3,21,3,26" -locations_default(#20519,#10000,3,21,3,26) -hasLocation(#20518,#20519) -regexp_const_value(#20518,"module") -#20520=* -exprs(#20520,8,#20516,-10,"{ type: ""json"" }") -#20521=@"loc,{#10000},3,34,3,49" -locations_default(#20521,#10000,3,34,3,49) -hasLocation(#20520,#20521) -enclosing_stmt(#20520,#20516) -expr_containers(#20520,#20001) -#20522=* -properties(#20522,#20520,0,0,"type: ""json""") -#20523=@"loc,{#10000},3,36,3,47" -locations_default(#20523,#10000,3,36,3,47) -hasLocation(#20522,#20523) +exprs(#20518,4,#20515,1,"""json""") +hasLocation(#20518,#20093) +enclosing_stmt(#20518,#20509) +expr_containers(#20518,#20001) +literals("json","""json""",#20518) +#20519=* +regexpterm(#20519,14,#20518,0,"json") +#20520=@"loc,{#10000},2,44,2,47" +locations_default(#20520,#10000,2,44,2,47) +hasLocation(#20519,#20520) +regexp_const_value(#20519,"json") +#20521=* +exprs(#20521,85,#20509,0,"* as v1") +#20522=@"loc,{#10000},2,8,2,14" +locations_default(#20522,#10000,2,8,2,14) +hasLocation(#20521,#20522) +enclosing_stmt(#20521,#20509) +expr_containers(#20521,#20001) +#20523=* +exprs(#20523,78,#20521,1,"v1") +hasLocation(#20523,#20079) +enclosing_stmt(#20523,#20509) +expr_containers(#20523,#20001) +literals("v1","v1",#20523) +decl(#20523,#20486) +typedecl(#20523,#20489) +namespacedecl(#20523,#20492) #20524=* -exprs(#20524,83,#20516,0,"v2") -hasLocation(#20524,#20103) -enclosing_stmt(#20524,#20516) -expr_containers(#20524,#20001) +stmts(#20524,27,#20001,2,"import ... son"" };") +hasLocation(#20524,#20011) +stmt_containers(#20524,#20001) #20525=* -exprs(#20525,0,#20524,0,"v2") -hasLocation(#20525,#20103) -enclosing_stmt(#20525,#20516) +exprs(#20525,4,#20524,-1,"""module""") +hasLocation(#20525,#20109) +enclosing_stmt(#20525,#20524) expr_containers(#20525,#20001) -literals("v2","v2",#20525) +literals("module","""module""",#20525) #20526=* -exprs(#20526,78,#20524,1,"v2") -hasLocation(#20526,#20103) -enclosing_stmt(#20526,#20516) -expr_containers(#20526,#20001) -literals("v2","v2",#20526) -decl(#20526,#20487) -typedecl(#20526,#20490) -namespacedecl(#20526,#20493) -#20527=* -stmts(#20527,27,#20001,3,"import ... son"" };") -hasLocation(#20527,#20013) -stmt_containers(#20527,#20001) +regexpterm(#20526,14,#20525,0,"module") +#20527=@"loc,{#10000},3,21,3,26" +locations_default(#20527,#10000,3,21,3,26) +hasLocation(#20526,#20527) +regexp_const_value(#20526,"module") #20528=* -exprs(#20528,4,#20527,-1,"""module""") -hasLocation(#20528,#20131) -enclosing_stmt(#20528,#20527) +exprs(#20528,8,#20524,-10,"{ type: ""json"" }") +#20529=@"loc,{#10000},3,34,3,49" +locations_default(#20529,#10000,3,34,3,49) +hasLocation(#20528,#20529) +enclosing_stmt(#20528,#20524) expr_containers(#20528,#20001) -literals("module","""module""",#20528) -#20529=* -regexpterm(#20529,14,#20528,0,"module") -#20530=@"loc,{#10000},4,17,4,22" -locations_default(#20530,#10000,4,17,4,22) -hasLocation(#20529,#20530) -regexp_const_value(#20529,"module") -#20531=* -exprs(#20531,8,#20527,-10,"{ type: ""json"" }") -#20532=@"loc,{#10000},4,30,4,45" -locations_default(#20532,#10000,4,30,4,45) -hasLocation(#20531,#20532) -enclosing_stmt(#20531,#20527) -expr_containers(#20531,#20001) +#20530=* +properties(#20530,#20528,0,0,"type: ""json""") +#20531=@"loc,{#10000},3,36,3,47" +locations_default(#20531,#10000,3,36,3,47) +hasLocation(#20530,#20531) +#20532=* +exprs(#20532,0,#20530,0,"type") +hasLocation(#20532,#20115) +enclosing_stmt(#20532,#20524) +expr_containers(#20532,#20001) +literals("type","type",#20532) #20533=* -properties(#20533,#20531,0,0,"type: ""json""") -#20534=@"loc,{#10000},4,32,4,43" -locations_default(#20534,#10000,4,32,4,43) -hasLocation(#20533,#20534) -#20535=* -exprs(#20535,84,#20527,0,"v3") -hasLocation(#20535,#20127) -enclosing_stmt(#20535,#20527) -expr_containers(#20535,#20001) +exprs(#20533,4,#20530,1,"""json""") +hasLocation(#20533,#20119) +enclosing_stmt(#20533,#20524) +expr_containers(#20533,#20001) +literals("json","""json""",#20533) +#20534=* +regexpterm(#20534,14,#20533,0,"json") +#20535=@"loc,{#10000},3,43,3,46" +locations_default(#20535,#10000,3,43,3,46) +hasLocation(#20534,#20535) +regexp_const_value(#20534,"json") #20536=* -exprs(#20536,78,#20535,1,"v3") -hasLocation(#20536,#20127) -enclosing_stmt(#20536,#20527) +exprs(#20536,83,#20524,0,"v2") +hasLocation(#20536,#20103) +enclosing_stmt(#20536,#20524) expr_containers(#20536,#20001) -literals("v3","v3",#20536) -decl(#20536,#20488) -typedecl(#20536,#20491) -namespacedecl(#20536,#20494) #20537=* -stmts(#20537,30,#20001,4,"export ... son"" };") -hasLocation(#20537,#20017) -stmt_containers(#20537,#20001) +exprs(#20537,0,#20536,0,"v2") +hasLocation(#20537,#20103) +enclosing_stmt(#20537,#20524) +expr_containers(#20537,#20001) +literals("v2","v2",#20537) #20538=* -exprs(#20538,4,#20537,-2,"""module""") -hasLocation(#20538,#20157) -enclosing_stmt(#20538,#20537) +exprs(#20538,78,#20536,1,"v2") +hasLocation(#20538,#20103) +enclosing_stmt(#20538,#20524) expr_containers(#20538,#20001) -literals("module","""module""",#20538) +literals("v2","v2",#20538) +decl(#20538,#20487) +typedecl(#20538,#20490) +namespacedecl(#20538,#20493) #20539=* -regexpterm(#20539,14,#20538,0,"module") -#20540=@"loc,{#10000},6,21,6,26" -locations_default(#20540,#10000,6,21,6,26) -hasLocation(#20539,#20540) -regexp_const_value(#20539,"module") +stmts(#20539,27,#20001,3,"import ... son"" };") +hasLocation(#20539,#20013) +stmt_containers(#20539,#20001) +#20540=* +exprs(#20540,4,#20539,-1,"""module""") +hasLocation(#20540,#20131) +enclosing_stmt(#20540,#20539) +expr_containers(#20540,#20001) +literals("module","""module""",#20540) #20541=* -exprs(#20541,8,#20537,-10,"{ type: ""json"" }") -#20542=@"loc,{#10000},6,34,6,49" -locations_default(#20542,#10000,6,34,6,49) +regexpterm(#20541,14,#20540,0,"module") +#20542=@"loc,{#10000},4,17,4,22" +locations_default(#20542,#10000,4,17,4,22) hasLocation(#20541,#20542) -enclosing_stmt(#20541,#20537) -expr_containers(#20541,#20001) +regexp_const_value(#20541,"module") #20543=* -properties(#20543,#20541,0,0,"type: ""json""") -#20544=@"loc,{#10000},6,36,6,47" -locations_default(#20544,#10000,6,36,6,47) +exprs(#20543,8,#20539,-10,"{ type: ""json"" }") +#20544=@"loc,{#10000},4,30,4,45" +locations_default(#20544,#10000,4,30,4,45) hasLocation(#20543,#20544) +enclosing_stmt(#20543,#20539) +expr_containers(#20543,#20001) #20545=* -exprs(#20545,86,#20537,0,"v4") -hasLocation(#20545,#20151) -enclosing_stmt(#20545,#20537) -expr_containers(#20545,#20001) -#20546=* -exprs(#20546,0,#20545,0,"v4") -hasLocation(#20546,#20151) -enclosing_stmt(#20546,#20537) -expr_containers(#20546,#20001) -literals("v4","v4",#20546) +properties(#20545,#20543,0,0,"type: ""json""") +#20546=@"loc,{#10000},4,32,4,43" +locations_default(#20546,#10000,4,32,4,43) +hasLocation(#20545,#20546) #20547=* -exprs(#20547,0,#20545,1,"v4") -hasLocation(#20547,#20151) -enclosing_stmt(#20547,#20537) +exprs(#20547,0,#20545,0,"type") +hasLocation(#20547,#20137) +enclosing_stmt(#20547,#20539) expr_containers(#20547,#20001) -literals("v4","v4",#20547) +literals("type","type",#20547) #20548=* -stmts(#20548,28,#20001,5,"export ... son"" };") -hasLocation(#20548,#20019) -stmt_containers(#20548,#20001) +exprs(#20548,4,#20545,1,"""json""") +hasLocation(#20548,#20141) +enclosing_stmt(#20548,#20539) +expr_containers(#20548,#20001) +literals("json","""json""",#20548) #20549=* -exprs(#20549,4,#20548,0,"""module""") -hasLocation(#20549,#20179) -enclosing_stmt(#20549,#20548) -expr_containers(#20549,#20001) -literals("module","""module""",#20549) -#20550=* -regexpterm(#20550,14,#20549,0,"module") -#20551=@"loc,{#10000},7,16,7,21" -locations_default(#20551,#10000,7,16,7,21) -hasLocation(#20550,#20551) -regexp_const_value(#20550,"module") +regexpterm(#20549,14,#20548,0,"json") +#20550=@"loc,{#10000},4,39,4,42" +locations_default(#20550,#10000,4,39,4,42) +hasLocation(#20549,#20550) +regexp_const_value(#20549,"json") +#20551=* +exprs(#20551,84,#20539,0,"v3") +hasLocation(#20551,#20127) +enclosing_stmt(#20551,#20539) +expr_containers(#20551,#20001) #20552=* -exprs(#20552,8,#20548,-10,"{ type: ""json"" }") -#20553=@"loc,{#10000},7,29,7,44" -locations_default(#20553,#10000,7,29,7,44) -hasLocation(#20552,#20553) -enclosing_stmt(#20552,#20548) +exprs(#20552,78,#20551,1,"v3") +hasLocation(#20552,#20127) +enclosing_stmt(#20552,#20539) expr_containers(#20552,#20001) +literals("v3","v3",#20552) +decl(#20552,#20488) +typedecl(#20552,#20491) +namespacedecl(#20552,#20494) +#20553=* +stmts(#20553,30,#20001,4,"export ... son"" };") +hasLocation(#20553,#20017) +stmt_containers(#20553,#20001) #20554=* -properties(#20554,#20552,0,0,"type: ""json""") -#20555=@"loc,{#10000},7,31,7,42" -locations_default(#20555,#10000,7,31,7,42) -hasLocation(#20554,#20555) -#20556=* -stmts(#20556,30,#20001,6,"export ... son"" };") -hasLocation(#20556,#20021) -stmt_containers(#20556,#20001) +exprs(#20554,4,#20553,-2,"""module""") +hasLocation(#20554,#20157) +enclosing_stmt(#20554,#20553) +expr_containers(#20554,#20001) +literals("module","""module""",#20554) +#20555=* +regexpterm(#20555,14,#20554,0,"module") +#20556=@"loc,{#10000},6,21,6,26" +locations_default(#20556,#10000,6,21,6,26) +hasLocation(#20555,#20556) +regexp_const_value(#20555,"module") #20557=* -exprs(#20557,4,#20556,-2,"""module""") -hasLocation(#20557,#20205) -enclosing_stmt(#20557,#20556) +exprs(#20557,8,#20553,-10,"{ type: ""json"" }") +#20558=@"loc,{#10000},6,34,6,49" +locations_default(#20558,#10000,6,34,6,49) +hasLocation(#20557,#20558) +enclosing_stmt(#20557,#20553) expr_containers(#20557,#20001) -literals("module","""module""",#20557) -#20558=* -regexpterm(#20558,14,#20557,0,"module") -#20559=@"loc,{#10000},8,22,8,27" -locations_default(#20559,#10000,8,22,8,27) -hasLocation(#20558,#20559) -regexp_const_value(#20558,"module") -#20560=* -exprs(#20560,8,#20556,-10,"{ type: ""json"" }") -#20561=@"loc,{#10000},8,35,8,50" -locations_default(#20561,#10000,8,35,8,50) -hasLocation(#20560,#20561) -enclosing_stmt(#20560,#20556) -expr_containers(#20560,#20001) +#20559=* +properties(#20559,#20557,0,0,"type: ""json""") +#20560=@"loc,{#10000},6,36,6,47" +locations_default(#20560,#10000,6,36,6,47) +hasLocation(#20559,#20560) +#20561=* +exprs(#20561,0,#20559,0,"type") +hasLocation(#20561,#20163) +enclosing_stmt(#20561,#20553) +expr_containers(#20561,#20001) +literals("type","type",#20561) #20562=* -properties(#20562,#20560,0,0,"type: ""json""") -#20563=@"loc,{#10000},8,37,8,48" -locations_default(#20563,#10000,8,37,8,48) -hasLocation(#20562,#20563) -#20564=* -exprs(#20564,96,#20556,0,"* as v5") -#20565=@"loc,{#10000},8,8,8,14" -locations_default(#20565,#10000,8,8,8,14) -hasLocation(#20564,#20565) -enclosing_stmt(#20564,#20556) -expr_containers(#20564,#20001) +exprs(#20562,4,#20559,1,"""json""") +hasLocation(#20562,#20167) +enclosing_stmt(#20562,#20553) +expr_containers(#20562,#20001) +literals("json","""json""",#20562) +#20563=* +regexpterm(#20563,14,#20562,0,"json") +#20564=@"loc,{#10000},6,43,6,46" +locations_default(#20564,#10000,6,43,6,46) +hasLocation(#20563,#20564) +regexp_const_value(#20563,"json") +#20565=* +exprs(#20565,86,#20553,0,"v4") +hasLocation(#20565,#20151) +enclosing_stmt(#20565,#20553) +expr_containers(#20565,#20001) #20566=* -exprs(#20566,0,#20564,1,"v5") -hasLocation(#20566,#20201) -enclosing_stmt(#20566,#20556) +exprs(#20566,0,#20565,0,"v4") +hasLocation(#20566,#20151) +enclosing_stmt(#20566,#20553) expr_containers(#20566,#20001) -literals("v5","v5",#20566) +literals("v4","v4",#20566) #20567=* -stmts(#20567,22,#20001,7,"const v ... "" } });") -hasLocation(#20567,#20025) -stmt_containers(#20567,#20001) +exprs(#20567,0,#20565,1,"v4") +hasLocation(#20567,#20151) +enclosing_stmt(#20567,#20553) +expr_containers(#20567,#20001) +literals("v4","v4",#20567) #20568=* -exprs(#20568,64,#20567,0,"v6 = im ... n"" } })") -#20569=@"loc,{#10000},10,7,10,57" -locations_default(#20569,#10000,10,7,10,57) -hasLocation(#20568,#20569) -enclosing_stmt(#20568,#20567) -expr_containers(#20568,#20001) +stmts(#20568,28,#20001,5,"export ... son"" };") +hasLocation(#20568,#20019) +stmt_containers(#20568,#20001) +#20569=* +exprs(#20569,4,#20568,0,"""module""") +hasLocation(#20569,#20179) +enclosing_stmt(#20569,#20568) +expr_containers(#20569,#20001) +literals("module","""module""",#20569) #20570=* -exprs(#20570,78,#20568,0,"v6") -hasLocation(#20570,#20223) -enclosing_stmt(#20570,#20567) -expr_containers(#20570,#20001) -literals("v6","v6",#20570) -decl(#20570,#20495) -#20571=* -exprs(#20571,99,#20568,1,"import( ... n"" } })") -#20572=@"loc,{#10000},10,12,10,57" -locations_default(#20572,#10000,10,12,10,57) -hasLocation(#20571,#20572) -enclosing_stmt(#20571,#20567) -expr_containers(#20571,#20001) -#20573=* -exprs(#20573,4,#20571,0,"""module""") -hasLocation(#20573,#20231) -enclosing_stmt(#20573,#20567) -expr_containers(#20573,#20001) -literals("module","""module""",#20573) +regexpterm(#20570,14,#20569,0,"module") +#20571=@"loc,{#10000},7,16,7,21" +locations_default(#20571,#10000,7,16,7,21) +hasLocation(#20570,#20571) +regexp_const_value(#20570,"module") +#20572=* +exprs(#20572,8,#20568,-10,"{ type: ""json"" }") +#20573=@"loc,{#10000},7,29,7,44" +locations_default(#20573,#10000,7,29,7,44) +hasLocation(#20572,#20573) +enclosing_stmt(#20572,#20568) +expr_containers(#20572,#20001) #20574=* -regexpterm(#20574,14,#20573,0,"module") -#20575=@"loc,{#10000},10,20,10,25" -locations_default(#20575,#10000,10,20,10,25) +properties(#20574,#20572,0,0,"type: ""json""") +#20575=@"loc,{#10000},7,31,7,42" +locations_default(#20575,#10000,7,31,7,42) hasLocation(#20574,#20575) -regexp_const_value(#20574,"module") #20576=* -exprs(#20576,8,#20571,1,"{ ""with ... on"" } }") -#20577=@"loc,{#10000},10,29,10,56" -locations_default(#20577,#10000,10,29,10,56) -hasLocation(#20576,#20577) -enclosing_stmt(#20576,#20567) +exprs(#20576,0,#20574,0,"type") +hasLocation(#20576,#20185) +enclosing_stmt(#20576,#20568) expr_containers(#20576,#20001) +literals("type","type",#20576) +#20577=* +exprs(#20577,4,#20574,1,"""json""") +hasLocation(#20577,#20189) +enclosing_stmt(#20577,#20568) +expr_containers(#20577,#20001) +literals("json","""json""",#20577) #20578=* -properties(#20578,#20576,0,0,"""with"": ... json"" }") -#20579=@"loc,{#10000},10,31,10,54" -locations_default(#20579,#10000,10,31,10,54) +regexpterm(#20578,14,#20577,0,"json") +#20579=@"loc,{#10000},7,38,7,41" +locations_default(#20579,#10000,7,38,7,41) hasLocation(#20578,#20579) +regexp_const_value(#20578,"json") #20580=* -exprs(#20580,4,#20578,0,"""with""") -hasLocation(#20580,#20237) -enclosing_stmt(#20580,#20567) -expr_containers(#20580,#20001) -literals("with","""with""",#20580) +stmts(#20580,30,#20001,6,"export ... son"" };") +hasLocation(#20580,#20021) +stmt_containers(#20580,#20001) #20581=* -regexpterm(#20581,14,#20580,0,"with") -#20582=@"loc,{#10000},10,32,10,35" -locations_default(#20582,#10000,10,32,10,35) -hasLocation(#20581,#20582) -regexp_const_value(#20581,"with") -#20583=* -exprs(#20583,8,#20578,1,"{ type: ""json"" }") -#20584=@"loc,{#10000},10,39,10,54" -locations_default(#20584,#10000,10,39,10,54) -hasLocation(#20583,#20584) -enclosing_stmt(#20583,#20567) -expr_containers(#20583,#20001) -#20585=* -properties(#20585,#20583,0,0,"type: ""json""") -#20586=@"loc,{#10000},10,41,10,52" -locations_default(#20586,#10000,10,41,10,52) -hasLocation(#20585,#20586) -#20587=* -exprs(#20587,0,#20585,0,"type") -hasLocation(#20587,#20243) -enclosing_stmt(#20587,#20567) -expr_containers(#20587,#20001) -literals("type","type",#20587) +exprs(#20581,4,#20580,-2,"""module""") +hasLocation(#20581,#20205) +enclosing_stmt(#20581,#20580) +expr_containers(#20581,#20001) +literals("module","""module""",#20581) +#20582=* +regexpterm(#20582,14,#20581,0,"module") +#20583=@"loc,{#10000},8,22,8,27" +locations_default(#20583,#10000,8,22,8,27) +hasLocation(#20582,#20583) +regexp_const_value(#20582,"module") +#20584=* +exprs(#20584,8,#20580,-10,"{ type: ""json"" }") +#20585=@"loc,{#10000},8,35,8,50" +locations_default(#20585,#10000,8,35,8,50) +hasLocation(#20584,#20585) +enclosing_stmt(#20584,#20580) +expr_containers(#20584,#20001) +#20586=* +properties(#20586,#20584,0,0,"type: ""json""") +#20587=@"loc,{#10000},8,37,8,48" +locations_default(#20587,#10000,8,37,8,48) +hasLocation(#20586,#20587) #20588=* -exprs(#20588,4,#20585,1,"""json""") -hasLocation(#20588,#20247) -enclosing_stmt(#20588,#20567) +exprs(#20588,0,#20586,0,"type") +hasLocation(#20588,#20211) +enclosing_stmt(#20588,#20580) expr_containers(#20588,#20001) -literals("json","""json""",#20588) +literals("type","type",#20588) #20589=* -regexpterm(#20589,14,#20588,0,"json") -#20590=@"loc,{#10000},10,48,10,51" -locations_default(#20590,#10000,10,48,10,51) -hasLocation(#20589,#20590) -regexp_const_value(#20589,"json") -#20591=* -stmts(#20591,27,#20001,8,"import ""module"";") -#20592=@"loc,{#10000},12,1,12,16" -locations_default(#20592,#10000,12,1,12,16) -hasLocation(#20591,#20592) -stmt_containers(#20591,#20001) -#20593=* -exprs(#20593,4,#20591,-1,"""module""") -hasLocation(#20593,#20259) -enclosing_stmt(#20593,#20591) -expr_containers(#20593,#20001) -literals("module","""module""",#20593) +exprs(#20589,4,#20586,1,"""json""") +hasLocation(#20589,#20215) +enclosing_stmt(#20589,#20580) +expr_containers(#20589,#20001) +literals("json","""json""",#20589) +#20590=* +regexpterm(#20590,14,#20589,0,"json") +#20591=@"loc,{#10000},8,44,8,47" +locations_default(#20591,#10000,8,44,8,47) +hasLocation(#20590,#20591) +regexp_const_value(#20590,"json") +#20592=* +exprs(#20592,96,#20580,0,"* as v5") +#20593=@"loc,{#10000},8,8,8,14" +locations_default(#20593,#10000,8,8,8,14) +hasLocation(#20592,#20593) +enclosing_stmt(#20592,#20580) +expr_containers(#20592,#20001) #20594=* -regexpterm(#20594,14,#20593,0,"module") -#20595=@"loc,{#10000},12,9,12,14" -locations_default(#20595,#10000,12,9,12,14) -hasLocation(#20594,#20595) -regexp_const_value(#20594,"module") +exprs(#20594,0,#20592,1,"v5") +hasLocation(#20594,#20201) +enclosing_stmt(#20594,#20580) +expr_containers(#20594,#20001) +literals("v5","v5",#20594) +#20595=* +stmts(#20595,22,#20001,7,"const v ... "" } });") +hasLocation(#20595,#20025) +stmt_containers(#20595,#20001) #20596=* -stmts(#20596,2,#20001,9,"assert( ... on"" });") -#20597=@"loc,{#10000},13,1,13,25" -locations_default(#20597,#10000,13,1,13,25) +exprs(#20596,64,#20595,0,"v6 = im ... n"" } })") +#20597=@"loc,{#10000},10,7,10,57" +locations_default(#20597,#10000,10,7,10,57) hasLocation(#20596,#20597) -stmt_containers(#20596,#20001) +enclosing_stmt(#20596,#20595) +expr_containers(#20596,#20001) #20598=* -exprs(#20598,13,#20596,0,"assert( ... son"" })") -#20599=@"loc,{#10000},13,1,13,24" -locations_default(#20599,#10000,13,1,13,24) -hasLocation(#20598,#20599) -enclosing_stmt(#20598,#20596) +exprs(#20598,78,#20596,0,"v6") +hasLocation(#20598,#20223) +enclosing_stmt(#20598,#20595) expr_containers(#20598,#20001) -#20600=* -exprs(#20600,79,#20598,-1,"assert") -hasLocation(#20600,#20263) -enclosing_stmt(#20600,#20596) -expr_containers(#20600,#20001) -literals("assert","assert",#20600) -#20601=@"var;{assert};{#20000}" -variables(#20601,"assert",#20000) -bind(#20600,#20601) +literals("v6","v6",#20598) +decl(#20598,#20495) +#20599=* +exprs(#20599,99,#20596,1,"import( ... n"" } })") +#20600=@"loc,{#10000},10,12,10,57" +locations_default(#20600,#10000,10,12,10,57) +hasLocation(#20599,#20600) +enclosing_stmt(#20599,#20595) +expr_containers(#20599,#20001) +#20601=* +exprs(#20601,4,#20599,0,"""module""") +hasLocation(#20601,#20231) +enclosing_stmt(#20601,#20595) +expr_containers(#20601,#20001) +literals("module","""module""",#20601) #20602=* -exprs(#20602,8,#20598,0,"{ type: ""json"" }") -#20603=@"loc,{#10000},13,8,13,23" -locations_default(#20603,#10000,13,8,13,23) +regexpterm(#20602,14,#20601,0,"module") +#20603=@"loc,{#10000},10,20,10,25" +locations_default(#20603,#10000,10,20,10,25) hasLocation(#20602,#20603) -enclosing_stmt(#20602,#20596) -expr_containers(#20602,#20001) +regexp_const_value(#20602,"module") #20604=* -properties(#20604,#20602,0,0,"type: ""json""") -#20605=@"loc,{#10000},13,10,13,21" -locations_default(#20605,#10000,13,10,13,21) +exprs(#20604,8,#20599,1,"{ ""with ... on"" } }") +#20605=@"loc,{#10000},10,29,10,56" +locations_default(#20605,#10000,10,29,10,56) hasLocation(#20604,#20605) +enclosing_stmt(#20604,#20595) +expr_containers(#20604,#20001) #20606=* -exprs(#20606,0,#20604,0,"type") -hasLocation(#20606,#20269) -enclosing_stmt(#20606,#20596) -expr_containers(#20606,#20001) -literals("type","type",#20606) -#20607=* -exprs(#20607,4,#20604,1,"""json""") -hasLocation(#20607,#20273) -enclosing_stmt(#20607,#20596) -expr_containers(#20607,#20001) -literals("json","""json""",#20607) +properties(#20606,#20604,0,0,"""with"": ... json"" }") +#20607=@"loc,{#10000},10,31,10,54" +locations_default(#20607,#10000,10,31,10,54) +hasLocation(#20606,#20607) #20608=* -regexpterm(#20608,14,#20607,0,"json") -#20609=@"loc,{#10000},13,17,13,20" -locations_default(#20609,#10000,13,17,13,20) -hasLocation(#20608,#20609) -regexp_const_value(#20608,"json") -#20610=* -stmts(#20610,27,#20001,10,"import ... son"" };") -hasLocation(#20610,#20035) -stmt_containers(#20610,#20001) +exprs(#20608,4,#20606,0,"""with""") +hasLocation(#20608,#20237) +enclosing_stmt(#20608,#20595) +expr_containers(#20608,#20001) +literals("with","""with""",#20608) +#20609=* +regexpterm(#20609,14,#20608,0,"with") +#20610=@"loc,{#10000},10,32,10,35" +locations_default(#20610,#10000,10,32,10,35) +hasLocation(#20609,#20610) +regexp_const_value(#20609,"with") #20611=* -exprs(#20611,4,#20610,-1,"""module""") -hasLocation(#20611,#20283) -enclosing_stmt(#20611,#20610) +exprs(#20611,8,#20606,1,"{ type: ""json"" }") +#20612=@"loc,{#10000},10,39,10,54" +locations_default(#20612,#10000,10,39,10,54) +hasLocation(#20611,#20612) +enclosing_stmt(#20611,#20595) expr_containers(#20611,#20001) -literals("module","""module""",#20611) -#20612=* -regexpterm(#20612,14,#20611,0,"module") -#20613=@"loc,{#10000},15,9,15,14" -locations_default(#20613,#10000,15,9,15,14) -hasLocation(#20612,#20613) -regexp_const_value(#20612,"module") -#20614=* -exprs(#20614,8,#20610,-10,"{ type: ""json"" }") -#20615=@"loc,{#10000},15,24,15,39" -locations_default(#20615,#10000,15,24,15,39) -hasLocation(#20614,#20615) -enclosing_stmt(#20614,#20610) -expr_containers(#20614,#20001) +#20613=* +properties(#20613,#20611,0,0,"type: ""json""") +#20614=@"loc,{#10000},10,41,10,52" +locations_default(#20614,#10000,10,41,10,52) +hasLocation(#20613,#20614) +#20615=* +exprs(#20615,0,#20613,0,"type") +hasLocation(#20615,#20243) +enclosing_stmt(#20615,#20595) +expr_containers(#20615,#20001) +literals("type","type",#20615) #20616=* -properties(#20616,#20614,0,0,"type: ""json""") -#20617=@"loc,{#10000},15,26,15,37" -locations_default(#20617,#10000,15,26,15,37) -hasLocation(#20616,#20617) -#20618=* -stmts(#20618,27,#20001,11,"import ... son"" };") -hasLocation(#20618,#20037) -stmt_containers(#20618,#20001) +exprs(#20616,4,#20613,1,"""json""") +hasLocation(#20616,#20247) +enclosing_stmt(#20616,#20595) +expr_containers(#20616,#20001) +literals("json","""json""",#20616) +#20617=* +regexpterm(#20617,14,#20616,0,"json") +#20618=@"loc,{#10000},10,48,10,51" +locations_default(#20618,#10000,10,48,10,51) +hasLocation(#20617,#20618) +regexp_const_value(#20617,"json") #20619=* -exprs(#20619,4,#20618,-1,"""module""") -hasLocation(#20619,#20309) -enclosing_stmt(#20619,#20618) -expr_containers(#20619,#20001) -literals("module","""module""",#20619) -#20620=* -regexpterm(#20620,14,#20619,0,"module") -#20621=@"loc,{#10000},16,22,16,27" -locations_default(#20621,#10000,16,22,16,27) -hasLocation(#20620,#20621) -regexp_const_value(#20620,"module") +stmts(#20619,27,#20001,8,"import ""module"";") +#20620=@"loc,{#10000},12,1,12,16" +locations_default(#20620,#10000,12,1,12,16) +hasLocation(#20619,#20620) +stmt_containers(#20619,#20001) +#20621=* +exprs(#20621,4,#20619,-1,"""module""") +hasLocation(#20621,#20259) +enclosing_stmt(#20621,#20619) +expr_containers(#20621,#20001) +literals("module","""module""",#20621) #20622=* -exprs(#20622,8,#20618,-10,"{ type: ""json"" }") -#20623=@"loc,{#10000},16,37,16,52" -locations_default(#20623,#10000,16,37,16,52) +regexpterm(#20622,14,#20621,0,"module") +#20623=@"loc,{#10000},12,9,12,14" +locations_default(#20623,#10000,12,9,12,14) hasLocation(#20622,#20623) -enclosing_stmt(#20622,#20618) -expr_containers(#20622,#20001) +regexp_const_value(#20622,"module") #20624=* -properties(#20624,#20622,0,0,"type: ""json""") -#20625=@"loc,{#10000},16,39,16,50" -locations_default(#20625,#10000,16,39,16,50) +stmts(#20624,2,#20001,9,"assert( ... on"" });") +#20625=@"loc,{#10000},13,1,13,25" +locations_default(#20625,#10000,13,1,13,25) hasLocation(#20624,#20625) +stmt_containers(#20624,#20001) #20626=* -exprs(#20626,85,#20618,0,"* as v1") -#20627=@"loc,{#10000},16,8,16,14" -locations_default(#20627,#10000,16,8,16,14) +exprs(#20626,13,#20624,0,"assert( ... son"" })") +#20627=@"loc,{#10000},13,1,13,24" +locations_default(#20627,#10000,13,1,13,24) hasLocation(#20626,#20627) -enclosing_stmt(#20626,#20618) +enclosing_stmt(#20626,#20624) expr_containers(#20626,#20001) #20628=* -exprs(#20628,78,#20626,1,"v1") -hasLocation(#20628,#20305) -enclosing_stmt(#20628,#20618) +exprs(#20628,79,#20626,-1,"assert") +hasLocation(#20628,#20263) +enclosing_stmt(#20628,#20624) expr_containers(#20628,#20001) -literals("v1","v1",#20628) -decl(#20628,#20486) -typedecl(#20628,#20489) -namespacedecl(#20628,#20492) -#20629=* -stmts(#20629,27,#20001,12,"import ... son"" };") -hasLocation(#20629,#20039) -stmt_containers(#20629,#20001) +literals("assert","assert",#20628) +#20629=@"var;{assert};{#20000}" +variables(#20629,"assert",#20000) +bind(#20628,#20629) #20630=* -exprs(#20630,4,#20629,-1,"""module""") -hasLocation(#20630,#20335) -enclosing_stmt(#20630,#20629) +exprs(#20630,8,#20626,0,"{ type: ""json"" }") +#20631=@"loc,{#10000},13,8,13,23" +locations_default(#20631,#10000,13,8,13,23) +hasLocation(#20630,#20631) +enclosing_stmt(#20630,#20624) expr_containers(#20630,#20001) -literals("module","""module""",#20630) -#20631=* -regexpterm(#20631,14,#20630,0,"module") -#20632=@"loc,{#10000},17,21,17,26" -locations_default(#20632,#10000,17,21,17,26) -hasLocation(#20631,#20632) -regexp_const_value(#20631,"module") -#20633=* -exprs(#20633,8,#20629,-10,"{ type: ""json"" }") -#20634=@"loc,{#10000},17,36,17,51" -locations_default(#20634,#10000,17,36,17,51) -hasLocation(#20633,#20634) -enclosing_stmt(#20633,#20629) -expr_containers(#20633,#20001) +#20632=* +properties(#20632,#20630,0,0,"type: ""json""") +#20633=@"loc,{#10000},13,10,13,21" +locations_default(#20633,#10000,13,10,13,21) +hasLocation(#20632,#20633) +#20634=* +exprs(#20634,0,#20632,0,"type") +hasLocation(#20634,#20269) +enclosing_stmt(#20634,#20624) +expr_containers(#20634,#20001) +literals("type","type",#20634) #20635=* -properties(#20635,#20633,0,0,"type: ""json""") -#20636=@"loc,{#10000},17,38,17,49" -locations_default(#20636,#10000,17,38,17,49) -hasLocation(#20635,#20636) -#20637=* -exprs(#20637,83,#20629,0,"v2") -hasLocation(#20637,#20329) -enclosing_stmt(#20637,#20629) -expr_containers(#20637,#20001) +exprs(#20635,4,#20632,1,"""json""") +hasLocation(#20635,#20273) +enclosing_stmt(#20635,#20624) +expr_containers(#20635,#20001) +literals("json","""json""",#20635) +#20636=* +regexpterm(#20636,14,#20635,0,"json") +#20637=@"loc,{#10000},13,17,13,20" +locations_default(#20637,#10000,13,17,13,20) +hasLocation(#20636,#20637) +regexp_const_value(#20636,"json") #20638=* -exprs(#20638,0,#20637,0,"v2") -hasLocation(#20638,#20329) -enclosing_stmt(#20638,#20629) -expr_containers(#20638,#20001) -literals("v2","v2",#20638) +stmts(#20638,27,#20001,10,"import ... son"" };") +hasLocation(#20638,#20035) +stmt_containers(#20638,#20001) #20639=* -exprs(#20639,78,#20637,1,"v2") -hasLocation(#20639,#20329) -enclosing_stmt(#20639,#20629) +exprs(#20639,4,#20638,-1,"""module""") +hasLocation(#20639,#20283) +enclosing_stmt(#20639,#20638) expr_containers(#20639,#20001) -literals("v2","v2",#20639) -decl(#20639,#20487) -typedecl(#20639,#20490) -namespacedecl(#20639,#20493) +literals("module","""module""",#20639) #20640=* -stmts(#20640,27,#20001,13,"import ... son"" };") -hasLocation(#20640,#20041) -stmt_containers(#20640,#20001) -#20641=* -exprs(#20641,4,#20640,-1,"""module""") -hasLocation(#20641,#20357) -enclosing_stmt(#20641,#20640) -expr_containers(#20641,#20001) -literals("module","""module""",#20641) +regexpterm(#20640,14,#20639,0,"module") +#20641=@"loc,{#10000},15,9,15,14" +locations_default(#20641,#10000,15,9,15,14) +hasLocation(#20640,#20641) +regexp_const_value(#20640,"module") #20642=* -regexpterm(#20642,14,#20641,0,"module") -#20643=@"loc,{#10000},18,17,18,22" -locations_default(#20643,#10000,18,17,18,22) +exprs(#20642,8,#20638,-10,"{ type: ""json"" }") +#20643=@"loc,{#10000},15,24,15,39" +locations_default(#20643,#10000,15,24,15,39) hasLocation(#20642,#20643) -regexp_const_value(#20642,"module") +enclosing_stmt(#20642,#20638) +expr_containers(#20642,#20001) #20644=* -exprs(#20644,8,#20640,-10,"{ type: ""json"" }") -#20645=@"loc,{#10000},18,32,18,47" -locations_default(#20645,#10000,18,32,18,47) +properties(#20644,#20642,0,0,"type: ""json""") +#20645=@"loc,{#10000},15,26,15,37" +locations_default(#20645,#10000,15,26,15,37) hasLocation(#20644,#20645) -enclosing_stmt(#20644,#20640) -expr_containers(#20644,#20001) #20646=* -properties(#20646,#20644,0,0,"type: ""json""") -#20647=@"loc,{#10000},18,34,18,45" -locations_default(#20647,#10000,18,34,18,45) -hasLocation(#20646,#20647) +exprs(#20646,0,#20644,0,"type") +hasLocation(#20646,#20289) +enclosing_stmt(#20646,#20638) +expr_containers(#20646,#20001) +literals("type","type",#20646) +#20647=* +exprs(#20647,4,#20644,1,"""json""") +hasLocation(#20647,#20293) +enclosing_stmt(#20647,#20638) +expr_containers(#20647,#20001) +literals("json","""json""",#20647) #20648=* -exprs(#20648,84,#20640,0,"v3") -hasLocation(#20648,#20353) -enclosing_stmt(#20648,#20640) -expr_containers(#20648,#20001) -#20649=* -exprs(#20649,78,#20648,1,"v3") -hasLocation(#20649,#20353) -enclosing_stmt(#20649,#20640) -expr_containers(#20649,#20001) -literals("v3","v3",#20649) -decl(#20649,#20488) -typedecl(#20649,#20491) -namespacedecl(#20649,#20494) +regexpterm(#20648,14,#20647,0,"json") +#20649=@"loc,{#10000},15,33,15,36" +locations_default(#20649,#10000,15,33,15,36) +hasLocation(#20648,#20649) +regexp_const_value(#20648,"json") #20650=* -stmts(#20650,30,#20001,14,"export ... son"" };") -hasLocation(#20650,#20045) +stmts(#20650,27,#20001,11,"import ... son"" };") +hasLocation(#20650,#20037) stmt_containers(#20650,#20001) #20651=* -exprs(#20651,4,#20650,-2,"""module""") -hasLocation(#20651,#20383) +exprs(#20651,4,#20650,-1,"""module""") +hasLocation(#20651,#20309) enclosing_stmt(#20651,#20650) expr_containers(#20651,#20001) literals("module","""module""",#20651) #20652=* regexpterm(#20652,14,#20651,0,"module") -#20653=@"loc,{#10000},20,21,20,26" -locations_default(#20653,#10000,20,21,20,26) +#20653=@"loc,{#10000},16,22,16,27" +locations_default(#20653,#10000,16,22,16,27) hasLocation(#20652,#20653) regexp_const_value(#20652,"module") #20654=* exprs(#20654,8,#20650,-10,"{ type: ""json"" }") -#20655=@"loc,{#10000},20,36,20,51" -locations_default(#20655,#10000,20,36,20,51) +#20655=@"loc,{#10000},16,37,16,52" +locations_default(#20655,#10000,16,37,16,52) hasLocation(#20654,#20655) enclosing_stmt(#20654,#20650) expr_containers(#20654,#20001) #20656=* properties(#20656,#20654,0,0,"type: ""json""") -#20657=@"loc,{#10000},20,38,20,49" -locations_default(#20657,#10000,20,38,20,49) +#20657=@"loc,{#10000},16,39,16,50" +locations_default(#20657,#10000,16,39,16,50) hasLocation(#20656,#20657) #20658=* -exprs(#20658,86,#20650,0,"v4") -hasLocation(#20658,#20377) +exprs(#20658,0,#20656,0,"type") +hasLocation(#20658,#20315) enclosing_stmt(#20658,#20650) expr_containers(#20658,#20001) +literals("type","type",#20658) #20659=* -exprs(#20659,0,#20658,0,"v4") -hasLocation(#20659,#20377) +exprs(#20659,4,#20656,1,"""json""") +hasLocation(#20659,#20319) enclosing_stmt(#20659,#20650) expr_containers(#20659,#20001) -literals("v4","v4",#20659) +literals("json","""json""",#20659) #20660=* -exprs(#20660,0,#20658,1,"v4") -hasLocation(#20660,#20377) -enclosing_stmt(#20660,#20650) -expr_containers(#20660,#20001) -literals("v4","v4",#20660) -#20661=* -stmts(#20661,28,#20001,15,"export ... son"" };") -hasLocation(#20661,#20047) -stmt_containers(#20661,#20001) +regexpterm(#20660,14,#20659,0,"json") +#20661=@"loc,{#10000},16,46,16,49" +locations_default(#20661,#10000,16,46,16,49) +hasLocation(#20660,#20661) +regexp_const_value(#20660,"json") #20662=* -exprs(#20662,4,#20661,0,"""module""") -hasLocation(#20662,#20405) -enclosing_stmt(#20662,#20661) +exprs(#20662,85,#20650,0,"* as v1") +#20663=@"loc,{#10000},16,8,16,14" +locations_default(#20663,#10000,16,8,16,14) +hasLocation(#20662,#20663) +enclosing_stmt(#20662,#20650) expr_containers(#20662,#20001) -literals("module","""module""",#20662) -#20663=* -regexpterm(#20663,14,#20662,0,"module") -#20664=@"loc,{#10000},21,16,21,21" -locations_default(#20664,#10000,21,16,21,21) -hasLocation(#20663,#20664) -regexp_const_value(#20663,"module") +#20664=* +exprs(#20664,78,#20662,1,"v1") +hasLocation(#20664,#20305) +enclosing_stmt(#20664,#20650) +expr_containers(#20664,#20001) +literals("v1","v1",#20664) +decl(#20664,#20486) +typedecl(#20664,#20489) +namespacedecl(#20664,#20492) #20665=* -exprs(#20665,8,#20661,-10,"{ type: ""json"" }") -#20666=@"loc,{#10000},21,31,21,46" -locations_default(#20666,#10000,21,31,21,46) -hasLocation(#20665,#20666) -enclosing_stmt(#20665,#20661) -expr_containers(#20665,#20001) +stmts(#20665,27,#20001,12,"import ... son"" };") +hasLocation(#20665,#20039) +stmt_containers(#20665,#20001) +#20666=* +exprs(#20666,4,#20665,-1,"""module""") +hasLocation(#20666,#20335) +enclosing_stmt(#20666,#20665) +expr_containers(#20666,#20001) +literals("module","""module""",#20666) #20667=* -properties(#20667,#20665,0,0,"type: ""json""") -#20668=@"loc,{#10000},21,33,21,44" -locations_default(#20668,#10000,21,33,21,44) +regexpterm(#20667,14,#20666,0,"module") +#20668=@"loc,{#10000},17,21,17,26" +locations_default(#20668,#10000,17,21,17,26) hasLocation(#20667,#20668) +regexp_const_value(#20667,"module") #20669=* -stmts(#20669,30,#20001,16,"export ... son"" };") -hasLocation(#20669,#20049) -stmt_containers(#20669,#20001) -#20670=* -exprs(#20670,4,#20669,-2,"""module""") -hasLocation(#20670,#20431) -enclosing_stmt(#20670,#20669) -expr_containers(#20670,#20001) -literals("module","""module""",#20670) +exprs(#20669,8,#20665,-10,"{ type: ""json"" }") +#20670=@"loc,{#10000},17,36,17,51" +locations_default(#20670,#10000,17,36,17,51) +hasLocation(#20669,#20670) +enclosing_stmt(#20669,#20665) +expr_containers(#20669,#20001) #20671=* -regexpterm(#20671,14,#20670,0,"module") -#20672=@"loc,{#10000},22,22,22,27" -locations_default(#20672,#10000,22,22,22,27) +properties(#20671,#20669,0,0,"type: ""json""") +#20672=@"loc,{#10000},17,38,17,49" +locations_default(#20672,#10000,17,38,17,49) hasLocation(#20671,#20672) -regexp_const_value(#20671,"module") #20673=* -exprs(#20673,8,#20669,-10,"{ type: ""json"" }") -#20674=@"loc,{#10000},22,37,22,52" -locations_default(#20674,#10000,22,37,22,52) -hasLocation(#20673,#20674) -enclosing_stmt(#20673,#20669) +exprs(#20673,0,#20671,0,"type") +hasLocation(#20673,#20341) +enclosing_stmt(#20673,#20665) expr_containers(#20673,#20001) +literals("type","type",#20673) +#20674=* +exprs(#20674,4,#20671,1,"""json""") +hasLocation(#20674,#20345) +enclosing_stmt(#20674,#20665) +expr_containers(#20674,#20001) +literals("json","""json""",#20674) #20675=* -properties(#20675,#20673,0,0,"type: ""json""") -#20676=@"loc,{#10000},22,39,22,50" -locations_default(#20676,#10000,22,39,22,50) +regexpterm(#20675,14,#20674,0,"json") +#20676=@"loc,{#10000},17,45,17,48" +locations_default(#20676,#10000,17,45,17,48) hasLocation(#20675,#20676) +regexp_const_value(#20675,"json") #20677=* -exprs(#20677,96,#20669,0,"* as v5") -#20678=@"loc,{#10000},22,8,22,14" -locations_default(#20678,#10000,22,8,22,14) -hasLocation(#20677,#20678) -enclosing_stmt(#20677,#20669) +exprs(#20677,83,#20665,0,"v2") +hasLocation(#20677,#20329) +enclosing_stmt(#20677,#20665) expr_containers(#20677,#20001) +#20678=* +exprs(#20678,0,#20677,0,"v2") +hasLocation(#20678,#20329) +enclosing_stmt(#20678,#20665) +expr_containers(#20678,#20001) +literals("v2","v2",#20678) #20679=* -exprs(#20679,0,#20677,1,"v5") -hasLocation(#20679,#20427) -enclosing_stmt(#20679,#20669) +exprs(#20679,78,#20677,1,"v2") +hasLocation(#20679,#20329) +enclosing_stmt(#20679,#20665) expr_containers(#20679,#20001) -literals("v5","v5",#20679) +literals("v2","v2",#20679) +decl(#20679,#20487) +typedecl(#20679,#20490) +namespacedecl(#20679,#20493) #20680=* -stmts(#20680,22,#20001,17,"const v ... "" } });") -hasLocation(#20680,#20053) +stmts(#20680,27,#20001,13,"import ... son"" };") +hasLocation(#20680,#20041) stmt_containers(#20680,#20001) #20681=* -exprs(#20681,64,#20680,0,"v6 = im ... n"" } })") -#20682=@"loc,{#10000},24,7,24,57" -locations_default(#20682,#10000,24,7,24,57) -hasLocation(#20681,#20682) +exprs(#20681,4,#20680,-1,"""module""") +hasLocation(#20681,#20357) enclosing_stmt(#20681,#20680) expr_containers(#20681,#20001) -#20683=* -exprs(#20683,78,#20681,0,"v6") -hasLocation(#20683,#20449) -enclosing_stmt(#20683,#20680) -expr_containers(#20683,#20001) -literals("v6","v6",#20683) -decl(#20683,#20495) +literals("module","""module""",#20681) +#20682=* +regexpterm(#20682,14,#20681,0,"module") +#20683=@"loc,{#10000},18,17,18,22" +locations_default(#20683,#10000,18,17,18,22) +hasLocation(#20682,#20683) +regexp_const_value(#20682,"module") #20684=* -exprs(#20684,99,#20681,1,"import( ... n"" } })") -#20685=@"loc,{#10000},24,12,24,57" -locations_default(#20685,#10000,24,12,24,57) +exprs(#20684,8,#20680,-10,"{ type: ""json"" }") +#20685=@"loc,{#10000},18,32,18,47" +locations_default(#20685,#10000,18,32,18,47) hasLocation(#20684,#20685) enclosing_stmt(#20684,#20680) expr_containers(#20684,#20001) #20686=* -exprs(#20686,4,#20684,0,"""module""") -hasLocation(#20686,#20457) -enclosing_stmt(#20686,#20680) -expr_containers(#20686,#20001) -literals("module","""module""",#20686) -#20687=* -regexpterm(#20687,14,#20686,0,"module") -#20688=@"loc,{#10000},24,20,24,25" -locations_default(#20688,#10000,24,20,24,25) -hasLocation(#20687,#20688) -regexp_const_value(#20687,"module") +properties(#20686,#20684,0,0,"type: ""json""") +#20687=@"loc,{#10000},18,34,18,45" +locations_default(#20687,#10000,18,34,18,45) +hasLocation(#20686,#20687) +#20688=* +exprs(#20688,0,#20686,0,"type") +hasLocation(#20688,#20363) +enclosing_stmt(#20688,#20680) +expr_containers(#20688,#20001) +literals("type","type",#20688) #20689=* -exprs(#20689,8,#20684,1,"{ asser ... on"" } }") -#20690=@"loc,{#10000},24,29,24,56" -locations_default(#20690,#10000,24,29,24,56) -hasLocation(#20689,#20690) +exprs(#20689,4,#20686,1,"""json""") +hasLocation(#20689,#20367) enclosing_stmt(#20689,#20680) expr_containers(#20689,#20001) -#20691=* -properties(#20691,#20689,0,0,"assert: ... json"" }") -#20692=@"loc,{#10000},24,31,24,54" -locations_default(#20692,#10000,24,31,24,54) -hasLocation(#20691,#20692) +literals("json","""json""",#20689) +#20690=* +regexpterm(#20690,14,#20689,0,"json") +#20691=@"loc,{#10000},18,41,18,44" +locations_default(#20691,#10000,18,41,18,44) +hasLocation(#20690,#20691) +regexp_const_value(#20690,"json") +#20692=* +exprs(#20692,84,#20680,0,"v3") +hasLocation(#20692,#20353) +enclosing_stmt(#20692,#20680) +expr_containers(#20692,#20001) #20693=* -exprs(#20693,0,#20691,0,"assert") -hasLocation(#20693,#20463) +exprs(#20693,78,#20692,1,"v3") +hasLocation(#20693,#20353) enclosing_stmt(#20693,#20680) expr_containers(#20693,#20001) -literals("assert","assert",#20693) +literals("v3","v3",#20693) +decl(#20693,#20488) +typedecl(#20693,#20491) +namespacedecl(#20693,#20494) #20694=* -exprs(#20694,8,#20691,1,"{ type: ""json"" }") -#20695=@"loc,{#10000},24,39,24,54" -locations_default(#20695,#10000,24,39,24,54) -hasLocation(#20694,#20695) -enclosing_stmt(#20694,#20680) -expr_containers(#20694,#20001) +stmts(#20694,30,#20001,14,"export ... son"" };") +hasLocation(#20694,#20045) +stmt_containers(#20694,#20001) +#20695=* +exprs(#20695,4,#20694,-2,"""module""") +hasLocation(#20695,#20383) +enclosing_stmt(#20695,#20694) +expr_containers(#20695,#20001) +literals("module","""module""",#20695) #20696=* -properties(#20696,#20694,0,0,"type: ""json""") -#20697=@"loc,{#10000},24,41,24,52" -locations_default(#20697,#10000,24,41,24,52) +regexpterm(#20696,14,#20695,0,"module") +#20697=@"loc,{#10000},20,21,20,26" +locations_default(#20697,#10000,20,21,20,26) hasLocation(#20696,#20697) +regexp_const_value(#20696,"module") #20698=* -exprs(#20698,0,#20696,0,"type") -hasLocation(#20698,#20469) -enclosing_stmt(#20698,#20680) +exprs(#20698,8,#20694,-10,"{ type: ""json"" }") +#20699=@"loc,{#10000},20,36,20,51" +locations_default(#20699,#10000,20,36,20,51) +hasLocation(#20698,#20699) +enclosing_stmt(#20698,#20694) expr_containers(#20698,#20001) -literals("type","type",#20698) -#20699=* -exprs(#20699,4,#20696,1,"""json""") -hasLocation(#20699,#20473) -enclosing_stmt(#20699,#20680) -expr_containers(#20699,#20001) -literals("json","""json""",#20699) #20700=* -regexpterm(#20700,14,#20699,0,"json") -#20701=@"loc,{#10000},24,48,24,51" -locations_default(#20701,#10000,24,48,24,51) +properties(#20700,#20698,0,0,"type: ""json""") +#20701=@"loc,{#10000},20,38,20,49" +locations_default(#20701,#10000,20,38,20,49) hasLocation(#20700,#20701) -regexp_const_value(#20700,"json") #20702=* -entry_cfg_node(#20702,#20001) -#20703=@"loc,{#10000},1,1,1,0" -locations_default(#20703,#10000,1,1,1,0) -hasLocation(#20702,#20703) +exprs(#20702,0,#20700,0,"type") +hasLocation(#20702,#20389) +enclosing_stmt(#20702,#20694) +expr_containers(#20702,#20001) +literals("type","type",#20702) +#20703=* +exprs(#20703,4,#20700,1,"""json""") +hasLocation(#20703,#20393) +enclosing_stmt(#20703,#20694) +expr_containers(#20703,#20001) +literals("json","""json""",#20703) #20704=* -exit_cfg_node(#20704,#20001) -hasLocation(#20704,#20483) -successor(#20680,#20683) -successor(#20686,#20684) -successor(#20684,#20681) -successor(#20683,#20686) -successor(#20681,#20704) -successor(#20669,#20670) -successor(#20677,#20679) -successor(#20679,#20680) -successor(#20670,#20677) -successor(#20661,#20662) -successor(#20662,#20669) -successor(#20650,#20651) -successor(#20658,#20659) -successor(#20660,#20661) -successor(#20659,#20660) -successor(#20651,#20658) -successor(#20640,#20650) -successor(#20629,#20640) -successor(#20618,#20629) -successor(#20610,#20618) -successor(#20596,#20600) -successor(#20602,#20606) -successor(#20607,#20604) -successor(#20606,#20607) -successor(#20604,#20598) -successor(#20600,#20602) -successor(#20598,#20610) -successor(#20591,#20596) -successor(#20567,#20570) -successor(#20573,#20571) -successor(#20571,#20568) -successor(#20570,#20573) -successor(#20568,#20591) -successor(#20556,#20557) -successor(#20564,#20566) +regexpterm(#20704,14,#20703,0,"json") +#20705=@"loc,{#10000},20,45,20,48" +locations_default(#20705,#10000,20,45,20,48) +hasLocation(#20704,#20705) +regexp_const_value(#20704,"json") +#20706=* +exprs(#20706,86,#20694,0,"v4") +hasLocation(#20706,#20377) +enclosing_stmt(#20706,#20694) +expr_containers(#20706,#20001) +#20707=* +exprs(#20707,0,#20706,0,"v4") +hasLocation(#20707,#20377) +enclosing_stmt(#20707,#20694) +expr_containers(#20707,#20001) +literals("v4","v4",#20707) +#20708=* +exprs(#20708,0,#20706,1,"v4") +hasLocation(#20708,#20377) +enclosing_stmt(#20708,#20694) +expr_containers(#20708,#20001) +literals("v4","v4",#20708) +#20709=* +stmts(#20709,28,#20001,15,"export ... son"" };") +hasLocation(#20709,#20047) +stmt_containers(#20709,#20001) +#20710=* +exprs(#20710,4,#20709,0,"""module""") +hasLocation(#20710,#20405) +enclosing_stmt(#20710,#20709) +expr_containers(#20710,#20001) +literals("module","""module""",#20710) +#20711=* +regexpterm(#20711,14,#20710,0,"module") +#20712=@"loc,{#10000},21,16,21,21" +locations_default(#20712,#10000,21,16,21,21) +hasLocation(#20711,#20712) +regexp_const_value(#20711,"module") +#20713=* +exprs(#20713,8,#20709,-10,"{ type: ""json"" }") +#20714=@"loc,{#10000},21,31,21,46" +locations_default(#20714,#10000,21,31,21,46) +hasLocation(#20713,#20714) +enclosing_stmt(#20713,#20709) +expr_containers(#20713,#20001) +#20715=* +properties(#20715,#20713,0,0,"type: ""json""") +#20716=@"loc,{#10000},21,33,21,44" +locations_default(#20716,#10000,21,33,21,44) +hasLocation(#20715,#20716) +#20717=* +exprs(#20717,0,#20715,0,"type") +hasLocation(#20717,#20411) +enclosing_stmt(#20717,#20709) +expr_containers(#20717,#20001) +literals("type","type",#20717) +#20718=* +exprs(#20718,4,#20715,1,"""json""") +hasLocation(#20718,#20415) +enclosing_stmt(#20718,#20709) +expr_containers(#20718,#20001) +literals("json","""json""",#20718) +#20719=* +regexpterm(#20719,14,#20718,0,"json") +#20720=@"loc,{#10000},21,40,21,43" +locations_default(#20720,#10000,21,40,21,43) +hasLocation(#20719,#20720) +regexp_const_value(#20719,"json") +#20721=* +stmts(#20721,30,#20001,16,"export ... son"" };") +hasLocation(#20721,#20049) +stmt_containers(#20721,#20001) +#20722=* +exprs(#20722,4,#20721,-2,"""module""") +hasLocation(#20722,#20431) +enclosing_stmt(#20722,#20721) +expr_containers(#20722,#20001) +literals("module","""module""",#20722) +#20723=* +regexpterm(#20723,14,#20722,0,"module") +#20724=@"loc,{#10000},22,22,22,27" +locations_default(#20724,#10000,22,22,22,27) +hasLocation(#20723,#20724) +regexp_const_value(#20723,"module") +#20725=* +exprs(#20725,8,#20721,-10,"{ type: ""json"" }") +#20726=@"loc,{#10000},22,37,22,52" +locations_default(#20726,#10000,22,37,22,52) +hasLocation(#20725,#20726) +enclosing_stmt(#20725,#20721) +expr_containers(#20725,#20001) +#20727=* +properties(#20727,#20725,0,0,"type: ""json""") +#20728=@"loc,{#10000},22,39,22,50" +locations_default(#20728,#10000,22,39,22,50) +hasLocation(#20727,#20728) +#20729=* +exprs(#20729,0,#20727,0,"type") +hasLocation(#20729,#20437) +enclosing_stmt(#20729,#20721) +expr_containers(#20729,#20001) +literals("type","type",#20729) +#20730=* +exprs(#20730,4,#20727,1,"""json""") +hasLocation(#20730,#20441) +enclosing_stmt(#20730,#20721) +expr_containers(#20730,#20001) +literals("json","""json""",#20730) +#20731=* +regexpterm(#20731,14,#20730,0,"json") +#20732=@"loc,{#10000},22,46,22,49" +locations_default(#20732,#10000,22,46,22,49) +hasLocation(#20731,#20732) +regexp_const_value(#20731,"json") +#20733=* +exprs(#20733,96,#20721,0,"* as v5") +#20734=@"loc,{#10000},22,8,22,14" +locations_default(#20734,#10000,22,8,22,14) +hasLocation(#20733,#20734) +enclosing_stmt(#20733,#20721) +expr_containers(#20733,#20001) +#20735=* +exprs(#20735,0,#20733,1,"v5") +hasLocation(#20735,#20427) +enclosing_stmt(#20735,#20721) +expr_containers(#20735,#20001) +literals("v5","v5",#20735) +#20736=* +stmts(#20736,22,#20001,17,"const v ... "" } });") +hasLocation(#20736,#20053) +stmt_containers(#20736,#20001) +#20737=* +exprs(#20737,64,#20736,0,"v6 = im ... n"" } })") +#20738=@"loc,{#10000},24,7,24,57" +locations_default(#20738,#10000,24,7,24,57) +hasLocation(#20737,#20738) +enclosing_stmt(#20737,#20736) +expr_containers(#20737,#20001) +#20739=* +exprs(#20739,78,#20737,0,"v6") +hasLocation(#20739,#20449) +enclosing_stmt(#20739,#20736) +expr_containers(#20739,#20001) +literals("v6","v6",#20739) +decl(#20739,#20495) +#20740=* +exprs(#20740,99,#20737,1,"import( ... n"" } })") +#20741=@"loc,{#10000},24,12,24,57" +locations_default(#20741,#10000,24,12,24,57) +hasLocation(#20740,#20741) +enclosing_stmt(#20740,#20736) +expr_containers(#20740,#20001) +#20742=* +exprs(#20742,4,#20740,0,"""module""") +hasLocation(#20742,#20457) +enclosing_stmt(#20742,#20736) +expr_containers(#20742,#20001) +literals("module","""module""",#20742) +#20743=* +regexpterm(#20743,14,#20742,0,"module") +#20744=@"loc,{#10000},24,20,24,25" +locations_default(#20744,#10000,24,20,24,25) +hasLocation(#20743,#20744) +regexp_const_value(#20743,"module") +#20745=* +exprs(#20745,8,#20740,1,"{ asser ... on"" } }") +#20746=@"loc,{#10000},24,29,24,56" +locations_default(#20746,#10000,24,29,24,56) +hasLocation(#20745,#20746) +enclosing_stmt(#20745,#20736) +expr_containers(#20745,#20001) +#20747=* +properties(#20747,#20745,0,0,"assert: ... json"" }") +#20748=@"loc,{#10000},24,31,24,54" +locations_default(#20748,#10000,24,31,24,54) +hasLocation(#20747,#20748) +#20749=* +exprs(#20749,0,#20747,0,"assert") +hasLocation(#20749,#20463) +enclosing_stmt(#20749,#20736) +expr_containers(#20749,#20001) +literals("assert","assert",#20749) +#20750=* +exprs(#20750,8,#20747,1,"{ type: ""json"" }") +#20751=@"loc,{#10000},24,39,24,54" +locations_default(#20751,#10000,24,39,24,54) +hasLocation(#20750,#20751) +enclosing_stmt(#20750,#20736) +expr_containers(#20750,#20001) +#20752=* +properties(#20752,#20750,0,0,"type: ""json""") +#20753=@"loc,{#10000},24,41,24,52" +locations_default(#20753,#10000,24,41,24,52) +hasLocation(#20752,#20753) +#20754=* +exprs(#20754,0,#20752,0,"type") +hasLocation(#20754,#20469) +enclosing_stmt(#20754,#20736) +expr_containers(#20754,#20001) +literals("type","type",#20754) +#20755=* +exprs(#20755,4,#20752,1,"""json""") +hasLocation(#20755,#20473) +enclosing_stmt(#20755,#20736) +expr_containers(#20755,#20001) +literals("json","""json""",#20755) +#20756=* +regexpterm(#20756,14,#20755,0,"json") +#20757=@"loc,{#10000},24,48,24,51" +locations_default(#20757,#10000,24,48,24,51) +hasLocation(#20756,#20757) +regexp_const_value(#20756,"json") +#20758=* +entry_cfg_node(#20758,#20001) +#20759=@"loc,{#10000},1,1,1,0" +locations_default(#20759,#10000,1,1,1,0) +hasLocation(#20758,#20759) +#20760=* +exit_cfg_node(#20760,#20001) +hasLocation(#20760,#20483) +successor(#20736,#20739) +successor(#20742,#20740) +successor(#20740,#20737) +successor(#20739,#20742) +successor(#20737,#20760) +successor(#20721,#20722) +successor(#20733,#20735) +successor(#20735,#20736) +successor(#20722,#20733) +successor(#20709,#20710) +successor(#20710,#20721) +successor(#20694,#20695) +successor(#20706,#20707) +successor(#20708,#20709) +successor(#20707,#20708) +successor(#20695,#20706) +successor(#20680,#20694) +successor(#20665,#20680) +successor(#20650,#20665) +successor(#20638,#20650) +successor(#20624,#20628) +successor(#20630,#20634) +successor(#20635,#20632) +successor(#20634,#20635) +successor(#20632,#20626) +successor(#20628,#20630) +successor(#20626,#20638) +successor(#20619,#20624) +successor(#20595,#20598) +successor(#20601,#20599) +successor(#20599,#20596) +successor(#20598,#20601) +successor(#20596,#20619) +successor(#20580,#20581) +successor(#20592,#20594) +successor(#20594,#20595) +successor(#20581,#20592) +successor(#20568,#20569) +successor(#20569,#20580) +successor(#20553,#20554) +successor(#20565,#20566) +successor(#20567,#20568) successor(#20566,#20567) -successor(#20557,#20564) -successor(#20548,#20549) -successor(#20549,#20556) -successor(#20537,#20538) -successor(#20545,#20546) -successor(#20547,#20548) -successor(#20546,#20547) -successor(#20538,#20545) -successor(#20527,#20537) -successor(#20516,#20527) -successor(#20505,#20516) -successor(#20497,#20505) -successor(#20648,#20497) -successor(#20637,#20648) -successor(#20626,#20637) -successor(#20535,#20626) -successor(#20524,#20535) -successor(#20513,#20524) -successor(#20702,#20513) +successor(#20554,#20565) +successor(#20539,#20553) +successor(#20524,#20539) +successor(#20509,#20524) +successor(#20497,#20509) +successor(#20692,#20497) +successor(#20677,#20692) +successor(#20662,#20677) +successor(#20551,#20662) +successor(#20536,#20551) +successor(#20521,#20536) +successor(#20758,#20521) numlines(#10000,24,18,2) filetype(#10000,"typescript") diff --git a/javascript/ql/integration-tests/diagnostics/internal-error/fail-native.cjs b/javascript/ql/integration-tests/diagnostics/internal-error/fail-native.cjs new file mode 100644 index 000000000000..6dd5449b1acb --- /dev/null +++ b/javascript/ql/integration-tests/diagnostics/internal-error/fail-native.cjs @@ -0,0 +1,7 @@ +const childProcess = require("node:child_process"); +const spawn = childProcess.spawn; +childProcess.spawn = function (...args) { + const child = spawn.apply(this, args); + child.kill(); + return child; +}; diff --git a/javascript/ql/integration-tests/diagnostics/internal-error/test.py b/javascript/ql/integration-tests/diagnostics/internal-error/test.py index 25a95e763ee8..c9e36473f627 100644 --- a/javascript/ql/integration-tests/diagnostics/internal-error/test.py +++ b/javascript/ql/integration-tests/diagnostics/internal-error/test.py @@ -1,2 +1,6 @@ -def test(codeql, javascript): - codeql.database.create(_assert_failure=True) +import os + + +def test(codeql, javascript, cwd): + os.environ["NODE_OPTIONS"] = f'--require "{cwd / "fail-native.cjs"}"' + codeql.database.create(source_root="src", _assert_failure=True) diff --git a/javascript/ql/test/library-tests/TypeScript/SourceImports/test.expected b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.expected new file mode 100644 index 000000000000..769e9dfee998 --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.expected @@ -0,0 +1,5 @@ +imports +| tst.ts:1:1:1:40 | import ... .wasm"; | +references +| tst.ts:1:15:1:18 | wasm | +| tst.ts:3:1:3:4 | wasm | diff --git a/javascript/ql/test/library-tests/TypeScript/SourceImports/test.ql b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.ql new file mode 100644 index 000000000000..0997989d37ec --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/SourceImports/test.ql @@ -0,0 +1,5 @@ +import javascript + +query predicate imports(ImportDeclaration decl) { any() } + +query predicate references(VarRef ref) { any() } diff --git a/javascript/ql/test/library-tests/TypeScript/SourceImports/tst.ts b/javascript/ql/test/library-tests/TypeScript/SourceImports/tst.ts new file mode 100644 index 000000000000..d1c1fa88139a --- /dev/null +++ b/javascript/ql/test/library-tests/TypeScript/SourceImports/tst.ts @@ -0,0 +1,3 @@ +import source wasm from "./module.wasm"; + +wasm; diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 684bcbb8c3ba..1410e12330bb 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -84,7 +84,7 @@ CodeQLPackInfo = provider( "pack_prefix": "A prefix to add to all paths, IF the user requests so. We omit it for local installation targets of single packs (but not pack groups)", "files": "PackageFilegroupInfo provider with list of all files in this pack (CODEQL_PLATFORM in paths unresolved)", "zips": "A _ZipInfo provider to include in the pack, (CODEQL_PLATFORM unresolved).", - "arch_overrides": "A list of files that should be included in the arch-specific bit, even though the path doesn't contain CODEQL_PLATFORM.", + "arch_overrides": "Files or imported zip prefixes to include in the arch-specific bit, even without CODEQL_PLATFORM in their paths.", }, ) @@ -220,7 +220,7 @@ _codeql_pack_info = rule( allow_files = [".zip"], ), "prefix": attr.string(doc = "Prefix to add to all files."), - "arch_overrides": attr.string_list(doc = "A list of files that should be included in the arch package regardless of the path, specify the path _without_ `prefix`."), + "arch_overrides": attr.string_list(doc = "Files or imported zip prefixes to include in the arch package regardless of the path, specified _without_ `prefix`."), }, provides = [CodeQLPackInfo], ) @@ -293,6 +293,8 @@ def _codeql_pack_group_extract_zips_impl(ctx): # for each zip file, resolve whether it's filtered out or not by the current kind, and add the pack prefix for zip, zip_prefix in src.zips.zips_to_prefixes.items(): zip_kind, expanded_prefix = _expand_path(paths.join(prefix, zip_prefix), platform) + if zip_kind == "common" and zip_prefix in src.arch_overrides: + zip_kind = "arch" if include_all_files or zip_kind == ctx.attr.kind: zips_to_prefixes[zip] = expanded_prefix @@ -465,13 +467,13 @@ def codeql_pack( The distinction between arch-specific and common contents is made based on whether the paths (including possible prefixes added by rules) contain the special `{CODEQL_PLATFORM}` placeholder, which in case it is present will also be replaced by the appropriate platform (`linux64`, `linux-arm64`, `win64` or `osx64`). - Specific file paths can be placed in the arch-specific package by adding them to `arch_overrides`, even if their - path doesn't contain the `CODEQL_PLATFORM` placeholder. + Specific file paths or imported zip prefixes can be placed in the arch-specific package by adding them to + `arch_overrides`, even if their path doesn't contain the `CODEQL_PLATFORM` placeholder. The codeql pack rules will expand the `{CODEQL_PLATFORM}` marker in paths, and use that to split the files into a common and an arch-specific part. This placeholder will be replaced by the appropriate platform (`linux64`, `linux-arm64`, `win64` or `osx64`). - `arch_overrides` is a list of files that should be included in the arch-specific bits of the pack, even if their path doesn't - contain the `{CODEQL_PLATFORM}` marker. + `arch_overrides` is a list of files or imported zip prefixes that should be included in the arch-specific bits of the + pack, even if their path doesn't contain the `{CODEQL_PLATFORM}` marker. All files in the pack will be prefixed with `name`, unless `pack_prefix` is set, then is used instead. This rule also provides a convenient installer target named `-installer`, with a path governed by `install_dest`,