Harden DT PR runs

.azure-pipelines/typescript-pr-dt.yml

@@ -31,6 +31,8 @@ resources:
       ref: refs/heads/master
 
 variables:
+  - name: dtslintShardCount
+    value: 16
   - name: CI
     value: true
   - name: skipComponentGovernanceDetection
@@ -51,7 +53,7 @@ jobs:
     workspace:
       clean: all
     strategy:
-      parallel: 16
+      parallel: ${{ variables.dtslintShardCount }}
     steps:
       - checkout: self
         path: s/DefinitelyTyped-tools
@@ -105,6 +107,9 @@ jobs:
           verify_sha base "$(git rev-parse HEAD^1)" "$EXPECTED_BASE_SHA"
           verify_sha head "$(git rev-parse HEAD^2)" "$EXPECTED_HEAD_SHA"
         displayName: Verify TypeScript checkout
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/TypeScript
         env:
           EXPECTED_BASE_SHA: ${{ parameters.expected_base_sha }}
@@ -116,32 +121,50 @@ jobs:
           npm install --global "$(jq -r '.packageManager' package.json)"
           pnpm install --frozen-lockfile
         displayName: Install DefinitelyTyped tools dependencies
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/DefinitelyTyped-tools
 
       - bash: pnpm build
         displayName: Build DefinitelyTyped tools
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/DefinitelyTyped-tools
 
       - bash: npm ci
         displayName: Install PR TypeScript dependencies
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/TypeScript
 
       - bash: |
           set -euo pipefail
           npm run build
           npx hereby build:api
         displayName: Build PR TypeScript
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/TypeScript
 
       - bash: |
           set -euo pipefail
           npm install --global "$(jq -r '.packageManager' package.json)"
           pnpm install
         displayName: Install DefinitelyTyped dependencies
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/DefinitelyTyped
 
       - bash: bash .azure-pipelines/scripts/run-typescript-pr-dt.sh PR
         displayName: Run dtslint with PR TypeScript
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/DefinitelyTyped-tools
         env:
           SHARD_COUNT: $(System.TotalJobsInPhase)
@@ -152,30 +175,45 @@ jobs:
           git clean -xdf
           git switch --detach "$EXPECTED_BASE_SHA"
         displayName: Prepare base TypeScript checkout
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/TypeScript
         env:
           EXPECTED_BASE_SHA: ${{ parameters.expected_base_sha }}
 
       - bash: npm ci
         displayName: Install base TypeScript dependencies
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/TypeScript
 
       - bash: |
           set -euo pipefail
           npm run build
           npx hereby build:api
         displayName: Build base TypeScript
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/TypeScript
 
       - bash: bash .azure-pipelines/scripts/run-typescript-pr-dt.sh main
         displayName: Run dtslint with base TypeScript
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/DefinitelyTyped-tools
         env:
           SHARD_COUNT: $(System.TotalJobsInPhase)
           SHARD_ID: $(System.JobPositionInPhase)
 
       - bash: bash .azure-pipelines/scripts/compare-typescript-pr-dt.sh
         displayName: Compare dtslint logs
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/DefinitelyTyped-tools
 
       - publish: $(Pipeline.Workspace)/pr/prFailures$(System.JobPositionInPhase).json
@@ -229,10 +267,32 @@ jobs:
           pnpm install --frozen-lockfile
           pnpm build
         displayName: Build DefinitelyTyped tools
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/DefinitelyTyped-tools
 
+      # Restricted commands still allow task.complete; validate in a separate trusted task.
+      - bash: |
+          node packages/dtslint-runner/dist/validate-results.js \
+            "$EXPECTED_SHARD_COUNT" \
+            "$(Pipeline.Workspace)/mainFailures" \
+            "$(Pipeline.Workspace)/prFailures"
+        displayName: Validate successful DT artifacts
+        condition: and(succeeded(), eq(variables['runResult'], 'Succeeded'))
+        target:
+          commands: restricted
+          settableVariables: none
+        workingDirectory: $(Pipeline.Workspace)/s/DefinitelyTyped-tools
+        env:
+          EXPECTED_SHARD_COUNT: ${{ variables.dtslintShardCount }}
+
       - task: AzureCLI@2
         displayName: Create GitHub token
+        target:
+          commands: restricted
+          settableVariables:
+            - GH_TOKEN
         inputs:
           azureSubscription: TypeScript Public CI
           scriptType: bash
@@ -266,15 +326,20 @@ jobs:
               "$status"
               "$(Pipeline.Workspace)/mainFailures"
               "$(Pipeline.Workspace)/prFailures"
+              "$EXPECTED_SHARD_COUNT"
             )
           else
             args+=("$status")
           fi
           node packages/dtslint-runner/dist/post-results.js "${args[@]}"
         displayName: Post DT results
+        target:
+          commands: restricted
+          settableVariables: none
         workingDirectory: $(Pipeline.Workspace)/s/DefinitelyTyped-tools
         env:
           DISTINCT_ID: ${{ parameters.distinct_id }}
+          EXPECTED_SHARD_COUNT: ${{ variables.dtslintShardCount }}
           GH_TOKEN: $(GH_TOKEN)
           REQUESTING_USER: ${{ parameters.requesting_user }}
           RUN_RESULT: $(runResult)
@@ -283,6 +348,9 @@ jobs:
 
       - bash: node /tmp/create-github-app-token.cjs
         displayName: Revoke GitHub token
+        target:
+          commands: restricted
+          settableVariables: none
         condition: always()
         continueOnError: true
         env:

.changeset/shaggy-rivers-look.md

@@ -0,0 +1,5 @@
+---
+
+---
+
+

packages/dtslint-runner/src/post-results.ts

@@ -1,21 +1,31 @@
 import { Octokit } from "@octokit/rest";
-import { readFileSync } from "fs";
-import glob = require("glob");
 import type { Failure } from "./add-github-links";
+import { readComparison } from "./validate-results";
 
 type Errors = Failure[];
 
-// Args: [auth token] [buildId] [status comment] [user to tag] [issue] [distinct id] [job status] [?main errors file] [?branch errors file]
+// Args: [auth token] [buildId] [status comment] [user to tag] [issue] [distinct id] [job status]
+// Successful runs also require: [main errors directory] [branch errors directory] [expected shard count]
 export async function main() {
-  const [auth, buildId, statusCommentId, userToTag, issue, distinctId, status, mainErrorsPath, branchErrorsPath] =
-    process.argv.slice(2);
+  const [
+    auth,
+    buildId,
+    statusCommentId,
+    userToTag,
+    issue,
+    distinctId,
+    status,
+    mainErrorsPath,
+    branchErrorsPath,
+    expectedShardCount,
+  ] = process.argv.slice(2);
   if (!auth) throw new Error("First argument must be a GitHub auth token.");
   if (!buildId) throw new Error("Second argument must be a build id.");
   if (!statusCommentId) throw new Error("Third argument must be a GitHub comment id.");
   if (!userToTag) throw new Error("Fourth argument must be a GitHub username.");
   if (!issue) throw new Error("Fifth argument must be a TypeScript issue/PR number.");
   if (!distinctId) throw new Error("Sixth argument must be a distinct ID.");
-  if (!status) throw new Error("Seventh argument must be a status ('ok' or 'fail').");
+  if (status !== "ok" && status !== "fail") throw new Error("Seventh argument must be a status ('ok' or 'fail').");
 
   const gh = new Octokit({ auth });
   let checkLogsMessage = "";
@@ -38,20 +48,11 @@ export async function main() {
       ];
       emoji = "❌";
     } else {
-      const mainErrors: Errors = [];
-      if (mainErrorsPath) {
-        const mainFiles = glob.sync(`**/*.json`, { cwd: mainErrorsPath, absolute: true });
-        for (const file of mainFiles) {
-          mainErrors.push(...(JSON.parse(readFileSync(file, "utf-8")) as Errors));
-        }
-      }
-      const branchErrors: Errors = [];
-      if (branchErrorsPath) {
-        const branchFiles = glob.sync(`**/*.json`, { cwd: branchErrorsPath, absolute: true });
-        for (const file of branchFiles) {
-          branchErrors.push(...(JSON.parse(readFileSync(file, "utf-8")) as Errors));
-        }
-      }
+      const { main: mainErrors, branch: branchErrors } = readComparison(
+        mainErrorsPath,
+        branchErrorsPath,
+        expectedShardCount,
+      );
 
       const reports = getDiffReports(mainErrors, branchErrors);
       if (reports.length) {

packages/dtslint-runner/src/validate-results.ts

@@ -0,0 +1,66 @@
+import { lstatSync, readFileSync } from "fs";
+import glob = require("glob");
+import type { Failure } from "./add-github-links";
+
+export function readComparison(
+  mainErrorsPath: string | undefined,
+  branchErrorsPath: string | undefined,
+  expectedShardCount: string | undefined,
+): { main: Failure[]; branch: Failure[] } {
+  try {
+    const shardCount = Number(expectedShardCount);
+    if (
+      !mainErrorsPath ||
+      !branchErrorsPath ||
+      !Number.isSafeInteger(shardCount) ||
+      shardCount < 1 ||
+      String(shardCount) !== expectedShardCount
+    ) {
+      throw new Error();
+    }
+    return {
+      main: readFailures(mainErrorsPath, "main", shardCount),
+      branch: readFailures(branchErrorsPath, "pr", shardCount),
+    };
+  } catch {
+    // JSON parse and filesystem errors can contain artifact-controlled command markers.
+    throw new Error("Missing or invalid DT failure artifacts.");
+  }
+}
+
+function readFailures(directory: string, prefix: "main" | "pr", shardCount: number): Failure[] {
+  const result: Failure[] = [];
+  for (let shard = 1; shard <= shardCount; shard++) {
+    const files = glob.sync(`**/${prefix}Failures${shard}.json`, { cwd: directory, absolute: true });
+    if (files.length !== 1 || !lstatSync(files[0]).isFile()) {
+      throw new Error();
+    }
+    const failures: unknown = JSON.parse(readFileSync(files[0], "utf8"));
+    if (!Array.isArray(failures) || !failures.every(isFailure)) {
+      throw new Error();
+    }
+    result.push(...failures);
+  }
+  return result;
+}
+
+function isFailure(value: unknown): value is Failure {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    "path" in value &&
+    typeof value.path === "string" &&
+    "error" in value &&
+    typeof value.error === "string"
+  );
+}
+
+if (require.main === module) {
+  const [shardCount, mainErrorsPath, branchErrorsPath] = process.argv.slice(2);
+  try {
+    readComparison(mainErrorsPath, branchErrorsPath, shardCount);
+  } catch {
+    console.error("Missing or invalid DT failure artifacts.");
+    process.exitCode = 1;
+  }
+}