Faster website build

.github/actions/website-cache/action.yml

@@ -0,0 +1,25 @@
+name: Restore website build work
+description: Reuse compiled examples and Webpack work without caching generated pages
+
+outputs:
+  key:
+    description: Primary key to use when checkpointing the completed build
+    value: ${{ steps.restore.outputs.cache-primary-key }}
+
+runs:
+  using: composite
+  steps:
+    - name: Get Node version
+      id: node
+      shell: bash
+      run: echo "version=$(node --version)" >> "$GITHUB_OUTPUT"
+
+    - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+      id: restore
+      with:
+        path: |
+          node_modules/.cache/twoslash
+          packages/typescriptlang-org/.cache/webpack
+        key: website-v2-${{ runner.os }}-${{ runner.arch }}-${{ steps.node.outputs.version }}-${{ github.run_id }}-${{ github.run_attempt }}
+        restore-keys: |
+          website-v2-${{ runner.os }}-${{ runner.arch }}-${{ steps.node.outputs.version }}-

.github/actions/website-cache/prune-twoslash-cache.js

@@ -0,0 +1,51 @@
+const fs = require("fs")
+const path = require("path")
+
+const MAX_BYTES = 64 * 1024 * 1024
+
+function pruneTwoslashCache(directory) {
+  let entries
+  try {
+    if (!fs.lstatSync(directory).isDirectory()) {
+      throw new Error(`Expected a Twoslash cache directory: ${directory}`)
+    }
+    entries = fs.readdirSync(directory, { withFileTypes: true })
+  } catch (error) {
+    if (error.code !== "ENOENT") throw error
+    return { bytes: 0, removed: 0, removedBytes: 0 }
+  }
+
+  const files = entries.map(entry => {
+    if (!entry.isFile() || !/^[a-f0-9]{40}\.json$/.test(entry.name)) {
+      throw new Error(`Unexpected Twoslash cache entry: ${entry.name}`)
+    }
+    const filename = path.join(directory, entry.name)
+    const { size, mtimeMs } = fs.statSync(filename)
+    return { filename, size, mtimeMs }
+  })
+  files.sort((a, b) => a.mtimeMs - b.mtimeMs || a.filename.localeCompare(b.filename))
+
+  let bytes = files.reduce((total, file) => total + file.size, 0)
+  let removed = 0
+  let removedBytes = 0
+  for (const file of files) {
+    if (bytes <= MAX_BYTES) break
+    fs.unlinkSync(file.filename)
+    bytes -= file.size
+    removedBytes += file.size
+    removed++
+  }
+  return { bytes, removed, removedBytes }
+}
+
+module.exports = { MAX_BYTES, pruneTwoslashCache }
+
+if (require.main === module) {
+  const directory = path.resolve(__dirname, "../../../node_modules/.cache/twoslash")
+  const { bytes, removed, removedBytes } = pruneTwoslashCache(directory)
+  console.log(
+    `Twoslash cache: ${(bytes / 1024 / 1024).toFixed(2)} MiB retained; ` +
+      `removed ${removed} entries (${(removedBytes / 1024 / 1024).toFixed(2)} MiB). ` +
+      `Limit: ${MAX_BYTES / 1024 / 1024} MiB.`
+  )
+}

.github/actions/website-cache/save/action.yml

@@ -0,0 +1,28 @@
+name: Save website build work
+description: Prune least-recently-used compiled examples and checkpoint the successful build
+
+inputs:
+  key:
+    description: Primary key returned by the website cache restore action
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Prune Twoslash cache to 64 MiB
+      shell: bash
+      env:
+        CACHE_KEY: ${{ inputs.key }}
+      run: |
+        if [[ -z "$CACHE_KEY" ]]; then
+          echo "::error::Missing website cache restore key"
+          exit 1
+        fi
+        node "$GITHUB_ACTION_PATH/../prune-twoslash-cache.js"
+
+    - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
+      with:
+        path: |
+          node_modules/.cache/twoslash
+          packages/typescriptlang-org/.cache/webpack
+        key: ${{ inputs.key }}

.github/workflows/CI.yml

@@ -11,7 +11,7 @@
     shell: bash
 
 jobs:
-  tests:
+  site:
     strategy:
       fail-fast: false
       matrix:
@@ -24,53 +24,111 @@
 
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version: "20.x"
-      - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+          cache: pnpm
 
-      - run: pnpm install
+      - run: pnpm install --frozen-lockfile
 
       # Grab localizations
       - run: pnpm docs-sync pull microsoft/TypeScript-Website-localizations#main 1
 
       # Build the packages
       - run: pnpm bootstrap
-      - run: pnpm build
+
+      - name: Restore website build caches
+        id: website-cache
+        uses: ./.github/actions/website-cache
 
       # Verify it compiles
       - run: pnpm build-site
+
+      - name: Save website build caches
+        uses: ./.github/actions/website-cache/save
+        with:
+          key: ${{ steps.website-cache.outputs.key }}
 
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: github.event_name == 'pull_request' && matrix.os == 'ubuntu-latest'
         with:
           name: site
           path: packages/typescriptlang-org/public
 
-      # Run all the package's tests
-      - run: pnpm test
+      # Validate documentation and types generated by Gatsby
+      - run: pnpm test:ci:site
+
+      - name: Save website validation caches
+        uses: ./.github/actions/website-cache/save
+        with:
+          key: ${{ steps.website-cache.outputs.key }}-tests
+
+      - run: |
+          git add .
+          if ! git diff --staged --exit-code --quiet; then
+            echo "This PR is missing some generated changes. Please update locally or merge the patch artifact."
+            echo ""
+            git diff --staged
+            git diff --staged > missing.patch
+            exit 1
+          fi
+        name: Check for uncommitted changes
+        id: check-diff
+        if: github.event_name == 'pull_request'
+
+      - name: Upload diff artifact
+        if: ${{ failure() && steps.check-diff.conclusion == 'failure' }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: missing.patch-site-${{ matrix.os }}
+          path: missing.patch
+
+  tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "20.x"
+          cache: pnpm
+
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm docs-sync pull microsoft/TypeScript-Website-localizations#main 1
+      - run: pnpm bootstrap
+      - run: pnpm test:ci:packages
 
       - run: |
           git add .
           if ! git diff --staged --exit-code --quiet; then
             echo "This PR is missing some generated changes. Please update locally or merge the patch artifact."
             echo ""
             git diff --staged
             git diff --staged > missing.patch
             exit 1
           fi
         name: Check for uncommitted changes
         id: check-diff
         if: github.event_name == 'pull_request'
 
       - name: Upload diff artifact
         if: ${{ failure() && steps.check-diff.conclusion == 'failure' }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: missing.patch
+          name: missing.patch-packages-${{ matrix.os }}
           path: missing.patch
 
   changesets:
     name: changesets
     runs-on: ubuntu-latest
     steps:
@@ -103,6 +161,7 @@
     runs-on: ubuntu-latest
     if: ${{ always() }}
     needs:
+      - site
       - tests
       - changesets
 

.github/workflows/deploy-prod-static.yml

@@ -31,33 +31,44 @@ jobs:
         with:
           # Fetch the full history, to build attribution.json
           fetch-depth: 0
+      - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+
       - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
         with:
           node-version: "20.x"
+          cache: pnpm
 
-      - uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Pull localizations
+        run: pnpm docs-sync pull microsoft/TypeScript-Website-localizations#main 1
 
-      # Builds the modules, and boostraps the other modules
-      - name: Build website
-        run: |
-          pnpm install
-          pnpm docs-sync pull microsoft/TypeScript-Website-localizations#main 1
-          pnpm bootstrap
-          pnpm run --filter=typescriptlang-org setup-playground-cache-bust
-          pnpm build
+      - name: Build packages
+        run: pnpm bootstrap
 
-      - name: Makes the site
-        run: |
-          pnpm build-site
-          cp -r packages/typescriptlang-org/public site
+      - name: Restore website build caches
+        id: website-cache
+        uses: ./.github/actions/website-cache
+
+      - name: Set up playground cache busting
+        run: pnpm run --filter=typescriptlang-org setup-playground-cache-bust
+
+      - name: Build static site
+        run: pnpm build-site
+
+      - name: Save website build caches
+        uses: ./.github/actions/website-cache/save
+        with:
+          key: ${{ steps.website-cache.outputs.key }}
 
       - name: Setup Pages
         uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v3
         with:
-          path: './site'
+          path: packages/typescriptlang-org/public
 
   deploy:
     environment:

.github/workflows/publish-packages.yml

@@ -35,7 +35,6 @@ jobs:
         run: |
           pnpm install
           pnpm bootstrap
-          pnpm build
 
       - uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
         with:

package.json

@@ -41,7 +41,8 @@
       "webpack": "^5.105.3"
     },
     "patchedDependencies": {
-      "[email protected]": "patches/[email protected]"
+      "[email protected]": "patches/[email protected]",
+      "[email protected]": "patches/[email protected]"
     },
     "ignoredBuiltDependencies": [
       "@parcel/watcher",
@@ -67,7 +68,7 @@
   },
   "scripts": {
     "ci:publish": "pnpm changeset git-tag && pnpm publish -r",
-    "bootstrap": "pnpm -r run bootstrap && BOOTSTRAPPING=true pnpm -r run build",
+    "bootstrap": "pnpm -r --filter=!@typescript/ata --filter=!@typescript/vfs --filter=!@typescript/twoslash --filter=!playground-examples --filter=!tsconfig-reference run bootstrap && pnpm build",
     "start": "concurrently -p \"[{name}]\" -n \"BUILD,SITE\" -c \"bgBlue.bold,bgMagenta.bold\" \"node watcher.js\" \"pnpm run --filter=typescriptlang-org start\"",
     "build": "pnpm run --filter=!typescriptlang-org build",
     "build-site": "pnpm run --filter=typescriptlang-org build",
@@ -76,6 +77,9 @@
     "clean": "pnpm run --filter=typescriptlang-org gatsby clean",
     "clean-twoslash": "rm -rf packages/.cache/twoslash",
     "test": "CI=true pnpm -r run test",
+    "test:ci": "pnpm test:ci:site && pnpm test:ci:packages",
+    "test:ci:site": "CI=true pnpm -r --filter=glossary --filter=tsconfig-reference run lint && CI=true pnpm --filter=typescriptlang-org test",
+    "test:ci:packages": "CI=true pnpm -r --filter=!root --filter=!typescriptlang-org --filter=!glossary --filter=!tsconfig-reference --no-sort run test",
     "update-test-snapshots": "CI=true pnpm run --filter=@typescript/twoslash --filter=@typescript/vfs test -u"
   },
   "repository": "microsoft/TypeScript-Website",

packages/tsconfig-reference/package.json

@@ -12,7 +12,7 @@
     "generate:md:tsconfig": "tsc --build && node scripts/tsconfig/generateMarkdown",
     "generate:md:cli": "tsc --build && node scripts/cli/generateMarkdown",
     "generate:md:msbuild": "tsc --build && node scripts/msbuild/generateMarkdown",
-    "generate:md": "pnpm generate:md:tsconfig && pnpm generate:md:cli && pnpm generate:md:tsconfig",
+    "generate:md": "pnpm generate:md:tsconfig && pnpm generate:md:cli",
     "download": "node scripts/schema/downloadSchemaBase.js",
     "bootstrap": "pnpm generate:json",
     "build": "pnpm generate:json && pnpm generate:md",