From 8351328b6333efb47274e0a4a64e4428e0a87b69 Mon Sep 17 00:00:00 2001 From: Doddanna17 Date: Sat, 17 Oct 2026 21:09:06 +0530 Subject: [PATCH] fix(scripts): reconcile publish conflicts and wait for visibility npm processes publishes asynchronously (publish-time malware scanning), so a successful `npm publish` only means the upload was accepted and the version appears on the registry minutes-to-hours later. The release pipeline assumed immediate visibility, which produced three failures: a hard-coded sibling wait that expired while the version was still being processed, reruns that collided with versions already published (Rekor 409, npm E403), and runs that reported green while packages were missing. verify-release.ts now treats the registry, not the publish exit code, as the source of truth: it waits for the exact version to appear, reconciles a publish conflict when that version is already published instead of bumping it, classifies npm publish-over failures as recoverable, requires bitgo to carry shrinkwrap metadata, warns when the dist-tag has not moved, and fails the run in every mode when a package is missing. Both release workflows get explicit visibility budgets with job timeouts above them, and Verify Publish regenerates bitgo's shrinkwrap when recovery re-publishes it. Also runs the TypeScript script tests in CI: `test:scripts` only matched `*.test.js`, so the shrinkwrap and prepare-release suites never executed. Ticket: SI-2040 --- .github/workflows/npmjs-release.yml | 9 +- .github/workflows/publish.yml | 11 + package.json | 2 +- scripts/tests/verify-release.test.ts | 343 +++++++++++++++++++++++++++ scripts/verify-release.ts | 256 ++++++++++++++++---- 5 files changed, 570 insertions(+), 51 deletions(-) create mode 100644 scripts/tests/verify-release.test.ts diff --git a/.github/workflows/npmjs-release.yml b/.github/workflows/npmjs-release.yml index 0e27423d3c7..e792b8f15cb 100644 --- a/.github/workflows/npmjs-release.yml +++ b/.github/workflows/npmjs-release.yml @@ -214,7 +214,14 @@ jobs: - verify-back-merge if: ${{ always() && needs.get-release-context.result != 'failure' && needs.get-recovery-context.result != 'failure' && needs.verify-back-merge.result != 'failure' }} runs-on: ${{ vars.BASE_RUNNER_TYPE || 'ubuntu-latest' }} - timeout-minutes: 60 + # npm processes publishes asynchronously (publish-time malware scanning), so newly published + # siblings can take minutes — occasionally much longer (measured: 55 minutes for a + # @bitgo-beta/sdk-coin-tao beta) — to become visible before bitgo's shrinkwrap can be built. + # These budgets cover that tail; the job timeout must stay comfortably above them. + timeout-minutes: 180 + env: + BITGO_SIBLING_AVAILABILITY_TIMEOUT_MINUTES: 120 + BITGO_PUBLISH_VISIBILITY_TIMEOUT_MINUTES: 120 environment: npmjs-release steps: - name: Checkout repository diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9426b3ce3b3..73385672b65 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,6 +35,14 @@ jobs: # a deployment branch policy on this environment, so the job cannot run # from arbitrary PR branches even though it is manually dispatched. environment: beta-alpha-release + # npm processes publishes asynchronously (publish-time malware scanning), so a version can take + # minutes — occasionally much longer (measured: 55 minutes for @bitgo-beta/sdk-coin-tao@1.0.0-beta.1477) + # — to become visible on the registry after `npm publish` succeeds. Size both waits to cover that + # tail with headroom, and keep the job timeout comfortably above them. + timeout-minutes: 180 + env: + BITGO_SIBLING_AVAILABILITY_TIMEOUT_MINUTES: 120 + BITGO_PUBLISH_VISIBILITY_TIMEOUT_MINUTES: 120 steps: - uses: socketdev/action@4337a545deecc20f19a909e52db7a2f6ba292f42 # v1 @@ -126,3 +134,6 @@ jobs: env: NPM_CONFIG_PROVENANCE: true RECOVERY_MODE: ${{ inputs.recovery-mode }} + # A recovery re-publish of bitgo must regenerate its shrinkwrap, otherwise the published + # tarball has none and consumers silently lose the pinned workspace siblings. + BITGO_GENERATE_SHRINKWRAP: true diff --git a/package.json b/package.json index 09ac3f0d39c..4dc9bb9d695 100644 --- a/package.json +++ b/package.json @@ -321,7 +321,7 @@ "prepare-commit-msg": "node ./scripts/prepare-commit-msg.js", "test:prepare-release": "mocha --require tsx ./scripts/tests/prepareRelease/prepare-release-main.test.ts", "test:shrinkwrap": "mocha --require tsx ./scripts/tests/generate-bitgo-shrinkwrap.test.ts", - "test:scripts": "mocha './scripts/tests/**/*.test.js'" + "test:scripts": "mocha --import=tsx './scripts/tests/**/*.test.js' './scripts/tests/*.test.ts'" }, "dependencies": { "axios": "1.20.0", diff --git a/scripts/tests/verify-release.test.ts b/scripts/tests/verify-release.test.ts new file mode 100644 index 00000000000..a58f7728f04 --- /dev/null +++ b/scripts/tests/verify-release.test.ts @@ -0,0 +1,343 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, it } from 'mocha'; + +import { + getVisibilityTimeoutMs, + isRecoverablePublishConflict, + verify, + verifyPackage, + waitForVersionInfo, + type VerifyDeps, + type VersionInfo, +} from '../../scripts/verify-release'; + +/** A clock that only moves when the code under test sleeps, so polling costs no real time. */ +function fakeClock() { + let time = 0; + return { + now: () => time, + sleep: async (delayMs: number) => { + time += delayMs; + }, + }; +} + +function makeDeps(overrides: Partial = {}) { + const clock = fakeClock(); + const base: VerifyDeps = { + fetchVersionInfo: async () => ({ found: true, hasShrinkwrap: true }), + publish: async () => ({ stdout: 'published' }), + sleep: clock.sleep, + now: clock.now, + getDistTags: async () => ({}), + getLernaModules: async () => [], + visibilityTimeoutMs: 60 * 60_000, + visibilityDelayMs: 30_000, + }; + return { deps: { ...base, ...overrides }, clock }; +} + +function tempModule(name: string, version: string, dirName = 'module'): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'verify-release-')); + const dir = path.join(root, dirName); + fs.mkdirSync(dir); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify({ name, version }, null, 2) + '\n'); + return dir; +} + +function readVersion(dir: string): string { + return JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8')).version; +} + +function publishOverError(name: string, version: string): Error { + return Object.assign(new Error('execa: Command failed with exit code 1: npm publish'), { + stdout: '', + stderr: + `npm error code E403\n` + + `npm error 403 403 Forbidden - PUT https://registry.npmjs.org/${name.replace('/', '%2f')}\n` + + `npm error 403 You cannot publish over the previously published versions: ${version}.`, + }); +} + +describe('verify-release publish conflict classification', function () { + it('classifies the Rekor duplicate-provenance conflict', function () { + assert.equal(isRecoverablePublishConflict('lerna ERR! TLOG_CREATE_ENTRY_ERROR error creating tlog entry'), true); + }); + + it('classifies a 409 transparency-log conflict', function () { + assert.equal( + isRecoverablePublishConflict('Error: (409) an equivalent entry already exists in the transparency log'), + true + ); + assert.equal(isRecoverablePublishConflict('409 Conflict while writing to the transparency log'), true); + }); + + it('classifies a previously-staged-version conflict', function () { + assert.equal(isRecoverablePublishConflict('You cannot publish over a previously staged version'), true); + }); + + it('classifies npm publish-over failures', function () { + const output = + 'npm error code E403\n' + + 'npm error 403 You cannot publish over the previously published versions: 1.0.0-beta.1708.'; + assert.equal(isRecoverablePublishConflict(output), true); + assert.equal(isRecoverablePublishConflict('npm ERR! code EPUBLISHCONFLICT'), true); + }); + + it('does not classify unrelated publish failures as recoverable', function () { + assert.equal(isRecoverablePublishConflict('npm error code E401 Unable to authenticate'), false); + assert.equal(isRecoverablePublishConflict('ENOSPC no space left on device'), false); + }); +}); + +describe('verify-release visibility budget', function () { + it('defaults to one hour when unset', function () { + assert.equal(getVisibilityTimeoutMs({}), 60 * 60_000); + }); + + it('reads the budget from the environment', function () { + assert.equal(getVisibilityTimeoutMs({ BITGO_PUBLISH_VISIBILITY_TIMEOUT_MINUTES: '15' }), 15 * 60_000); + }); + + it('rejects an invalid budget', function () { + assert.throws(() => getVisibilityTimeoutMs({ BITGO_PUBLISH_VISIBILITY_TIMEOUT_MINUTES: 'soon' })); + assert.throws(() => getVisibilityTimeoutMs({ BITGO_PUBLISH_VISIBILITY_TIMEOUT_MINUTES: '-1' })); + }); +}); + +describe('waitForVersionInfo', function () { + it('returns immediately when the version is already visible', async function () { + const { deps } = makeDeps(); + let fetches = 0; + let sleeps = 0; + + const info = await waitForVersionInfo('@bitgo-beta/sdk-core', '1.2.3-beta.4', { + ...deps, + fetchVersionInfo: async () => { + fetches += 1; + return { found: true, hasShrinkwrap: true }; + }, + sleep: async () => { + sleeps += 1; + }, + }); + + assert.deepEqual(info, { found: true, hasShrinkwrap: true }); + assert.equal(fetches, 1); + assert.equal(sleeps, 0); + }); + + it('polls until the version appears, as npm publish-time processing resolves', async function () { + const { deps } = makeDeps(); + let fetches = 0; + const sleeps: number[] = []; + + const info = await waitForVersionInfo('@bitgo-beta/sdk-coin-tao', '1.0.0-beta.1477', { + ...deps, + fetchVersionInfo: async () => { + fetches += 1; + return { found: fetches >= 3, hasShrinkwrap: true }; + }, + sleep: async (delayMs) => { + sleeps.push(delayMs); + }, + }); + + assert.equal(info.found, true); + assert.equal(fetches, 3); + assert.deepEqual(sleeps, [30_000, 30_000]); + }); + + it('gives up at the budget and reports the version as not visible', async function () { + const { deps } = makeDeps({ visibilityTimeoutMs: 2 * 60_000 }); + let fetches = 0; + + const info = await waitForVersionInfo('@bitgo-beta/sdk-coin-tao', '1.0.0-beta.1477', { + ...deps, + fetchVersionInfo: async (): Promise => { + fetches += 1; + return { found: false, hasShrinkwrap: false }; + }, + }); + + assert.equal(info.found, false); + // 2 minute budget at a 30s delay: checks at 0s, 30s, 60s, 90s and 120s; the 120s check exhausts + // the budget (there is no room left for another sleep). + assert.equal(fetches, 5); + }); +}); + +describe('verifyPackage reconciliation', function () { + it('treats an already-published version as done without publishing it again', async function () { + const dir = tempModule('@bitgo-beta/sdk-core', '1.2.3-beta.4'); + const published: string[] = []; + const { deps } = makeDeps({ + publish: async (cwd) => { + published.push(cwd); + return { stdout: '' }; + }, + getDistTags: async () => ({ beta: '1.2.3-beta.4' }), + }); + + assert.equal(await verifyPackage(dir, 'beta', deps), true); + assert.deepEqual(published, []); + }); + + it('warns but does not fail when the dist-tag has not moved to the published version', async function () { + const dir = tempModule('@bitgo-beta/sdk-core', '1.2.3-beta.4'); + const { deps } = makeDeps({ getDistTags: async () => ({ beta: '1.2.3-beta.3' }) }); + + assert.equal(await verifyPackage(dir, 'beta', deps), true); + }); + + it('publishes a missing version and accepts it once the registry serves it', async function () { + const dir = tempModule('@bitgo-beta/sdk-core', '1.2.3-beta.4'); + const published: string[] = []; + let visible = false; + const { deps } = makeDeps({ + fetchVersionInfo: async () => ({ found: visible, hasShrinkwrap: true }), + publish: async () => { + published.push('publish'); + visible = true; + return { stdout: '' }; + }, + }); + + assert.equal(await verifyPackage(dir, 'beta', deps), true); + assert.deepEqual(published, ['publish']); + }); + + it('reconciles an npm publish-over conflict instead of bumping the version', async function () { + const dir = tempModule('@bitgo-beta/sdk-opensslbytes', '1.0.0-beta.1708'); + let visible = false; + let publishCalls = 0; + const { deps } = makeDeps({ + fetchVersionInfo: async () => ({ found: visible, hasShrinkwrap: true }), + publish: async () => { + publishCalls += 1; + // The registry already holds this exact version, so npm refuses the upload; the version + // becomes visible on the next read (lag), which is what the reconcile path waits for. + visible = true; + throw publishOverError('@bitgo-beta/sdk-opensslbytes', '1.0.0-beta.1708'); + }, + }); + + assert.equal(await verifyPackage(dir, 'beta', deps), true); + assert.equal(publishCalls, 1); + // The version must not be bumped: the release's own version is already published. + assert.equal(readVersion(dir), '1.0.0-beta.1708'); + }); + + it('fails the package when publish reports success but the version never becomes visible', async function () { + const dir = tempModule('@bitgo-beta/sdk-core', '1.2.3-beta.4'); + const { deps } = makeDeps({ + visibilityTimeoutMs: 0, + fetchVersionInfo: async () => ({ found: false, hasShrinkwrap: false }), + publish: async () => ({ stdout: '' }), + }); + + assert.equal(await verifyPackage(dir, 'beta', deps), false); + }); + + it('fails the package on an unrelated publish error', async function () { + const dir = tempModule('@bitgo-beta/sdk-core', '1.2.3-beta.4'); + const { deps } = makeDeps({ + fetchVersionInfo: async () => ({ found: false, hasShrinkwrap: false }), + publish: async () => { + throw Object.assign(new Error('npm publish failed'), { stderr: 'npm error code E401' }); + }, + }); + + assert.equal(await verifyPackage(dir, 'beta', deps), false); + }); + + it('accepts a bitgo version only when it carries shrinkwrap metadata', async function () { + const dir = tempModule('@bitgo-beta/bitgo', '14.2.1-beta.2203', 'bitgo'); + const { deps } = makeDeps({ + fetchVersionInfo: async () => ({ found: true, hasShrinkwrap: true }), + }); + + assert.equal(await verifyPackage(dir, 'beta', deps), true); + }); + + it('re-publishes a bitgo version that is on npm without shrinkwrap metadata', async function () { + const dir = tempModule('@bitgo-beta/bitgo', '14.2.1-beta.2203', 'bitgo'); + let published = false; + const { deps } = makeDeps({ + fetchVersionInfo: async () => ({ found: true, hasShrinkwrap: published }), + publish: async () => { + published = true; + return { stdout: '' }; + }, + }); + + assert.equal(await verifyPackage(dir, 'beta', deps), true); + assert.equal(published, true); + }); + + it('skips private modules other than bitgo', async function () { + const dir = tempModule('@bitgo-beta/private-tool', '1.0.0', 'private-tool'); + const pkgPath = path.join(dir, 'package.json'); + fs.writeFileSync( + pkgPath, + JSON.stringify({ name: '@bitgo-beta/private-tool', version: '1.0.0', private: true }, null, 2) + '\n' + ); + + const { deps } = makeDeps({ + fetchVersionInfo: async () => { + throw new Error('should not be queried'); + }, + }); + + assert.equal(await verifyPackage(dir, 'beta', deps), true); + }); +}); + +describe('verify fail-closed behaviour', function () { + it('fails the run when a package could not be published, in normal mode too', async function () { + const good = tempModule('@bitgo-beta/sdk-core', '1.2.3-beta.4', 'sdk-core'); + const bad = tempModule('@bitgo-beta/sdk-coin-tao', '1.0.0-beta.1477', 'sdk-coin-tao'); + const previousExitCode = process.exitCode; + + try { + process.exitCode = undefined; + const { deps } = makeDeps({ + getLernaModules: async () => [{ location: good }, { location: bad }], + fetchVersionInfo: async (name: string) => ({ + found: name === '@bitgo-beta/sdk-core', + hasShrinkwrap: true, + }), + publish: async () => { + throw Object.assign(new Error('npm publish failed'), { stderr: 'npm error code E401' }); + }, + }); + + const failed = await verify('beta', deps); + + assert.deepEqual(failed, ['sdk-coin-tao']); + assert.equal(process.exitCode, 1); + } finally { + process.exitCode = previousExitCode; + } + }); + + it('reports no failures and leaves the exit code untouched when everything is published', async function () { + const dir = tempModule('@bitgo-beta/sdk-core', '1.2.3-beta.4', 'sdk-core'); + const previousExitCode = process.exitCode; + + try { + process.exitCode = undefined; + const { deps } = makeDeps({ getLernaModules: async () => [{ location: dir }] }); + + const failed = await verify('beta', deps); + + assert.deepEqual(failed, []); + assert.equal(process.exitCode, undefined); + } finally { + process.exitCode = previousExitCode; + } + }); +}); diff --git a/scripts/verify-release.ts b/scripts/verify-release.ts index 4b74d9274f9..e345ae1edaa 100644 --- a/scripts/verify-release.ts +++ b/scripts/verify-release.ts @@ -5,58 +5,202 @@ import path from 'path'; import { inc } from 'semver'; import { getLernaModules, getDistTags } from './prepareRelease'; -let lernaModuleLocations: string[] = []; - const RECOVERY_MODE = process.env.RECOVERY_MODE === 'true'; const MAX_RECOVERY_ATTEMPTS = 5; -async function getLernaModuleLocations(): Promise { - const modules = await getLernaModules(); - lernaModuleLocations = modules.map(({ location }) => location); +/** + * npm processes publishes asynchronously (publish-time malware scanning, since 2026-07-28): + * a successful `npm publish` only means the upload was accepted, and the version becomes + * visible on the registry later — minutes in the normal case, hours at the tail. Every + * decision below therefore waits for the version to actually appear rather than trusting + * the publish exit code, and "already published" is reconciled instead of re-published. + */ +const visibilityTimeoutEnvVar = 'BITGO_PUBLISH_VISIBILITY_TIMEOUT_MINUTES'; +const defaultVisibilityTimeoutMs = 60 * 60_000; +const visibilityDelayMs = 30_000; +const npmRegistryUrl = 'https://registry.npmjs.org'; + +export type VersionInfo = { + /** Whether the registry serves this exact version. */ + found: boolean; + /** Only meaningful for `bitgo`; whether the published version carries shrinkwrap metadata. */ + hasShrinkwrap: boolean; +}; + +export type FetchVersionInfo = (name: string, version: string) => Promise; +export type Publish = (cwd: string, preid: string) => Promise<{ stdout: string }>; +export type Sleep = (delayMs: number) => Promise; + +const defaultFetchVersionInfo: FetchVersionInfo = async (name, version) => { + const url = `${npmRegistryUrl}/${encodeURIComponent(name)}/${encodeURIComponent(version)}`; + const response = await fetch(url); + if (response.status === 404 || response.status === 410) { + return { found: false, hasShrinkwrap: false }; + } + if (!response.ok) { + throw new Error(`Failed ${url}: ${response.status} ${response.statusText}`); + } + const body = (await response.json()) as { _hasShrinkwrap?: unknown }; + return { found: true, hasShrinkwrap: body._hasShrinkwrap === true }; +}; + +const defaultPublish: Publish = async (cwd, preid) => { + const result = await execa('npm', ['publish', '--tag', preid, '--provenance'], { cwd }); + return { stdout: result.stdout ?? '' }; +}; + +export function getVisibilityTimeoutMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env[visibilityTimeoutEnvVar]; + if (raw === undefined || raw.trim() === '') return defaultVisibilityTimeoutMs; + const minutes = Number(raw); + if (!Number.isFinite(minutes) || minutes < 0) { + throw new Error(`${visibilityTimeoutEnvVar} must be zero or more minutes, got "${raw}".`); + } + return minutes * 60_000; } -function isRecoverablePublishConflict(output: string): boolean { +export type VerifyDeps = { + fetchVersionInfo: FetchVersionInfo; + publish: Publish; + sleep: Sleep; + now: () => number; + getDistTags: (packageName: string) => Promise>; + getLernaModules: () => Promise>; + visibilityTimeoutMs: number; + visibilityDelayMs: number; +}; + +export const defaultVerifyDeps: VerifyDeps = { + fetchVersionInfo: defaultFetchVersionInfo, + publish: defaultPublish, + sleep: (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)), + now: Date.now, + getDistTags, + getLernaModules, + visibilityTimeoutMs: getVisibilityTimeoutMs(), + visibilityDelayMs, +}; + +/** + * Polls until the exact version is served by the registry, or the budget runs out. + * Returns the last observation, so callers decide what a timeout means. + */ +export async function waitForVersionInfo( + name: string, + version: string, + deps: Pick< + VerifyDeps, + 'fetchVersionInfo' | 'sleep' | 'now' | 'visibilityTimeoutMs' | 'visibilityDelayMs' + > = defaultVerifyDeps +): Promise { + const startedAt = deps.now(); + let checks = 0; + for (;;) { + checks += 1; + const info = await deps.fetchVersionInfo(name, version); + if (info.found) { + if (checks > 1) { + console.log(`${name}@${version} became visible on npm after ${Math.round((deps.now() - startedAt) / 1000)}s.`); + } + return info; + } + console.log( + `${name}@${version} is not visible on npm yet (check ${checks}, ` + + `${Math.round((deps.now() - startedAt) / 1000)}s elapsed).` + ); + const elapsedMs = deps.now() - startedAt; + if (elapsedMs + deps.visibilityDelayMs > deps.visibilityTimeoutMs) { + console.warn( + `${name}@${version} is still not visible after ${Math.round(elapsedMs / 1000)}s ` + + `(budget ${Math.round(deps.visibilityTimeoutMs / 1000)}s); npm publish-time processing can take longer.` + ); + return info; + } + await deps.sleep(deps.visibilityDelayMs); + } +} + +export function isRecoverablePublishConflict(output: string): boolean { return ( output.includes('TLOG_CREATE_ENTRY_ERROR') || output.includes('previously staged version') || (output.includes('409 Conflict') && output.includes('transparency log')) || - (output.includes('(409)') && output.includes('transparency log')) + (output.includes('(409)') && output.includes('transparency log')) || + // npm rejects a publish whose exact version already exists on the registry. + output.includes('EPUBLISHCONFLICT') || + output.includes('cannot publish over the previously published versions') || + (output.includes('E403') && output.includes('previously published')) ); } +/** + * A published `bitgo` without shrinkwrap metadata is unusable by consumers (npm only honors + * a shipped shrinkwrap when the registry reports `_hasShrinkwrap`), so it does not count as + * published and must be re-published. + */ +function isUsablePublishedVersion(name: string, version: string, info: VersionInfo, isBitgo: boolean): boolean { + if (!info.found) { + console.log(`${name}@${version} is not visible on npm yet.`); + return false; + } + if (isBitgo && !info.hasShrinkwrap) { + console.warn(`${name}@${version} is on npm but without shrinkwrap metadata.`); + return false; + } + return true; +} + // retries a stuck package's publish, bumping its version each time to sidestep an already-logged Rekor entry for that exact tarball digest async function publishWithRecovery( cwd: string, json: any, preid: string, -): Promise<{ stdout: string; exitCode: number }> { + deps: VerifyDeps = defaultVerifyDeps +): Promise<{ stdout: string }> { + const isBitgo = path.basename(cwd) === 'bitgo'; const maxAttempts = RECOVERY_MODE ? MAX_RECOVERY_ATTEMPTS : 1; for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { - return await execa( - 'npm', - ['publish', '--tag', preid, '--provenance'], - { cwd }, - ); + const { stdout } = await deps.publish(cwd, preid); + console.log(stdout); + // A successful publish only means the upload was accepted; confirm the registry serves it. + const info = await waitForVersionInfo(json.name, json.version, deps); + if (!isUsablePublishedVersion(json.name, json.version, info, isBitgo)) { + throw new Error( + `${json.name}@${json.version}: npm reported success but the version is not usable on the registry yet.` + ); + } + return { stdout }; } catch (e: any) { - const output = `${e.stdout ?? ''}\n${e.stderr ?? ''}`; - if (!RECOVERY_MODE || !isRecoverablePublishConflict(output)) { + const output = `${e.stdout ?? ''}\n${e.stderr ?? ''}\n${e.message ?? ''}`; + if (!isRecoverablePublishConflict(output)) { throw e; } + // Reconcile first: the conflict usually means a previous attempt already published this + // exact version and the registry had not caught up when we looked. + const info = await waitForVersionInfo(json.name, json.version, deps); + if (isUsablePublishedVersion(json.name, json.version, info, isBitgo)) { + console.log( + `${json.name}@${json.version} is already published and usable — treating the publish conflict as resolved.` + ); + return { stdout: output }; + } if (attempt === maxAttempts) { throw new Error( - `${json.name}: still hitting a publish conflict after ${maxAttempts} version bumps`, + `${json.name}@${json.version} could not be published: the version is taken but not usable` + + (maxAttempts > 1 + ? ` after ${maxAttempts} version bumps.` + : '. Re-run with recovery-mode to bump and retry.') ); } const next = inc(json.version, 'prerelease', undefined, preid); assert(typeof next === 'string', `Failed to increment version for ${json.name}`); json.version = next; - writeFileSync( - path.join(cwd, 'package.json'), - JSON.stringify(json, null, 2) + '\n', - ); + writeFileSync(path.join(cwd, 'package.json'), JSON.stringify(json, null, 2) + '\n'); console.warn( - `${json.name}: publish conflict, retrying with bumped version ${json.version} (attempt ${attempt + 1}/${maxAttempts})`, + `${json.name}: publish conflict, retrying with bumped version ${json.version} (attempt ${ + attempt + 1 + }/${maxAttempts})` ); } } @@ -64,53 +208,67 @@ async function publishWithRecovery( throw new Error(`${json.name}: exhausted publish attempts`); } -async function verifyPackage(dir: string, preid = 'beta'): Promise { +export async function verifyPackage( + dir: string, + preid = 'beta', + deps: VerifyDeps = defaultVerifyDeps +): Promise { const cwd = dir; - const json = JSON.parse( - readFileSync(path.join(cwd, 'package.json'), { encoding: 'utf-8' }), - ); + const json = JSON.parse(readFileSync(path.join(cwd, 'package.json'), { encoding: 'utf-8' })); // bitgo is private but still published via lerna --include-private. if (json.private && path.basename(cwd) !== 'bitgo') { return true; } + const isBitgo = path.basename(cwd) === 'bitgo'; + try { - const distTags = await getDistTags(json.name); - if (json.version !== distTags[preid]) { - console.log( - `${json.name} missing. Expected ${json.version}, latest is ${distTags[preid]}`, - ); - const { stdout, exitCode } = await publishWithRecovery(cwd, json, preid); - console.log(stdout); - return exitCode === 0; - } else { + const info = await waitForVersionInfo(json.name, json.version, deps); + if (isUsablePublishedVersion(json.name, json.version, info, isBitgo)) { console.log(`${json.name} matches expected version ${json.version}`); + // The version is the source of truth; a dist-tag that has not moved is worth flagging because + // consumers resolve `@` through it, but it must not fail an otherwise complete publish. + try { + const distTags = await deps.getDistTags(json.name); + if (distTags[preid] !== json.version) { + console.warn( + `${json.name}: ${json.version} is published but the "${preid}" dist-tag points at ` + + `${distTags[preid] ?? 'nothing'} — consumers using @${preid} will not get this release.` + ); + } + } catch (tagError) { + console.warn(`Failed to check the "${preid}" dist-tag for ${json.name}`, tagError); + } + return true; } + + console.log(`${json.name} missing. Expected ${json.version}`); + await publishWithRecovery(cwd, json, preid, deps); return true; } catch (e) { - console.warn(`Failed to fetch dist tags for ${json.name}`, e); + console.warn(`Failed to verify ${json.name}`, e); return false; } } -async function verify(preid?: string) { - await getLernaModuleLocations(); - let anyFailed = false; - for (let i = 0; i < lernaModuleLocations.length; i++) { - const dir = lernaModuleLocations[i]; - if (!(await verifyPackage(dir, preid))) { - console.error('Failed to verify outstanding packages.'); - if (!RECOVERY_MODE) { - return; - } - // keep bumping/retrying the rest of the packages instead of aborting on the first stuck one - anyFailed = true; +export async function verify(preid?: string, deps: VerifyDeps = defaultVerifyDeps): Promise { + const modules = await deps.getLernaModules(); + const failed: string[] = []; + // Check every package before failing so the log lists everything that is missing, not just the first. + for (const { location: dir } of modules) { + if (!(await verifyPackage(dir, preid, deps))) { + failed.push(path.basename(dir)); } } - if (anyFailed) { + if (failed.length > 0) { + console.error(`Failed to verify outstanding packages: ${failed.join(', ')}`); + // Fail closed in every mode: a run must never be green while packages are missing from npm. process.exitCode = 1; } + return failed; } // e.g. for alpha releases: `npx tsx ./scripts/verify-release.ts alpha` -verify(process.argv.slice(2)[0]); +if (require.main === module) { + verify(process.argv.slice(2)[0]); +}