From c66a1f0f395a1abf5f39778150eef573f7717f11 Mon Sep 17 00:00:00 2001 From: ashtonanderson Date: Thu, 5 Nov 2026 03:44:12 +0000 Subject: [PATCH 1/2] feat(analytics): what visitors wait for on the analysis page, and when it dies Three PostHog events, so that round 10 and later changes can be judged on real visitors, by engine, build, device and release: - analysis_engine_load: the first position of a visit (Maia, the chart, Stockfish's first depth, d12, d18, lines settled, search done). - analysis_engine_steps: a summary of the positions moved to after it, sent when the page is hidden or left: medians, and how many reached each stage. d18 now means the best line, so "lines settled" is what compares across round 10. - analysis_page_ended_unexpectedly: the last analysis page in this tab was never left, as when a phone runs out of memory, and whether it was on screen. stockfish_engine_ready and _failed also give the engine and its memory retries, and Maia's start is reported as maia_engine_ready and _failed, with its restarts. Nothing reads the page: a few timestamps per engine update. Co-Authored-By: Claude Opus 5.5 --- src/components/Analysis/MovesByRating.tsx | 7 +- src/constants/analysis.ts | 2 + src/contexts/MaiaEngineContext.tsx | 21 ++ src/contexts/StockfishEngineContext.tsx | 5 +- .../useEngineAnalysis.ts | 20 ++ src/lib/analysisTiming.test.ts | 181 ++++++++++++ src/lib/analysisTiming.ts | 271 ++++++++++++++++++ src/lib/analytics.ts | 49 +++- src/lib/engine/maia.ts | 5 + src/lib/engine/relaxedSimd.ts | 10 + src/lib/engine/stockfish.ts | 19 +- src/types/analysis.ts | 2 + 12 files changed, 576 insertions(+), 16 deletions(-) create mode 100644 src/lib/analysisTiming.test.ts create mode 100644 src/lib/analysisTiming.ts create mode 100644 src/lib/engine/relaxedSimd.ts diff --git a/src/components/Analysis/MovesByRating.tsx b/src/components/Analysis/MovesByRating.tsx index 5e9b16d9..017ab233 100644 --- a/src/components/Analysis/MovesByRating.tsx +++ b/src/components/Analysis/MovesByRating.tsx @@ -1,5 +1,6 @@ import dynamic from 'next/dynamic' -import { memo, useMemo, useState } from 'react' +import { memo, useEffect, useMemo, useState } from 'react' +import { analysisChartDrawn } from 'src/lib/analysisTiming' import { ColorSanMapping } from 'src/types' import { useContentStable } from 'src/hooks/useContentStable' @@ -35,6 +36,10 @@ const MovesByRatingChart: React.FC = ({ setShown({ moves, colorSanMapping, positionKey }) } + useEffect(() => { + if (shown.moves?.length) analysisChartDrawn(shown.positionKey) + }, [shown]) + const moveKeys = useMemo(() => { if (!shown.moves?.length) return [] return Object.keys(shown.moves[0]).filter((move) => move !== 'rating') diff --git a/src/constants/analysis.ts b/src/constants/analysis.ts index f0b199cc..cbf6d9e0 100644 --- a/src/constants/analysis.ts +++ b/src/constants/analysis.ts @@ -17,6 +17,8 @@ export const MOVE_CLASSIFICATION_THRESHOLDS = { GOOD_THRESHOLD: -0.05, } as const +// The build the analysis page runs, on PostHog's engine events. +export const STOCKFISH_ENGINE = 'stockfish-19-smallnet' as const export const DEFAULT_MAIA_MODEL = 'maia_kdd_1500' as const export const MAIA_MIN_RATING = 600 as const export const MAIA_MAX_RATING = 2600 as const diff --git a/src/contexts/MaiaEngineContext.tsx b/src/contexts/MaiaEngineContext.tsx index 735171d4..b82dbe56 100644 --- a/src/contexts/MaiaEngineContext.tsx +++ b/src/contexts/MaiaEngineContext.tsx @@ -1,3 +1,4 @@ +import { trackMaiaEngineFailed, trackMaiaEngineReady } from 'src/lib/analytics' import Maia from '../lib/engine/maia' import { MaiaStatus, MaiaEngine } from 'src/types' import React, { @@ -63,6 +64,26 @@ export const MaiaEngineContextProvider: React.FC<{ children: ReactNode }> = ({ return await maia.clearStorage() }, [maia]) + // How Maia's start went, once a page (a download doesn't count as a start). + const startedAt = useRef(Date.now()) + const startReported = useRef(false) + useEffect(() => { + if (status === 'no-cache' || status === 'downloading') { + startReported.current = true + } + if (startReported.current || (status !== 'ready' && status !== 'error')) { + return + } + startReported.current = true + const page = window.location.pathname.split('/')[1] || 'home' + const seconds = (Date.now() - startedAt.current) / 1000 + if (status === 'ready') { + trackMaiaEngineReady(seconds, page, maia.restartCount) + } else { + trackMaiaEngineFailed(error ?? '', seconds, page, maia.restartCount) + } + }, [status, error, maia]) + // Toast notifications for Maia model status useEffect(() => { return () => { diff --git a/src/contexts/StockfishEngineContext.tsx b/src/contexts/StockfishEngineContext.tsx index 4576a543..ee8ac0ca 100644 --- a/src/contexts/StockfishEngineContext.tsx +++ b/src/contexts/StockfishEngineContext.tsx @@ -17,7 +17,7 @@ import { STOCKFISH_DEBUG_RERUN_EVENT, STOCKFISH_DEBUG_RERUN_KEY, } from 'src/constants/analysis' -import Engine from 'src/lib/engine/stockfish' +import Engine, { stockfishMemoryRetries } from 'src/lib/engine/stockfish' import { trackStockfishEngineFailed, trackStockfishEngineReady, @@ -112,12 +112,13 @@ const reportEngineOutcome = (engine: Engine, status: StockfishStatus) => { const page = window.location.pathname.split('/')[1] || 'home' const seconds = (Date.now() - startedAt) / 1000 if (status === 'ready') { - trackStockfishEngineReady(seconds, page) + trackStockfishEngineReady(seconds, page, stockfishMemoryRetries()) } else { trackStockfishEngineFailed( engine.initializationErrorDetail ?? engine.initializationError ?? '', seconds, page, + stockfishMemoryRetries(), ) } } diff --git a/src/hooks/useAnalysisController/useEngineAnalysis.ts b/src/hooks/useAnalysisController/useEngineAnalysis.ts index 91887f29..bf7d4946 100644 --- a/src/hooks/useAnalysisController/useEngineAnalysis.ts +++ b/src/hooks/useAnalysisController/useEngineAnalysis.ts @@ -1,6 +1,12 @@ import { Chess } from 'chess.ts' import { fetchOpeningBookMoves } from 'src/api' import { startTransition, useEffect, useContext, useRef, useState } from 'react' +import { + analysisMaiaStage, + analysisPositionShown, + analysisStockfishUpdate, + startAnalysisTiming, +} from 'src/lib/analysisTiming' import { MAIA_MODELS, MAIA_RATINGS } from 'src/constants/common' import { STOCKFISH_DEBUG_RERUN_EVENT, @@ -87,6 +93,18 @@ export const useEngineAnalysis = ( } }, []) + useEffect(() => { + if (!enabled || !window.location.pathname.startsWith('/analysis')) return + return startAnalysisTiming({ isPhone: isPhone() }) + }, [enabled]) + + // Timed before the engine effects below, for the position they start on. + useEffect(() => { + // Full-game analysis moves through the game by itself, at other depths. + if (!currentNode || !enabled || targetDepth !== 18) return + analysisPositionShown(currentNode.fen) + }, [currentNode, enabled, targetDepth]) + async function inferenceMaiaModel( board: Chess, models: string[], @@ -239,6 +257,7 @@ export const useEngineAnalysis = ( { ...currentNode.analysis.maia, ...maiaEvaluations }, currentMaiaModel, ) + analysisMaiaStage(nodeFen, stages.indexOf(stage)) if (stage === stages[0]) { setAnalysisState((state) => state + 1) } else { @@ -380,6 +399,7 @@ export const useEngineAnalysis = ( break } nodeForAnalysis.addStockfishAnalysis(evaluation, currentMaiaModel) + analysisStockfishUpdate(nodeForAnalysis.fen, evaluation) // The first update renders at once, later ones at most every // STOCKFISH_RENDER_INTERVAL_MS (the last one always renders). const wait = diff --git a/src/lib/analysisTiming.test.ts b/src/lib/analysisTiming.test.ts new file mode 100644 index 00000000..6a1a91da --- /dev/null +++ b/src/lib/analysisTiming.test.ts @@ -0,0 +1,181 @@ +import { + analysisChartDrawn, + analysisMaiaStage, + analysisPositionShown, + analysisStockfishUpdate, + startAnalysisTiming, +} from './analysisTiming' +import { capture } from 'src/lib/posthogClient' + +jest.mock('src/lib/posthogClient', () => ({ capture: jest.fn() })) + +const captured = capture as jest.Mock +const events = (name: string) => + captured.mock.calls.filter(([event]) => event === name).map(([, p]) => p) + +const A = 'a' +const B = 'b' +const C = 'c' + +const setVisibility = (state: 'visible' | 'hidden') => { + Object.defineProperty(document, 'visibilityState', { + configurable: true, + get: () => state, + }) + document.dispatchEvent(new Event('visibilitychange')) +} + +const search = (fen: string) => { + analysisStockfishUpdate(fen, { depth: 6, partial: true }) + analysisMaiaStage(fen, 0) + analysisStockfishUpdate(fen, { depth: 18, partial: true }) + analysisMaiaStage(fen, 1) + analysisChartDrawn(fen) + analysisStockfishUpdate(fen, { depth: 18, partial: true, lines_done: true }) + analysisStockfishUpdate(fen, { depth: 18 }) +} + +describe('analysisTiming', () => { + let now = 0 + let stop: () => void + + beforeEach(() => { + now = 1000 + captured.mockClear() + jest.spyOn(performance, 'now').mockImplementation(() => (now += 100)) + sessionStorage.clear() + setVisibility('visible') + stop = startAnalysisTiming({ isPhone: true }) + }) + + afterEach(() => { + stop() + jest.restoreAllMocks() + }) + + it('reports the first position as a load, once it ends', () => { + analysisPositionShown(A) + search(A) + expect(events('analysis_engine_load')).toHaveLength(0) + analysisPositionShown(B) + + const [load] = events('analysis_engine_load') + expect(load).toMatchObject({ + is_phone: true, + engine: 'stockfish-19-smallnet', + stockfish_first_ms: 100, + maia_first_ms: 200, + d12_ms: 300, + d18_ms: 300, + maia_chart_data_ms: 400, + chart_ms: 500, + lines_settled_ms: 600, + stockfish_done_ms: 700, + stockfish_ran: true, + maia_ran: true, + }) + }) + + it('summarises later positions when the page is hidden', () => { + analysisPositionShown(A) + analysisPositionShown(B) + search(B) + analysisPositionShown(C) + analysisStockfishUpdate(C, { depth: 12, partial: true }) + setVisibility('hidden') + + const [steps] = events('analysis_engine_steps') + expect(steps).toMatchObject({ + steps: 2, + d12_reached: 2, + d18_reached: 1, + lines_settled_reached: 1, + stockfish_done_reached: 1, + d18_median_ms: 300, + }) + }) + + it('takes nothing that arrives after the page is hidden', () => { + analysisPositionShown(A) + analysisStockfishUpdate(A, { depth: 6, partial: true }) + setVisibility('hidden') + analysisStockfishUpdate(A, { depth: 18 }) + + const [load] = events('analysis_engine_load') + expect(load.d18_ms).toBeNull() + expect(load.stockfish_done_ms).toBeNull() + }) + + it('leaves out a position where neither engine ran', () => { + analysisPositionShown(A) + analysisPositionShown(B) + analysisPositionShown(C) + setVisibility('hidden') + expect(events('analysis_engine_load')).toHaveLength(0) + expect(events('analysis_engine_steps')).toHaveLength(0) + }) + + it('ignores updates from a position no longer on show', () => { + analysisPositionShown(A) + analysisPositionShown(B) + analysisStockfishUpdate(A, { depth: 18 }) + setVisibility('hidden') + expect(events('analysis_engine_steps')).toHaveLength(0) + }) + + it('reports nothing unexpected when a page was left normally', () => { + window.dispatchEvent(new Event('pagehide')) + stop() + stop = startAnalysisTiming({ isPhone: true }) + expect(events('analysis_page_ended_unexpectedly')).toHaveLength(0) + }) + + it('reports nothing when the page is hidden after being left, as on a reload', () => { + window.dispatchEvent(new Event('pagehide')) + setVisibility('hidden') + expect(sessionStorage.getItem('maia.analysisPageAlive')).toBeNull() + setVisibility('visible') + stop() + stop = startAnalysisTiming({ isPhone: true }) + expect(events('analysis_page_ended_unexpectedly')).toHaveLength(0) + }) + + it('marks the page alive again on coming back from the back/forward cache', () => { + window.dispatchEvent(new Event('pagehide')) + expect(sessionStorage.getItem('maia.analysisPageAlive')).toBeNull() + window.dispatchEvent(new Event('pageshow')) + expect(sessionStorage.getItem('maia.analysisPageAlive')).not.toBeNull() + }) + + it('reports an earlier page that was never left', () => { + // A page that dies runs no pagehide, and leaves its mark behind. + const mark = sessionStorage.getItem('maia.analysisPageAlive') as string + stop() + sessionStorage.setItem('maia.analysisPageAlive', mark) + stop = startAnalysisTiming({ isPhone: true }) + + const [ended] = events('analysis_page_ended_unexpectedly') + expect(ended).toMatchObject({ was_visible: true }) + }) + + it('tells a page lost in the background from one lost on screen', () => { + setVisibility('hidden') + const mark = sessionStorage.getItem('maia.analysisPageAlive') as string + stop() + sessionStorage.setItem('maia.analysisPageAlive', mark) + setVisibility('visible') + stop = startAnalysisTiming({ isPhone: true }) + + const [ended] = events('analysis_page_ended_unexpectedly') + expect(ended).toMatchObject({ was_visible: false }) + }) + + it('does nothing when no analysis page is mounted', () => { + stop() + analysisPositionShown(A) + search(A) + analysisPositionShown(B) + expect(events('analysis_engine_load')).toHaveLength(0) + stop = startAnalysisTiming({ isPhone: true }) + }) +}) diff --git a/src/lib/analysisTiming.ts b/src/lib/analysisTiming.ts new file mode 100644 index 00000000..5b681c27 --- /dev/null +++ b/src/lib/analysisTiming.ts @@ -0,0 +1,271 @@ +import { STOCKFISH_ENGINE } from 'src/constants/analysis' +import { hasRelaxedSimd } from 'src/lib/engine/relaxedSimd' +import { + trackAnalysisEngineLoad, + trackAnalysisEngineSteps, + trackAnalysisPageEndedUnexpectedly, +} from 'src/lib/analytics' + +// What visitors wait for on the analysis page, for PostHog. Nothing here reads +// the page, so it costs a few timestamps per engine update. +// +// - analysis_engine_load: the first position of a visit. +// - analysis_engine_steps: a summary of the positions moved to after it, sent +// when the page is hidden or left, and every MAX_STEPS positions. +// - analysis_page_ended_unexpectedly: the last analysis page in this tab was +// never left, as when a phone runs out of memory and reloads it. +// +// "d18" means the best line has reached depth 18, and "lines settled" that the +// lines on show have, which is what compares with a build before round 10. + +const MAX_STEPS = 30 +const ALIVE_KEY = 'maia.analysisPageAlive' + +type Position = { + fen: string + kind: 'load' | 'step' + start: number + sinceNavigation: boolean + first?: number + d12?: number + d18?: number + lines?: number + done?: number + maiaFirst?: number + maiaChartData?: number + chart?: number +} + +type Row = { + d12?: number + d18?: number + lines?: number + done?: number + maiaFirst?: number + chart?: number +} + +let active = false +let isPhone = false +let loadTaken = false +let position: Position | null = null +let rows: Row[] = [] +let leaving = false +let startedAt = 0 + +const median = (values: (number | undefined)[]) => { + const known = values.filter((v): v is number => v !== undefined) + if (known.length === 0) return null + known.sort((a, b) => a - b) + return known[Math.floor((known.length - 1) / 2)] +} + +const count = (values: (number | undefined)[]) => + values.filter((v) => v !== undefined).length + +const device = () => { + const nav = navigator as Navigator & { + deviceMemory?: number + connection?: { effectiveType?: string } + } + return { + engine: STOCKFISH_ENGINE, + engine_build: hasRelaxedSimd() ? 'relaxed-simd' : 'plain', + is_phone: isPhone, + cores: nav.hardwareConcurrency ?? null, + device_memory_gb: nav.deviceMemory ?? null, + connection: nav.connection?.effectiveType ?? null, + release: process.env.NEXT_PUBLIC_VERCEL_GIT_COMMIT_SHA?.slice(0, 7) ?? null, + } +} + +const relative = (p: Position, time?: number) => + time === undefined ? null : Math.round(time - p.start) + +const flushSteps = () => { + if (rows.length === 0) return + const sent = rows + rows = [] + const column = (key: keyof Row) => sent.map((row) => row[key]) + trackAnalysisEngineSteps({ + ...device(), + steps: sent.length, + maia_first_median_ms: median(column('maiaFirst')), + chart_median_ms: median(column('chart')), + d12_median_ms: median(column('d12')), + d18_median_ms: median(column('d18')), + lines_settled_median_ms: median(column('lines')), + stockfish_done_median_ms: median(column('done')), + d12_reached: count(column('d12')), + d18_reached: count(column('d18')), + lines_settled_reached: count(column('lines')), + stockfish_done_reached: count(column('done')), + }) +} + +// Ends the position on show, with what it reached. The page being hidden ends +// it too: timers and engines slow down there, so later times would mean nothing. +const closePosition = () => { + const p = position + position = null + if (!p) return + // Nothing ran: saved analysis, or the next move came at once. + if (p.first === undefined && p.maiaFirst === undefined) return + + if (p.kind === 'load') { + trackAnalysisEngineLoad({ + ...device(), + arrival: p.sinceNavigation ? 'page_load' : 'in_app', + position_shown_ms: p.sinceNavigation ? Math.round(p.start) : null, + stockfish_ran: p.first !== undefined, + maia_ran: p.maiaFirst !== undefined, + maia_first_ms: relative(p, p.maiaFirst), + maia_chart_data_ms: relative(p, p.maiaChartData), + chart_ms: relative(p, p.chart), + stockfish_first_ms: relative(p, p.first), + d12_ms: relative(p, p.d12), + d18_ms: relative(p, p.d18), + lines_settled_ms: relative(p, p.lines), + stockfish_done_ms: relative(p, p.done), + }) + return + } + + rows.push({ + d12: relative(p, p.d12) ?? undefined, + d18: relative(p, p.d18) ?? undefined, + lines: relative(p, p.lines) ?? undefined, + done: relative(p, p.done) ?? undefined, + maiaFirst: relative(p, p.maiaFirst) ?? undefined, + chart: relative(p, p.chart) ?? undefined, + }) + if (rows.length >= MAX_STEPS) flushSteps() +} + +const markAlive = () => { + // The page also becomes hidden after pagehide: that is not being alive. + if (leaving) return + try { + sessionStorage.setItem( + ALIVE_KEY, + JSON.stringify({ + at: startedAt, + visible: document.visibilityState === 'visible', + }), + ) + } catch { + // Without session storage there is nothing to compare next time. + } +} + +const reportEarlierPage = () => { + try { + const earlier = sessionStorage.getItem(ALIVE_KEY) + if (!earlier) return + const { at, visible } = JSON.parse(earlier) + trackAnalysisPageEndedUnexpectedly({ + ...device(), + was_visible: !!visible, + seconds_since_it_started: Math.round((Date.now() - at) / 1000), + }) + } catch { + // As above. + } +} + +const onVisibilityChange = () => { + markAlive() + if (document.visibilityState === 'hidden') { + closePosition() + flushSteps() + } +} + +const onPageHide = () => { + leaving = true + closePosition() + flushSteps() + try { + sessionStorage.removeItem(ALIVE_KEY) + } catch { + // As above. + } +} + +// Back from the browser's back/forward cache. +const onPageShow = () => { + leaving = false + markAlive() +} + +/** While a page with the analysis engines is mounted. Returns its cleanup. */ +export const startAnalysisTiming = (options: { isPhone: boolean }) => { + if (typeof window === 'undefined') return () => undefined + active = true + isPhone = options.isPhone + loadTaken = false + leaving = false + startedAt = Date.now() + reportEarlierPage() + markAlive() + document.addEventListener('visibilitychange', onVisibilityChange) + window.addEventListener('pagehide', onPageHide) + window.addEventListener('pageshow', onPageShow) + return () => { + active = false + onPageHide() + document.removeEventListener('visibilitychange', onVisibilityChange) + window.removeEventListener('pagehide', onPageHide) + window.removeEventListener('pageshow', onPageShow) + } +} + +const isFullPageLoad = () => { + const navigation = performance.getEntriesByType?.('navigation')[0] + return !!navigation && new URL(navigation.name).pathname === location.pathname +} + +/** A position came on show. */ +export const analysisPositionShown = (fen: string) => { + if (!active || position?.fen === fen) return + closePosition() + const kind = loadTaken ? 'step' : 'load' + loadTaken = true + if (document.visibilityState !== 'visible') return + position = { + fen, + kind, + start: performance.now(), + sinceNavigation: kind === 'load' && isFullPageLoad(), + } +} + +const current = (fen: string) => (position?.fen === fen ? position : null) + +export const analysisStockfishUpdate = ( + fen: string, + update: { depth: number; partial?: boolean; lines_done?: boolean }, +) => { + const p = current(fen) + if (!p) return + const now = performance.now() + p.first ??= now + if (update.depth >= 12) p.d12 ??= now + if (update.depth >= 18) p.d18 ??= now + if (update.lines_done) p.lines ??= now + if (!update.partial) p.done ??= now +} + +/** Stage 0 is the selected rating; stage 1 is what the chart is drawn from. */ +export const analysisMaiaStage = (fen: string, stage: number) => { + const p = current(fen) + if (!p) return + const now = performance.now() + if (stage === 0) p.maiaFirst ??= now + if (stage === 1) p.maiaChartData ??= now +} + +export const analysisChartDrawn = (fen: string | undefined) => { + const p = fen ? current(fen) : null + if (p && p.maiaChartData !== undefined) p.chart ??= performance.now() +} diff --git a/src/lib/analytics.ts b/src/lib/analytics.ts index c27f67b8..b76cda92 100644 --- a/src/lib/analytics.ts +++ b/src/lib/analytics.ts @@ -1,6 +1,7 @@ 'use client' import { capture } from 'src/lib/posthogClient' +import { STOCKFISH_ENGINE } from 'src/constants/analysis' type EventProperties = Record @@ -61,10 +62,13 @@ export const trackStockfishEngineLoading = (page: string) => { export const trackStockfishEngineReady = ( loadTimeSeconds: number, page: string, + memoryRetries = 0, ) => { safeTrack('stockfish_engine_ready', { load_time_seconds: loadTimeSeconds, page, + engine: STOCKFISH_ENGINE, + memory_retries: memoryRetries, }) } @@ -72,11 +76,14 @@ export const trackStockfishEngineFailed = ( error: string, loadTimeSeconds: number, page: string, + memoryRetries = 0, ) => { safeTrack('stockfish_engine_failed', { error: error.slice(0, 500), load_time_seconds: loadTimeSeconds, page, + engine: STOCKFISH_ENGINE, + memory_retries: memoryRetries, }) } @@ -84,11 +91,47 @@ export const trackMaiaEngineLoading = (page: string) => { safeTrack('maia_engine_loading', { page }) } -export const trackMaiaEngineReady = (loadTimeSeconds: number, page: string) => { - safeTrack('maia_engine_ready', { load_time_seconds: loadTimeSeconds, page }) +export const trackMaiaEngineReady = ( + loadTimeSeconds: number, + page: string, + restarts = 0, +) => { + safeTrack('maia_engine_ready', { + load_time_seconds: loadTimeSeconds, + page, + restarts, + }) +} + +export const trackMaiaEngineFailed = ( + error: string, + loadTimeSeconds: number, + page: string, + restarts = 0, +) => { + safeTrack('maia_engine_failed', { + error: error.slice(0, 500), + load_time_seconds: loadTimeSeconds, + page, + restarts, + }) +} + +// The analysis page's timings: see src/lib/analysisTiming.ts +export const trackAnalysisEngineLoad = (properties: EventProperties) => { + safeTrack('analysis_engine_load', properties) +} + +export const trackAnalysisEngineSteps = (properties: EventProperties) => { + safeTrack('analysis_engine_steps', properties) +} + +export const trackAnalysisPageEndedUnexpectedly = ( + properties: EventProperties, +) => { + safeTrack('analysis_page_ended_unexpectedly', properties) } -// Opening Selection & Drill Events export const trackOpeningSelectionModalOpened = ( source: 'page_load' | 'change_selections_button', existingSelectionsCount: number, diff --git a/src/lib/engine/maia.ts b/src/lib/engine/maia.ts index 6111a193..c35bdd55 100644 --- a/src/lib/engine/maia.ts +++ b/src/lib/engine/maia.ts @@ -185,6 +185,11 @@ class Maia { setTimeout(() => this.initialize(modelUrl, modelVersion), delay) } + /** How many times Maia has started again for lack of memory. */ + public get restartCount() { + return this.restarts + } + public async downloadModel() { if (!this.worker) throw new Error('Worker not initialized') if (this.downloadPromise) { diff --git a/src/lib/engine/relaxedSimd.ts b/src/lib/engine/relaxedSimd.ts new file mode 100644 index 00000000..b7714450 --- /dev/null +++ b/src/lib/engine/relaxedSimd.ts @@ -0,0 +1,10 @@ +// i8x16.relaxed_swizzle, the probe from wasm-feature-detect. The relaxed-SIMD +// build searches ~30% more positions a second; Chrome and Firefox have it. +export const hasRelaxedSimd = () => + typeof WebAssembly !== 'undefined' && + WebAssembly.validate( + new Uint8Array([ + 0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 15, + 1, 13, 0, 65, 1, 253, 15, 65, 2, 253, 15, 253, 128, 2, 11, + ]), + ) diff --git a/src/lib/engine/stockfish.ts b/src/lib/engine/stockfish.ts index 81ce3314..186ff0a5 100644 --- a/src/lib/engine/stockfish.ts +++ b/src/lib/engine/stockfish.ts @@ -1,5 +1,6 @@ import { Chess } from 'chess.ts' import { cpToWinrate } from 'src/lib' +import { hasRelaxedSimd } from './relaxedSimd' import StockfishWeb from '@lichess-org/stockfish-web' import { StockfishDiagnostics, @@ -1105,6 +1106,7 @@ class Engine { runContext.fen, ) restEval.partial = true + restEval.lines_done = true this.attachDiagnostics(restEval, { runContext, strategy: 'staged-root-probe', @@ -1175,6 +1177,7 @@ class Engine { runContext.fen, ) maiaDeepenedEval.partial = true + maiaDeepenedEval.lines_done = true this.attachDiagnostics(maiaDeepenedEval, { runContext, strategy: 'staged-root-probe', @@ -1210,6 +1213,7 @@ class Engine { rootScores, runContext.fen, ) + finalEval.lines_done = true this.attachDiagnostics(finalEval, { runContext, strategy: 'staged-root-probe', @@ -2035,6 +2039,10 @@ export const sharedWasmMemory = ( } } +let memoryRetries = 0 +/** How many times this page has waited for memory to start Stockfish. */ +export const stockfishMemoryRetries = () => memoryRetries + export const reserveWasmMemory = async ( lo: number, delaysMs = STOCKFISH_MEMORY_RETRY_DELAYS_MS, @@ -2044,6 +2052,7 @@ export const reserveWasmMemory = async ( return sharedWasmMemory(lo) } catch (e) { if (!(e instanceof RangeError) || attempt >= delaysMs.length) throw e + memoryRetries += 1 await new Promise((resolve) => setTimeout(resolve, delaysMs[attempt])) } } @@ -2116,16 +2125,6 @@ const setNnueBuffer = ( /** Stockfish's script or its net could not be fetched. */ class StockfishFileError extends Error {} -// i8x16.relaxed_swizzle, the probe from wasm-feature-detect. The relaxed-SIMD -// build searches ~30% more positions a second; Chrome and Firefox have it. -const hasRelaxedSimd = () => - WebAssembly.validate( - new Uint8Array([ - 0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3, 2, 1, 0, 10, 15, - 1, 13, 0, 65, 1, 253, 15, 65, 2, 253, 15, 253, 128, 2, 11, - ]), - ) - /** * Stockfish 19 with its single 1.2 MB net (@lichess-org/stockfish-web's * smallnet build). Stockfish 17's nets were 78 MB, which took minutes to diff --git a/src/types/analysis.ts b/src/types/analysis.ts index 1dc1d993..bfba0b39 100644 --- a/src/types/analysis.ts +++ b/src/types/analysis.ts @@ -63,6 +63,8 @@ export interface StockfishEvaluation { * and the position is not done even if it has reached the target depth. */ partial?: boolean + /** The lines on show have reached the target depth. */ + lines_done?: boolean } export interface CachedEngineAnalysisEntry { From 3366784cd304bd754b50154ff7f12cd0506f84a7 Mon Sep 17 00:00:00 2001 From: ashtonanderson Date: Thu, 5 Nov 2026 04:08:36 +0000 Subject: [PATCH 2/2] docs: LOADING.md: round 11, the events, how to read them, and queries Co-Authored-By: Claude Opus 5.5 --- LOADING.md | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/LOADING.md b/LOADING.md index 164cae5b..02ed6fda 100644 --- a/LOADING.md +++ b/LOADING.md @@ -1413,6 +1413,78 @@ loads both runtimes at once. **Not known yet:** how this behaves on real Android phones, and for real visitors. No field data was used. The iPhone numbers are one phone and few runs. +### Round 11: what real visitors get, in PostHog + +Rounds 1–10 measured emulated devices and one iPhone. These events show real visitors, +split by `engine`, `engine_build`, `is_phone`, `cores`, `release` and PostHog's own +`$os` and `$browser`. Nothing reads the page: `src/lib/analysisTiming.ts` takes a few +timestamps per engine update. Times are in milliseconds, and are left empty when the +visitor moved on or hid the page first. + +| event | sent | properties | +| ----------------------------------- | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `analysis_engine_load` | once a visit, for the first position | `position_shown_ms`, `maia_first_ms`, `maia_chart_data_ms`, `chart_ms`, `stockfish_first_ms`, `d12_ms`, `d18_ms`, `lines_settled_ms`, `stockfish_done_ms`, `arrival`, `stockfish_ran`, `maia_ran` | +| `analysis_engine_steps` | when the page is hidden or left, and every 30 moves | `steps`, the `_median_ms` of `maia_first`, `chart`, `d12`, `d18`, `lines_settled`, `stockfish_done`, and how many positions `_reached` each Stockfish stage | +| `analysis_page_ended_unexpectedly` | on the next analysis page in the tab | `was_visible`, `seconds_since_it_started` | +| `stockfish_engine_ready`, `_failed` | once an engine, as before | `load_time_seconds`, and now `engine` and `memory_retries` | +| `maia_engine_ready`, `_failed` | once a page, when Maia starts from its saved model | `load_time_seconds`, `restarts` | + +Reading them: + +- **"d18" is not comparable across round 10:** use `lines_settled` for that. `d18_ms` says + when the best line got there. +- **A crash:** `analysis_page_ended_unexpectedly` with `was_visible` true means the last + analysis page in this tab was on screen and never left. With it false, the browser may + just have dropped a page in the background. It undercounts: it can only report when the + visitor comes back to an analysis page in that tab, so one who leaves after a crash, or + sees Safari's "A problem repeatedly occurred", isn't counted until they return. +- **Saved analysis:** `stockfish_ran` false means the search didn't run for that position, + so its times say nothing. Filter on it for engine times. +- **Before and after round 10:** only `stockfish_engine_ready`, `stockfish_engine_failed`, + `$web_vitals` and `$exception` existed before it went live (2026-11-05 03:30 UTC). The + new events have no "before", so they compare later changes against round 10. + +Checked in a browser with PostHog's requests captured: the events and values above arrive; +a normal reload reports no crash; and a page that loses its exit handlers, as a killed one +would, is reported on the next load with `was_visible` true. PostHog ignores automated +browsers, so this needed the test browser to hide that it is one. + +No slowdown within the noise, against the same code without these events: six paired +loads per device (test browsers, whose events PostHog drops, so sending itself isn't +counted). Desktop: first depth 0.00 s, depth 12 0.00 s, depth 18 0.00 s (paired medians). +Phone: first depth 0.56 s, depth 12 0.61 s and depth 18 0.71 s sooner, which is noise in +the build's favour (3–4 of 6 pairs). + +Queries for PostHog's SQL editor: + +```sql +-- Crash rate: analysis pages lost on screen, per analysis page load, by day and phone +SELECT toDate(timestamp) AS day, properties.is_phone AS phone, + countIf(event = 'analysis_page_ended_unexpectedly' AND properties.was_visible = true) AS lost_on_screen, + countIf(event = 'analysis_engine_load') AS loads +FROM events +WHERE event IN ('analysis_page_ended_unexpectedly', 'analysis_engine_load') AND timestamp > now() - INTERVAL 14 DAY +GROUP BY day, phone ORDER BY day + +-- What a visitor waits for on a return visit, by phone and operating system +SELECT properties.is_phone AS phone, properties.$os AS os, count() AS loads, + quantile(0.5)(toFloat(properties.stockfish_first_ms)) AS first_depth, + quantile(0.5)(toFloat(properties.lines_settled_ms)) AS lines_settled, + quantile(0.5)(toFloat(properties.chart_ms)) AS chart, + quantile(0.9)(toFloat(properties.lines_settled_ms)) AS lines_settled_p90 +FROM events +WHERE event = 'analysis_engine_load' AND properties.stockfish_ran = true AND timestamp > now() - INTERVAL 14 DAY +GROUP BY phone, os ORDER BY loads DESC + +-- Round 10, before and after: Stockfish's start, and how often it failed +SELECT timestamp >= toDateTime('2026-11-05 03:30:00') AS after_round_10, + countIf(event = 'stockfish_engine_ready') AS ready, countIf(event = 'stockfish_engine_failed') AS failed, + quantileIf(0.5)(toFloat(properties.load_time_seconds), event = 'stockfish_engine_ready') AS median_ready_seconds +FROM events +WHERE event IN ('stockfish_engine_ready', 'stockfish_engine_failed') AND timestamp > now() - INTERVAL 30 DAY +GROUP BY after_round_10 +``` + ## What changed | commit | change | effect |