diff --git a/.github/workflows/frontend_tests.yml b/.github/workflows/frontend_tests.yml index 016cfee624..cb696adb8f 100644 --- a/.github/workflows/frontend_tests.yml +++ b/.github/workflows/frontend_tests.yml @@ -86,6 +86,10 @@ jobs: config: playwright.paired-outcomes.config.ts shard: 1 total: 1 + - project: scenario-hierarchy + config: playwright.scenario-hierarchy.config.ts + shard: 1 + total: 1 outputs: expected-reports: ${{ strategy.job-total }} permissions: @@ -131,7 +135,7 @@ jobs: run: npx playwright install --with-deps chromium - name: Build the isolated fixture frontend - if: matrix.project == 'analytics' || matrix.project == 'paired-outcomes' + if: matrix.project == 'analytics' || matrix.project == 'paired-outcomes' || matrix.project == 'scenario-hierarchy' run: npm run build - name: Run E2E shard (no credentials needed) diff --git a/frontend/e2e/fixtures/scenario_hierarchy_backend.py b/frontend/e2e/fixtures/scenario_hierarchy_backend.py new file mode 100644 index 0000000000..ee7b4d17b0 --- /dev/null +++ b/frontend/e2e/fixtures/scenario_hierarchy_backend.py @@ -0,0 +1,145 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Disposable saved Scenario run with compound results for hierarchy browser acceptance tests.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from typing import TYPE_CHECKING +from unittest.mock import patch +from uuid import UUID + +from frontend.e2e.fixtures import analytics_backend + +if TYPE_CHECKING: + from pyrit.memory import SQLiteMemory + +RUN_ID = UUID(int=0x3602) +CHILD_COUNT = 520 +MISSING_POSITION = 1 +DUPLICATE_POSITION = 3 + + +async def _seed_async(memory: SQLiteMemory) -> None: + from pyrit.common.utils import to_sha256 + from pyrit.models import ( + SCENARIO_RUN_PLAN_METADATA_KEY, + AtomicAttackIdentifier, + AttackIdentifier, + AttackOutcome, + AttackResult, + AttackResultRole, + ScenarioIdentifier, + ScenarioResult, + ScenarioRunPlan, + ScenarioRunPlanAtomicGroup, + ScenarioRunPlanGroupKind, + ScenarioRunPlanSeedGroup, + ScenarioRunState, + TargetIdentifier, + ) + + memory.disable_embedding() + timestamp = datetime(2026, 10, 28, tzinfo=UTC) + target = TargetIdentifier(class_name="MockPromptTarget", class_module="hierarchy_browser_fixture") + identifier = AtomicAttackIdentifier.build( + attack_identifier=AttackIdentifier( + class_name="SequentialAttack", + class_module="pyrit.executor.attack", + objective_target=target, + ) + ) + plan = ScenarioRunPlan( + atomic_groups=[ + ScenarioRunPlanAtomicGroup( + id=name, + atomic_attack_name=name, + display_group=display, + technique_eval_hash=f"{name}-eval", + kind=ScenarioRunPlanGroupKind.ATTACK, + seed_group_ids=["seed"], + ) + for name, display in [("sequential", "Sequential"), ("single", "Single turn")] + ], + seed_groups=[ScenarioRunPlanSeedGroup(id="seed", objective="seed", objective_sha256=to_sha256("seed"))], + ) + + def result( + index: int, + group: str, + *, + objective: str, + outcome: AttackOutcome = AttackOutcome.FAILURE, + child_ids: list[str] | None = None, + attempt_index: int | None = None, + ) -> AttackResult: + return AttackResult( + attack_result_id=str(UUID(int=0x360200000 + index)), + conversation_id=str(UUID(int=0x360300000 + index)), + objective=objective, + outcome=outcome, + timestamp=timestamp + timedelta(seconds=index), + atomic_attack_identifier=identifier, + attribution_parent_id=str(RUN_ID), + attribution_data={ + "seed_group_id": "seed", + "parent_eval_hash": f"{group}-eval", + "parent_collection": group, + # Like SequentialAttack: children share the parent's attribution plus their position. + **({"attempt_index": attempt_index} if attempt_index is not None else {}), + "result_role": ( + AttackResultRole.ORCHESTRATION.value + if child_ids is not None + else AttackResultRole.TARGET_FACING.value + ), + }, + metadata={"child_attack_result_ids": child_ids} if child_ids is not None else {}, + ) + + grandchildren = [ + result(1000 + i, "sequential", objective=f"Nested step {i + 1}", attempt_index=i + 1) for i in range(2) + ] + children = [ + result( + 10 + i, + "sequential", + objective=f"Step {i + 1}", + outcome=AttackOutcome.SUCCESS if i == 4 else AttackOutcome.FAILURE, + child_ids=[g.attack_result_id for g in grandchildren] if i == 0 else None, + attempt_index=i + 1, + ) + for i in range(CHILD_COUNT) + ] + child_ids = [child.attack_result_id for child in children] + child_ids[MISSING_POSITION] = str(UUID(int=0x3602DEAD)) + child_ids[DUPLICATE_POSITION] = child_ids[0] + parent = result(1, "sequential", objective="Sequential parent", child_ids=child_ids) + single = result(2, "single", objective="Single-turn attempt", outcome=AttackOutcome.SUCCESS) + + rows = {"sequential": [parent, *children, *grandchildren], "single": [single]} + await memory.add_attack_results_to_memory_async(attack_results=[row for group in rows.values() for row in group]) + await memory.add_scenario_results_to_memory_async( + scenario_results=[ + ScenarioResult( + id=RUN_ID, + scenario_identifier=ScenarioIdentifier( + class_name="HierarchyBrowserScenario", + class_module="hierarchy_browser_fixture", + version=1, + objective_target=target, + ), + scenario_description="Offline compound-result hierarchy acceptance fixture.", + attack_results=rows, + metadata={SCENARIO_RUN_PLAN_METADATA_KEY: plan.model_dump(mode="json")}, + scenario_run_state=ScenarioRunState.COMPLETED, + creation_time=timestamp, + completion_time=timestamp + timedelta(minutes=1), + ) + ] + ) + + +if __name__ == "__main__": + with patch.object(analytics_backend, "_seed_async", _seed_async): + analytics_backend.main() diff --git a/frontend/e2e/scenario-hierarchy.spec.ts b/frontend/e2e/scenario-hierarchy.spec.ts new file mode 100644 index 0000000000..2f96db0bd3 --- /dev/null +++ b/frontend/e2e/scenario-hierarchy.spec.ts @@ -0,0 +1,86 @@ +import { expect, test, type Page } from '@playwright/test' +import { scenarioRunRoutePath } from '../src/utils/routeParams' + +// Seeded by e2e/fixtures/scenario_hierarchy_backend.py +const run = scenarioRunRoutePath('00000000-0000-0000-0000-000000003602') + +async function openHierarchy(page: Page) { + await page.goto(run) + const section = page.getByRole('region', { name: 'Result hierarchy', exact: true }) + await expect(section.getByText('Sequential parent', { exact: true })).toBeVisible() + return section +} + +test('browses a compound result through the real API, page by page, with stable totals', async ({ page }, testInfo) => { + const errors: string[] = [] + page.on('pageerror', (error) => errors.push(error.message)) + const section = await openHierarchy(page) + const totals = section.getByLabel('Run totals') + const before = await totals.textContent() + + const firstPage = page.waitForResponse((response) => /\/hierarchy\/nodes\/[^/?]+\?limit=200$/.test(response.url())) + await section.getByRole('button', { name: 'Expand children of sequential: Sequential parent' }).click() + expect((await firstPage).status()).toBe(200) + + const table = section.getByRole('table', { name: 'Sequential saved results' }) + const row = (text: string) => table.getByRole('row').filter({ hasText: text }).first() + await expect(table.getByText('Step 1', { exact: true })).toBeVisible() + await expect(row('#2')).toContainText('Unresolved reference') + await expect(row('#4')).toContainText('Unresolved reference') + // Children keep the parent's columns: outcome text sits under "Attack outcome", not nested in the row. + await expect(row('Step 5').getByRole('cell').nth(2)).toHaveText('Success') + await expect(section.getByText('Showing 200 of 520 saved children')).toBeVisible() + + for (const shown of [400, 520]) { + const next = page.waitForResponse((response) => response.url().includes('children_cursor=')) + await section.getByRole('button', { name: 'Load more children' }).click() + expect((await next).status()).toBe(200) + await expect(section.getByText(`Showing ${shown} of 520 saved children`)).toBeVisible() + } + await expect(section.getByRole('button', { name: 'Load more children' })).toHaveCount(0) + await expect(table.getByText('Step 520', { exact: true })).toBeVisible() + + await table.getByRole('button', { name: 'Expand children of #1 sequential: Step 1' }).click() + await expect(table.getByText('Nested step 2', { exact: true })).toBeVisible() + expect(await totals.textContent()).toBe(before) + + await section.getByRole('button', { name: 'Open details for single: Single-turn attempt' }).click() + await expect(page.getByRole('dialog', { name: 'single' })).toBeVisible() + await page.getByRole('button', { name: 'Close' }).click() + await expect(page.getByRole('dialog')).toHaveCount(0) + + await page.getByRole('heading', { name: 'Result hierarchy', exact: true }).evaluate((heading) => + heading.scrollIntoView({ block: 'start' }), + ) + await page.screenshot({ path: testInfo.outputPath('scenario-hierarchy-desktop.png') }) + expect(errors).toEqual([]) +}) + +test.describe('on a narrow touch screen', () => { + // Touch targets grow only for coarse pointers, as in touch-targets.spec.ts. + test.use({ hasTouch: true }) + + test('keeps the hierarchy usable at a narrow viewport and from the keyboard', async ({ page }, testInfo) => { + await page.setViewportSize({ width: 390, height: 844 }) + const section = await openHierarchy(page) + + const expand = section.getByRole('button', { name: 'Expand children of sequential: Sequential parent' }) + await expand.focus() + await page.keyboard.press('Enter') + // The same button is renamed once expanded, so look it up by its new name. + await expect(section.getByRole('button', { name: 'Collapse children of sequential: Sequential parent' })).toHaveAttribute( + 'aria-expanded', + 'true', + ) + await expect(section.getByText('Showing 200 of 520 saved children')).toBeVisible() + + const loadMore = section.getByRole('button', { name: 'Load more children' }) + const size = await loadMore.boundingBox() + expect(size?.height).toBeGreaterThanOrEqual(44) + const overflow = await section.evaluate((element) => element.scrollWidth > element.clientWidth) + expect(overflow).toBe(false) + + await section.scrollIntoViewIfNeeded() + await page.screenshot({ path: testInfo.outputPath('scenario-hierarchy-mobile.png') }) + }) +}) diff --git a/frontend/package.json b/frontend/package.json index d89a062900..2613321d2e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -17,6 +17,7 @@ "test:coverage": "jest --coverage", "test:e2e": "playwright test", "test:e2e:paired-outcomes": "npm run build && playwright test --config playwright.paired-outcomes.config.ts", + "test:e2e:scenario-hierarchy": "npm run build && playwright test --config playwright.scenario-hierarchy.config.ts", "test:e2e:analytics": "npm run build && playwright test --config playwright.analytics.config.ts", "test:e2e:seeded": "playwright test --project seeded", "test:e2e:live": "E2E_LIVE_MODE=true playwright test --project live", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index db313d78eb..0c01eee691 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -26,7 +26,7 @@ const E2E_BACKEND_URL = `http://127.0.0.1:${E2E_BACKEND_PORT}`; export default defineConfig({ testDir: "./e2e", - testIgnore: ["**/analytics.spec.ts", "**/paired-outcomes.spec.ts"], + testIgnore: ["**/analytics.spec.ts", "**/paired-outcomes.spec.ts", "**/scenario-hierarchy.spec.ts"], fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, diff --git a/frontend/playwright.scenario-hierarchy.config.ts b/frontend/playwright.scenario-hierarchy.config.ts new file mode 100644 index 0000000000..ffe4eda1d2 --- /dev/null +++ b/frontend/playwright.scenario-hierarchy.config.ts @@ -0,0 +1,22 @@ +import process from 'node:process' +import { defineConfig } from '@playwright/test' +import analytics from './playwright.analytics.config' + +const port = Number(process.env.PYRIT_E2E_HIERARCHY_PORT ?? '18498') +if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('Invalid scenario hierarchy test port') +const url = `http://127.0.0.1:${port}` + +export default defineConfig({ + ...analytics, + testMatch: 'scenario-hierarchy.spec.ts', + use: { + ...analytics.use, + baseURL: url, + storageState: { cookies: [], origins: [{ origin: url, localStorage: [{ name: 'pyrit-tour-completed', value: 'true' }] }] }, + }, + projects: [{ name: 'scenario-hierarchy', use: { viewport: { width: 1440, height: 1050 } } }], + webServer: { + command: `cd .. && uv run --no-sync python -m frontend.e2e.fixtures.scenario_hierarchy_backend --port ${port}`, + env: { PYTHONIOENCODING: 'utf-8' }, url: `${url}/api/health`, reuseExistingServer: false, timeout: 120_000, + }, +}) diff --git a/frontend/src/components/AttackResults/attackAttemptFormatting.ts b/frontend/src/components/AttackResults/attackAttemptFormatting.ts index 2287a26138..21fabc443c 100644 --- a/frontend/src/components/AttackResults/attackAttemptFormatting.ts +++ b/frontend/src/components/AttackResults/attackAttemptFormatting.ts @@ -19,6 +19,11 @@ export function formatAttackSuccess(outcome: ScenarioProgressResult['outcome']): return outcome.replace(/^\w/, (letter) => letter.toUpperCase()) } +/** The attack's outcome as a word, for places headed "Attack outcome" rather than "Attack success". */ +export function formatAttackOutcome(outcome: ScenarioProgressResult['outcome']): string { + return outcome.replace(/^\w/, (letter) => letter.toUpperCase()) +} + export function formatScore(score: ScenarioProgressResult['score']): string { if (!score || score.status === 'undetermined') { return 'Undetermined' diff --git a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx index 6a2d728c3c..f52b71b0f9 100644 --- a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx @@ -24,6 +24,7 @@ jest.mock('@/services/api', () => ({ scenariosApi: { cancelRun: jest.fn(), estimateRun: jest.fn(), + getRunHierarchy: jest.fn(() => new Promise(() => {})), getScenario: jest.fn(), listCatalog: jest.fn(), startRun: jest.fn(), diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts b/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts new file mode 100644 index 0000000000..25af9c77d4 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.styles.ts @@ -0,0 +1,87 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTarget } from '@/styles/touchTargets' + +export const useScenarioResultHierarchyStyles = makeStyles({ + root: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalM, + minWidth: 0, + }, + totals: { + display: 'flex', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalL, + }, + hint: { + color: tokens.colorNeutralForeground3, + }, + group: { + display: 'flex', + flexDirection: 'column', + gap: tokens.spacingVerticalXS, + paddingTop: tokens.spacingVerticalS, + borderTop: `${tokens.strokeWidthThin} solid ${tokens.colorNeutralStroke2}`, + minWidth: 0, + }, + // Wide tables scroll inside their own box instead of widening the page on narrow screens. + tableScroll: { + width: '100%', + overflowX: 'auto', + border: `1px solid ${tokens.colorNeutralStroke2}`, + borderRadius: tokens.borderRadiusLarge, + backgroundColor: tokens.colorNeutralBackground1, + }, + table: { + minWidth: '52rem', + tableLayout: 'auto', + }, + resultCell: { + display: 'flex', + alignItems: 'center', + gap: tokens.spacingHorizontalS, + minWidth: '18rem', + }, + outcomeCell: { + display: 'flex', + flexDirection: 'column', + alignItems: 'flex-start', + gap: tokens.spacingVerticalXXS, + }, + nowrap: { + whiteSpace: 'nowrap', + }, + expandButton: { + flexShrink: 0, + ...mobileTouchTarget, + }, + expandSpacer: { + width: '2rem', + flexShrink: 0, + ...mobileTouchTarget, + }, + position: { + color: tokens.colorNeutralForeground3, + fontVariantNumeric: 'tabular-nums', + }, + objective: { + minWidth: 0, + overflowWrap: 'anywhere', + }, + unresolved: { + color: tokens.colorNeutralForeground3, + fontStyle: 'italic', + overflowWrap: 'anywhere', + }, + footer: { + display: 'flex', + alignItems: 'center', + flexWrap: 'wrap', + gap: tokens.spacingHorizontalS, + paddingTop: tokens.spacingVerticalXS, + }, + touchTarget: { + ...mobileTouchTarget, + }, +}) diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx b/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx new file mode 100644 index 0000000000..9f89f196ab --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.test.tsx @@ -0,0 +1,452 @@ +import { render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' + +import { scenariosApi } from '@/services/api' +import type { + ScenarioHierarchyNode, + ScenarioHierarchyProjection, + ScenarioProgressCounts, + ScenarioRunState, +} from '@/types' + +import ScenarioResultHierarchy from './ScenarioResultHierarchy' + +jest.mock('@/services/api', () => ({ + scenariosApi: { getRunHierarchy: jest.fn(), getRunHierarchyNode: jest.fn() }, +})) + +const getRoots = scenariosApi.getRunHierarchy as jest.Mock +const getNode = scenariosApi.getRunHierarchyNode as jest.Mock + +const TOTALS: ScenarioProgressCounts = { + completed: 7, + planned: 9, + succeeded: 2, + success_percentage: 29, + errors: 1, + retries: 4, +} + +function node(id: string, overrides: Partial = {}): ScenarioHierarchyNode { + return { + attack_result_id: id, + conversation_id: `conv-${id}`, + result_role: 'target_facing', + atomic_attack_name: `attack ${id}`, + atomic_group_id: 'g1', + seed_group_id: 's1', + objective: `objective ${id}`, + outcome: 'failure', + execution_time_ms: 1000, + executed_turns: 1, + timestamp: '2026-10-28T00:00:00Z', + total_retries: 0, + retries: [], + retries_truncated: false, + child_attack_result_ids: [], + child_references_truncated: false, + children: [], + children_truncated: false, + child_offset: 0, + unresolved_child_attack_result_ids: [], + unresolved_child_references_truncated: false, + conversations: [], + conversations_truncated: false, + ...overrides, + } +} + +function projection(roots: ScenarioHierarchyNode[], overrides: Partial = {}) { + return { + scenario_result_id: 'run', + scenario_name: 'scenario', + scenario_version: 1, + status: 'COMPLETED', + groups: [ + { + atomic_group_id: 'g1', + atomic_attack_name: 'sequential', + display_group: 'Sequential', + roots, + counts: TOTALS, + }, + ], + overall_counts: TOTALS, + diagnostics: { + failed_attacks: [], + failed_attacks_count: 0, + failed_attacks_truncated: false, + attack_retries: [], + attack_retries_count: 0, + attack_retries_truncated: false, + total_retries: 4, + rootless_result_ids: [], + rootless_results_count: 0, + rootless_results_truncated: false, + }, + truncated: false, + has_more: false, + next_cursor: null, + ...overrides, + } satisfies ScenarioHierarchyProjection +} + +function cursorError(status: number) { + return { isAxiosError: true, response: { status, data: { detail: 'Cursor does not match' } }, message: 'x' } +} + +/** An orchestration parent whose children the preview deferred. */ +const PARENT = node('parent', { + result_role: 'orchestration', + child_attack_result_ids: ['c1', 'missing'], + children_truncated: true, + children_count: 3, +}) + +function mount(onOpenResult = jest.fn(), runStatus: ScenarioRunState | null = null) { + const view = (status: ScenarioRunState | null) => ( + + + + ) + const { rerender } = render(view(runStatus)) + return { onOpenResult, setRunStatus: (status: ScenarioRunState) => rerender(view(status)) } +} + +const resultsTable = () => screen.getByRole('table', { name: 'Sequential saved results' }) +/** The row whose first cell shows this text. */ +const rowWith = (text: string) => within(resultsTable()).getByText(text).closest('tr') as HTMLElement + +const totalsText = () => screen.getByLabelText('Run totals').textContent + +beforeEach(() => { + getRoots.mockReset() + getNode.mockReset() +}) + +test('loads children by position, keeps unresolved references visible, and pages with the node cursor', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValue(projection([PARENT])) + getNode + .mockResolvedValueOnce( + node('parent', { + result_role: 'orchestration', + child_attack_result_ids: ['c1', 'missing'], + children: [node('c1', { child_index: 0, parent_attack_result_id: 'parent' })], + children_count: 3, + next_children_cursor: 'children-2', + }), + ) + .mockResolvedValueOnce( + node('parent', { + result_role: 'orchestration', + child_offset: 2, + child_attack_result_ids: ['c3'], + children: [node('c3', { child_index: 2, parent_attack_result_id: 'parent' })], + children_count: 3, + }), + ) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + await screen.findByText('objective c1') + + expect(getNode).toHaveBeenLastCalledWith('run', 'parent', { limit: 200 }, expect.any(AbortSignal)) + expect(within(rowWith('objective c1')).getByText('#1')).toBeInTheDocument() + expect(within(rowWith('Unresolved reference: missing')).getByText('#2')).toBeInTheDocument() + expect(screen.getByText('Showing 2 of 3 saved children')).toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Load more children' })) + + expect(await screen.findByText('objective c3')).toBeInTheDocument() + expect(getNode).toHaveBeenLastCalledWith( + 'run', + 'parent', + { limit: 200, children_cursor: 'children-2' }, + expect.any(AbortSignal), + ) + expect(within(rowWith('objective c3')).getByText('#3')).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Load more children' })).not.toBeInTheDocument() + expect(getRoots).toHaveBeenCalledTimes(1) +}) + +test('run totals come from the backend and do not change as roots and children load', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([PARENT], { has_more: true, next_cursor: 'roots-2' })) + .mockResolvedValueOnce(projection([node('second-root')])) + getNode.mockResolvedValue( + node('parent', { + result_role: 'orchestration', + child_attack_result_ids: ['c1'], + children: [node('c1', { child_index: 0 })], + children_count: 1, + }), + ) + mount() + + await screen.findByLabelText('Run totals') + const before = totalsText() + expect(before).toContain('Completed: 7 of 9') + + await user.click(screen.getByRole('button', { name: 'Load more results' })) + await screen.findByText('objective second-root') + expect(getRoots).toHaveBeenLastCalledWith('run', { max_nodes: 1000, cursor: 'roots-2' }, expect.any(AbortSignal)) + + await user.click(screen.getByRole('button', { name: 'Expand children of attack parent: objective parent' })) + await screen.findByText('objective c1') + + expect(totalsText()).toBe(before) +}) + +test('expanding a node whose preview is complete makes no request', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValue( + projection([ + node('parent', { + result_role: 'orchestration', + child_attack_result_ids: ['c1'], + children: [node('c1', { child_index: 0 })], + children_count: 1, + }), + ]), + ) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + + expect(screen.getByText('objective c1')).toBeInTheDocument() + expect(getNode).not.toHaveBeenCalled() +}) + +test('a stale child cursor asks to reload the children and restarts paging from the first page', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValue(projection([PARENT])) + getNode + .mockResolvedValueOnce( + node('parent', { + child_attack_result_ids: ['c1'], + children: [node('c1', { child_index: 0 })], + children_count: 3, + next_children_cursor: 'children-2', + }), + ) + .mockRejectedValueOnce(cursorError(400)) + .mockResolvedValueOnce( + node('parent', { + child_attack_result_ids: ['c9'], + children: [node('c9', { child_index: 0 })], + children_count: 1, + }), + ) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + await user.click(await screen.findByRole('button', { name: 'Load more children' })) + + expect(await screen.findByText("This result's saved children changed since they were loaded.")).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Load more children' })).not.toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Reload children' })) + + expect(await screen.findByText('objective c9')).toBeInTheDocument() + expect(screen.queryByText('objective c1')).not.toBeInTheDocument() + expect(getNode).toHaveBeenLastCalledWith('run', 'parent', { limit: 200 }, expect.any(AbortSignal)) +}) + +test('a stale root cursor asks to reload the hierarchy', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([node('r1')], { has_more: true, next_cursor: 'roots-2' })) + .mockRejectedValueOnce(cursorError(400)) + .mockResolvedValueOnce(projection([node('fresh')])) + mount() + + await user.click(await screen.findByRole('button', { name: 'Load more results' })) + expect(await screen.findByText("This run's saved results changed since they were loaded.")).toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Reload hierarchy' })) + + expect(await screen.findByText('objective fresh')).toBeInTheDocument() + expect(getRoots).toHaveBeenLastCalledWith('run', { max_nodes: 1000 }, expect.any(AbortSignal)) +}) + +test('a failed first load can be retried', async () => { + const user = userEvent.setup() + getRoots.mockRejectedValueOnce(cursorError(500)).mockResolvedValueOnce(projection([node('r1')])) + mount() + + await user.click(await screen.findByRole('button', { name: 'Retry' })) + + expect(await screen.findByText('objective r1')).toBeInTheDocument() +}) + +test('shows empty, truncated and rootless states without inventing roots', async () => { + getRoots.mockResolvedValue({ + ...projection([], { truncated: true }), + groups: [], + diagnostics: { ...projection([]).diagnostics, rootless_result_ids: ['x', 'y'], rootless_results_count: 2 }, + }) + mount() + + expect(await screen.findByText(/2 saved results have no root in this run/)).toHaveTextContent('Their IDs: x, y.') + expect(screen.getByText(/Some results are not in this preview/)).toBeInTheDocument() + expect(screen.queryByText('No saved results are available for this run yet.')).not.toBeInTheDocument() + expect(screen.queryByRole('button', { name: /Expand/ })).not.toBeInTheDocument() +}) + +test('opens details for a node with the clicked button as the focus target', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValue(projection([node('r1')])) + const { onOpenResult } = mount() + + const button = await screen.findByRole('button', { name: 'Open details for attack r1: objective r1' }) + await user.click(button) + + expect(onOpenResult).toHaveBeenCalledWith(expect.objectContaining({ attack_result_id: 'r1' }), button) +}) + +test('an empty run says so', async () => { + getRoots.mockResolvedValue({ ...projection([]), groups: [] }) + mount() + + expect(await screen.findByText('No saved results are available for this run yet.')).toBeInTheDocument() +}) + +test('shows the totals the backend reports, not totals counted from loaded rows', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([node('r1')], { has_more: true, next_cursor: 'roots-2' })) + .mockResolvedValueOnce(projection([node('r2')], { overall_counts: { ...TOTALS, completed: 8 } })) + mount() + + await user.click(await screen.findByRole('button', { name: 'Load more results' })) + await screen.findByText('objective r2') + + expect(totalsText()).toContain('Completed: 8 of 9') +}) + +test('reloading after expanding starts from a collapsed tree with no stale children', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([PARENT], { has_more: true, next_cursor: 'roots-2' })) + .mockRejectedValueOnce(cursorError(400)) + .mockResolvedValueOnce(projection([PARENT])) + getNode.mockResolvedValue( + node('parent', { child_attack_result_ids: ['c1'], children: [node('c1', { child_index: 0 })], children_count: 1 }), + ) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + await screen.findByText('objective c1') + await user.click(screen.getByRole('button', { name: 'Load more results' })) + await user.click(await screen.findByRole('button', { name: 'Reload hierarchy' })) + + const expand = await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' }) + expect(expand).toHaveAttribute('aria-expanded', 'false') + expect(screen.queryByText('objective c1')).not.toBeInTheDocument() + + await user.click(expand) + expect(await screen.findByText('objective c1')).toBeInTheDocument() + expect(getNode).toHaveBeenCalledTimes(2) +}) + +test('notes that a run still in progress is a snapshot and can be reloaded', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([node('r1')], { status: 'IN_PROGRESS' })) + .mockResolvedValueOnce(projection([node('r1'), node('r2')], { status: 'COMPLETED' })) + mount() + + expect(await screen.findByText(/This run is still in progress/)).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Reload hierarchy' })) + + expect(await screen.findByText('objective r2')).toBeInTheDocument() + expect(screen.queryByText(/This run is still in progress/)).not.toBeInTheDocument() +}) + +test('a reference back to an ancestor stays unresolved even when the cycle arrives across fetched pages', async () => { + const user = userEvent.setup() + // Saved links P -> A -> B -> A; each node's children come from its own request. + const deferred = (id: string, childId: string, extra: Partial = {}) => + node(id, { + result_role: 'orchestration', + child_attack_result_ids: [childId], + children_truncated: true, + children_count: 1, + ...extra, + }) + const page = (id: string, child: ScenarioHierarchyNode) => + node(id, { result_role: 'orchestration', child_attack_result_ids: [child.attack_result_id], children: [child], children_count: 1 }) + getRoots.mockResolvedValue(projection([deferred('P', 'A')])) + getNode + .mockResolvedValueOnce(page('P', deferred('A', 'B', { child_index: 0 }))) + .mockResolvedValueOnce(page('A', deferred('B', 'A', { child_index: 0 }))) + .mockResolvedValueOnce(page('B', deferred('A', 'B', { child_index: 0 }))) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack P: objective P' })) + await user.click(await screen.findByRole('button', { name: 'Expand children of #1 attack A: objective A' })) + await user.click(await screen.findByRole('button', { name: 'Expand children of #1 attack B: objective B' })) + + const cycle = await screen.findByText(/Unresolved reference: A \(it points back to a result above it\)/) + expect(within(cycle.closest('tr') as HTMLElement).getByText('#1')).toBeInTheDocument() + // Only the first A can be expanded; the repeat has no expand control. + expect(screen.getAllByRole('button', { name: /children of #1 attack A:/ })).toHaveLength(1) + expect(getNode).toHaveBeenCalledTimes(3) +}) + +test('a failed run that is resumed offers to reload the hierarchy it loaded before', async () => { + const user = userEvent.setup() + getRoots + .mockResolvedValueOnce(projection([node('r1')], { status: 'FAILED' })) + .mockResolvedValueOnce(projection([node('r1'), node('r2')], { status: 'IN_PROGRESS' })) + const { setRunStatus } = mount(jest.fn(), 'FAILED') + + await screen.findByText('objective r1') + expect(screen.queryByRole('button', { name: 'Reload hierarchy' })).not.toBeInTheDocument() + + setRunStatus('QUEUED') + expect(screen.getByText(/This run is still in progress/)).toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Reload hierarchy' })) + expect(await screen.findByText('objective r2')).toBeInTheDocument() + + // Finishing after the reload is a change too; the snapshot is no longer current. + setRunStatus('COMPLETED') + expect(screen.getByText("This run's status changed since this hierarchy loaded.")).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Reload hierarchy' })).toBeInTheDocument() +}) + +test('shows results in aligned columns with the attack outcome spelled out', async () => { + const user = userEvent.setup() + getRoots.mockResolvedValue( + projection([ + node('parent', { + result_role: 'orchestration', + outcome: 'success', + child_attack_result_ids: ['c1'], + children: [node('c1', { child_index: 0, outcome: 'error', error_message: 'Target timed out', total_retries: 2 })], + children_count: 1, + }), + ]), + ) + mount() + + await user.click(await screen.findByRole('button', { name: 'Expand children of attack parent: objective parent' })) + await screen.findByText('objective c1') + + expect(within(resultsTable()).getAllByRole('columnheader').map((header) => header.textContent)).toEqual([ + 'Result / objective', + 'Result type', + 'Attack outcome', + 'Duration', + 'Retries', + 'Actions', + ]) + // A child row has the same cells in the same columns as its parent; only the first column is indented. + const cells = (text: string) => within(rowWith(text)).getAllByRole('cell').map((cell) => cell.textContent) + expect(cells('objective parent')).toEqual(['objective parent', 'Orchestration', 'Success', '1s', '0', 'Details']) + expect(cells('objective c1')).toEqual(['#1objective c1', 'Target-facing', 'ErrorTarget timed out', '1s', '2', 'Details']) + expect(screen.getByText(/Each row is one saved attack result/)).toHaveTextContent('not chat messages or retries') +}) diff --git a/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx b/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx new file mode 100644 index 0000000000..00158badff --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioResultHierarchy.tsx @@ -0,0 +1,417 @@ +import type { CSSProperties } from 'react' + +import { + Badge, + Button, + MessageBar, + MessageBarActions, + MessageBarBody, + Table, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow, + Text, +} from '@fluentui/react-components' +import { ChevronDownRegular, ChevronRightRegular } from '@fluentui/react-icons' + +import OutcomeBadge from '@/components/OutcomeBadge' +import { formatAttackOutcome, formatDuration } from '@/components/AttackResults/attackAttemptFormatting' +import { + useScenarioResultHierarchy, + type HierarchyChildrenState, + type UseScenarioResultHierarchyResult, +} from '@/hooks/useScenarioResultHierarchy' +import type { ScenarioHierarchyNode, ScenarioProgressCounts, ScenarioRunState } from '@/types' + +import { childSlots, hasChildReferences, resultRoleLabel } from './scenarioHierarchy' +import { useScenarioResultHierarchyStyles } from './ScenarioResultHierarchy.styles' + +interface ScenarioResultHierarchyProps { + readonly scenarioResultId: string + /** The run's current state, which can change (for example on resume) after the hierarchy loaded. */ + readonly runStatus?: ScenarioRunState | null + readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void +} + +/** Read-only browser for a run's saved result lineage: orchestration parents and their ordered children. */ +export default function ScenarioResultHierarchy({ + scenarioResultId, + runStatus = null, + onOpenResult, +}: ScenarioResultHierarchyProps) { + const styles = useScenarioResultHierarchyStyles() + const hierarchy = useScenarioResultHierarchy(scenarioResultId) + const { roots } = hierarchy + + // Plain status text, not a Spinner: a progressbar role here would be read as run progress. + if (roots.status === 'loading') { + return ( + + Loading result hierarchy... + + ) + } + + if (roots.status === 'error') { + return ( + + Could not load the result hierarchy: {roots.error} + + + + + ) + } + + const rootless = roots.diagnostics?.rootless_results_count ?? 0 + const rootlessSample = roots.diagnostics?.rootless_result_ids ?? [] + const currentStatus = runStatus ?? roots.runStatus + const runActive = currentStatus !== null && ACTIVE_RUN_STATES.has(currentStatus) + const statusChanged = runStatus !== null && roots.runStatus !== null && runStatus !== roots.runStatus + + return ( +
+ {roots.overallCounts && } + + Totals cover the whole run as reported by the server; they are not counted from the rows shown here. + + {(runActive || statusChanged) && ( +
+ + {runActive + ? 'This run is still in progress, so this hierarchy shows the results saved when it loaded.' + : "This run's status changed since this hierarchy loaded."} + + +
+ )} + + {roots.groups.length === 0 ? ( + rootless === 0 && No saved results are available for this run yet. + ) : ( + <> + + Each row is one saved attack result. An orchestration row summarizes a compound attack; expand it to + see its saved component results, numbered in order. These are separate attack results, not chat + messages or retries. + + {roots.groups.map((group) => ( +
+ {group.display_group} + + {group.atomic_attack_name} ยท {formatCounts(group.counts)} + +
+ + + + Result / objective + Result type + Attack outcome + Duration + Retries + Actions + + + + {group.roots.map((node, i) => { + const key = `${group.atomic_group_id}:${i}:${node.attack_result_id}` + return ( + + ) + })} + +
+
+
+ ))} + + )} + + {roots.truncated && ( + + Some results are not in this preview, such as deeper levels or results it could not place. Expand a + result to load its saved children. + + )} + {rootless > 0 && ( + + {rootless} saved {rootless === 1 ? 'result has' : 'results have'} no root in this run (for example, a + cycle of references) and {rootless === 1 ? 'is' : 'are'} not shown in the tree. + {rootlessSample.length > 0 && + ` ${roots.diagnostics?.rootless_results_truncated ? 'Some of their IDs' : 'Their IDs'}: ${rootlessSample.join(', ')}.`} + + )} + + {roots.staleCursor ? ( + + This run's saved results changed since they were loaded. + + + + + ) : roots.error ? ( + + Could not load more results: {roots.error} + + + + + ) : null} + {roots.nextCursor && !roots.staleCursor && !roots.error && ( +
+ +
+ )} +
+ ) +} + +interface HierarchyNodeRowsProps { + readonly nodeKey: string + readonly node: ScenarioHierarchyNode + /** Absolute position under the parent, or null for a root. */ + readonly position: number | null + readonly depth: number + /** Results on the path above this one; a child that is one of them closes a cycle. */ + readonly ancestors: ReadonlySet + readonly hierarchy: UseScenarioResultHierarchyResult + readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void +} + +/** A result's row, followed by its child rows when expanded. Only the first column is indented. */ +function HierarchyNodeRows({ + nodeKey, + node, + position, + depth, + ancestors, + hierarchy, + onOpenResult, +}: HierarchyNodeRowsProps) { + const styles = useScenarioResultHierarchyStyles() + const isExpanded = hierarchy.expanded.has(nodeKey) + const expandable = hasChildReferences(node) + const label = nodeLabel(node, position) + + return ( + <> + + +
+ {expandable ? ( +
+ + + + {resultRoleLabel(node.result_role)} + + + +
+ + {node.outcome === 'error' && (node.error_message || node.error_type) && ( + + {node.error_message ?? node.error_type} + + )} +
+ + {formatDuration(node.execution_time_ms)} + {node.total_retries} + + + + + {expandable && isExpanded && ( + + )} + + ) +} + +interface ChildRowsProps { + readonly parentKey: string + readonly parent: ScenarioHierarchyNode + readonly depth: number + /** The parent and everything above it. */ + readonly ancestors: ReadonlySet + readonly state: HierarchyChildrenState | undefined + readonly hierarchy: UseScenarioResultHierarchyResult + readonly onOpenResult: (node: ScenarioHierarchyNode, trigger: HTMLElement) => void +} + +function ChildRows({ parentKey, parent, depth, ancestors, state, hierarchy, onOpenResult }: ChildRowsProps) { + const styles = useScenarioResultHierarchyStyles() + const pages = state?.pages ?? [] + const slots = pages.flatMap(childSlots) + const loaded = slots.length + const total = state?.childrenCount ?? null + + return ( + <> + {slots.map((slot) => { + // A fetched page can't know the path to it, so a reference back up the path is caught here. + const cycle = slot.kind === 'child' && ancestors.has(slot.node.attack_result_id) + if (slot.kind === 'child' && !cycle) { + return ( + + ) + } + const id = slot.kind === 'child' ? slot.node.attack_result_id : slot.attackResultId + return ( + + +
+ + #{slot.position + 1} + + Unresolved reference: {id} + {cycle && ' (it points back to a result above it)'} + +
+ + + ) + })} + + +
+ {state?.loading && ( + + Loading children... + + )} + {!state?.loading && pages.length > 0 && ( + + {total !== null ? `Showing ${loaded} of ${total} saved children` : `Showing ${loaded} saved children`} + + )} + {state?.staleCursor ? ( + + This result's saved children changed since they were loaded. + + + + + ) : state?.error ? ( + + Could not load children: {state.error} + + + + + ) : null} + {state?.nextCursor && !state.loading && !state.staleCursor && !state.error && ( + + )} +
+ + + + ) +} + +function RunTotals({ counts }: { readonly counts: ScenarioProgressCounts }) { + const styles = useScenarioResultHierarchyStyles() + return ( +
+ Completed: {counts.planned !== null ? `${counts.completed} of ${counts.planned}` : counts.completed} + Succeeded: {counts.succeeded} + Errors: {counts.errors} + Retries: {counts.retries} +
+ ) +} + +const ACTIVE_RUN_STATES: ReadonlySet = new Set(['CREATED', 'QUEUED', 'IN_PROGRESS']) +const NO_ANCESTORS: ReadonlySet = new Set() +const COLUMN_COUNT = 6 +const LABEL_OBJECTIVE_LENGTH = 60 +const INDENT_STEP = '1.5rem' + +/** Nesting depth, shown only in the first column so the other columns stay aligned. */ +function indent(depth: number): CSSProperties { + return { paddingInlineStart: `calc(${depth} * ${INDENT_STEP})` } +} + +/** Accessible name for a result; sibling results usually share an attack name, so add position and objective. */ +function nodeLabel(node: ScenarioHierarchyNode, position: number | null): string { + const objective = + node.objective.length > LABEL_OBJECTIVE_LENGTH + ? `${node.objective.slice(0, LABEL_OBJECTIVE_LENGTH)}...` + : node.objective + const prefix = position === null ? '' : `#${position + 1} ` + return `${prefix}${node.atomic_attack_name}: ${objective}` +} + +function formatCounts(counts: ScenarioProgressCounts): string { + const completed = counts.planned !== null ? `${counts.completed} of ${counts.planned}` : `${counts.completed}` + return `${completed} completed, ${counts.succeeded} succeeded, ${counts.errors} errors, ${counts.retries} retries` +} diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx index 52fec6b6d7..374d84c0ad 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunPage.test.tsx @@ -14,6 +14,7 @@ import { useScenarioQueue } from '@/hooks/useScenarioQueue' import { scenariosApi } from '@/services/api' import type { ScenarioComponentIdentity, + ScenarioHierarchyNode, ScenarioProgressHeader, ScenarioProgressSummary, ScenarioProgressResult, @@ -43,6 +44,9 @@ jest.mock('@/services/api', () => ({ cancelRun: jest.fn(), resumeRun: jest.fn(), getRunProgress: jest.fn(), + // The hierarchy section has its own tests; keep it loading so these tests cover the rest of the page. + getRunHierarchy: jest.fn(() => new Promise(() => {})), + getRunHierarchyNode: jest.fn(), }, })) @@ -151,6 +155,47 @@ const ATTEMPT: ScenarioProgressResult = { }, } +function hierarchyNode(overrides: Partial = {}): ScenarioHierarchyNode { + return { + attack_result_id: 'hierarchy-only', + conversation_id: 'conversation-h', + result_role: 'orchestration', + atomic_attack_name: 'sequential-parent', + atomic_group_id: 'atomic-1', + seed_group_id: 'seed-unplanned', + objective: 'Objective carried by the hierarchy node', + outcome: 'failure', + execution_time_ms: 1200, + executed_turns: 0, + timestamp: '2026-01-01T00:00:00Z', + total_retries: 0, + retries: [], + retries_truncated: false, + child_attack_result_ids: [], + child_references_truncated: false, + children: [], + children_truncated: false, + child_offset: 0, + unresolved_child_attack_result_ids: [], + unresolved_child_references_truncated: false, + conversations: [], + conversations_truncated: false, + ...overrides, + } +} + +function mockHierarchy(roots: ScenarioHierarchyNode[], status: ScenarioRunState = 'COMPLETED'): void { + const counts = { completed: 1, planned: 1, succeeded: 0, success_percentage: 0, errors: 0, retries: 0 } + ;(scenariosApi.getRunHierarchy as jest.Mock).mockResolvedValueOnce({ + status, + groups: [{ atomic_group_id: 'atomic-1', atomic_attack_name: 'a', display_group: 'Group', roots, counts }], + overall_counts: counts, + diagnostics: { rootless_results_count: 0 }, + truncated: false, + has_more: false, + }) +} + const SUMMARY: ScenarioProgressSummary = { overall: { completed: 1, @@ -687,6 +732,7 @@ describe('ScenarioRunPage', () => { 'Overall progress', 'Paired outcomes', 'Atomic attack groups', + 'Result hierarchy', 'Objective Scorer', 'Techniques', 'Objectives', @@ -1219,6 +1265,59 @@ describe('ScenarioRunPage', () => { } }) + it('opens details for a hierarchy result that the progress feed has not delivered yet', async () => { + const user = userEvent.setup() + mockHierarchy([hierarchyNode()]) + renderPage() + + await user.click(await screen.findByRole('button', { name: 'Open details for sequential-parent: Objective carried by the hierarchy node' })) + + const dialog = await screen.findByRole('dialog', { name: 'sequential-parent' }) + expect(within(dialog).getByText('Objective carried by the hierarchy node')).toBeInTheDocument() + }) + + it('keeps the hierarchy objective when the progress row for an unplanned seed is already loaded', async () => { + const user = userEvent.setup() + // The progress row wins as the selected attempt, but it carries no objective and its seed is not in the plan. + const progressRow = { ...ATTEMPT, attack_result_id: 'unplanned-result', seed_group_id: 'seed-unplanned' } + mockHookState(makeState({ results: [ATTEMPT, progressRow] })) + mockHierarchy([ + hierarchyNode({ + attack_result_id: 'unplanned-result', + atomic_attack_name: 'attack-technique', + objective: 'Objective stored on the saved result', + }), + ]) + renderPage() + + await user.click(await screen.findByRole('button', { name: 'Open details for attack-technique: Objective stored on the saved result' })) + + const dialog = await screen.findByRole('dialog', { name: 'attack-technique' }) + expect(within(dialog).getByText('Objective stored on the saved result')).toBeInTheDocument() + expect(within(dialog).queryByText(/Objective text unavailable/)).not.toBeInTheDocument() + }) + + it('offers to reload the result hierarchy after a failed run is resumed', async () => { + const user = userEvent.setup() + const failedState = makeState({ run: { ...makeState().run, status: 'FAILED' } }) + mockHookState(failedState) + mockResumeRun.mockResolvedValueOnce({ ...failedState.run, status: 'QUEUED' }) + mockApplyRunSummary.mockImplementationOnce(() => { + mockHookState({ ...failedState, run: { ...failedState.run, status: 'QUEUED' } }) + }) + mockHierarchy([hierarchyNode()], 'FAILED') + renderPage() + + const hierarchy = screen.getByRole('region', { name: 'Result hierarchy' }) + await within(hierarchy).findByText('Objective carried by the hierarchy node') + expect(within(hierarchy).queryByRole('button', { name: 'Reload hierarchy' })).not.toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Resume run' })) + + expect(await within(hierarchy).findByRole('button', { name: 'Reload hierarchy' })).toBeInTheDocument() + expect(within(hierarchy).getByText(/This run is still in progress/)).toBeInTheDocument() + }) + it('shows descriptive technique details without result metrics', async () => { const user = userEvent.setup() renderPage() diff --git a/frontend/src/components/Scenarios/ScenarioRunPage.tsx b/frontend/src/components/Scenarios/ScenarioRunPage.tsx index 0c96338a58..80b7981c67 100644 --- a/frontend/src/components/Scenarios/ScenarioRunPage.tsx +++ b/frontend/src/components/Scenarios/ScenarioRunPage.tsx @@ -51,6 +51,7 @@ import { useScenarioQueue } from '@/hooks/useScenarioQueue' import { scenariosApi } from '@/services/api' import { toApiError } from '@/services/errors' import type { + ScenarioHierarchyNode, ScenarioProgressCounts, ScenarioProgressHeader, ScenarioProgressResult, @@ -75,6 +76,7 @@ import { ObjectiveDetailsDialog, TechniqueDetailsDialog } from './ScenarioRunDia import { useScenarioRunPageStyles } from './ScenarioRunPage.styles' import ScenarioQueue from './ScenarioQueue' import PairedOutcomes from './PairedOutcomes' +import ScenarioResultHierarchy from './ScenarioResultHierarchy' const CLOCK_REFRESH_INTERVAL_MS = 1_000 const MAX_VISIBLE_ATTEMPTS_PER_GROUP = 100 @@ -129,6 +131,7 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu const [atomicGroupsExpandedChoice, setAtomicGroupsExpandedChoice] = useState(null) const [expandedGroupIds, setExpandedGroupIds] = useState>(new Set()) const detailsTriggerRef = useRef(null) + const [hierarchySelection, setHierarchySelection] = useState(null) const navigationState = location.state as { fromScenarioHistory?: boolean scenarioHistorySearch?: string @@ -163,9 +166,12 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu () => new Map(state.summary?.atomic_groups.map((group) => [group.id, group]) ?? []), [state.summary], ) - const selectedAttempt = useMemo( - () => state.results.find((attempt) => attempt.attack_result_id === attackResultId) ?? null, - [attackResultId, state.results], + // A hierarchy node can be opened before the paged progress feed has delivered it. + const selectedAttempt = useMemo( + () => + state.results.find((attempt) => attempt.attack_result_id === attackResultId) ?? + (hierarchySelection?.attack_result_id === attackResultId ? hierarchySelection : null), + [attackResultId, state.results, hierarchySelection], ) const selectedAttemptGroup = selectedAttempt ? plannedAtomicGroups.get(selectedAttempt.atomic_group_id) @@ -199,6 +205,12 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu navigate(scenarioRunAttackRoutePath(scenarioResultId, attempt.attack_result_id), { state: location.state }) } + const openResultDetails = (node: ScenarioHierarchyNode, trigger: HTMLElement): void => { + detailsTriggerRef.current = trigger + setHierarchySelection(node) + navigate(scenarioRunAttackRoutePath(scenarioResultId, node.attack_result_id), { state: location.state }) + } + const toggleDisplayGroup = (groupId: string): void => { setExpandedGroupIds((current) => { const updated = new Set(current) @@ -638,6 +650,18 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu +
+ + Result hierarchy + + +
+
Objective Scorer @@ -790,7 +814,13 @@ function ScenarioRunPageContent({ scenarioResultId, attackResultId }: ScenarioRu = {}): ScenarioHierarchyNode { + return { + attack_result_id: 'ar', + conversation_id: '', + result_role: 'target_facing', + atomic_attack_name: 'attack', + atomic_group_id: 'g1', + seed_group_id: 's1', + objective: 'objective', + outcome: 'success', + execution_time_ms: 0, + executed_turns: 0, + timestamp: '2026-10-28T00:00:00Z', + total_retries: 0, + retries: [], + retries_truncated: false, + child_attack_result_ids: [], + child_references_truncated: false, + children: [], + children_truncated: false, + child_offset: 0, + unresolved_child_attack_result_ids: [], + unresolved_child_references_truncated: false, + conversations: [], + conversations_truncated: false, + ...overrides, + } +} + +const counts = (completed: number) => ({ + completed, + planned: 10, + succeeded: 0, + success_percentage: null, + errors: 0, + retries: 0, +}) + +function group(id: string, roots: string[], completed: number): ScenarioHierarchyGroup { + return { + atomic_group_id: id, + atomic_attack_name: id, + display_group: id, + roots: roots.map((rootId) => node({ attack_result_id: rootId, atomic_group_id: id })), + counts: counts(completed), + } +} + +describe('childSlots', () => { + it('keeps every saved position, including unresolved and duplicate references', () => { + // a resolves at 200; 'missing' never resolves; the second 'a' is a duplicate and stays unresolved + const slots = childSlots({ + offset: 200, + referenceIds: ['a', 'missing', 'a'], + children: [node({ attack_result_id: 'a', child_index: 200 })], + }) + + expect(slots.map((slot) => [slot.position, slot.kind])).toEqual([ + [200, 'child'], + [201, 'unresolved'], + [202, 'unresolved'], + ]) + expect(slots[2]).toMatchObject({ kind: 'unresolved', attackResultId: 'a' }) + }) + + it('lays out a page where no reference resolves', () => { + const slots = childSlots({ offset: 0, referenceIds: ['x', 'y'], children: [] }) + + expect(slots.every((slot) => slot.kind === 'unresolved')).toBe(true) + expect(slots.map((slot) => slot.position)).toEqual([0, 1]) + }) +}) + +describe('mergeRootPage', () => { + it('appends roots to known groups and keeps the latest exact counts', () => { + const first = mergeRootPage([], { groups: [group('g1', ['r1'], 3)] } as ScenarioHierarchyProjection) + const merged = mergeRootPage(first, { + groups: [group('g1', ['r2'], 3), group('g2', ['r3'], 1)], + } as ScenarioHierarchyProjection) + + expect(merged.map((g) => [g.atomic_group_id, g.roots.map((r) => r.attack_result_id)])).toEqual([ + ['g1', ['r1', 'r2']], + ['g2', ['r3']], + ]) + expect(merged[0].counts.completed).toBe(3) + expect(first[0].roots).toHaveLength(1) + }) +}) + +describe('previewHasAllChildren', () => { + it('needs a request when the preview deferred or paged the children', () => { + expect(previewHasAllChildren(node({ child_attack_result_ids: ['a'], children_count: 1 }))).toBe(true) + expect(previewHasAllChildren(node({ child_attack_result_ids: ['a'], children_truncated: true }))).toBe(false) + expect(previewHasAllChildren(node({ child_attack_result_ids: ['a'], next_children_cursor: 'c' }))).toBe(false) + expect(previewHasAllChildren(node({ child_attack_result_ids: ['a'], children_count: 3 }))).toBe(false) + }) +}) diff --git a/frontend/src/components/Scenarios/scenarioHierarchy.ts b/frontend/src/components/Scenarios/scenarioHierarchy.ts new file mode 100644 index 0000000000..79202943f6 --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioHierarchy.ts @@ -0,0 +1,100 @@ +import type { + AttackResultRole, + ScenarioHierarchyGroup, + ScenarioHierarchyNode, + ScenarioHierarchyProjection, +} from '@/types' + +/** One saved child-reference position under a parent: a loaded child, or a reference that did not resolve. */ +export type HierarchyChildSlot = + | { readonly position: number; readonly kind: 'child'; readonly node: ScenarioHierarchyNode } + | { readonly position: number; readonly kind: 'unresolved'; readonly attackResultId: string } + +/** A loaded page of a parent's saved child references, in saved order. */ +export interface HierarchyChildPage { + readonly offset: number + readonly referenceIds: readonly string[] + readonly children: readonly ScenarioHierarchyNode[] +} + +/** Builds a child page from a node returned by either hierarchy endpoint. */ +export function childPageFromNode(node: ScenarioHierarchyNode): HierarchyChildPage { + return { + offset: node.child_offset, + referenceIds: node.child_attack_result_ids, + children: node.children, + } +} + +/** + * Lays out every saved reference position on a page. + * + * Resolved children are matched by their absolute `child_index`, not by id, because a duplicate + * reference to the same result is unresolved and still occupies its own position. + */ +export function childSlots(page: HierarchyChildPage): HierarchyChildSlot[] { + const childrenByPosition = new Map() + for (const child of page.children) { + if (child.child_index !== null && child.child_index !== undefined) { + childrenByPosition.set(child.child_index, child) + } + } + return page.referenceIds.map((attackResultId, i) => { + const position = page.offset + i + const node = childrenByPosition.get(position) + return node ? { position, kind: 'child', node } : { position, kind: 'unresolved', attackResultId } + }) +} + +/** + * Whether the tree preview already holds every saved child reference of a node, so expanding it + * needs no request. + */ +export function previewHasAllChildren(node: ScenarioHierarchyNode): boolean { + return ( + !node.children_truncated && + !node.child_references_truncated && + !node.next_children_cursor && + node.child_offset === 0 && + (node.children_count === null || + node.children_count === undefined || + node.children_count === node.child_attack_result_ids.length) + ) +} + +/** Whether a node has saved child references at all. */ +export function hasChildReferences(node: ScenarioHierarchyNode): boolean { + return node.child_attack_result_ids.length > 0 || (node.children_count ?? 0) > 0 +} + +/** + * Appends a page of roots to the groups already loaded. Groups keep their first-seen order; + * counts come from the latest page because they are exact run totals, not page totals. + */ +export function mergeRootPage( + groups: readonly ScenarioHierarchyGroup[], + page: ScenarioHierarchyProjection, +): ScenarioHierarchyGroup[] { + const merged = groups.map((group) => ({ ...group, roots: [...group.roots] })) + const indexById = new Map(merged.map((group, i) => [group.atomic_group_id, i])) + for (const group of page.groups) { + const i = indexById.get(group.atomic_group_id) + if (i === undefined) { + indexById.set(group.atomic_group_id, merged.length) + merged.push({ ...group, roots: [...group.roots] }) + } else { + merged[i] = { ...merged[i], counts: group.counts, roots: [...merged[i].roots, ...group.roots] } + } + } + return merged +} + +const ROLE_LABELS: Record = { + orchestration: 'Orchestration', + target_facing: 'Target-facing', + unknown: 'Unknown role', +} + +export function resultRoleLabel(role: AttackResultRole): string { + return ROLE_LABELS[role] ?? ROLE_LABELS.unknown +} diff --git a/frontend/src/hooks/useScenarioResultHierarchy.ts b/frontend/src/hooks/useScenarioResultHierarchy.ts new file mode 100644 index 0000000000..37c5ecaeec --- /dev/null +++ b/frontend/src/hooks/useScenarioResultHierarchy.ts @@ -0,0 +1,325 @@ +import { useCallback, useEffect, useRef, useState } from 'react' + +import { + childPageFromNode, + mergeRootPage, + previewHasAllChildren, + type HierarchyChildPage, +} from '@/components/Scenarios/scenarioHierarchy' +import { scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { + ScenarioHierarchyDiagnostics, + ScenarioHierarchyGroup, + ScenarioHierarchyNode, + ScenarioProgressCounts, + ScenarioRunState, +} from '@/types' + +export const HIERARCHY_MAX_NODES = 1000 +export const HIERARCHY_CHILD_PAGE_SIZE = 200 + +export interface HierarchyRootState { + readonly status: 'loading' | 'ready' | 'error' + /** Run state when the hierarchy was loaded; a run that is still going keeps adding results. */ + readonly runStatus: ScenarioRunState | null + readonly groups: ScenarioHierarchyGroup[] + readonly overallCounts: ScenarioProgressCounts | null + readonly diagnostics: ScenarioHierarchyDiagnostics | null + /** The preview left out deferred descendants, deep levels or results it could not place. */ + readonly truncated: boolean + readonly nextCursor: string | null + readonly loadingMore: boolean + readonly error: string | null + /** The root cursor no longer matches the saved run; paging has to restart. */ + readonly staleCursor: boolean +} + +export interface HierarchyChildrenState { + readonly pages: HierarchyChildPage[] + readonly childrenCount: number | null + readonly nextCursor: string | null + readonly loading: boolean + readonly error: string | null + /** The child cursor no longer matches the parent's saved references; paging has to restart. */ + readonly staleCursor: boolean +} + +const INITIAL_ROOTS: HierarchyRootState = { + status: 'loading', + runStatus: null, + groups: [], + overallCounts: null, + diagnostics: null, + truncated: false, + nextCursor: null, + loadingMore: false, + error: null, + staleCursor: false, +} + +const EMPTY_CHILDREN: HierarchyChildrenState = { + pages: [], + childrenCount: null, + nextCursor: null, + loading: false, + error: null, + staleCursor: false, +} + +export interface UseScenarioResultHierarchyResult { + readonly roots: HierarchyRootState + readonly children: ReadonlyMap + readonly expanded: ReadonlySet + /** Expands or collapses a node; expanding shows its children, loading them if needed. */ + readonly toggleNode: (key: string, node: ScenarioHierarchyNode) => void + readonly reloadRoots: () => void + readonly loadMoreRoots: () => void + /** Shows a node's children, from the preview when it is complete and otherwise from the API. */ + readonly openChildren: (key: string, node: ScenarioHierarchyNode) => void + readonly loadMoreChildren: (key: string, node: ScenarioHierarchyNode) => void + readonly restartChildren: (key: string, node: ScenarioHierarchyNode) => void +} + +/** State is not reset when `scenarioResultId` changes; mount a new instance per run (for example with a `key`). */ +export function useScenarioResultHierarchy(scenarioResultId: string): UseScenarioResultHierarchyResult { + const [roots, setRoots] = useState(INITIAL_ROOTS) + const [children, setChildren] = useState>(new Map()) + const [expanded, setExpanded] = useState>(new Set()) + const [rootEpoch, setRootEpoch] = useState(0) + const controllers = useRef(new Set()) + /** The child request in flight for each node, so a restart can drop a page that is still loading. */ + const childControllers = useRef(new Map()) + + const track = useCallback((): AbortController => { + const controller = new AbortController() + controllers.current.add(controller) + return controller + }, []) + + const release = useCallback((controller: AbortController) => { + controllers.current.delete(controller) + }, []) + + useEffect(() => { + const active = controllers.current + return () => { + for (const controller of active) { + controller.abort() + } + active.clear() + } + }, []) + + useEffect(() => { + const controller = track() + scenariosApi + .getRunHierarchy(scenarioResultId, { max_nodes: HIERARCHY_MAX_NODES }, controller.signal) + .then((page) => { + setRoots({ + status: 'ready', + runStatus: page.status, + groups: mergeRootPage([], page), + overallCounts: page.overall_counts, + diagnostics: page.diagnostics, + truncated: page.truncated, + nextCursor: page.has_more ? page.next_cursor ?? null : null, + loadingMore: false, + error: null, + staleCursor: false, + }) + }) + .catch((err: unknown) => { + if (controller.signal.aborted) { + return + } + setRoots({ ...INITIAL_ROOTS, status: 'error', error: toApiError(err).detail }) + }) + .finally(() => release(controller)) + return () => controller.abort() + }, [scenarioResultId, rootEpoch, track, release]) + + const reloadRoots = useCallback(() => { + // Late responses from before the reload must not land in the fresh tree. + for (const controller of controllers.current) { + controller.abort() + } + controllers.current.clear() + childControllers.current.clear() + setRoots(INITIAL_ROOTS) + setChildren(new Map()) + setExpanded(new Set()) + setRootEpoch((epoch) => epoch + 1) + }, []) + + const loadMoreRoots = useCallback(() => { + const cursor = roots.nextCursor + if (!cursor || roots.loadingMore) { + return + } + const controller = track() + setRoots((state) => ({ ...state, loadingMore: true, error: null, staleCursor: false })) + scenariosApi + .getRunHierarchy(scenarioResultId, { max_nodes: HIERARCHY_MAX_NODES, cursor }, controller.signal) + .then((page) => { + setRoots((state) => ({ + ...state, + runStatus: page.status, + groups: mergeRootPage(state.groups, page), + overallCounts: page.overall_counts, + diagnostics: page.diagnostics, + truncated: state.truncated || page.truncated, + nextCursor: page.has_more ? page.next_cursor ?? null : null, + loadingMore: false, + })) + }) + .catch((err: unknown) => { + if (controller.signal.aborted) { + return + } + const error = toApiError(err) + setRoots((state) => ({ + ...state, + loadingMore: false, + error: error.detail, + staleCursor: error.status === 400, + })) + }) + .finally(() => release(controller)) + }, [roots.nextCursor, roots.loadingMore, scenarioResultId, track, release]) + + const fetchChildren = useCallback( + (key: string, node: ScenarioHierarchyNode, cursor: string | null) => { + childControllers.current.get(key)?.abort() + const controller = track() + childControllers.current.set(key, controller) + setChildren((map) => { + const next = new Map(map) + const current = cursor ? map.get(key) ?? EMPTY_CHILDREN : EMPTY_CHILDREN + next.set(key, { ...current, loading: true, error: null, staleCursor: false }) + return next + }) + scenariosApi + .getRunHierarchyNode( + scenarioResultId, + node.attack_result_id, + cursor + ? { limit: HIERARCHY_CHILD_PAGE_SIZE, children_cursor: cursor } + : { limit: HIERARCHY_CHILD_PAGE_SIZE }, + controller.signal, + ) + .then((page) => { + setChildren((map) => { + const next = new Map(map) + const current = map.get(key) ?? EMPTY_CHILDREN + next.set(key, { + pages: [...current.pages, childPageFromNode(page)], + childrenCount: page.children_count ?? null, + nextCursor: page.next_children_cursor ?? null, + loading: false, + error: null, + staleCursor: false, + }) + return next + }) + }) + .catch((err: unknown) => { + if (controller.signal.aborted) { + return + } + const error = toApiError(err) + setChildren((map) => { + const next = new Map(map) + const current = map.get(key) ?? EMPTY_CHILDREN + next.set(key, { + ...current, + loading: false, + error: error.detail, + staleCursor: cursor !== null && error.status === 400, + }) + return next + }) + }) + .finally(() => { + release(controller) + if (childControllers.current.get(key) === controller) { + childControllers.current.delete(key) + } + }) + }, + [scenarioResultId, track, release], + ) + + const openChildren = useCallback( + (key: string, node: ScenarioHierarchyNode) => { + const existing = children.get(key) + if (existing && (existing.pages.length > 0 || existing.loading)) { + return + } + if (previewHasAllChildren(node)) { + setChildren((map) => { + const next = new Map(map) + next.set(key, { + ...EMPTY_CHILDREN, + pages: [childPageFromNode(node)], + childrenCount: node.children_count ?? node.child_attack_result_ids.length, + }) + return next + }) + return + } + fetchChildren(key, node, null) + }, + [children, fetchChildren], + ) + + const loadMoreChildren = useCallback( + (key: string, node: ScenarioHierarchyNode) => { + const state = children.get(key) + if (!state?.nextCursor || state.loading) { + return + } + fetchChildren(key, node, state.nextCursor) + }, + [children, fetchChildren], + ) + + const restartChildren = useCallback( + (key: string, node: ScenarioHierarchyNode) => { + // Descendants loaded under the old child pages may no longer match; forget them too. + const prefix = `${key}/` + setChildren((map) => new Map([...map].filter(([childKey]) => !childKey.startsWith(prefix)))) + setExpanded((keys) => new Set([...keys].filter((childKey) => !childKey.startsWith(prefix)))) + fetchChildren(key, node, null) + }, + [fetchChildren], + ) + + const toggleNode = useCallback( + (key: string, node: ScenarioHierarchyNode) => { + if (expanded.has(key)) { + setExpanded((keys) => { + const next = new Set(keys) + next.delete(key) + return next + }) + return + } + setExpanded((keys) => new Set(keys).add(key)) + openChildren(key, node) + }, + [expanded, openChildren], + ) + + return { + roots, + children, + expanded, + toggleNode, + reloadRoots, + loadMoreRoots, + openChildren, + loadMoreChildren, + restartChildren, + } +} diff --git a/frontend/src/services/api.test.ts b/frontend/src/services/api.test.ts index c40ce0d65b..bd368decf4 100644 --- a/frontend/src/services/api.test.ts +++ b/frontend/src/services/api.test.ts @@ -1252,6 +1252,52 @@ describe("api service", () => { }); }); + it("loads a hierarchy root page with its root cursor", async () => { + const projection = { + scenario_result_id: "sr-1", + groups: [], + has_more: true, + next_cursor: "roots-2", + }; + (apiClient.get as jest.Mock).mockResolvedValueOnce({ data: projection }); + const controller = new AbortController(); + + const result = await scenariosApi.getRunHierarchy( + "sr/1", + { max_nodes: 200, cursor: "roots-1" }, + controller.signal, + ); + + expect(apiClient.get).toHaveBeenCalledWith("/scenarios/runs/sr%2F1/hierarchy", { + params: { max_nodes: 200, cursor: "roots-1" }, + signal: controller.signal, + }); + expect(result).toBe(projection); + }); + + it("loads a node's children with the node's own children cursor", async () => { + const node = { + attack_result_id: "ar/1", + child_offset: 200, + next_children_cursor: "children-3", + }; + (apiClient.get as jest.Mock).mockResolvedValueOnce({ data: node }); + const controller = new AbortController(); + + const result = await scenariosApi.getRunHierarchyNode( + "sr/1", + "ar/1", + { limit: 200, children_cursor: "children-2" }, + controller.signal, + ); + + expect(apiClient.get).toHaveBeenCalledWith( + "/scenarios/runs/sr%2F1/hierarchy/nodes/ar%2F1", + { params: { limit: 200, children_cursor: "children-2" }, signal: controller.signal }, + ); + expect(result).toBe(node); + }); + it("cancels a scenario run by id", async () => { const mockResponse = { data: { diff --git a/frontend/src/services/api.ts b/frontend/src/services/api.ts index a8ddd7dc4c..6d2ed1bcba 100644 --- a/frontend/src/services/api.ts +++ b/frontend/src/services/api.ts @@ -66,6 +66,8 @@ import type { ScenarioRunSummary, ScenarioRunListResponse, ScenarioRunProgress, + ScenarioHierarchyNode, + ScenarioHierarchyProjection, PairedOutcomeStatistics, ScenarioQueueSnapshot, ScenarioRunState, @@ -724,6 +726,33 @@ export const scenariosApi = { return response.data }, + /** Loads one page of hierarchy roots. `cursor` continues roots only, never a node's children. */ + getRunHierarchy: async ( + scenarioResultId: string, + params?: { max_nodes?: number; cursor?: string }, + signal?: AbortSignal, + ): Promise => { + const response = await apiClient.get( + `/scenarios/runs/${encodeURIComponent(scenarioResultId)}/hierarchy`, + { params, signal }, + ) + return response.data + }, + + /** Loads one page of a node's saved children, continued with that node's `next_children_cursor`. */ + getRunHierarchyNode: async ( + scenarioResultId: string, + attackResultId: string, + params?: { limit?: number; children_cursor?: string }, + signal?: AbortSignal, + ): Promise => { + const response = await apiClient.get( + `/scenarios/runs/${encodeURIComponent(scenarioResultId)}/hierarchy/nodes/${encodeURIComponent(attackResultId)}`, + { params, signal }, + ) + return response.data + }, + getPairedOutcomes: async ( scenarioResultId: string, params: { baseline_group_id: string; attack_group_id: string }, diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index a962d2bd32..399834e866 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -2098,6 +2098,106 @@ export interface ScenarioAtomicGroupProgress extends ScenarioProgressCounts { technique_details?: ScenarioAttackTechniqueDetails | null } +/** What a saved attack result represents; not proof that the objective target was called. */ +export type AttackResultRole = 'target_facing' | 'orchestration' | 'unknown' + +export interface ScenarioOrchestrationParentSummary { + attack_result_id: string + result_role: AttackResultRole + outcome: AttackOutcome + outcome_reason?: string | null + attempt_index?: number | null + execution_time_ms: number + executed_turns: number + completion_policy?: string | null + child_attack_result_ids: string[] + child_references_truncated: boolean + error_type?: string | null + error_message?: string | null + timestamp: string + total_retries: number + retries: RetryEvent[] + retries_truncated: boolean +} + +export interface ScenarioHierarchyNode { + attack_result_id: string + conversation_id: string + parent_attack_result_id?: string | null + /** Absolute position under the immediate parent. */ + child_index?: number | null + result_role: AttackResultRole + attempt_index?: number | null + atomic_attack_name: string + atomic_group_id: string + seed_group_id: string + objective: string + outcome: AttackOutcome + outcome_reason?: string | null + execution_time_ms: number + executed_turns: number + timestamp: string + score?: ScenarioProgressScore | null + error_type?: string | null + error_message?: string | null + total_retries: number + retries: RetryEvent[] + retries_truncated: boolean + child_attack_result_ids: string[] + child_references_truncated: boolean + children: ScenarioHierarchyNode[] + children_truncated: boolean + /** Full saved child-reference count; null when not known. */ + children_count?: number | null + /** Position of the first child-reference on this page. */ + child_offset: number + /** Continuation for this node's children only; never for roots. */ + next_children_cursor?: string | null + unresolved_child_attack_result_ids: string[] + unresolved_child_references_truncated: boolean + conversations: ConversationSummary[] + conversations_count?: number | null + conversations_truncated: boolean + orchestration?: ScenarioOrchestrationParentSummary | null +} + +export interface ScenarioHierarchyGroup { + atomic_group_id: string + atomic_attack_name: string + display_group: string + technique?: ScenarioAttackTechniqueDetails | null + roots: ScenarioHierarchyNode[] + counts: ScenarioProgressCounts +} + +export interface ScenarioHierarchyDiagnostics { + failed_attacks: AttackErrorSummary[] + failed_attacks_count: number + failed_attacks_truncated: boolean + attack_retries: AttackRetrySummary[] + attack_retries_count: number + attack_retries_truncated: boolean + total_retries: number + rootless_result_ids: string[] + rootless_results_count: number + rootless_results_truncated: boolean +} + +export interface ScenarioHierarchyProjection { + scenario_result_id: string + scenario_name: string + scenario_version: number + status: ScenarioRunState + groups: ScenarioHierarchyGroup[] + /** Exact run totals; independent of which roots or children are loaded. */ + overall_counts: ScenarioProgressCounts + diagnostics: ScenarioHierarchyDiagnostics + /** Root continuation only. */ + has_more: boolean + next_cursor?: string | null + truncated: boolean +} + export interface ScenarioObjectiveScorerMetrics { accuracy: number accuracy_standard_error?: number | null diff --git a/pyrit/backend/README.md b/pyrit/backend/README.md index 85bb9c9765..f7a65fed4d 100644 --- a/pyrit/backend/README.md +++ b/pyrit/backend/README.md @@ -75,7 +75,8 @@ and transcript reads, including hierarchy node expansion. ### Scenario result hierarchy These read-only APIs provide the backend foundation for compound-attack browsing. -The hierarchy GUI is separate work; the current scanner still shows flat executions. +The Scenario run page's "Result hierarchy" section browses them read-only; saved +conversation evidence for each node is separate follow-up work. `GET /api/scenarios/runs/{scenario_result_id}/hierarchy?max_nodes=1000` returns a bounded tree preview grouped by atomic attack group. Roots are saved results with