diff --git a/frontend/src/components/Scenarios/ScenarioDetail.tsx b/frontend/src/components/Scenarios/ScenarioDetail.tsx index 600be2eafa..2cc0db0d4a 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.tsx @@ -1,9 +1,8 @@ -import { type FormEvent, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' +import { useCallback, useEffect, useLayoutEffect, useMemo, useReducer, useRef, useState } from 'react' import { Badge, Button, - Checkbox, Dialog, DialogActions, DialogBody, @@ -11,21 +10,17 @@ import { DialogSurface, DialogTitle, Field, - Radio, MessageBar, MessageBarBody, - mergeClasses, Select, Spinner, Switch, Text, Tooltip, - ToggleButton, } from '@fluentui/react-components' import { ArrowLeftRegular, ArrowSyncRegular, - InfoRegular, SettingsRegular, } from '@fluentui/react-icons' import { Link, useNavigate, useParams, useSearchParams } from 'react-router' @@ -38,79 +33,61 @@ import { useConversationDraft } from '@/hooks/useConversationDraft' import { useRuntime } from '@/hooks/useRuntime' import { useRunTechniqueEditor } from '@/hooks/useRunTechniqueEditor' import { useScenarioDatasets } from '@/hooks/useScenarioDatasets' +import { useScenarioDetail } from '@/hooks/useScenarioDetail' +import { useScenarioLaunch } from '@/hooks/useScenarioLaunch' +import { useScenarioRunEstimate } from '@/hooks/useScenarioRunEstimate' import ParameterField from '@/components/Parameters/ParameterField' import SingleStepSpinButton from '@/components/Parameters/SingleStepSpinButton' -import { - buildParametersFromForm, - getInitialFormValues, - type ParameterFormValue, -} from '@/components/Parameters/parameterForm' +import type { ParameterFormValue } from '@/components/Parameters/parameterForm' import type { ViewName } from '@/components/Sidebar/Navigation' -import { attacksApi, scenariosApi } from '@/services/api' +import { attacksApi } from '@/services/api' import { toApiError } from '@/services/errors' import type { AddMessageResponse, Parameter, RegisteredScenario, - RunScenarioRequest, - ScenarioRunEstimate, - ScenarioRunEstimateResult, - ScenarioRunSizeEstimateRequest, - ScenarioRunEstimateState, - ScenarioTechniqueSummary, + ScenarioDatasetOptions, ScenarioDatasetRequest, ScenarioDatasetSettings, - ScenarioDatasetOptions, - ScenarioRunSizeEstimateResponse, + ScenarioRunSummary, TargetInstance, - TechniqueOverride, } from '@/types' import { newDraftMessage, toConversationDraft } from '@/utils/conversationDraft' import { routerPathParamValue, scenarioRunRoutePath } from '@/utils/routeParams' -import { sameTarget, targetModelName } from '@/utils/targetIdentity' +import { targetModelName } from '@/utils/targetIdentity' import { useScenarioDetailStyles } from './ScenarioDetail.styles' -import { ScenarioRunEstimateDetails } from './ScenarioRunEstimate' +import ScenarioEstimateSection from './ScenarioEstimateSection' +import ScenarioRunPreviewDialog from './ScenarioRunPreviewDialog' +import ScenarioTechniqueSection from './ScenarioTechniqueSection' import { scenarioDatasetDefaults, scenarioDatasetOptions } from './datasetPresentation' import { normalizeScenarioMarkdown } from './scenarioMarkdown' -import { mapScenarioRunEstimate } from './scenarioRunEstimateAdapter' -import { techniqueSetName } from './scenarioTechniqueSets' +import { + BASELINE_TECHNIQUE, + exceedsTechniqueLimit, + initialScenarioLaunchForm, + reduceScenarioLaunchForm, + type ScenarioLaunchFormAction, + type ScenarioLaunchFormRules, + type ScenarioLaunchFormState, + type SelectableTechnique, + uniqueTechniqueOptions, +} from './scenarioLaunchForm' +import { + buildEstimateRequest, + buildRunRequest, + CONVERSATION_PARAMETER_NAMES, + MAX_MAX_CONCURRENCY, + MAX_MAX_RETRIES, + MIN_MAX_CONCURRENCY, + MIN_MAX_RETRIES, + OBJECTIVE_PARAMETER_NAMES, + type ScenarioLaunchInput, + scenarioDynamicParameters, + supportsParameter, +} from './scenarioLaunchRequest' import ScenarioDatasetTable from './ScenarioDatasetTable' -/** - * Common/opaque parameters the base scenario declares via - * `Scenario._common_scenario_parameters` — the launch form exposes a - * purpose-built control for each of these (target, techniques, datasets, - * labels, concurrency, retries, baseline), and `technique_converters` has no - * UI at all. They're hidden from the dynamic scenario-specific parameter list. - */ -const COMMON_SCENARIO_PARAMETER_NAMES = new Set([ - 'objective_target', - 'scenario_techniques', - 'technique_converters', - 'local_technique_factory', - 'dataset_config', - 'dataset_overrides', - 'memory_labels', - 'max_concurrency', - 'max_retries', - 'include_baseline', -]) - -const MIN_MAX_CONCURRENCY = 1 -const MAX_MAX_CONCURRENCY = 100 -const MIN_MAX_RETRIES = 0 -const MAX_MAX_RETRIES = 20 -const DEFAULT_MAX_CONCURRENCY = 10 -const DEFAULT_MAX_RETRIES = 0 -const ESTIMATE_DEBOUNCE_MS = 300 -const CONVERSATION_PARAMETER_NAMES = new Set(['prepended_conversation_id', 'apply_converters_to_prepended_history']) -const OBJECTIVE_PARAMETER_NAMES = new Set(['objective', 'harm_categories']) - -function supportsParameter(scenario: RegisteredScenario, name: string): boolean { - return scenario.supported_parameter_names.includes(name) -} - function targetOptionLabel(target: TargetInstance): string { const modelName = targetModelName(target) return modelName @@ -127,292 +104,6 @@ function resolveSpinButtonValue(data: { value?: number | null; displayValue?: st return Number.isFinite(parsed) ? parsed : previous } -type LoadStatus = 'loading' | 'success' | 'not-found' | 'error' - -interface TechniqueOptions { - techniques: ScenarioTechniqueSummary[] - defaultTechniques: string[] -} - -function uniqueTechniqueOptions(scenario: RegisteredScenario): TechniqueOptions { - const aggregateNames = new Set(scenario.aggregate_techniques) - const summariesByName = new Map( - scenario.technique_summaries.map((summary) => [summary.name, summary]), - ) - const techniques: ScenarioTechniqueSummary[] = [] - const seen = new Set() - for (const name of scenario.all_techniques) { - if (!aggregateNames.has(name) && !seen.has(name)) { - techniques.push(summariesByName.get(name) ?? { name, description: null, tags: [] }) - seen.add(name) - } - } - const concreteNames = new Set(techniques.map((technique) => technique.name)) - const defaultTechniques = scenario.default_techniques.filter((name) => concreteNames.has(name)) - if (defaultTechniques.length === 0 && concreteNames.has(scenario.default_technique)) { - defaultTechniques.push(scenario.default_technique) - } - return { techniques, defaultTechniques } -} - -interface SelectableTechnique extends ScenarioTechniqueSummary { - isBaseline: boolean - disabled: boolean -} - -const BASELINE_TECHNIQUE: ScenarioTechniqueSummary = { - name: 'baseline', - description: 'Sends each objective directly to the target for comparison.', - tags: ['baseline', 'single_turn'], -} - -function formatParameterPreview(value: ParameterFormValue | undefined): string { - if (Array.isArray(value)) { - return value.length > 0 ? value.join(', ') : 'Not set' - } - if (typeof value === 'object') { - return value.type || 'Not set' - } - return value?.trim() || 'Not set' -} - -function formatEffectiveParameterPreview( - parameterName: string, - value: ParameterFormValue | undefined, - estimate: ScenarioRunEstimate | undefined, -): string { - const configuredValue = formatParameterPreview(value) - if (configuredValue !== 'Not set') { - return configuredValue - } - const effectiveValue = estimate?.effectiveParameters[parameterName] - if (Array.isArray(effectiveValue)) { - return effectiveValue.length > 0 ? effectiveValue.join(', ') : 'Not set' - } - return effectiveValue?.toString() ?? 'Not set' -} -function estimateFromState(state: ScenarioRunEstimateState): ScenarioRunEstimate | undefined { - switch (state.status) { - case 'available': - case 'conditional': - case 'refreshing': - case 'stale': - return state.estimate - default: - return undefined - } -} - -function formatAtomicAttackCount(state: ScenarioRunEstimateState): string { - const estimate = estimateFromState(state) - if (!estimate) { - return state.status === 'loading' ? 'Calculating...' : 'Unknown' - } - const prefix = estimate.approximate ? 'About ' : '' - if (estimate.total !== null) { - return `${estimate.approximate ? 'Up to ' : ''}${estimate.total.toLocaleString()}` - } - if (estimate.minimum != null && estimate.maximum != null) { - return estimate.minimum === estimate.maximum - ? `${prefix}${estimate.minimum.toLocaleString()}` - : `${prefix}${estimate.minimum.toLocaleString()}-${estimate.maximum.toLocaleString()}` - } - if (estimate.minimum != null) { - return `At least ${prefix.toLowerCase()}${estimate.minimum.toLocaleString()}` - } - if (estimate.maximum != null) { - return `Up to ${estimate.maximum.toLocaleString()}` - } - return 'Varies' -} - -function estimateNotes(state: ScenarioRunEstimateState): string | null { - const estimate = estimateFromState(state) - if (!estimate) { - return state.status === 'unavailable' ? state.note ?? state.label : null - } - const notes = [ - estimate.note, - ...estimate.components.map((component) => component.note), - ].filter((note): note is string => Boolean(note)) - return notes.length > 0 ? notes.join('\n\n') : null -} - -interface BuildEstimateRequestInput { - scenario: RegisteredScenario - targetName: string - adversarialTargetName: string - techniques: string[] - dynamicParameters: Parameter[] - scenarioParamValues: Record - datasetEdits: ScenarioDatasetRequest - datasetError: string | null - includeBaseline: boolean - techniqueOverride?: TechniqueOverride - techniqueEditError?: string - supportsPrependedConversation?: boolean -} - -interface BuildRunRequestInput extends BuildEstimateRequestInput { - operationId: string | null - maxConcurrency: number - maxRetries: number - labels: Record -} - -type BuildEstimateRequestResult = - | { - ok: true - request: ScenarioRunSizeEstimateRequest - } - | { - ok: false - error: string - } - -type BuildRunRequestResult = - | { - ok: true - request: RunScenarioRequest - } - | { - ok: false - error: string - } - -type SuccessfulEstimateResult = Extract< - ScenarioRunEstimateResult, - { status: 'available' | 'conditional' } -> - -type EstimateRequestState = - | { - status: 'resolved' - requestKey: string - result: ScenarioRunEstimateResult - datasetSettings?: ScenarioDatasetSettings - } - | { - status: 'error' - requestKey: string - error: string - } - -function buildEstimateRequest({ - scenario, - targetName, - adversarialTargetName, - techniques, - dynamicParameters, - scenarioParamValues, - datasetEdits, - datasetError, - includeBaseline, - techniqueOverride, - techniqueEditError, - supportsPrependedConversation = true, -}: BuildEstimateRequestInput): BuildEstimateRequestResult { - if (techniqueEditError) return { ok: false, error: techniqueEditError } - if (supportsParameter(scenario, 'scenario_techniques') && techniques.length === 0) { - return { ok: false, error: 'Select at least one technique.' } - } - if (supportsParameter(scenario, 'scenario_techniques') - && scenario.max_concrete_techniques != null && techniques.length > scenario.max_concrete_techniques) { - return { ok: false, error: `Select at most ${scenario.max_concrete_techniques} concrete technique(s).` } - } - - let scenarioParams: Record | null = null - if (dynamicParameters.length > 0) { - const history = scenarioParamValues.prepended_conversation_id - if (!supportsPrependedConversation && typeof history === 'string' && history.trim()) { - return { ok: false, error: 'This technique does not support a prepended conversation. Clear the saved conversation before launch.' } - } - const result = buildParametersFromForm( - dynamicParameters.filter((parameter) => supportsPrependedConversation || !CONVERSATION_PARAMETER_NAMES.has(parameter.name)), - scenarioParamValues, - ) - if (!result.ok) { - return result - } - scenarioParams = result.parameters - } - - const supportsDatasets = supportsParameter(scenario, 'dataset_config') - if (supportsDatasets && datasetError) return { ok: false, error: datasetError } - const request: ScenarioRunSizeEstimateRequest = {} - if (techniqueOverride) request.technique_override = techniqueOverride - if (supportsParameter(scenario, 'scenario_techniques')) { - request.techniques = techniques - } - if (supportsParameter(scenario, 'include_baseline')) { - request.include_baseline = includeBaseline - } - if (supportsParameter(scenario, 'objective_target') && targetName) { - request.target_name = targetName - } - if (scenario.uses_default_adversarial_target && adversarialTargetName) { - request.adversarial_target_name = adversarialTargetName - } - if (supportsDatasets && Object.keys(datasetEdits).length > 0) { - request.datasets = datasetEdits - } - if (scenarioParams) { - request.scenario_params = scenarioParams - } - return { ok: true, request } -} - -function buildRunRequest(input: BuildRunRequestInput): BuildRunRequestResult { - if (supportsParameter(input.scenario, 'objective_target') && !input.targetName) { - return { ok: false, error: 'Select a target.' } - } - const estimateResult = buildEstimateRequest(input) - if (!estimateResult.ok) { - return estimateResult - } - if ( - supportsParameter(input.scenario, 'max_concurrency') && ( - !Number.isInteger(input.maxConcurrency) - || input.maxConcurrency < MIN_MAX_CONCURRENCY - || input.maxConcurrency > MAX_MAX_CONCURRENCY - ) - ) { - return { - ok: false, - error: `Max concurrency must be an integer from ${MIN_MAX_CONCURRENCY} to ${MAX_MAX_CONCURRENCY}.`, - } - } - if ( - supportsParameter(input.scenario, 'max_retries') && ( - !Number.isInteger(input.maxRetries) - || input.maxRetries < MIN_MAX_RETRIES - || input.maxRetries > MAX_MAX_RETRIES - ) - ) { - return { - ok: false, - error: `Max retries must be an integer from ${MIN_MAX_RETRIES} to ${MAX_MAX_RETRIES}.`, - } - } - - const estimateRequest = estimateResult.request - const request: RunScenarioRequest = { - scenario_name: input.scenario.scenario_name, - ...estimateRequest, - } - if (input.operationId) request.operation_id = input.operationId - if (supportsParameter(input.scenario, 'max_concurrency')) { - request.max_concurrency = input.maxConcurrency - } - if (supportsParameter(input.scenario, 'max_retries')) { - request.max_retries = input.maxRetries - } - if (supportsParameter(input.scenario, 'memory_labels')) { - request.labels = input.labels - } - return { ok: true, request } -} - interface ScenarioDetailProps { targets: TargetInstance[] defaultObjectiveTarget: TargetInstance | null @@ -453,47 +144,7 @@ function ScenarioDetailContent({ }: ScenarioDetailContentProps) { const styles = useScenarioDetailStyles() const decodedScenarioName = routerPathParamValue(encodedScenarioName) - - const [scenario, setScenario] = useState(null) - const { generation, ready } = useRuntime() - const [scenarioStatus, setScenarioStatus] = useState('loading') - const [scenarioError, setScenarioError] = useState(null) - const [refetchCount, setRefetchCount] = useState(0) - - useEffect(() => { - if (!ready) return - let cancelled = false - scenariosApi - .getScenario(decodedScenarioName) - .then((data) => { - if (cancelled) return - if (!Array.isArray(data.supported_parameter_names)) { - setScenario(null) - setScenarioStatus('error') - setScenarioError('The scenario catalog has no input support metadata. Update the backend and reload.') - return - } - setScenario(data) - setScenarioStatus('success') - setScenarioError(null) - }) - .catch((err: unknown) => { - if (cancelled) return - const apiError = toApiError(err) - setScenario(null) - setScenarioStatus(apiError.status === 404 ? 'not-found' : 'error') - setScenarioError(apiError.status === 404 ? null : apiError.detail) - }) - return () => { - cancelled = true - } - }, [decodedScenarioName, refetchCount, generation, ready]) - - const handleRetry = (): void => { - setScenarioStatus('loading') - setScenarioError(null) - setRefetchCount((count) => count + 1) - } + const { status: scenarioStatus, scenario, error: scenarioError, retry } = useScenarioDetail(decodedScenarioName) if (scenarioStatus === 'loading') { return ( @@ -536,7 +187,7 @@ function ScenarioDetailContent({ className={styles.touchTarget} appearance="primary" icon={} - onClick={handleRetry} + onClick={retry} data-testid="retry-btn" > Retry @@ -603,13 +254,7 @@ function ScenarioLaunchForm({ () => uniqueTechniqueOptions(scenario), [scenario], ) - const dynamicParameters = useMemo( - () => scenario.supported_parameters.filter( - (parameter) => supportsParameter(scenario, parameter.name) - && !COMMON_SCENARIO_PARAMETER_NAMES.has(parameter.name), - ), - [scenario], - ) + const dynamicParameters = useMemo(() => scenarioDynamicParameters(scenario), [scenario]) const conversationConvertersParameter = dynamicParameters.some((parameter) => parameter.name === 'prepended_conversation_id') ? dynamicParameters.find((parameter) => parameter.name === 'apply_converters_to_prepended_history') : undefined @@ -621,25 +266,53 @@ function ScenarioLaunchForm({ const supportsRetries = supportsParameter(scenario, 'max_retries') const isBaselineForbidden = scenario.baseline_policy === 'forbidden' const singleTechnique = scenario.max_concrete_techniques === 1 - - const [targetName, setTargetName] = useState(() => { - if (defaultObjectiveTarget && targets.some((target: TargetInstance) => - sameTarget(target, defaultObjectiveTarget))) { - return defaultObjectiveTarget.target_registry_name - } - return '' - }) const adversarialTargets = targets.filter( (target: TargetInstance) => target.capabilities?.supports_multi_turn === true, ) - const [adversarialTargetName, setAdversarialTargetName] = useState(() => { - if (defaultAdversarialTarget && adversarialTargets.some((target: TargetInstance) => - sameTarget(target, defaultAdversarialTarget))) { - return defaultAdversarialTarget.target_registry_name - } - return '' - }) - const [selectedTechniques, setSelectedTechniques] = useState(() => defaultTechniques) + + const formRules = useMemo(() => ({ + techniqueNames: techniqueOptions.map((technique) => technique.name), + maxConcreteTechniques: scenario.max_concrete_techniques ?? null, + singleTechnique, + }), [techniqueOptions, scenario.max_concrete_techniques, singleTechnique]) + const [form, dispatch] = useReducer( + (state: ScenarioLaunchFormState, action: ScenarioLaunchFormAction) => + reduceScenarioLaunchForm(state, action, formRules), + undefined, + () => initialScenarioLaunchForm({ + scenario, + targets, + defaultObjectiveTarget, + adversarialTargets, + defaultAdversarialTarget, + defaultTechniques, + dynamicParameters, + initialConversationId, + }), + ) + const { + targetName, + adversarialTargetName, + selectedTechniques, + baselineChecked, + datasetEdits, + datasetError, + maxConcurrency, + maxRetries, + scenarioParamValues, + } = form + const setDatasetEdits = useCallback( + (edits: ScenarioDatasetRequest) => dispatch({ type: 'setDatasetEdits', edits }), + [], + ) + const setDatasetError = useCallback( + (error: string | null) => dispatch({ type: 'setDatasetError', error }), + [], + ) + const updateScenarioParam = (name: string, value: ParameterFormValue): void => { + dispatch({ type: 'setParam', name, value }) + } + const techniqueEditor = useRunTechniqueEditor( singleTechnique && supportsTechniques && selectedTechniques.length === 1 ? selectedTechniques[0] : undefined, runtime.generation, runtime.ready, @@ -660,13 +333,69 @@ function ScenarioLaunchForm({ && !targets.some((target) => target.target_registry_name === targetName) ) || (supportsTechniques && selectedTechniques.some((name) => !techniqueOptions.some((technique) => technique.name === name))) - const [baselineChecked, setBaselineChecked] = useState( - () => !isBaselineForbidden && scenario.include_baseline_by_default, + + // The one effective configuration both the estimate and the launch read. + const launchInput = useMemo(() => ({ + scenario, + targetName, + adversarialTargetName, + techniques: selectedTechniques, + techniqueOverride: techniqueEditor.override, + techniqueEditError, + supportsPrependedConversation, + dynamicParameters, + scenarioParamValues, + datasetEdits, + datasetError, + maxConcurrency, + maxRetries, + includeBaseline: isBaselineForbidden ? false : baselineChecked, + labels, + operationId, + }), [ + adversarialTargetName, + baselineChecked, + datasetEdits, + datasetError, + dynamicParameters, + isBaselineForbidden, + labels, + maxConcurrency, + maxRetries, + operationId, + scenario, + scenarioParamValues, + selectedTechniques, + supportsPrependedConversation, + targetName, + techniqueEditError, + techniqueEditor.override, + ]) + const estimateResult = useMemo(() => buildEstimateRequest(launchInput), [launchInput]) + const requestResult = useMemo(() => buildRunRequest(launchInput), [launchInput]) + const estimateRequest = useMemo( + () => estimateResult.ok ? estimateResult.request : null, + [estimateResult], + ) + const estimateRequestKey = useMemo( + () => estimateRequest === null + ? null + : JSON.stringify({ scenarioName: scenario.scenario_name, request: estimateRequest, generation: runtime.generation }), + [estimateRequest, scenario.scenario_name, runtime.generation], ) - const [datasetEdits, setDatasetEdits] = useState({}) - const [datasetError, setDatasetError] = useState(null) const datasetMetadata = useScenarioDatasets() - const [datasetProjection, setDatasetProjection] = useState(null) + const runEstimate = useScenarioRunEstimate({ + scenarioName: scenario.scenario_name, + request: estimateRequest, + requestKey: estimateRequestKey, + invalidReason: estimateResult.ok ? null : estimateResult.error, + ready: runtime.ready, + paused: staleSelection, + }) + const estimateState = runEstimate.state + const currentDatasetEstimate = runEstimate.current + + const datasetProjection = runEstimate.datasetProjection const datasetDefaults: ScenarioDatasetSettings = datasetProjection?.dataset_defaults ?? scenarioDatasetDefaults(scenario) const effectiveDatasets = datasetProjection?.effective_datasets ?? datasetDefaults @@ -682,15 +411,21 @@ function ScenarioLaunchForm({ reloadDatasetMetadata() } }, [datasetSelectionKey, reloadDatasetMetadata]) - const [maxConcurrency, setMaxConcurrency] = useState(DEFAULT_MAX_CONCURRENCY) - const [maxRetries, setMaxRetries] = useState(DEFAULT_MAX_RETRIES) - const [scenarioParamValues, setScenarioParamValues] = useState>(() => - getInitialFormValues(dynamicParameters, initialConversationId ? { prepended_conversation_id: initialConversationId } : undefined), - ) - const [validationError, setValidationError] = useState(null) - const [apiError, setApiError] = useState(null) - const [submitting, setSubmitting] = useState(false) - const [previewOpen, setPreviewOpen] = useState(false) + const confirmedDatasetSettings = currentDatasetEstimate?.status === 'resolved' + ? currentDatasetEstimate.datasetSettings : undefined + const checkingRefreshPolicies = supportsDatasets && selectedDatasetNames.length > 0 && currentDatasetEstimate === null + const unconfirmedRefreshPolicies = supportsDatasets && selectedDatasetNames.length > 0 + && !checkingRefreshPolicies && confirmedDatasetSettings === undefined + const refreshDatasets = [...new Set((confirmedDatasetSettings?.sources ?? []) + .filter((source) => source.fetch === 'always').map((source) => source.name))] + const editedRefreshDatasets = refreshDatasets.filter((name) => { + const metadata = datasetMetadata.items.find((item) => item.name === name) + return metadata?.loaded && (!metadata.origins?.length + || metadata.origins.some((origin) => origin === 'user' || origin === 'unknown')) + }) + const uncheckedRefreshDatasets = refreshDatasets.filter((name) => datasetMetadata.loading || datasetMetadata.error + || !datasetMetadata.items.some((item) => item.name === name)) + const newAttackContext = useMemo(() => ({ generation: runtime.generation, ready: runtime.ready && defaultsReady && !staleSelection, @@ -709,8 +444,33 @@ function ScenarioLaunchForm({ const configuredConversationId = scenarioParamValues.prepended_conversation_id const canEditConversation = typeof configuredConversationId === 'string' && Boolean(configuredConversationId.trim()) + const { + previewOpen, + submitting, + apiError, + validationError, + clearValidationError, + isSubmitting, + launchButtonRef, + submitForm, + dismissPreview, + changePreviewOpen, + confirmLaunch, + } = useScenarioLaunch({ + requestResult, + submitBlocked: conversationBusy || Boolean(techniqueEditError), + launchBlocked: conversationBusy || Boolean(techniqueEditError) || !runtime.ready || !defaultsReady + || staleSelection || unavailableSelection || checkingRefreshPolicies, + onLaunched: (summary: ScenarioRunSummary) => { + datasetMetadata.reload() + navigate(scenarioRunRoutePath(summary.scenario_result_id), { + state: { scenarioName: scenario.scenario_name }, + }) + }, + }) + const beginConversation = async (parameterName: string, edit: boolean): Promise => { - if (conversationBusy || previewOpen || isSubmittingRef.current) return + if (conversationBusy || previewOpen || isSubmitting()) return const sequence = ++conversationLoadSequence.current const initialContext = conversationContextRef.current setConversationParameter(parameterName) @@ -740,30 +500,6 @@ function ScenarioLaunchForm({ if (sequence === conversationLoadSequence.current) setLoadingConversation(false) } } - const [estimateRequestState, setEstimateRequestState] = useState(null) - const [lastGoodEstimate, setLastGoodEstimate] = useState(null) - // Synchronous guard against a double-submit racing ahead of the state update. - const isSubmittingRef = useRef(false) - const estimateSequenceRef = useRef(0) - const launchButtonRef = useRef(null) - - const restoreLaunchFocus = (): void => { - const focus = () => { - launchButtonRef.current?.focus() - } - if (typeof requestAnimationFrame === 'function') { - requestAnimationFrame(focus) - } else { - setTimeout(focus, 0) - } - } - - const handleDismissPreview = (): void => { - if (!submitting) { - setPreviewOpen(false) - restoreLaunchFocus() - } - } const selectableTechniques = useMemo( () => [ @@ -780,220 +516,21 @@ function ScenarioLaunchForm({ ], [isBaselineForbidden, supportsBaseline, supportsTechniques, techniqueOptions], ) - const techniques = selectedTechniques const datasetSizeLabel = !datasetOptions.total_limit ? 'Not applicable' : effectiveDatasets.max_total === 'all' ? 'No total limit' : String(effectiveDatasets.max_total) - const estimateResult = useMemo( - () => buildEstimateRequest({ - scenario, - targetName, - adversarialTargetName, - techniques, - techniqueOverride: techniqueEditor.override, - techniqueEditError, - supportsPrependedConversation, - dynamicParameters, - scenarioParamValues, - datasetEdits, - datasetError, - includeBaseline: isBaselineForbidden ? false : baselineChecked, - }), - [ - adversarialTargetName, - baselineChecked, - datasetEdits, - datasetError, - dynamicParameters, - isBaselineForbidden, - scenario, - scenarioParamValues, - targetName, - techniques, - techniqueEditor.override, - techniqueEditError, - supportsPrependedConversation, - ], - ) - const requestResult = useMemo( - () => buildRunRequest({ - scenario, - targetName, - adversarialTargetName, - techniques, - techniqueOverride: techniqueEditor.override, - techniqueEditError, - supportsPrependedConversation, - dynamicParameters, - scenarioParamValues, - datasetEdits, - datasetError, - maxConcurrency, - maxRetries, - includeBaseline: isBaselineForbidden ? false : baselineChecked, - labels, - operationId, - }), - [ - adversarialTargetName, - baselineChecked, - datasetEdits, - datasetError, - dynamicParameters, - isBaselineForbidden, - labels, - maxConcurrency, - maxRetries, - operationId, - scenario, - scenarioParamValues, - targetName, - techniques, - techniqueEditor.override, - techniqueEditError, - supportsPrependedConversation, - ], - ) - const estimateRequest = useMemo( - () => estimateResult.ok ? estimateResult.request : null, - [estimateResult], - ) - const estimateRequestKey = useMemo( - () => estimateRequest === null - ? null - : JSON.stringify({ scenarioName: scenario.scenario_name, request: estimateRequest, generation: runtime.generation }), - [estimateRequest, scenario.scenario_name, runtime.generation], - ) - - useEffect(() => { - if (!runtime.ready || staleSelection || estimateRequest === null || estimateRequestKey === null) { - return - } - - const requestSequence = estimateSequenceRef.current + 1 - estimateSequenceRef.current = requestSequence - const controller = new AbortController() - - const debounceTimer = window.setTimeout(() => { - scenariosApi - .estimateRun(scenario.scenario_name, estimateRequest, controller.signal) - .then((response) => { - if ( - controller.signal.aborted - || requestSequence !== estimateSequenceRef.current - ) { - return - } - const result = mapScenarioRunEstimate(response, 'request') - setDatasetProjection(response) - setEstimateRequestState({ - status: 'resolved', - requestKey: estimateRequestKey, - result, - datasetSettings: response.effective_datasets, - }) - if (result.status === 'available' || result.status === 'conditional') { - setLastGoodEstimate(result) - } - }) - .catch((err: unknown) => { - if ( - controller.signal.aborted - || requestSequence !== estimateSequenceRef.current - ) { - return - } - const error = toApiError(err) - setEstimateRequestState({ - status: 'error', - requestKey: estimateRequestKey, - error: error.detail, - }) - if (estimateRequest.datasets && (error.status === 400 || error.status === 422)) { - const { datasets, ...defaultRequest } = estimateRequest - void datasets - // Keep invalid edits visible while refreshing parameter-aware allowed sources. - void scenariosApi.estimateRun(scenario.scenario_name, defaultRequest, controller.signal) - .then((response) => { - if (controller.signal.aborted || requestSequence !== estimateSequenceRef.current) return - setDatasetProjection((previous) => ({ - ...response, - effective_datasets: previous?.effective_datasets ?? response.effective_datasets, - })) - }) - .catch(() => { /* The original validation error remains authoritative. */ }) - } - }) - }, ESTIMATE_DEBOUNCE_MS) - - return () => { - window.clearTimeout(debounceTimer) - controller.abort() - } - }, [estimateRequest, estimateRequestKey, scenario.scenario_name, runtime.ready, staleSelection]) - - let estimateState: ScenarioRunEstimateState - if (!estimateResult.ok) { - estimateState = { - status: 'unavailable', - scope: 'request', - label: 'Complete the required configuration to request an estimate.', - note: estimateResult.error, - } - } else if ( - estimateRequestState?.requestKey === estimateRequestKey - && estimateRequestState.status === 'resolved' - ) { - estimateState = estimateRequestState.result - } else if ( - estimateRequestState?.requestKey === estimateRequestKey - && estimateRequestState.status === 'error' - ) { - estimateState = lastGoodEstimate - ? { - status: 'stale', - estimate: lastGoodEstimate.estimate, - label: 'Showing the last successful estimate.', - error: estimateRequestState.error, - } - : { - status: 'unavailable', - scope: 'request', - label: 'The backend estimate could not be refreshed.', - note: estimateRequestState.error, - } - } else if (lastGoodEstimate) { - estimateState = { - status: 'refreshing', - estimate: lastGoodEstimate.estimate, - label: 'Updating for the current configuration…', - } - } else { - estimateState = { status: 'loading', scope: 'request' } - } const handleTechniqueChange = (technique: SelectableTechnique, checked: boolean): void => { if (technique.isBaseline) { - setBaselineChecked(checked) + dispatch({ type: 'setBaseline', checked }) } else { if (singleTechnique && checked && technique.name !== selectedTechniques[0] && techniqueEditor.hasChanges) { setPendingTechnique(technique.name) return } if (singleTechnique && technique.name !== selectedTechniques[0]) techniqueEditor.discard() - setSelectedTechniques((current) => { - if (checked) { - if (singleTechnique) return [technique.name] - if (scenario.max_concrete_techniques != null && current.length >= scenario.max_concrete_techniques) { - return current - } - return current.includes(technique.name) - ? current - : [...current, technique.name] - } - return current.filter((name) => name !== technique.name) - }) + dispatch({ type: 'setTechnique', name: technique.name, checked }) } - setValidationError(null) + clearValidationError() } const isTechniqueSelected = (technique: SelectableTechnique): boolean => ( @@ -1004,76 +541,18 @@ function ScenarioLaunchForm({ const members = selectableTechniques.filter( (technique) => !technique.disabled && technique.tags.includes(tag), ) - const shouldSelect = members.some((technique) => !isTechniqueSelected(technique)) - const memberNames = new Set( - members.filter((technique) => !technique.isBaseline).map((technique) => technique.name), - ) - if (shouldSelect && scenario.max_concrete_techniques != null - && new Set([...selectedTechniques, ...memberNames]).size > scenario.max_concrete_techniques) { + const select = members.some((technique) => !isTechniqueSelected(technique)) + const memberNames = members.filter((technique) => !technique.isBaseline).map((technique) => technique.name) + if (select && exceedsTechniqueLimit(selectedTechniques, memberNames, scenario.max_concrete_techniques ?? null)) { return } - setSelectedTechniques((current) => { - const selected = new Set(current) - for (const name of memberNames) { - if (shouldSelect) selected.add(name) - else selected.delete(name) - } - return techniqueOptions.map((technique) => technique.name).filter((name) => selected.has(name)) + dispatch({ + type: 'toggleTag', + memberNames, + includesBaseline: members.some((technique) => technique.isBaseline), + select, }) - if (members.some((technique) => technique.isBaseline)) { - setBaselineChecked(shouldSelect) - } - setValidationError(null) - } - - const updateScenarioParam = (name: string, value: ParameterFormValue): void => { - setScenarioParamValues((current) => ({ ...current, [name]: value })) - } - - const handleLaunchConfirmed = async (): Promise => { - if (isSubmittingRef.current || conversationBusy || techniqueEditError || !runtime.ready || !defaultsReady || staleSelection || unavailableSelection - || checkingRefreshPolicies) { - return - } - - setApiError(null) - if (!requestResult.ok) { - setValidationError(requestResult.error) - setPreviewOpen(false) - return - } - - isSubmittingRef.current = true - setSubmitting(true) - setValidationError(null) - - try { - const summary = await scenariosApi.startRun(requestResult.request) - datasetMetadata.reload() - setPreviewOpen(false) - navigate(scenarioRunRoutePath(summary.scenario_result_id), { - state: { scenarioName: scenario.scenario_name }, - }) - } catch (err) { - setApiError(toApiError(err).detail) - setPreviewOpen(false) - restoreLaunchFocus() - } finally { - isSubmittingRef.current = false - setSubmitting(false) - } - } - - const handleFormSubmit = (event: FormEvent): void => { - event.preventDefault() - if (conversationBusy || techniqueEditError || isSubmittingRef.current) return - setApiError(null) - if (!requestResult.ok) { - setValidationError(requestResult.error) - return - } - setValidationError(null) - setPreviewOpen(true) + clearValidationError() } const techniqueSelectionOverLimit = supportsTechniques && scenario.max_concrete_techniques != null @@ -1083,30 +562,10 @@ function ScenarioLaunchForm({ : techniqueSelectionOverLimit ? `Select at most ${scenario.max_concrete_techniques} concrete technique(s).` : null - const displayedEstimateNotes = estimateNotes(estimateState) - const displayedEstimate = estimateFromState(estimateState) const selectedTechniqueCount = (supportsTechniques ? selectedTechniques.length : 0) + (supportsBaseline && baselineChecked && !isBaselineForbidden ? 1 : 0) const previewDatasets = (datasetEdits.sources ?? datasetDefaults.sources).map((source) => source.name) - const previewDatasetNames = previewDatasets.length > 0 ? previewDatasets : scenario.default_datasets const previewFilters = { ...datasetDefaults.filters, ...datasetEdits.filters } - const previewHarmCategories = previewFilters.harm_categories ?? [] - const previewDataTypes = previewFilters.data_types ?? [] - const currentDatasetEstimate = estimateRequestState?.requestKey === estimateRequestKey ? estimateRequestState : null - const confirmedDatasetSettings = currentDatasetEstimate?.status === 'resolved' - ? currentDatasetEstimate.datasetSettings : undefined - const checkingRefreshPolicies = supportsDatasets && selectedDatasetNames.length > 0 && currentDatasetEstimate === null - const unconfirmedRefreshPolicies = supportsDatasets && selectedDatasetNames.length > 0 - && !checkingRefreshPolicies && confirmedDatasetSettings === undefined - const refreshDatasets = [...new Set((confirmedDatasetSettings?.sources ?? []) - .filter((source) => source.fetch === 'always').map((source) => source.name))] - const editedRefreshDatasets = refreshDatasets.filter((name) => { - const metadata = datasetMetadata.items.find((item) => item.name === name) - return metadata?.loaded && (!metadata.origins?.length - || metadata.origins.some((origin) => origin === 'user' || origin === 'unknown')) - }) - const uncheckedRefreshDatasets = refreshDatasets.filter((name) => datasetMetadata.loading || datasetMetadata.error - || !datasetMetadata.items.some((item) => item.name === name)) const renderScenarioParameter = (parameter: Parameter): React.ReactElement => (
@@ -1169,7 +628,7 @@ function ScenarioLaunchForm({ id={formId} className={styles.formColumn} aria-label="Scenario run configuration" - onSubmit={handleFormSubmit} + onSubmit={submitForm} noValidate > {validationError && !requestResult.ok && ( @@ -1183,8 +642,7 @@ function ScenarioLaunchForm({ PyRIT was reinitialized. Review the refreshed target and technique selections before starting. Your parameter drafts have been preserved. + onClick={() => dispatch({ type: 'clearPrependedConversation' })}>Clear saved conversation } } @@ -1400,7 +783,7 @@ function ScenarioLaunchForm({ targets={adversarialTargets} value={adversarialTargetName} onChange={(target: TargetInstance | null) => { - setAdversarialTargetName(target?.target_registry_name ?? '') + dispatch({ type: 'setAdversarialTarget', name: target?.target_registry_name ?? '' }) }} label="Adversarial Target" hint="The registered target this scenario uses to generate attacks." @@ -1415,7 +798,7 @@ function ScenarioLaunchForm({ min={MIN_MAX_CONCURRENCY} max={MAX_MAX_CONCURRENCY} disabled={submitting} - onChange={(_, data) => setMaxConcurrency(resolveSpinButtonValue(data, maxConcurrency))} + onChange={(_, data) => dispatch({ type: 'setMaxConcurrency', value: resolveSpinButtonValue(data, maxConcurrency) })} data-testid="max-concurrency-input" /> } @@ -1426,7 +809,7 @@ function ScenarioLaunchForm({ min={MIN_MAX_RETRIES} max={MAX_MAX_RETRIES} disabled={submitting} - onChange={(_, data) => setMaxRetries(resolveSpinButtonValue(data, maxRetries))} + onChange={(_, data) => dispatch({ type: 'setMaxRetries', value: resolveSpinButtonValue(data, maxRetries) })} data-testid="max-retries-input" /> } @@ -1434,70 +817,15 @@ function ScenarioLaunchForm({ )} -
-
- - Run estimate - - {displayedEstimateNotes && ( - -
-
-
-
Total atomic attacks
-
{formatAtomicAttackCount(estimateState)}
-
- {supportsDatasets &&
-
Dataset size
-
- {datasetSizeLabel} -
-
} - {(supportsTechniques || supportsBaseline) &&
-
Number techniques
-
{selectedTechniqueCount}
-
} - {dynamicParameters.map((parameter) => ( -
-
{parameter.name}
-
- {formatEffectiveParameterPreview( - parameter.name, - scenarioParamValues[parameter.name], - displayedEstimate, - )} -
-
- ))} -
- {estimateState.status === 'refreshing' && ( -
- - Updating estimate... -
- )} - {estimateState.status === 'stale' && ( - - {estimateState.error} - - )} -
+
@@ -1551,160 +879,39 @@ function ScenarioLaunchForm({ } - { - if (!submitting) { - setPreviewOpen(data.open) - if (!data.open) { - restoreLaunchFocus() - } - } + submitting={submitting} + confirmDisabled={submitting || conversationBusy || !runtime.ready || !defaultsReady || !requestResult.ok + || checkingRefreshPolicies} + onOpenChange={changePreviewOpen} + onCancel={dismissPreview} + onConfirm={() => void confirmLaunch()} + refresh={{ + checking: checkingRefreshPolicies, + unconfirmed: unconfirmedRefreshPolicies, + refreshed: refreshDatasets, + edited: editedRefreshDatasets, + unchecked: uncheckedRefreshDatasets, }} - > - - - Run preview - - {checkingRefreshPolicies && Checking dataset refresh policies...} - {unconfirmedRefreshPolicies && - - Cannot confirm which datasets will refresh. The backend settings could not be checked for this - request. A refresh replaces the full stored dataset, can remove manual edits, and can prevent - older runs from resuming. Filters and objective limits do not limit replacement. - - } - {refreshDatasets.length > 0 && - - Refresh will replace the full stored datasets: {refreshDatasets.join(', ')}. - Filters and objective limits do not limit replacement. This can remove manual edits - and prevent older runs from resuming. Use a new dataset name to keep both batches. - {editedRefreshDatasets.length > 0 && - User or unknown-origin data will be replaced: {editedRefreshDatasets.join(', ')}. - } - {uncheckedRefreshDatasets.length > 0 && - Existing data could not be checked for: {uncheckedRefreshDatasets.join(', ')}. - } - - } -
-
- {supportsObjectiveTarget && <> -
Objective Target
-
{targetName}
- } - {scenario.uses_default_adversarial_target && ( - <> -
Adversarial Target
-
{adversarialTargetName || 'Use server default'}
- - )} -
- {(supportsTechniques || supportsBaseline) &&
-
Techniques
-
-
- {supportsBaseline && baselineChecked && !isBaselineForbidden - && baseline} - {supportsTechniques && selectedTechniques.map((name) => ( - {name} - ))} -
-
-
} - {techniqueEditor.override &&
-
Run-local technique edits
-
The registered {techniqueEditor.override.source_name} stays unchanged. - 
{JSON.stringify(techniqueEditor.override.changes, null, 2)}
-
-
} - {supportsDatasets && <> -
-
Datasets
-
-
- - {previewDatasetNames.length > 0 - ? previewDatasetNames.join(', ') - : 'No datasets declared'} - - - {previewDatasets.length > 0 ? 'Custom override' : 'Scenario defaults'} - {` - dataset size: ${datasetSizeLabel}`} - -
-
-
-
-
Dataset filters
-
- {previewHarmCategories.length > 0 || previewDataTypes.length > 0 ? ( -
- {previewHarmCategories.length > 0 && ( -
-
Harm categories
-
{previewHarmCategories.join(', ')}
-
- )} - {previewDataTypes.length > 0 && ( -
-
Data types
-
{previewDataTypes.join(', ')}
-
- )} -
- ) : ( - 'None' - )} -
-
- } -
-
Parameters
-
- {dynamicParameters.length > 0 ? ( -
- {dynamicParameters.map((parameter) => ( -
-
{parameter.name}
-
- {formatEffectiveParameterPreview( - parameter.name, - scenarioParamValues[parameter.name], - displayedEstimate, - )} -
-
- ))} -
- ) : ( - 'No scenario-specific parameters' - )} -
-
-
- - - - - - - - - + objectiveTarget={supportsObjectiveTarget ? targetName : null} + adversarialTarget={scenario.uses_default_adversarial_target ? adversarialTargetName || 'Use server default' : null} + techniques={supportsTechniques || supportsBaseline ? { + baseline: supportsBaseline && baselineChecked && !isBaselineForbidden, + names: supportsTechniques ? selectedTechniques : [], + } : null} + techniqueOverride={techniqueEditor.override} + datasets={supportsDatasets ? { + names: previewDatasets.length > 0 ? previewDatasets : scenario.default_datasets, + custom: previewDatasets.length > 0, + sizeLabel: datasetSizeLabel, + harmCategories: previewFilters.harm_categories ?? [], + dataTypes: previewFilters.data_types ?? [], + } : null} + dynamicParameters={dynamicParameters} + scenarioParamValues={scenarioParamValues} + estimateState={estimateState} + />
diff --git a/frontend/src/components/Scenarios/ScenarioEstimateSection.tsx b/frontend/src/components/Scenarios/ScenarioEstimateSection.tsx new file mode 100644 index 0000000000..ddc6ecb460 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioEstimateSection.tsx @@ -0,0 +1,103 @@ +import { Button, Spinner, Text, Tooltip, mergeClasses } from '@fluentui/react-components' +import { InfoRegular } from '@fluentui/react-icons' + +import type { ParameterFormValue } from '@/components/Parameters/parameterForm' +import type { Parameter, ScenarioRunEstimateState } from '@/types' + +import { useScenarioDetailStyles } from './ScenarioDetail.styles' +import { + estimateFromState, + estimateNotes, + formatAtomicAttackCount, + formatEffectiveParameterPreview, +} from './scenarioLaunchPresentation' + +interface ScenarioEstimateSectionProps { + state: ScenarioRunEstimateState + showDatasetSize: boolean + datasetSizeLabel: string + showTechniqueCount: boolean + techniqueCount: number + dynamicParameters: Parameter[] + scenarioParamValues: Record +} + +export default function ScenarioEstimateSection({ + state, + showDatasetSize, + datasetSizeLabel, + showTechniqueCount, + techniqueCount, + dynamicParameters, + scenarioParamValues, +}: ScenarioEstimateSectionProps) { + const styles = useScenarioDetailStyles() + const notes = estimateNotes(state) + const estimate = estimateFromState(state) + return ( +
+
+ + Run estimate + + {notes && ( + +
+
+
+
Total atomic attacks
+
{formatAtomicAttackCount(state)}
+
+ {showDatasetSize &&
+
Dataset size
+
+ {datasetSizeLabel} +
+
} + {showTechniqueCount &&
+
Number techniques
+
{techniqueCount}
+
} + {dynamicParameters.map((parameter) => ( +
+
{parameter.name}
+
+ {formatEffectiveParameterPreview( + parameter.name, + scenarioParamValues[parameter.name], + estimate, + )} +
+
+ ))} +
+ {state.status === 'refreshing' && ( +
+ + Updating estimate... +
+ )} + {state.status === 'stale' && ( + + {state.error} + + )} +
+ ) +} diff --git a/frontend/src/components/Scenarios/ScenarioRunPreviewDialog.tsx b/frontend/src/components/Scenarios/ScenarioRunPreviewDialog.tsx new file mode 100644 index 0000000000..02209f6b84 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioRunPreviewDialog.tsx @@ -0,0 +1,225 @@ +import { + Badge, + Button, + Dialog, + DialogActions, + DialogBody, + DialogContent, + DialogSurface, + DialogTitle, + MessageBar, + MessageBarBody, + Text, +} from '@fluentui/react-components' + +import type { ParameterFormValue } from '@/components/Parameters/parameterForm' +import type { Parameter, ScenarioRunEstimateState, TechniqueOverride } from '@/types' + +import { useScenarioDetailStyles } from './ScenarioDetail.styles' +import { ScenarioRunEstimateDetails } from './ScenarioRunEstimate' +import { estimateFromState, formatEffectiveParameterPreview } from './scenarioLaunchPresentation' + +/** Which datasets a confirmed run would replace, and how sure the backend answer is. */ +export interface DatasetRefreshPreview { + checking: boolean + unconfirmed: boolean + refreshed: string[] + edited: string[] + unchecked: string[] +} + +/** The datasets section of the run preview, or null when the scenario takes no dataset input. */ +export interface DatasetSelectionPreview { + names: string[] + custom: boolean + sizeLabel: string + harmCategories: string[] + dataTypes: string[] +} + +interface ScenarioRunPreviewDialogProps { + open: boolean + submitting: boolean + confirmDisabled: boolean + onOpenChange: (open: boolean) => void + onCancel: () => void + onConfirm: () => void + refresh: DatasetRefreshPreview + /** The objective target name, or null when the scenario takes no target. */ + objectiveTarget: string | null + /** The adversarial target shown, or null when the scenario has no adversarial target. */ + adversarialTarget: string | null + /** The selected techniques, or null when the scenario takes neither techniques nor a baseline. */ + techniques: { baseline: boolean, names: string[] } | null + techniqueOverride?: TechniqueOverride + datasets: DatasetSelectionPreview | null + dynamicParameters: Parameter[] + scenarioParamValues: Record + estimateState: ScenarioRunEstimateState +} + +export default function ScenarioRunPreviewDialog({ + open, + submitting, + confirmDisabled, + onOpenChange, + onCancel, + onConfirm, + refresh, + objectiveTarget, + adversarialTarget, + techniques, + techniqueOverride, + datasets, + dynamicParameters, + scenarioParamValues, + estimateState, +}: ScenarioRunPreviewDialogProps) { + const styles = useScenarioDetailStyles() + const estimate = estimateFromState(estimateState) + return ( + onOpenChange(data.open)}> + + + Run preview + + {refresh.checking && Checking dataset refresh policies...} + {refresh.unconfirmed && + + Cannot confirm which datasets will refresh. The backend settings could not be checked for this + request. A refresh replaces the full stored dataset, can remove manual edits, and can prevent + older runs from resuming. Filters and objective limits do not limit replacement. + + } + {refresh.refreshed.length > 0 && + + Refresh will replace the full stored datasets: {refresh.refreshed.join(', ')}. + Filters and objective limits do not limit replacement. This can remove manual edits + and prevent older runs from resuming. Use a new dataset name to keep both batches. + {refresh.edited.length > 0 && + User or unknown-origin data will be replaced: {refresh.edited.join(', ')}. + } + {refresh.unchecked.length > 0 && + Existing data could not be checked for: {refresh.unchecked.join(', ')}. + } + + } +
+
+ {objectiveTarget !== null && <> +
Objective Target
+
{objectiveTarget}
+ } + {adversarialTarget !== null && ( + <> +
Adversarial Target
+
{adversarialTarget}
+ + )} +
+ {techniques &&
+
Techniques
+
+
+ {techniques.baseline && baseline} + {techniques.names.map((name) => ( + {name} + ))} +
+
+
} + {techniqueOverride &&
+
Run-local technique edits
+
The registered {techniqueOverride.source_name} stays unchanged. + 
{JSON.stringify(techniqueOverride.changes, null, 2)}
+
+
} + {datasets && <> +
+
Datasets
+
+
+ + {datasets.names.length > 0 + ? datasets.names.join(', ') + : 'No datasets declared'} + + + {datasets.custom ? 'Custom override' : 'Scenario defaults'} + {` - dataset size: ${datasets.sizeLabel}`} + +
+
+
+
+
Dataset filters
+
+ {datasets.harmCategories.length > 0 || datasets.dataTypes.length > 0 ? ( +
+ {datasets.harmCategories.length > 0 && ( +
+
Harm categories
+
{datasets.harmCategories.join(', ')}
+
+ )} + {datasets.dataTypes.length > 0 && ( +
+
Data types
+
{datasets.dataTypes.join(', ')}
+
+ )} +
+ ) : ( + 'None' + )} +
+
+ } +
+
Parameters
+
+ {dynamicParameters.length > 0 ? ( +
+ {dynamicParameters.map((parameter) => ( +
+
{parameter.name}
+
+ {formatEffectiveParameterPreview( + parameter.name, + scenarioParamValues[parameter.name], + estimate, + )} +
+
+ ))} +
+ ) : ( + 'No scenario-specific parameters' + )} +
+
+
+ + + + + + + + + + ) +} diff --git a/frontend/src/components/Scenarios/ScenarioTechniqueSection.tsx b/frontend/src/components/Scenarios/ScenarioTechniqueSection.tsx new file mode 100644 index 0000000000..99cef4e072 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioTechniqueSection.tsx @@ -0,0 +1,126 @@ +import { Badge, Checkbox, Radio, Text, ToggleButton } from '@fluentui/react-components' + +import { useScenarioDetailStyles } from './ScenarioDetail.styles' +import { exceedsTechniqueLimit, type SelectableTechnique } from './scenarioLaunchForm' +import { techniqueSetName } from './scenarioTechniqueSets' + +interface ScenarioTechniqueSectionProps { + formId: string + techniques: SelectableTechnique[] + selectedTechniques: string[] + supportsTechniques: boolean + singleTechnique: boolean + maxConcreteTechniques: number | null + selectionOverLimit: boolean + selectionError: string | null + disabled: boolean + isSelected: (technique: SelectableTechnique) => boolean + onTechniqueChange: (technique: SelectableTechnique, checked: boolean) => void + onTagChange: (tag: string) => void +} + +export default function ScenarioTechniqueSection({ + formId, + techniques, + selectedTechniques, + supportsTechniques, + singleTechnique, + maxConcreteTechniques, + selectionOverLimit, + selectionError, + disabled, + isSelected, + onTechniqueChange, + onTagChange, +}: ScenarioTechniqueSectionProps) { + const styles = useScenarioDetailStyles() + return ( +
+ + Techniques + + {supportsTechniques && + {singleTechnique + ? 'Select exactly one technique. Baseline comparison is optional and separate.' + : 'Select individual techniques, or use a tag to select or clear all techniques with that tag.'} + } + {selectionError && ( + + {selectionError} + + )} +
+ {techniques.map((technique) => { + const selected = isSelected(technique) + return ( +
+ {singleTechnique && !technique.isBaseline ? ( + onTechniqueChange(technique, true)} + data-testid={`technique-${technique.name}`} + /> + ) : = maxConcreteTechniques)} + onChange={(_, data) => onTechniqueChange(technique, data.checked === true)} + data-testid={technique.isBaseline ? 'baseline-checkbox' : `technique-${technique.name}`} + />} +
+ {technique.description && ( + {technique.description} + )} + {technique.tags.length > 0 && ( +
+ {technique.tags.map((tag) => { + const tagMembers = techniques.filter( + (candidate) => !candidate.disabled && candidate.tags.includes(tag), + ) + const tagSelected = tagMembers.length > 0 && tagMembers.every(isSelected) + const exceedsLimit = exceedsTechniqueLimit( + selectedTechniques, + tagMembers.filter((candidate) => !candidate.isBaseline).map((candidate) => candidate.name), + maxConcreteTechniques, + ) + if (singleTechnique) { + return {techniqueSetName(tag)} + } + return ( + onTagChange(tag)} + aria-label={`${tagSelected ? 'Clear' : 'Select'} ${techniqueSetName(tag)} techniques`} + > + {techniqueSetName(tag)} + + ) + })} +
+ )} + {technique.disabled && ( + + This scenario does not support a baseline comparison. + + )} +
+
+ ) + })} +
+
+ ) +} diff --git a/frontend/src/components/Scenarios/scenarioLaunchForm.test.ts b/frontend/src/components/Scenarios/scenarioLaunchForm.test.ts new file mode 100644 index 0000000000..a0f2bbac1e --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioLaunchForm.test.ts @@ -0,0 +1,155 @@ +import type { RegisteredScenario, TargetInstance } from '@/types' + +import { + initialScenarioLaunchForm, + reduceScenarioLaunchForm, + type ScenarioLaunchFormAction, + type ScenarioLaunchFormRules, + type ScenarioLaunchFormState, +} from './scenarioLaunchForm' + +const RULES: ScenarioLaunchFormRules = { + techniqueNames: ['a', 'b', 'c'], + maxConcreteTechniques: null, + singleTechnique: false, +} + +function state(overrides: Partial = {}): ScenarioLaunchFormState { + return { + targetName: '', + adversarialTargetName: '', + selectedTechniques: ['a'], + baselineChecked: false, + datasetEdits: {}, + datasetError: null, + maxConcurrency: 10, + maxRetries: 0, + scenarioParamValues: {}, + ...overrides, + } +} + +function reduce( + current: ScenarioLaunchFormState, + actions: ScenarioLaunchFormAction[], + rules: ScenarioLaunchFormRules = RULES, +): ScenarioLaunchFormState { + return actions.reduce((next, action) => reduceScenarioLaunchForm(next, action, rules), current) +} + +function target(name: string): TargetInstance { + return { target_registry_name: name, target_type: 'OpenAIChatTarget', identifier: { hash: `${name}-hash` } } as TargetInstance +} + +describe('scenario launch form', () => { + it('starts from defaults that are still available', () => { + const scenario = { + baseline_policy: 'enabled', + include_baseline_by_default: true, + } as RegisteredScenario + const initial = initialScenarioLaunchForm({ + scenario, + targets: [target('chat')], + defaultObjectiveTarget: target('gone'), + adversarialTargets: [target('chat')], + defaultAdversarialTarget: target('chat'), + defaultTechniques: ['a'], + dynamicParameters: [], + initialConversationId: '', + }) + + expect(initial).toMatchObject({ + targetName: '', + adversarialTargetName: 'chat', + selectedTechniques: ['a'], + baselineChecked: true, + maxConcurrency: 10, + maxRetries: 0, + }) + }) + + it('never starts with a forbidden baseline', () => { + const initial = initialScenarioLaunchForm({ + scenario: { baseline_policy: 'forbidden', include_baseline_by_default: true } as RegisteredScenario, + targets: [], + defaultObjectiveTarget: null, + adversarialTargets: [], + defaultAdversarialTarget: null, + defaultTechniques: [], + dynamicParameters: [], + initialConversationId: '', + }) + + expect(initial.baselineChecked).toBe(false) + }) + + it('adds and removes techniques without duplicates', () => { + const next = reduce(state(), [ + { type: 'setTechnique', name: 'b', checked: true }, + { type: 'setTechnique', name: 'b', checked: true }, + { type: 'setTechnique', name: 'a', checked: false }, + ]) + + expect(next.selectedTechniques).toEqual(['b']) + }) + + it('replaces the selection when only one technique is allowed', () => { + const next = reduce(state(), [{ type: 'setTechnique', name: 'c', checked: true }], { + ...RULES, singleTechnique: true, maxConcreteTechniques: 1, + }) + + expect(next.selectedTechniques).toEqual(['c']) + }) + + it('stops adding techniques at the limit', () => { + const next = reduce(state(), [{ type: 'setTechnique', name: 'b', checked: true }], { + ...RULES, maxConcreteTechniques: 1, + }) + + expect(next.selectedTechniques).toEqual(['a']) + }) + + it('selects a tag in display order and includes the baseline', () => { + const next = reduce(state({ selectedTechniques: ['c'] }), [ + { type: 'toggleTag', memberNames: ['b', 'a'], includesBaseline: true, select: true }, + ]) + + expect(next.selectedTechniques).toEqual(['a', 'b', 'c']) + expect(next.baselineChecked).toBe(true) + }) + + it('clears a tag and leaves the baseline alone when it is not a member', () => { + const next = reduce(state({ selectedTechniques: ['a', 'b', 'c'], baselineChecked: true }), [ + { type: 'toggleTag', memberNames: ['a', 'b'], includesBaseline: false, select: false }, + ]) + + expect(next.selectedTechniques).toEqual(['c']) + expect(next.baselineChecked).toBe(true) + }) + + it('ignores a tag that would go over the limit', () => { + const current = state() + + const next = reduce(current, [ + { type: 'toggleTag', memberNames: ['b', 'c'], includesBaseline: true, select: true }, + ], { ...RULES, maxConcreteTechniques: 2 }) + + expect(next).toBe(current) + }) + + it('drops techniques that are no longer offered', () => { + const next = reduce(state({ selectedTechniques: ['a', 'removed'] }), [{ type: 'keepAvailableTechniques' }]) + + expect(next.selectedTechniques).toEqual(['a']) + }) + + it('clears a saved conversation together with its converter switch', () => { + const next = reduce(state({ + scenarioParamValues: { prepended_conversation_id: 'saved', apply_converters_to_prepended_history: 'true', objective: 'x' }, + }), [{ type: 'clearPrependedConversation' }]) + + expect(next.scenarioParamValues).toEqual({ + prepended_conversation_id: '', apply_converters_to_prepended_history: 'false', objective: 'x', + }) + }) +}) diff --git a/frontend/src/components/Scenarios/scenarioLaunchForm.ts b/frontend/src/components/Scenarios/scenarioLaunchForm.ts new file mode 100644 index 0000000000..77dc518de1 --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioLaunchForm.ts @@ -0,0 +1,204 @@ +import { getInitialFormValues, type ParameterFormValue } from '@/components/Parameters/parameterForm' +import type { + Parameter, + RegisteredScenario, + ScenarioDatasetRequest, + ScenarioTechniqueSummary, + TargetInstance, +} from '@/types' +import { sameTarget } from '@/utils/targetIdentity' + +import { DEFAULT_MAX_CONCURRENCY, DEFAULT_MAX_RETRIES } from './scenarioLaunchRequest' + +export interface TechniqueOptions { + techniques: ScenarioTechniqueSummary[] + defaultTechniques: string[] +} + +export function uniqueTechniqueOptions(scenario: RegisteredScenario): TechniqueOptions { + const aggregateNames = new Set(scenario.aggregate_techniques) + const summariesByName = new Map( + scenario.technique_summaries.map((summary) => [summary.name, summary]), + ) + const techniques: ScenarioTechniqueSummary[] = [] + const seen = new Set() + for (const name of scenario.all_techniques) { + if (!aggregateNames.has(name) && !seen.has(name)) { + techniques.push(summariesByName.get(name) ?? { name, description: null, tags: [] }) + seen.add(name) + } + } + const concreteNames = new Set(techniques.map((technique) => technique.name)) + const defaultTechniques = scenario.default_techniques.filter((name) => concreteNames.has(name)) + if (defaultTechniques.length === 0 && concreteNames.has(scenario.default_technique)) { + defaultTechniques.push(scenario.default_technique) + } + return { techniques, defaultTechniques } +} + +export interface SelectableTechnique extends ScenarioTechniqueSummary { + isBaseline: boolean + disabled: boolean +} + +export const BASELINE_TECHNIQUE: ScenarioTechniqueSummary = { + name: 'baseline', + description: 'Sends each objective directly to the target for comparison.', + tags: ['baseline', 'single_turn'], +} + +/** The related launch-form fields that change together through explicit actions. */ +export interface ScenarioLaunchFormState { + targetName: string + adversarialTargetName: string + selectedTechniques: string[] + baselineChecked: boolean + datasetEdits: ScenarioDatasetRequest + datasetError: string | null + maxConcurrency: number + maxRetries: number + scenarioParamValues: Record +} + +/** Technique selection rules fixed for one scenario. */ +export interface ScenarioLaunchFormRules { + /** Concrete technique names in display order. */ + techniqueNames: string[] + maxConcreteTechniques: number | null + singleTechnique: boolean +} + +export type ScenarioLaunchFormAction = + | { type: 'setTarget', name: string } + | { type: 'setAdversarialTarget', name: string } + | { type: 'setTechnique', name: string, checked: boolean } + | { type: 'replaceTechniques', names: string[] } + | { type: 'keepAvailableTechniques' } + | { type: 'setBaseline', checked: boolean } + | { type: 'toggleTag', memberNames: string[], includesBaseline: boolean, select: boolean } + | { type: 'setDatasetEdits', edits: ScenarioDatasetRequest } + | { type: 'setDatasetError', error: string | null } + | { type: 'setMaxConcurrency', value: number } + | { type: 'setMaxRetries', value: number } + | { type: 'setParam', name: string, value: ParameterFormValue } + | { type: 'clearPrependedConversation' } + +interface InitialFormInput { + scenario: RegisteredScenario + targets: TargetInstance[] + defaultObjectiveTarget: TargetInstance | null + adversarialTargets: TargetInstance[] + defaultAdversarialTarget: TargetInstance | null + defaultTechniques: string[] + dynamicParameters: Parameter[] + initialConversationId: string +} + +function defaultTargetName(target: TargetInstance | null, available: TargetInstance[]): string { + return target && available.some((candidate) => sameTarget(candidate, target)) ? target.target_registry_name : '' +} + +export function initialScenarioLaunchForm({ + scenario, + targets, + defaultObjectiveTarget, + adversarialTargets, + defaultAdversarialTarget, + defaultTechniques, + dynamicParameters, + initialConversationId, +}: InitialFormInput): ScenarioLaunchFormState { + return { + targetName: defaultTargetName(defaultObjectiveTarget, targets), + adversarialTargetName: defaultTargetName(defaultAdversarialTarget, adversarialTargets), + selectedTechniques: defaultTechniques, + baselineChecked: scenario.baseline_policy !== 'forbidden' && scenario.include_baseline_by_default, + datasetEdits: {}, + datasetError: null, + maxConcurrency: DEFAULT_MAX_CONCURRENCY, + maxRetries: DEFAULT_MAX_RETRIES, + scenarioParamValues: getInitialFormValues( + dynamicParameters, + initialConversationId ? { prepended_conversation_id: initialConversationId } : undefined, + ), + } +} + +/** True when selecting every named technique would go over the scenario's technique limit. */ +export function exceedsTechniqueLimit( + selected: string[], + added: Iterable, + maxConcreteTechniques: number | null, +): boolean { + return maxConcreteTechniques != null && new Set([...selected, ...added]).size > maxConcreteTechniques +} + +function selectTechnique( + current: string[], + name: string, + checked: boolean, + rules: ScenarioLaunchFormRules, +): string[] { + if (!checked) return current.filter((selected) => selected !== name) + if (rules.singleTechnique) return [name] + if (rules.maxConcreteTechniques != null && current.length >= rules.maxConcreteTechniques) return current + return current.includes(name) ? current : [...current, name] +} + +export function reduceScenarioLaunchForm( + state: ScenarioLaunchFormState, + action: ScenarioLaunchFormAction, + rules: ScenarioLaunchFormRules, +): ScenarioLaunchFormState { + switch (action.type) { + case 'setTarget': + return { ...state, targetName: action.name } + case 'setAdversarialTarget': + return { ...state, adversarialTargetName: action.name } + case 'setTechnique': + return { ...state, selectedTechniques: selectTechnique(state.selectedTechniques, action.name, action.checked, rules) } + case 'replaceTechniques': + return { ...state, selectedTechniques: action.names } + case 'keepAvailableTechniques': + return { + ...state, + selectedTechniques: state.selectedTechniques.filter((name) => rules.techniqueNames.includes(name)), + } + case 'setBaseline': + return { ...state, baselineChecked: action.checked } + case 'toggleTag': { + if (action.select && exceedsTechniqueLimit(state.selectedTechniques, action.memberNames, rules.maxConcreteTechniques)) { + return state + } + const selected = new Set(state.selectedTechniques) + for (const name of action.memberNames) { + if (action.select) selected.add(name) + else selected.delete(name) + } + return { + ...state, + selectedTechniques: rules.techniqueNames.filter((name) => selected.has(name)), + baselineChecked: action.includesBaseline ? action.select : state.baselineChecked, + } + } + case 'setDatasetEdits': + return { ...state, datasetEdits: action.edits } + case 'setDatasetError': + return { ...state, datasetError: action.error } + case 'setMaxConcurrency': + return { ...state, maxConcurrency: action.value } + case 'setMaxRetries': + return { ...state, maxRetries: action.value } + case 'setParam': + return { ...state, scenarioParamValues: { ...state.scenarioParamValues, [action.name]: action.value } } + case 'clearPrependedConversation': + return { + ...state, + scenarioParamValues: { + ...state.scenarioParamValues, + prepended_conversation_id: '', + apply_converters_to_prepended_history: 'false', + }, + } + } +} diff --git a/frontend/src/components/Scenarios/scenarioLaunchPresentation.ts b/frontend/src/components/Scenarios/scenarioLaunchPresentation.ts new file mode 100644 index 0000000000..ca0c7ef50f --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioLaunchPresentation.ts @@ -0,0 +1,75 @@ +import type { ParameterFormValue } from '@/components/Parameters/parameterForm' +import type { ScenarioRunEstimate, ScenarioRunEstimateState } from '@/types' + +function formatParameterPreview(value: ParameterFormValue | undefined): string { + if (Array.isArray(value)) { + return value.length > 0 ? value.join(', ') : 'Not set' + } + if (typeof value === 'object') { + return value.type || 'Not set' + } + return value?.trim() || 'Not set' +} + +export function formatEffectiveParameterPreview( + parameterName: string, + value: ParameterFormValue | undefined, + estimate: ScenarioRunEstimate | undefined, +): string { + const configuredValue = formatParameterPreview(value) + if (configuredValue !== 'Not set') { + return configuredValue + } + const effectiveValue = estimate?.effectiveParameters[parameterName] + if (Array.isArray(effectiveValue)) { + return effectiveValue.length > 0 ? effectiveValue.join(', ') : 'Not set' + } + return effectiveValue?.toString() ?? 'Not set' +} + +export function estimateFromState(state: ScenarioRunEstimateState): ScenarioRunEstimate | undefined { + switch (state.status) { + case 'available': + case 'conditional': + case 'refreshing': + case 'stale': + return state.estimate + default: + return undefined + } +} + +export function formatAtomicAttackCount(state: ScenarioRunEstimateState): string { + const estimate = estimateFromState(state) + if (!estimate) { + return state.status === 'loading' ? 'Calculating...' : 'Unknown' + } + const prefix = estimate.approximate ? 'About ' : '' + if (estimate.total !== null) { + return `${estimate.approximate ? 'Up to ' : ''}${estimate.total.toLocaleString()}` + } + if (estimate.minimum != null && estimate.maximum != null) { + return estimate.minimum === estimate.maximum + ? `${prefix}${estimate.minimum.toLocaleString()}` + : `${prefix}${estimate.minimum.toLocaleString()}-${estimate.maximum.toLocaleString()}` + } + if (estimate.minimum != null) { + return `At least ${prefix.toLowerCase()}${estimate.minimum.toLocaleString()}` + } + if (estimate.maximum != null) { + return `Up to ${estimate.maximum.toLocaleString()}` + } + return 'Varies' +} + +export function estimateNotes(state: ScenarioRunEstimateState): string | null { + const estimate = estimateFromState(state) + if (!estimate) { + return state.status === 'unavailable' ? state.note ?? state.label : null + } + const notes = [ + estimate.note, + ...estimate.components.map((component) => component.note), + ].filter((note): note is string => Boolean(note)) + return notes.length > 0 ? notes.join('\n\n') : null +} diff --git a/frontend/src/components/Scenarios/scenarioLaunchRequest.test.ts b/frontend/src/components/Scenarios/scenarioLaunchRequest.test.ts new file mode 100644 index 0000000000..e012693cb1 --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioLaunchRequest.test.ts @@ -0,0 +1,124 @@ +import type { Parameter, RegisteredScenario } from '@/types' + +import { + buildEstimateRequest, + buildRunRequest, + scenarioDynamicParameters, + type ScenarioLaunchInput, +} from './scenarioLaunchRequest' + +const OBJECTIVE: Parameter = { + name: 'objective', description: 'The objective.', param_type: 'str', choices: null, default: null, required: false, +} as unknown as Parameter +const HISTORY: Parameter = { + name: 'prepended_conversation_id', description: 'History.', param_type: 'str', choices: null, default: null, + required: false, +} as unknown as Parameter + +function makeScenario(overrides: Partial = {}): RegisteredScenario { + return { + scenario_name: 'garak.api_key', + supported_parameter_names: [ + 'objective_target', 'scenario_techniques', 'dataset_config', 'memory_labels', + 'max_concurrency', 'max_retries', 'include_baseline', 'objective', 'prepended_conversation_id', + ], + supported_parameters: [OBJECTIVE, HISTORY], + uses_default_adversarial_target: true, + max_concrete_techniques: null, + ...overrides, + } as RegisteredScenario +} + +function makeInput(overrides: Partial = {}): ScenarioLaunchInput { + const scenario = overrides.scenario ?? makeScenario() + return { + scenario, + targetName: 'target', + adversarialTargetName: 'adversary', + techniques: ['get_key'], + dynamicParameters: scenarioDynamicParameters(scenario), + scenarioParamValues: { objective: 'steal a key', prepended_conversation_id: '' }, + datasetEdits: {}, + datasetError: null, + includeBaseline: false, + operationId: 'operation', + maxConcurrency: 4, + maxRetries: 1, + labels: { team: 'red' }, + ...overrides, + } +} + +describe('scenario launch requests', () => { + it('builds the run request from the same input as the estimate, plus run-only settings', () => { + const input = makeInput({ datasetEdits: { max_total: 3 }, includeBaseline: true }) + + const estimate = buildEstimateRequest(input) + const run = buildRunRequest(input) + + expect(estimate.ok && run.ok).toBe(true) + if (!estimate.ok || !run.ok) return + expect(run.request).toEqual({ + scenario_name: 'garak.api_key', + ...estimate.request, + operation_id: 'operation', + max_concurrency: 4, + max_retries: 1, + labels: { team: 'red' }, + }) + expect(estimate.request).toEqual({ + techniques: ['get_key'], + include_baseline: true, + target_name: 'target', + adversarial_target_name: 'adversary', + datasets: { max_total: 3 }, + scenario_params: { objective: 'steal a key' }, + }) + }) + + it('leaves out inputs the scenario does not take', () => { + const scenario = makeScenario({ + supported_parameter_names: ['objective_target'], + supported_parameters: [], + uses_default_adversarial_target: false, + }) + + const run = buildRunRequest(makeInput({ scenario, datasetEdits: { max_total: 3 } })) + + expect(run).toEqual({ ok: true, request: { scenario_name: 'garak.api_key', target_name: 'target', operation_id: 'operation' } }) + }) + + it('omits unchanged datasets and an unset operation', () => { + const run = buildRunRequest(makeInput({ operationId: null })) + + expect(run.ok && 'datasets' in run.request).toBe(false) + expect(run.ok && 'operation_id' in run.request).toBe(false) + }) + + it.each<[string, Partial, string]>([ + ['no target', { targetName: '' }, 'Select a target.'], + ['no technique', { techniques: [] }, 'Select at least one technique.'], + ['too many techniques', { + scenario: makeScenario({ max_concrete_techniques: 1 }), techniques: ['get_key', 'complete_key'], + }, 'Select at most 1 concrete technique(s).'], + ['pending technique edits', { techniqueEditError: 'Wait for valid technique settings before launch.' }, + 'Wait for valid technique settings before launch.'], + ['invalid datasets', { datasetError: 'Bad dataset limit.' }, 'Bad dataset limit.'], + ['an unsupported conversation', { + supportsPrependedConversation: false, + scenarioParamValues: { objective: '', prepended_conversation_id: 'saved' }, + }, 'This technique does not support a prepended conversation. Clear the saved conversation before launch.'], + ['concurrency out of range', { maxConcurrency: 0 }, 'Max concurrency must be an integer from 1 to 100.'], + ['fractional retries', { maxRetries: 1.5 }, 'Max retries must be an integer from 0 to 20.'], + ])('rejects %s with the message the form shows', (_, overrides, error) => { + expect(buildRunRequest(makeInput(overrides))).toEqual({ ok: false, error }) + }) + + it('estimates without a target, since only the launch needs one', () => { + expect(buildEstimateRequest(makeInput({ targetName: '' })).ok).toBe(true) + }) + + it('keeps run-only limits out of the estimate', () => { + expect(buildEstimateRequest(makeInput({ maxConcurrency: 0, maxRetries: -1 })).ok).toBe(true) + }) +}) diff --git a/frontend/src/components/Scenarios/scenarioLaunchRequest.ts b/frontend/src/components/Scenarios/scenarioLaunchRequest.ts new file mode 100644 index 0000000000..175cd6a3cd --- /dev/null +++ b/frontend/src/components/Scenarios/scenarioLaunchRequest.ts @@ -0,0 +1,196 @@ +import { buildParametersFromForm, type ParameterFormValue } from '@/components/Parameters/parameterForm' +import type { + Parameter, + RegisteredScenario, + RunScenarioRequest, + ScenarioDatasetRequest, + ScenarioRunSizeEstimateRequest, + TechniqueOverride, +} from '@/types' + +/** + * Common/opaque parameters the base scenario declares via + * `Scenario._common_scenario_parameters` — the launch form exposes a + * purpose-built control for each of these (target, techniques, datasets, + * labels, concurrency, retries, baseline), and `technique_converters` has no + * UI at all. They're hidden from the dynamic scenario-specific parameter list. + */ +export const COMMON_SCENARIO_PARAMETER_NAMES = new Set([ + 'objective_target', + 'scenario_techniques', + 'technique_converters', + 'local_technique_factory', + 'dataset_config', + 'dataset_overrides', + 'memory_labels', + 'max_concurrency', + 'max_retries', + 'include_baseline', +]) + +export const MIN_MAX_CONCURRENCY = 1 +export const MAX_MAX_CONCURRENCY = 100 +export const MIN_MAX_RETRIES = 0 +export const MAX_MAX_RETRIES = 20 +export const DEFAULT_MAX_CONCURRENCY = 10 +export const DEFAULT_MAX_RETRIES = 0 +export const CONVERSATION_PARAMETER_NAMES = new Set(['prepended_conversation_id', 'apply_converters_to_prepended_history']) +export const OBJECTIVE_PARAMETER_NAMES = new Set(['objective', 'harm_categories']) + +export function supportsParameter(scenario: RegisteredScenario, name: string): boolean { + return scenario.supported_parameter_names.includes(name) +} + +/** The scenario-specific parameters the launch form renders, without the common ones. */ +export function scenarioDynamicParameters(scenario: RegisteredScenario): Parameter[] { + return scenario.supported_parameters.filter( + (parameter) => supportsParameter(scenario, parameter.name) + && !COMMON_SCENARIO_PARAMETER_NAMES.has(parameter.name), + ) +} + +/** + * The one effective launch configuration. The estimate and the run request are + * both built from it, so the preview always describes what gets launched. + */ +export interface ScenarioLaunchInput { + scenario: RegisteredScenario + targetName: string + adversarialTargetName: string + techniques: string[] + dynamicParameters: Parameter[] + scenarioParamValues: Record + datasetEdits: ScenarioDatasetRequest + datasetError: string | null + includeBaseline: boolean + techniqueOverride?: TechniqueOverride + techniqueEditError?: string + supportsPrependedConversation?: boolean + operationId: string | null + maxConcurrency: number + maxRetries: number + labels: Record +} + +export type BuildEstimateRequestResult = + | { ok: true, request: ScenarioRunSizeEstimateRequest } + | { ok: false, error: string } + +export type BuildRunRequestResult = + | { ok: true, request: RunScenarioRequest } + | { ok: false, error: string } + +export function buildEstimateRequest({ + scenario, + targetName, + adversarialTargetName, + techniques, + dynamicParameters, + scenarioParamValues, + datasetEdits, + datasetError, + includeBaseline, + techniqueOverride, + techniqueEditError, + supportsPrependedConversation = true, +}: ScenarioLaunchInput): BuildEstimateRequestResult { + if (techniqueEditError) return { ok: false, error: techniqueEditError } + if (supportsParameter(scenario, 'scenario_techniques') && techniques.length === 0) { + return { ok: false, error: 'Select at least one technique.' } + } + if (supportsParameter(scenario, 'scenario_techniques') + && scenario.max_concrete_techniques != null && techniques.length > scenario.max_concrete_techniques) { + return { ok: false, error: `Select at most ${scenario.max_concrete_techniques} concrete technique(s).` } + } + + let scenarioParams: Record | null = null + if (dynamicParameters.length > 0) { + const history = scenarioParamValues.prepended_conversation_id + if (!supportsPrependedConversation && typeof history === 'string' && history.trim()) { + return { ok: false, error: 'This technique does not support a prepended conversation. Clear the saved conversation before launch.' } + } + const result = buildParametersFromForm( + dynamicParameters.filter((parameter) => supportsPrependedConversation || !CONVERSATION_PARAMETER_NAMES.has(parameter.name)), + scenarioParamValues, + ) + if (!result.ok) { + return result + } + scenarioParams = result.parameters + } + + const supportsDatasets = supportsParameter(scenario, 'dataset_config') + if (supportsDatasets && datasetError) return { ok: false, error: datasetError } + const request: ScenarioRunSizeEstimateRequest = {} + if (techniqueOverride) request.technique_override = techniqueOverride + if (supportsParameter(scenario, 'scenario_techniques')) { + request.techniques = techniques + } + if (supportsParameter(scenario, 'include_baseline')) { + request.include_baseline = includeBaseline + } + if (supportsParameter(scenario, 'objective_target') && targetName) { + request.target_name = targetName + } + if (scenario.uses_default_adversarial_target && adversarialTargetName) { + request.adversarial_target_name = adversarialTargetName + } + if (supportsDatasets && Object.keys(datasetEdits).length > 0) { + request.datasets = datasetEdits + } + if (scenarioParams) { + request.scenario_params = scenarioParams + } + return { ok: true, request } +} + +/** The run request is the estimate request plus the run-only settings, never a separate interpretation. */ +export function buildRunRequest(input: ScenarioLaunchInput): BuildRunRequestResult { + if (supportsParameter(input.scenario, 'objective_target') && !input.targetName) { + return { ok: false, error: 'Select a target.' } + } + const estimateResult = buildEstimateRequest(input) + if (!estimateResult.ok) { + return estimateResult + } + if ( + supportsParameter(input.scenario, 'max_concurrency') && ( + !Number.isInteger(input.maxConcurrency) + || input.maxConcurrency < MIN_MAX_CONCURRENCY + || input.maxConcurrency > MAX_MAX_CONCURRENCY + ) + ) { + return { + ok: false, + error: `Max concurrency must be an integer from ${MIN_MAX_CONCURRENCY} to ${MAX_MAX_CONCURRENCY}.`, + } + } + if ( + supportsParameter(input.scenario, 'max_retries') && ( + !Number.isInteger(input.maxRetries) + || input.maxRetries < MIN_MAX_RETRIES + || input.maxRetries > MAX_MAX_RETRIES + ) + ) { + return { + ok: false, + error: `Max retries must be an integer from ${MIN_MAX_RETRIES} to ${MAX_MAX_RETRIES}.`, + } + } + + const request: RunScenarioRequest = { + scenario_name: input.scenario.scenario_name, + ...estimateResult.request, + } + if (input.operationId) request.operation_id = input.operationId + if (supportsParameter(input.scenario, 'max_concurrency')) { + request.max_concurrency = input.maxConcurrency + } + if (supportsParameter(input.scenario, 'max_retries')) { + request.max_retries = input.maxRetries + } + if (supportsParameter(input.scenario, 'memory_labels')) { + request.labels = input.labels + } + return { ok: true, request } +} diff --git a/frontend/src/hooks/useScenarioDetail.test.tsx b/frontend/src/hooks/useScenarioDetail.test.tsx new file mode 100644 index 0000000000..d01d4fe321 --- /dev/null +++ b/frontend/src/hooks/useScenarioDetail.test.tsx @@ -0,0 +1,88 @@ +import { act, renderHook, waitFor } from '@testing-library/react' + +import { scenariosApi } from '@/services/api' +import type { RegisteredScenario } from '@/types' + +import { useScenarioDetail } from './useScenarioDetail' + +jest.mock('@/services/api', () => ({ scenariosApi: { getScenario: jest.fn() } })) +let runtime = { ready: true, generation: 'one' } +jest.mock('./useRuntime', () => ({ useRuntime: () => runtime })) +const getScenario = jest.mocked(scenariosApi.getScenario) + +const SCENARIO = { scenario_name: 'garak.api_key', supported_parameter_names: ['objective_target'] } as RegisteredScenario + +describe('useScenarioDetail', () => { + beforeEach(() => { + getScenario.mockReset() + runtime = { ready: true, generation: 'one' } + }) + + it('loads the scenario', async () => { + getScenario.mockResolvedValue(SCENARIO) + const view = renderHook(() => useScenarioDetail('garak.api_key')) + + expect(view.result.current.status).toBe('loading') + await waitFor(() => expect(view.result.current.status).toBe('success')) + expect(view.result.current.scenario).toBe(SCENARIO) + expect(getScenario).toHaveBeenCalledWith('garak.api_key') + }) + + it('reports a missing scenario without an error message', async () => { + getScenario.mockRejectedValue({ isAxiosError: true, response: { status: 404, data: { detail: 'Not found' } } }) + const view = renderHook(() => useScenarioDetail('gone')) + + await waitFor(() => expect(view.result.current.status).toBe('not-found')) + expect(view.result.current.error).toBeNull() + }) + + it('rejects a catalog entry without input metadata', async () => { + getScenario.mockResolvedValue({ scenario_name: 'old' } as RegisteredScenario) + const view = renderHook(() => useScenarioDetail('old')) + + await waitFor(() => expect(view.result.current.status).toBe('error')) + expect(view.result.current.error).toMatch(/no input support metadata/) + expect(view.result.current.scenario).toBeNull() + }) + + it('retries after a failure', async () => { + getScenario + .mockRejectedValueOnce({ isAxiosError: true, response: { status: 500, data: { detail: 'boom' } } }) + .mockResolvedValueOnce(SCENARIO) + const view = renderHook(() => useScenarioDetail('garak.api_key')) + await waitFor(() => expect(view.result.current.error).toBe('boom')) + + act(() => view.result.current.retry()) + expect(view.result.current.status).toBe('loading') + + await waitFor(() => expect(view.result.current.status).toBe('success')) + expect(getScenario).toHaveBeenCalledTimes(2) + }) + + it('waits for the runtime and reloads for a new generation', async () => { + getScenario.mockResolvedValue(SCENARIO) + runtime = { ready: false, generation: 'one' } + const view = renderHook(() => useScenarioDetail('garak.api_key')) + expect(getScenario).not.toHaveBeenCalled() + + runtime = { ready: true, generation: 'two' } + view.rerender() + await waitFor(() => expect(view.result.current.status).toBe('success')) + runtime = { ready: true, generation: 'three' } + view.rerender() + + await waitFor(() => expect(getScenario).toHaveBeenCalledTimes(2)) + }) + + it('ignores an answer that arrives after unmount', async () => { + let finish: (scenario: RegisteredScenario) => void = () => {} + getScenario.mockReturnValue(new Promise((resolve) => { finish = resolve })) + const view = renderHook(() => useScenarioDetail('garak.api_key')) + const before = view.result.current + + view.unmount() + await act(async () => { finish(SCENARIO) }) + + expect(view.result.current).toBe(before) + }) +}) diff --git a/frontend/src/hooks/useScenarioDetail.ts b/frontend/src/hooks/useScenarioDetail.ts new file mode 100644 index 0000000000..bde5580afc --- /dev/null +++ b/frontend/src/hooks/useScenarioDetail.ts @@ -0,0 +1,61 @@ +import { useCallback, useEffect, useState } from 'react' + +import { useRuntime } from '@/hooks/useRuntime' +import { scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { RegisteredScenario } from '@/types' + +export type ScenarioLoadStatus = 'loading' | 'success' | 'not-found' | 'error' + +export interface ScenarioDetailState { + status: ScenarioLoadStatus + scenario: RegisteredScenario | null + error: string | null + retry: () => void +} + +/** Load one registered scenario for the current runtime generation, ignoring answers for an older request. */ +export function useScenarioDetail(scenarioName: string): ScenarioDetailState { + const { generation, ready } = useRuntime() + const [scenario, setScenario] = useState(null) + const [status, setStatus] = useState('loading') + const [error, setError] = useState(null) + const [refetchCount, setRefetchCount] = useState(0) + + useEffect(() => { + if (!ready) return + let cancelled = false + scenariosApi + .getScenario(scenarioName) + .then((data) => { + if (cancelled) return + if (!Array.isArray(data.supported_parameter_names)) { + setScenario(null) + setStatus('error') + setError('The scenario catalog has no input support metadata. Update the backend and reload.') + return + } + setScenario(data) + setStatus('success') + setError(null) + }) + .catch((err: unknown) => { + if (cancelled) return + const apiError = toApiError(err) + setScenario(null) + setStatus(apiError.status === 404 ? 'not-found' : 'error') + setError(apiError.status === 404 ? null : apiError.detail) + }) + return () => { + cancelled = true + } + }, [scenarioName, refetchCount, generation, ready]) + + const retry = useCallback((): void => { + setStatus('loading') + setError(null) + setRefetchCount((count) => count + 1) + }, []) + + return { status, scenario, error, retry } +} diff --git a/frontend/src/hooks/useScenarioLaunch.test.tsx b/frontend/src/hooks/useScenarioLaunch.test.tsx new file mode 100644 index 0000000000..bdf46a1241 --- /dev/null +++ b/frontend/src/hooks/useScenarioLaunch.test.tsx @@ -0,0 +1,139 @@ +import type { FormEvent } from 'react' + +import { act, renderHook } from '@testing-library/react' + +import type { BuildRunRequestResult } from '@/components/Scenarios/scenarioLaunchRequest' +import { scenariosApi } from '@/services/api' +import type { ScenarioRunSummary } from '@/types' + +import { useScenarioLaunch } from './useScenarioLaunch' + +jest.mock('@/services/api', () => ({ scenariosApi: { startRun: jest.fn() } })) +const startRun = jest.mocked(scenariosApi.startRun) + +const VALID: BuildRunRequestResult = { ok: true, request: { scenario_name: 'scenario', target_name: 'target' } } +const SUMMARY = { scenario_result_id: 'run-1' } as ScenarioRunSummary + +interface Props { + requestResult?: BuildRunRequestResult + submitBlocked?: boolean + launchBlocked?: boolean +} + +function renderLaunch(initial: Props = {}) { + const onLaunched = jest.fn() + const view = renderHook( + ({ requestResult = VALID, submitBlocked = false, launchBlocked = false }: Props) => useScenarioLaunch({ + requestResult, submitBlocked, launchBlocked, onLaunched, + }), + { initialProps: initial }, + ) + return { view, onLaunched } +} + +function submitEvent(): FormEvent { + return { preventDefault: jest.fn() } as unknown as FormEvent +} + +describe('useScenarioLaunch', () => { + beforeEach(() => startRun.mockReset()) + + it('opens the confirmation only for a valid request', () => { + const { view } = renderLaunch({ requestResult: { ok: false, error: 'Select a target.' } }) + act(() => view.result.current.submitForm(submitEvent())) + expect(view.result.current.previewOpen).toBe(false) + expect(view.result.current.validationError).toBe('Select a target.') + + view.rerender({ requestResult: VALID }) + act(() => view.result.current.submitForm(submitEvent())) + expect(view.result.current.previewOpen).toBe(true) + expect(view.result.current.validationError).toBeNull() + }) + + it('does not open the confirmation while submitting is blocked', () => { + const { view } = renderLaunch({ submitBlocked: true }) + act(() => view.result.current.submitForm(submitEvent())) + + expect(view.result.current.previewOpen).toBe(false) + }) + + it('starts one run when confirm is pressed twice in the same tick', async () => { + let finish: (summary: ScenarioRunSummary) => void = () => {} + startRun.mockReturnValue(new Promise((resolve) => { finish = resolve })) + const { view, onLaunched } = renderLaunch() + act(() => view.result.current.submitForm(submitEvent())) + + let first: Promise = Promise.resolve() + let second: Promise = Promise.resolve() + act(() => { + first = view.result.current.confirmLaunch() + second = view.result.current.confirmLaunch() + }) + expect(view.result.current.submitting).toBe(true) + await act(async () => { + finish(SUMMARY) + await Promise.all([first, second]) + }) + + expect(startRun).toHaveBeenCalledTimes(1) + expect(startRun).toHaveBeenCalledWith(VALID.ok ? VALID.request : undefined) + expect(onLaunched).toHaveBeenCalledWith(SUMMARY) + expect(view.result.current.previewOpen).toBe(false) + expect(view.result.current.submitting).toBe(false) + }) + + it('keeps the form and shows the error when the launch fails', async () => { + startRun.mockRejectedValue({ isAxiosError: true, response: { status: 409, data: { detail: 'Already running.' } } }) + const { view, onLaunched } = renderLaunch() + act(() => view.result.current.submitForm(submitEvent())) + + await act(async () => { await view.result.current.confirmLaunch() }) + + expect(onLaunched).not.toHaveBeenCalled() + expect(view.result.current.apiError).toBe('Already running.') + expect(view.result.current.previewOpen).toBe(false) + expect(view.result.current.submitting).toBe(false) + }) + + it('does not start a run while launching is blocked', async () => { + const { view } = renderLaunch({ launchBlocked: true }) + + await act(async () => { await view.result.current.confirmLaunch() }) + + expect(startRun).not.toHaveBeenCalled() + }) + + it('closes the confirmation for a request that became invalid', async () => { + const { view } = renderLaunch() + act(() => view.result.current.submitForm(submitEvent())) + view.rerender({ requestResult: { ok: false, error: 'Select at least one technique.' } }) + + await act(async () => { await view.result.current.confirmLaunch() }) + + expect(startRun).not.toHaveBeenCalled() + expect(view.result.current.previewOpen).toBe(false) + expect(view.result.current.validationError).toBe('Select at least one technique.') + }) + + it('keeps the confirmation open while a launch is in flight', async () => { + startRun.mockReturnValue(new Promise(() => {})) + const { view } = renderLaunch() + act(() => view.result.current.submitForm(submitEvent())) + act(() => { void view.result.current.confirmLaunch() }) + + act(() => view.result.current.dismissPreview()) + act(() => view.result.current.changePreviewOpen(false)) + + expect(view.result.current.previewOpen).toBe(true) + }) + + it('closes the confirmation on cancel', () => { + const { view } = renderLaunch() + act(() => view.result.current.submitForm(submitEvent())) + + act(() => view.result.current.dismissPreview()) + + expect(view.result.current.previewOpen).toBe(false) + expect(startRun).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/hooks/useScenarioLaunch.ts b/frontend/src/hooks/useScenarioLaunch.ts new file mode 100644 index 0000000000..0e3d3158c7 --- /dev/null +++ b/frontend/src/hooks/useScenarioLaunch.ts @@ -0,0 +1,130 @@ +import { type FormEvent, type RefObject, useRef, useState } from 'react' + +import type { BuildRunRequestResult } from '@/components/Scenarios/scenarioLaunchRequest' +import { scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { ScenarioRunSummary } from '@/types' + +interface UseScenarioLaunchInput { + /** The run request built from the current form, or why it can't be built. */ + requestResult: BuildRunRequestResult + /** True when the form can't open the confirmation, for example while a conversation is being edited. */ + submitBlocked: boolean + /** True when a confirmed launch must not start, for example while the runtime is not ready. */ + launchBlocked: boolean + onLaunched: (summary: ScenarioRunSummary) => void +} + +export interface ScenarioLaunch { + previewOpen: boolean + submitting: boolean + apiError: string | null + validationError: string | null + clearValidationError: () => void + /** Synchronous submit state, for guards that run before React re-renders. */ + isSubmitting: () => boolean + launchButtonRef: RefObject + submitForm: (event: FormEvent) => void + dismissPreview: () => void + changePreviewOpen: (open: boolean) => void + confirmLaunch: () => Promise +} + +function focusSoon(ref: RefObject): void { + const focus = () => { + ref.current?.focus() + } + if (typeof requestAnimationFrame === 'function') { + requestAnimationFrame(focus) + } else { + setTimeout(focus, 0) + } +} + +/** Own the confirmation dialog and the single in-flight launch for a scenario run. */ +export function useScenarioLaunch({ + requestResult, + submitBlocked, + launchBlocked, + onLaunched, +}: UseScenarioLaunchInput): ScenarioLaunch { + const [previewOpen, setPreviewOpen] = useState(false) + const [submitting, setSubmitting] = useState(false) + const [apiError, setApiError] = useState(null) + const [validationError, setValidationError] = useState(null) + // Synchronous guard against a double-submit racing ahead of the state update. + const isSubmittingRef = useRef(false) + const launchButtonRef = useRef(null) + + const submitForm = (event: FormEvent): void => { + event.preventDefault() + if (submitBlocked || isSubmittingRef.current) return + setApiError(null) + if (!requestResult.ok) { + setValidationError(requestResult.error) + return + } + setValidationError(null) + setPreviewOpen(true) + } + + const dismissPreview = (): void => { + if (!submitting) { + setPreviewOpen(false) + focusSoon(launchButtonRef) + } + } + + const changePreviewOpen = (open: boolean): void => { + if (!submitting) { + setPreviewOpen(open) + if (!open) { + focusSoon(launchButtonRef) + } + } + } + + const confirmLaunch = async (): Promise => { + if (isSubmittingRef.current || launchBlocked) { + return + } + + setApiError(null) + if (!requestResult.ok) { + setValidationError(requestResult.error) + setPreviewOpen(false) + return + } + + isSubmittingRef.current = true + setSubmitting(true) + setValidationError(null) + + try { + const summary = await scenariosApi.startRun(requestResult.request) + setPreviewOpen(false) + onLaunched(summary) + } catch (err) { + setApiError(toApiError(err).detail) + setPreviewOpen(false) + focusSoon(launchButtonRef) + } finally { + isSubmittingRef.current = false + setSubmitting(false) + } + } + + return { + previewOpen, + submitting, + apiError, + validationError, + clearValidationError: () => setValidationError(null), + isSubmitting: () => isSubmittingRef.current, + launchButtonRef, + submitForm, + dismissPreview, + changePreviewOpen, + confirmLaunch, + } +} diff --git a/frontend/src/hooks/useScenarioRunEstimate.test.tsx b/frontend/src/hooks/useScenarioRunEstimate.test.tsx new file mode 100644 index 0000000000..de7f440b92 --- /dev/null +++ b/frontend/src/hooks/useScenarioRunEstimate.test.tsx @@ -0,0 +1,154 @@ +import { act, renderHook } from '@testing-library/react' + +import { scenariosApi } from '@/services/api' +import type { ScenarioRunSizeEstimateRequest, ScenarioRunSizeEstimateResponse } from '@/types' + +import { ESTIMATE_DEBOUNCE_MS, useScenarioRunEstimate } from './useScenarioRunEstimate' + +jest.mock('@/services/api', () => ({ scenariosApi: { estimateRun: jest.fn() } })) +const estimateRun = jest.mocked(scenariosApi.estimateRun) + +function response(total: number, maxTotal: number | 'all' = 'all'): ScenarioRunSizeEstimateResponse { + return { + dataset_size: { kind: 'bounded', value: total }, + dataset_limit: { state: 'scenario_default' }, + estimated_attack_count: total, + components: [{ label: 'Attacks', count: total, is_baseline: false, note: null }], + datasets: [], + note: null, + effective_datasets: { sources: [], max_per_dataset: 'all', max_total: maxTotal, fetch: 'if_missing', filters: {} }, + } as unknown as ScenarioRunSizeEstimateResponse +} + +interface Props { + request: ScenarioRunSizeEstimateRequest | null + ready?: boolean + paused?: boolean + invalidReason?: string | null +} + +function renderEstimate(initial: Props) { + return renderHook( + ({ request, ready = true, paused = false, invalidReason = null }: Props) => useScenarioRunEstimate({ + scenarioName: 'scenario', + request, + requestKey: request === null ? null : JSON.stringify(request), + invalidReason, + ready, + paused, + }), + { initialProps: initial }, + ) +} + +async function settle(milliseconds = ESTIMATE_DEBOUNCE_MS): Promise { + await act(async () => { + jest.advanceTimersByTime(milliseconds) + await Promise.resolve() + await Promise.resolve() + }) +} + +function deferred() { + let resolve: (value: T) => void = () => {} + let reject: (error: unknown) => void = () => {} + const promise = new Promise((done, fail) => { resolve = done; reject = fail }) + return { promise, resolve, reject } +} + +describe('useScenarioRunEstimate', () => { + beforeEach(() => { + jest.useFakeTimers() + estimateRun.mockReset() + }) + afterEach(() => jest.useRealTimers()) + + it('sends only the last request after rapid changes', async () => { + estimateRun.mockResolvedValue(response(5)) + const view = renderEstimate({ request: { techniques: ['a'] } }) + view.rerender({ request: { techniques: ['a', 'b'] } }) + view.rerender({ request: { techniques: ['b'] } }) + + await settle(ESTIMATE_DEBOUNCE_MS - 1) + expect(estimateRun).not.toHaveBeenCalled() + await settle(1) + + expect(estimateRun).toHaveBeenCalledTimes(1) + expect(estimateRun.mock.calls[0][1]).toEqual({ techniques: ['b'] }) + expect(view.result.current.state.status).toBe('available') + }) + + it('drops an older answer that arrives after a newer request', async () => { + const slow = deferred() + estimateRun.mockReturnValueOnce(slow.promise).mockResolvedValueOnce(response(9)) + const view = renderEstimate({ request: { techniques: ['a'] } }) + await settle() + view.rerender({ request: { techniques: ['b'] } }) + await settle() + + await act(async () => { slow.resolve(response(1)) }) + + expect(view.result.current.current?.status).toBe('resolved') + expect(view.result.current.state.status === 'available' && view.result.current.state.estimate.total).toBe(9) + }) + + it('keeps the last good estimate as stale when a refresh fails', async () => { + estimateRun.mockResolvedValueOnce(response(4)).mockRejectedValueOnce({ isAxiosError: true, response: { status: 500, data: { detail: 'down' } } }) + const view = renderEstimate({ request: { techniques: ['a'] } }) + await settle() + view.rerender({ request: { techniques: ['b'] } }) + + expect(view.result.current.state.status).toBe('refreshing') + await settle() + + const state = view.result.current.state + expect(state.status).toBe('stale') + expect(state.status === 'stale' && state.estimate.total).toBe(4) + }) + + it('refreshes allowed datasets without the rejected edits and keeps the edited settings', async () => { + estimateRun + .mockResolvedValueOnce(response(4, 7)) + .mockRejectedValueOnce({ isAxiosError: true, response: { status: 422, data: { detail: 'bad source' } } }) + .mockResolvedValueOnce(response(4, 'all')) + const view = renderEstimate({ request: { techniques: ['a'] } }) + await settle() + view.rerender({ request: { techniques: ['a'], datasets: { sources: [{ name: 'nope' }] } } }) + await settle() + + expect(estimateRun).toHaveBeenCalledTimes(3) + expect(estimateRun.mock.calls[2][1]).toEqual({ techniques: ['a'] }) + expect(view.result.current.datasetProjection?.effective_datasets?.max_total).toBe(7) + }) + + it('reports why no request could be built instead of calling the backend', async () => { + const view = renderEstimate({ request: null, invalidReason: 'Select a target.' }) + await settle() + + expect(estimateRun).not.toHaveBeenCalled() + expect(view.result.current.state).toMatchObject({ status: 'unavailable', note: 'Select a target.' }) + }) + + it.each([ + ['not ready', { ready: false }], + ['paused', { paused: true }], + ])('waits while %s', async (_, props) => { + renderEstimate({ request: { techniques: ['a'] }, ...props }) + await settle() + + expect(estimateRun).not.toHaveBeenCalled() + }) + + it('cancels the pending request on unmount', async () => { + const pending = deferred() + estimateRun.mockReturnValueOnce(pending.promise) + const view = renderEstimate({ request: { techniques: ['a'] } }) + await settle() + const signal = estimateRun.mock.calls[0][2] as AbortSignal + + view.unmount() + + expect(signal.aborted).toBe(true) + await act(async () => { pending.resolve(response(2)) }) + }) +}) diff --git a/frontend/src/hooks/useScenarioRunEstimate.ts b/frontend/src/hooks/useScenarioRunEstimate.ts new file mode 100644 index 0000000000..67c7d2f3f3 --- /dev/null +++ b/frontend/src/hooks/useScenarioRunEstimate.ts @@ -0,0 +1,161 @@ +import { useEffect, useRef, useState } from 'react' + +import { mapScenarioRunEstimate } from '@/components/Scenarios/scenarioRunEstimateAdapter' +import { scenariosApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { + ScenarioDatasetSettings, + ScenarioRunEstimateResult, + ScenarioRunEstimateState, + ScenarioRunSizeEstimateRequest, + ScenarioRunSizeEstimateResponse, +} from '@/types' + +export const ESTIMATE_DEBOUNCE_MS = 300 + +type SuccessfulEstimateResult = Extract + +/** The backend answer for one request key. */ +export type EstimateRequestState = + | { status: 'resolved', requestKey: string, result: ScenarioRunEstimateResult, datasetSettings?: ScenarioDatasetSettings } + | { status: 'error', requestKey: string, error: string } + +interface UseScenarioRunEstimateInput { + scenarioName: string + /** The request to estimate, or null when the form can't produce one. */ + request: ScenarioRunSizeEstimateRequest | null + /** Identifies the request; answers for any other key are stale. */ + requestKey: string | null + /** Why no request could be built, shown while ``request`` is null. */ + invalidReason: string | null + ready: boolean + /** Pauses estimation, for example while a reinitialized runtime awaits review. */ + paused: boolean +} + +export interface ScenarioRunEstimate { + state: ScenarioRunEstimateState + /** The latest dataset projection from the backend, if any. */ + datasetProjection: ScenarioRunSizeEstimateResponse | null + /** The backend answer for the current request, or null while it's pending. */ + current: EstimateRequestState | null +} + +/** + * Debounce estimate requests and keep only the answer for the newest request. + * Older answers, aborted requests, and answers after unmount are dropped. + */ +export function useScenarioRunEstimate({ + scenarioName, + request, + requestKey, + invalidReason, + ready, + paused, +}: UseScenarioRunEstimateInput): ScenarioRunEstimate { + const [requestState, setRequestState] = useState(null) + const [lastGoodEstimate, setLastGoodEstimate] = useState(null) + const [datasetProjection, setDatasetProjection] = useState(null) + const sequenceRef = useRef(0) + + useEffect(() => { + if (!ready || paused || request === null || requestKey === null) { + return + } + + const requestSequence = sequenceRef.current + 1 + sequenceRef.current = requestSequence + const controller = new AbortController() + const isStale = (): boolean => controller.signal.aborted || requestSequence !== sequenceRef.current + + const debounceTimer = window.setTimeout(() => { + scenariosApi + .estimateRun(scenarioName, request, controller.signal) + .then((response) => { + if (isStale()) return + const result = mapScenarioRunEstimate(response, 'request') + setDatasetProjection(response) + setRequestState({ status: 'resolved', requestKey, result, datasetSettings: response.effective_datasets }) + if (result.status === 'available' || result.status === 'conditional') { + setLastGoodEstimate(result) + } + }) + .catch((err: unknown) => { + if (isStale()) return + const error = toApiError(err) + setRequestState({ status: 'error', requestKey, error: error.detail }) + if (request.datasets && (error.status === 400 || error.status === 422)) { + const { datasets, ...defaultRequest } = request + void datasets + // Keep invalid edits visible while refreshing parameter-aware allowed sources. + void scenariosApi.estimateRun(scenarioName, defaultRequest, controller.signal) + .then((response) => { + if (isStale()) return + setDatasetProjection((previous) => ({ + ...response, + effective_datasets: previous?.effective_datasets ?? response.effective_datasets, + })) + }) + .catch(() => { /* The original validation error remains authoritative. */ }) + } + }) + }, ESTIMATE_DEBOUNCE_MS) + + return () => { + window.clearTimeout(debounceTimer) + controller.abort() + } + }, [request, requestKey, scenarioName, ready, paused]) + + const current = requestState?.requestKey === requestKey ? requestState : null + return { + state: estimateState({ invalidReason: request === null ? invalidReason : null, current, lastGoodEstimate }), + datasetProjection, + current, + } +} + +function estimateState({ + invalidReason, + current, + lastGoodEstimate, +}: { + invalidReason: string | null + current: EstimateRequestState | null + lastGoodEstimate: SuccessfulEstimateResult | null +}): ScenarioRunEstimateState { + if (invalidReason !== null) { + return { + status: 'unavailable', + scope: 'request', + label: 'Complete the required configuration to request an estimate.', + note: invalidReason, + } + } + if (current?.status === 'resolved') { + return current.result + } + if (current?.status === 'error') { + return lastGoodEstimate + ? { + status: 'stale', + estimate: lastGoodEstimate.estimate, + label: 'Showing the last successful estimate.', + error: current.error, + } + : { + status: 'unavailable', + scope: 'request', + label: 'The backend estimate could not be refreshed.', + note: current.error, + } + } + if (lastGoodEstimate) { + return { + status: 'refreshing', + estimate: lastGoodEstimate.estimate, + label: 'Updating for the current configuration…', + } + } + return { status: 'loading', scope: 'request' } +}