diff --git a/pyrit/analytics/outcome_statistics.py b/pyrit/analytics/outcome_statistics.py index 5fcfdf723b..2828eda250 100644 --- a/pyrit/analytics/outcome_statistics.py +++ b/pyrit/analytics/outcome_statistics.py @@ -6,6 +6,7 @@ from __future__ import annotations from collections import Counter +import math from typing import TYPE_CHECKING from pyrit.models import AttackOutcome, OutcomeStatistics @@ -16,6 +17,17 @@ from pyrit.models import AttackStats + +def _calculate_confidence_interval(successes: int, total: int, z: float = 1.96) -> tuple[float | None, float | None]: + """Calculate the Wilson score interval for a proportion.""" + if total == 0: + return None, None + p = successes / total + denominator = 1 + z**2 / total + center = (p + z**2 / (2 * total)) / denominator + margin = z * math.sqrt((p * (1 - p) / total) + (z**2 / (4 * total**2))) / denominator + return max(0.0, center - margin), min(1.0, center + margin) + def compute_outcome_statistics(counts: Mapping[str, int] | Mapping[AttackOutcome, int]) -> OutcomeStatistics: """ Calculate both success rates for an already selected population. @@ -42,6 +54,7 @@ def compute_outcome_statistics(counts: Mapping[str, int] | Mapping[AttackOutcome errors = values.get(AttackOutcome.ERROR, 0) decided = successes + failures total = decided + undetermined + errors + ci_lower, ci_upper = _calculate_confidence_interval(successes, decided) return OutcomeStatistics( successes=successes, failures=failures, @@ -50,6 +63,8 @@ def compute_outcome_statistics(counts: Mapping[str, int] | Mapping[AttackOutcome total_decided=decided, total_results=total, success_rate=_rate(successes=successes, total=decided), + confidence_interval_lower=ci_lower, + confidence_interval_upper=ci_upper, success_rate_all=_rate(successes=successes, total=total), decided_share=_rate(successes=decided, total=total), outcome_shares={ diff --git a/pyrit/models/analytics.py b/pyrit/models/analytics.py index 4518c46598..d85e667836 100644 --- a/pyrit/models/analytics.py +++ b/pyrit/models/analytics.py @@ -165,6 +165,9 @@ def _validate_rate( # Preserve the existing class/constructor identity while sharing it beyond attack reports. + confidence_interval_lower: float | None = None + confidence_interval_upper: float | None = None + OutcomeStatistics = AttackAnalyticsStatistics diff --git a/pyrit/score/scorer_evaluation/scorer_evaluator.py b/pyrit/score/scorer_evaluation/scorer_evaluator.py index d3e65ea0ec..1a6ef953ad 100644 --- a/pyrit/score/scorer_evaluation/scorer_evaluator.py +++ b/pyrit/score/scorer_evaluation/scorer_evaluator.py @@ -862,6 +862,7 @@ def _compute_metrics( accuracy = (true_positive + true_negative) / gold_scores.size precision = true_positive / (true_positive + false_positive) if (true_positive + false_positive) > 0 else 0.0 recall = true_positive / (true_positive + false_negative) if (true_positive + false_negative) > 0 else 0.0 + specificity = true_negative / (true_negative + false_positive) if (true_negative + false_positive) > 0 else 0.0 f1_score = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0 num_responses = all_human_scores.shape[1] @@ -874,6 +875,7 @@ def _compute_metrics( accuracy_standard_error=np.sqrt(accuracy * (1 - accuracy) / gold_scores.size), precision=precision, recall=recall, + specificity=specificity, f1_score=f1_score, num_scorer_trials=num_scorer_trials, dataset_name=dataset_name, diff --git a/pyrit/score/scorer_evaluation/scorer_metrics.py b/pyrit/score/scorer_evaluation/scorer_metrics.py index 0f5ede65ee..e3891cb8ad 100644 --- a/pyrit/score/scorer_evaluation/scorer_metrics.py +++ b/pyrit/score/scorer_evaluation/scorer_metrics.py @@ -287,6 +287,7 @@ class ObjectiveScorerMetrics(ScorerMetrics): f1_score: float precision: float recall: float + specificity: float @dataclass