Feat: Report specificity and confidence intervals in analytics

pyrit/analytics/outcome_statistics.py

@@ -6,6 +6,7 @@
 from __future__ import annotations
 
 from collections import Counter
+import math
 from typing import TYPE_CHECKING
 
 from pyrit.models import AttackOutcome, OutcomeStatistics
@@ -16,6 +17,17 @@
     from pyrit.models import AttackStats
 
 
+
+def _calculate_confidence_interval(successes: int, total: int, z: float = 1.96) -> tuple[float | None, float | None]:
+    """Calculate the Wilson score interval for a proportion."""
+    if total == 0:
+        return None, None
+    p = successes / total
+    denominator = 1 + z**2 / total
+    center = (p + z**2 / (2 * total)) / denominator
+    margin = z * math.sqrt((p * (1 - p) / total) + (z**2 / (4 * total**2))) / denominator
+    return max(0.0, center - margin), min(1.0, center + margin)
+
 def compute_outcome_statistics(counts: Mapping[str, int] | Mapping[AttackOutcome, int]) -> OutcomeStatistics:
     """
     Calculate both success rates for an already selected population.
@@ -42,6 +54,7 @@ def compute_outcome_statistics(counts: Mapping[str, int] | Mapping[AttackOutcome
     errors = values.get(AttackOutcome.ERROR, 0)
     decided = successes + failures
     total = decided + undetermined + errors
+    ci_lower, ci_upper = _calculate_confidence_interval(successes, decided)
     return OutcomeStatistics(
         successes=successes,
         failures=failures,
@@ -50,6 +63,8 @@ def compute_outcome_statistics(counts: Mapping[str, int] | Mapping[AttackOutcome
         total_decided=decided,
         total_results=total,
         success_rate=_rate(successes=successes, total=decided),
+        confidence_interval_lower=ci_lower,
+        confidence_interval_upper=ci_upper,
         success_rate_all=_rate(successes=successes, total=total),
         decided_share=_rate(successes=decided, total=total),
         outcome_shares={

pyrit/models/analytics.py

@@ -165,6 +165,9 @@ def _validate_rate(
 
 
 # Preserve the existing class/constructor identity while sharing it beyond attack reports.
+    confidence_interval_lower: float | None = None
+    confidence_interval_upper: float | None = None
+
 OutcomeStatistics = AttackAnalyticsStatistics
 
 

pyrit/score/scorer_evaluation/scorer_evaluator.py

@@ -862,6 +862,7 @@ def _compute_metrics(
         accuracy = (true_positive + true_negative) / gold_scores.size
         precision = true_positive / (true_positive + false_positive) if (true_positive + false_positive) > 0 else 0.0
         recall = true_positive / (true_positive + false_negative) if (true_positive + false_negative) > 0 else 0.0
+        specificity = true_negative / (true_negative + false_positive) if (true_negative + false_positive) > 0 else 0.0
         f1_score = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0.0
 
         num_responses = all_human_scores.shape[1]
@@ -874,6 +875,7 @@ def _compute_metrics(
             accuracy_standard_error=np.sqrt(accuracy * (1 - accuracy) / gold_scores.size),
             precision=precision,
             recall=recall,
+            specificity=specificity,
             f1_score=f1_score,
             num_scorer_trials=num_scorer_trials,
             dataset_name=dataset_name,

pyrit/score/scorer_evaluation/scorer_metrics.py

@@ -287,6 +287,7 @@ class ObjectiveScorerMetrics(ScorerMetrics):
     f1_score: float
     precision: float
     recall: float
+    specificity: float
 
 
 @dataclass