diff --git a/pyrit/score/scorer_evaluation/scorer_metrics_io.py b/pyrit/score/scorer_evaluation/scorer_metrics_io.py index 0ec19239be..01fea7beea 100644 --- a/pyrit/score/scorer_evaluation/scorer_metrics_io.py +++ b/pyrit/score/scorer_evaluation/scorer_metrics_io.py @@ -43,6 +43,17 @@ } +def _harm_metrics_path(harm_category: str) -> Path: + """ + Return the registry file for a harm category, including canonical category names. + + Returns: + Path: The JSONL metrics file for the category. + """ + file_name = _HARM_METRICS_FILES_BY_CATEGORY.get(harm_category, f"{harm_category}_metrics.jsonl") + return SCORER_EVALS_PATH / "harm" / file_name + + def _metrics_to_registry_dict(metrics: ScorerMetrics) -> dict[str, Any]: """ Convert metrics to a dictionary suitable for registry storage. @@ -123,8 +134,7 @@ def get_all_harm_metrics( Access metrics via `entry.metrics.mean_absolute_error`, `entry.metrics.harm_category`, etc. Access scorer info via `entry.scorer_identifier.class_name`, etc. """ - file_path = SCORER_EVALS_PATH / "harm" / f"{harm_category}_metrics.jsonl" - return _load_metrics_from_file(file_path=file_path, metrics_class=HarmScorerMetrics) + return _load_metrics_from_file(file_path=_harm_metrics_path(harm_category), metrics_class=HarmScorerMetrics) def _load_metrics_from_file( @@ -222,8 +232,7 @@ def find_harm_metrics_by_eval_hash( if file_path is None: if harm_category is None: raise ValueError("Either harm_category or file_path must be provided.") - file_name = _HARM_METRICS_FILES_BY_CATEGORY.get(harm_category, f"{harm_category}_metrics.jsonl") - file_path = SCORER_EVALS_PATH / "harm" / file_name + file_path = _harm_metrics_path(harm_category) return _find_metrics_by_eval_hash(file_path=file_path, eval_hash=eval_hash, metrics_class=HarmScorerMetrics) diff --git a/tests/unit/score/test_scorer_metrics_io.py b/tests/unit/score/test_scorer_metrics_io.py index a73c484c39..9802b63815 100644 --- a/tests/unit/score/test_scorer_metrics_io.py +++ b/tests/unit/score/test_scorer_metrics_io.py @@ -12,6 +12,7 @@ import pytest from pyrit.models import ComponentIdentifier +from pyrit.models.harm_category import HarmCategory from pyrit.score.scorer_evaluation.scorer_metrics import ( HarmScorerMetrics, ObjectiveScorerMetrics, @@ -327,6 +328,35 @@ def test_get_all_harm_metrics(): assert results[0].metrics.mean_absolute_error == 0.08 +@pytest.mark.parametrize( + ("harm_category", "expected_file_name"), + [ + ("REPRESENTATIONAL", "representational_metrics.jsonl"), + ("SEXUAL_CONTENT", "sexual_metrics.jsonl"), + ("violence", "violence_metrics.jsonl"), + ], +) +def test_get_all_harm_metrics_reads_the_same_file_as_the_eval_hash_lookup( + *, harm_category: str, expected_file_name: str +) -> None: + with patch("pyrit.score.scorer_evaluation.scorer_metrics_io._load_jsonl", return_value=[]) as mock_load: + get_all_harm_metrics(harm_category=harm_category) + find_harm_metrics_by_eval_hash(eval_hash="missing", harm_category=harm_category) + + assert [call[0][0].name for call in mock_load.call_args_list] == [expected_file_name, expected_file_name] + + +@pytest.mark.parametrize( + "scorer_harm_category", + [HarmCategory.SEXUAL_CONTENT.name, HarmCategory.REPRESENTATIONAL.name, "hate_speech", "self_harm", "violence"], +) +def test_get_all_harm_metrics_finds_registry_entries_for_scorer_categories(*, scorer_harm_category: str) -> None: + results = get_all_harm_metrics(harm_category=scorer_harm_category) + + assert results + assert all(result.metrics.harm_category for result in results) + + # --- add_evaluation_results tests ---