Skip to content
Open
2 changes: 1 addition & 1 deletion pyrit/executor/attack/multi_turn/crescendo.py
Original file line number Diff line number Diff line change
Expand Up @@ -814,7 +814,7 @@ async def _perform_backtrack_if_refused_async(
"""
context.last_response_was_refusal = False
# Check for refusal using the scorer (handles blocked/error responses internally)
refusal_score = await self._check_refusal_async(context, prompt_sent)
refusal_score = await self._check_refusal_async(context, context.objective)
is_refusal = score_is_true(refusal_score)
self._logger.debug(f"Refusal check: {is_refusal} - {(refusal_score.score_rationale or '')[:100]}...")
context.last_response_was_refusal = is_refusal
Expand Down
34 changes: 34 additions & 0 deletions tests/unit/executor/attack/multi_turn/test_crescendo.py
Original file line number Diff line number Diff line change
Expand Up @@ -1228,6 +1228,40 @@ async def test_check_refusal_scores_error_responses(
assert "skip_on_error_result" not in call_kwargs
assert "role_filter" not in call_kwargs

async def test_backtrack_refusal_check_receives_the_attack_objective(
self,
mock_objective_target: MagicMock,
mock_adversarial_chat: MagicMock,
mock_refusal_scorer: MagicMock,
basic_context: CrescendoAttackContext,
sample_response: Message,
refusal_score: Score,
):
"""The refusal judge must evaluate deflection against the attack objective.

The sent prompt used to be passed as the objective, so the judge saw the
adversarial prompt itself labeled as the conversation objective.
"""
adversarial_config = AttackAdversarialConfig(target=mock_adversarial_chat)
scoring_config = AttackScoringConfig(refusal_scorer=mock_refusal_scorer)

attack = CrescendoAttack(
objective_target=mock_objective_target,
attack_adversarial_config=adversarial_config,
attack_scoring_config=scoring_config,
)

basic_context.last_response = sample_response
mock_refusal_scorer.score_async.return_value = [refusal_score]

with patch.object(attack, "_backtrack_memory_async", new_callable=AsyncMock, return_value="new_conv_id"):
await attack._perform_backtrack_if_refused_async(
context=basic_context, prompt_sent="Long adversarial prompt text"
)

expectation = mock_refusal_scorer.score_async.call_args.kwargs["expectation"]
assert expectation.objective == "Test objective"


@pytest.mark.usefixtures("patch_central_database")
class TestBacktrackingLogic:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -325,8 +325,10 @@ async def score_objective(**_kwargs):
"response-9",
"response-10-final",
]
# The refusal judge renders its criterion as the conversation objective,
# so it receives the attack objective rather than the prompt that was sent.
assert [call.kwargs["expectation"].objective for call in refusal_scorer.score_async.await_args_list] == [
f"question-{attempt}" for attempt in range(1, 13)
_OBJECTIVE for _ in range(12)
]
objective_inputs = [call.kwargs["response"].get_value() for call in score_response.await_args_list]
assert refusal_scorer.score_async.await_count == 12
Expand Down Expand Up @@ -550,8 +552,8 @@ async def test_accepted_text_response_consumes_seed_before_followup(self, tmp_pa
assert [message.get_value() for message in refusal_inputs] == ["first text response", "final text response"]
assert [message.get_value() for message in objective_inputs] == ["first text response", "final text response"]
assert [call.kwargs["expectation"].objective for call in refusal_scorer.score_async.await_args_list] == [
"question-1",
"question-2",
_OBJECTIVE,
_OBJECTIVE,
]
assert [call.kwargs["expectation"].objective for call in score_response.await_args_list] == [
_OBJECTIVE,
Expand Down Expand Up @@ -725,9 +727,9 @@ async def test_first_turn_content_filter_retries_seed_then_consumes_it(self, tmp
]
assert refusal_inputs[0].get_piece().response_error == "blocked"
assert [call.kwargs["expectation"].objective for call in refusal_scorer.score_async.await_args_list] == [
"question-1",
"question-2",
"question-3",
_OBJECTIVE,
_OBJECTIVE,
_OBJECTIVE,
]
assert [call.kwargs["response"].get_value() for call in score_response.await_args_list] == [
"accepted retry response",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -519,7 +519,9 @@ async def test_crescendo_keeps_refusal_criterion_separate_async(self) -> None:
assert target.prompt_sent == ["seed prompt"]
assert objective.calls[0][1] is supplied
assert auxiliary.calls[0][1] is supplied
assert refusal.calls[0][1] == ScoringExpectation(objective="seed prompt")
# The refusal judge renders its criterion as the conversation objective,
# so it receives the attack objective, not the prompt that was sent.
assert refusal.calls[0][1] == ScoringExpectation(objective="attack objective")
assert refusal.calls[0][0] == objective.calls[0][0]

async def test_chunked_scores_combined_content_with_auxiliary_async(self, sqlite_instance: SQLiteMemory) -> None:
Expand Down