FEAT: Use SeedDataset-backed inline data for GCG

doc/code/executor/gcg/1_gcg_azure_ml.ipynb

@@ -57,9 +57,15 @@
    "source": [
     "import os\n",
     "\n",
-    "from pyrit.setup.initialization import _load_environment_files\n",
+    "from pyrit.setup import ConfigurationLoader, initialize_pyrit_async\n",
     "\n",
-    "_load_environment_files(env_files=None)\n",
+    "pyrit_config = ConfigurationLoader.load_with_overrides()\n",
+    "await initialize_pyrit_async(\n",
+    "    memory_db_type=\"InMemory\",\n",
+    "    load_defaults=False,\n",
+    "    env_files=pyrit_config.resolve_env_files(),\n",
+    "    env_akv_ref=pyrit_config.resolve_env_akv_ref(),\n",
+    ")\n",
     "\n",
     "subscription_id = os.environ.get(\"AZURE_ML_SUBSCRIPTION_ID\")\n",
     "resource_group = os.environ.get(\"AZURE_ML_RESOURCE_GROUP\")\n",
@@ -111,9 +117,9 @@
    ],
    "source": [
     "from azure.ai.ml import MLClient\n",
-    "from azure.identity import AzureCliCredential\n",
+    "from azure.identity import DefaultAzureCredential\n",
     "\n",
-    "ml_client = MLClient(AzureCliCredential(), subscription_id, resource_group, workspace)"
+    "ml_client = MLClient(DefaultAzureCredential(), subscription_id, resource_group, workspace)"
    ]
   },
   {
@@ -192,18 +198,90 @@
     "Docker-installed package (Python's `-m` flag puts the cwd at the front of `sys.path`).\n",
     "\n",
     "The new public API takes a typed ``GCGConfig`` (strategy) and a separate\n",
-    "``GCGDataConfig`` (CSV paths/counts). We build both locally with whatever\n",
-    "overrides we want, serialize each into a JSON file the AML job can read as\n",
-    "an input, and ship those paths through the job command. Defaults come from\n",
-    "the dataclasses in ``pyrit.executor.promptgen.gcg.config``; goals and targets\n",
-    "flow into ``GCGGenerator.execute_async`` at runtime, not through the config.\n",
+    "``GCGDataConfig`` (resolved goals and targets). We load the bundled\n",
+    "``adv_bench`` dataset through the standard ``SeedDatasetProvider``; each seed\n",
+    "already preserves its canonical target completion in ``metadata[\"gcg_target\"]``.\n",
+    "We materialize those values inline via ``from_seed_dataset``, so the serialized\n",
+    "JSON the AML job reads already embeds the data and the compute container never\n",
+    "fetches it from the public internet. We build both configs locally, serialize\n",
+    "each into a JSON file the AML job mounts as an input, and ship those paths\n",
+    "through the job command. Defaults come from the dataclasses in\n",
+    "``pyrit.executor.promptgen.gcg.config``; goals and targets flow into\n",
+    "``GCGGenerator.execute_async`` at runtime, not through the config.\n",
+    "\n",
+    "AdvBench already provides a target completion for each objective. For another\n",
+    "ordinary ``SeedDataset`` without those values, use\n",
+    "``create_gcg_data_config_async`` with an adversarial target and a\n",
+    "``completion_scorer``. The scorer must return True only for a suitable\n",
+    "affirmative prefix, not a refusal, unrelated response, or complete answer.\n",
+    "Provided ``gcg_target`` strings are trusted and preserved without scoring;\n",
+    "missing, None, or blank values trigger generation. Other metadata types are\n",
+    "rejected rather than converted to strings.\n",
+    "\n",
+    "With your configured ``adversarial_target`` and ``scoring_target``:\n",
+    "\n",
+    "```python\n",
+    "from pyrit.executor.promptgen.gcg import create_gcg_data_config_async\n",
+    "from pyrit.score import (\n",
+    "    SelfAskTrueFalseScorer,\n",
+    "    TrueFalseQuestion,\n",
+    "    render_true_false_system_prompt,\n",
+    ")\n",
+    "\n",
+    "question = TrueFalseQuestion(\n",
+    "    true_description=\"A short response prefix that directly begins complying with the objective.\",\n",
+    "    false_description=\"A refusal, deflection, unrelated response, or complete answer.\",\n",
+    ")\n",
+    "prefix_scorer = SelfAskTrueFalseScorer(\n",
+    "    chat_target=scoring_target,\n",
+    "    system_prompt=render_true_false_system_prompt(question=question),\n",
+    "    question=question,\n",
+    ")\n",
+    "data_config = await create_gcg_data_config_async(\n",
+    "    dataset=objectives,\n",
+    "    adversarial_target=adversarial_target,\n",
+    "    completion_scorer=prefix_scorer,\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "Optional ``completion_prompt=SeedPrompt(...)`` replaces the default generation\n",
+    "prompt using Jinja variables ``objective`` and ``max_length``. Generation uses\n",
+    "the final assistant response, not intermediate tool outputs, and requires a\n",
+    "single error-free text piece. Empty, overlong, or scorer-rejected completions\n",
+    "raise an error without truncation or automatic retries. The default generated\n",
+    "completion limit is 256 characters; provided metadata is not length-limited.\n",
+    "All resolved strings are serialized locally, so neither the adversarial target\n",
+    "nor the scorer is needed by the AML job.\n",
+    "\n",
+    "**Migration from v1.1.0:** ``GCGDataConfig(train_data=..., test_data=...,\n",
+    "n_train_data=..., n_test_data=...)`` and the ``random_seed`` argument to\n",
+    "``load_goals_and_targets`` are no longer supported. Load public or private\n",
+    "datasets before submitting the job and supply each completion as\n",
+    "``metadata[\"gcg_target\"]`` (or generate it as above). Select and split seeds\n",
+    "locally instead of supplying CSV URLs and row counts. For example, after\n",
+    "loading ``objectives`` through the standard provider:\n",
+    "\n",
+    "```python\n",
+    "from random import Random\n",
+    "from pyrit.models import SeedDataset\n",
+    "\n",
+    "seeds = list(objectives.seeds)\n",
+    "Random(42).shuffle(seeds)\n",
+    "data_config = GCGDataConfig.from_seed_dataset(\n",
+    "    SeedDataset(seeds=seeds[:50]),\n",
+    "    test_dataset=SeedDataset(seeds=seeds[50:60]),\n",
+    ")\n",
+    "```\n",
+    "\n",
+    "This makes the submitted split explicit and reproducible. The compute-side\n",
+    "``load_goals_and_targets(data=data_config)`` only reads the inline lists.\n",
     "\n",
     "We also have to specify a GPU compute target. In our experience, a GPU instance with\n",
     "at least 24GB of vRAM is required (e.g., Standard_NC24ads_A100_v4).\n",
     "\n",
     "Depending on the compute instance you use, you may encounter \"out of memory\" errors.\n",
-    "In this case, we recommend training on a smaller model or lowering ``data.n_train_data``\n",
-    "or ``algorithm.batch_size``."
+    "In this case, we recommend training on a smaller model or lowering the number\n",
+    "of goals selected below or ``algorithm.batch_size``."
    ]
   },
   {
@@ -215,6 +293,7 @@
    "source": [
     "import tempfile\n",
     "\n",
+    "from pyrit.datasets import SeedDatasetProvider\n",
     "from pyrit.executor.promptgen.gcg import (\n",
     "    GCGAlgorithmConfig,\n",
     "    GCGConfig,\n",
@@ -228,11 +307,9 @@
     "    algorithm=GCGAlgorithmConfig(n_steps=5, batch_size=64, test_steps=1),\n",
     "    output=GCGOutputConfig(result_prefix=\"gcg_suffix\"),\n",
     ")\n",
-    "data_config = GCGDataConfig(\n",
-    "    train_data=(\"https://raw.githubusercontent.com/llm-attacks/llm-attacks/main/data/advbench/harmful_behaviors.csv\"),\n",
-    "    n_train_data=5,\n",
-    "    n_test_data=0,\n",
-    ")\n",
+    "objectives = (await SeedDatasetProvider.fetch_datasets_async(dataset_names=[\"adv_bench\"]))[0]\n",
+    "objectives.seeds = list(objectives.seeds[:5])\n",
+    "data_config = GCGDataConfig.from_seed_dataset(objectives)\n",
     "\n",
     "# Write the configs into a tempdir so AML can mount them as separate job inputs.\n",
     "config_dir = Path(tempfile.mkdtemp(prefix=\"gcg-aml-config-\"))\n",
@@ -361,9 +438,13 @@
     "runs never share a file). For our config, that resolves to\n",
     "`gcg_suffix_<timestamp>_<id>.json` under\n",
     "`<download_dir>/named-outputs/results/` once we download. The\n",
-    "`controls` array in that file contains one entry per training step, and\n",
-    "the last entry is the final adversarial suffix that, appended to the user\n",
-    "prompt, was optimized to elicit the target response."
+    "`controls` array contains logged checkpoints, not an iteration count.\n",
+    "With the default individual-goal strategy, each of the five goals logs an\n",
+    "initial checkpoint plus five optimization checkpoints (`test_steps=1`):\n",
+    "30 entries for 25 optimization iterations in total. Other strategies,\n",
+    "checkpoint intervals, or early stopping can change that relationship.\n",
+    "The last entry is the final logged suffix. A nonempty suffix and finite loss\n",
+    "confirm a usable result artifact, not that the suffix achieves the objective."
    ]
   },
   {
@@ -420,6 +501,7 @@
    ],
    "source": [
     "import json\n",
+    "import math\n",
     "import tempfile\n",
     "import time\n",
     "from pathlib import Path\n",
@@ -456,8 +538,14 @@
     "final_suffix = log[\"controls\"][-1] if log[\"controls\"] else None\n",
     "final_loss = log[\"losses\"][-1] if log[\"losses\"] else None\n",
     "\n",
+    "if not isinstance(final_suffix, str) or not final_suffix.strip():\n",
+    "    raise ValueError(\"GCG result does not contain a nonempty final suffix.\")\n",
+    "if isinstance(final_loss, bool) or not isinstance(final_loss, (int, float)) or not math.isfinite(final_loss):\n",
+    "    raise ValueError(\"GCG result does not contain a finite final loss.\")\n",
+    "\n",
     "print(f\"Result file: {result_file.name}\")\n",
-    "print(f\"Steps run: {len(log['controls'])}\")\n",
+    "print(f\"Configured optimization steps per goal: {config.algorithm.n_steps}\")\n",
+    "print(f\"Logged checkpoints: {len(log['controls'])}\")\n",
     "print(f\"Final loss: {final_loss}\")\n",
     "print(f\"Generated suffix: {final_suffix!r}\")"
    ]

doc/code/executor/gcg/1_gcg_azure_ml.py

@@ -29,9 +29,15 @@
 # %%
 import os
 
-from pyrit.setup.initialization import _load_environment_files
-
-_load_environment_files(env_files=None)
+from pyrit.setup import ConfigurationLoader, initialize_pyrit_async
+
+pyrit_config = ConfigurationLoader.load_with_overrides()
+await initialize_pyrit_async(
+    memory_db_type="InMemory",
+    load_defaults=False,
+    env_files=pyrit_config.resolve_env_files(),
+    env_akv_ref=pyrit_config.resolve_env_akv_ref(),
+)
 
 subscription_id = os.environ.get("AZURE_ML_SUBSCRIPTION_ID")
 resource_group = os.environ.get("AZURE_ML_RESOURCE_GROUP")
@@ -56,9 +62,9 @@
 
 # %%
 from azure.ai.ml import MLClient
-from azure.identity import AzureCliCredential
+from azure.identity import DefaultAzureCredential
 
-ml_client = MLClient(AzureCliCredential(), subscription_id, resource_group, workspace)
+ml_client = MLClient(DefaultAzureCredential(), subscription_id, resource_group, workspace)
 
 # %% [markdown]
 # ## Create AML Environment
@@ -99,22 +105,95 @@
 # Docker-installed package (Python's `-m` flag puts the cwd at the front of `sys.path`).
 #
 # The new public API takes a typed ``GCGConfig`` (strategy) and a separate
-# ``GCGDataConfig`` (CSV paths/counts). We build both locally with whatever
-# overrides we want, serialize each into a JSON file the AML job can read as
-# an input, and ship those paths through the job command. Defaults come from
-# the dataclasses in ``pyrit.executor.promptgen.gcg.config``; goals and targets
-# flow into ``GCGGenerator.execute_async`` at runtime, not through the config.
+# ``GCGDataConfig`` (resolved goals and targets). We load the bundled
+# ``adv_bench`` dataset through the standard ``SeedDatasetProvider``; each seed
+# already preserves its canonical target completion in ``metadata["gcg_target"]``.
+# We materialize those values inline via ``from_seed_dataset``, so the serialized
+# JSON the AML job reads already embeds the data and the compute container never
+# fetches it from the public internet. We build both configs locally, serialize
+# each into a JSON file the AML job mounts as an input, and ship those paths
+# through the job command. Defaults come from the dataclasses in
+# ``pyrit.executor.promptgen.gcg.config``; goals and targets flow into
+# ``GCGGenerator.execute_async`` at runtime, not through the config.
+#
+# AdvBench already provides a target completion for each objective. For another
+# ordinary ``SeedDataset`` without those values, use
+# ``create_gcg_data_config_async`` with an adversarial target and a
+# ``completion_scorer``. The scorer must return True only for a suitable
+# affirmative prefix, not a refusal, unrelated response, or complete answer.
+# Provided ``gcg_target`` strings are trusted and preserved without scoring;
+# missing, None, or blank values trigger generation. Other metadata types are
+# rejected rather than converted to strings.
+#
+# With your configured ``adversarial_target`` and ``scoring_target``:
+#
+# ```python
+# from pyrit.executor.promptgen.gcg import create_gcg_data_config_async
+# from pyrit.score import (
+#     SelfAskTrueFalseScorer,
+#     TrueFalseQuestion,
+#     render_true_false_system_prompt,
+# )
+#
+# question = TrueFalseQuestion(
+#     true_description="A short response prefix that directly begins complying with the objective.",
+#     false_description="A refusal, deflection, unrelated response, or complete answer.",
+# )
+# prefix_scorer = SelfAskTrueFalseScorer(
+#     chat_target=scoring_target,
+#     system_prompt=render_true_false_system_prompt(question=question),
+#     question=question,
+# )
+# data_config = await create_gcg_data_config_async(
+#     dataset=objectives,
+#     adversarial_target=adversarial_target,
+#     completion_scorer=prefix_scorer,
+# )
+# ```
+#
+# Optional ``completion_prompt=SeedPrompt(...)`` replaces the default generation
+# prompt using Jinja variables ``objective`` and ``max_length``. Generation uses
+# the final assistant response, not intermediate tool outputs, and requires a
+# single error-free text piece. Empty, overlong, or scorer-rejected completions
+# raise an error without truncation or automatic retries. The default generated
+# completion limit is 256 characters; provided metadata is not length-limited.
+# All resolved strings are serialized locally, so neither the adversarial target
+# nor the scorer is needed by the AML job.
+#
+# **Migration from v1.1.0:** ``GCGDataConfig(train_data=..., test_data=...,
+# n_train_data=..., n_test_data=...)`` and the ``random_seed`` argument to
+# ``load_goals_and_targets`` are no longer supported. Load public or private
+# datasets before submitting the job and supply each completion as
+# ``metadata["gcg_target"]`` (or generate it as above). Select and split seeds
+# locally instead of supplying CSV URLs and row counts. For example, after
+# loading ``objectives`` through the standard provider:
+#
+# ```python
+# from random import Random
+# from pyrit.models import SeedDataset
+#
+# seeds = list(objectives.seeds)
+# Random(42).shuffle(seeds)
+# data_config = GCGDataConfig.from_seed_dataset(
+#     SeedDataset(seeds=seeds[:50]),
+#     test_dataset=SeedDataset(seeds=seeds[50:60]),
+# )
+# ```
+#
+# This makes the submitted split explicit and reproducible. The compute-side
+# ``load_goals_and_targets(data=data_config)`` only reads the inline lists.
 #
 # We also have to specify a GPU compute target. In our experience, a GPU instance with
 # at least 24GB of vRAM is required (e.g., Standard_NC24ads_A100_v4).
 #
 # Depending on the compute instance you use, you may encounter "out of memory" errors.
-# In this case, we recommend training on a smaller model or lowering ``data.n_train_data``
-# or ``algorithm.batch_size``.
+# In this case, we recommend training on a smaller model or lowering the number
+# of goals selected below or ``algorithm.batch_size``.
 
 # %%
 import tempfile
 
+from pyrit.datasets import SeedDatasetProvider
 from pyrit.executor.promptgen.gcg import (
     GCGAlgorithmConfig,
     GCGConfig,
@@ -128,11 +207,9 @@
     algorithm=GCGAlgorithmConfig(n_steps=5, batch_size=64, test_steps=1),
     output=GCGOutputConfig(result_prefix="gcg_suffix"),
 )
-data_config = GCGDataConfig(
-    train_data=("https://raw.githubusercontent.com/llm-attacks/llm-attacks/main/data/advbench/harmful_behaviors.csv"),
-    n_train_data=5,
-    n_test_data=0,
-)
+objectives = (await SeedDatasetProvider.fetch_datasets_async(dataset_names=["adv_bench"]))[0]
+objectives.seeds = list(objectives.seeds[:5])
+data_config = GCGDataConfig.from_seed_dataset(objectives)
 
 # Write the configs into a tempdir so AML can mount them as separate job inputs.
 config_dir = Path(tempfile.mkdtemp(prefix="gcg-aml-config-"))
@@ -183,12 +260,17 @@
 # runs never share a file). For our config, that resolves to
 # `gcg_suffix_<timestamp>_<id>.json` under
 # `<download_dir>/named-outputs/results/` once we download. The
-# `controls` array in that file contains one entry per training step, and
-# the last entry is the final adversarial suffix that, appended to the user
-# prompt, was optimized to elicit the target response.
+# `controls` array contains logged checkpoints, not an iteration count.
+# With the default individual-goal strategy, each of the five goals logs an
+# initial checkpoint plus five optimization checkpoints (`test_steps=1`):
+# 30 entries for 25 optimization iterations in total. Other strategies,
+# checkpoint intervals, or early stopping can change that relationship.
+# The last entry is the final logged suffix. A nonempty suffix and finite loss
+# confirm a usable result artifact, not that the suffix achieves the objective.
 
 # %%
 import json
+import math
 import tempfile
 import time
 from pathlib import Path
@@ -225,7 +307,13 @@
 final_suffix = log["controls"][-1] if log["controls"] else None
 final_loss = log["losses"][-1] if log["losses"] else None
 
+if not isinstance(final_suffix, str) or not final_suffix.strip():
+    raise ValueError("GCG result does not contain a nonempty final suffix.")
+if isinstance(final_loss, bool) or not isinstance(final_loss, (int, float)) or not math.isfinite(final_loss):
+    raise ValueError("GCG result does not contain a finite final loss.")
+
 print(f"Result file: {result_file.name}")
-print(f"Steps run: {len(log['controls'])}")
+print(f"Configured optimization steps per goal: {config.algorithm.n_steps}")
+print(f"Logged checkpoints: {len(log['controls'])}")
 print(f"Final loss: {final_loss}")
 print(f"Generated suffix: {final_suffix!r}")