FEAT Add speciesist bias remote dataset loader

doc/bibliography.md

@@ -5,5 +5,5 @@ All academic papers, research blogs, and technical reports referenced throughout
 :::{dropdown} Citation Keys
 :class: hidden-citations
 
-[@aakanksha2024multilingual; @abughallous2026semguard; @adversaai2023universal; @agarwal2026systematization; @ahn2025puzzled; @andriushchenko2024tense; @anthropic2024manyshot; @aqrawi2024singleturncrescendo; @assert2026; @atr2026; @banerjee2025safeinfer; @bethany2024mathprompt; @bhardwaj2023harmfulqa; @bhardwaj2024homer; @bhatt2023cyberseceval; @boucher2023trojan; @brahman2024coconot; @bryan2025agentictaxonomy; @bullwinkel2025airtlessons; @bullwinkel2025repeng; @bullwinkel2026trigger; @chao2023pair; @chao2024jailbreakbench; @choi2026xlsafetybench; @cui2024orbench; @darkbench2025; @derczynski2024garak; @ding2023wolf; @dong2025sata; @embracethered2024unicode; @embracethered2025sneakybits; @evtimov2025wasp; @gehman2020realtoxicityprompts; @ghosh2025aegis; @ghosh2025ailuminate; @gong2025figstep; @gupta2024walledeval; @haider2024phi3safety; @han2024medsafetybench; @han2024wildguard; @hiddenlayer2025policypuppetry; @hines2024spotlighting; @huang2024bijectionlearning; @hughes2024bestofn; @inie2025summon; @ji2023beavertails; @ji2024pkusaferlhf; @jiang2025sosbench; @jones2025computeruse; @kim2023propile; @kingma2014adam; @knight2025fortress; @li2024drattack; @li2024mossbench; @li2024saladbench; @li2024wmdp; @lin2023toxicchat; @liu2024flipattack; @liu2024mmsafetybench; @lopez2024pyrit; @luo2024jailbreakv; @lutz2026pyrit; @lv2024codechameleon; @mazeika2023tdc; @mazeika2024harmbench; @mckee2024transparency; @mehrotra2023tap; @microsoft2024skeletonkey; @odin2024; @palaskar2025vlsu; @pavlova2024goat; @perez2022modelwritten; @pfohl2024equitymedqa; @promptfoo2025ccp; @ren2024codeattack; @ren2025maskbenchmarkdisentanglinghonesty; @robustintelligence2024bypass; @roccia2024promptintel; @rottger2023xstest; @rottger2025msts; @russinovich2024crescendo; @russinovich2025cca; @russinovich2025price; @scheuerman2025transphobia; @shaikh2022second; @shayegani2025computeruse; @shen2023donotanything; @sheshadri2024lat; @souly2024strongreject; @stok2023ansi; @tan2026comicjailbreak; @tang2025multilingual; @tedeschi2024alert; @vantaylor2024socialbias; @vidgen2023simplesafetytests; @wang2023decodingtrust; @wang2023donotanswer; @wang2025siuo; @wang2026visualleakbench; @wei2023jailbroken; @xie2024sorrybench; @yu2023gptfuzzer; @yuan2023cipherchat; @zeng2024persuasion; @zeng2024shieldgemma; @zhang2024cbtbench; @ziems2022mic; @zong2024vlguard; @zou2023gcg]
+[@aakanksha2024multilingual; @abughallous2026semguard; @adversaai2023universal; @agarwal2026systematization; @ahn2025puzzled; @andriushchenko2024tense; @anthropic2024manyshot; @aqrawi2024singleturncrescendo; @assert2026; @atr2026; @banerjee2025safeinfer; @bethany2024mathprompt; @bhardwaj2023harmfulqa; @bhardwaj2024homer; @bhatt2023cyberseceval; @boucher2023trojan; @brahman2024coconot; @bryan2025agentictaxonomy; @bullwinkel2025airtlessons; @bullwinkel2025repeng; @bullwinkel2026trigger; @chao2023pair; @chao2024jailbreakbench; @choi2026xlsafetybench; @cui2024orbench; @darkbench2025; @derczynski2024garak; @ding2023wolf; @dong2025sata; @embracethered2024unicode; @embracethered2025sneakybits; @evtimov2025wasp; @gehman2020realtoxicityprompts; @ghosh2025aegis; @ghosh2025ailuminate; @gong2025figstep; @gupta2024walledeval; @haider2024phi3safety; @han2024medsafetybench; @han2024wildguard; @hiddenlayer2025policypuppetry; @hines2024spotlighting; @huang2024bijectionlearning; @hughes2024bestofn; @inie2025summon; @ji2023beavertails; @ji2024pkusaferlhf; @jiang2025sosbench; @jones2025computeruse; @kim2023propile; @kingma2014adam; @knight2025fortress; @li2024drattack; @li2024mossbench; @li2024saladbench; @li2024wmdp; @lin2023toxicchat; @liu2024flipattack; @liu2024mmsafetybench; @lopez2024pyrit; @luo2024jailbreakv; @lutz2026pyrit; @lv2024codechameleon; @mazeika2023tdc; @mazeika2024harmbench; @mckee2024transparency; @mehrotra2023tap; @microsoft2024skeletonkey; @odin2024; @palaskar2025vlsu; @pavlova2024goat; @perez2022modelwritten; @pfohl2024equitymedqa; @promptfoo2025ccp; @ren2024codeattack; @ren2025maskbenchmarkdisentanglinghonesty; @robustintelligence2024bypass; @roccia2024promptintel; @rottger2023xstest; @rottger2025msts; @russinovich2024crescendo; @russinovich2025cca; @russinovich2025price; @scheuerman2025transphobia; @shaikh2022second; @shayegani2025computeruse; @shen2023donotanything; @sheshadri2024lat; @souly2024strongreject; @stok2023ansi; @tan2026comicjailbreak; @tang2025multilingual; @tedeschi2024alert; @tuckerdavis2026speciesistbias; @vantaylor2024socialbias; @vidgen2023simplesafetytests; @wang2023decodingtrust; @wang2023donotanswer; @wang2025siuo; @wang2026visualleakbench; @wei2023jailbroken; @xie2024sorrybench; @yu2023gptfuzzer; @yuan2023cipherchat; @zeng2024persuasion; @zeng2024shieldgemma; @zhang2024cbtbench; @ziems2022mic; @zong2024vlguard; @zou2023gcg]
 :::

doc/code/datasets/1_loading_datasets.ipynb

@@ -52,6 +52,7 @@
     "SIUO [@wang2025siuo],\n",
     "SORRY-Bench [@xie2024sorrybench],\n",
     "SOSBench [@jiang2025sosbench],\n",
+    "Speciesist Bias Prompts [@tuckerdavis2026speciesistbias],\n",
     "StrongREJECT [@souly2024strongreject],\n",
     "TDC23 [@mazeika2023tdc],\n",
     "ToxicChat [@lin2023toxicchat],\n",
@@ -208,6 +209,7 @@
        " 'siuo',\n",
        " 'sorry_bench',\n",
        " 'sosbench',\n",
+       " 'speciesist_bias',\n",
        " 'strong_reject',\n",
        " 'tdc23_redteaming',\n",
        " 'toxic_chat',\n",

doc/code/datasets/1_loading_datasets.py

@@ -56,6 +56,7 @@
 # SIUO [@wang2025siuo],
 # SORRY-Bench [@xie2024sorrybench],
 # SOSBench [@jiang2025sosbench],
+# Speciesist Bias Prompts [@tuckerdavis2026speciesistbias],
 # StrongREJECT [@souly2024strongreject],
 # TDC23 [@mazeika2023tdc],
 # ToxicChat [@lin2023toxicchat],

doc/references.bib

@@ -139,6 +139,14 @@ @article{inie2025summon
   url       = {https://doi.org/10.1371/journal.pone.0314658},
 }
 
+@misc{tuckerdavis2026speciesistbias,
+  title     = {Speciesist Bias Prompts},
+  author    = {Sam Tucker-Davis},
+  year      = {2026},
+  url       = {https://huggingface.co/datasets/stuckdavis/speciesist-bias-prompts},
+  note      = {Hugging Face dataset, version 0.2.0},
+}
+
 @misc{vantaylor2024socialbias,
   title     = {A Red-Teaming Repository of Existing Social Bias Prompts},
   author    = {Simone Van Taylor},

pyrit/datasets/seed_datasets/remote/__init__.py

@@ -137,6 +137,7 @@
     from pyrit.datasets.seed_datasets.remote.siuo_dataset import SIUOCategory, _SIUODataset
     from pyrit.datasets.seed_datasets.remote.sorry_bench_dataset import _SorryBenchDataset
     from pyrit.datasets.seed_datasets.remote.sosbench_dataset import _SOSBenchDataset
+    from pyrit.datasets.seed_datasets.remote.speciesist_bias_dataset import _SpeciesistBiasDataset
     from pyrit.datasets.seed_datasets.remote.strong_reject_dataset import _StrongRejectDataset
     from pyrit.datasets.seed_datasets.remote.tdc23_redteaming_dataset import _TDC23RedteamingDataset
     from pyrit.datasets.seed_datasets.remote.toxic_chat_dataset import _ToxicChatDataset
@@ -278,6 +279,7 @@
     "_SIUODataset": "pyrit.datasets.seed_datasets.remote.siuo_dataset",
     "_SOSBenchDataset": "pyrit.datasets.seed_datasets.remote.sosbench_dataset",
     "_SorryBenchDataset": "pyrit.datasets.seed_datasets.remote.sorry_bench_dataset",
+    "_SpeciesistBiasDataset": "pyrit.datasets.seed_datasets.remote.speciesist_bias_dataset",
     "_StrongRejectDataset": "pyrit.datasets.seed_datasets.remote.strong_reject_dataset",
     "_TDC23RedteamingDataset": "pyrit.datasets.seed_datasets.remote.tdc23_redteaming_dataset",
     "_ToxicChatDataset": "pyrit.datasets.seed_datasets.remote.toxic_chat_dataset",

pyrit/datasets/seed_datasets/remote/speciesist_bias_dataset.py

@@ -0,0 +1,153 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import logging
+from typing import Any
+
+from typing_extensions import override
+
+from pyrit.datasets.seed_datasets.remote.remote_dataset_loader import (
+    _RemoteDatasetLoader,
+)
+from pyrit.models import Modality, SeedDataset, SeedPrompt, SeedUnion
+from pyrit.models.harm_category import HarmCategory
+
+logger = logging.getLogger(__name__)
+
+HF_DATASET_NAME: str = "stuckdavis/speciesist-bias-prompts"
+# Commit SHA of the v0.2.0 tag on the Hugging Face dataset repository.
+HF_REVISION: str = "6e0cfa08f4b88de3aef80ad9f0015072901f60dc"
+HF_SOURCE_URL: str = f"https://huggingface.co/datasets/{HF_DATASET_NAME}"
+
+_AUTHORS: list[str] = ["Sam Tucker-Davis"]
+
+_REQUIRED_KEYS: tuple[str, ...] = ("id", "prompt", "prompt_type", "species", "harm_category")
+
+
+class _SpeciesistBiasDataset(_RemoteDatasetLoader):
+    """
+    Loader for the Speciesist Bias Prompts dataset.
+
+    Prompts for checking whether a model treats the interests of different animal
+    species inconsistently or dismissively. Most rows are comparative prompts that
+    name two species in one prompt, so a single response can show uneven treatment.
+    A smaller set of split pairs fills one template with one species at a time; the
+    two halves share a ``pair_id`` in metadata and are emitted as independent seeds.
+    Each row also carries a ``framing`` (``implicit`` everyday task or ``explicit``
+    question) and a ``bias_signal``: one sentence on what a biased answer would look
+    like, which a scorer or an attack objective can use.
+
+    The dataset is for checking inconsistency and dismissiveness, not diet advice.
+
+    PyRIT has no species-bias harm category, so every seed uses ``HarmCategory.OTHER``
+    and keeps the source label ``speciesist_bias`` in ``metadata["harm_category"]``.
+    This loader does not implement a scorer.
+
+    References:
+        - Dataset card: [@tuckerdavis2026speciesistbias]
+        - HuggingFace: https://huggingface.co/datasets/stuckdavis/speciesist-bias-prompts
+
+    License: CC BY 4.0.
+    """
+
+    HF_DATASET_NAME: str = HF_DATASET_NAME
+    HF_REVISION: str = HF_REVISION
+
+    HARM_CATEGORY_ALIAS_OVERRIDES: dict[str, list[HarmCategory]] = {
+        "speciesist_bias": [HarmCategory.OTHER],
+    }
+
+    # Metadata
+    modalities: tuple[Modality, ...] = (Modality.TEXT,)
+    size: str = "medium"  # 77 comparative prompts and 16 split pairs (109 rows)
+    tags: frozenset[str] = frozenset({"safety", "bias", "ethics"})
+    harm_categories: list[str] = ["speciesist_bias"]
+
+    _DESCRIPTION: str = (
+        "Prompts naming one or two animal species, for checking whether a model treats similar "
+        "interests of different species inconsistently or dismissively. For checking inconsistency "
+        "and dismissiveness, not diet advice."
+    )
+
+    @property
+    @override
+    def dataset_name(self) -> str:
+        """The dataset name."""
+        return "speciesist_bias"
+
+    @override
+    async def _fetch_dataset_async(self, *, cache: bool = True) -> SeedDataset:
+        """
+        Fetch the Speciesist Bias Prompts dataset and return it as a SeedDataset.
+
+        Args:
+            cache: Whether to cache the fetched dataset. Defaults to True.
+
+        Returns:
+            SeedDataset: One SeedPrompt per row.
+
+        Raises:
+            ValueError: If a row is missing a required field or the dataset is empty.
+        """
+        logger.info(f"Loading Speciesist Bias Prompts dataset from {self.HF_DATASET_NAME}@{self.HF_REVISION}")
+
+        data = await self._fetch_from_huggingface_async(
+            dataset_name=self.HF_DATASET_NAME,
+            split="train",
+            cache=cache,
+            revision=self.HF_REVISION,
+        )
+
+        seed_prompts: list[SeedUnion] = [self._build_seed_prompt(row=dict(row)) for row in data]
+
+        if not seed_prompts:
+            raise ValueError("SeedDataset cannot be empty. The Speciesist Bias Prompts dataset returned no rows.")
+
+        logger.info(f"Successfully loaded {len(seed_prompts)} prompts from Speciesist Bias Prompts dataset")
+
+        return SeedDataset(seeds=seed_prompts, dataset_name=self.dataset_name)
+
+    def _build_seed_prompt(self, *, row: dict[str, Any]) -> SeedPrompt:
+        """
+        Build a SeedPrompt from one dataset row.
+
+        Args:
+            row (dict[str, Any]): A raw dataset row.
+
+        Returns:
+            SeedPrompt: The constructed seed prompt.
+
+        Raises:
+            ValueError: If the row is missing a required field.
+        """
+        missing = [key for key in _REQUIRED_KEYS if row.get(key) in (None, "", [])]
+        if missing:
+            raise ValueError(f"Speciesist Bias Prompts row {row.get('id')!r} is missing required field(s): {missing}")
+
+        metadata: dict[str, str | list[str]] = {
+            "item_id": str(row["id"]),
+            "harm_category": str(row["harm_category"]),
+            "prompt_type": str(row["prompt_type"]),
+            "species": [str(s) for s in row["species"]],
+            "contexts": [str(c) for c in row.get("contexts") or []],
+            "task": str(row.get("task") or ""),
+            "framing": str(row.get("framing") or ""),
+            "bias_signal": str(row.get("bias_signal") or ""),
+            "source": str(row.get("source") or ""),
+            "version": str(row.get("version") or ""),
+        }
+        if row.get("pair_id"):
+            metadata["pair_id"] = str(row["pair_id"])
+
+        return SeedPrompt(
+            value=str(row["prompt"]),
+            data_type="text",
+            dataset_name=self.dataset_name,
+            harm_categories=self._standardize_harm_categories(
+                str(row["harm_category"]), alias_overrides=self.HARM_CATEGORY_ALIAS_OVERRIDES
+            ),
+            description=self._DESCRIPTION,
+            source=HF_SOURCE_URL,
+            authors=_AUTHORS,
+            metadata=metadata,
+        )