feat(AIC-3534): add a TypeSafe Jev judge handler

.github/workflows/release-please.yml

@@ -32,6 +32,7 @@ on:
           - packages/openai-messages
           - packages/langchain-agents
           - packages/langchain-messages
+          - packages/typesafe
       dry_run:
         description: 'Is this a dry run. If so no package will be published.'
         type: boolean
@@ -62,6 +63,8 @@ jobs:
       package-langchain-agents-tag-name: ${{ steps.release.outputs['packages/langchain-agents--tag_name'] }}
       package-langchain-messages-released: ${{ steps.release.outputs['packages/langchain-messages--release_created'] }}
       package-langchain-messages-tag-name: ${{ steps.release.outputs['packages/langchain-messages--tag_name'] }}
+      package-typesafe-released: ${{ steps.release.outputs['packages/typesafe--release_created'] }}
+      package-typesafe-tag-name: ${{ steps.release.outputs['packages/typesafe--tag_name'] }}
     steps:
       - uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0
         id: release
@@ -316,6 +319,37 @@ jobs:
           password: ${{ env.PYPI_AUTH_TOKEN }}
           packages-dir: packages/langchain-messages/dist/
 
+  release-typesafe:
+    runs-on: ubuntu-latest
+    needs: ['release-please']
+    permissions:
+      contents: read
+      id-token: write
+      attestations: write
+    if: ${{ needs.release-please.outputs.package-typesafe-released == 'true' }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/ci
+        with:
+          workspace_path: packages/typesafe
+      - uses: ./.github/actions/build
+        with:
+          workspace_path: packages/typesafe
+      - name: Attest build provenance
+        uses: actions/attest@v4
+        with:
+          subject-path: 'packages/typesafe/dist/*'
+      - uses: launchdarkly/gh-actions/actions/[email protected]
+        name: 'Get PyPI token'
+        with:
+          aws_assume_role: ${{ vars.AWS_ROLE_ARN }}
+          ssm_parameter_pairs: '/production/common/releasing/pypi/token = PYPI_AUTH_TOKEN'
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+        with:
+          password: ${{ env.PYPI_AUTH_TOKEN }}
+          packages-dir: packages/typesafe/dist/
+
   # --- Manual publish ---
 
   manual-publish:

.release-please-manifest.json

@@ -6,5 +6,6 @@
   "packages/openai-agents": "0.2.4",
   "packages/openai-messages": "0.2.3",
   "packages/langchain-agents": "0.2.4",
-  "packages/langchain-messages": "0.2.3"
+  "packages/langchain-messages": "0.2.3",
+  "packages/typesafe": "0.1.0"
 }

.sdk_metadata.json

@@ -90,6 +90,17 @@
         "aiCompletionConfig": { "introduced": "0.1" }
       }
     },
+    "python-ai-typesafe": {
+      "name": "TypeSafe Jev AI SDK for Python",
+      "type": "ai-provider",
+      "path": "packages/typesafe",
+      "languages": ["Python"],
+      "aiSdkNames": ["launchdarkly-ai-typesafe"],
+      "releases": { "tag-prefix": "launchdarkly-ai-typesafe-" },
+      "features": {
+        "aiCompletionConfig": { "introduced": "0.1" }
+      }
+    },
     "python-ai-langchain-agents": {
       "name": "LangChain Agents AI SDK for Python",
       "type": "ai-provider",

README.md

@@ -85,6 +85,7 @@ Tier 0 — Core Client           (launchdarkly-ai-server)
 | [`launchdarkly-ai-claude-agents`](packages/claude-agents/README.md) | Anthropic | `agent` | Claude Agent SDK — agentic loop with MCP tool support |
 | [`launchdarkly-ai-langchain-messages`](packages/langchain-messages/README.md) | `*` (any) | `messages` | Any `BaseChatModel` via LangChain `bind_tools` loop |
 | [`launchdarkly-ai-langchain-agents`](packages/langchain-agents/README.md) | `*` (any) | `agent` | LangGraph `StateGraph` — managed ReAct loop |
+| [`launchdarkly-ai-typesafe`](packages/typesafe/README.md) | TypeSafe | `messages` | TypeSafe Jev judge handler |
 
 ## Quick Start
 

examples/register.py

@@ -22,6 +22,7 @@
 from launchdarkly_ai_openai_agents import create_openai_agent_handler
 from launchdarkly_ai_openai_messages import create_openai_messages_handler
 from launchdarkly_ai_server import global_registry
+from launchdarkly_ai_typesafe import create_typesafe_handler
 
 global_registry.register(
     handlers=[
@@ -31,6 +32,7 @@
         create_claude_messages_handler(),
         create_langchain_messages_handler(),
         create_langchain_agents_handler(),
+        create_typesafe_handler(),
     ],
     tools={
         # LD documentation agent tools

packages/client/src/launchdarkly_ai_server/conversation.py

@@ -36,6 +36,7 @@ class _JudgeEvalCapture:
     released: bool = False
     span: Any = None
     pending_end: Callable[[], None] | None = None
+    mirrored: bool = False
 
 
 # Every tracer this SDK creates is named "@launchdarkly/ai-<package>". The processor is registered
@@ -82,7 +83,12 @@ def _conversation_id_from(ctx: otel_context.Context | None) -> str | None:
 
 
 def _record_evaluation(
-    span: Any, name: str, score: float, explanation: str | None = None
+    span: Any,
+    name: str,
+    score: float,
+    explanation: str | None = None,
+    *,
+    mirror_attributes: bool = True,
 ) -> None:
     """Write the judge score as a ``gen_ai.evaluation.result`` event plus mirrored attributes.
 
@@ -99,6 +105,10 @@ def _record_evaluation(
     if explanation:
         attrs["gen_ai.evaluation.explanation"] = explanation
     span.add_event("gen_ai.evaluation.result", attrs)
+    # A classifier records one event per label. Span attributes hold a single name and
+    # score, so only the first evaluation is mirrored. Later events keep their own attributes.
+    if not mirror_attributes:
+        return
     for key, value in attrs.items():
         span.set_attribute(key, value)
 
@@ -237,9 +247,18 @@ async def with_judge_evaluation(name: str) -> AsyncIterator[RecordEvaluation]:
     """
     capture = _JudgeEvalCapture(name=name)
 
-    def record(score: float, explanation: str | None = None) -> None:
+    def record(
+        score: float, explanation: str | None = None, name: str | None = None
+    ) -> None:
         if capture.span is not None:
-            _record_evaluation(capture.span, capture.name, score, explanation)
+            _record_evaluation(
+                capture.span,
+                name or capture.name,
+                score,
+                explanation,
+                mirror_attributes=not capture.mirrored,
+            )
+            capture.mirrored = True
 
     token = otel_context.attach(otel_context.set_value(_EVAL_KEY, capture))
     try:

packages/client/src/launchdarkly_ai_server/evaluations/runner.py

@@ -17,6 +17,7 @@
     build_message_history,
     numeric_score,
     parse_judge_response,
+    typesafe_judge_entries,
 )
 from ..lifecycle import extract_variation
 from ..trajectory import (
@@ -968,10 +969,17 @@ async def _run_ld_judge_for_result(
                 "invalid_judge_output",
                 "judge handler result must be a mapping",
             )
+        judge_output = result.get("output", result.get("response"))
         try:
-            raw_score, reason = parse_judge_response(
-                result.get("output", result.get("response"))
-            )
+            entries = typesafe_judge_entries(judge_output)
+            if entries is not None:
+                # A criterion event is one score. The remaining labels are
+                # available on the online judge_results path. The reason is the
+                # question key: ingest drops a COMPLETE judge event with an
+                # empty reason, and a Jev label has no prose verdict.
+                raw_score, reason = entries[0]["score"], entries[0]["key"]
+            else:
+                raw_score, reason = parse_judge_response(judge_output)
         except ValueError as error:
             return self._criterion_error_result(
                 base, started_clock, "invalid_judge_output", str(error)

packages/client/src/launchdarkly_ai_server/judge_scoring.py

@@ -65,6 +65,55 @@ def numeric_score(score: Any) -> float | None:
     return value if isfinite(value) else None
 
 
+def typesafe_judge_entries(raw: Any) -> list[dict[str, Any]] | None:
+    """Parse a TypeSafe handler payload into question key, score, event key, and reason.
+
+    Returns ``None`` when ``raw`` is not a TypeSafe multi-result payload, so a
+    classic ``{"score", "reasoning"}`` judge stays on :func:`parse_judge_response`.
+    Raises ``ValueError`` when the payload claims to be TypeSafe but is malformed.
+    """
+    parsed: Any
+    if isinstance(raw, str):
+        parsed = parse_json_with_possible_fences(raw)
+    elif isinstance(raw, Mapping):
+        parsed = raw
+    else:
+        parsed = None
+    if not isinstance(parsed, Mapping) or parsed.get("kind") != "typesafe":
+        return None
+    results = parsed.get("results")
+    if not isinstance(results, list) or not results:
+        raise ValueError("TypeSafe judge output is missing results")
+    entries: list[dict[str, Any]] = []
+    for item in results:
+        if (
+            not isinstance(item, Mapping)
+            or not isinstance(item.get("key"), str)
+            or not item["key"]
+        ):
+            raise ValueError("TypeSafe judge result is missing a key")
+        score = numeric_score(item.get("score"))
+        if score is None:
+            raise ValueError(
+                f"TypeSafe judge result {item['key']!r} is missing a finite score"
+            )
+        event_key = item.get("eventKey")
+        if not isinstance(event_key, str) or not event_key:
+            raise ValueError(
+                f"TypeSafe judge result {item['key']!r} is missing an eventKey"
+            )
+        reason = item.get("reason")
+        entries.append(
+            {
+                "key": item["key"],
+                "score": score,
+                "eventKey": event_key,
+                "reason": reason if isinstance(reason, str) else "",
+            }
+        )
+    return entries
+
+
 def parse_judge_response(raw: Any) -> tuple[Any, str]:
     """Parse a judge model response into ``(score, reasoning)``.