diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index bbf1273c..cd332d88 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -32,6 +32,7 @@ on: - packages/openai-messages - packages/langchain-agents - packages/langchain-messages + - packages/typesafe dry_run: description: 'Is this a dry run. If so no package will be published.' type: boolean @@ -62,6 +63,8 @@ jobs: package-langchain-agents-tag-name: ${{ steps.release.outputs['packages/langchain-agents--tag_name'] }} package-langchain-messages-released: ${{ steps.release.outputs['packages/langchain-messages--release_created'] }} package-langchain-messages-tag-name: ${{ steps.release.outputs['packages/langchain-messages--tag_name'] }} + package-typesafe-released: ${{ steps.release.outputs['packages/typesafe--release_created'] }} + package-typesafe-tag-name: ${{ steps.release.outputs['packages/typesafe--tag_name'] }} steps: - uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0 id: release @@ -316,6 +319,37 @@ jobs: password: ${{ env.PYPI_AUTH_TOKEN }} packages-dir: packages/langchain-messages/dist/ + release-typesafe: + runs-on: ubuntu-latest + needs: ['release-please'] + permissions: + contents: read + id-token: write + attestations: write + if: ${{ needs.release-please.outputs.package-typesafe-released == 'true' }} + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/ci + with: + workspace_path: packages/typesafe + - uses: ./.github/actions/build + with: + workspace_path: packages/typesafe + - name: Attest build provenance + uses: actions/attest@v4 + with: + subject-path: 'packages/typesafe/dist/*' + - uses: launchdarkly/gh-actions/actions/release-secrets@release-secrets-v1.2.0 + name: 'Get PyPI token' + with: + aws_assume_role: ${{ vars.AWS_ROLE_ARN }} + ssm_parameter_pairs: '/production/common/releasing/pypi/token = PYPI_AUTH_TOKEN' + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + password: ${{ env.PYPI_AUTH_TOKEN }} + packages-dir: packages/typesafe/dist/ + # --- Manual publish --- manual-publish: diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 1fd39bc5..30bf630b 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -6,5 +6,6 @@ "packages/openai-agents": "0.2.4", "packages/openai-messages": "0.2.3", "packages/langchain-agents": "0.2.4", - "packages/langchain-messages": "0.2.3" + "packages/langchain-messages": "0.2.3", + "packages/typesafe": "0.1.0" } diff --git a/.sdk_metadata.json b/.sdk_metadata.json index 5d7708f3..13993ba9 100644 --- a/.sdk_metadata.json +++ b/.sdk_metadata.json @@ -90,6 +90,17 @@ "aiCompletionConfig": { "introduced": "0.1" } } }, + "python-ai-typesafe": { + "name": "TypeSafe Jev AI SDK for Python", + "type": "ai-provider", + "path": "packages/typesafe", + "languages": ["Python"], + "aiSdkNames": ["launchdarkly-ai-typesafe"], + "releases": { "tag-prefix": "launchdarkly-ai-typesafe-" }, + "features": { + "aiCompletionConfig": { "introduced": "0.1" } + } + }, "python-ai-langchain-agents": { "name": "LangChain Agents AI SDK for Python", "type": "ai-provider", diff --git a/README.md b/README.md index 0e693665..409e032e 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,7 @@ Tier 0 — Core Client (launchdarkly-ai-server) | [`launchdarkly-ai-claude-agents`](packages/claude-agents/README.md) | Anthropic | `agent` | Claude Agent SDK — agentic loop with MCP tool support | | [`launchdarkly-ai-langchain-messages`](packages/langchain-messages/README.md) | `*` (any) | `messages` | Any `BaseChatModel` via LangChain `bind_tools` loop | | [`launchdarkly-ai-langchain-agents`](packages/langchain-agents/README.md) | `*` (any) | `agent` | LangGraph `StateGraph` — managed ReAct loop | +| [`launchdarkly-ai-typesafe`](packages/typesafe/README.md) | TypeSafe | `messages` | TypeSafe Jev judge handler | ## Quick Start diff --git a/examples/register.py b/examples/register.py index c5ff6a11..8f58d6a1 100644 --- a/examples/register.py +++ b/examples/register.py @@ -22,6 +22,7 @@ from launchdarkly_ai_openai_agents import create_openai_agent_handler from launchdarkly_ai_openai_messages import create_openai_messages_handler from launchdarkly_ai_server import global_registry +from launchdarkly_ai_typesafe import create_typesafe_handler global_registry.register( handlers=[ @@ -31,6 +32,7 @@ create_claude_messages_handler(), create_langchain_messages_handler(), create_langchain_agents_handler(), + create_typesafe_handler(), ], tools={ # LD documentation agent tools diff --git a/packages/client/src/launchdarkly_ai_server/conversation.py b/packages/client/src/launchdarkly_ai_server/conversation.py index d3af0e9e..ba19a9bb 100644 --- a/packages/client/src/launchdarkly_ai_server/conversation.py +++ b/packages/client/src/launchdarkly_ai_server/conversation.py @@ -36,6 +36,7 @@ class _JudgeEvalCapture: released: bool = False span: Any = None pending_end: Callable[[], None] | None = None + mirrored: bool = False # Every tracer this SDK creates is named "@launchdarkly/ai-". The processor is registered @@ -82,7 +83,12 @@ def _conversation_id_from(ctx: otel_context.Context | None) -> str | None: def _record_evaluation( - span: Any, name: str, score: float, explanation: str | None = None + span: Any, + name: str, + score: float, + explanation: str | None = None, + *, + mirror_attributes: bool = True, ) -> None: """Write the judge score as a ``gen_ai.evaluation.result`` event plus mirrored attributes. @@ -99,6 +105,10 @@ def _record_evaluation( if explanation: attrs["gen_ai.evaluation.explanation"] = explanation span.add_event("gen_ai.evaluation.result", attrs) + # A classifier records one event per label. Span attributes hold a single name and + # score, so only the first evaluation is mirrored. Later events keep their own attributes. + if not mirror_attributes: + return for key, value in attrs.items(): span.set_attribute(key, value) @@ -237,9 +247,18 @@ async def with_judge_evaluation(name: str) -> AsyncIterator[RecordEvaluation]: """ capture = _JudgeEvalCapture(name=name) - def record(score: float, explanation: str | None = None) -> None: + def record( + score: float, explanation: str | None = None, name: str | None = None + ) -> None: if capture.span is not None: - _record_evaluation(capture.span, capture.name, score, explanation) + _record_evaluation( + capture.span, + name or capture.name, + score, + explanation, + mirror_attributes=not capture.mirrored, + ) + capture.mirrored = True token = otel_context.attach(otel_context.set_value(_EVAL_KEY, capture)) try: diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index b12bab91..084a94d1 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -17,6 +17,7 @@ build_message_history, numeric_score, parse_judge_response, + typesafe_judge_entries, ) from ..lifecycle import extract_variation from ..trajectory import ( @@ -968,10 +969,17 @@ async def _run_ld_judge_for_result( "invalid_judge_output", "judge handler result must be a mapping", ) + judge_output = result.get("output", result.get("response")) try: - raw_score, reason = parse_judge_response( - result.get("output", result.get("response")) - ) + entries = typesafe_judge_entries(judge_output) + if entries is not None: + # A criterion event is one score. The remaining labels are + # available on the online judge_results path. The reason is the + # question key: ingest drops a COMPLETE judge event with an + # empty reason, and a Jev label has no prose verdict. + raw_score, reason = entries[0]["score"], entries[0]["key"] + else: + raw_score, reason = parse_judge_response(judge_output) except ValueError as error: return self._criterion_error_result( base, started_clock, "invalid_judge_output", str(error) diff --git a/packages/client/src/launchdarkly_ai_server/judge_scoring.py b/packages/client/src/launchdarkly_ai_server/judge_scoring.py index 99fc3a05..b782e497 100644 --- a/packages/client/src/launchdarkly_ai_server/judge_scoring.py +++ b/packages/client/src/launchdarkly_ai_server/judge_scoring.py @@ -65,6 +65,55 @@ def numeric_score(score: Any) -> float | None: return value if isfinite(value) else None +def typesafe_judge_entries(raw: Any) -> list[dict[str, Any]] | None: + """Parse a TypeSafe handler payload into question key, score, event key, and reason. + + Returns ``None`` when ``raw`` is not a TypeSafe multi-result payload, so a + classic ``{"score", "reasoning"}`` judge stays on :func:`parse_judge_response`. + Raises ``ValueError`` when the payload claims to be TypeSafe but is malformed. + """ + parsed: Any + if isinstance(raw, str): + parsed = parse_json_with_possible_fences(raw) + elif isinstance(raw, Mapping): + parsed = raw + else: + parsed = None + if not isinstance(parsed, Mapping) or parsed.get("kind") != "typesafe": + return None + results = parsed.get("results") + if not isinstance(results, list) or not results: + raise ValueError("TypeSafe judge output is missing results") + entries: list[dict[str, Any]] = [] + for item in results: + if ( + not isinstance(item, Mapping) + or not isinstance(item.get("key"), str) + or not item["key"] + ): + raise ValueError("TypeSafe judge result is missing a key") + score = numeric_score(item.get("score")) + if score is None: + raise ValueError( + f"TypeSafe judge result {item['key']!r} is missing a finite score" + ) + event_key = item.get("eventKey") + if not isinstance(event_key, str) or not event_key: + raise ValueError( + f"TypeSafe judge result {item['key']!r} is missing an eventKey" + ) + reason = item.get("reason") + entries.append( + { + "key": item["key"], + "score": score, + "eventKey": event_key, + "reason": reason if isinstance(reason, str) else "", + } + ) + return entries + + def parse_judge_response(raw: Any) -> tuple[Any, str]: """Parse a judge model response into ``(score, reasoning)``. diff --git a/packages/client/src/launchdarkly_ai_server/judges.py b/packages/client/src/launchdarkly_ai_server/judges.py index 56d7e032..bcb06e16 100644 --- a/packages/client/src/launchdarkly_ai_server/judges.py +++ b/packages/client/src/launchdarkly_ai_server/judges.py @@ -10,6 +10,7 @@ build_message_history, numeric_score, parse_judge_response, + typesafe_judge_entries, ) from .types import ( AiConfigRep, @@ -40,6 +41,32 @@ def _provider_matches(handler: ProviderHandler, provider: str | None) -> bool: ) +def _handler_for( + handlers: list[ProviderHandler], provider: str | None, mode: str +) -> ProviderHandler | None: + """Exact ``(provider, mode)`` match, then a same-mode wildcard. + + A wildcard such as LangChain's ``('*', 'messages')`` must not hide a + handler registered for the judge's own provider. + """ + exact = next( + (handler for handler in handlers if handler.provides_for == (provider, mode)), + None, + ) + if exact is not None: + return exact + return next( + ( + handler + for handler in handlers + if handler.provides_for + and handler.provides_for[0] == "*" + and handler.provides_for[1] == mode + ), + None, + ) + + logger = logging.getLogger(__name__) @@ -101,35 +128,18 @@ async def run_judges( ) # Select judge handler. Priority: - # 1. Exact provider + mode (or wildcard provider + same mode) - # 2. Agent-mode handler for same provider / wildcard (messages-mode fallback) - # 3. Parent handler when it covers the same provider or is a wildcard + # 1. Exact provider + mode + # 2. Wildcard provider + same mode + # 3. Exact provider agent handler, then a wildcard agent handler + # 4. Parent handler when it covers the same provider or is a wildcard # When falling back to an agent-mode handler for a messages-mode judge # config, collapse messages into a single instructions block. judge_handler: ProviderHandler = handler collapse_messages = False if handlers: - exact = next( - ( - h - for h in handlers - if _provider_matches(h, judge_provider) - and h.provides_for - and h.provides_for[1] == judge_mode - ), - None, - ) + exact = _handler_for(handlers, judge_provider, judge_mode) agent_fallback = ( - next( - ( - h - for h in handlers - if _provider_matches(h, judge_provider) - and h.provides_for - and h.provides_for[1] == "agent" - ), - None, - ) + _handler_for(handlers, judge_provider, "agent") if not exact and judge_mode == "messages" else None ) @@ -145,6 +155,13 @@ async def run_judges( and handler.provides_for is not None and handler.provides_for[1] == "agent" ) + else: + logger.warning( + "Judge '%s' skipped: no handler provides for provider %r", + judge_key, + judge_provider, + ) + continue effective_judge_config = ( _collapse_messages_to_instructions(judge_ai_config) @@ -171,12 +188,39 @@ async def run_judges( variables={ "message_history": message_history, "response_to_evaluate": llm_response, + **({"input": user_input} if user_input else {}), }, ) + usage = to_usage_dict(result["usage"]) + entries = typesafe_judge_entries(result["response"]) + if entries is not None: + # One Jev call. Every label is returned with that call's + # full usage, and each label is tracked under its eventKey. + from .lifecycle import get_client + + client = get_client() + for entry in entries: + result_key = f"{judge_key}.{entry['key']}" + score = entry["score"] + judge_results[result_key] = JudgeResult( + usage=usage, + response=entry["reason"], + score=score, + event_key=entry["eventKey"], + ) + record_evaluation(score, None, result_key) + client.track( + entry["eventKey"], + to_ld_context(client, user_context), + {**base_track_data, "judgeConfigKey": result_key}, + score, + ) + continue + score, reasoning = parse_judge_response(result["response"]) judge_results[judge_key] = JudgeResult( - usage=to_usage_dict(result["usage"]), + usage=usage, response=reasoning, score=score, ) @@ -267,27 +311,9 @@ async def build_judge_tasks( collapse_messages = False if handlers: - exact = next( - ( - h - for h in handlers - if _provider_matches(h, judge_provider) - and h.provides_for - and h.provides_for[1] == judge_mode - ), - None, - ) + exact = _handler_for(handlers, judge_provider, judge_mode) agent_fallback = ( - next( - ( - h - for h in handlers - if _provider_matches(h, judge_provider) - and h.provides_for - and h.provides_for[1] == "agent" - ), - None, - ) + _handler_for(handlers, judge_provider, "agent") if not exact and judge_mode == "messages" else None ) @@ -353,26 +379,9 @@ async def run_judge( """ from .tracking import execute_and_track - def _matches(h: ProviderHandler) -> bool: - return _provider_matches(h, task.judge_provider) - - exact = next( - ( - h - for h in handlers - if _matches(h) and h.provides_for and h.provides_for[1] == task.judge_mode - ), - None, - ) + exact = _handler_for(handlers, task.judge_provider, task.judge_mode) agent_fallback = ( - next( - ( - h - for h in handlers - if _matches(h) and h.provides_for and h.provides_for[1] == "agent" - ), - None, - ) + _handler_for(handlers, task.judge_provider, "agent") if task.judge_mode == "messages" and not exact else None ) @@ -409,9 +418,54 @@ def _matches(h: ProviderHandler) -> bool: **(task.variables or {}), "message_history": message_history, "response_to_evaluate": task.actual_output, + **({"input": task.user_input} if task.user_input else {}), }, ) + usage = to_usage_dict(result["usage"]) + try: + entries = typesafe_judge_entries(result["response"]) + except ValueError: + return None + if entries is not None: + first = entries[0] + result_key = f"{task.config_key}.{first['key']}" + expanded = { + f"{task.config_key}.{entry['key']}": JudgeResult( + usage=usage, + response=entry["reason"], + score=entry["score"], + event_key=entry["eventKey"], + ) + for entry in entries + } + metrics = [ + { + "eventKey": entry["eventKey"], + "score": entry["score"], + "judgeConfigKey": f"{task.config_key}.{entry['key']}", + } + for entry in entries + ] + for entry in entries: + record_evaluation( + entry["score"], None, f"{task.config_key}.{entry['key']}" + ) + typesafe_track_data: TrackData = { + **omit_model_stamps(task.parent_track_data), + **result["track_data"], + "judgeConfigKey": result_key, + } + return JudgeRunResult( + score=first["score"], + response=first["reason"], + usage=usage, + track_data=typesafe_track_data, + results=expanded, + metrics=metrics, + event_key=first["eventKey"], + ) + try: score, reasoning = parse_judge_response(result["response"]) except ValueError: @@ -427,10 +481,6 @@ def _matches(h: ProviderHandler) -> bool: metric_score, reasoning if judge_handler.capture_content else None, ) - raw_usage = result["usage"] - - usage = to_usage_dict(raw_usage) - # A judge without a pinned model config must not inherit the parent's # modelKey / modelVersion; every other parent-only key (graphKey, ...) # is still carried over. diff --git a/packages/client/src/launchdarkly_ai_server/types.py b/packages/client/src/launchdarkly_ai_server/types.py index 761faf5b..fd8bcfff 100644 --- a/packages/client/src/launchdarkly_ai_server/types.py +++ b/packages/client/src/launchdarkly_ai_server/types.py @@ -92,7 +92,8 @@ class Message: AiConfigRep = dict[str, Any] """ Raw AI config dict as returned by ``parse_ai_config``. Fields include -``model``, ``provider``, and at least one of ``instructions`` / ``messages``. +``model``, ``provider``, and at least one of ``instructions``, ``messages``, +or a non-empty ``classifiers`` list. """ VariationMeta = dict[str, Any] @@ -229,6 +230,8 @@ class JudgeResult: usage: UsageDict response: str score: float + event_key: str | None = None + """Metric name for this score. Set when the judge produced one event per label.""" @dataclass @@ -333,6 +336,20 @@ class JudgeRunResult: ``get_client().track(task.evaluation_metric_key, ctx, track_data, score)`` from the main thread after the worker finishes. """ + results: dict[str, JudgeResult] | None = None + """ + Present when one judge call produced several scores. Keys are + ``config_key.question_key``. ``score`` and ``response`` are the first + entry. Every entry carries the call's full usage and, for a classifier, + its ``event_key``. + """ + metrics: list[dict[str, Any]] | None = None + """ + One LaunchDarkly metric per classifier. When this is set, track each + entry's ``eventKey`` and do not also track ``evaluation_metric_key``. + """ + event_key: str | None = None + """Metric name for ``score``. Set to the first label's event key when one call produced several.""" # Stream events diff --git a/packages/client/src/launchdarkly_ai_server/types_validation.py b/packages/client/src/launchdarkly_ai_server/types_validation.py index acdcce42..ff97c4fa 100644 --- a/packages/client/src/launchdarkly_ai_server/types_validation.py +++ b/packages/client/src/launchdarkly_ai_server/types_validation.py @@ -52,8 +52,10 @@ def parse_ai_config(raw: Any) -> ParseResult: has_instructions = isinstance(raw.get("instructions"), str) messages = raw.get("messages") has_messages = isinstance(messages, list) and len(messages) > 0 + classifiers = raw.get("classifiers") + has_classifiers = isinstance(classifiers, list) and len(classifiers) > 0 - if not has_instructions and not has_messages: + if not has_instructions and not has_messages and not has_classifiers: return ParseFailure( success=False, error={ diff --git a/packages/client/tests/test_client.py b/packages/client/tests/test_client.py index d7357308..b1811d10 100644 --- a/packages/client/tests/test_client.py +++ b/packages/client/tests/test_client.py @@ -1044,6 +1044,7 @@ async def test_returns_judge_task_with_correct_shape( assert task.collapse_messages is False assert task.parent_track_data is not None assert task.parent_track_data.get("configKey") == "flag" + assert task.user_input == "q" async def test_sets_collapse_messages_for_agent_fallback( self, mock_ld_client: MagicMock diff --git a/packages/client/tests/test_conversation.py b/packages/client/tests/test_conversation.py index 83b5f8bf..37044bac 100644 --- a/packages/client/tests/test_conversation.py +++ b/packages/client/tests/test_conversation.py @@ -133,6 +133,26 @@ async def test_end_time_is_the_handler_call_not_the_release(self) -> None: # Allow scheduling slack, but nothing close to the 50ms of post-end work. assert span.end_time - ended_at < 20_000_000 + async def test_later_labels_keep_the_first_evaluation_attributes(self) -> None: + async with with_judge_evaluation("jev-judge") as record: + with _tracer.start_as_current_span("invoke_agent"): + pass + record(0.25, None, "jev-judge.accuracy") + record(0.93, None, "jev-judge.tone") + span = next(s for s in finished() if s.name == "invoke_agent") + assert span.attributes is not None + assert span.attributes["gen_ai.evaluation.name"] == "jev-judge.accuracy" + assert span.attributes["gen_ai.evaluation.score.value"] == 0.25 + events = [e for e in span.events if e.name == "gen_ai.evaluation.result"] + assert [e.attributes["gen_ai.evaluation.name"] for e in events] == [ + "jev-judge.accuracy", + "jev-judge.tone", + ] + assert [e.attributes["gen_ai.evaluation.score.value"] for e in events] == [ + 0.25, + 0.93, + ] + class TestProcessorScope: """The processor is registered on the *global* provider, so it sees every span in the process. diff --git a/packages/client/tests/test_judge_scoring.py b/packages/client/tests/test_judge_scoring.py index 4613b679..a0472a02 100644 --- a/packages/client/tests/test_judge_scoring.py +++ b/packages/client/tests/test_judge_scoring.py @@ -4,7 +4,10 @@ import pytest -from launchdarkly_ai_server.judge_scoring import parse_judge_response +from launchdarkly_ai_server.judge_scoring import ( + parse_judge_response, + typesafe_judge_entries, +) class TestParseJudgeResponse: @@ -52,3 +55,29 @@ def test_score_returned_untouched_for_caller_policy(self) -> None: def test_rejects_non_object_responses(self, raw: Any) -> None: with pytest.raises(ValueError, match="Invalid JSON from judge"): parse_judge_response(raw) + + +def test_typesafe_entries_leave_classic_judges_alone() -> None: + assert typesafe_judge_entries('{"score": 0.9, "reasoning": "solid"}') is None + + +def test_typesafe_entries_keep_label_order() -> None: + raw = ( + '{"kind":"typesafe","results":[' + '{"key":"migration_intent","eventKey":"$ld:ai:judge:jev:migration_intent","score":0.25},' + '{"key":"migration","eventKey":"$ld:ai:judge:jev:migration","score":0.9}]}' + ) + assert typesafe_judge_entries(raw) == [ + { + "key": "migration_intent", + "score": 0.25, + "eventKey": "$ld:ai:judge:jev:migration_intent", + "reason": "", + }, + { + "key": "migration", + "score": 0.9, + "eventKey": "$ld:ai:judge:jev:migration", + "reason": "", + }, + ] diff --git a/packages/client/tests/test_judges.py b/packages/client/tests/test_judges.py index 5d560948..0c6a3463 100644 --- a/packages/client/tests/test_judges.py +++ b/packages/client/tests/test_judges.py @@ -194,6 +194,58 @@ async def recording_fn( assert effective.get("instructions") is not None assert effective.get("messages") == [] + async def test_exact_provider_beats_an_earlier_wildcard( + self, mock_ld_client: MagicMock + ) -> None: + seen: list[str] = [] + + def handler_for(name: str, provider: str) -> ProviderHandler: + async def fn( + config, user_input, tool_handlers, variables, history=None + ) -> dict: # type: ignore[override] + seen.append(name) + return { + "output": '{"score": 0.5, "reasoning": "ok"}', + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + return ProviderHandler(fn=fn, provides_for=(provider, "messages")) # type: ignore[arg-type] + + wildcard = handler_for("wildcard", "*") + typesafe = handler_for("typesafe", "TypeSafe") + mock_ld_client.variation = AsyncMock( + return_value={ + "model": {"name": "jev"}, + "provider": {"name": "TypeSafe"}, + "instructions": "judge", + "_ldMeta": { + "enabled": True, + "variationKey": "j1", + "version": 1, + "mode": "judge", + }, + } + ) + config = { + "model": {"name": "gpt-4"}, + "provider": {"name": "OpenAI"}, + "instructions": "hi", + "judgeConfiguration": {"judges": [{"key": "jev", "samplingRate": 1.0}]}, + } + import random + + with patch.object(random, "random", return_value=0.0): + await run_judges( + config=config, + user_context=CONTEXT, + handler=wildcard, + handlers=[wildcard, typesafe], + user_input="question", + llm_response="answer", + base_track_data={}, + ) + assert seen == ["typesafe"] + async def test_exact_agent_handler_fallback_collapses_messages( self, mock_ld_client: MagicMock ) -> None: @@ -587,3 +639,175 @@ async def test_keeps_judge_own_model_stamps( assert result is not None assert result.track_data["modelKey"] == "judge-model" assert result.track_data["modelVersion"] == 2 + + +class TestTypesafeJudgeResults: + async def test_expands_labels_and_tracks_each_event_key( + self, mock_ld_client: MagicMock + ) -> None: + import json + import random + + payload = json.dumps( + { + "kind": "typesafe", + "results": [ + { + "key": "migration_intent", + "eventKey": "$ld:ai:judge:jev:migration_intent", + "score": 0.25, + "reason": "Unclear", + }, + { + "key": "migration", + "eventKey": "$ld:ai:judge:jev:migration", + "score": 0.9, + "reason": "Yes", + }, + ], + } + ) + + seen_variables: list[dict[str, Any]] = [] + + async def fn( + config, user_input, tool_handlers, variables, history=None + ) -> dict: # type: ignore[override] + seen_variables.append(variables) + return { + "output": payload, + "usage": {"input_tokens": 100, "output_tokens": 20}, + } + + handler = ProviderHandler(fn=fn, provides_for=("TypeSafe", "messages")) # type: ignore[arg-type] + mock_ld_client.variation = AsyncMock( + return_value={ + "model": {"name": "jev"}, + "provider": {"name": "TypeSafe"}, + "instructions": "judge", + "evaluationMetricKey": "judge-metric", + "_ldMeta": { + "enabled": True, + "variationKey": "j1", + "version": 1, + "mode": "judge", + }, + } + ) + with patch.object(random, "random", return_value=0.0): + result = await run_judges( + config={ + "model": {"name": "gpt-4"}, + "provider": {"name": "OpenAI"}, + "instructions": "hi", + "judgeConfiguration": { + "judges": [{"key": "jev-judge", "samplingRate": 1.0}] + }, + }, + user_context=CONTEXT, + handler=handler, + user_input="question", + llm_response="answer", + base_track_data={"runId": "x"}, + ) + + assert seen_variables[0]["input"] == "question" + assert seen_variables[0]["response_to_evaluate"] == "answer" + assert set(result) == { + "jev-judge.migration_intent", + "jev-judge.migration", + } + assert result["jev-judge.migration_intent"].score == 0.25 + assert result["jev-judge.migration"].score == 0.9 + assert result["jev-judge.migration_intent"].response == "Unclear" + assert result["jev-judge.migration"].response == "Yes" + assert result["jev-judge.migration_intent"].usage.input == 100 + assert result["jev-judge.migration"].usage.input == 100 + assert result["jev-judge.migration_intent"].usage.output == 20 + metric_calls = [ + call + for call in mock_ld_client.track.call_args_list + if str(call.args[0]).startswith("$ld:ai:judge:jev:") + ] + token_calls = [ + call + for call in mock_ld_client.track.call_args_list + if call.args[0] == "$ld:ai:tokens:total" + ] + assert [call.args[0] for call in metric_calls] == [ + "$ld:ai:judge:jev:migration_intent", + "$ld:ai:judge:jev:migration", + ] + assert metric_calls[0].args[2]["judgeConfigKey"] == "jev-judge.migration_intent" + assert metric_calls[0].args[3] == 0.25 + assert metric_calls[1].args[3] == 0.9 + assert len(token_calls) == 1 + + async def test_run_judge_keeps_the_first_label_event_key( + self, mock_ld_client: MagicMock + ) -> None: + import json + + from launchdarkly_ai_server import JudgeTask, run_judge + + payload = json.dumps( + { + "kind": "typesafe", + "results": [ + { + "key": "migration_intent", + "eventKey": "$ld:ai:judge:jev:migration_intent", + "score": 0.25, + "reason": "Unclear", + }, + { + "key": "migration", + "eventKey": "$ld:ai:judge:jev:migration", + "score": 0.9, + "reason": "Yes", + }, + ], + } + ) + + seen_variables: list[dict[str, Any]] = [] + + async def fn( + config, user_input, tool_handlers, variables, history=None + ) -> dict: # type: ignore[override] + seen_variables.append(variables) + return { + "output": payload, + "usage": {"input_tokens": 100, "output_tokens": 20}, + } + + handler = ProviderHandler(fn=fn, provides_for=("TypeSafe", "messages")) # type: ignore[arg-type] + result = await run_judge( + JudgeTask( + config_key="jev-judge", + judge_config={ + "model": {"name": "jev"}, + "provider": {"name": "TypeSafe"}, + "instructions": "judge", + }, + judge_meta={ + "enabled": True, + "variationKey": "j1", + "version": 1, + "mode": "judge", + }, + actual_output="answer", + user_input="question", + user_context=CONTEXT, + judge_provider="TypeSafe", + judge_mode="messages", + collapse_messages=False, + parent_track_data={"runId": "x"}, + ), + [handler], + ) + assert result is not None + assert seen_variables[0]["input"] == "question" + assert result.event_key == "$ld:ai:judge:jev:migration_intent" + assert result.score == 0.25 + assert result.response == "Unclear" diff --git a/packages/client/tests/test_schema.py b/packages/client/tests/test_schema.py index fad75cf0..7a6a50d5 100644 --- a/packages/client/tests/test_schema.py +++ b/packages/client/tests/test_schema.py @@ -32,6 +32,24 @@ def test_fails_with_neither(self) -> None: result = parse_ai_config(raw) assert result.success is False + def test_accepts_classifiers_without_instructions(self) -> None: + raw = { + "model": {"name": "jev-latest", "parameters": {}}, + "provider": {"name": "TypeSafe"}, + "messages": [], + "classifiers": [ + { + "key": "accuracy", + "eventKey": "$ld:ai:judge:jev:accuracy", + "instructions": "How accurate?", + "type": "score", + "criteria": ["Great", "Ok", "Bad"], + } + ], + } + result = parse_ai_config(raw) + assert result.success is True + def test_fails_with_empty_messages(self) -> None: raw = { "model": {"name": "gpt-4"}, diff --git a/packages/typesafe/CHANGELOG.md b/packages/typesafe/CHANGELOG.md new file mode 100644 index 00000000..20b78eb4 --- /dev/null +++ b/packages/typesafe/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## 0.1.0 + +* add a TypeSafe Jev handler for judge configs diff --git a/packages/typesafe/README.md b/packages/typesafe/README.md new file mode 100644 index 00000000..42433da6 --- /dev/null +++ b/packages/typesafe/README.md @@ -0,0 +1,19 @@ +# `launchdarkly-ai-typesafe` + +Judge handler for TypeSafe Jev. A judge AI Config whose provider is `TypeSafe` is routed here. Judge mode already normalizes to `messages`, so the handler registers as `("TypeSafe", "messages")`. + +Jev does not return the `{score, reasoning}` JSON other judges use. The questions are the variation's `classifiers` list, sent as Jev `noul`, `choice`, and `score` questions. `extract_typesafe_questions` is the only reader of that list. + +Each label becomes its own `judge_results` entry, keyed `judge_key.label_key`, and carries that label's `event_key`. The entry's `response` is the selected label value. Every entry carries the full token usage of the one Jev call. LaunchDarkly token telemetry is recorded once for that call. Each label is tracked on its own `eventKey`. + +The client reads `TYPESAFE_API_KEY` from the environment. + +```python +from launchdarkly_ai_typesafe import create_typesafe_handler +from launchdarkly_ai_server import config + +result = await config( + key="my-config", + handler=create_typesafe_handler(), +).invoke(user_input, context) +``` diff --git a/packages/typesafe/agents.md b/packages/typesafe/agents.md new file mode 100644 index 00000000..e976a1e8 --- /dev/null +++ b/packages/typesafe/agents.md @@ -0,0 +1,19 @@ +# Agent Guide — `launchdarkly-ai-typesafe` + +This Tier 1 handler sends a judge config to TypeSafe Jev. + +## Invariants + +- Routing metadata is `("TypeSafe", "messages")`. Judge mode normalizes to `messages`. +- Questions come only from `extract_typesafe_questions`, which reads the variation's `classifiers` list. +- The handler does not ask for `{score, reasoning}`. That block is stripped from `message_history` before it becomes Jev state. +- One `system_one` call returns one result per label. The handler's `output` is `{"kind":"typesafe","results":[...]}`. The client expands that into `judge_results`. +- Auth is `TYPESAFE_API_KEY`. The factory does not take a client or a key. +- No tools and no streaming. +- Telemetry is `invoke_agent` → `chat `. Content is gated by `capture_content`. + +## Files + +- `questions.py` — classifier extraction, state construction, and answer-to-score mapping. +- `handler.py` — Jev call and spans. +- `__init__.py` — package registration and exports. diff --git a/packages/typesafe/pyproject.toml b/packages/typesafe/pyproject.toml new file mode 100644 index 00000000..ccfa92d1 --- /dev/null +++ b/packages/typesafe/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "launchdarkly-ai-typesafe" +version = "0.1.0" +requires-python = ">=3.12" +dependencies = [ + "launchdarkly-ai-server", + "opentelemetry-api>=1.25", + "typesafe-sdk>=0.6", +] +description = "TypeSafe Jev handler for LaunchDarkly AI SDK judges" +readme = "README.md" +license = "Apache-2.0" +authors = [{name = "LaunchDarkly", email = "team@launchdarkly.com"}] +keywords = ["launchdarkly", "ai", "typesafe", "jev"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Libraries", +] + +[project.urls] +Homepage = "https://github.com/launchdarkly/python-ai-sdk" +Repository = "https://github.com/launchdarkly/python-ai-sdk" +"Bug Tracker" = "https://github.com/launchdarkly/python-ai-sdk/issues" + +[build-system] +requires = ["hatchling>=1.32,<2"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/launchdarkly_ai_typesafe"] diff --git a/packages/typesafe/src/launchdarkly_ai_typesafe/__init__.py b/packages/typesafe/src/launchdarkly_ai_typesafe/__init__.py new file mode 100644 index 00000000..51149c07 --- /dev/null +++ b/packages/typesafe/src/launchdarkly_ai_typesafe/__init__.py @@ -0,0 +1,25 @@ +"""LaunchDarkly AI SDK integration for TypeSafe Jev.""" + +__version__ = "0.1.0" # x-release-please-version + +from launchdarkly_ai_server import register_ai_sdk_package + +from .handler import create_typesafe_handler +from .questions import ( + TypesafeQuestionError, + build_typesafe_state, + extract_typesafe_questions, + reason_from_answer, + score_from_answer, +) + +__all__ = [ + "TypesafeQuestionError", + "build_typesafe_state", + "create_typesafe_handler", + "extract_typesafe_questions", + "reason_from_answer", + "score_from_answer", +] + +register_ai_sdk_package("launchdarkly-ai-typesafe", __version__) diff --git a/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py b/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py new file mode 100644 index 00000000..61b36344 --- /dev/null +++ b/packages/typesafe/src/launchdarkly_ai_typesafe/handler.py @@ -0,0 +1,163 @@ +"""TypeSafe Jev handler for LaunchDarkly AI judges.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from typing import Any + +from opentelemetry import trace +from opentelemetry.trace import Status, StatusCode +from typesafe_sdk import AsyncTypeSafeClient, Choice, Noul, Score + +from launchdarkly_ai_server import ( + AiConfigRep, + ProviderHandler, + create_handler, + set_input_content_attributes, + set_ld_span_attributes, + set_model_identity_attributes, + set_output_content_attributes, + set_usage_span_attributes, + text_message, +) +from launchdarkly_ai_server.utils import SpanUsage + +from .questions import ( + build_typesafe_state, + extract_typesafe_questions, + reason_from_answer, + score_from_answer, + typesafe_output, +) + +# The conversation processor only holds spans from tracers named "@launchdarkly/...". +_TRACER = trace.get_tracer("@launchdarkly/ai-typesafe") + + +def _sdk_question(question: Mapping[str, Any]) -> Any: + instructions = question["instructions"] + if question["type"] == "noul": + criteria = question.get("criteria") + if criteria: + return Noul(instructions=instructions, criteria=criteria) + return Noul(instructions=instructions) + if question["type"] == "choice": + return Choice(instructions=instructions, criteria=question["criteria"]) + return Score(instructions=instructions, criteria=question["criteria"]) + + +def _usage_of(usage: Any) -> dict[str, int]: + if usage is None: + return {"input_tokens": 0, "output_tokens": 0} + if isinstance(usage, Mapping): + input_tokens = usage.get("input_tokens", usage.get("inputTokens", 0)) + output_tokens = usage.get("output_tokens", usage.get("outputTokens", 0)) + else: + input_tokens = getattr(usage, "input_tokens", None) + if input_tokens is None: + input_tokens = getattr(usage, "inputTokens", 0) + output_tokens = getattr(usage, "output_tokens", None) + if output_tokens is None: + output_tokens = getattr(usage, "outputTokens", 0) + return { + "input_tokens": int(input_tokens or 0), + "output_tokens": int(output_tokens or 0), + } + + +def _answer_for(response: Any, key: str) -> Any: + answers = getattr(response, "answers", None) + if isinstance(answers, Mapping) and key in answers: + return answers[key] + if isinstance(response, Mapping): + mapped = response.get("answers") + if isinstance(mapped, Mapping): + return mapped.get(key) + return None + + +def create_typesafe_handler(*, capture_content: bool = False) -> ProviderHandler: + """Build a handler with ``provides_for = ('TypeSafe', 'messages')``. + + Judge mode normalizes to messages, so a judge variation whose provider is + ``TypeSafe`` selects this handler. ``TYPESAFE_API_KEY`` is read by the + TypeSafe client from the environment. + """ + + async def _call( + config: AiConfigRep, + user_input: str | None = None, + tool_handlers: dict[str, Any] | None = None, + variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + del tool_handlers + questions = extract_typesafe_questions(config) + state = build_typesafe_state( + user_input=user_input, variables=variables, history=history + ) + model = config.get("model") + model_name = model.get("name") if isinstance(model, Mapping) else None + span_model = model_name or "jev" + root = _TRACER.start_span("invoke_agent") + chat = _TRACER.start_span( + f"chat {span_model}", context=trace.set_span_in_context(root) + ) + set_ld_span_attributes(root, dict(variables) if variables else None) + set_model_identity_attributes(root, "typesafe", span_model) + set_model_identity_attributes(chat, "typesafe", span_model) + if capture_content: + set_input_content_attributes( + chat, + True, + messages=[text_message("user", json.dumps(state))], + ) + try: + async with AsyncTypeSafeClient() as client: + response = await client.system_one( + state=state, + questions={ + question["key"]: _sdk_question(question) + for question in questions + }, + model=model_name or None, + ) + results = [] + for question in questions: + answer = _answer_for(response, question["key"]) + results.append( + { + "key": question["key"], + "eventKey": question["eventKey"], + "score": score_from_answer(question, answer), + "reason": reason_from_answer(question, answer), + } + ) + output = typesafe_output(results) + usage = _usage_of(getattr(response, "usage", None)) + span_usage = SpanUsage( + input=usage["input_tokens"], output=usage["output_tokens"] + ) + set_usage_span_attributes(chat, span_usage) + set_usage_span_attributes(root, span_usage) + if capture_content: + set_output_content_attributes( + chat, True, [text_message("assistant", output)] + ) + chat.set_status(Status(StatusCode.OK)) + root.set_status(Status(StatusCode.OK)) + return {"output": output, "usage": usage} + except Exception as exc: + chat.record_exception(exc) + root.record_exception(exc) + chat.set_status(Status(StatusCode.ERROR, str(exc))) + root.set_status(Status(StatusCode.ERROR, str(exc))) + raise + finally: + chat.end() + root.end() + + return create_handler( + ("TypeSafe", "messages"), _call, capture_content=capture_content + ) diff --git a/packages/typesafe/src/launchdarkly_ai_typesafe/questions.py b/packages/typesafe/src/launchdarkly_ai_typesafe/questions.py new file mode 100644 index 00000000..793e4d0b --- /dev/null +++ b/packages/typesafe/src/launchdarkly_ai_typesafe/questions.py @@ -0,0 +1,302 @@ +"""Question extraction and Jev answer scoring for a TypeSafe judge. + +``extract_typesafe_questions`` is the only reader of the judge's ``classifiers`` +list. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Any, Literal + +from launchdarkly_ai_server.judge_scoring import FORMATTING_INSTRUCTIONS + +QuestionType = Literal["noul", "choice", "score"] +_QUESTION_TYPES = {"noul", "choice", "score"} + + +class TypesafeQuestionError(ValueError): + """The judge config does not describe a usable set of Jev questions.""" + + +def extract_typesafe_questions(config: Mapping[str, Any]) -> list[dict[str, Any]]: + """Return the Jev questions declared on ``config['classifiers']``.""" + labels = config.get("classifiers") + if not isinstance(labels, list) or not labels: + raise TypesafeQuestionError("classifiers must be a non-empty list") + + seen: set[str] = set() + seen_events: set[str] = set() + questions: list[dict[str, Any]] = [] + for index, label in enumerate(labels): + if not isinstance(label, Mapping): + raise TypesafeQuestionError(f"classifiers[{index}] must be an object") + key = label.get("key") + if not isinstance(key, str) or not key.strip(): + raise TypesafeQuestionError(f"classifiers[{index}] is missing a key") + if key in seen: + raise TypesafeQuestionError(f"classifiers has a duplicate key {key!r}") + seen.add(key) + event_key = label.get("eventKey") + if not isinstance(event_key, str) or not event_key.strip(): + raise TypesafeQuestionError( + f"classifiers label {key!r} is missing an eventKey" + ) + if event_key in seen_events: + raise TypesafeQuestionError( + f"classifiers has a duplicate eventKey {event_key!r}" + ) + seen_events.add(event_key) + question_type = label.get("type") + if question_type not in _QUESTION_TYPES: + raise TypesafeQuestionError( + f"classifiers label {key!r} has an unsupported type" + ) + instructions = label.get("instructions") + if not isinstance(instructions, str) or not instructions.strip(): + raise TypesafeQuestionError( + f"classifiers label {key!r} is missing instructions" + ) + question: dict[str, Any] = { + "key": key, + "eventKey": event_key, + "type": question_type, + "instructions": instructions, + } + criteria = _criteria_for(key, question_type, label.get("criteria")) + if criteria is not None: + question["criteria"] = criteria + questions.append(question) + return questions + + +def _criteria_for(key: str, question_type: str, raw: Any) -> Any: + if question_type == "noul": + if raw is None: + return None + if not isinstance(raw, Mapping): + raise TypesafeQuestionError( + f"classifiers label {key!r} has invalid noul criteria" + ) + criteria: dict[str, str | None] = {} + for name in ("true", "false"): + if name not in raw: + continue + description = raw[name] + if description is not None and not isinstance(description, str): + raise TypesafeQuestionError( + f"classifiers label {key!r} has a non-string noul description" + ) + criteria[name] = description + return criteria or None + if question_type == "choice": + if not isinstance(raw, Mapping) or not raw: + raise TypesafeQuestionError( + f"classifiers label {key!r} requires choice criteria" + ) + choice_criteria: dict[str, str | None] = {} + for name, description in raw.items(): + if not isinstance(name, str) or not name.strip(): + raise TypesafeQuestionError( + f"classifiers label {key!r} has an empty choice" + ) + if description is not None and not isinstance(description, str): + raise TypesafeQuestionError( + f"classifiers label {key!r} has a non-string choice description" + ) + choice_criteria[name] = description + return choice_criteria + if not isinstance(raw, list) or len(raw) < 2: + raise TypesafeQuestionError( + f"classifiers label {key!r} requires at least two score levels" + ) + levels: list[str | None] = [] + for level in raw: + if level is not None and not isinstance(level, str): + raise TypesafeQuestionError( + f"classifiers label {key!r} has a non-string score level" + ) + levels.append(level) + return levels + + +def without_formatting_instructions(value: Any) -> str | None: + """Drop the shared judge JSON-format block. Jev does not use it.""" + if not isinstance(value, str): + return None + index = value.find(FORMATTING_INSTRUCTIONS) + text = value if index == -1 else value[:index] + text = text.strip() + return text or None + + +def build_typesafe_state( + *, + user_input: str | None = None, + variables: Mapping[str, Any] | None = None, + history: Sequence[Mapping[str, Any]] | None = None, +) -> dict[str, Any]: + """The state sent to Jev: the conversation, without score-format instructions.""" + variables = variables or {} + state: dict[str, Any] = {} + + def assign(key: str, value: Any) -> None: + if value is None or value == "": + return + state[key] = value + + assign("input", variables.get("input")) + output = variables.get("response_to_evaluate", user_input) + assign("output", output) + assign( + "message_history", + without_formatting_instructions(variables.get("message_history")), + ) + assign("trajectory", variables.get("trajectory")) + assign("expected_output", variables.get("expected_output")) + assign("ground_truth_context", variables.get("ground_truth_context")) + if user_input and user_input != output: + assign("user_input", user_input) + if history: + state["history"] = list(history) + return state + + +def _read(answer: Any, field: str) -> Any: + if isinstance(answer, Mapping): + return answer.get(field) + return getattr(answer, field, None) + + +def _finite(value: Any) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + number = float(value) + if number != number or number in (float("inf"), float("-inf")): + return None + return number + + +def _clamp(value: float) -> float: + return min(1.0, max(0.0, value)) + + +def score_from_answer(question: Mapping[str, Any], answer: Any) -> float: + """Map one Jev answer onto the 0–1 score stored in ``judge_results``.""" + key = question["key"] + question_type = question["type"] + if question_type == "noul": + noul = _finite(_read(answer, "noul")) + if noul is None: + raise TypesafeQuestionError( + f"TypeSafe answer {key!r} is missing a noul probability" + ) + return _clamp(noul) + if question_type == "choice": + selected = _read(answer, "choice") + probabilities = _read(answer, "probabilities") + if isinstance(selected, str) and isinstance(probabilities, Mapping): + selected_probability = _finite(probabilities.get(selected)) + if selected_probability is not None: + return _clamp(selected_probability) + confidence = _finite(_read(answer, "confidence")) + if confidence is not None: + return _clamp(confidence) + if isinstance(selected, str) and selected: + return 1.0 + raise TypesafeQuestionError(f"TypeSafe answer {key!r} is missing a choice") + + score = _finite(_read(answer, "score")) + if score is None: + raise TypesafeQuestionError(f"TypeSafe answer {key!r} is missing a score") + legend = _read(answer, "legend") + criteria = question.get("criteria") + levels = ( + len(legend) + if isinstance(legend, Mapping) + else len(criteria) + if isinstance(criteria, list) + else 0 + ) + # Jev's score is an expected rubric index. A value in 0–1 is the low end of + # a longer rubric, not a probability. Two levels divide by 1. + if levels > 1: + return _clamp(score / (levels - 1)) + return _clamp(score) + + +def _criterion_text(value: Any) -> str | None: + if isinstance(value, str) and value.strip(): + return value + return None + + +def _legend_level(key: Any) -> int | None: + if isinstance(key, bool): + return None + if isinstance(key, int): + return key + if isinstance(key, str) and key.lstrip("-").isdigit(): + return int(key) + return None + + +def _nearest_key(score: float, keys: Sequence[int]) -> int: + """Integer closest to ``score``. Ties resolve to the lower key.""" + return min(keys, key=lambda key: (abs(key - score), key)) + + +def reason_from_answer(question: Mapping[str, Any], answer: Any) -> str: + """The selected label's value, stored as the judge result reason. + + A noul at or above 0.5 selects ``true``. A score uses the legend entry + nearest the raw score. + """ + question_type = question.get("type") + criteria = question.get("criteria") + if question_type == "noul": + noul = _finite(_read(answer, "noul")) + side = "true" if noul is not None and noul >= 0.5 else "false" + if isinstance(criteria, Mapping): + described = _criterion_text(criteria.get(side)) + if described is not None: + return described + return side + if question_type == "choice": + selected = _read(answer, "choice") + name = selected if isinstance(selected, str) else "" + if name and isinstance(criteria, Mapping): + described = _criterion_text(criteria.get(name)) + if described is not None: + return described + return name + + score = _finite(_read(answer, "score")) + if score is None: + return "" + legend = _read(answer, "legend") + if isinstance(legend, Mapping) and legend: + keys: list[int] = [] + for legend_key in legend: + level = _legend_level(legend_key) + if level is not None: + keys.append(level) + if keys: + level = _nearest_key(score, keys) + described = _criterion_text(legend.get(level)) + if described is None: + described = _criterion_text(legend.get(str(level))) + if described is not None: + return described + if isinstance(criteria, list) and criteria: + level = _nearest_key(score, list(range(len(criteria)))) + described = _criterion_text(criteria[level]) + if described is not None: + return described + return "" + + +def typesafe_output(results: list[dict[str, Any]]) -> str: + import json + + return json.dumps({"kind": "typesafe", "results": results}) diff --git a/packages/typesafe/tests/test_handler.py b/packages/typesafe/tests/test_handler.py new file mode 100644 index 00000000..2b9bec5e --- /dev/null +++ b/packages/typesafe/tests/test_handler.py @@ -0,0 +1,125 @@ +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from launchdarkly_ai_server.judge_scoring import FORMATTING_INSTRUCTIONS +from launchdarkly_ai_typesafe import create_typesafe_handler + +CLASSIFIERS = [ + { + "key": "tone", + "eventKey": "$ld:ai:judge:jev:tone", + "instructions": "Is the tone appropriate?", + "type": "noul", + "criteria": {"true": "Polite", "false": "Harmful"}, + }, + { + "key": "accuracy", + "eventKey": "$ld:ai:judge:jev:accuracy", + "instructions": "How accurate is the response?", + "type": "score", + "criteria": ["Great", "Ok", "Bad"], + }, +] + + +def _client(response: object) -> MagicMock: + client = MagicMock() + client.system_one = AsyncMock(return_value=response) + entered = MagicMock() + entered.__aenter__ = AsyncMock(return_value=client) + entered.__aexit__ = AsyncMock(return_value=None) + factory = MagicMock(return_value=entered) + return factory + + +@pytest.mark.asyncio +async def test_provides_for_typesafe_messages() -> None: + handler = create_typesafe_handler() + assert handler.provides_for == ("TypeSafe", "messages") + assert handler.capture_content is False + + +@pytest.mark.asyncio +async def test_sends_labels_and_returns_one_score_per_label() -> None: + response = SimpleNamespace( + answers={ + "tone": SimpleNamespace(noul=0.25), + "accuracy": SimpleNamespace(score=1), + }, + usage=SimpleNamespace(input_tokens=100, output_tokens=20), + ) + factory = _client(response) + handler = create_typesafe_handler() + with patch("launchdarkly_ai_typesafe.handler.AsyncTypeSafeClient", factory): + result = await handler( + { + "model": {"name": "jev-latest"}, + "provider": {"name": "TypeSafe"}, + "messages": [], + "classifiers": CLASSIFIERS, + }, + "assistant output", + None, + { + "input": "user question", + "response_to_evaluate": "assistant output", + "message_history": ( + "user question\n\nassistant output\n\n" + FORMATTING_INSTRUCTIONS + ), + }, + ) + + state = factory.return_value.__aenter__.return_value.system_one.await_args.kwargs + request_state = state["state"] + assert state["model"] == "jev-latest" + tone = state["questions"]["tone"] + assert tone.instructions == "Is the tone appropriate?" + assert tone.criteria["true"] == "Polite" + assert state["questions"]["accuracy"].criteria == ["Great", "Ok", "Bad"] + assert "valid JSON format" not in str(request_state) + assert request_state["input"] == "user question" + assert request_state["output"] == "assistant output" + assert json.loads(result["output"]) == { + "kind": "typesafe", + "results": [ + { + "key": "tone", + "eventKey": "$ld:ai:judge:jev:tone", + "score": 0.25, + "reason": "Harmful", + }, + { + "key": "accuracy", + "eventKey": "$ld:ai:judge:jev:accuracy", + "score": 0.5, + "reason": "Ok", + }, + ], + } + assert result["usage"] == {"input_tokens": 100, "output_tokens": 20} + + +@pytest.mark.asyncio +async def test_propagates_client_failure() -> None: + factory = _client(SimpleNamespace()) + factory.return_value.__aenter__.return_value.system_one.side_effect = RuntimeError( + "typesafe down" + ) + handler = create_typesafe_handler() + with ( + patch("launchdarkly_ai_typesafe.handler.AsyncTypeSafeClient", factory), + pytest.raises(RuntimeError, match="typesafe down"), + ): + await handler( + { + "model": {"name": "jev"}, + "messages": [], + "classifiers": CLASSIFIERS, + "provider": {"name": "TypeSafe"}, + "instructions": "ignored", + }, + "output", + ) diff --git a/packages/typesafe/tests/test_questions.py b/packages/typesafe/tests/test_questions.py new file mode 100644 index 00000000..86384b53 --- /dev/null +++ b/packages/typesafe/tests/test_questions.py @@ -0,0 +1,167 @@ +import pytest + +from launchdarkly_ai_server.judge_scoring import FORMATTING_INSTRUCTIONS +from launchdarkly_ai_typesafe.questions import ( + TypesafeQuestionError, + build_typesafe_state, + extract_typesafe_questions, + reason_from_answer, + score_from_answer, +) + +CLASSIFIERS = [ + { + "key": "accuracy", + "eventKey": "$ld:ai:judge:jev:accuracy", + "instructions": "How accurate is the response?", + "type": "score", + "criteria": ["Great", "Ok", "Bad"], + }, + { + "key": "tone", + "eventKey": "$ld:ai:judge:jev:tone", + "instructions": "Is the tone appropriate?", + "type": "noul", + "criteria": {"true": "Polite", "false": "Harmful"}, + }, + { + "key": "route", + "eventKey": "$ld:ai:judge:jev:route", + "instructions": "What is the tone?", + "type": "choice", + "criteria": {"calm": "Calm", "angry": "Angry"}, + }, +] + + +def _config(labels: object = CLASSIFIERS) -> dict: + return { + "model": {"name": "jev-latest"}, + "provider": {"name": "TypeSafe"}, + "messages": [], + "classifiers": labels, + } + + +def test_extracts_all_three_question_types() -> None: + assert extract_typesafe_questions(_config()) == [ + { + "key": "accuracy", + "eventKey": "$ld:ai:judge:jev:accuracy", + "type": "score", + "instructions": "How accurate is the response?", + "criteria": ["Great", "Ok", "Bad"], + }, + { + "key": "tone", + "eventKey": "$ld:ai:judge:jev:tone", + "type": "noul", + "instructions": "Is the tone appropriate?", + "criteria": {"true": "Polite", "false": "Harmful"}, + }, + { + "key": "route", + "eventKey": "$ld:ai:judge:jev:route", + "type": "choice", + "instructions": "What is the tone?", + "criteria": {"calm": "Calm", "angry": "Angry"}, + }, + ] + + +def test_rejects_a_config_without_labels() -> None: + with pytest.raises(TypesafeQuestionError, match="classifiers"): + extract_typesafe_questions(_config([])) + with pytest.raises(TypesafeQuestionError, match="classifiers"): + extract_typesafe_questions( + { + "model": {"name": "jev"}, + "provider": {"name": "TypeSafe"}, + "instructions": "unused", + } + ) + + +def test_state_drops_formatting_instructions() -> None: + state = build_typesafe_state( + user_input="the answer", + variables={ + "input": "the question", + "response_to_evaluate": "the answer", + "message_history": f"the question\n\n{FORMATTING_INSTRUCTIONS}", + "trajectory": "called search", + }, + history=[{"role": "user", "content": "earlier"}], + ) + assert state == { + "input": "the question", + "output": "the answer", + "message_history": "the question", + "trajectory": "called search", + "history": [{"role": "user", "content": "earlier"}], + } + assert "valid JSON format" not in str(state) + + +def test_score_mapping() -> None: + noul = {"key": "migration", "type": "noul", "instructions": "migration?"} + choice = {"key": "tone", "type": "choice", "instructions": "tone?"} + scored = { + "key": "urgency", + "type": "score", + "instructions": "urgency?", + "criteria": ["a", "b", "c"], + } + assert score_from_answer(noul, {"noul": 0.82}) == 0.82 + assert ( + score_from_answer( + choice, + { + "choice": "angry", + "probabilities": {"calm": 0.2, "angry": 0.8}, + "confidence": 0.1, + }, + ) + == 0.8 + ) + legend = {0: "a", 1: "b", 2: "c"} + assert score_from_answer(scored, {"score": 0.4, "legend": legend}) == pytest.approx( + 0.2 + ) + assert score_from_answer(scored, {"score": 1, "legend": legend}) == 0.5 + assert score_from_answer(scored, {"score": 1.7, "legend": legend}) == pytest.approx( + 0.85 + ) + binary = {**scored, "criteria": ["a", "b"]} + assert score_from_answer(binary, {"score": 0.4, "legend": {0: "a", 1: "b"}}) == 0.4 + + +def test_reason_is_the_selected_label_value() -> None: + noul = { + "key": "tone", + "type": "noul", + "instructions": "tone?", + "criteria": {"true": "Polite", "false": "Harmful"}, + } + choice = { + "key": "route", + "type": "choice", + "instructions": "route?", + "criteria": {"calm": "A neutral message", "angry": None}, + } + scored = { + "key": "urgency", + "type": "score", + "instructions": "urgency?", + "criteria": ["Great", "Ok", "Bad"], + } + legend = {0: "Great", 1: "Ok", 2: "Bad"} + assert reason_from_answer(noul, {"noul": 0.5}) == "Polite" + assert reason_from_answer(noul, {"noul": 0.49}) == "Harmful" + assert reason_from_answer({"key": "tone", "type": "noul"}, {"noul": 0.9}) == "true" + assert reason_from_answer(choice, {"choice": "calm"}) == "A neutral message" + assert reason_from_answer(choice, {"choice": "angry"}) == "angry" + assert reason_from_answer(scored, {"score": 0.4, "legend": legend}) == "Great" + assert reason_from_answer(scored, {"score": 1.5, "legend": legend}) == "Ok" + assert reason_from_answer(scored, {"score": 1.7, "legend": legend}) == "Bad" + assert reason_from_answer(scored, {"score": 1}) == "Ok" diff --git a/pyproject.toml b/pyproject.toml index 99e24585..d5acfe6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,4 +66,5 @@ known-first-party = [ "launchdarkly_ai_langchain_messages", "launchdarkly_ai_openai_agents", "launchdarkly_ai_openai_messages", + "launchdarkly_ai_typesafe", ] diff --git a/release-please-config.json b/release-please-config.json index 9d2a04e3..87d7d098 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -89,6 +89,17 @@ "src/launchdarkly_ai_langchain_messages/__init__.py" ], "component": "launchdarkly-ai-langchain-messages" + }, + "packages/typesafe": { + "release-type": "python", + "versioning": "default", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true, + "include-v-in-tag": false, + "extra-files": [ + "src/launchdarkly_ai_typesafe/__init__.py" + ], + "component": "launchdarkly-ai-typesafe" } } } diff --git a/uv.lock b/uv.lock index d8fba767..f52de56d 100644 --- a/uv.lock +++ b/uv.lock @@ -21,6 +21,7 @@ members = [ "launchdarkly-ai-openai-messages", "launchdarkly-ai-python", "launchdarkly-ai-server", + "launchdarkly-ai-typesafe", ] [manifest.dependency-groups] @@ -506,6 +507,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/f3/1db7aa2bc2524062192bb0e0323969492d1883152a232fe36eea65f4e35c/httpcore2-2.13.1.tar.gz", hash = "sha256:e0aa977abe17e69a3b820a24542a6fa88702676d83880b8d194dcd18408e5103", size = 68071, upload-time = "2026-09-23T07:47:22.372Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/ba/a4568248771ce81957bfb7cc600264a40fbcda092391ee1c415c50be4bea/httpcore2-2.13.1-py3-none-any.whl", hash = "sha256:e1e05d4f25f7d7d496bfb96748f6f4b67657b03da069b3a68c36069f3db73d0a", size = 83423, upload-time = "2026-09-23T07:47:19.365Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -530,6 +544,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, ] +[[package]] +name = "httpx2" +version = "2.13.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/44/474bef2a0e9d90f1715d32cb98b0738695ca17ba324095fb2497ed7fbd59/httpx2-2.13.1.tar.gz", hash = "sha256:e48744a19e3af5ee48313d0ce5fe941d5422fae5705ea922a4aabf94d7800dfa", size = 100405, upload-time = "2026-09-23T07:47:23.052Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/9c/6fe8931fd9f381042a9e4c7d5a7b4cbf7016b252bec0c99a49fce42c3326/httpx2-2.13.1-py3-none-any.whl", hash = "sha256:6dff50fabc270ee5fd25d845d0b078ed20564579744d6d962850975996d2f9a4", size = 95597, upload-time = "2026-09-23T07:47:20.995Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "identify" version = "2.6.19" @@ -844,7 +884,7 @@ wheels = [ [[package]] name = "launchdarkly-ai-claude-agents" -version = "0.2.3" +version = "0.2.4" source = { editable = "packages/claude-agents" } dependencies = [ { name = "anthropic" }, @@ -880,7 +920,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-langchain-agents" -version = "0.2.3" +version = "0.2.4" source = { editable = "packages/langchain-agents" } dependencies = [ { name = "langchain-core" }, @@ -916,7 +956,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-openai-agents" -version = "0.2.3" +version = "0.2.4" source = { editable = "packages/openai-agents" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -952,7 +992,7 @@ requires-dist = [ [[package]] name = "launchdarkly-ai-python" -version = "0.1.7" +version = "0.1.8" source = { editable = "packages/ai" } dependencies = [ { name = "launchdarkly-ai-server" }, @@ -972,7 +1012,7 @@ provides-extras = ["otel"] [[package]] name = "launchdarkly-ai-server" -version = "0.2.3" +version = "0.2.4" source = { editable = "packages/client" } dependencies = [ { name = "opentelemetry-api" }, @@ -992,6 +1032,23 @@ requires-dist = [ ] provides-extras = ["otel"] +[[package]] +name = "launchdarkly-ai-typesafe" +version = "0.1.0" +source = { editable = "packages/typesafe" } +dependencies = [ + { name = "launchdarkly-ai-server" }, + { name = "opentelemetry-api" }, + { name = "typesafe-sdk" }, +] + +[package.metadata] +requires-dist = [ + { name = "launchdarkly-ai-server", editable = "packages/client" }, + { name = "opentelemetry-api", specifier = ">=1.25" }, + { name = "typesafe-sdk", specifier = ">=0.6" }, +] + [[package]] name = "launchdarkly-eventsource" version = "1.7.1" @@ -2209,6 +2266,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d8/8e/bb97bb0c71802080bfc8952937d174e49cfc50de5c951dd47b2496f0dcdb/tqdm-4.68.3-py3-none-any.whl", hash = "sha256:39832cc2def2789a6f29df83f172db7416cea70052c0907a57801c5f2fdccb03", size = 78337, upload-time = "2026-06-17T07:36:50.132Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + +[[package]] +name = "typesafe-sdk" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "httpx2" }, + { name = "pydantic" }, + { name = "pydantic-core" }, + { name = "tenacity" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/ac/8bad2c02c038b85039cdb198ba106650d54bade0c0c986c68dc6e9cef8c3/typesafe_sdk-0.7.2.tar.gz", hash = "sha256:2434f4af5cdfc45d6ab5ff27048e8b257207f786b60456e69995e345b32e77e1", size = 23169, upload-time = "2026-09-26T21:20:24.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/5e/2b97ba9b0b77612e3a480ddb60c0c0325b4809ef57a7b565341d65ef7bec/typesafe_sdk-0.7.2-py3-none-any.whl", hash = "sha256:0a961148187d52e18276ed7f2d02617cfac48e3b97673cb631a8749397d43d1e", size = 36448, upload-time = "2026-09-26T21:20:23.511Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0"