From 56d7ab34f60ce1fddced6824ed826eec10f5a45b Mon Sep 17 00:00:00 2001 From: mattt Date: Fri, 20 Nov 2026 02:19:54 -0800 Subject: [PATCH 1/2] Mark evaluations of the Foundation Models instructions adapter --- Sources/AnyLanguageModel/Models/SystemLanguageModel.swift | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift index a1e2fc81..e2036761 100644 --- a/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/SystemLanguageModel.swift @@ -398,8 +398,14 @@ struct FoundationModelsDynamicInstructionsAdapter: FoundationModels.DynamicInstructions { let session: LanguageModelSession + /// Whether the current task is evaluating instructions through an adapter's ``body``. + /// Tests read this to tell the evaluations that Foundation Models makes from any others. + @TaskLocal static var isEvaluating = false + var body: some FoundationModels.DynamicInstructions { - let requestContext = session.resolvedRequestContext() + let requestContext = Self.$isEvaluating.withValue(true) { + session.resolvedRequestContext() + } if let instructions = requestContext.instructions { instructions.toFoundationModels() } From 51139bfb47d3f44a5d064821422a11e2e14b5ea2 Mon Sep 17 00:00:00 2001 From: mattt Date: Fri, 20 Nov 2026 02:19:54 -0800 Subject: [PATCH 2/2] Require native evaluations of dynamic instructions to come through the adapter --- .../NativeProfileHistoryTests.swift | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/Tests/AnyLanguageModelTests/NativeProfileHistoryTests.swift b/Tests/AnyLanguageModelTests/NativeProfileHistoryTests.swift index 99abfd56..cd3da325 100644 --- a/Tests/AnyLanguageModelTests/NativeProfileHistoryTests.swift +++ b/Tests/AnyLanguageModelTests/NativeProfileHistoryTests.swift @@ -592,9 +592,13 @@ import Testing @Test func ordinaryDynamicInstructionsAreEvaluatedOnlyByTheNativeSession() throws { guard #available(macOS 27.0, iOS 27.0, visionOS 27.0, *) else { return } let evaluations = Locked(0) + let nativeEvaluations = Locked(0) let session = AnyLanguageModel.LanguageModelSession( model: MockLanguageModel(), - dynamicInstructions: CountingProjectionInstructions(evaluations: evaluations) + dynamicInstructions: CountingProjectionInstructions( + evaluations: evaluations, + nativeEvaluations: nativeEvaluations + ) ) #expect(evaluations.withLock { $0 } == 0) _ = try makeFoundationModelsRequest( @@ -605,10 +609,13 @@ import Testing ) // Foundation Models can evaluate the instructions more than once // when it creates the native session; macOS 27.2 adds one evaluation. + // Every evaluation must come through the adapter. let evaluationsAfterRequest = evaluations.withLock { $0 } - #expect(evaluationsAfterRequest >= 1) + #expect(nativeEvaluations.withLock { $0 } >= 1) + #expect(nativeEvaluations.withLock { $0 } == evaluationsAfterRequest) _ = FoundationModelsDynamicInstructionsAdapter(session: session).body #expect(evaluations.withLock { $0 } == evaluationsAfterRequest + 1) + #expect(nativeEvaluations.withLock { $0 } == evaluationsAfterRequest + 1) } } @@ -932,12 +939,23 @@ import Testing private struct CountingProjectionInstructions: AnyLanguageModel.DynamicInstructions { let evaluations: Locked + /// The evaluations made through `FoundationModelsDynamicInstructionsAdapter.body`. + var nativeEvaluations: Locked? = nil var body: some AnyLanguageModel.DynamicInstructions { - let count = evaluations.withLock { + let count = recordEvaluation() + AnyLanguageModel.Instructions("Revision \(count)") + } + + private func recordEvaluation() -> Int { + if #available(macOS 27.0, iOS 27.0, visionOS 27.0, watchOS 27.0, *), + FoundationModelsDynamicInstructionsAdapter.isEvaluating + { + nativeEvaluations?.withLock { $0 += 1 } + } + return evaluations.withLock { $0 += 1; return $0 } - AnyLanguageModel.Instructions("Revision \(count)") } } #endif