Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 33 additions & 7 deletions Sources/AnyLanguageModel/Models/GeminiLanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@ public struct GeminiLanguageModel: LanguageModel, ProfileRequestIdentifiedLangua
serverTools: effectiveServerTools
)
let params = try createGenerateContentParams(
systemInstruction: requestContext.transcript.geminiSystemInstruction,
contents: try geminiContents(
from: requestContext,
saying: schemaInstructions,
Expand Down Expand Up @@ -491,6 +492,7 @@ public struct GeminiLanguageModel: LanguageModel, ProfileRequestIdentifiedLangua
serverTools: effectiveServerTools
)
let params = try createGenerateContentParams(
systemInstruction: requestContext.transcript.geminiSystemInstruction,
contents: try geminiContents(
from: requestContext,
saying: schemaInstructions,
Expand Down Expand Up @@ -621,6 +623,7 @@ private func convertSchemaToGeminiFormat(_ schema: GenerationSchema) throws -> J
}

private func createGenerateContentParams<Content: Generable>(
systemInstruction: String?,
contents: [GeminiContent],
tools: [GeminiTool]?,
generating type: Content.Type,
Expand All @@ -633,6 +636,12 @@ private func createGenerateContentParams<Content: Generable>(
"contents": try JSONValue(contents)
]

if let systemInstruction {
params["systemInstruction"] = .object([
"parts": .array([.object(["text": .string(systemInstruction)])])
])
}

if let tools, !tools.isEmpty {
params["tools"] = try .array(tools.map { try $0.jsonValue })

Expand Down Expand Up @@ -872,17 +881,34 @@ private func geminiFunctionResponseData(from url: URL) throws -> GeminiInlineDat
// MARK: - Supporting Types

extension Transcript {
/// The text of the transcript's instructions, for the request's `systemInstruction` field,
/// or `nil` if it has none.
///
/// Gemini takes instructions in that field, outside `contents`,
/// so they don't go to the model as something the user said.
/// The field takes only text, so image segments are left out.
/// Each instructions entry is kept, in order.
fileprivate var geminiSystemInstruction: String? {
let text = compactMap { entry -> String? in
guard case .instructions(let instructions) = entry else { return nil }
return instructions.segments.compactMap { segment -> String? in
switch segment {
case .text(let text): text.content
case .structure(let structure): structure.content.jsonString
case .image: nil
}
}.joined(separator: "\n")
}.filter { !$0.isEmpty }.joined(separator: "\n")
return text.isEmpty ? nil : text
}

fileprivate func toGeminiContent() throws -> [GeminiContent] {
var messages = [GeminiContent]()
for item in geminiReplayEntries {
switch item {
case .instructions(let instructions):
messages.append(
.init(
role: .user,
parts: convertSegmentsToGeminiParts(instructions.segments)
)
)
case .instructions:
// Sent in the request's `systemInstruction` field; see `geminiSystemInstruction`.
continue
case .prompt(let prompt):
messages.append(
.init(
Expand Down
153 changes: 153 additions & 0 deletions Tests/AnyLanguageModelTests/GeminiSystemInstructionsTests.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
import Foundation
import Testing

@testable import AnyLanguageModel

/// Instructions go to Gemini in the request's `systemInstruction` field,
/// not in `contents` as something the user said.
@Suite("Gemini system instructions")
struct GeminiSystemInstructionsTests {
private static let instructions = "You are a helpful assistant. Be concise."

@Test(arguments: [false, true])
func instructionsAreSentAsSystemInstruction(streaming: Bool) async throws {
let transport = FakeTransport(replies: [try Self.reply(parts: [["text": "Hi"]], streaming: streaming)])
let session = LanguageModelSession(model: Self.model(transport), instructions: Self.instructions)

if streaming {
_ = try await session.streamResponse(to: "Hello").collect()
} else {
_ = try await session.respond(to: "Hello")
}

let body = try #require(try transport.bodies.map(Self.json).first)
#expect(Self.systemInstruction(of: body) == Self.instructions)
let contents = try #require(body["contents"] as? [[String: Any]])
#expect(contents.count == 1)
#expect(contents.first?["role"] as? String == "user")
#expect(Self.text(of: contents.first) == "Hello")
}

@Test(arguments: [false, true])
func eachToolRoundSendsInstructionsAsSystemInstruction(streaming: Bool) async throws {
let transport = FakeTransport(replies: [
try Self.reply(
parts: [["functionCall": ["name": "getWeather", "args": ["city": "Paris"]]]],
streaming: streaming
),
try Self.reply(parts: [["text": "Sunny"]], streaming: streaming),
])
let session = LanguageModelSession(
model: Self.model(transport),
tools: [WeatherTool()],
instructions: Self.instructions
)

if streaming {
_ = try await session.streamResponse(to: "Hello").collect()
} else {
_ = try await session.respond(to: "Hello")
}

let bodies = try transport.bodies.map(Self.json)
try #require(bodies.count == 2)
for body in bodies {
#expect(Self.systemInstruction(of: body) == Self.instructions)
let contents = try #require(body["contents"] as? [[String: Any]])
#expect(contents.first?["role"] as? String == "user")
#expect(Self.text(of: contents.first) == "Hello")
#expect(!contents.contains { Self.text(of: $0) == Self.instructions })
}
}

@Test(arguments: [false, true])
func structuredGenerationSendsInstructionsAsSystemInstruction(streaming: Bool) async throws {
let transport = FakeTransport(replies: [
try Self.reply(parts: [["text": #"{"forecast":"Sunny"}"#]], streaming: streaming)
])
let session = LanguageModelSession(model: Self.model(transport), instructions: Self.instructions)

if streaming {
_ = try await session.streamResponse(to: "Hello", generating: Forecast.self).collect()
} else {
_ = try await session.respond(to: "Hello", generating: Forecast.self)
}

let body = try #require(try transport.bodies.map(Self.json).first)
#expect(Self.systemInstruction(of: body) == Self.instructions)
let contents = try #require(body["contents"] as? [[String: Any]])
#expect(contents.first?["role"] as? String == "user")
#expect(Self.text(of: contents.first)?.hasPrefix("Hello") == true)
#expect(!contents.contains { Self.text(of: $0) == Self.instructions })
}

@Test func instructionsEntriesAreJoinedWithoutImages() async throws {
let transport = FakeTransport(replies: [try Self.reply(parts: [["text": "Hi"]], streaming: false)])
let image = Transcript.ImageSegment(data: Data([0x89, 0x50, 0x4E, 0x47]), mimeType: "image/png")
let transcript = Transcript(entries: [
.instructions(
.init(
segments: [.text(.init(content: "First.")), .image(image), .text(.init(content: "Second."))],
toolDefinitions: []
)
),
.instructions(.init(segments: [.text(.init(content: "Third."))], toolDefinitions: [])),
])
let session = LanguageModelSession(model: Self.model(transport), transcript: transcript)

_ = try await session.respond(to: "Hello")

let body = try #require(try transport.bodies.map(Self.json).first)
#expect(Self.systemInstruction(of: body) == "First.\nSecond.\nThird.")
let contents = try #require(body["contents"] as? [[String: Any]])
#expect(contents.count == 1)
#expect(Self.text(of: contents.first) == "Hello")
}

@Test func noInstructionsOmitsSystemInstruction() async throws {
let transport = FakeTransport(replies: [try Self.reply(parts: [["text": "Hi"]], streaming: false)])
let session = LanguageModelSession(model: Self.model(transport))

_ = try await session.respond(to: "Hello")

let body = try #require(try transport.bodies.map(Self.json).first)
#expect(body["systemInstruction"] == nil)
let contents = try #require(body["contents"] as? [[String: Any]])
#expect(contents.count == 1)
#expect(Self.text(of: contents.first) == "Hello")
}

// MARK: Fixtures

@Generable
struct Forecast {
var forecast: String
}

private static func model(_ transport: FakeTransport) -> GeminiLanguageModel {
GeminiLanguageModel(apiKey: "test", model: "test", transport: transport)
}

private static func json(_ data: Data) throws -> [String: Any] {
try #require(try JSONSerialization.jsonObject(with: data) as? [String: Any])
}

/// The joined text parts of a content.
private static func text(of content: [String: Any]?) -> String? {
guard let parts = content?["parts"] as? [[String: Any]] else { return nil }
return parts.compactMap { $0["text"] as? String }.joined()
}

/// The joined text parts of a request's system instruction.
private static func systemInstruction(of body: [String: Any]) -> String? {
text(of: body["systemInstruction"] as? [String: Any])
}

private static func reply(parts: [[String: Any]], streaming: Bool) throws -> FakeTransport.Reply {
let object: [String: Any] = [
"candidates": [["content": ["role": "model", "parts": parts], "finishReason": "STOP"]]
]
let body = String(decoding: try JSONSerialization.data(withJSONObject: object), as: UTF8.self)
return .init(chunks: [streaming ? "data: \(body)\n\n" : body])
}
}
Loading