Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions Sources/AnyLanguageModel/Models/OpenAILanguageModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,10 @@ public struct OpenAILanguageModel: LanguageModel, ProfileRequestIdentifiedLangua
/// The default base URL for OpenAI's API.
public static let defaultBaseURL = URL(string: "https://api.openai.com/v1/")!

/// A request body, its keys sorted: the same tools and messages encode to the same bytes
/// every time, so a provider's prompt cache, which matches an exact prefix (OpenAI's, Azure's),
/// can hit. Unsorted, dictionary order differs per request.
/// Encodes a request body with its object keys sorted at every level,
/// so the same request encodes to the same bytes every time.
/// This keeps the prompt prefix stable,
/// which lets a provider's prompt cache hit when it matches on that prefix.
static func encodeBody(_ value: some Encodable) throws -> Data {
let encoder = JSONEncoder()
encoder.outputFormatting = .sortedKeys
Expand Down
6 changes: 0 additions & 6 deletions Tests/AnyLanguageModelTests/OpenAILanguageModelTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,6 @@ struct OpenAILanguageModelTests {
}
}

@Test func requestBodyKeysAreSorted() throws {
let value = ["zeta": 1, "alpha": 2, "mu": 3, "beta": 4, "omega": 5]
let body = try OpenAILanguageModel.encodeBody(value)
#expect(String(decoding: body, as: UTF8.self) == #"{"alpha":2,"beta":4,"mu":3,"omega":5,"zeta":1}"#)
}

@Suite("OpenAILanguageModel Chat Completions API", .enabled(if: openaiAPIKey?.isEmpty == false))
struct ChatCompletionsTests {
private let apiKey = openaiAPIKey!
Expand Down
54 changes: 54 additions & 0 deletions Tests/AnyLanguageModelTests/OpenAIToolCallHistoryTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -702,6 +702,60 @@ import Testing
}
}

@Test(arguments: [OpenAILanguageModel.APIVariant.chatCompletions, .responses], [false, true])
func requestBodyKeysAreSortedAtEveryLevel(
variant: OpenAILanguageModel.APIVariant,
streaming: Bool
) async throws {
let completed = responsesResponse.replacingOccurrences(of: "\n", with: " ")
let responsesStream =
#"data: {"type": "response.output_text.delta", "delta": "Done."}"# + "\n\n"
+ #"data: {"type": "response.completed", "response": \#(completed)}"# + "\n\n"
let reply =
switch (variant, streaming) {
case (.chatCompletions, false): chatCompletion
case (.chatCompletions, true): textStream("Done.")
case (.responses, false): responsesResponse
case (.responses, true): responsesStream
}
ToolCallHistoryURLProtocol.reset(responses: [reply])
let model = OpenAILanguageModel(
apiKey: "test",
model: "test",
apiVariant: variant,
transport: ToolCallHistoryURLProtocol.makeSession()
)
var options = GenerationOptions()
options[custom: OpenAILanguageModel.self] = .init(
extraBody: [
"vendor_options": .object([
"zeta": .int(1),
"alpha": .object(["omega": .bool(true), "beta": .string("b")]),
"mu": .array([.object(["y": .null, "b": .double(0.5)])]),
])
]
)
let session = LanguageModelSession(model: model, tools: [ReorderedWeatherTool()])
if streaming {
_ = try await session.streamResponse(to: "Weather in Paris?", options: options).collect()
} else {
_ = try await session.respond(to: "Weather in Paris?", options: options)
}

let body = try #require(ToolCallHistoryURLProtocol.recordedBodies.last)
// The body has the nested objects whose key order the check covers.
let json = try #require(try JSONSerialization.jsonObject(with: body) as? [String: Any])
#expect(json["vendor_options"] is [String: Any])
let tools = try #require(json["tools"] as? [[String: Any]])
#expect(tools.count == 1)
// Re-encoding the body with sorted keys changes no byte only if every object,
// at every level, already has its keys sorted.
let encoder = JSONEncoder()
encoder.outputFormatting = .sortedKeys
let sorted = try encoder.encode(try JSONDecoder().decode(JSONValue.self, from: body))
#expect(String(decoding: body, as: UTF8.self) == String(decoding: sorted, as: UTF8.self))
}

@Test(arguments: Transcript.Reasoning.Origin.allCases.filter { $0 != .chatCompletions })
func otherProvidersReasoningIsNotSent(origin: Transcript.Reasoning.Origin) async throws {
ToolCallHistoryURLProtocol.reset(responses: [chatCompletion])
Expand Down
Loading