feat(api): add agent turn items and usage source grouping

.castiron.stats.yml

@@ -1,8 +1,8 @@
 schema_version: 1
-generation_id: 4a45859f-2ddc-4ae8-8d48-7d66ac4e16fe
-openapi_spec_hash: 8aa23d19137079c724365bd6cadd0858
-openapi_transformed_spec_hash: d591fedadfdd68837534b2574782bf81
-config_hash: 70e6e763ed8ac8c4038de193a96c771d
-codegen_sha: 460e3ec26544481b476774331018f126c57454d6
-codegen_hash: c35eb5de9f3bd898d3bf8745619d12d9ffcc5bc4e3ca332d978e6b42813243fd
-public_codegen_sha: 0f5e1cc6093a03afec0cc7ad21d18921436e173f
+generation_id: b5e92b57-640d-4fdf-b97b-b5e824207bf5
+openapi_spec_hash: 0824fadcd64901a2a03bb44fd10bfa80
+openapi_transformed_spec_hash: 794495a7295d5744dbec1d4557949fcd
+config_hash: 10ea2c74c82b64939d70ba94d3ed6f57
+codegen_sha: bdad42acfefe450652583d12afe24bf6f44d83bc
+codegen_hash: 234c93f7f62b5b847336b7ad8fbe7cf7e5df4d9fffff6acc00abb0bb23e20ed9
+public_codegen_sha: c834768776c04fcd3c11594e9d9be6ae95572fee

openai-java-core/src/main/kotlin/com/openai/helpers/ChatCompletionAccumulator.kt

@@ -56,6 +56,29 @@ class ChatCompletionAccumulator private constructor() {
      */
     private val messageRefusals = mutableMapOf<Long, StringBuilder>()
 
+    private val messageAudio = mutableMapOf<Long, AudioState>()
+
+    private class AudioState {
+        var id: String? = null
+        var expiresAt: Long? = null
+        var data: StringBuilder? = null
+        var transcript: StringBuilder? = null
+        val additionalProperties = mutableMapOf<String, JsonValue>()
+
+        fun isComplete() = id != null && expiresAt != null && data != null && transcript != null
+
+        fun build(): ChatCompletionAudio {
+            if (!isComplete()) throw OpenAIInvalidDataException("Incomplete streamed audio.")
+            return ChatCompletionAudio.builder()
+                .id(id!!)
+                .data(data.toString())
+                .transcript(transcript.toString())
+                .expiresAt(expiresAt!!)
+                .putAllAdditionalProperties(additionalProperties)
+                .build()
+        }
+    }
+
     /**
      * The builders for the [ChatCompletion.Choice.Logprobs] of each choice. These are only
      * accumulated if the option to report log probabilities was enabled in the request. When the
@@ -143,11 +166,12 @@ class ChatCompletionAccumulator private constructor() {
      * Accumulates a streamed chunk and uses it to construct a [ChatCompletion]. When all chunks
      * have been accumulated, the chat completion can be retrieved by calling [chatCompletion].
      *
-     * The last chunk is identified as that which provides the `finishReason`. At that point, the
-     * [ChatCompletion] is created and available. However, if the request was configured to include
-     * the usage details, one additional chunk may be accumulated which provides those usage details
-     * and the [ChatCompletion] will be recreated to reflect them. After that, no more chunks of any
-     * kind may be accumulated.
+     * The last chunk provides a `finishReason`, or an expiry-only audio update completing the
+     * accumulated audio. A finish reason received before the complete audio is retained. At that
+     * point, the [ChatCompletion] is created and available. However, if the request was configured
+     * to include the usage details, one additional chunk may be accumulated which provides those
+     * usage details and the [ChatCompletion] will be recreated to reflect them. After that, no more
+     * chunks of any kind may be accumulated.
      *
      * @return The given [chunk] for convenience, such as when chaining method calls.
      * @throws IllegalStateException If [accumulate] is called again after the last chunk has been
@@ -219,20 +243,27 @@ class ChatCompletionAccumulator private constructor() {
                 // chunks have been received, presumptively build the `ChatCompletion`. (One further
                 // usage chunk _might_ be notified.)
                 isFinished[index] = true
-            } else {
+            } else if (isTerminalAudioUpdate(choice) && messageAudio[index]?.isComplete() == true) {
+                if (isFinished[index] != true) {
+                    choiceBuilder.finishReason(ChatCompletion.Choice.FinishReason.STOP)
+                }
+                isFinished[index] = true
+            } else if (isFinished[index] != true) {
                 choiceBuilder.finishReason(JsonNull.of())
             }
         }
 
         if (
-            chunk.choices().any { it.finishReason().isPresent } &&
-                choiceBuilders.keys.all { isFinished[it] == true }
+            chunk.choices().any { it.finishReason().isPresent || isTerminalAudioUpdate(it) } &&
+                choiceBuilders.keys.all { isFinished[it] == true } &&
+                messageAudio.values.all { it.isComplete() }
         ) {
             chatCompletion = chatCompletionBuilder.choices(buildChoices()).build()
             // Release mutable storage only after the whole chunk and final build succeed.
             toolCallFunctionArgs.clear()
             messageContents.clear()
             messageRefusals.clear()
+            messageAudio.clear()
         }
 
         return chunk
@@ -270,6 +301,21 @@ class ChatCompletionAccumulator private constructor() {
     internal fun accumulateMessage(index: Long, delta: ChatCompletionChunk.Choice.Delta) {
         val messageBuilder = messageBuilders.getOrPut(index) { ChatCompletionMessage.builder() }
 
+        delta.audio().ifPresent { audio ->
+            val state = messageAudio.getOrPut(index) { AudioState() }
+            audio.id().ifPresent { state.id = it }
+            audio.expiresAt().ifPresent { state.expiresAt = it }
+            audio.data().ifPresent {
+                val buffer = state.data ?: StringBuilder().also { state.data = it }
+                buffer.append(it)
+            }
+            audio.transcript().ifPresent {
+                val buffer = state.transcript ?: StringBuilder().also { state.transcript = it }
+                buffer.append(it)
+            }
+            state.additionalProperties.putAll(audio._additionalProperties())
+        }
+
         delta.content().ifPresent { messageContents.getOrPut(index) { StringBuilder() }.append(it) }
         delta.refusal().ifPresent { messageRefusals.getOrPut(index) { StringBuilder() }.append(it) }
         // The `role` defaults to "assistant", so if no other `role` is set on the delta, there is
@@ -320,6 +366,14 @@ class ChatCompletionAccumulator private constructor() {
     ): ChatCompletionChunk.Choice.Delta.ToolCall.Function =
         function.orElseThrow { OpenAIInvalidDataException("Tool call chunk missing function.") }
 
+    private fun isTerminalAudioUpdate(choice: ChatCompletionChunk.Choice): Boolean =
+        choice.delta().audio().getOrNull()?.let {
+            it.expiresAt().isPresent &&
+                !it.id().isPresent &&
+                !it.data().isPresent &&
+                !it.transcript().isPresent
+        } ?: false
+
     private fun buildChoices() =
         choiceBuilders.entries
             .sortedBy { it.key }
@@ -338,6 +392,7 @@ class ChatCompletionAccumulator private constructor() {
             .content(messageContents[index]?.toString())
             .refusal(messageRefusals[index]?.toString())
             .toolCalls(buildToolCalls(index))
+            .apply { messageAudio[index]?.let { audio(it.build()) } }
             .build()
 
     private fun buildToolCalls(index: Long): List<ChatCompletionMessageToolCall> =

openai-java-core/src/main/kotlin/com/openai/models/ErrorObject.kt

@@ -308,6 +308,7 @@ private constructor(
     private constructor(
         private val detailedExplanation: JsonField<String>,
         private val errorType: JsonField<ErrorType>,
+        private val reviewTarget: JsonField<String>,
         private val steer: JsonField<Steer>,
         private val additionalProperties: MutableMap<String, JsonValue>,
     ) {
@@ -320,8 +321,11 @@ private constructor(
             @JsonProperty("error_type")
             @ExcludeMissing
             errorType: JsonField<ErrorType> = JsonMissing.of(),
+            @JsonProperty("review_target")
+            @ExcludeMissing
+            reviewTarget: JsonField<String> = JsonMissing.of(),
             @JsonProperty("steer") @ExcludeMissing steer: JsonField<Steer> = JsonMissing.of(),
-        ) : this(detailedExplanation, errorType, steer, mutableMapOf())
+        ) : this(detailedExplanation, errorType, reviewTarget, steer, mutableMapOf())
 
         /**
          * The public explanation for this block.
@@ -340,6 +344,14 @@ private constructor(
          */
         fun errorType(): Optional<ErrorType> = errorType.getOptional("error_type")
 
+        /**
+         * An opaque target for explicitly continuing this review, or null when unavailable.
+         *
+         * @throws OpenAIInvalidDataException if the JSON field has an unexpected type (e.g. if the
+         *   server responded with an unexpected value).
+         */
+        fun reviewTarget(): Optional<String> = reviewTarget.getOptional("review_target")
+
         /**
          * An optional public continuation instruction.
          *
@@ -367,6 +379,16 @@ private constructor(
         @ExcludeMissing
         fun _errorType(): JsonField<ErrorType> = errorType
 
+        /**
+         * Returns the raw JSON value of [reviewTarget].
+         *
+         * Unlike [reviewTarget], this method doesn't throw if the JSON field has an unexpected
+         * type.
+         */
+        @JsonProperty("review_target")
+        @ExcludeMissing
+        fun _reviewTarget(): JsonField<String> = reviewTarget
+
         /**
          * Returns the raw JSON value of [steer].
          *
@@ -397,13 +419,15 @@ private constructor(
 
             private var detailedExplanation: JsonField<String> = JsonMissing.of()
             private var errorType: JsonField<ErrorType> = JsonMissing.of()
+            private var reviewTarget: JsonField<String> = JsonMissing.of()
             private var steer: JsonField<Steer> = JsonMissing.of()
             private var additionalProperties: MutableMap<String, JsonValue> = mutableMapOf()
 
             @JvmSynthetic
             internal fun from(misalignment: Misalignment) = apply {
                 detailedExplanation = misalignment.detailedExplanation
                 errorType = misalignment.errorType
+                reviewTarget = misalignment.reviewTarget
                 steer = misalignment.steer
                 additionalProperties = misalignment.additionalProperties.toMutableMap()
             }
@@ -444,6 +468,25 @@ private constructor(
              */
             fun errorType(value: String) = errorType(ErrorType.of(value))
 
+            /** An opaque target for explicitly continuing this review, or null when unavailable. */
+            fun reviewTarget(reviewTarget: String?) =
+                reviewTarget(JsonField.ofNullable(reviewTarget))
+
+            /** Alias for calling [Builder.reviewTarget] with `reviewTarget.orElse(null)`. */
+            fun reviewTarget(reviewTarget: Optional<String>) =
+                reviewTarget(reviewTarget.getOrNull())
+
+            /**
+             * Sets [Builder.reviewTarget] to an arbitrary JSON value.
+             *
+             * You should usually call [Builder.reviewTarget] with a well-typed [String] value
+             * instead. This method is primarily for setting the field to an undocumented or not yet
+             * supported value.
+             */
+            fun reviewTarget(reviewTarget: JsonField<String>) = apply {
+                this.reviewTarget = reviewTarget
+            }
+
             /** An optional public continuation instruction. */
             fun steer(steer: Steer) = steer(JsonField.of(steer))
 
@@ -484,6 +527,7 @@ private constructor(
                 Misalignment(
                     detailedExplanation,
                     errorType,
+                    reviewTarget,
                     steer,
                     additionalProperties.toMutableMap(),
                 )
@@ -507,6 +551,7 @@ private constructor(
 
             detailedExplanation()
             errorType()
+            reviewTarget()
             steer().ifPresent { it.validate() }
             validated = true
         }
@@ -529,6 +574,7 @@ private constructor(
         internal fun validity(): Int =
             (if (detailedExplanation.asKnown().isPresent) 1 else 0) +
                 (if (errorType.asKnown().isPresent) 1 else 0) +
+                (if (reviewTarget.asKnown().isPresent) 1 else 0) +
                 (steer.asKnown().getOrNull()?.validity() ?: 0)
 
         /** An optional classification; clients must accept additional values. */
@@ -875,18 +921,19 @@ private constructor(
             return other is Misalignment &&
                 detailedExplanation == other.detailedExplanation &&
                 errorType == other.errorType &&
+                reviewTarget == other.reviewTarget &&
                 steer == other.steer &&
                 additionalProperties == other.additionalProperties
         }
 
         private val hashCode: Int by lazy {
-            Objects.hash(detailedExplanation, errorType, steer, additionalProperties)
+            Objects.hash(detailedExplanation, errorType, reviewTarget, steer, additionalProperties)
         }
 
         override fun hashCode(): Int = hashCode
 
         override fun toString() =
-            "Misalignment{detailedExplanation=$detailedExplanation, errorType=$errorType, steer=$steer, additionalProperties=$additionalProperties}"
+            "Misalignment{detailedExplanation=$detailedExplanation, errorType=$errorType, reviewTarget=$reviewTarget, steer=$steer, additionalProperties=$additionalProperties}"
     }
 
     override fun equals(other: Any?): Boolean {