fix(api)!: represent multi-graders as named grader maps

.castiron.stats.yml

@@ -1,8 +1,8 @@
 schema_version: 1
-generation_id: cf94c82f-0604-4dfc-aff0-cec482e84953
-openapi_spec_hash: ce912f79d7f8cb19a7a515513c214b8a
-openapi_transformed_spec_hash: 352b5a7a028f990b6475b6593e381eee
-config_hash: dde98d9c41cd6f4163320f92c70f83e6
-codegen_sha: 326831af0c0fdf859335a189df2b6716e4b605d4
-codegen_hash: 00e7f6a53e1a6a1adf02385bc368fce999fdd775d1f1c28615f790c7a4109ad0
-public_codegen_sha: c2abe7c282c83c8fb9a7330254235c20b67a2b10
+generation_id: b6102435-2392-433a-b905-ce6c6297f1de
+openapi_spec_hash: a7dc885441a2a4a3afa82352ab442ed0
+openapi_transformed_spec_hash: 6c704acfb38ca363f191d758b4da5f20
+config_hash: fbf4242a57b6c023ef78a63466ab5c6d
+codegen_sha: ab0dea4dcd308905cb0d4e7ccabec1be9d006249
+codegen_hash: 4d85fb2e9b0d0f3408e7546585bb3f43c93799f9a4f3759e08a20dc294d30479
+public_codegen_sha: 7d887b1b5dc07c1dfa91236a0f89b1dd0c1e141e

api_reference/openapi.transformed.yml

@@ -42203,12 +42203,23 @@ components:
           type: string
           description: The name of the grader.
         graders:
-          anyOf:
-          - $ref: '#/components/schemas/GraderStringCheck'
-          - $ref: '#/components/schemas/GraderTextSimilarity'
-          - $ref: '#/components/schemas/GraderPython'
-          - $ref: '#/components/schemas/GraderScoreModel'
-          - $ref: '#/components/schemas/GraderLabelModel'
+          type: object
+          minProperties: 1
+          additionalProperties:
+            discriminator:
+              propertyName: type
+              mapping:
+                string_check: '#/components/schemas/GraderStringCheck'
+                text_similarity: '#/components/schemas/GraderTextSimilarity'
+                python: '#/components/schemas/GraderPython'
+                score_model: '#/components/schemas/GraderScoreModel'
+                label_model: '#/components/schemas/GraderLabelModel'
+            anyOf:
+            - $ref: '#/components/schemas/GraderStringCheck'
+            - $ref: '#/components/schemas/GraderTextSimilarity'
+            - $ref: '#/components/schemas/GraderPython'
+            - $ref: '#/components/schemas/GraderScoreModel'
+            - $ref: '#/components/schemas/GraderLabelModel'
         calculate_output:
           type: string
           description: A formula to calculate the output based on grader results.
@@ -42224,23 +42235,23 @@ components:
           {
             "type": "multi",
             "name": "example multi grader",
-            "graders": [
-              {
+            "graders": {
+              "text_similarity_score": {
                 "type": "text_similarity",
                 "name": "example text similarity grader",
                 "input": "The graded text",
                 "reference": "The reference text",
                 "evaluation_metric": "fuzzy_match"
               },
-              {
+              "string_check_score": {
                 "type": "string_check",
                 "name": "Example string check grader",
                 "input": "{{sample.output_text}}",
                 "reference": "{{item.label}}",
                 "operation": "eq"
               }
-            ],
-            "calculate_output": "0.5 * text_similarity_score +  0.5 * string_check_score)"
+            },
+            "calculate_output": "0.5 * text_similarity_score + 0.5 * string_check_score"
           }
     GraderPython:
       type: object

lib/openai/models/graders/multi_grader.rb

@@ -11,11 +11,9 @@ class MultiGrader < OpenAI::Internal::Type::BaseModel
         required :calculate_output, String
 
         # @!attribute graders
-        #   A StringCheckGrader object that performs a string comparison between input and
-        #   reference using a specified operation.
         #
-        #   @return [OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::Graders::TextSimilarityGrader, OpenAI::Models::Graders::PythonGrader, OpenAI::Models::Graders::ScoreModelGrader, OpenAI::Models::Graders::LabelModelGrader]
-        required :graders, union: -> { OpenAI::Graders::MultiGrader::Graders }
+        #   @return [Hash{Symbol=>OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::Graders::TextSimilarityGrader, OpenAI::Models::Graders::PythonGrader, OpenAI::Models::Graders::ScoreModelGrader, OpenAI::Models::Graders::LabelModelGrader}]
+        required :graders, -> { OpenAI::Internal::Type::HashOf[union: OpenAI::Graders::MultiGrader::Grader] }
 
         # @!attribute name
         #   The name of the grader.
@@ -36,9 +34,7 @@ class MultiGrader < OpenAI::Internal::Type::BaseModel
         #   @param calculate_output [String]
         #     A formula to calculate the output based on grader results.
         #
-        #   @param graders [OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::Graders::TextSimilarityGrader, OpenAI::Models::Graders::PythonGrader, OpenAI::Models::Graders::ScoreModelGrader, OpenAI::Models::Graders::LabelModelGrader]
-        #     A StringCheckGrader object that performs a string comparison between input and
-        #     reference using a specified operation.
+        #   @param graders [Hash{Symbol=>OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::Graders::TextSimilarityGrader, OpenAI::Models::Graders::PythonGrader, OpenAI::Models::Graders::ScoreModelGrader, OpenAI::Models::Graders::LabelModelGrader}]
         #
         #   @param name [String]
         #     The name of the grader.
@@ -48,26 +44,26 @@ class MultiGrader < OpenAI::Internal::Type::BaseModel
 
         # A StringCheckGrader object that performs a string comparison between input and
         # reference using a specified operation.
-        #
-        # @see OpenAI::Models::Graders::MultiGrader#graders
-        module Graders
+        module Grader
           extend OpenAI::Internal::Type::Union
 
+          discriminator :type
+
           # A StringCheckGrader object that performs a string comparison between input and reference using a specified operation.
-          variant -> { OpenAI::Graders::StringCheckGrader }
+          variant :string_check, -> { OpenAI::Graders::StringCheckGrader }
 
           # A TextSimilarityGrader object which grades text based on similarity metrics.
-          variant -> { OpenAI::Graders::TextSimilarityGrader }
+          variant :text_similarity, -> { OpenAI::Graders::TextSimilarityGrader }
 
           # A PythonGrader object that runs a python script on the input.
-          variant -> { OpenAI::Graders::PythonGrader }
+          variant :python, -> { OpenAI::Graders::PythonGrader }
 
           # A ScoreModelGrader object that uses a model to assign a score to the input.
-          variant -> { OpenAI::Graders::ScoreModelGrader }
+          variant :score_model, -> { OpenAI::Graders::ScoreModelGrader }
 
           # A LabelModelGrader object which uses a model to assign labels to each item
           # in the evaluation.
-          variant -> { OpenAI::Graders::LabelModelGrader }
+          variant :label_model, -> { OpenAI::Graders::LabelModelGrader }
 
           # @!method self.variants
           #   @return [Array(OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::Graders::TextSimilarityGrader, OpenAI::Models::Graders::PythonGrader, OpenAI::Models::Graders::ScoreModelGrader, OpenAI::Models::Graders::LabelModelGrader)]

lib/openai/resources/admin/organization.rb

@@ -4,9 +4,6 @@ module OpenAI
   module Resources
     class Admin
       class Organization
-        # @return [OpenAI::Resources::Admin::Organization::ExternalStorage]
-        attr_reader :external_storage
-
         # List user actions and configuration changes within this organization.
         # @return [OpenAI::Resources::Admin::Organization::AuditLogs]
         attr_reader :audit_logs
@@ -32,6 +29,9 @@ class Organization
         # @return [OpenAI::Resources::Admin::Organization::DataRetention]
         attr_reader :data_retention
 
+        # @return [OpenAI::Resources::Admin::Organization::ExternalStorage]
+        attr_reader :external_storage
+
         # @return [OpenAI::Resources::Admin::Organization::SpendLimit]
         attr_reader :spend_limit
 
@@ -49,7 +49,6 @@ class Organization
         # @param client [OpenAI::Client]
         def initialize(client:)
           @client = client
-          @external_storage = OpenAI::Resources::Admin::Organization::ExternalStorage.new(client: client)
           @audit_logs = OpenAI::Resources::Admin::Organization::AuditLogs.new(client: client)
           @admin_api_keys = OpenAI::Resources::Admin::Organization::AdminAPIKeys.new(client: client)
           @usage = OpenAI::Resources::Admin::Organization::Usage.new(client: client)
@@ -58,6 +57,7 @@ def initialize(client:)
           @groups = OpenAI::Resources::Admin::Organization::Groups.new(client: client)
           @roles = OpenAI::Resources::Admin::Organization::Roles.new(client: client)
           @data_retention = OpenAI::Resources::Admin::Organization::DataRetention.new(client: client)
+          @external_storage = OpenAI::Resources::Admin::Organization::ExternalStorage.new(client: client)
           @spend_limit = OpenAI::Resources::Admin::Organization::SpendLimit.new(client: client)
           @spend_alerts = OpenAI::Resources::Admin::Organization::SpendAlerts.new(client: client)
           @certificates = OpenAI::Resources::Admin::Organization::Certificates.new(client: client)

lib/openai/resources/safety.rb

@@ -3,19 +3,19 @@
 module OpenAI
   module Resources
     class Safety
-      # @return [OpenAI::Resources::Safety::Alerts]
-      attr_reader :alerts
-
       # @return [OpenAI::Resources::Safety::Cases]
       attr_reader :cases
 
+      # @return [OpenAI::Resources::Safety::Alerts]
+      attr_reader :alerts
+
       # @api private
       #
       # @param client [OpenAI::Client]
       def initialize(client:)
         @client = client
-        @alerts = OpenAI::Resources::Safety::Alerts.new(client: client)
         @cases = OpenAI::Resources::Safety::Cases.new(client: client)
+        @alerts = OpenAI::Resources::Safety::Alerts.new(client: client)
       end
     end
   end

rbi/openai/models/graders/multi_grader.rbi

@@ -20,17 +20,18 @@ module OpenAI
         sig { returns(String) }
         attr_accessor :calculate_output
 
-        # A StringCheckGrader object that performs a string comparison between input and
-        # reference using a specified operation.
         sig {
           returns(
-            T.any(
-              OpenAI::Graders::StringCheckGrader,
-              OpenAI::Graders::TextSimilarityGrader,
-              OpenAI::Graders::PythonGrader,
-              OpenAI::Graders::ScoreModelGrader,
-              OpenAI::Graders::LabelModelGrader
-            )
+            T::Hash[
+              Symbol,
+              T.any(
+                OpenAI::Graders::StringCheckGrader,
+                OpenAI::Graders::TextSimilarityGrader,
+                OpenAI::Graders::PythonGrader,
+                OpenAI::Graders::ScoreModelGrader,
+                OpenAI::Graders::LabelModelGrader
+              )
+            ]
           )
         }
         attr_accessor :graders
@@ -50,13 +51,16 @@ module OpenAI
 
             calculate_output: String,
 
-            graders: T.any(
-              OpenAI::Graders::StringCheckGrader::OrHash,
-              OpenAI::Graders::TextSimilarityGrader::OrHash,
-              OpenAI::Graders::PythonGrader::OrHash,
-              OpenAI::Graders::ScoreModelGrader::OrHash,
-              OpenAI::Graders::LabelModelGrader::OrHash
-            ),
+            graders: T::Hash[
+              Symbol,
+              T.any(
+                OpenAI::Graders::StringCheckGrader::OrHash,
+                OpenAI::Graders::TextSimilarityGrader::OrHash,
+                OpenAI::Graders::PythonGrader::OrHash,
+                OpenAI::Graders::ScoreModelGrader::OrHash,
+                OpenAI::Graders::LabelModelGrader::OrHash
+              )
+            ],
 
             name: String,
 
@@ -69,8 +73,6 @@ module OpenAI
           # A formula to calculate the output based on grader results.
           calculate_output:,
 
-          # A StringCheckGrader object that performs a string comparison between input and
-          # reference using a specified operation.
           graders:,
 
           # The name of the grader.
@@ -86,13 +88,16 @@ module OpenAI
           override.returns(
             {
               calculate_output: String,
-              graders: T.any(
-                OpenAI::Graders::StringCheckGrader,
-                OpenAI::Graders::TextSimilarityGrader,
-                OpenAI::Graders::PythonGrader,
-                OpenAI::Graders::ScoreModelGrader,
-                OpenAI::Graders::LabelModelGrader
-              ),
+              graders: T::Hash[
+                Symbol,
+                T.any(
+                  OpenAI::Graders::StringCheckGrader,
+                  OpenAI::Graders::TextSimilarityGrader,
+                  OpenAI::Graders::PythonGrader,
+                  OpenAI::Graders::ScoreModelGrader,
+                  OpenAI::Graders::LabelModelGrader
+                )
+              ],
               name: String,
               type: Symbol
             }
@@ -103,7 +108,7 @@ module OpenAI
 
         # A StringCheckGrader object that performs a string comparison between input and
         # reference using a specified operation.
-        module Graders
+        module Grader
           extend OpenAI::Internal::Type::Union
 
           Variants = T.type_alias {
@@ -116,7 +121,7 @@ module OpenAI
             )
           }
 
-          sig { override.returns(T::Array[OpenAI::Graders::MultiGrader::Graders::Variants]) }
+          sig { override.returns(T::Array[OpenAI::Graders::MultiGrader::Grader::Variants]) }
           def self.variants
           end
 

rbi/openai/resources/admin/organization.rbi

@@ -7,9 +7,6 @@ module OpenAI
 
       class Organization
 
-        sig { returns(OpenAI::Resources::Admin::Organization::ExternalStorage) }
-        attr_reader :external_storage
-
         # List user actions and configuration changes within this organization.
         sig { returns(OpenAI::Resources::Admin::Organization::AuditLogs) }
         attr_reader :audit_logs
@@ -35,6 +32,9 @@ module OpenAI
         sig { returns(OpenAI::Resources::Admin::Organization::DataRetention) }
         attr_reader :data_retention
 
+        sig { returns(OpenAI::Resources::Admin::Organization::ExternalStorage) }
+        attr_reader :external_storage
+
         sig { returns(OpenAI::Resources::Admin::Organization::SpendLimit) }
         attr_reader :spend_limit
 

rbi/openai/resources/safety.rbi

@@ -5,12 +5,12 @@ module OpenAI
 
     class Safety
 
-      sig { returns(OpenAI::Resources::Safety::Alerts) }
-      attr_reader :alerts
-
       sig { returns(OpenAI::Resources::Safety::Cases) }
       attr_reader :cases
 
+      sig { returns(OpenAI::Resources::Safety::Alerts) }
+      attr_reader :alerts
+
       # @api private
       sig { params(client: OpenAI::Client).returns(T.attached_class) }
       def self.new(client:)

sig/openai/models/graders/multi_grader.rbs

@@ -6,7 +6,7 @@ module OpenAI
       type multi_grader =
         {
           calculate_output: String,
-          graders: OpenAI::Models::Graders::MultiGrader::graders,
+          graders: ::Hash[Symbol, OpenAI::Models::Graders::MultiGrader::grader],
           name: String,
           ?type: :multi
         }
@@ -15,9 +15,7 @@ module OpenAI
         # A formula to calculate the output based on grader results.
         attr_accessor calculate_output: String
 
-        # A StringCheckGrader object that performs a string comparison between input and
-        # reference using a specified operation.
-        attr_accessor graders: OpenAI::Models::Graders::MultiGrader::graders
+        attr_accessor graders: ::Hash[Symbol, OpenAI::Models::Graders::MultiGrader::grader]
 
         # The name of the grader.
         attr_accessor name: String
@@ -27,38 +25,35 @@ module OpenAI
 
         # @param calculate_output
         #   A formula to calculate the output based on grader results.
-        # @param graders
-        #   A StringCheckGrader object that performs a string comparison between input and
-        #   reference using a specified operation.
         # @param name
         #   The name of the grader.
         # @param type
         #   The object type, which is always `multi`.
         def initialize: (
           calculate_output: String,
-          graders: OpenAI::Models::Graders::MultiGrader::graders,
+          graders: ::Hash[Symbol, OpenAI::Models::Graders::MultiGrader::grader],
           name: String,
           ?type: :multi
         ) -> void
 
         def to_hash: -> {
           calculate_output: String,
-          graders: OpenAI::Models::Graders::MultiGrader::graders,
+          graders: ::Hash[Symbol, OpenAI::Models::Graders::MultiGrader::grader],
           name: String,
           ?type: :multi
         }
 
-        type graders =
+        type grader =
           OpenAI::Graders::StringCheckGrader
           | OpenAI::Graders::TextSimilarityGrader
           | OpenAI::Graders::PythonGrader
           | OpenAI::Graders::ScoreModelGrader
           | OpenAI::Graders::LabelModelGrader
 
-        module Graders
+        module Grader
           extend OpenAI::Internal::Type::Union
 
-          def self?.variants: -> ::Array[OpenAI::Models::Graders::MultiGrader::graders]
+          def self?.variants: -> ::Array[OpenAI::Models::Graders::MultiGrader::grader]
         end
       end
     end