From 34732dbf341e7029c4e0ba3d8468400ac2e33261 Mon Sep 17 00:00:00 2001 From: markstuart-oai <323302876+markstuart-oai@users.noreply.github.com> Date: Sun, 20 Sep 2026 17:44:33 +0000 Subject: [PATCH] fix(api)!: represent multi-graders as named grader maps Castiron-Internal-PR: https://github.com/openai/openai-ruby-internal/pull/129 Castiron-Source-SHA: 8ebce350cab9bb13fc02e62385fd567346d44d85 Castiron-Public-Base-SHA: 33435479c2d98603b93c810183d314bc55cd1ee7 --- .castiron.stats.yml | 14 +-- api_reference/openapi.transformed.yml | 33 ++++-- lib/openai/models/graders/multi_grader.rb | 26 ++-- lib/openai/resources/admin/organization.rb | 8 +- lib/openai/resources/safety.rb | 8 +- rbi/openai/models/graders/multi_grader.rbi | 59 +++++----- rbi/openai/resources/admin/organization.rbi | 6 +- rbi/openai/resources/safety.rbi | 6 +- sig/openai/models/graders/multi_grader.rbs | 19 ++- sig/openai/resources/admin/organization.rbs | 4 +- sig/openai/resources/safety.rbs | 4 +- test/openai/multi_grader_test.rb | 124 ++++++++++++++++++++ 12 files changed, 221 insertions(+), 90 deletions(-) create mode 100644 test/openai/multi_grader_test.rb diff --git a/.castiron.stats.yml b/.castiron.stats.yml index 516b8e5e0..d2c8f747e 100644 --- a/.castiron.stats.yml +++ b/.castiron.stats.yml @@ -1,8 +1,8 @@ schema_version: 1 -generation_id: cf94c82f-0604-4dfc-aff0-cec482e84953 -openapi_spec_hash: ce912f79d7f8cb19a7a515513c214b8a -openapi_transformed_spec_hash: 352b5a7a028f990b6475b6593e381eee -config_hash: dde98d9c41cd6f4163320f92c70f83e6 -codegen_sha: 326831af0c0fdf859335a189df2b6716e4b605d4 -codegen_hash: 00e7f6a53e1a6a1adf02385bc368fce999fdd775d1f1c28615f790c7a4109ad0 -public_codegen_sha: c2abe7c282c83c8fb9a7330254235c20b67a2b10 +generation_id: b6102435-2392-433a-b905-ce6c6297f1de +openapi_spec_hash: a7dc885441a2a4a3afa82352ab442ed0 +openapi_transformed_spec_hash: 6c704acfb38ca363f191d758b4da5f20 +config_hash: fbf4242a57b6c023ef78a63466ab5c6d +codegen_sha: ab0dea4dcd308905cb0d4e7ccabec1be9d006249 +codegen_hash: 4d85fb2e9b0d0f3408e7546585bb3f43c93799f9a4f3759e08a20dc294d30479 +public_codegen_sha: 7d887b1b5dc07c1dfa91236a0f89b1dd0c1e141e diff --git a/api_reference/openapi.transformed.yml b/api_reference/openapi.transformed.yml index 30f3e6581..12e5960e9 100644 --- a/api_reference/openapi.transformed.yml +++ b/api_reference/openapi.transformed.yml @@ -42203,12 +42203,23 @@ components: type: string description: The name of the grader. graders: - anyOf: - - $ref: '#/components/schemas/GraderStringCheck' - - $ref: '#/components/schemas/GraderTextSimilarity' - - $ref: '#/components/schemas/GraderPython' - - $ref: '#/components/schemas/GraderScoreModel' - - $ref: '#/components/schemas/GraderLabelModel' + type: object + minProperties: 1 + additionalProperties: + discriminator: + propertyName: type + mapping: + string_check: '#/components/schemas/GraderStringCheck' + text_similarity: '#/components/schemas/GraderTextSimilarity' + python: '#/components/schemas/GraderPython' + score_model: '#/components/schemas/GraderScoreModel' + label_model: '#/components/schemas/GraderLabelModel' + anyOf: + - $ref: '#/components/schemas/GraderStringCheck' + - $ref: '#/components/schemas/GraderTextSimilarity' + - $ref: '#/components/schemas/GraderPython' + - $ref: '#/components/schemas/GraderScoreModel' + - $ref: '#/components/schemas/GraderLabelModel' calculate_output: type: string description: A formula to calculate the output based on grader results. @@ -42224,23 +42235,23 @@ components: { "type": "multi", "name": "example multi grader", - "graders": [ - { + "graders": { + "text_similarity_score": { "type": "text_similarity", "name": "example text similarity grader", "input": "The graded text", "reference": "The reference text", "evaluation_metric": "fuzzy_match" }, - { + "string_check_score": { "type": "string_check", "name": "Example string check grader", "input": "{{sample.output_text}}", "reference": "{{item.label}}", "operation": "eq" } - ], - "calculate_output": "0.5 * text_similarity_score + 0.5 * string_check_score)" + }, + "calculate_output": "0.5 * text_similarity_score + 0.5 * string_check_score" } GraderPython: type: object diff --git a/lib/openai/models/graders/multi_grader.rb b/lib/openai/models/graders/multi_grader.rb index b118eb3c2..11d1889e1 100644 --- a/lib/openai/models/graders/multi_grader.rb +++ b/lib/openai/models/graders/multi_grader.rb @@ -11,11 +11,9 @@ class MultiGrader < OpenAI::Internal::Type::BaseModel required :calculate_output, String # @!attribute graders - # A StringCheckGrader object that performs a string comparison between input and - # reference using a specified operation. # - # @return [OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::Graders::TextSimilarityGrader, OpenAI::Models::Graders::PythonGrader, OpenAI::Models::Graders::ScoreModelGrader, OpenAI::Models::Graders::LabelModelGrader] - required :graders, union: -> { OpenAI::Graders::MultiGrader::Graders } + # @return [Hash{Symbol=>OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::Graders::TextSimilarityGrader, OpenAI::Models::Graders::PythonGrader, OpenAI::Models::Graders::ScoreModelGrader, OpenAI::Models::Graders::LabelModelGrader}] + required :graders, -> { OpenAI::Internal::Type::HashOf[union: OpenAI::Graders::MultiGrader::Grader] } # @!attribute name # The name of the grader. @@ -36,9 +34,7 @@ class MultiGrader < OpenAI::Internal::Type::BaseModel # @param calculate_output [String] # A formula to calculate the output based on grader results. # - # @param graders [OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::Graders::TextSimilarityGrader, OpenAI::Models::Graders::PythonGrader, OpenAI::Models::Graders::ScoreModelGrader, OpenAI::Models::Graders::LabelModelGrader] - # A StringCheckGrader object that performs a string comparison between input and - # reference using a specified operation. + # @param graders [Hash{Symbol=>OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::Graders::TextSimilarityGrader, OpenAI::Models::Graders::PythonGrader, OpenAI::Models::Graders::ScoreModelGrader, OpenAI::Models::Graders::LabelModelGrader}] # # @param name [String] # The name of the grader. @@ -48,26 +44,26 @@ class MultiGrader < OpenAI::Internal::Type::BaseModel # A StringCheckGrader object that performs a string comparison between input and # reference using a specified operation. - # - # @see OpenAI::Models::Graders::MultiGrader#graders - module Graders + module Grader extend OpenAI::Internal::Type::Union + discriminator :type + # A StringCheckGrader object that performs a string comparison between input and reference using a specified operation. - variant -> { OpenAI::Graders::StringCheckGrader } + variant :string_check, -> { OpenAI::Graders::StringCheckGrader } # A TextSimilarityGrader object which grades text based on similarity metrics. - variant -> { OpenAI::Graders::TextSimilarityGrader } + variant :text_similarity, -> { OpenAI::Graders::TextSimilarityGrader } # A PythonGrader object that runs a python script on the input. - variant -> { OpenAI::Graders::PythonGrader } + variant :python, -> { OpenAI::Graders::PythonGrader } # A ScoreModelGrader object that uses a model to assign a score to the input. - variant -> { OpenAI::Graders::ScoreModelGrader } + variant :score_model, -> { OpenAI::Graders::ScoreModelGrader } # A LabelModelGrader object which uses a model to assign labels to each item # in the evaluation. - variant -> { OpenAI::Graders::LabelModelGrader } + variant :label_model, -> { OpenAI::Graders::LabelModelGrader } # @!method self.variants # @return [Array(OpenAI::Models::Graders::StringCheckGrader, OpenAI::Models::Graders::TextSimilarityGrader, OpenAI::Models::Graders::PythonGrader, OpenAI::Models::Graders::ScoreModelGrader, OpenAI::Models::Graders::LabelModelGrader)] diff --git a/lib/openai/resources/admin/organization.rb b/lib/openai/resources/admin/organization.rb index 9325a4af9..6f76d6133 100644 --- a/lib/openai/resources/admin/organization.rb +++ b/lib/openai/resources/admin/organization.rb @@ -4,9 +4,6 @@ module OpenAI module Resources class Admin class Organization - # @return [OpenAI::Resources::Admin::Organization::ExternalStorage] - attr_reader :external_storage - # List user actions and configuration changes within this organization. # @return [OpenAI::Resources::Admin::Organization::AuditLogs] attr_reader :audit_logs @@ -32,6 +29,9 @@ class Organization # @return [OpenAI::Resources::Admin::Organization::DataRetention] attr_reader :data_retention + # @return [OpenAI::Resources::Admin::Organization::ExternalStorage] + attr_reader :external_storage + # @return [OpenAI::Resources::Admin::Organization::SpendLimit] attr_reader :spend_limit @@ -49,7 +49,6 @@ class Organization # @param client [OpenAI::Client] def initialize(client:) @client = client - @external_storage = OpenAI::Resources::Admin::Organization::ExternalStorage.new(client: client) @audit_logs = OpenAI::Resources::Admin::Organization::AuditLogs.new(client: client) @admin_api_keys = OpenAI::Resources::Admin::Organization::AdminAPIKeys.new(client: client) @usage = OpenAI::Resources::Admin::Organization::Usage.new(client: client) @@ -58,6 +57,7 @@ def initialize(client:) @groups = OpenAI::Resources::Admin::Organization::Groups.new(client: client) @roles = OpenAI::Resources::Admin::Organization::Roles.new(client: client) @data_retention = OpenAI::Resources::Admin::Organization::DataRetention.new(client: client) + @external_storage = OpenAI::Resources::Admin::Organization::ExternalStorage.new(client: client) @spend_limit = OpenAI::Resources::Admin::Organization::SpendLimit.new(client: client) @spend_alerts = OpenAI::Resources::Admin::Organization::SpendAlerts.new(client: client) @certificates = OpenAI::Resources::Admin::Organization::Certificates.new(client: client) diff --git a/lib/openai/resources/safety.rb b/lib/openai/resources/safety.rb index 901e5d385..0086eacf0 100644 --- a/lib/openai/resources/safety.rb +++ b/lib/openai/resources/safety.rb @@ -3,19 +3,19 @@ module OpenAI module Resources class Safety - # @return [OpenAI::Resources::Safety::Alerts] - attr_reader :alerts - # @return [OpenAI::Resources::Safety::Cases] attr_reader :cases + # @return [OpenAI::Resources::Safety::Alerts] + attr_reader :alerts + # @api private # # @param client [OpenAI::Client] def initialize(client:) @client = client - @alerts = OpenAI::Resources::Safety::Alerts.new(client: client) @cases = OpenAI::Resources::Safety::Cases.new(client: client) + @alerts = OpenAI::Resources::Safety::Alerts.new(client: client) end end end diff --git a/rbi/openai/models/graders/multi_grader.rbi b/rbi/openai/models/graders/multi_grader.rbi index 2df1dfa6f..86a5fbc8e 100644 --- a/rbi/openai/models/graders/multi_grader.rbi +++ b/rbi/openai/models/graders/multi_grader.rbi @@ -20,17 +20,18 @@ module OpenAI sig { returns(String) } attr_accessor :calculate_output - # A StringCheckGrader object that performs a string comparison between input and - # reference using a specified operation. sig { returns( - T.any( - OpenAI::Graders::StringCheckGrader, - OpenAI::Graders::TextSimilarityGrader, - OpenAI::Graders::PythonGrader, - OpenAI::Graders::ScoreModelGrader, - OpenAI::Graders::LabelModelGrader - ) + T::Hash[ + Symbol, + T.any( + OpenAI::Graders::StringCheckGrader, + OpenAI::Graders::TextSimilarityGrader, + OpenAI::Graders::PythonGrader, + OpenAI::Graders::ScoreModelGrader, + OpenAI::Graders::LabelModelGrader + ) + ] ) } attr_accessor :graders @@ -50,13 +51,16 @@ module OpenAI calculate_output: String, - graders: T.any( - OpenAI::Graders::StringCheckGrader::OrHash, - OpenAI::Graders::TextSimilarityGrader::OrHash, - OpenAI::Graders::PythonGrader::OrHash, - OpenAI::Graders::ScoreModelGrader::OrHash, - OpenAI::Graders::LabelModelGrader::OrHash - ), + graders: T::Hash[ + Symbol, + T.any( + OpenAI::Graders::StringCheckGrader::OrHash, + OpenAI::Graders::TextSimilarityGrader::OrHash, + OpenAI::Graders::PythonGrader::OrHash, + OpenAI::Graders::ScoreModelGrader::OrHash, + OpenAI::Graders::LabelModelGrader::OrHash + ) + ], name: String, @@ -69,8 +73,6 @@ module OpenAI # A formula to calculate the output based on grader results. calculate_output:, - # A StringCheckGrader object that performs a string comparison between input and - # reference using a specified operation. graders:, # The name of the grader. @@ -86,13 +88,16 @@ module OpenAI override.returns( { calculate_output: String, - graders: T.any( - OpenAI::Graders::StringCheckGrader, - OpenAI::Graders::TextSimilarityGrader, - OpenAI::Graders::PythonGrader, - OpenAI::Graders::ScoreModelGrader, - OpenAI::Graders::LabelModelGrader - ), + graders: T::Hash[ + Symbol, + T.any( + OpenAI::Graders::StringCheckGrader, + OpenAI::Graders::TextSimilarityGrader, + OpenAI::Graders::PythonGrader, + OpenAI::Graders::ScoreModelGrader, + OpenAI::Graders::LabelModelGrader + ) + ], name: String, type: Symbol } @@ -103,7 +108,7 @@ module OpenAI # A StringCheckGrader object that performs a string comparison between input and # reference using a specified operation. - module Graders + module Grader extend OpenAI::Internal::Type::Union Variants = T.type_alias { @@ -116,7 +121,7 @@ module OpenAI ) } - sig { override.returns(T::Array[OpenAI::Graders::MultiGrader::Graders::Variants]) } + sig { override.returns(T::Array[OpenAI::Graders::MultiGrader::Grader::Variants]) } def self.variants end diff --git a/rbi/openai/resources/admin/organization.rbi b/rbi/openai/resources/admin/organization.rbi index c5b58b4ff..d26de70ba 100644 --- a/rbi/openai/resources/admin/organization.rbi +++ b/rbi/openai/resources/admin/organization.rbi @@ -7,9 +7,6 @@ module OpenAI class Organization - sig { returns(OpenAI::Resources::Admin::Organization::ExternalStorage) } - attr_reader :external_storage - # List user actions and configuration changes within this organization. sig { returns(OpenAI::Resources::Admin::Organization::AuditLogs) } attr_reader :audit_logs @@ -35,6 +32,9 @@ module OpenAI sig { returns(OpenAI::Resources::Admin::Organization::DataRetention) } attr_reader :data_retention + sig { returns(OpenAI::Resources::Admin::Organization::ExternalStorage) } + attr_reader :external_storage + sig { returns(OpenAI::Resources::Admin::Organization::SpendLimit) } attr_reader :spend_limit diff --git a/rbi/openai/resources/safety.rbi b/rbi/openai/resources/safety.rbi index b8efbfae6..fe85088bc 100644 --- a/rbi/openai/resources/safety.rbi +++ b/rbi/openai/resources/safety.rbi @@ -5,12 +5,12 @@ module OpenAI class Safety - sig { returns(OpenAI::Resources::Safety::Alerts) } - attr_reader :alerts - sig { returns(OpenAI::Resources::Safety::Cases) } attr_reader :cases + sig { returns(OpenAI::Resources::Safety::Alerts) } + attr_reader :alerts + # @api private sig { params(client: OpenAI::Client).returns(T.attached_class) } def self.new(client:) diff --git a/sig/openai/models/graders/multi_grader.rbs b/sig/openai/models/graders/multi_grader.rbs index 5b3c36c2a..39284403d 100644 --- a/sig/openai/models/graders/multi_grader.rbs +++ b/sig/openai/models/graders/multi_grader.rbs @@ -6,7 +6,7 @@ module OpenAI type multi_grader = { calculate_output: String, - graders: OpenAI::Models::Graders::MultiGrader::graders, + graders: ::Hash[Symbol, OpenAI::Models::Graders::MultiGrader::grader], name: String, ?type: :multi } @@ -15,9 +15,7 @@ module OpenAI # A formula to calculate the output based on grader results. attr_accessor calculate_output: String - # A StringCheckGrader object that performs a string comparison between input and - # reference using a specified operation. - attr_accessor graders: OpenAI::Models::Graders::MultiGrader::graders + attr_accessor graders: ::Hash[Symbol, OpenAI::Models::Graders::MultiGrader::grader] # The name of the grader. attr_accessor name: String @@ -27,38 +25,35 @@ module OpenAI # @param calculate_output # A formula to calculate the output based on grader results. - # @param graders - # A StringCheckGrader object that performs a string comparison between input and - # reference using a specified operation. # @param name # The name of the grader. # @param type # The object type, which is always `multi`. def initialize: ( calculate_output: String, - graders: OpenAI::Models::Graders::MultiGrader::graders, + graders: ::Hash[Symbol, OpenAI::Models::Graders::MultiGrader::grader], name: String, ?type: :multi ) -> void def to_hash: -> { calculate_output: String, - graders: OpenAI::Models::Graders::MultiGrader::graders, + graders: ::Hash[Symbol, OpenAI::Models::Graders::MultiGrader::grader], name: String, ?type: :multi } - type graders = + type grader = OpenAI::Graders::StringCheckGrader | OpenAI::Graders::TextSimilarityGrader | OpenAI::Graders::PythonGrader | OpenAI::Graders::ScoreModelGrader | OpenAI::Graders::LabelModelGrader - module Graders + module Grader extend OpenAI::Internal::Type::Union - def self?.variants: -> ::Array[OpenAI::Models::Graders::MultiGrader::graders] + def self?.variants: -> ::Array[OpenAI::Models::Graders::MultiGrader::grader] end end end diff --git a/sig/openai/resources/admin/organization.rbs b/sig/openai/resources/admin/organization.rbs index fcedc6219..589bd92cb 100644 --- a/sig/openai/resources/admin/organization.rbs +++ b/sig/openai/resources/admin/organization.rbs @@ -2,8 +2,6 @@ module OpenAI module Resources class Admin class Organization - attr_reader external_storage: OpenAI::Resources::Admin::Organization::ExternalStorage - attr_reader audit_logs: OpenAI::Resources::Admin::Organization::AuditLogs attr_reader admin_api_keys: OpenAI::Resources::Admin::Organization::AdminAPIKeys @@ -20,6 +18,8 @@ module OpenAI attr_reader data_retention: OpenAI::Resources::Admin::Organization::DataRetention + attr_reader external_storage: OpenAI::Resources::Admin::Organization::ExternalStorage + attr_reader spend_limit: OpenAI::Resources::Admin::Organization::SpendLimit attr_reader spend_alerts: OpenAI::Resources::Admin::Organization::SpendAlerts diff --git a/sig/openai/resources/safety.rbs b/sig/openai/resources/safety.rbs index 8569b7e90..9b9257931 100644 --- a/sig/openai/resources/safety.rbs +++ b/sig/openai/resources/safety.rbs @@ -1,10 +1,10 @@ module OpenAI module Resources class Safety - attr_reader alerts: OpenAI::Resources::Safety::Alerts - attr_reader cases: OpenAI::Resources::Safety::Cases + attr_reader alerts: OpenAI::Resources::Safety::Alerts + def initialize: (client: OpenAI::Client) -> void end end diff --git a/test/openai/multi_grader_test.rb b/test/openai/multi_grader_test.rb new file mode 100644 index 000000000..e29ff827b --- /dev/null +++ b/test/openai/multi_grader_test.rb @@ -0,0 +1,124 @@ +# frozen_string_literal: true + +require_relative "test_helper" + +class OpenAI::Test::MultiGraderTest < Minitest::Test + extend Minitest::Serial + include WebMock::API + + def before_all + super + WebMock.enable! + end + + def after_all + WebMock.disable! + super + end + + def teardown + WebMock.reset! + super + end + + def test_coerces_named_values_with_string_and_symbol_keys + [grader_values, grader_values.transform_keys(&:to_s)].each do |graders| + payload = multi_payload.merge(graders: graders) + state = OpenAI::Internal::Type::Converter.new_coerce_state + parsed = OpenAI::Internal::Type::Converter.coerce(OpenAI::Graders::MultiGrader, payload, state: state) + + assert_nil(state.fetch(:error)) + assert_named_graders(parsed) + assert_equal(JSON.parse(JSON.generate(payload)), JSON.parse(parsed.to_json)) + end + end + + def test_validate_and_run_send_named_graders + raw_graders = JSON.parse(JSON.generate(grader_values)) + multi = OpenAI::Graders::MultiGrader.new(**multi_payload.merge(graders: raw_graders)) + wire = JSON.parse(JSON.generate(multi_payload)) + validate_request = stub_request(:post, "http://localhost/fine_tuning/alpha/graders/validate") + .with { |request| JSON.parse(request.body) == {"grader" => wire} } + .to_return_json(body: {grader: wire}) + run_request = stub_request(:post, "http://localhost/fine_tuning/alpha/graders/run") + .with { |request| JSON.parse(request.body) == {"grader" => wire, "model_sample" => "synthetic"} } + .to_return_json(body: {reward: 1.0, sub_rewards: {}, metadata: {}, model_grader_token_usage_per_model: {}}) + client = OpenAI::Client.new(base_url: "http://localhost", api_key: "synthetic", max_retries: 0) + + assert_same(raw_graders, multi.graders) + + response = client.fine_tuning.alpha.graders.validate(grader: multi) + client.fine_tuning.alpha.graders.run(grader: multi, model_sample: "synthetic") + + assert_instance_of(OpenAI::Graders::MultiGrader, response.grader) + assert_named_graders(response.grader) + assert_equal(wire, JSON.parse(response.grader.to_json)) + assert_requested(validate_request, times: 1) + assert_requested(run_request, times: 1) + end + + def test_flat_and_array_graders_do_not_coerce_as_a_dictionary + standalone = grader_values.fetch(:compliant) + [standalone, [standalone]].each do |invalid| + parsed = OpenAI::Internal::Type::Converter.coerce( + OpenAI::Graders::MultiGrader, + multi_payload.merge(graders: invalid) + ) + + assert_raises(OpenAI::Errors::ConversionError) { parsed.graders } + end + end + + private + + def assert_named_graders(multi) + expected = { + compliant: OpenAI::Graders::StringCheckGrader, + similarity: OpenAI::Graders::TextSimilarityGrader, + python_score: OpenAI::Graders::PythonGrader, + model_score: OpenAI::Graders::ScoreModelGrader, + label_score: OpenAI::Graders::LabelModelGrader + } + + assert_equal(expected.keys.sort, multi.graders.keys.sort) + expected.each { |key, model| assert_instance_of(model, multi.graders.fetch(key)) } + end + + def multi_payload + {type: "multi", name: "Combined", calculate_output: "0.5 * compliant", graders: grader_values} + end + + def grader_values + { + compliant: { + type: "string_check", + name: "A display name, not the formula key", + input: "{{sample.output_json.compliant}}", + reference: "{{item.compliant}}", + operation: "eq" + }, + similarity: { + type: "text_similarity", + name: "Similarity", + input: "text", + reference: "text", + evaluation_metric: "fuzzy_match" + }, + python_score: {type: "python", name: "Python", source: "def grade(sample, item):\n return 1.0"}, + model_score: { + type: "score_model", + name: "Score", + model: "synthetic-model", + input: [{role: "user", content: "Score the sample"}] + }, + label_score: { + type: "label_model", + name: "Label", + model: "synthetic-model", + input: [{role: "user", content: "Label the sample"}], + labels: ["pass", "fail"], + passing_labels: ["pass"] + } + } + end +end