From 778459fc5910e7b54b2f0e74c7dcdb6d968d6800 Mon Sep 17 00:00:00 2001 From: ZeroPoint95 <329227198+ZeroPoint95@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:09:41 -0400 Subject: [PATCH 1/2] fix: price qualified shell models on their selected service --- cmd/codeaf/carried.go | 15 +- cmd/codeaf/carried_price_test.go | 249 ++++++++++++++++++++++++++++++ internal/manual/chat/delegates.md | 4 +- 3 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 cmd/codeaf/carried_price_test.go diff --git a/cmd/codeaf/carried.go b/cmd/codeaf/carried.go index 821540366b..73054a6a65 100644 --- a/cmd/codeaf/carried.go +++ b/cmd/codeaf/carried.go @@ -171,7 +171,7 @@ func profileRoad() (carriedRoad, error) { settings.Models.Warmed(context.Background()) return session.ResolveProgramShellModels(word, v3TaskModels(settings.Models)(), sources) }, - modelPrice: settings.Models.PriceNow, + modelPrice: func(model string) (float64, float64, bool) { return carriedModelPrice(settings, model) }, defaultSeat: func() (string, error) { // A shell run needs the profile's work seat only when nobody pinned // a model for this invocation. A fresh profile can still use --high. @@ -212,6 +212,19 @@ func profileRoad() (carriedRoad, error) { }, nil } +// carriedModelPrice prices the same service and bare id the adapter calls. +// A connection qualifier is not a vendor prefix; only the selected default +// service owns the shared catalog. Other services keep the unpriced bound +// rather than borrowing an OpenRouter tariff for a coincidentally equal id. +func carriedModelPrice(settings config.Config, model string) (float64, float64, bool) { + model, _ = roles.SplitEffort(model) + service, bare := settings.Sources.OrDefault(settings.APIKey, settings.BaseURL).For(model) + if !strings.EqualFold(strings.TrimSpace(service.Source.ID), modelsource.DefaultID) { + return 0, 0, false + } + return settings.Models.PriceNow(bare) +} + // carriedAdapters is one adapter per model a shell run's program asks for, // built once and kept for the run. type carriedAdapters struct { diff --git a/cmd/codeaf/carried_price_test.go b/cmd/codeaf/carried_price_test.go new file mode 100644 index 0000000000..ecc72e47e4 --- /dev/null +++ b/cmd/codeaf/carried_price_test.go @@ -0,0 +1,249 @@ +package main + +import ( + "context" + "fmt" + "io" + "math" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/agentfield/sdk/go/ai" + "github.com/Agent-Field/codeaf/internal/catalog" + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/modelsource" + "github.com/Agent-Field/codeaf/internal/provider" + "github.com/Agent-Field/codeaf/internal/provider/modelapi" +) + +// Keep this on the shell's actual profile road: the adapter already strips +// the connection and effort, but admission used to price the unsplit word. +func pricedProfileRoad(t *testing.T) carriedRoad { + t.Helper() + profile := t.TempDir() + t.Setenv("CODEAF_HOME", profile) + t.Setenv(config.ProfileDirEnv, profile) + t.Setenv("OPENROUTER_API_KEY", "sk-or-v1-fixture") + t.Setenv("CODEAF_BASE_URL", config.DefaultBaseURL) + options := catalog.Options{Dir: profile, BaseURL: config.DefaultBaseURL} + if err := catalog.Remember(options, []catalog.Model{ + {ID: "~fixture/floating-latest", PromptPrice: 0.000001, CompletionPrice: 0.000002}, + {ID: "vendor/native", PromptPrice: 0.000003, CompletionPrice: 0.000004}, + {ID: "vendor/unpriced", PriceUnknown: true}, + }); err != nil { + t.Fatal(err) + } + old := sharedCatalog + sharedCatalog = func(config.Config) *catalog.Catalog { return catalog.Recall(options) } + t.Cleanup(func() { sharedCatalog = old }) + road, err := profileRoad() + if err != nil { + t.Fatal(err) + } + return road +} + +func TestShellPricingUsesTheSelectedServiceWithoutStrippingVendorIdentity(t *testing.T) { + road := pricedProfileRoad(t) + // The road's catalog can be handed to a differently ordered set without + // changing any rows. A matching id on a direct service is still unpriced. + settings, err := config.Load() + if err != nil { + t.Fatal(err) + } + settings.Models = sharedCatalog(settings) + custom := modelsource.Connected{Source: modelsource.Source{ID: "custom", Written: "private"}} + defaultService := modelsource.Connected{Source: modelsource.DefaultSource(config.DefaultBaseURL)} + for _, tc := range []struct { + name, model string + sources modelsource.Set + known bool + }{ + {"qualified-direct", "private/vendor/native", modelsource.NewSet(defaultService, custom), false}, + {"direct-default", "vendor/native", modelsource.NewSet(custom, defaultService), false}, + {"explicit-openrouter", "openrouter/vendor/native", modelsource.NewSet(custom, defaultService), true}, + {"unconnected-prefix", "private/vendor/native", modelsource.NewSet(defaultService), false}, + {"vendor-prefix", "vendor/native", modelsource.NewSet(defaultService), true}, + {"openrouter-vendor-on-direct", "openrouter/vendor/native", modelsource.NewSet(custom), false}, + } { + t.Run(tc.name, func(t *testing.T) { + settings.Sources = tc.sources + in, out, known := carriedModelPrice(settings, tc.model) + if known != tc.known || (known && (in != 0.000003 || out != 0.000004)) || (!known && (in != 0 || out != 0)) { + t.Fatalf("price(%q) = (%g, %g, %v), known want %v", tc.model, in, out, known, tc.known) + } + }) + } + if _, _, known := road.modelPrice("native"); known { + t.Fatal("a vendor suffix alone borrowed a listed model's price") + } +} + +type pricingCompleter func(context.Context, []ai.Message, ...ai.Option) (*ai.Response, error) + +func (f pricingCompleter) CompleteWithMessages(ctx context.Context, messages []ai.Message, options ...ai.Option) (*ai.Response, error) { + return f(ctx, messages, options...) +} + +// A tiny positive receipt used to make two half-ceiling reservations no +// longer fit. Exercise the actual API and road, with controlled completions, +// so recognizing a price proves admission and settlement rather than only +// string normalization. Unknown models must still be held behind a call. +func TestShellQualifiedPricingRestoresAdmissionAndKeepsUnknownBounds(t *testing.T) { + for _, tc := range []struct { + name, model string + held bool + }{ + {"canonical", "~fixture/floating-latest:low", false}, + {"qualified", "openrouter/~fixture/floating-latest:low", false}, + {"unknown", "openrouter/vendor/missing:low", true}, + {"unpublished-price", "openrouter/vendor/unpriced", true}, + } { + t.Run(tc.name, func(t *testing.T) { + road := pricedProfileRoad(t) + started := make(chan struct{}, 2) + release := make(chan struct{}) + var released sync.Once + releaseCalls := func() { released.Do(func() { close(release) }) } + server, err := modelapi.Open(modelapi.Config{Ceiling: 0.25, ModelPrice: road.modelPrice, + Serves: func(string) bool { return true }, + CompleterFor: func(model string) modelapi.Completer { + return pricingCompleter(func(ctx context.Context, messages []ai.Message, _ ...ai.Option) (*ai.Response, error) { + if messages[0].Content[0].Text == "hold" { + started <- struct{}{} + select { + case <-release: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + if sink := provider.BillingSinkFrom(ctx); sink != nil { + sink(provider.Billed{Model: model, PromptTokens: 1, CompletionTokens: 1, Cost: 0.01}) + } + return &ai.Response{Model: model, Choices: []ai.Choice{{Message: ai.Message{Role: "assistant", Content: []ai.ContentPart{{Type: "text", Text: "ok"}}}, FinishReason: "stop"}}}, nil + }) + }, + }) + if err != nil { + t.Fatal(err) + } + defer server.Close() + // Unblock upstreams before closing the server, even on a failed assertion. + defer releaseCalls() + api := server.API() + type reply struct { + status int + held string + err error + } + post := func(model, message string, cap int) reply { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + body := fmt.Sprintf(`{"model":%q,"messages":[{"role":"user","content":%q}],"max_completion_tokens":%d}`, model, message, cap) + request, err := http.NewRequestWithContext(ctx, http.MethodPost, modelapi.ChatURL(api.BaseURL), strings.NewReader(body)) + if err != nil { + return reply{err: err} + } + request.Header.Set("Authorization", "Bearer "+api.Token) + request.Header.Set("Content-Type", "application/json") + response, err := http.DefaultClient.Do(request) + if err != nil { + return reply{err: err} + } + defer response.Body.Close() + _, err = io.Copy(io.Discard, response.Body) + return reply{response.StatusCode, response.Header.Get(modelapi.HeldHeader), err} + } + if got := post("~fixture/floating-latest", "seed", 1); got.err != nil || got.status != 200 { + t.Fatalf("seed = %+v", got) + } + first, second := make(chan reply, 1), make(chan reply, 1) + go func() { first <- post(tc.model, "hold", 1) }() + select { + case <-started: + case got := <-first: + t.Fatalf("first call did not start: %+v", got) + case <-time.After(3 * time.Second): + t.Fatal("first upstream did not start") + } + go func() { second <- post(tc.model, "hold", 1) }() + if tc.held { + select { + case got := <-second: + if got.err != nil || got.status != 429 || got.held != "ceiling" { + t.Fatalf("unknown second = %+v", got) + } + case <-started: + t.Fatal("an unknown price admitted both calls despite recorded spend") + case <-time.After(3 * time.Second): + t.Fatal("unknown call was not held") + } + } else { + select { + case <-started: + case got := <-second: + t.Fatalf("priced second call did not start: %+v", got) + case <-time.After(3 * time.Second): + t.Fatal("priced second upstream did not start") + } + } + // Both admitted calls can now settle. + releaseCalls() + if got := <-first; got.err != nil || got.status != 200 { + t.Fatalf("first = %+v", got) + } + if !tc.held { + if got := <-second; got.err != nil || got.status != 200 { + t.Fatalf("second = %+v", got) + } + } + wantSpent := 0.03 + if tc.held { + wantSpent = 0.02 + } + if got := server.Spent(); math.Abs(got-wantSpent) > 1e-9 { + t.Fatalf("spent = %g, want %g", got, wantSpent) + } + wantHeld := 0 + if tc.held { + wantHeld = 1 + } + if server.HeldAtCeiling() != wantHeld { + t.Fatalf("held = %d, want %d", server.HeldAtCeiling(), wantHeld) + } + // A known, oversized reservation remains refused before upstream. + if got := post("openrouter/~fixture/floating-latest:low", "too expensive", 200000); got.err != nil || got.status != 402 { + t.Fatalf("oversized priced call = %+v", got) + } + if server.RefusedAtCeiling() != 1 || math.Abs(server.Spent()-wantSpent) > 1e-9 { + t.Fatal("refusal forwarded a call or changed spend") + } + }) + } +} + +func TestShellPricesCanonicalAndQualifiedModelsOnTheSameProfileRoad(t *testing.T) { + road := pricedProfileRoad(t) + for _, tc := range []struct { + model string + in, out float64 + known bool + }{ + {"~fixture/floating-latest", 0.000001, 0.000002, true}, + {"~fixture/floating-latest:low", 0.000001, 0.000002, true}, + {"openrouter/~fixture/floating-latest:low", 0.000001, 0.000002, true}, + {"openrouter/vendor/native", 0.000003, 0.000004, true}, + {"vendor/native", 0.000003, 0.000004, true}, + {"vendor/missing", 0, 0, false}, + {"openrouter/vendor/missing:low", 0, 0, false}, + {"openrouter/vendor/unpriced", 0, 0, false}, + } { + in, out, known := road.modelPrice(tc.model) + if in != tc.in || out != tc.out || known != tc.known { + t.Errorf("price(%q) = (%g, %g, %v), want (%g, %g, %v)", tc.model, in, out, known, tc.in, tc.out, tc.known) + } + } +} diff --git a/internal/manual/chat/delegates.md b/internal/manual/chat/delegates.md index ba3041488e..c638496280 100644 --- a/internal/manual/chat/delegates.md +++ b/internal/manual/chat/delegates.md @@ -168,7 +168,9 @@ An open chat's `/budget conversation` change binds its next proposal, run and tu as soon as the setting receipt appears. Before forwarding a call, codeaf reserves the larger estimate from the requested model and its possible fallback seat when both have known prices, using input size and output -cap; if either price is unknown, it uses the unpriced bound. It refuses a call whose +cap. A shell model's connection prefix and thinking level do not change its +published price on that service; another service never borrows OpenRouter's price. +If either price is unknown, it uses the unpriced bound. It refuses a call whose estimate would cross the ceiling, and a call that would fit once calls already in flight have ended (a program that makes many calls at once, as sec does) is asked to wait for them instead and goes on when one ends. An From c3ed9dbdbc4b791a23bf53a8e982a001eceb7bc6 Mon Sep 17 00:00:00 2001 From: ZeroPoint95 <329227198+ZeroPoint95@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:22:27 -0400 Subject: [PATCH 2/2] docs: record qualified shell model pricing for #1819 --- .../1819-qualified-shell-model-pricing.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/changes/unreleased/1819-qualified-shell-model-pricing.md diff --git a/docs/changes/unreleased/1819-qualified-shell-model-pricing.md b/docs/changes/unreleased/1819-qualified-shell-model-pricing.md new file mode 100644 index 0000000000..d1b2bf70cb --- /dev/null +++ b/docs/changes/unreleased/1819-qualified-shell-model-pricing.md @@ -0,0 +1,13 @@ +--- +kind: fixed +title: shell programs price qualified model names on their selected service +pr: 1819 +surface: [engine, docs] +invalidates: + - "Shell admission priced connection-qualified model words as unknown even when OpenRouter published their tariff. It now resolves the connection and thinking level the same way the provider adapter does." + - "A matching bare id on another connected service could borrow the shared OpenRouter tariff. Other services now retain the conservative unknown-price bound." +--- + +Unknown and unpublished prices still use the existing conservative reservation. +Receipt settlement, fallback reservation policy, and the run's cost and time ceilings are +unchanged.