Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 72 additions & 0 deletions .github/workflows/model-metadata-sync.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
name: Model metadata sync

on:
schedule:
# Keep newly released models from waiting for an unrelated code change.
- cron: '43 4 * * *'
workflow_dispatch:

permissions:
contents: write
pull-requests: write

concurrency:
group: model-metadata-sync
cancel-in-progress: true

jobs:
sync:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Check out the repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0

- name: Set up Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: '24'

- name: Install validation dependencies
run: npm ci --ignore-scripts

- name: Refresh models.dev facts
run: npm run sync:model-metadata

- id: changes
name: Detect catalog changes
run: |
if git diff --quiet; then
echo "changed=false" >> "$GITHUB_OUTPUT"
else
echo "changed=true" >> "$GITHUB_OUTPUT"
fi

- name: Validate generated catalog contracts
if: steps.changes.outputs.changed == 'true'
run: |
npm --workspace @maka/core run build
node -e "import('./packages/core/dist/provider-registry.js')"

- name: Open or update the sync pull request
if: steps.changes.outputs.changed == 'true'
env:
GH_TOKEN: ${{ github.token }}
BRANCH: automation/models-dev-sync
run: |
git config user.name github-actions[bot]
git config user.email 41898282+github-actions[bot]@users.noreply.github.com
git switch -C "$BRANCH"
git add packages/core/src/model-metadata.generated.ts packages/runtime/src/telemetry/model-pricing.generated.ts
git commit -m "chore(models): refresh models.dev metadata"
git push --force origin "HEAD:$BRANCH"

if [[ "$(gh pr list --head "$BRANCH" --state open --json number --jq length)" == "0" ]]; then
gh pr create \

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] Bind the existing-PR lookup to this repository branch

gh pr list --head "$BRANCH" filters by branch name and does not support an owner-qualified head. A fork PR whose head is also named automation/models-dev-sync therefore makes this count nonzero; the workflow then force-pushes the repository branch but skips gh pr create, leaving that branch without its own sync PR. Query REST or GraphQL with the repository owner as part of the head identity (and exclude cross-repository PRs) before deciding that the automation PR already exists.

--base main \
--head "$BRANCH" \
--title "chore(models): refresh models.dev metadata" \
--body "Automated refresh of bundled model capabilities and pricing from models.dev."
fi
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
"astryx:theme": "node scripts/build-astryx-theme.mjs",
"astryx:surface-inventory": "node scripts/check-astryx-surface-inventory.mjs",
"astryx:surface-inventory:write": "node scripts/generate-astryx-surface-inventory.mjs",
"sync:model-metadata": "node scripts/sync-model-metadata.mjs",
"sync:model-metadata": "node scripts/sync-model-metadata.mjs && biome format --write --files-max-size=5000000 packages/core/src/model-metadata.generated.ts packages/runtime/src/telemetry/model-pricing.generated.ts",
"generate:bundled-skills": "node scripts/gen-bundled-skill-catalog.mjs",
"cost:deepseek-baseline": "node scripts/deepseek-live-cost-baseline.mjs",
"computer-use": "node scripts/computer-use.mjs",
Expand Down
4 changes: 3 additions & 1 deletion packages/core/src/__tests__/model-metadata.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,11 @@ describe('openAiAdapterApiProtocol', () => {
assert.equal(openAiAdapterApiProtocol('gpt-4o'), 'openai-chat');
});

it('routes only xAI Grok 4.5 through Responses', () => {
it('routes xAI Responses-generation Grok models by family contract', () => {
assert.equal(openAiAdapterApiProtocol('grok-4.5', 'xai'), 'openai-responses');
assert.equal(openAiAdapterApiProtocol('grok-4.5', 'xai-oauth'), 'openai-responses');
assert.equal(openAiAdapterApiProtocol('grok-4.6', 'xai'), 'openai-responses');
assert.equal(openAiAdapterApiProtocol('grok-5', 'xai'), 'openai-responses');
assert.equal(openAiAdapterApiProtocol('grok-4.3', 'xai'), 'openai-chat');
assert.equal(openAiAdapterApiProtocol('grok-4.5', 'openai'), 'openai-chat');
});
Expand Down
29 changes: 29 additions & 0 deletions packages/core/src/__tests__/model-thinking.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,39 @@ import {
resolveThinkingLevel,
deriveThinkingChoices,
thinkingOptionsForModel,
thinkingOptionsForConnection,
thinkingVariantsForConnection,
thinkingVariantsForModel,
} from '../model-thinking.js';

test('provider inventory reasoning controls outrank the bundled snapshot', () => {
const connection = {
providerType: 'github-copilot',
models: [
{
id: 'grok-4.6',
capabilities: { reasoning: true },
thinkingOptions: { efforts: ['low', 'xhigh'] },
},
],
} as const;
assert.deepEqual(thinkingOptionsForConnection(connection, 'grok-4.6'), {
efforts: ['low', 'xhigh'],
});
assert.deepEqual(thinkingVariantsForConnection(connection, 'grok-4.6'), ['low', 'xhigh']);

assert.equal(
thinkingOptionsForConnection(
{
providerType: 'xai',
models: [{ id: 'grok-4.5', capabilities: { reasoning: false } }],
},
'grok-4.5',
),
undefined,
);
});

test('declarable relay levels are every intensity tier but off', () => {
// `off` is a disable-wire encoding (reasoning_effort 'none'), not an
// intensity tier — a hybrid UI/data contract keeps it out of declarations.
Expand Down
16 changes: 14 additions & 2 deletions packages/core/src/__tests__/runtime-policy-codec.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -383,12 +383,24 @@ test('relay model profiles round-trip canonical entries and drafts, strictly', (
test('normalizes exact bounded model discovery results', () => {
assert.deepEqual(
normalizeConnectionModelDiscoveryResult({
models: [{ id: 'gpt-5', capabilities: { chat: true } }],
models: [
{
id: 'gpt-5',
capabilities: { chat: true },
thinkingOptions: { efforts: ['low', 'high'] },
},
],
source: 'fetched',
fetchedAt: 42,
}),
{
models: [{ id: 'gpt-5', capabilities: { chat: true } }],
models: [
{
id: 'gpt-5',
capabilities: { chat: true },
thinkingOptions: { efforts: ['low', 'high'] },
},
],
source: 'fetched',
fetchedAt: 42,
},
Expand Down
5 changes: 5 additions & 0 deletions packages/core/src/llm-connections.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,11 @@ export interface ModelInfo {
/** Provider-hosted live web search, using this exact model and connection. */
webSearch?: boolean;
};
/** Exact reasoning controls advertised by this provider's model inventory. */
thinkingOptions?: {
efforts?: string[];
toggle?: boolean;
};
/** Multimodal input/output support from provider catalog metadata. */
modalities?: {
input: Array<'text' | 'image' | 'audio' | 'pdf'>;
Expand Down
Loading
Loading