Skip to content

Execute dynamic tools through llm-providers #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stackbilt-admin merged 1 commit into from
Jun 2, 2026
Merged

Execute dynamic tools through llm-providers #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 30 additions & 52 deletions web/src/kernel/dynamic-tools.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
// No eval(). No code execution. Just parameterized prompts.

import { type EdgeEnv } from './dispatch.js';
import { buildLLMProviderFactory } from './provider-factory.js';
import type { ToolExecutor, ToolStatus } from '../schema-enums.js';

// ─── Types ──────────────────────────────────────────────────
Expand Down Expand Up @@ -160,58 +161,20 @@ export async function executeDynamicTool(
): Promise<ToolInvocationResult> {
const rendered = renderPrompt(tool.prompt_template, inputs);
const start = Date.now();
let text = '';
let cost = 0;

if (tool.executor === 'workers_ai' && env.ai) {
// Workers AI — free inference
const result = await env.ai.run('@cf/meta/llama-3.1-8b-instruct' as Parameters<Ai['run']>[0], {
messages: [
{ role: 'system', content: 'You are a focused tool. Answer precisely. No preamble.' },
{ role: 'user', content: rendered },
],
max_tokens: 1024,
}) as { response?: string };
text = result.response ?? '';
cost = 0;
} else if (tool.executor === 'groq' && env.groqApiKey) {
// Groq — fast, cheap
const res = await fetch('https://api.groq.com/openai/v1/chat/completions', {
method: 'POST',
headers: {
'Authorization': `Bearer ${env.groqApiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
model: env.groqModel ?? 'llama-3.3-70b-versatile',
messages: [
{ role: 'system', content: 'You are a focused tool. Answer precisely. No preamble.' },
{ role: 'user', content: rendered },
],
max_tokens: 1024,
}),
signal: AbortSignal.timeout(15_000),
});
if (!res.ok) throw new Error(`Groq error: ${res.status}`);
const data = await res.json() as { choices: Array<{ message: { content: string } }> };
text = data.choices[0]?.message?.content ?? '';
cost = 0.001; // ~$0.001 per Groq call
} else {
// Default: Workers AI fallback (always available on CF Workers)
if (env.ai) {
const result = await env.ai.run('@cf/meta/llama-3.1-8b-instruct' as Parameters<Ai['run']>[0], {
messages: [
{ role: 'system', content: 'You are a focused tool. Answer precisely. No preamble.' },
{ role: 'user', content: rendered },
],
max_tokens: 1024,
}) as { response?: string };
text = result.response ?? '';
cost = 0;
} else {
throw new Error(`Executor "${tool.executor}" not available — no API key or binding`);
}
}
const executor = tool.executor as ToolExecutor;
const model = resolveDynamicToolModel(executor, env);
const factory = buildLLMProviderFactory(env);
const result = await factory.generateResponse({
model,
messages: [
{ role: 'system', content: 'You are a focused tool. Answer precisely. No preamble.' },
{ role: 'user', content: rendered },
],
maxTokens: 1024,
temperature: 0.2,
});
const text = result.message ?? '';
const cost = result.usage.cost;

const latencyMs = Date.now() - start;

Expand All @@ -229,6 +192,21 @@ export async function executeDynamicTool(
return { text, cost, latency_ms: latencyMs, executor: tool.executor };
}

function resolveDynamicToolModel(executor: ToolExecutor, env: EdgeEnv): string {
if (executor === 'workers_ai') {
if (!env.ai) throw new Error('Executor "workers_ai" not available — no AI binding');
return '@cf/meta/llama-3.1-8b-instruct';
}

if (executor === 'groq') {
if (!env.groqApiKey) throw new Error('Executor "groq" not available — no Groq API key');
return env.groqModel ?? 'llama-3.3-70b-versatile';
}

if (!env.ai) throw new Error(`Executor "${executor}" not available — no AI binding`);
return env.gptOssModel;
}

// ─── Lifecycle (GC + Promotion) ─────────────────────────────

const MAX_ACTIVE_TOOLS = 50;
Expand Down
131 changes: 131 additions & 0 deletions web/tests/dynamic-tools-provider.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { EdgeEnv } from '../src/kernel/dispatch.js';
import type { DynamicTool } from '../src/kernel/dynamic-tools.js';

const mocks = vi.hoisted(() => ({
buildLLMProviderFactory: vi.fn(),
generateResponse: vi.fn(),
}));

vi.mock('../src/kernel/provider-factory.js', () => ({
buildLLMProviderFactory: mocks.buildLLMProviderFactory,
}));

const { executeDynamicTool } = await import('../src/kernel/dynamic-tools.js');

function makeDb() {
const run = vi.fn().mockResolvedValue({});
const bind = vi.fn().mockReturnValue({ run });
const prepare = vi.fn().mockReturnValue({ bind });
return { db: { prepare } as unknown as D1Database, prepare, bind, run };
}

function makeEnv(overrides?: Partial<EdgeEnv>): EdgeEnv {
const { db } = makeDb();
return {
db,
anthropicApiKey: 'anthropic-test',
claudeModel: 'claude-sonnet',
opusModel: 'claude-opus',
gptOssModel: '@cf/openai/gpt-oss-120b',
groqApiKey: 'groq-test',
groqModel: 'llama-3.3-70b-versatile',
groqResponseModel: 'llama-3.1-8b-instant',
groqGptOssModel: 'openai/gpt-oss-120b',
bizopsFetcher: {} as Fetcher,
bizopsToken: 'bizops-test',
resendApiKey: 'resend-test',
resendApiKeyPersonal: 'resend-personal-test',
githubToken: 'github-test',
githubRepo: 'Stackbilt-dev/aegis-oss',
braveApiKey: 'brave-test',
notifyEmail: 'notify@test.dev',
baseUrl: 'https://aegis.test',
ai: { run: vi.fn() } as unknown as Ai,
anthropicBaseUrl: 'https://api.anthropic.com',
groqBaseUrl: 'https://api.groq.com',
...overrides,
};
}

function makeTool(overrides?: Partial<DynamicTool>): DynamicTool {
return {
id: 'tool-1',
name: 'summarize_thing',
description: 'Summarize a thing',
input_schema: '{}',
prompt_template: 'Summarize {{thing}}',
executor: 'gpt_oss',
created_by: 'test',
status: 'active',
ttl_days: null,
use_count: 2,
last_used_at: null,
avg_latency_ms: 10,
avg_cost: 0.001,
created_at: '2026-06-02T00:00:00.000Z',
updated_at: '2026-06-02T00:00:00.000Z',
expires_at: null,
...overrides,
};
}

describe('executeDynamicTool provider execution', () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.buildLLMProviderFactory.mockReturnValue({ generateResponse: mocks.generateResponse });
mocks.generateResponse.mockResolvedValue({
message: 'provider output',
usage: { cost: 0.004, inputTokens: 10, outputTokens: 5, totalTokens: 15 },
model: '@cf/openai/gpt-oss-120b',
provider: 'cloudflare',
responseTime: 50,
});
});

it('executes gpt_oss dynamic tools through the provider factory', async () => {
const db = makeDb();
const env = makeEnv({ db: db.db });

const result = await executeDynamicTool(makeTool(), { thing: 'AEGIS' }, env);

expect(mocks.buildLLMProviderFactory).toHaveBeenCalledWith(env);
expect(mocks.generateResponse).toHaveBeenCalledWith({
model: '@cf/openai/gpt-oss-120b',
messages: [
{ role: 'system', content: 'You are a focused tool. Answer precisely. No preamble.' },
{ role: 'user', content: 'Summarize AEGIS' },
],
maxTokens: 1024,
temperature: 0.2,
});
expect(db.bind).toHaveBeenCalledWith(3, expect.any(Number), 0.002, 'tool-1');
expect(result).toEqual({
text: 'provider output',
cost: 0.004,
latency_ms: expect.any(Number),
executor: 'gpt_oss',
});
});

it('maps workers_ai dynamic tools to the Workers AI provider model', async () => {
await executeDynamicTool(makeTool({ executor: 'workers_ai' }), { thing: 'AEGIS' }, makeEnv());

expect(mocks.generateResponse).toHaveBeenCalledWith(expect.objectContaining({
model: '@cf/meta/llama-3.1-8b-instruct',
}));
});

it('maps groq dynamic tools to the configured Groq model', async () => {
await executeDynamicTool(makeTool({ executor: 'groq' }), { thing: 'AEGIS' }, makeEnv({ groqModel: 'llama-test' }));

expect(mocks.generateResponse).toHaveBeenCalledWith(expect.objectContaining({
model: 'llama-test',
}));
});

it('fails gpt_oss dynamic tools without an AI binding', async () => {
await expect(executeDynamicTool(makeTool(), { thing: 'AEGIS' }, makeEnv({ ai: undefined })))
.rejects.toThrow('Executor "gpt_oss" not available');
});
});