From 08c41ac1e7bda6bdde88a970036cd3c6cd92244c Mon Sep 17 00:00:00 2001 From: Aegis Date: Tue, 2 Jun 2026 10:37:01 -0500 Subject: [PATCH] fix: remove raw groq logprobs call --- web/src/groq.ts | 63 ++++++------------------------------------ web/tests/groq.test.ts | 35 +++++------------------ 2 files changed, 16 insertions(+), 82 deletions(-) diff --git a/web/src/groq.ts b/web/src/groq.ts index 65832f2..9452542 100755 --- a/web/src/groq.ts +++ b/web/src/groq.ts @@ -44,10 +44,10 @@ export async function askGroq( } } -// ─── Logprobs-enabled classification ───────────────────────── -// Requests logprobs from Groq's OpenAI-compatible API and computes -// geometric mean of token probabilities for the classification value. -// Returns both self-reported confidence and token-level confidence. +// ─── Classification confidence helper ──────────────────────── +// Provider-backed replacement for the former raw Groq logprobs call. +// llm-providers does not expose provider logprob request options yet, so +// tokenConfidence mirrors self-reported confidence until that contract lands. export interface LogprobClassification { pattern: string; @@ -64,65 +64,20 @@ export async function askGroqWithLogprobs( userPrompt: string, baseUrl = 'https://api.groq.com', ): Promise { - const response = await fetch(`${baseUrl}/openai/v1/chat/completions`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - temperature: 0.1, - max_tokens: 200, - response_format: { type: 'json_object' }, - logprobs: true, - top_logprobs: 3, - messages: [ - { role: 'system', content: systemPrompt }, - { role: 'user', content: userPrompt }, - ], - }), - }); - - if (!response.ok) { - const errText = await response.text(); - throw new Error(`Groq logprobs API error ${response.status}: ${errText}`); - } - - interface LogprobToken { - token: string; - logprob: number; - } - - const data = await response.json<{ - choices: { - message: { content: string }; - logprobs?: { content?: LogprobToken[] }; - }[]; - }>(); - - const raw = data.choices[0]?.message?.content ?? '{}'; - const parsed = JSON.parse(raw) as { + const { parsed } = await askGroqJson<{ pattern?: string; complexity?: number; needs_tools?: boolean; confidence?: number; - }; - - // Compute geometric mean of token logprobs - let tokenConfidence = 0.5; // fallback if no logprobs - const logprobTokens = data.choices[0]?.logprobs?.content; - if (logprobTokens && logprobTokens.length > 0) { - const sumLogprobs = logprobTokens.reduce((sum, t) => sum + t.logprob, 0); - tokenConfidence = Math.exp(sumLogprobs / logprobTokens.length); - } + }>(apiKey, model, systemPrompt, userPrompt, baseUrl, { maxTokens: 200, temperature: 0.1 }); + const confidence = parsed.confidence ?? 0.5; return { pattern: (parsed.pattern ?? 'general_knowledge').toLowerCase().replace(/[^a-z_]/g, ''), complexity: parsed.complexity ?? 2, needs_tools: parsed.needs_tools ?? false, - selfReportedConfidence: parsed.confidence ?? 0.5, - tokenConfidence, + selfReportedConfidence: confidence, + tokenConfidence: confidence, }; } diff --git a/web/tests/groq.test.ts b/web/tests/groq.test.ts index 822c50c..20e8110 100755 --- a/web/tests/groq.test.ts +++ b/web/tests/groq.test.ts @@ -1,5 +1,5 @@ // Groq helper tests — askGroq, askGroqJson, askGroqWithLogprobs, probeConsistency -// Mocks provider factory and fetch() to test API interaction without real calls +// Mocks provider factory to test API interaction without real calls import { describe, it, expect, vi, beforeEach } from 'vitest'; @@ -22,10 +22,6 @@ vi.mock('../src/kernel/memory/index.js', () => ({ }, })); -// Mock global fetch -const mockFetch = vi.fn(); -vi.stubGlobal('fetch', mockFetch); - const { askGroq, askGroqJson, askGroqWithLogprobs, probeConsistency } = await import('../src/groq.js'); function providerResponse(content: unknown, usage = { inputTokens: 100, outputTokens: 50, totalTokens: 150, cost: 0.001 }) { @@ -38,15 +34,6 @@ function providerResponse(content: unknown, usage = { inputTokens: 100, outputTo }; } -function groqLogprobResponse(content: string, logprobs: Array<{ token: string; logprob: number }>) { - return new Response(JSON.stringify({ - choices: [{ - message: { content }, - logprobs: { content: logprobs }, - }], - }), { status: 200, headers: { 'Content-Type': 'application/json' } }); -} - describe('askGroq', () => { beforeEach(() => { vi.clearAllMocks(); @@ -144,39 +131,31 @@ describe('askGroqWithLogprobs', () => { providerMocks.createLLMProviderFactory.mockReturnValue({ generateResponse: providerMocks.generateResponse }); }); - it('parses classification with token confidence', async () => { - mockFetch.mockResolvedValue(groqLogprobResponse( + it('parses classification with provider-backed confidence', async () => { + providerMocks.generateResponse.mockResolvedValue(providerResponse( '{"pattern":"greeting","complexity":0,"needs_tools":false,"confidence":0.99}', - [ - { token: '{"', logprob: -0.01 }, - { token: 'pattern', logprob: -0.02 }, - { token: '":"', logprob: -0.01 }, - { token: 'greeting', logprob: -0.05 }, - ], )); const result = await askGroqWithLogprobs('key', 'model', 'sys', 'user'); expect(result.pattern).toBe('greeting'); expect(result.complexity).toBe(0); expect(result.needs_tools).toBe(false); expect(result.selfReportedConfidence).toBe(0.99); - expect(result.tokenConfidence).toBeGreaterThan(0); - expect(result.tokenConfidence).toBeLessThanOrEqual(1); + expect(result.tokenConfidence).toBe(0.99); }); it('defaults missing fields', async () => { - mockFetch.mockResolvedValue(groqLogprobResponse('{}', [])); + providerMocks.generateResponse.mockResolvedValue(providerResponse('{}')); const result = await askGroqWithLogprobs('key', 'model', 'sys', 'user'); expect(result.pattern).toBe('general_knowledge'); expect(result.complexity).toBe(2); expect(result.needs_tools).toBe(false); expect(result.selfReportedConfidence).toBe(0.5); - expect(result.tokenConfidence).toBe(0.5); // fallback when no logprobs + expect(result.tokenConfidence).toBe(0.5); }); it('sanitizes pattern string', async () => { - mockFetch.mockResolvedValue(groqLogprobResponse( + providerMocks.generateResponse.mockResolvedValue(providerResponse( '{"pattern":"Greeting!123"}', - [{ token: 'x', logprob: -0.1 }], )); const result = await askGroqWithLogprobs('key', 'model', 'sys', 'user'); expect(result.pattern).toBe('greeting');