From b7c7caaddf892027ad9e820b5b5eb9665cf1fd48 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 6 Sep 2026 09:45:04 +0000 Subject: [PATCH] Add OpenAI-compatible API base URLs in Settings Thread a chat endpoint and an optional transcription endpoint through the REST client so Azure, OpenRouter, Groq, LM Studio, Ollama, and a local Whisper server work without an env var. Structured output falls back from json_schema when the provider rejects it. OPENAI_BASE_URL still wins. Closes #46 Co-authored-by: Jeremy Smith --- CHANGELOG.md | 9 + README.md | 12 +- src/main/pipeline/index.ts | 2 +- src/main/pipeline/openai.ts | 165 +++++++++++++++--- src/main/pipeline/wholeVideo.ts | 2 +- src/main/settings.ts | 30 ++++ src/renderer/src/components/SettingsModal.tsx | 55 +++++- src/shared/types.ts | 19 ++ tests/openai.test.ts | 121 ++++++++++++- 9 files changed, 379 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 332dcf1..84540b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ still pre-1.0, minor bumps carry new features and patch bumps carry fixes. ## [Unreleased] +### Added + +- **OpenAI-compatible API endpoints.** Settings now takes a chat base URL + (Azure, OpenRouter, Groq, LM Studio, Ollama) and an optional separate + transcription URL, so a local Whisper server can sit next to a hosted LLM. + Structured-output calls fall back from `json_schema` to `json_object` (and + then a plain JSON completion) when the provider does not support OpenAI's + strict schema mode. `OPENAI_BASE_URL` still overrides Settings when set. + ### Removed - **The WorkVivo posting integration.** It was specific to one organisation's diff --git a/README.md b/README.md index 5012cb9..e261bc4 100644 --- a/README.md +++ b/README.md @@ -216,9 +216,13 @@ Signing and notarisation are wanted; see are configurable in Settings, including a cheaper legacy option. **Can I run it against a local or non-OpenAI model?** -Not today. The client in `src/main/pipeline/openai.ts` targets the OpenAI REST -API. A pluggable endpoint would be a welcome contribution, and local Whisper is -the obvious first step. +Yes, if it speaks the OpenAI REST shape. Set the API base URL in Settings +(or `OPENAI_BASE_URL`) to Azure OpenAI, OpenRouter, Groq, LM Studio, Ollama, +or anything else with `/v1/chat/completions`. Transcription can point at a +separate local Whisper server (faster-whisper, whisper.cpp’s compatible +endpoint) — it must return **word-level timestamps**, because captions and +tighten-cuts depend on them. Bundled in-process Whisper (no server at all) +is still on the roadmap. ## Roadmap @@ -226,7 +230,7 @@ Each of these is an open issue, so the discussion and the detail live there. Con - [Multi-language caption translation](https://github.com/JeremySNR/clip-forge/issues/49) - [Manual zoom keyframes on the timeline](https://github.com/JeremySNR/clip-forge/issues/50) -- [OpenAI-compatible endpoints and local Whisper](https://github.com/JeremySNR/clip-forge/issues/46), so transcription can run free and offline +- Bundled on-device Whisper, so transcription needs no server at all - [Size-targeted export](https://github.com/JeremySNR/clip-forge/issues/48) ("fit under N MB"), where the encoder work is already done - Direct publishing and scheduling to socials (needs an audited TikTok/YouTube app) diff --git a/src/main/pipeline/index.ts b/src/main/pipeline/index.ts index bb84c81..a2f061a 100644 --- a/src/main/pipeline/index.ts +++ b/src/main/pipeline/index.ts @@ -138,7 +138,7 @@ export async function analyzeProject( ): Promise { const apiKey = getApiKey() if (!apiKey) { - throw new Error('No OpenAI API key configured. Add one in Settings before generating clips.') + throw new Error('No API key configured. Add one in Settings before generating clips.') } const settings = getModelPreferences() const workDir = join(tmpdir(), 'clipforge', `job-${project.id}`) diff --git a/src/main/pipeline/openai.ts b/src/main/pipeline/openai.ts index 4ab7bac..58edda7 100644 --- a/src/main/pipeline/openai.ts +++ b/src/main/pipeline/openai.ts @@ -12,10 +12,13 @@ import { setTimeout as sleep } from 'node:timers/promises' export const DEFAULT_OPENAI_API_BASE = 'https://api.openai.com/v1' /** - * Resolve the OpenAI REST base URL from OPENAI_BASE_URL. Empty, whitespace, - * relative values like "/v1", and other non-absolute URLs fall back to the - * default — otherwise fetch() posts to "/v1/audio/transcriptions" and Whisper - * fails with HTTP 404 "Invalid URL". + * Resolve an OpenAI-compatible REST base URL. Empty, whitespace, relative + * values like "/v1", and other non-absolute URLs fall back to the default — + * otherwise fetch() posts to "/v1/audio/transcriptions" and Whisper fails + * with HTTP 404 "Invalid URL". + * + * `OPENAI_BASE_URL` still wins when set (cloud agents, CI). Settings values + * are applied through `configureOpenAiEndpoints`. */ export function resolveOpenAiApiBase( envBase: string | undefined = process.env.OPENAI_BASE_URL @@ -26,7 +29,7 @@ export function resolveOpenAiApiBase( const trimmed = raw.replace(/\/$/, '') if (!/^https?:\/\//i.test(trimmed)) { console.warn( - `[clipforge] OPENAI_BASE_URL must be an absolute https URL (got ${JSON.stringify(raw)}); using ${DEFAULT_OPENAI_API_BASE}` + `[clipforge] API base URL must be an absolute http(s) URL (got ${JSON.stringify(raw)}); using ${DEFAULT_OPENAI_API_BASE}` ) return DEFAULT_OPENAI_API_BASE } @@ -41,13 +44,45 @@ export function resolveOpenAiApiBase( return trimmed } catch { console.warn( - `[clipforge] OPENAI_BASE_URL is invalid (${JSON.stringify(raw)}); using ${DEFAULT_OPENAI_API_BASE}` + `[clipforge] API base URL is invalid (${JSON.stringify(raw)}); using ${DEFAULT_OPENAI_API_BASE}` ) return DEFAULT_OPENAI_API_BASE } } -const API_BASE = resolveOpenAiApiBase() +/** + * Settings-sourced bases. Env vars still take precedence so a cloud/CI + * `OPENAI_BASE_URL` cannot be silently overridden by a leftover Settings field. + */ +let settingsChatBase: string | undefined +let settingsTranscriptionBase: string | undefined + +export function configureOpenAiEndpoints(opts: { + chatBase?: string + transcriptionBase?: string +}): void { + settingsChatBase = opts.chatBase?.trim() || undefined + settingsTranscriptionBase = opts.transcriptionBase?.trim() || undefined +} + +/** Chat completions base (analysis, captions, B-roll, visual scoring). */ +export function chatApiBase(): string { + return resolveOpenAiApiBase(process.env.OPENAI_BASE_URL || settingsChatBase) +} + +/** + * Whisper transcription base. A dedicated transcription URL (env or Settings) + * wins, then the shared chat base, then the OpenAI default — so a local + * Whisper server can sit next to a hosted LLM. + */ +export function transcriptionApiBase(): string { + return resolveOpenAiApiBase( + process.env.OPENAI_TRANSCRIPTION_BASE_URL || + process.env.OPENAI_BASE_URL || + settingsTranscriptionBase || + settingsChatBase + ) +} export class OpenAIError extends Error { constructor( @@ -128,7 +163,7 @@ async function raiseForStatus(res: Response, context: string): Promise { /* non-JSON error body */ } if (res.status === 401) { - throw new OpenAIError('OpenAI rejected the API key. Check it in Settings.', 401) + throw new OpenAIError('The API rejected the API key. Check it in Settings.', 401) } throw new OpenAIError(`${context} failed (HTTP ${res.status})${detail ? `: ${detail}` : ''}`, res.status) } @@ -184,7 +219,7 @@ export async function transcribeAudioFile( if (opts.contextPrompt) form.append('prompt', opts.contextPrompt) if (opts.language && opts.language !== 'auto') form.append('language', opts.language) - const res = await fetch(`${API_BASE}/audio/transcriptions`, { + const res = await fetch(`${transcriptionApiBase()}/audio/transcriptions`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}` }, body: form, @@ -216,20 +251,91 @@ export async function chatJSON( ): Promise { return withRetries( async () => { - const res = await fetch(`${API_BASE}/chat/completions`, { + const content = await completeChatContent( + apiKey, + model, + messages, + schemaName, + schema, + signal + ) + try { + return JSON.parse(content) as T + } catch { + throw new OpenAIError('Analysis returned invalid JSON') + } + }, + { signal } + ) +} + +function looksLikeUnsupportedFormat(err: unknown): boolean { + if (!(err instanceof OpenAIError) || (err.status !== 400 && err.status !== 422)) return false + const m = err.message.toLowerCase() + return /response_format|json_schema|json_object|strict|not supported|unknown parameter|unrecognized|invalid parameter/.test( + m + ) +} + +/** + * Compatible endpoints (Ollama, LM Studio, some Groq/OpenRouter models) often + * reject OpenAI's strict json_schema. Try that first, then json_object, then + * a bare completion with an instruction to return JSON. + */ +async function completeChatContent( + apiKey: string, + model: string, + messages: ChatMessage[], + schemaName: string, + schema: Record, + signal?: AbortSignal +): Promise { + const formats: Array<{ label: string; body: Record }> = [ + { + label: 'json_schema', + body: { + model, + messages, + response_format: { + type: 'json_schema', + json_schema: { name: schemaName, strict: true, schema } + } + } + }, + { + label: 'json_object', + body: { + model, + messages, + response_format: { type: 'json_object' } + } + }, + { + label: 'plain', + body: { + model, + messages: [ + ...messages, + { + role: 'user', + content: + 'Return only a JSON object matching the requested schema. No markdown, no commentary.' + } + ] + } + } + ] + + let lastError: unknown + for (const format of formats) { + try { + const res = await fetch(`${chatApiBase()}/chat/completions`, { method: 'POST', headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ - model, - messages, - response_format: { - type: 'json_schema', - json_schema: { name: schemaName, strict: true, schema } - } - }), + body: JSON.stringify(format.body), signal: withTimeout(CHAT_TIMEOUT_MS, signal) }) await raiseForStatus(res, 'Analysis') @@ -238,12 +344,19 @@ export async function chatJSON( } const content = body.choices?.[0]?.message?.content if (!content) throw new OpenAIError('Analysis returned an empty response') - try { - return JSON.parse(content) as T - } catch { - throw new OpenAIError('Analysis returned invalid JSON') - } - }, - { signal } - ) + return extractJsonText(content) + } catch (err) { + lastError = err + if (signal?.aborted) throw err + if (!looksLikeUnsupportedFormat(err)) throw err + } + } + throw lastError +} + +/** Strip optional markdown fences some local models wrap JSON in. */ +export function extractJsonText(content: string): string { + const trimmed = content.trim() + const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i) + return fenced ? fenced[1].trim() : trimmed } diff --git a/src/main/pipeline/wholeVideo.ts b/src/main/pipeline/wholeVideo.ts index e58b5e0..f331d2c 100644 --- a/src/main/pipeline/wholeVideo.ts +++ b/src/main/pipeline/wholeVideo.ts @@ -48,7 +48,7 @@ export async function captionWholeVideo( // offline is allowed. const apiKey = getApiKey() if (!apiKey && !project.transcript) { - throw new Error('No OpenAI API key configured. Add one in Settings before transcribing.') + throw new Error('No API key configured. Add one in Settings before transcribing.') } const settings = getModelPreferences() const workDir = join(tmpdir(), 'clipforge', `caption-${project.id}`) diff --git a/src/main/settings.ts b/src/main/settings.ts index c17350a..743e284 100644 --- a/src/main/settings.ts +++ b/src/main/settings.ts @@ -13,6 +13,7 @@ import type { import { getGpuStatus } from './pipeline/encoders' import { clearImportCookiesFile, getImportCookiesPath } from './cookies' import { DEFAULT_BRAND_COLORS } from '@shared/captionStyles' +import { configureOpenAiEndpoints } from './pipeline/openai' interface StoredSettings { @@ -22,6 +23,8 @@ interface StoredSettings { /** ISO-639-1 language code forced on Whisper, or 'auto' to auto-detect. */ transcriptionLanguage: string analysisModel: string + openaiBaseUrl: string + transcriptionBaseUrl: string encoder: EncoderPreference quality: QualityPreference branding: BrandingSettings @@ -54,6 +57,8 @@ const DEFAULTS: StoredSettings = { // theirs — or 'auto' — in Settings. transcriptionLanguage: 'en', analysisModel: 'gpt-5.4-mini', + openaiBaseUrl: '', + transcriptionBaseUrl: '', encoder: 'auto', quality: 'standard', branding: DEFAULT_BRANDING, @@ -67,6 +72,18 @@ function settingsPath(): string { let cache: StoredSettings | null = null +function applyEndpoints(s: StoredSettings): void { + configureOpenAiEndpoints({ + chatBase: s.openaiBaseUrl, + transcriptionBase: s.transcriptionBaseUrl + }) +} + +function storedBaseUrl(raw: unknown): string { + if (typeof raw !== 'string') return '' + return raw.trim().replace(/\/$/, '') +} + function load(): StoredSettings { if (cache) return cache try { @@ -75,6 +92,8 @@ function load(): StoredSettings { cache = { ...DEFAULTS, ...parsed, + openaiBaseUrl: storedBaseUrl(parsed.openaiBaseUrl), + transcriptionBaseUrl: storedBaseUrl(parsed.transcriptionBaseUrl), // Nested objects: merge so settings saved before new fields stay valid. branding: { ...DEFAULT_BRANDING, @@ -83,17 +102,20 @@ function load(): StoredSettings { }, brandVoice: { ...DEFAULT_BRAND_VOICE, ...(parsed.brandVoice ?? {}) } } + applyEndpoints(cache) return cache } } catch { /* corrupted settings fall back to defaults */ } cache = { ...DEFAULTS } + applyEndpoints(cache) return cache } function persist(s: StoredSettings): void { cache = s + applyEndpoints(s) mkdirSync(app.getPath('userData'), { recursive: true }) writeFileSync(settingsPath(), JSON.stringify(s, null, 2), 'utf8') } @@ -127,6 +149,7 @@ export function getApiKey(): string { export async function getSettings(): Promise { const s = load() + applyEndpoints(s) const key = getApiKey() return { hasApiKey: key.length > 0, @@ -135,6 +158,9 @@ export async function getSettings(): Promise { transcriptionModel: s.transcriptionModel, transcriptionLanguage: s.transcriptionLanguage, analysisModel: s.analysisModel, + openaiBaseUrl: s.openaiBaseUrl, + transcriptionBaseUrl: s.transcriptionBaseUrl, + openaiBaseUrlFromEnv: Boolean(process.env.OPENAI_BASE_URL?.trim()), encoder: s.encoder, quality: s.quality, gpu: await getGpuStatus(), @@ -199,6 +225,10 @@ export async function updateSettings(update: SettingsUpdate): Promise s.refreshSettings) const [apiKey, setApiKey] = useState('') const [model, setModel] = useState(settings?.analysisModel ?? 'gpt-5.4-mini') + const [openaiBaseUrl, setOpenaiBaseUrl] = useState(settings?.openaiBaseUrl ?? '') + const [transcriptionBaseUrl, setTranscriptionBaseUrl] = useState( + settings?.transcriptionBaseUrl ?? '' + ) const [saved, setSaved] = useState(false) const [saving, setSaving] = useState(false) const [gpuProgress, setGpuProgress] = useState(null) @@ -101,7 +106,9 @@ export default function SettingsModal(): React.JSX.Element { try { await saveSettings({ ...(apiKey.trim() ? { apiKey: apiKey.trim() } : {}), - analysisModel: model + analysisModel: model, + openaiBaseUrl, + transcriptionBaseUrl }) setApiKey('') setSaved(true) @@ -170,7 +177,7 @@ export default function SettingsModal(): React.JSX.Element {

Used for Whisper transcription and clip analysis. Stored encrypted on this machine - and never sent anywhere except the OpenAI API. + and never sent anywhere except the API base below (OpenAI by default).

{settings !== null && !settings.keyStorageSecure && (

@@ -195,6 +202,50 @@ export default function SettingsModal(): React.JSX.Element { +

+ +

+ OpenAI-compatible endpoint for analysis (and transcription, unless you set a + separate one below). Azure OpenAI, OpenRouter, Groq, LM Studio, Ollama. Leave + blank for api.openai.com. +

+ {settings?.openaiBaseUrlFromEnv && ( +

+ OPENAI_BASE_URL is set in the environment, so it overrides these fields until + you unset it. +

+ )} + setOpenaiBaseUrl(e.target.value)} + placeholder="https://api.openai.com/v1" + disabled={settings?.openaiBaseUrlFromEnv} + className="mt-2.5 w-full rounded-xl border border-surface-600 bg-surface-850 px-3.5 py-2.5 text-sm text-zinc-200 placeholder:text-zinc-600 focus:border-white/25 focus:outline-none disabled:opacity-50" + /> + +

+ Point Whisper at a local server (faster-whisper, whisper.cpp’s OpenAI-compatible + endpoint) while clip finding stays on the chat base above. Needs word-level + timestamps. Leave blank to use the same URL. +

+ setTranscriptionBaseUrl(e.target.value)} + placeholder="Same as API base URL" + disabled={settings?.openaiBaseUrlFromEnv} + className="mt-2 w-full rounded-xl border border-surface-600 bg-surface-850 px-3.5 py-2.5 text-sm text-zinc-200 placeholder:text-zinc-600 focus:border-white/25 focus:outline-none disabled:opacity-50" + /> +
+