From e68d70f1d7f67c0d527c75246d3e0c3be31ea7a3 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Sun, 18 Oct 2026 09:07:40 +0800 Subject: [PATCH 01/12] feat(polish): add ASR-driven polish strategies Add a bounded Rhai policy between corrected transcription and LLM polishing, including provider-aware request overrides, shared and Linux settings, encrypted preference sync, documentation, and regression coverage. Co-Authored-By: Claude Opus 5 --- docs/polish-strategy.md | 77 ++++ openless-all/app/Cargo.lock | 79 ++++ .../app/crates/openless-core/Cargo.toml | 1 + .../src/asr/gemini/tests/omni.rs | 2 + .../openless-core/src/cloud_providers.rs | 139 ++++++- .../src/cloud_sync_e2ee_documents/registry.rs | 14 + .../openless-core/src/dictation_context.rs | 22 +- .../openless-core/src/dictation_engine.rs | 175 ++++++++- .../src/experimental_asr/tests.rs | 7 +- .../app/crates/openless-core/src/lib.rs | 9 +- .../crates/openless-core/src/llm_gemini.rs | 30 ++ .../crates/openless-core/src/llm_protocol.rs | 115 +++++- .../openless-core/src/local_asr_service.rs | 22 +- .../app/crates/openless-core/src/omni.rs | 51 ++- .../app/crates/openless-core/src/polish.rs | 337 +++++++++++++++- .../openless-core/src/polish_strategy.rs | 368 ++++++++++++++++++ .../app/crates/openless-core/src/ports.rs | 12 + .../crates/openless-core/src/shared_types.rs | 37 ++ openless-all/app/linux-egui/src/i18n.rs | 40 ++ .../linux-egui/src/linux_app/settings_save.rs | 8 + openless-all/app/linux-egui/src/main.rs | 2 + .../linux-egui/src/ui/frontend/settings.rs | 46 +++ .../linux-egui/src/ui/frontend/view_model.rs | 10 + openless-all/app/src/i18n/de.ts | 5 + openless-all/app/src/i18n/en.ts | 7 + openless-all/app/src/i18n/es.ts | 5 + openless-all/app/src/i18n/fr.ts | 5 + openless-all/app/src/i18n/ja.ts | 5 + openless-all/app/src/i18n/ko.ts | 5 + openless-all/app/src/i18n/zh-CN.ts | 5 + openless-all/app/src/i18n/zh-TW.ts | 5 + openless-all/app/src/lib/ipc/mock-data.ts | 2 + openless-all/app/src/lib/types.ts | 4 + .../src/pages/settings/DataStorageSection.tsx | 26 ++ 34 files changed, 1614 insertions(+), 63 deletions(-) create mode 100644 docs/polish-strategy.md create mode 100644 openless-all/app/crates/openless-core/src/polish_strategy.rs diff --git a/docs/polish-strategy.md b/docs/polish-strategy.md new file mode 100644 index 000000000..28b834db5 --- /dev/null +++ b/docs/polish-strategy.md @@ -0,0 +1,77 @@ +# ASR polish strategy + +OpenLess can optionally run a local strategy after ASR has finished and correction rules have been applied, but before the polish LLM request is sent. Enable **Use custom polish strategy** in Settings → Privacy → Data storage, then enter a Rhai function named `strategy`. + +The feature is disabled by default. When it is disabled, empty, or fails, OpenLess keeps the normal polish and short-transcript rules. A strategy failure never prevents dictation. + +## Input + +The function receives one map. `charCount` counts non-whitespace Unicode scalar values, using the same rule as the built-in short-input setting. `audioDurationMs` is the final audio duration in milliseconds. + +```rhai +text // corrected ASR text +charCount // integer, non-whitespace Rust char count +audioDurationMs // integer, milliseconds +language // string, or () when unavailable +confidence // floating-point value, or () when unavailable +asrProvider // provider label, or an inferred provider type +asrModel // string, or () when unavailable +``` + +The function must return a map with an `action` field: + +```rhai +// Do not send an LLM request. +#{ action: "skip" } + +// Keep the normal polish request and configured parameters. +#{ action: "polish" } + +// Polish, applying only the listed per-request overrides. +#{ action: "request", temperature: 0.4, maxTokens: 400, reasoningEffort: "low" } +``` + +A typical short-input policy is: + +```rhai +fn strategy(input) { + if input["charCount"] <= 3 { + #{ action: "skip" } + } else { + #{ action: "polish" } + } +} +``` + +The reasoning example below keeps longer English dictations cheaper while preserving ordinary polish for other languages: + +```rhai +fn strategy(input) { + if input["language"] == "en" && input["charCount"] > 160 { + #{ action: "request", reasoningEffort: "low" } + } else { + #{ action: "polish" } + } +} +``` + +## Request mapping + +Only `temperature` (`0..=2`), `maxTokens`, and `reasoningEffort` (`none`, `low`, `medium`, or `high`) are accepted. They apply to this request only; they do not rewrite channel credentials or session preferences. + +- Chat Completions uses `temperature`, `max_tokens`, and provider-specific reasoning controls; official OpenAI reasoning models use `max_completion_tokens` for a `maxTokens` override. +- Responses uses `max_output_tokens` and the provider's `reasoning.effort` mapping. Official OpenAI reasoning models also omit `temperature`, including when a `none` override removes the reasoning object. +- Messages uses `max_tokens`; its existing thinking mode and budget validation remain authoritative. If a `maxTokens` override would be no larger than a configured budget-thinking request, it is omitted to avoid an invalid request. +- Gemini maps `temperature` to `generationConfig.temperature` and `maxTokens` to `generationConfig.maxOutputTokens`; `none` disables thinking and other effort values enable it. Gemini does not expose four distinct effort levels. Providers that expose only a boolean thinking switch likewise treat `low`/`medium`/`high` as enabled and cannot preserve the exact level. +- Codex uses Responses `max_output_tokens` and maps `reasoningEffort` to Responses `reasoning.effort`. +- Existing model capability rules still win. For example, official OpenAI chat models that do not support `reasoning_effort` receive no such field, and `gpt-5-pro` remains at its required `high` tier. Unsupported overrides are omitted or reduced to the configured provider capability, not retried as a second request. + +Other returned keys, malformed values, out-of-range numbers, and actions with incompatible overrides are rejected and the normal policy is used. + +## Limits and privacy + +The evaluator is embedded in Core and has no file, network, process, package, or credential APIs. The source is limited to 32 KiB, transcript input to 128 KiB, and execution is capped by operations, call depth, string size, and collection size. The script receives transcript metadata but never receives API keys, prompts, credentials, or document context. + +The script and enable switch are portable encrypted preferences. If cloud sync is enabled, another OpenLess device may receive and execute the same user-authored script, still under these limits. Disable the feature before syncing to a device where the policy is not desired. + +Missing provider metadata is represented by `()`. A script that depends on optional metadata should check for that value and return `polish` when it is unavailable. diff --git a/openless-all/app/Cargo.lock b/openless-all/app/Cargo.lock index f92c98e4f..c4fbf2d1c 100644 --- a/openless-all/app/Cargo.lock +++ b/openless-all/app/Cargo.lock @@ -149,6 +149,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", "getrandom 0.3.4", "once_cell", "version_check", @@ -1045,6 +1046,26 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -3652,6 +3673,9 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "portable-atomic", +] [[package]] name = "opaque-debug" @@ -3679,6 +3703,7 @@ dependencies = [ "parking_lot", "regex", "reqwest", + "rhai", "serde", "serde_json", "sha1", @@ -4554,6 +4579,34 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "rhai" +version = "1.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0334639972c0ea5a3fd366aa36116754a11431b619fec3ed559b3f73bcbcebf5" +dependencies = [ + "ahash", + "bitflags 2.13.2", + "num-traits", + "once_cell", + "rhai_codegen", + "smallvec", + "smartstring", + "thin-vec", + "web-time", +] + +[[package]] +name = "rhai_codegen" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd3a7535e50bf36857e7be7bec276d334e8c2dfa469c2201226fd01638ea5ca" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "ring" version = "0.17.14" @@ -4976,6 +5029,17 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + [[package]] name = "smithay-client-toolkit" version = "0.19.2" @@ -5225,6 +5289,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -5311,6 +5381,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tiny-skia" version = "0.11.4" diff --git a/openless-all/app/crates/openless-core/Cargo.toml b/openless-all/app/crates/openless-core/Cargo.toml index e58f79a95..01b90e2e0 100644 --- a/openless-all/app/crates/openless-core/Cargo.toml +++ b/openless-all/app/crates/openless-core/Cargo.toml @@ -23,6 +23,7 @@ md-5 = "0.10" once_cell = "1" parking_lot = "0.12" regex = "1" +rhai = { version = "1.22", default-features = true } reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls", "stream", "system-proxy"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/openless-all/app/crates/openless-core/src/asr/gemini/tests/omni.rs b/openless-all/app/crates/openless-core/src/asr/gemini/tests/omni.rs index d76a8cf97..bbc886d61 100644 --- a/openless-all/app/crates/openless-core/src/asr/gemini/tests/omni.rs +++ b/openless-all/app/crates/openless-core/src/asr/gemini/tests/omni.rs @@ -17,6 +17,8 @@ fn omni_provider(endpoint: String) -> OmniProvider { extra_headers: Default::default(), temperature: Some(0.3), thinking_enabled: true, + max_tokens_override: None, + reasoning_effort: None, }) } diff --git a/openless-all/app/crates/openless-core/src/cloud_providers.rs b/openless-all/app/crates/openless-core/src/cloud_providers.rs index 9372fc885..13cf8c354 100644 --- a/openless-all/app/crates/openless-core/src/cloud_providers.rs +++ b/openless-all/app/crates/openless-core/src/cloud_providers.rs @@ -180,6 +180,7 @@ impl PreparedTranscription for PreparedCloudTranscription { Ok(Arc::new(CloudTranscriptionSession { kind, asr_call_label, + language: preparation.context.asr.language.clone(), finished: AtomicBool::new(false), task_spawner, }) as Arc) @@ -205,6 +206,7 @@ enum CloudTranscriptionSessionKind { struct CloudTranscriptionSession { kind: CloudTranscriptionSessionKind, asr_call_label: crate::AsrCallLabel, + language: Option, finished: AtomicBool, task_spawner: Arc, } @@ -269,6 +271,13 @@ impl TranscriptionSession for CloudTranscriptionSession { Some(self.asr_call_label.clone()) } + fn transcript_metadata(&self) -> crate::ports::TranscriptMetadata { + crate::ports::TranscriptMetadata { + language: self.language.clone(), + confidence: None, + } + } + fn finish(&self) -> BoxFuture<'static, Result> { if self.finished.swap(true, Ordering::AcqRel) { return Box::pin(async { @@ -1398,10 +1407,17 @@ async fn build_cloud_polisher_provider( BackendError::new(BackendErrorCode::Provider, "LLM model is not configured") })?; if provider_type == crate::polish::CODEX_OAUTH_PROVIDER_ID { - let provider = crate::polish::CodexOAuthLLMProvider::new( - crate::polish::CodexOAuthConfig::new(model) - .with_thinking_enabled(context.polish.llm_thinking_enabled), - ); + let mut config = crate::polish::CodexOAuthConfig::new(model) + .with_thinking_enabled(context.polish.llm_thinking_enabled); + if let Some(effort) = strategy_reasoning_effort(context) { + // Codex uses the Responses API, whose effort enum does not accept + // `none`; omit that unsupported override rather than sending a + // request that the endpoint rejects. + let effort = (effort != "none").then_some(effort); + config = config.with_reasoning_effort(effort); + } + config = config.with_max_output_tokens(strategy_max_tokens(context)); + let provider = crate::polish::CodexOAuthLLMProvider::new(config); return Ok(CloudPolisherProvider::OpenAi( crate::polish::ActiveLLMProvider::Codex(provider), )); @@ -1438,17 +1454,22 @@ async fn build_cloud_polisher_provider( crate::endpoint_security::validate_http_endpoint(&endpoint) .map_err(|error| map_llm_error(error.to_string()))?; if provider_type == "gemini" { - let provider = crate::llm_gemini::GeminiProvider::new( + let thinking_enabled = strategy_thinking_enabled(context); + let mut config = crate::llm_gemini::GeminiConfig::new(api_key, model, endpoint.trim_end_matches('/')) - .with_thinking_enabled(context.polish.llm_thinking_enabled), - ); + .with_thinking_enabled(thinking_enabled) + .with_max_output_tokens(strategy_max_tokens(context)); + if let Some(temperature) = strategy_temperature(context) { + config = config.with_temperature(temperature); + } + let provider = crate::llm_gemini::GeminiProvider::new(config); return Ok(CloudPolisherProvider::Gemini(provider)); } let protocol = crate::llm_protocol::LlmProtocolConfig::load(credentials, channel_id, provider_type) .await?; - let temperature = read_channel_credential( + let configured_temperature = read_channel_credential( credentials, CredentialNamespace::Llm, channel_id, @@ -1459,6 +1480,17 @@ async fn build_cloud_polisher_provider( .map(parse_temperature) .transpose()? .flatten(); + let strategy_temperature = context + .polish + .polish_request_overrides + .as_ref() + .and_then(|overrides| overrides.temperature()); + let temperature = strategy_temperature.or_else(|| { + crate::polish::openai_compatible_temperature_for_provider( + provider_type, + configured_temperature, + ) + }); let extra_headers = read_channel_credential( credentials, CredentialNamespace::Llm, @@ -1479,10 +1511,9 @@ async fn build_cloud_polisher_provider( ) .with_protocol(protocol) .with_thinking_enabled(context.polish.llm_thinking_enabled) - .with_temperature(crate::polish::openai_compatible_temperature_for_provider( - provider_type, - temperature, - )) + .with_reasoning_effort(strategy_reasoning_effort(context)) + .with_max_tokens_override(strategy_max_tokens(context)) + .with_temperature(temperature) .with_extra_headers(extra_headers); Ok(CloudPolisherProvider::OpenAi( crate::polish::ActiveLLMProvider::OpenAI(crate::polish::OpenAICompatibleLLMProvider::new( @@ -1491,6 +1522,60 @@ async fn build_cloud_polisher_provider( )) } +fn strategy_temperature(context: &DictationContext) -> Option { + context + .polish + .polish_request_overrides + .as_ref() + .and_then(|overrides| overrides.temperature()) +} + +fn omni_temperature( + context: &DictationContext, + provider_type: &str, + configured_temperature: Option, +) -> Option { + strategy_temperature(context).or_else(|| { + crate::polish::openai_compatible_temperature_for_provider( + provider_type, + configured_temperature, + ) + }) +} + +fn strategy_reasoning_effort(context: &DictationContext) -> Option { + context + .polish + .polish_request_overrides + .as_ref() + .and_then(|overrides| overrides.reasoning_effort) + .map(|effort| match effort { + crate::polish_strategy::ReasoningEffort::None => "none", + crate::polish_strategy::ReasoningEffort::Low => "low", + crate::polish_strategy::ReasoningEffort::Medium => "medium", + crate::polish_strategy::ReasoningEffort::High => "high", + }) + .map(str::to_string) +} + +fn strategy_thinking_enabled(context: &DictationContext) -> bool { + context + .polish + .polish_request_overrides + .as_ref() + .and_then(|overrides| overrides.reasoning_effort) + .map(|effort| !matches!(effort, crate::polish_strategy::ReasoningEffort::None)) + .unwrap_or(context.polish.llm_thinking_enabled) +} + +fn strategy_max_tokens(context: &DictationContext) -> Option { + context + .polish + .polish_request_overrides + .as_ref() + .and_then(|overrides| overrides.max_tokens) +} + fn parse_temperature(value: &str) -> Result, BackendError> { let value = value.trim(); if value.is_empty() { @@ -1785,7 +1870,7 @@ async fn build_omni_provider( .map(parse_extra_headers) .transpose()? .unwrap_or_default(); - let temperature = read_channel_credential( + let configured_temperature = read_channel_credential( credentials, CredentialNamespace::Omni, provider_id, @@ -1796,17 +1881,17 @@ async fn build_omni_provider( .map(parse_temperature) .transpose()? .flatten(); + let temperature = omni_temperature(context, provider_type, configured_temperature); let config = crate::omni::OmniConfig { provider_id: provider_type.to_string(), base_url, api_key, model, extra_headers, - temperature: crate::polish::openai_compatible_temperature_for_provider( - provider_type, - temperature, - ), - thinking_enabled: context.polish.llm_thinking_enabled, + temperature, + thinking_enabled: strategy_thinking_enabled(context), + max_tokens_override: strategy_max_tokens(context), + reasoning_effort: strategy_reasoning_effort(context), }; let provider = crate::omni::OmniProvider::new(config) .with_transcription_hints(&context.polish.working_languages, &context.polish.hotwords); @@ -2503,6 +2588,24 @@ mod tests { use super::*; use crate::{InMemoryCredentialStore, ProviderInvocation, SecretValue}; + #[test] + fn strategy_temperature_overrides_configured_omni_temperature() { + let mut context = DictationContext::default(); + context.polish.polish_request_overrides = Some( + crate::polish_strategy::PolishRequestOverrides { + temperature_milli: Some(400), + max_tokens: None, + reasoning_effort: None, + }, + ); + + assert_eq!(omni_temperature(&context, "openai", Some(1.2)), Some(0.4)); + assert_eq!( + omni_temperature(&DictationContext::default(), "custom", Some(1.2)), + Some(1.2) + ); + } + struct ObservedTranslationStream { chunks: Mutex>, arrived: tokio::sync::Semaphore, diff --git a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs index 4ac190fef..b06d91a22 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs @@ -662,6 +662,20 @@ pub const PREFERENCE_FIELDS: &[PreferenceField] = &[ shape: PreferenceShape::Unsigned32, reason: "portable_setting", }, + PreferenceField { + rust_name: "polish_strategy_enabled", + key: "polishStrategyEnabled", + class: PreferenceClass::Portable, + shape: PreferenceShape::Boolean, + reason: "portable_setting", + }, + PreferenceField { + rust_name: "polish_strategy_script", + key: "polishStrategyScript", + class: PreferenceClass::Portable, + shape: PreferenceShape::Text, + reason: "portable_setting", + }, PreferenceField { rust_name: "start_minimized", key: "startMinimized", diff --git a/openless-all/app/crates/openless-core/src/dictation_context.rs b/openless-all/app/crates/openless-core/src/dictation_context.rs index 93a6cd75b..370c488d0 100644 --- a/openless-all/app/crates/openless-core/src/dictation_context.rs +++ b/openless-all/app/crates/openless-core/src/dictation_context.rs @@ -139,6 +139,10 @@ pub struct DictationPolishContext { /// Transcripts with at most this many non-whitespace characters skip /// polish (0 = off). See `DictationContext::skips_polish_for_short_text`. pub polish_skip_max_chars: u32, + /// User-defined post-ASR policy, captured when the session starts. + pub polish_strategy: crate::polish_strategy::PolishStrategyConfig, + /// Request-only overrides selected by the strategy for this transcript. + pub polish_request_overrides: Option, pub front_app: Option, pub cursor_context: Option, /// Newest-first turns captured when the session starts. @@ -350,6 +354,11 @@ impl DictationContext { llm_thinking_enabled: preferences.llm_thinking_enabled, context_window_minutes: preferences.polish_context_window_minutes, polish_skip_max_chars: preferences.polish_skip_max_chars, + polish_strategy: crate::polish_strategy::PolishStrategyConfig { + enabled: preferences.polish_strategy_enabled, + script: preferences.polish_strategy_script.clone(), + }, + polish_request_overrides: None, front_app: options.front_app.clone().and_then(non_blank_owned), cursor_context: options.cursor_context.clone().and_then(non_blank_owned), prior_turns, @@ -711,16 +720,23 @@ mod tests { #[test] fn textstream_stays_live_without_changing_the_users_batch_preference() { - let preferences = UserPreferences { stable_transcription_enabled: true, ..UserPreferences::default() }; + let preferences = UserPreferences { + stable_transcription_enabled: true, + ..UserPreferences::default() + }; let pack = builtin_style_pack_for_mode(PolishMode::Structured); for (provider, buffered) in [("experimental-textstream", false), ("volcengine", true)] { let context = DictationContext::capture( - &preferences, &pack, + &preferences, + &pack, DictationProviderInvocations::new( ProviderInvocation::for_provider(provider), ProviderInvocation::for_provider("openai"), ProviderInvocation::for_provider("omni"), - ), Vec::new(), Vec::new(), &DictationStartOptions::default(), + ), + Vec::new(), + Vec::new(), + &DictationStartOptions::default(), ); assert_eq!(context.recording.transcribe_after_stop, buffered); } diff --git a/openless-all/app/crates/openless-core/src/dictation_engine.rs b/openless-all/app/crates/openless-core/src/dictation_engine.rs index 376ab6b23..24167441a 100644 --- a/openless-all/app/crates/openless-core/src/dictation_engine.rs +++ b/openless-all/app/crates/openless-core/src/dictation_engine.rs @@ -619,6 +619,7 @@ impl DictationEngine for PipelineDictationEngine { let asr_started = std::time::Instant::now(); let mut asr_call_label = transcription.asr_call_label(); let transcription_result = transcription.finish().await; + let mut asr_metadata = transcription.transcript_metadata(); asr_call_label = transcription.asr_call_label().or(asr_call_label); for notification in transcription.take_progress_notifications() { publish_progress( @@ -664,8 +665,9 @@ impl DictationEngine for PipelineDictationEngine { ) .await { - Ok((transcript, label)) => { + Ok((transcript, label, metadata)) => { asr_call_label = label; + asr_metadata = metadata; transcript } Err((error, label)) => { @@ -760,8 +762,52 @@ impl DictationEngine for PipelineDictationEngine { is_final: true, }), )?; + // Issue #1225: evaluate the optional policy only after correction and + // final transcript publication, so the script sees what the user will + // actually receive and cannot affect raw ASR diagnostics. + let strategy_input = crate::polish_strategy::PolishStrategyInput { + char_count: transcript + .text + .chars() + .filter(|character| !character.is_whitespace()) + .count(), + text: transcript.text.clone(), + audio_duration_ms: transcript.duration_ms, + language: asr_metadata + .language + .clone() + .or_else(|| context.asr.language.clone()), + confidence: asr_metadata.confidence, + asr_provider: asr_call_label + .as_ref() + .map(|label| label.provider.clone()) + .unwrap_or_else(|| context.asr.provider_type.clone()), + asr_model: asr_call_label + .as_ref() + .and_then(|label| label.model.clone()) + .or_else(|| context.asr.model.clone()), + }; + let strategy_decision = (context.output_target + != crate::dictation_context::DictationOutputTarget::Qa) + .then(|| { + crate::polish_strategy::evaluate( + &context.polish.polish_strategy, + &strategy_input, + ) + }) + .flatten(); + let strategy_overrides = match &strategy_decision { + Some(crate::polish_strategy::PolishStrategyDecision::Request(overrides)) => { + Some(overrides.clone()) + } + _ => None, + }; + let strategy_skips = strategy_skips_polish(&context, strategy_decision.as_ref()); // Issue #1032: short transcripts skip polish (user-configured threshold). - let uses_polisher = context.uses_llm_polisher() + // A policy error or `polish` decision deliberately retains this legacy + // path; only an explicit `skip` suppresses it. + let uses_polisher = !strategy_skips + && context.uses_llm_polisher() && !context.skips_polish_for_short_text(&transcript.text); let (polish_output, polish_failed, polish_ms) = if uses_polisher { publish_progress( @@ -785,13 +831,21 @@ impl DictationEngine for PipelineDictationEngine { progress: Arc::clone(&progress), }); let polish_started = std::time::Instant::now(); + let polish_context = strategy_overrides.as_ref().map_or_else( + || Arc::clone(&context), + |overrides| { + let mut updated = (*context).clone(); + updated.polish.polish_request_overrides = Some(overrides.clone()); + Arc::new(updated) + }, + ); let output = if let Some(error) = context.deferred_llm_error.clone() { Err(error) } else { polisher .polish( session_id, - context, + polish_context, transcript.text.clone(), polish_partials, ) @@ -1108,6 +1162,7 @@ async fn retry_transcription( ( crate::ports::TranscriptOutput, Option, + crate::ports::TranscriptMetadata, ), (BackendError, Option), > { @@ -1175,6 +1230,7 @@ async fn retry_transcription( } transcription.consume_pcm_chunk(&pcm); let transcription_result = transcription.finish().await; + let metadata = transcription.transcript_metadata(); for notification in transcription.take_progress_notifications() { if let Err(error) = publish_progress( &session, @@ -1190,7 +1246,7 @@ async fn retry_transcription( session .transcription_finished .store(true, Ordering::Release); - return Ok((output, last_label)); + return Ok((output, last_label, metadata)); } Err(error) => { let _ = cancel_transcription_once(&session, transcription).await; @@ -1262,6 +1318,17 @@ fn publish_progress( progress.publish(session_id, event) } +fn strategy_skips_polish( + context: &DictationContext, + decision: Option<&crate::polish_strategy::PolishStrategyDecision>, +) -> bool { + matches!( + decision, + Some(crate::polish_strategy::PolishStrategyDecision::Skip) + ) && !context.polish.translation_active + && context.is_plain_dictation_output() +} + fn can_fallback_to_raw(policy: PolishFailurePolicy, error: &BackendError) -> bool { policy == PolishFailurePolicy::UseRawText && matches!( @@ -1607,6 +1674,13 @@ impl TranscriptionSession for BufferedTranscriptionSession { .and_then(|session| session.asr_call_label()) } + fn transcript_metadata(&self) -> crate::ports::TranscriptMetadata { + self.downstream() + .map_or_else(crate::ports::TranscriptMetadata::default, |session| { + session.transcript_metadata() + }) + } + fn take_progress_notifications(&self) -> Vec { self.downstream() .map_or_else(Vec::new, |session| session.take_progress_notifications()) @@ -3082,6 +3156,73 @@ mod tests { assert_eq!(result.polished_text, "raw text"); } + #[tokio::test] + async fn custom_strategy_skip_bypasses_the_polisher() { + let fixture = fixture_engine( + false, + Ok(crate::ports::PolishOutput::text("must not be used")), + None, + None, + ); + let mut context = DictationContext::default(); + context.polish.polish_strategy = crate::polish_strategy::PolishStrategyConfig { + enabled: true, + script: r#"fn strategy(input) { #{ action: "skip" } }"#.into(), + }; + let session_id = SessionId::new(); + fixture + .engine + .start(session_id, Arc::new(context), fixture.progress.clone()) + .await + .unwrap(); + let result = fixture + .engine + .finish(session_id, fixture.progress.clone()) + .await + .unwrap(); + assert_eq!(fixture.polish_calls.load(Ordering::SeqCst), 0); + assert_eq!(result.polished_text, "raw text"); + } + + #[tokio::test] + async fn custom_strategy_request_overrides_are_scoped_to_the_polish_call() { + let fixture = fixture_engine( + false, + Ok(crate::ports::PolishOutput::text("polished text")), + None, + None, + ); + let mut context = DictationContext::default(); + context.polish.polish_strategy = crate::polish_strategy::PolishStrategyConfig { + enabled: true, + script: r#"fn strategy(input) { #{ action: "request", maxTokens: 321 } }"#.into(), + }; + let context = Arc::new(context); + let session_id = SessionId::new(); + fixture + .engine + .start(session_id, Arc::clone(&context), fixture.progress.clone()) + .await + .unwrap(); + fixture + .engine + .finish(session_id, fixture.progress.clone()) + .await + .unwrap(); + + assert_eq!(context.polish.polish_request_overrides, None); + let polish_contexts = fixture.polish_contexts.lock().unwrap(); + assert_eq!(polish_contexts.len(), 1); + assert_eq!( + polish_contexts[0] + .polish + .polish_request_overrides + .as_ref() + .and_then(|overrides| overrides.max_tokens), + Some(321) + ); + } + #[tokio::test] async fn short_transcript_under_threshold_skips_polish() { // Issue #1032: "raw text" has 7 non-whitespace characters, within a limit of 8. @@ -3096,11 +3237,7 @@ mod tests { let session_id = SessionId::new(); fixture .engine - .start( - session_id, - Arc::new(context), - fixture.progress.clone(), - ) + .start(session_id, Arc::new(context), fixture.progress.clone()) .await .unwrap(); let result = fixture @@ -3127,11 +3264,7 @@ mod tests { let session_id = SessionId::new(); fixture .engine - .start( - session_id, - Arc::new(context), - fixture.progress.clone(), - ) + .start(session_id, Arc::new(context), fixture.progress.clone()) .await .unwrap(); let result = fixture @@ -3369,6 +3502,20 @@ mod tests { (engine, Arc::new(RecordingProgress::default()), starts) } + #[test] + fn strategy_skip_preserves_translation_and_voice_edit_semantics() { + let context = raw_dictation_context(); + let decision = crate::polish_strategy::PolishStrategyDecision::Skip; + assert!(strategy_skips_polish(&context, Some(&decision))); + + let mut translation = (*context).clone(); + translation.polish.translation_active = true; + assert!(!strategy_skips_polish(&translation, Some(&decision))); + + let voice_edit = context.with_output_target(DictationOutputTarget::VoiceEdit); + assert!(!strategy_skips_polish(&voice_edit, Some(&decision))); + } + fn raw_dictation_context() -> Arc { let mut context = DictationContext::default(); context.polish.mode = crate::types::PolishMode::Raw; diff --git a/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs b/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs index c294f1f38..08539ead7 100644 --- a/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs +++ b/openless-all/app/crates/openless-core/src/experimental_asr/tests.rs @@ -593,7 +593,12 @@ async fn optional_runtime_files_are_not_download_progress() { let directory = f.store.model_dir(&Model::TextStream.target()).unwrap(); std::fs::create_dir_all(directory.join(".runtime/cache")).unwrap(); let unrelated = std::fs::File::create(directory.join(".runtime/cache/wheels")).unwrap(); - unrelated.set_len(5 * 1024 * 1024 * 1024).unwrap(); + // A modestly oversized cache file is enough to prove optional runtime + // content is ignored, while remaining portable on Windows filesystems. + unrelated.set_len(5 * 1024 * 1024).unwrap(); + // Windows keeps open file handles from being renamed during the model + // staging commit; the cache file itself remains part of the fixture. + drop(unrelated); let status = f.service.status().unwrap(); assert_eq!(status[Model::TextStream.index()].downloaded_bytes, 0); f.service.set_enabled(Model::TextStream, true).unwrap(); diff --git a/openless-all/app/crates/openless-core/src/lib.rs b/openless-all/app/crates/openless-core/src/lib.rs index 0e3bca1c7..c432d3624 100644 --- a/openless-all/app/crates/openless-core/src/lib.rs +++ b/openless-all/app/crates/openless-core/src/lib.rs @@ -49,6 +49,7 @@ pub mod omni; pub mod output_cleaning; mod persistence; pub mod polish; +pub mod polish_strategy; pub mod ports; pub mod preferences; pub mod prompt_compose; @@ -291,6 +292,10 @@ pub use model_store::{ ModelTransport, ModelTransportRequest, ModelTransportResponse, ReqwestModelTransport, MODEL_PARTIAL_INDEX, MODEL_READY_SENTINEL, }; +pub use polish_strategy::{ + evaluate as evaluate_polish_strategy, PolishRequestOverrides, PolishStrategyConfig, + PolishStrategyDecision, PolishStrategyInput, ReasoningEffort, +}; pub use ports::PreparedTranscription; pub use ports::{ ActiveRecording, AudioConsumer, AudioRecorder, CaptureStartRequest, CapturedTextTarget, @@ -300,8 +305,8 @@ pub use ports::{ InsertWriteResult, NoopEditObservationAdapter, NoopHostActions, NoopHostContextAdapter, PendingAudioCapture, PolishOutput, RecordingArchive, RecordingControlSink, RecordingEvent, RecordingProgressSink, ResourceResolver, TextInserter, TextInsertionSession, TextPolisher, - TextStreamChunk, TextStreamSink, TranscriptOutput, TranscriptionEngine, TranscriptionSession, - UnsupportedTextInserter, VoiceCapture, + TextStreamChunk, TextStreamSink, TranscriptMetadata, TranscriptOutput, TranscriptionEngine, + TranscriptionSession, UnsupportedTextInserter, VoiceCapture, }; pub use preferences::PreferencesStore; pub use prompt_compose::{ diff --git a/openless-all/app/crates/openless-core/src/llm_gemini.rs b/openless-all/app/crates/openless-core/src/llm_gemini.rs index 329080e14..5df3157fa 100644 --- a/openless-all/app/crates/openless-core/src/llm_gemini.rs +++ b/openless-all/app/crates/openless-core/src/llm_gemini.rs @@ -47,6 +47,8 @@ pub struct GeminiConfig { /// think by its own default; false = send Gemini's native channel-level /// minimal thinking config. pub thinking_enabled: bool, + /// Optional per-request output token cap selected by the polish strategy. + pub max_output_tokens: Option, } impl GeminiConfig { @@ -62,6 +64,7 @@ impl GeminiConfig { temperature: DEFAULT_TEMPERATURE, request_timeout_secs: DEFAULT_REQUEST_TIMEOUT_SECS, thinking_enabled: false, + max_output_tokens: None, } } @@ -69,6 +72,16 @@ impl GeminiConfig { self.thinking_enabled = enabled; self } + + pub fn with_temperature(mut self, temperature: f32) -> Self { + self.temperature = temperature; + self + } + + pub fn with_max_output_tokens(mut self, max_tokens: Option) -> Self { + self.max_output_tokens = max_tokens; + self + } } pub struct GeminiProvider { @@ -309,6 +322,9 @@ impl GeminiProvider { /// `generationConfig` injection: temperature + channel-level thinkingConfig. fn build_generate_body(&self, system_prompt: &str, contents: Vec) -> Value { let mut generation_config = json!({ "temperature": self.config.temperature }); + if let Some(max_tokens) = self.config.max_output_tokens { + generation_config["maxOutputTokens"] = json!(max_tokens); + } if !self.config.thinking_enabled { generation_config["thinkingConfig"] = disabled_thinking_config(); } @@ -791,6 +807,20 @@ mod tests { assert_eq!(body["contents"][0]["role"], "user"); } + #[test] + fn strategy_request_overrides_gemini_generation_config() { + let cfg = GeminiConfig::new("k", "gemini-2.5-flash", "https://x/v1beta") + .with_temperature(0.7) + .with_max_output_tokens(Some(321)); + let provider = GeminiProvider::new(cfg); + let body = provider.build_generate_body("SYS", vec![user_content("hi")]); + let temperature = body["generationConfig"]["temperature"] + .as_f64() + .expect("Gemini temperature should be numeric"); + assert!((temperature - 0.7).abs() < 1e-6); + assert_eq!(body["generationConfig"]["maxOutputTokens"], 321); + } + #[test] fn build_generate_body_thinking_enabled_omits_thinking_config() { let cfg = GeminiConfig::new("k", "gemini-2.5-flash", "https://x/v1beta") diff --git a/openless-all/app/crates/openless-core/src/llm_protocol.rs b/openless-all/app/crates/openless-core/src/llm_protocol.rs index 3490807b5..dec70484b 100644 --- a/openless-all/app/crates/openless-core/src/llm_protocol.rs +++ b/openless-all/app/crates/openless-core/src/llm_protocol.rs @@ -259,6 +259,9 @@ pub(crate) fn request_body( LlmRequestFormat::Responses => { let mut body = json!({"model": config.model, "stream": stream, "store": false, "input": messages}); + if let Some(max_tokens) = config.max_tokens_override { + body["max_output_tokens"] = json!(max_tokens); + } let model = config .model .trim() @@ -270,16 +273,27 @@ pub(crate) fn request_body( || model.starts_with("gpt-3.5") || model.starts_with("chatgpt-4")) { - let effort = if model.starts_with("gpt-5-pro") - || model.contains("-pro") && model.starts_with("gpt-5.") - { - "high" - } else if config.thinking_enabled { - "medium" + let effort = if crate::polish::openai_model_is_gpt5_pro(&model) { + // gpt-5-pro only supports the high tier; a strategy must not + // override this model capability with low/medium/none. + Some("high") } else { - "low" + config.reasoning_effort.as_deref().or_else(|| { + if config.thinking_enabled { + Some("medium") + } else { + Some("low") + } + }) }; - body["reasoning"] = json!({"effort": effort}); + // `none` is the portable way for a strategy to disable reasoning. + // Responses does not accept it as an effort value, so omit the + // reasoning object rather than sending an invalid request. + if !matches!(effort, Some("none")) { + if let Some(effort) = effort { + body["reasoning"] = json!({"effort": effort}); + } + } } body } @@ -295,7 +309,19 @@ pub(crate) fn request_body( turns.push(message); } } - let mut body = json!({"model": config.model, "stream": stream, "messages": turns, "max_tokens": config.protocol.max_tokens}); + let max_tokens = config.max_tokens_override.filter(|max_tokens| { + let conflicts_with_thinking_budget = config.thinking_enabled + && config.protocol.messages_thinking == MessagesThinking::Budget + && config.protocol.thinking_budget >= *max_tokens; + if conflicts_with_thinking_budget { + log::debug!( + "[llm] omitting strategy maxTokens={} because it is not greater than the configured Messages thinking budget", + max_tokens + ); + } + !conflicts_with_thinking_budget + }); + let mut body = json!({"model": config.model, "stream": stream, "messages": turns, "max_tokens": max_tokens.unwrap_or(config.protocol.max_tokens)}); if !system.is_empty() { body["system"] = json!(system.join("\n\n")); } @@ -310,8 +336,14 @@ pub(crate) fn request_body( body } }; + let official_openai_reasoning_model_without_reasoning = config.protocol.format + == LlmRequestFormat::Responses + && config.provider_id.trim() == "openai" + && crate::polish::openai_model_uses_max_completion_tokens(&config.model) + && body.get("reasoning").is_none(); if body.get("reasoning").is_none() && !(config.protocol.format == LlmRequestFormat::Messages && config.thinking_enabled) + && !official_openai_reasoning_model_without_reasoning { if let Some(temperature) = config.temperature { body["temperature"] = crate::polish::temperature_json(temperature); @@ -759,6 +791,71 @@ mod tests { } } + #[test] + fn strategy_request_overrides_map_to_each_protocol() { + let messages = vec![json!({"role":"user","content":"hello"})]; + let mut config = OpenAICompatibleConfig::new( + "custom", + "test", + "https://example.com/v1", + "key", + "custom-model", + ) + .with_max_tokens_override(Some(321)) + .with_temperature(Some(0.4)); + + // Chat Completions is built by OpenAICompatibleLLMProvider::chat_body; + // request_body intentionally handles only the other protocol shapes. + config.protocol.format = LlmRequestFormat::Responses; + let responses = request_body(&config, true, messages.clone()); + assert_eq!(responses["max_output_tokens"], 321); + assert_eq!(responses["reasoning"]["effort"], "low"); + assert!(responses.get("temperature").is_none()); + + config.provider_id = "openai".into(); + config.model = "gpt-5-mini".into(); + config.reasoning_effort = Some("low".into()); + let responses = request_body(&config, true, messages.clone()); + assert_eq!(responses["max_output_tokens"], 321); + assert_eq!(responses["reasoning"]["effort"], "low"); + + config.reasoning_effort = Some("high".into()); + let responses = request_body(&config, true, messages.clone()); + assert_eq!(responses["reasoning"]["effort"], "high"); + + config.model = "gpt-5-pro".into(); + config.reasoning_effort = Some("low".into()); + let responses = request_body(&config, true, messages.clone()); + assert_eq!(responses["reasoning"]["effort"], "high"); + + config.model = "gpt-5-mini".into(); + config.reasoning_effort = Some("none".into()); + let responses = request_body(&config, true, messages.clone()); + assert!(responses.get("reasoning").is_none()); + assert!(responses.get("temperature").is_none()); + + config.model = "gpt-5-pro".into(); + config.reasoning_effort = Some("none".into()); + let responses = request_body(&config, true, messages.clone()); + assert_eq!(responses["reasoning"]["effort"], "high"); + + config.model = "gpt-5-mini".into(); + config.protocol.format = LlmRequestFormat::Messages; + let messages_body = request_body(&config, false, messages); + assert_eq!(messages_body["max_tokens"], 321); + assert!(messages_body.get("reasoning_effort").is_none()); + + config.thinking_enabled = true; + config.protocol.messages_thinking = MessagesThinking::Budget; + config.protocol.thinking_budget = 1024; + let messages_body = request_body( + &config, + false, + vec![json!({"role":"user","content":"hello"})], + ); + assert_eq!(messages_body["max_tokens"], config.protocol.max_tokens); + assert_eq!(messages_body["thinking"]["budget_tokens"], 1024); + } #[test] fn service_tier_is_validated_and_applied_only_to_ark_requests() { let mut protocol = LlmProtocolConfig::default(); diff --git a/openless-all/app/crates/openless-core/src/local_asr_service.rs b/openless-all/app/crates/openless-core/src/local_asr_service.rs index 06251efaf..255531fd6 100644 --- a/openless-all/app/crates/openless-core/src/local_asr_service.rs +++ b/openless-all/app/crates/openless-core/src/local_asr_service.rs @@ -9,7 +9,9 @@ use std::sync::Arc; use futures_util::future::BoxFuture; -use crate::credentials::{ChannelKind, ChannelMutation, ChannelMutationResult, CredentialStore}; +use crate::credentials::{ + ChannelKind, ChannelMutation, ChannelMutationResult, CredentialStore, ProviderSlot, +}; use crate::domains::{ LocalAsrActivationRequest, LocalAsrActivationResult, LocalAsrApi, LocalAsrModel, LocalAsrModelCard, LocalAsrRemoteInfo, LocalAsrRuntimeStatus, LocalAsrSettings, @@ -625,7 +627,23 @@ impl LocalAsrApi for LocalAsrService { if adapter.restart_required(request.target.runtime) { return Err(restart_required_error()); } - let previous_target = Self::target_for_active_preferences(&previous_preferences); + // Preferences can retain a local provider after an older process + // stopped, while the runtime itself is cold. Only retire the + // selected target when the credential directory confirms that the + // previous local channel is still the active, enabled owner. + let active_provider = match credentials.active_provider(ProviderSlot::Asr).await { + Ok(provider) => provider, + Err(error) if error.code == BackendErrorCode::Unsupported => String::new(), + Err(error) => return Err(error), + }; + let previous_channel_is_active = channels.iter().any(|channel| { + channel.id == active_provider + && channel.enabled + && channel.provider_type == previous_preferences.active_asr_provider + }); + let previous_target = previous_channel_is_active + .then(|| Self::target_for_active_preferences(&previous_preferences)) + .flatten(); if let Some(previous) = previous_target.filter(|target| { target != &request.target || previous_preferences.active_asr_provider != provider_type diff --git a/openless-all/app/crates/openless-core/src/omni.rs b/openless-all/app/crates/openless-core/src/omni.rs index 97a170443..7a2db1ceb 100644 --- a/openless-all/app/crates/openless-core/src/omni.rs +++ b/openless-all/app/crates/openless-core/src/omni.rs @@ -18,9 +18,10 @@ use base64::Engine; use serde_json::{json, Value}; use crate::polish::{ - append_utf8_sse_chunk, apply_openai_compatible_thinking_control, chat_completions_url, - extract_assistant_content, finish_utf8_sse_chunks, http_client_builder, - openai_model_omits_custom_temperature, safe_str_slice, send_with_transient_retry, LLMError, + append_utf8_sse_chunk, apply_openai_compatible_thinking_control_with_effort, + chat_completions_url, extract_assistant_content, finish_utf8_sse_chunks, http_client_builder, + openai_model_omits_custom_temperature, openai_model_uses_max_completion_tokens, safe_str_slice, + send_with_transient_retry, LLMError, }; pub const OMNI_GEMINI_PROVIDER_ID: &str = "gemini"; @@ -38,6 +39,10 @@ pub struct OmniConfig { pub extra_headers: HashMap, pub temperature: Option, pub thinking_enabled: bool, + /// Per-request token cap selected by the post-ASR strategy. + pub max_tokens_override: Option, + /// Per-request reasoning effort selected by the post-ASR strategy. + pub reasoning_effort: Option, } impl OmniConfig { @@ -100,16 +105,27 @@ impl OpenAICompatibleOmni { body["temperature"] = json!(temperature); } } + if let Some(max_tokens) = self.config.max_tokens_override { + let field = if self.config.provider_id.trim() == "openai" + && openai_model_uses_max_completion_tokens(&self.config.model) + { + "max_completion_tokens" + } else { + "max_tokens" + }; + body[field] = json!(max_tokens); + } // DashScope Omni requests text-only output. if self.config.audio_requires_data_url() { body["modalities"] = json!(["text"]); } - apply_openai_compatible_thinking_control( + apply_openai_compatible_thinking_control_with_effort( &mut body, &self.config.provider_id, &self.config.base_url, &self.config.model, self.config.thinking_enabled, + self.config.reasoning_effort.as_deref(), ); body } @@ -373,13 +389,18 @@ impl OmniProvider { label, }; } - let gemini_config = crate::llm_gemini::GeminiConfig::new( + let thinking_enabled = config + .reasoning_effort + .as_deref() + .map(|effort| effort != "none") + .unwrap_or(config.thinking_enabled); + let mut gemini_config = crate::llm_gemini::GeminiConfig::new( config.api_key.clone(), config.model.clone(), config.base_url.clone(), ) - .with_thinking_enabled(config.thinking_enabled); - let mut gemini_config = gemini_config; + .with_thinking_enabled(thinking_enabled) + .with_max_output_tokens(config.max_tokens_override); if let Some(temperature) = config.temperature { gemini_config.temperature = temperature; } @@ -492,6 +513,8 @@ mod tests { extra_headers: HashMap::new(), temperature: Some(0.3), thinking_enabled: false, + max_tokens_override: None, + reasoning_effort: None, } } @@ -758,6 +781,20 @@ mod tests { assert!(body.get("modalities").is_none()); } + #[test] + fn omni_body_applies_strategy_request_overrides() { + let mut omni = config(); + omni.model = "gpt-5-mini".into(); + omni.max_tokens_override = Some(321); + omni.reasoning_effort = Some("high".into()); + let provider = OpenAICompatibleOmni::new(omni); + let body = provider.omni_body(false, vec![json!({"role": "user", "content": "x"})]); + + assert_eq!(body["max_completion_tokens"], 321); + assert_eq!(body["reasoning_effort"], "high"); + assert!(body.get("max_tokens").is_none()); + } + #[test] fn omni_body_sets_text_modalities_for_dashscope() { let mut dashscope = config(); diff --git a/openless-all/app/crates/openless-core/src/polish.rs b/openless-all/app/crates/openless-core/src/polish.rs index 8f5078cac..e52c9d59f 100644 --- a/openless-all/app/crates/openless-core/src/polish.rs +++ b/openless-all/app/crates/openless-core/src/polish.rs @@ -116,6 +116,10 @@ pub struct OpenAICompatibleConfig { /// allowlist checks, but the official OpenAI channel skips ordinary chat models known /// not to support reasoning_effort. pub thinking_enabled: bool, + /// One-call normalized effort selected by the post-ASR strategy, if any. + pub reasoning_effort: Option, + /// One-call token cap selected by the post-ASR strategy. + pub max_tokens_override: Option, } impl OpenAICompatibleConfig { @@ -143,6 +147,8 @@ impl OpenAICompatibleConfig { temperature, request_timeout_secs: DEFAULT_REQUEST_TIMEOUT_SECS, thinking_enabled: false, + reasoning_effort: None, + max_tokens_override: None, } } @@ -151,6 +157,16 @@ impl OpenAICompatibleConfig { self } + pub fn with_reasoning_effort(mut self, effort: Option) -> Self { + self.reasoning_effort = effort; + self + } + + pub fn with_max_tokens_override(mut self, max_tokens: Option) -> Self { + self.max_tokens_override = max_tokens; + self + } + pub fn with_protocol(mut self, protocol: LlmProtocolConfig) -> Self { self.protocol = protocol; self @@ -834,12 +850,23 @@ impl OpenAICompatibleLLMProvider { body["temperature"] = temperature_json(temperature); } } - apply_openai_compatible_thinking_control( + if let Some(max_tokens) = self.config.max_tokens_override { + let field = if self.config.provider_id.trim() == "openai" + && openai_model_uses_max_completion_tokens(&self.config.model) + { + "max_completion_tokens" + } else { + "max_tokens" + }; + body[field] = json!(max_tokens); + } + apply_openai_compatible_thinking_control_with_effort( &mut body, &self.config.provider_id, &self.config.base_url, &self.config.model, self.config.thinking_enabled, + self.config.reasoning_effort.as_deref(), ); self.config .protocol @@ -1131,6 +1158,7 @@ pub struct CodexOAuthConfig { pub model: String, pub auth_path: Option, pub reasoning_effort: Option, + pub max_output_tokens: Option, pub text_verbosity: Option, pub request_timeout_secs: u64, } @@ -1142,6 +1170,7 @@ impl CodexOAuthConfig { model: normalize_codex_model(model.into().as_str()), auth_path: None, reasoning_effort: Some("medium".to_string()), + max_output_tokens: None, text_verbosity: Some("medium".to_string()), request_timeout_secs: DEFAULT_REQUEST_TIMEOUT_SECS, } @@ -1161,6 +1190,16 @@ impl CodexOAuthConfig { self.reasoning_effort = Some(if enabled { "medium" } else { "low" }.to_string()); self } + + pub fn with_reasoning_effort(mut self, effort: Option) -> Self { + self.reasoning_effort = effort; + self + } + + pub fn with_max_output_tokens(mut self, max_tokens: Option) -> Self { + self.max_output_tokens = max_tokens; + self + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -1434,6 +1473,9 @@ impl CodexOAuthLLMProvider { if let Some(effort) = self.config.reasoning_effort.as_deref() { body["reasoning"] = json!({ "effort": effort }); } + if let Some(max_tokens) = self.config.max_output_tokens { + body["max_output_tokens"] = json!(max_tokens); + } if let Some(verbosity) = self.config.text_verbosity.as_deref() { body["text"] = json!({ "verbosity": verbosity }); } @@ -1911,13 +1953,23 @@ fn unix_now_secs() -> u64 { .unwrap_or(0) } -pub(crate) fn apply_openai_compatible_thinking_control( +pub(crate) fn apply_openai_compatible_thinking_control_with_effort( body: &mut Value, provider_id: &str, base_url: &str, model: &str, thinking_enabled: bool, + override_effort: Option<&str>, ) { + // Providers with boolean thinking controls cannot represent four effort + // levels, but the strategy still needs to be able to enable or disable + // thinking for this request. Preserve the exact effort only in the + // provider-specific fields that support it; map low/medium/high to enabled + // for boolean controls. + let base_thinking_enabled = thinking_enabled; + let thinking_enabled = override_effort + .map(|effort| effort != "none") + .unwrap_or(thinking_enabled); if provider_id.trim() == "tencentTokenHub" { apply_tokenhub_chat_thinking_control(body, model, thinking_enabled); return; @@ -1947,10 +1999,51 @@ pub(crate) fn apply_openai_compatible_thinking_control( // Official OpenAI Chat Completions only accepts reasoning_effort on the // reasoning model families; ordinary chat models get a plain 400. Other // compatible channels keep sending it as declared per channel. - let effort = if provider_id.trim() == "openai" { - openai_chat_reasoning_effort(model, thinking_enabled) + let effort = if let Some(override_effort) = override_effort { + if provider_id.trim() == "openai" { + if openai_model_is_gpt5_pro(model) { + if override_effort != "high" { + log::debug!( + "[llm] preserving gpt-5-pro high reasoning effort instead of unsupported strategy effort={}", + override_effort + ); + } + Some("high") + } else if openai_model_supports_reasoning_effort(model, override_effort) { + Some(override_effort) + } else { + log::debug!( + "[llm] omitting unsupported strategy reasoning_effort provider={} model={} effort={}", + provider_id, + model, + override_effort + ); + openai_chat_reasoning_effort(model, base_thinking_enabled) + } + } else if override_effort == "none" { + // StepFun and other compatible reasoning-effort providers do + // not accept `none`; preserve the channel's existing tier. + log::debug!( + "[llm] preserving configured reasoning effort for unsupported strategy effort=none provider={} model={}", + provider_id, + model + ); + Some(if base_thinking_enabled { + "medium" + } else { + "low" + }) + } else { + Some(override_effort) + } + } else if provider_id.trim() == "openai" { + openai_chat_reasoning_effort(model, base_thinking_enabled) } else { - Some(if thinking_enabled { "medium" } else { "low" }) + Some(if base_thinking_enabled { + "medium" + } else { + "low" + }) }; if let Some(effort) = effort { body["reasoning_effort"] = json!(effort); @@ -1961,7 +2054,7 @@ pub(crate) fn apply_openai_compatible_thinking_control( } Some(ThinkingControl::OpenRouterReasoning) => { body["reasoning"] = json!({ - "effort": if thinking_enabled { "medium" } else { "none" }, + "effort": override_effort.unwrap_or(if thinking_enabled { "medium" } else { "none" }), // OpenLess QA/polish output only shows the final answer; reasoning // content, even if generated, must not reach the UI. "exclude": true, @@ -2142,6 +2235,33 @@ fn openai_model_supports_none_reasoning_effort(model: &str) -> bool { .any(|family| openai_model_matches_family(model, family)) } +pub(crate) fn openai_model_is_gpt5_pro(model: &str) -> bool { + let normalized = normalize_openai_model_id(model); + normalized.starts_with("gpt-5-pro") + || normalized.starts_with("gpt-5.") && normalized.contains("-pro") +} + +pub(crate) fn openai_model_uses_max_completion_tokens(model: &str) -> bool { + let normalized = normalize_openai_model_id(model); + normalized.starts_with("o1") + || normalized.starts_with("o3") + || normalized.starts_with("o4") + || normalized.starts_with("gpt-5") + || normalized.starts_with("gpt-6") +} + +fn openai_model_supports_reasoning_effort(model: &str, effort: &str) -> bool { + let normalized = normalize_openai_model_id(model); + if effort == "none" { + return openai_model_supports_none_reasoning_effort(&normalized); + } + normalized.starts_with("o1") + || normalized.starts_with("o3") + || normalized.starts_with("o4") + || normalized.starts_with("gpt-5") + || normalized.starts_with("gpt-6") +} + fn openai_chat_reasoning_effort(model: &str, thinking_enabled: bool) -> Option<&'static str> { let normalized = normalize_openai_model_id(model); @@ -3823,6 +3943,57 @@ mod tests { } } + #[test] + fn strategy_max_tokens_uses_completion_field_for_openai_reasoning_models() { + for model in ["o1", "o3-mini", "o4-mini", "gpt-5-mini", "gpt-6"] { + let provider = OpenAICompatibleLLMProvider::new( + OpenAICompatibleConfig::new( + "openai", + "OpenAI", + "https://api.openai.com/v1", + "k", + model, + ) + .with_max_tokens_override(Some(321)), + ); + let body = provider.chat_body(false, vec![json!({ "role": "user", "content": "hi" })]); + assert_eq!(body["max_completion_tokens"], 321, "{model}"); + assert!(body.get("max_tokens").is_none(), "{model}"); + } + + let custom = OpenAICompatibleLLMProvider::new( + OpenAICompatibleConfig::new( + "custom", + "Custom", + "https://example.test/v1", + "k", + "o3-mini", + ) + .with_max_tokens_override(Some(321)), + ); + let body = custom.chat_body(false, vec![json!({ "role": "user", "content": "hi" })]); + assert_eq!(body["max_tokens"], 321); + assert!(body.get("max_completion_tokens").is_none()); + } + + #[test] + fn strategy_reasoning_effort_preserves_gpt5_pro_high_capability() { + for effort in ["none", "low", "medium", "high"] { + let provider = OpenAICompatibleLLMProvider::new( + OpenAICompatibleConfig::new( + "openai", + "OpenAI", + "https://api.openai.com/v1", + "k", + "gpt-5-pro", + ) + .with_reasoning_effort(Some(effort.to_string())), + ); + let body = provider.chat_body(false, vec![json!({ "role": "user", "content": "hi" })]); + assert_eq!(body["reasoning_effort"], "high", "{effort}"); + } + } + #[test] fn openai_chat_body_lowers_reasoning_when_disabled_for_channel() { let provider = OpenAICompatibleLLMProvider::new(OpenAICompatibleConfig::new( @@ -3872,6 +4043,25 @@ mod tests { assert_eq!(body["reasoning"]["exclude"], true); } + #[test] + fn strategy_reasoning_effort_overrides_openrouter_default() { + let provider = OpenAICompatibleLLMProvider::new( + OpenAICompatibleConfig::new( + "openrouterFree", + "OpenRouter", + "https://openrouter.ai/api/v1", + "k", + "openai/gpt-5-mini", + ) + .with_reasoning_effort(Some("high".into())), + ); + + let body = provider.chat_body(true, vec![json!({ "role": "user", "content": "hi" })]); + + assert_eq!(body["reasoning"]["effort"], "high"); + assert_eq!(body["reasoning"]["exclude"], true); + } + #[test] fn openai_chat_body_adds_openrouter_reasoning_by_channel_not_model() { let provider = OpenAICompatibleLLMProvider::new(OpenAICompatibleConfig::new( @@ -3903,6 +4093,111 @@ mod tests { assert_eq!(body["thinking"]["type"], "disabled"); } + #[test] + fn strategy_reasoning_effort_controls_boolean_thinking_presets() { + let body = |provider_id: &str, endpoint: &str, model: &str, effort: &str| -> Value { + OpenAICompatibleLLMProvider::new( + OpenAICompatibleConfig::new(provider_id, provider_id, endpoint, "k", model) + .with_reasoning_effort(Some(effort.to_string())), + ) + .chat_body(false, vec![json!({ "role": "user", "content": "hi" })]) + }; + + assert_eq!( + body( + "deepseek", + "https://api.deepseek.com/v1", + "deepseek-chat", + "high" + )["thinking"]["type"], + "enabled" + ); + assert_eq!( + body( + "deepseek", + "https://api.deepseek.com/v1", + "deepseek-chat", + "none" + )["thinking"]["type"], + "disabled" + ); + assert_eq!( + body( + "alibabaCoding", + "https://coding-intl.dashscope.aliyuncs.com/v1", + "qwen3-coder", + "medium", + )["enable_thinking"], + true + ); + assert_eq!( + body( + "alibabaCoding", + "https://coding-intl.dashscope.aliyuncs.com/v1", + "qwen3-coder", + "none", + )["enable_thinking"], + false + ); + assert_eq!( + body( + "minimax", + "https://api.minimaxi.com/v1", + "MiniMax-M3", + "low" + )["thinking"]["type"], + "adaptive" + ); + assert_eq!( + body( + "minimax", + "https://api.minimaxi.com/v1", + "MiniMax-M3", + "none" + )["thinking"]["type"], + "disabled" + ); + assert_eq!( + body("lmstudio", "http://localhost:1234/v1", "model", "high")["chat_template_kwargs"] + ["enable_thinking"], + true + ); + assert_eq!( + body("lmstudio", "http://localhost:1234/v1", "model", "none")["chat_template_kwargs"] + ["enable_thinking"], + false + ); + } + + #[test] + fn strategy_reasoning_effort_controls_tokenhub_thinking() { + let body = |model: &str, effort: &str| { + OpenAICompatibleLLMProvider::new( + OpenAICompatibleConfig::new( + "tencentTokenHub", + "Tencent TokenHub", + "https://tokenhub.tencentmaas.com/v1", + "k", + model, + ) + .with_reasoning_effort(Some(effort.to_string())), + ) + .chat_body(false, vec![json!({ "role": "user", "content": "hi" })]) + }; + + assert_eq!( + body("deepseek-v4-pro", "high")["thinking"]["type"], + "enabled" + ); + assert_eq!( + body("deepseek-v4-pro", "none")["thinking"]["type"], + "disabled" + ); + assert_eq!(body("qwen3.5-plus", "low")["enable_thinking"], true); + assert_eq!(body("qwen3.5-plus", "none")["enable_thinking"], false); + assert_eq!(body("kimi-k3", "none").get("reasoning_effort"), None); + } + #[test] fn openai_chat_body_disables_minimax_thinking_by_preset() { // provider_id preset hits "minimax" -> takes the MiniMaxThinking branch; when disabled it @@ -4092,6 +4387,25 @@ mod tests { } } + #[test] + fn strategy_none_preserves_stepfun_reasoning_capability() { + let provider = OpenAICompatibleLLMProvider::new( + OpenAICompatibleConfig::new( + "stepfun", + "StepFun", + "https://api.stepfun.com/v1", + "k", + "step-3.7-flash", + ) + .with_thinking_enabled(true) + .with_reasoning_effort(Some("none".into())), + ); + + let body = provider.chat_body(false, vec![json!({ "role": "user", "content": "hi" })]); + + assert_eq!(body["reasoning_effort"], "medium"); + } + #[test] fn openai_chat_body_falls_back_to_base_url_for_custom_stepfun_endpoint() { // With the "custom" preset + a StepFun base_url, the base_url fallback identification @@ -4751,6 +5065,13 @@ mod tests { ); } + #[test] + fn codex_oauth_config_omits_unsupported_none_reasoning() { + let config = CodexOAuthConfig::new("gpt-5.5").with_reasoning_effort(None); + + assert_eq!(config.reasoning_effort, None); + } + #[test] fn codex_oauth_config_lowers_reasoning_when_thinking_disabled() { let config = CodexOAuthConfig::new("gpt-5.5").with_thinking_enabled(false); @@ -4779,6 +5100,7 @@ mod tests { assert!(request_text.contains(r#""role":"developer"#)); assert!(request_text.contains(r#""type":"input_text"#)); assert!(request_text.contains(r#""reasoning":{"effort":"medium"}"#)); + assert!(request_text.contains(r#""max_output_tokens":321"#)); assert!(!request_text.contains(r#""temperature":"#)); let body = concat!( @@ -4796,7 +5118,8 @@ mod tests { let provider = CodexOAuthLLMProvider::new( CodexOAuthConfig::new("gpt-5.5") .with_base_url(format!("http://{}", addr)) - .with_auth_path(auth_path.clone()), + .with_auth_path(auth_path.clone()) + .with_max_output_tokens(Some(321)), ); let deltas = StdMutex::new(String::new()); let output = provider diff --git a/openless-all/app/crates/openless-core/src/polish_strategy.rs b/openless-all/app/crates/openless-core/src/polish_strategy.rs new file mode 100644 index 000000000..66dd49e3c --- /dev/null +++ b/openless-all/app/crates/openless-core/src/polish_strategy.rs @@ -0,0 +1,368 @@ +//! Bounded, local policy evaluation between ASR and LLM polish. +//! +//! The policy is intentionally small: it cannot perform I/O or call host +//! functions, and it can only select the polish path or override a short +//! allowlist of request fields. Errors fail open to the normal pipeline. + +use std::fmt; + +use rhai::{Dynamic, Engine, Map, Scope}; + +const MAX_SCRIPT_BYTES: usize = 32 * 1024; +const MAX_INPUT_TEXT_BYTES: usize = 128 * 1024; +const MAX_OPERATIONS: u64 = 50_000; +const MAX_CALL_LEVELS: usize = 16; +const MAX_STRING_BYTES: usize = 256 * 1024; +const MAX_COLLECTION_ITEMS: usize = 256; +const MAX_OUTPUT_TOKENS: u32 = 1_000_000; + +/// A strategy configured for one captured dictation session. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PolishStrategyConfig { + pub enabled: bool, + pub script: String, +} + +impl Default for PolishStrategyConfig { + fn default() -> Self { + Self { + enabled: false, + script: String::new(), + } + } +} + +/// Metadata made available to the user policy after ASR completes. +#[derive(Debug, Clone, PartialEq)] +pub struct PolishStrategyInput { + pub text: String, + pub char_count: usize, + pub audio_duration_ms: u64, + pub language: Option, + pub confidence: Option, + pub asr_provider: String, + pub asr_model: Option, +} + +/// A bounded subset of parameters that can be changed for one polish call. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PolishRequestOverrides { + /// Temperature in thousandths, so the captured context remains Eq. + pub temperature_milli: Option, + pub max_tokens: Option, + pub reasoning_effort: Option, +} + +impl PolishRequestOverrides { + pub fn temperature(&self) -> Option { + self.temperature_milli.map(|value| value as f32 / 1_000.0) + } + + pub fn is_empty(&self) -> bool { + self.temperature_milli.is_none() + && self.max_tokens.is_none() + && self.reasoning_effort.is_none() + } +} + +/// Normalized reasoning values accepted by the policy API. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReasoningEffort { + None, + Low, + Medium, + High, +} + +/// Result of evaluating a user strategy. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PolishStrategyDecision { + Skip, + Polish, + Request(PolishRequestOverrides), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StrategyError { + Disabled, + Empty, + ScriptTooLarge, + InputTooLarge, + InvalidReturn, + Runtime, +} + +impl fmt::Display for StrategyError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Disabled => "disabled", + Self::Empty => "empty script", + Self::ScriptTooLarge => "script exceeds the size limit", + Self::InputTooLarge => "transcript exceeds the input limit", + Self::InvalidReturn => "invalid strategy return", + Self::Runtime => "script execution failed", + }) + } +} + +/// Evaluate a policy. Any error returns `None`, which means the caller should +/// retain the ordinary `uses_llm_polisher` and short-transcript decisions. +pub fn evaluate( + config: &PolishStrategyConfig, + input: &PolishStrategyInput, +) -> Option { + match evaluate_inner(config, input) { + Ok(decision) => Some(decision), + Err(error) => { + if !matches!(error, StrategyError::Disabled | StrategyError::Empty) { + log::warn!("[polish-strategy] {error}; falling back to normal polish policy"); + } + None + } + } +} + +fn evaluate_inner( + config: &PolishStrategyConfig, + input: &PolishStrategyInput, +) -> Result { + if !config.enabled { + return Err(StrategyError::Disabled); + } + if config.script.trim().is_empty() { + return Err(StrategyError::Empty); + } + if config.script.len() > MAX_SCRIPT_BYTES { + return Err(StrategyError::ScriptTooLarge); + } + if input.text.len() > MAX_INPUT_TEXT_BYTES { + return Err(StrategyError::InputTooLarge); + } + + let mut engine = Engine::new(); + engine.on_print(|_| {}); + engine.on_debug(|_, _, _| {}); + engine.set_max_operations(MAX_OPERATIONS); + engine.set_max_call_levels(MAX_CALL_LEVELS); + engine.set_max_string_size(MAX_STRING_BYTES); + engine.set_max_array_size(MAX_COLLECTION_ITEMS); + engine.set_max_map_size(MAX_COLLECTION_ITEMS); + + let mut scope = Scope::new(); + scope.push("input", input_map(input)); + let expression = format!("{}\nstrategy(input)", config.script); + let result = engine + .eval_with_scope::(&mut scope, &expression) + .map_err(|_| StrategyError::Runtime)?; + parse_decision(result) +} + +fn input_map(input: &PolishStrategyInput) -> Map { + let mut map = Map::new(); + map.insert("text".into(), input.text.clone().into()); + map.insert("charCount".into(), (input.char_count as i64).into()); + map.insert( + "audioDurationMs".into(), + (input.audio_duration_ms as i64).into(), + ); + map.insert( + "language".into(), + input + .language + .clone() + .map(Dynamic::from) + .unwrap_or(Dynamic::UNIT), + ); + map.insert( + "confidence".into(), + input + .confidence + .map(|value| Dynamic::from(value as f64)) + .unwrap_or(Dynamic::UNIT), + ); + map.insert("asrProvider".into(), input.asr_provider.clone().into()); + map.insert( + "asrModel".into(), + input + .asr_model + .clone() + .map(Dynamic::from) + .unwrap_or(Dynamic::UNIT), + ); + map +} + +fn parse_decision(result: Dynamic) -> Result { + let map = result + .try_cast::() + .ok_or(StrategyError::InvalidReturn)?; + if map.keys().any(|key| { + !matches!( + key.as_str(), + "action" | "temperature" | "maxTokens" | "reasoningEffort" + ) + }) { + return Err(StrategyError::InvalidReturn); + } + let action = map + .get("action") + .and_then(|value| value.clone().try_cast::()) + .ok_or(StrategyError::InvalidReturn)?; + let overrides = parse_overrides(&map)?; + match action.trim() { + "skip" if overrides.is_empty() => Ok(PolishStrategyDecision::Skip), + "polish" if overrides.is_empty() => Ok(PolishStrategyDecision::Polish), + "request" => Ok(PolishStrategyDecision::Request(overrides)), + _ => Err(StrategyError::InvalidReturn), + } +} + +fn parse_overrides(map: &Map) -> Result { + let temperature_milli = match map.get("temperature") { + None => None, + Some(value) => { + let value = numeric_f64(value).ok_or(StrategyError::InvalidReturn)?; + if !value.is_finite() || !(0.0..=2.0).contains(&value) { + return Err(StrategyError::InvalidReturn); + } + Some((value * 1_000.0).round() as u32) + } + }; + let max_tokens = match map.get("maxTokens") { + None => None, + Some(value) => { + let value = value + .clone() + .try_cast::() + .ok_or(StrategyError::InvalidReturn)?; + if !(1..=i64::from(MAX_OUTPUT_TOKENS)).contains(&value) { + return Err(StrategyError::InvalidReturn); + } + Some(value as u32) + } + }; + let reasoning_effort = match map.get("reasoningEffort") { + None => None, + Some(value) => { + let value = value + .clone() + .try_cast::() + .ok_or(StrategyError::InvalidReturn)?; + Some(match value.trim() { + "none" => ReasoningEffort::None, + "low" => ReasoningEffort::Low, + "medium" => ReasoningEffort::Medium, + "high" => ReasoningEffort::High, + _ => return Err(StrategyError::InvalidReturn), + }) + } + }; + Ok(PolishRequestOverrides { + temperature_milli, + max_tokens, + reasoning_effort, + }) +} + +fn numeric_f64(value: &Dynamic) -> Option { + value + .clone() + .try_cast::() + .or_else(|| value.clone().try_cast::().map(|value| value as f64)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn input(text: &str) -> PolishStrategyInput { + PolishStrategyInput { + text: text.to_string(), + char_count: text.chars().filter(|c| !c.is_whitespace()).count(), + audio_duration_ms: 1_250, + language: Some("en".into()), + confidence: Some(0.91), + asr_provider: "test-asr".into(), + asr_model: Some("tiny".into()), + } + } + + fn config(script: &str) -> PolishStrategyConfig { + PolishStrategyConfig { + enabled: true, + script: script.into(), + } + } + + #[test] + fn evaluates_skip_and_exposes_input() { + let decision = evaluate( + &config( + r#"fn strategy(input) { if input["charCount"] <= 3 && input["language"] == "en" { #{ action: "skip" } } else { #{ action: "polish" } } }"#, + ), + &input("hi "), + ); + assert_eq!(decision, Some(PolishStrategyDecision::Skip)); + } + + #[test] + fn evaluates_request_overrides() { + let decision = evaluate( + &config( + r#"fn strategy(input) { #{ action: "request", temperature: 0.4, maxTokens: 400, reasoningEffort: "low" } }"#, + ), + &input("a longer transcript"), + ); + assert_eq!( + decision, + Some(PolishStrategyDecision::Request(PolishRequestOverrides { + temperature_milli: Some(400), + max_tokens: Some(400), + reasoning_effort: Some(ReasoningEffort::Low), + })) + ); + } + + #[test] + fn malformed_or_disabled_scripts_fail_open() { + assert_eq!( + evaluate(&PolishStrategyConfig::default(), &input("hello")), + None + ); + assert_eq!( + evaluate( + &config(r#"fn strategy(input) { #{ action: "unknown" } }"#), + &input("hello") + ), + None + ); + assert_eq!( + evaluate( + &config(r#"fn strategy(input) { #{ action: "polish", unsupported: true } }"#), + &input("hello") + ), + None + ); + } + + #[test] + fn missing_values_are_unit_and_can_be_checked() { + let mut value = input("hello"); + value.language = None; + value.confidence = None; + value.asr_model = None; + let decision = evaluate( + &config( + r#"fn strategy(input) { if input["language"] == () && input["confidence"] == () { #{ action: "skip" } } else { #{ action: "polish" } } }"#, + ), + &value, + ); + assert_eq!(decision, Some(PolishStrategyDecision::Skip)); + } + + #[test] + fn script_size_limit_fails_open() { + let script = "x".repeat(MAX_SCRIPT_BYTES + 1); + assert_eq!(evaluate(&config(&script), &input("hello")), None); + } +} diff --git a/openless-all/app/crates/openless-core/src/ports.rs b/openless-all/app/crates/openless-core/src/ports.rs index 8337cf73a..b4138cc9e 100644 --- a/openless-all/app/crates/openless-core/src/ports.rs +++ b/openless-all/app/crates/openless-core/src/ports.rs @@ -602,12 +602,24 @@ pub struct TranscriptOutput { pub duration_ms: u64, } +#[derive(Debug, Clone, Default, PartialEq)] +pub struct TranscriptMetadata { + /// Detected language, when the ASR provider reports one. + pub language: Option, + /// Provider confidence in the final transcript, when available. + pub confidence: Option, +} + /// One provider transcription session that receives PCM while recording. pub trait TranscriptionSession: AudioConsumer { fn asr_call_label(&self) -> Option { None } + fn transcript_metadata(&self) -> TranscriptMetadata { + TranscriptMetadata::default() + } + /// Drain provider notices discovered during finalization. This lets a /// native adapter report facts such as Foundry GPU-to-CPU fallback without /// publishing UI events or inventing host-only callback policy. diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index 9ff98cb1f..ead704660 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -726,6 +726,12 @@ pub struct UserPreferences { /// always polish regardless of this value. #[serde(default = "default_polish_skip_max_chars")] pub polish_skip_max_chars: u32, + /// Enables the bounded post-ASR polish strategy hook. Disabled by default. + #[serde(default)] + pub polish_strategy_enabled: bool, + /// Rhai source for the bounded post-ASR polish strategy. + #[serde(default)] + pub polish_strategy_script: String, /// Start silently (no main window). Common for login-launch users who /// want the tray rather than a popped-up window. When on, every launch /// path skips the window (manual clicks included); users reach the main @@ -1125,6 +1131,10 @@ struct UserPreferencesWire { #[serde(default = "default_polish_skip_max_chars")] polish_skip_max_chars: u32, #[serde(default)] + polish_strategy_enabled: bool, + #[serde(default)] + polish_strategy_script: String, + #[serde(default)] start_minimized: bool, #[serde(default)] theme_mode: ThemeMode, @@ -1317,6 +1327,8 @@ impl Default for UserPreferencesWire { history_retention_days: prefs.history_retention_days, polish_context_window_minutes: prefs.polish_context_window_minutes, polish_skip_max_chars: prefs.polish_skip_max_chars, + polish_strategy_enabled: prefs.polish_strategy_enabled, + polish_strategy_script: prefs.polish_strategy_script, start_minimized: prefs.start_minimized, theme_mode: prefs.theme_mode, streaming_insert: prefs.streaming_insert, @@ -1534,6 +1546,8 @@ impl<'de> Deserialize<'de> for UserPreferences { history_retention_days: wire.history_retention_days, polish_context_window_minutes: wire.polish_context_window_minutes, polish_skip_max_chars: wire.polish_skip_max_chars, + polish_strategy_enabled: wire.polish_strategy_enabled, + polish_strategy_script: wire.polish_strategy_script, start_minimized: wire.start_minimized, theme_mode: wire.theme_mode, streaming_insert, @@ -1899,6 +1913,8 @@ impl Default for UserPreferences { history_retention_days: default_history_retention_days(), polish_context_window_minutes: default_polish_context_window_minutes(), polish_skip_max_chars: default_polish_skip_max_chars(), + polish_strategy_enabled: false, + polish_strategy_script: String::new(), start_minimized: false, theme_mode: ThemeMode::default(), streaming_insert: true, @@ -3545,6 +3561,27 @@ mod tests { assert_eq!(from_empty.paste_shortcut, PasteShortcut::CtrlV); } + #[test] + fn polish_strategy_defaults_disabled_and_round_trips() { + assert!(!UserPreferences::default().polish_strategy_enabled); + assert!(UserPreferences::default().polish_strategy_script.is_empty()); + let from_empty: UserPreferences = serde_json::from_str("{}").unwrap(); + assert!(!from_empty.polish_strategy_enabled); + assert!(from_empty.polish_strategy_script.is_empty()); + + let configured: UserPreferences = serde_json::from_str( + r#"{"polishStrategyEnabled":true,"polishStrategyScript":"fn strategy(input) { #{ action: \"polish\" } }"}"#, + ) + .unwrap(); + assert!(configured.polish_strategy_enabled); + assert!(configured.polish_strategy_script.contains("strategy")); + let saved = serde_json::to_value(&configured).unwrap(); + assert_eq!(saved["polishStrategyEnabled"], true); + assert!(saved["polishStrategyScript"] + .as_str() + .unwrap() + .contains("strategy")); + } #[test] fn polish_skip_max_chars_defaults_off_and_round_trips() { // Issue #1032: the short-input polish skip is opt-in, so existing users keep polishing. diff --git a/openless-all/app/linux-egui/src/i18n.rs b/openless-all/app/linux-egui/src/i18n.rs index 65374d7d1..ccb22d814 100644 --- a/openless-all/app/linux-egui/src/i18n.rs +++ b/openless-all/app/linux-egui/src/i18n.rs @@ -4062,6 +4062,46 @@ pub const CATALOG: &[Msg] = &[ "短い入力は整形をスキップ（最大文字数）", "짧은 입력은 다듬기 건너뛰기(최대 글자 수)", ), + }, + Msg { + key: "settings.recording.polish_strategy_enabled_desc", + text: row( + "在 ASR 后运行受限的本地策略；异常会回退到默认润色规则，脚本无法访问文件、网络或凭据。", + "在 ASR 後執行受限的本機策略；異常會回退到預設潤色規則，腳本無法存取檔案、網路或憑證。", + "Run a bounded local policy after ASR; errors fall back to normal polish and the script cannot access files, network, or credentials.", + "ASR 後に制限付きのローカル戦略を実行します。エラー時は通常の整形に戻り、外部アクセスはできません。", + "ASR 후 제한된 로컬 전략을 실행합니다. 오류는 일반 다듬기로 되돌아가며 외부 접근은 없습니다.", + ), + }, + Msg { + key: "settings.recording.polish_strategy_enabled_label", + text: row( + "使用自定义润色策略", + "使用自訂潤色策略", + "Use custom polish strategy", + "カスタム整形戦略を使う", + "사용자 지정 다듬기 전략 사용", + ), + }, + Msg { + key: "settings.recording.polish_strategy_script_desc", + text: row( + "定义 strategy(input)，返回 skip、polish 或 request。缺失的语言、置信度和模型值为 ()。", + "定義 strategy(input)，回傳 skip、polish 或 request。缺少的語言、信心度和模型值為 ()。", + "Define strategy(input) and return skip, polish, or request. Missing language, confidence, and model values are ().", + "strategy(input) を定義し、skip、polish、request のいずれかを返します。不明な値は () です。", + "strategy(input)을 정의하고 skip, polish 또는 request를 반환합니다. 없는 값은 ()입니다.", + ), + }, + Msg { + key: "settings.recording.polish_strategy_script_label", + text: row( + "润色策略脚本", + "潤色策略腳本", + "Polish strategy script", + "整形戦略スクリプト", + "다듬기 전략 스크립트", + ), }, Msg { key: "settings.recording.restore_clipboard_desc", diff --git a/openless-all/app/linux-egui/src/linux_app/settings_save.rs b/openless-all/app/linux-egui/src/linux_app/settings_save.rs index eec5a543b..b3be6d344 100644 --- a/openless-all/app/linux-egui/src/linux_app/settings_save.rs +++ b/openless-all/app/linux-egui/src/linux_app/settings_save.rs @@ -203,6 +203,9 @@ impl OpenLessEguiApp { frontend::view_model::SettingsField::RecordAudioForDebug => { preferences.record_audio_for_debug = !preferences.record_audio_for_debug; } + frontend::view_model::SettingsField::PolishStrategyEnabled => { + preferences.polish_strategy_enabled = !preferences.polish_strategy_enabled; + } frontend::view_model::SettingsField::StreamingSaveClipboard => { preferences.streaming_insert_save_clipboard = !preferences.streaming_insert_save_clipboard; @@ -512,6 +515,11 @@ impl OpenLessEguiApp { self.frontend_vm.settings.polish_skip_max_chars = parsed.to_string(); } + frontend::view_model::SettingsTextField::PolishStrategyScript => { + preferences.polish_strategy_script = text.chars().take(32_768).collect(); + self.frontend_vm.settings.polish_strategy_script = + preferences.polish_strategy_script.clone(); + } frontend::view_model::SettingsTextField::AudioRecordingMaxEntries => { preferences.audio_recording_max_entries = text .trim() diff --git a/openless-all/app/linux-egui/src/main.rs b/openless-all/app/linux-egui/src/main.rs index 6379202e5..43bd130e0 100644 --- a/openless-all/app/linux-egui/src/main.rs +++ b/openless-all/app/linux-egui/src/main.rs @@ -4102,6 +4102,7 @@ mod linux_app { s.mute_while_recording = prefs.mute_during_recording; s.audio_enhancement = prefs.audio_enhancement_enabled; s.audio_cue = prefs.audio_cue_on_record; + s.polish_strategy_enabled = prefs.polish_strategy_enabled; s.launch_at_login = prefs.launch_at_login; s.streaming_save_clipboard = prefs.streaming_insert_save_clipboard; s.record_audio_for_debug = prefs.record_audio_for_debug; @@ -4113,6 +4114,7 @@ mod linux_app { s.retention_days = prefs.history_retention_days.to_string(); s.polish_context_window = prefs.polish_context_window_minutes.to_string(); s.polish_skip_max_chars = prefs.polish_skip_max_chars.to_string(); + s.polish_strategy_script = prefs.polish_strategy_script.clone(); s.audio_recording_max_entries = prefs .audio_recording_max_entries .map(|value| value.to_string()) diff --git a/openless-all/app/linux-egui/src/ui/frontend/settings.rs b/openless-all/app/linux-egui/src/ui/frontend/settings.rs index bcfa58334..40da061e0 100644 --- a/openless-all/app/linux-egui/src/ui/frontend/settings.rs +++ b/openless-all/app/linux-egui/src/ui/frontend/settings.rs @@ -2131,6 +2131,30 @@ fn privacy(ui: &mut egui::Ui, vm: &mut FrontendViewModel, actions: &mut Vec { + void savePrefs({ ...prefs, polishStrategyScript: raw.slice(0, 32_768) }); + }; // History cap 200 is the current HISTORY_CAP (persistence.rs:32); the lower bound of 5 keeps a // user-entered 0 from wiping everything as soon as one entry is written; empty string means // unlimited, falling back to null → the backend applies its 200 default. @@ -101,6 +104,29 @@ export function DataStorageSection() { style={{ ...inputStyle, width: 80, textAlign: 'right' }} /> + + void savePrefs({ ...prefs, polishStrategyEnabled: next })} + /> + + +