Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ on:
- 'package.json'
- 'pnpm-lock.yaml'
- '.github/workflows/ci.yml'
pull_request:
branches: [master]
paths:
- 'src-tauri/**'
- 'src/**'
- 'package.json'
- 'pnpm-lock.yaml'
- '.github/workflows/ci.yml'
workflow_dispatch:

permissions:
Expand Down Expand Up @@ -94,6 +102,12 @@ jobs:
- name: Install frontend dependencies
run: pnpm install --frozen-lockfile

- name: Run checks and tests
if: matrix.platform == 'ubuntu-22.04'
run: |
pnpm check
pnpm test

# --no-bundle:只编译 Rust + 前端,跳过 msi/dmg/AppImage 打包,省时间;
# 但所有依赖 crate 仍被完整编译,足以填满热缓存。
- name: Build (no bundle)
Expand Down
5 changes: 5 additions & 0 deletions docs-site/advanced/rotation.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,11 @@ CC Mesh 在多个上游之间轮换请求,并为每个端点配备独立熔断
1. 连续失败达到 `failure_threshold`
2. 错误率在样本数 ≥ `min_requests` 时达到 `error_rate_threshold`

### 自定义设置与端点豁免

- **全局配置**:可在「设置」页面调整「熔断保护」总开关、连续失败阈值与基础冷却时间,修改后运行中的代理会自动平滑重启生效。
- **端点级免熔断**:在端点编辑弹窗中勾选「禁止熔断保护」后,该端点遇故障不会跳闸,始终保留在路由候选名单中。

## 429 限流降噪

429(瞬时限流)与 5xx/网络(端点坏了)分开处理,避免限流突发把端点打进 60-90s 长冷却:
Expand Down
7 changes: 4 additions & 3 deletions docs/KB/patterns/circuit-breaker.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,17 @@
- **请求驱动**:没有后台健康探测线程。Open → HalfOpen 发生在下一次真实请求经过 `is_available` / `allow_request` 时(惰性)。
- **运行期内存**:`BreakerRegistry` 挂在 `ProxyState` 上,代理启动时新建。停代理或重启进程即清空。
- **每端点独立**:A 熔断不影响 B;轮换与熔断都先按模型过滤,避免无关端点被误伤。
- **配置本期固定**:`CircuitBreakerConfig` 结构预留热更新,当前用默认常量。
- **支持全局与端点级配置**:可在「设置」页自定义全局总开关、失败阈值与冷却时长(平滑重启代理生效);可在端点编辑弹窗中单独开启「禁止熔断」豁免跳闸。

实现入口:

| 层 | 文件 | 职责 |
|----|------|------|
| 状态机 | `src-tauri/src/modules/proxy/circuit_breaker.rs` | 三态、许可、计数、选路过滤 |
| 状态机 | `src-tauri/src/modules/proxy/circuit_breaker.rs` | 三态、许可、计数、选路过滤、自定义选项构造 |
| 结果分类 | `src-tauri/src/modules/proxy/rotation.rs` | HTTP / 网络错误 → Retryable / NonRetryable |
| 接入 | `src-tauri/src/modules/proxy/forward.rs` | 选路、gate、上报、发健康事件 |
| 接入 | `src-tauri/src/modules/proxy/forward.rs` | 选路、gate、上报、发健康事件、单端点免熔断跳过 |
| 手动恢复 | `src-tauri/src/commands/endpoint.rs` | 连通性测试成功 → `force_close` |
| 配置设置 | `src/pages/Settings/_components/CircuitBreakerCard.tsx` + `EndpointForm.tsx` | 全局开关/阈值及单端点免熔断配置 |
| 对外 | `src-tauri/src/commands/health.rs` + 前端 `useEndpointHealth` | 卡片 Badge / 仪表盘状态点 |

---
Expand Down
2 changes: 1 addition & 1 deletion src-tauri/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 7 additions & 4 deletions src-tauri/src/commands/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,15 @@ pub async fn set_config(
config_repo::set_value(&conn, k, v)?;
}
let port_changed = patch.contains_key("port") && patch.get("port") != old_port.as_ref();
// 代理地址 / 启用代理 / 伪装 UA 变更需重建转发 client → 重启代理使其生效
let proxy_or_ua_changed = patch.contains_key("proxyUrl")
// 代理地址 / 启用代理 / 伪装 UA / 熔断配置变更需重建转发 client 或熔断器 → 重启代理使其生效
let proxy_or_breaker_changed = patch.contains_key("proxyUrl")
|| patch.contains_key("proxyEnabled")
|| patch.contains_key("openaiUa")
|| patch.contains_key("claudeCliUa");
port_changed || proxy_or_ua_changed
|| patch.contains_key("claudeCliUa")
|| patch.contains_key("circuitBreakerEnabled")
|| patch.contains_key("circuitBreakerFailureThreshold")
|| patch.contains_key("circuitBreakerTimeout");
port_changed || proxy_or_breaker_changed
};

if needs_restart {
Expand Down
1 change: 1 addition & 0 deletions src-tauri/src/commands/endpoint.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,7 @@ pub fn clone_endpoint(state: State<AppState>, id: i64) -> AppResult<Endpoint> {
header_overrides_enabled: src.header_overrides_enabled,
remark: src.remark,
fast: src.fast,
circuit_breaker_disabled: src.circuit_breaker_disabled,
};
endpoint_repo::create(&conn, &req)
}
Expand Down
12 changes: 9 additions & 3 deletions src-tauri/src/commands/health.rs
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,15 @@ pub fn get_endpoint_health(state: State<AppState>) -> AppResult<Vec<EndpointHeal
Some(h) => enabled
.iter()
.map(|e| {
h.state.breakers.health_of(&e.name).unwrap_or_else(|| {
EndpointHealthInfo::from_test_status(&e.name, &e.test_status)
})
if !h.state.breakers.is_enabled() || e.circuit_breaker_disabled {
let mut info = EndpointHealthInfo::from_test_status(&e.name, &e.test_status);
info.circuit = "closed".to_string();
info
} else {
h.state.breakers.health_of(&e.name).unwrap_or_else(|| {
EndpointHealthInfo::from_test_status(&e.name, &e.test_status)
})
}
})
.collect(),
None => enabled
Expand Down
9 changes: 9 additions & 0 deletions src-tauri/src/models/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ pub struct AppConfig {
pub openai_ua: String,
/// 转发到 Claude 端点时覆盖 User-Agent(空=透传客户端)。
pub claude_cli_ua: String,
/// 是否启用端点熔断保护(默认开)。
pub circuit_breaker_enabled: bool,
/// 熔断器连续失败触发阈值(次,默认 4)。
pub circuit_breaker_failure_threshold: u32,
/// 熔断器冷却时长(秒,默认 60)。
pub circuit_breaker_timeout: u64,
pub update: UpdateSettings,
pub webdav: WebDavConfig,
}
Expand All @@ -84,6 +90,9 @@ impl Default for AppConfig {
proxy_for_update: false,
openai_ua: ua::codex_probe_ua(),
claude_cli_ua: ua::CLAUDE_PROBE_UA.into(),
circuit_breaker_enabled: true,
circuit_breaker_failure_threshold: 4,
circuit_breaker_timeout: 60,
update: UpdateSettings::default(),
webdav: WebDavConfig::default(),
}
Expand Down
5 changes: 5 additions & 0 deletions src-tauri/src/models/endpoint.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ pub struct Endpoint {
pub fast: bool,
/// 快速队列内独立排序,不影响全局 sort_order。
pub fast_sort_order: i64,
/// 是否禁止该端点熔断保护(true 表示故障不跳闸,始终可路由)。
pub circuit_breaker_disabled: bool,
/// 测试状态:unknown / available / unavailable。
pub test_status: String,
pub created_at: String,
Expand Down Expand Up @@ -96,6 +98,8 @@ pub struct CreateEndpointRequest {
pub remark: String,
#[serde(default)]
pub fast: bool,
#[serde(default)]
pub circuit_breaker_disabled: bool,
}

#[derive(Debug, Clone, Default, Deserialize)]
Expand All @@ -117,6 +121,7 @@ pub struct UpdateEndpointRequest {
pub header_overrides_enabled: Option<bool>,
pub remark: Option<String>,
pub fast: Option<bool>,
pub circuit_breaker_disabled: Option<bool>,
}

fn default_true() -> bool {
Expand Down
1 change: 1 addition & 0 deletions src-tauri/src/modules/cc_switch_migration/importer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ pub async fn import(
header_overrides_enabled: false,
remark: p.remark.clone(),
fast: false,
circuit_breaker_disabled: false,
};
endpoint_repo::create(conn, &req)?;

Expand Down
116 changes: 111 additions & 5 deletions src-tauri/src/modules/proxy/circuit_breaker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -286,23 +286,53 @@ impl EndpointHealthInfo {
/// 按端点名池化的熔断器注册表(存 `ProxyState`,运行期内存态)。
/// 持两套 config preset:Claude 入站放宽,OpenAI/Responses 默认。状态按端点共享,仅阈值随入站而定。
pub struct BreakerRegistry {
enabled: bool,
config_default: CircuitBreakerConfig,
config_claude: CircuitBreakerConfig,
inner: Mutex<HashMap<String, BreakerInner>>,
}

impl BreakerRegistry {
/// 生产构造:默认 + Claude 两套 preset。
/// 生产构造:默认 + Claude 两套 preset(默认开启)。
#[allow(dead_code)]
pub fn new() -> Self {
Self::with_configs(
CircuitBreakerConfig::default(),
CircuitBreakerConfig::claude(),
)
}

/// 显式指定两套 preset(测试用)。
/// 根据用户配置项构造。
pub fn from_options(enabled: bool, failure_threshold: u32, timeout_secs: u64) -> Self {
let ft = failure_threshold.max(1);
let timeout = Duration::from_secs(timeout_secs.max(1));
let default_cfg = CircuitBreakerConfig {
failure_threshold: ft,
timeout,
..CircuitBreakerConfig::default()
};
let claude_cfg = CircuitBreakerConfig {
failure_threshold: ft.saturating_mul(2),
timeout: Duration::from_secs(timeout_secs.max(1).saturating_mul(3) / 2),
..CircuitBreakerConfig::claude()
};
Self::with_configs_and_enabled(enabled, default_cfg, claude_cfg)
}

/// 显式指定两套 preset(测试用,默认开启)。
#[allow(dead_code)]
pub fn with_configs(default: CircuitBreakerConfig, claude: CircuitBreakerConfig) -> Self {
Self::with_configs_and_enabled(true, default, claude)
}

/// 显式指定是否开启及两套 preset。
pub fn with_configs_and_enabled(
enabled: bool,
default: CircuitBreakerConfig,
claude: CircuitBreakerConfig,
) -> Self {
Self {
enabled,
config_default: default,
config_claude: claude,
inner: Mutex::new(HashMap::new()),
Expand All @@ -312,7 +342,12 @@ impl BreakerRegistry {
/// 测试用:两套 preset 设为同一份 config。
#[cfg(test)]
pub fn new_uniform(config: CircuitBreakerConfig) -> Self {
Self::with_configs(config, config)
Self::with_configs_and_enabled(true, config, config)
}

/// 是否开启熔断保护。
pub fn is_enabled(&self) -> bool {
self.enabled
}

fn cfg_for(&self, inbound: InboundKind) -> &CircuitBreakerConfig {
Expand All @@ -324,6 +359,9 @@ impl BreakerRegistry {

/// 选路过滤用:端点当前是否可选(不占半开许可)。Open 到期会惰性转 HalfOpen。
pub fn is_available(&self, name: &str, _inbound: InboundKind, now: Instant) -> bool {
if !self.enabled {
return true;
}
let mut g = self.inner.lock().unwrap();
let b = g.entry(name.to_string()).or_default();
b.maybe_half_open(now);
Expand All @@ -332,6 +370,12 @@ impl BreakerRegistry {

/// 发请求前取许可。HalfOpen 同一时刻只放行 1 个探测。
pub fn allow_request(&self, name: &str, _inbound: InboundKind, now: Instant) -> AllowResult {
if !self.enabled {
return AllowResult {
allowed: true,
used_half_open_permit: false,
};
}
let mut g = self.inner.lock().unwrap();
let b = g.entry(name.to_string()).or_default();
b.maybe_half_open(now);
Expand Down Expand Up @@ -363,6 +407,9 @@ impl BreakerRegistry {

/// 记录成功。返回是否发生状态转换(供调用方发事件)。
pub fn record_success(&self, name: &str, used_permit: bool, inbound: InboundKind) -> bool {
if !self.enabled {
return false;
}
let cfg = self.cfg_for(inbound);
let mut g = self.inner.lock().unwrap();
let b = g.entry(name.to_string()).or_default();
Expand Down Expand Up @@ -392,6 +439,9 @@ impl BreakerRegistry {
inbound: InboundKind,
kind: FailureKind,
) -> bool {
if !self.enabled {
return false;
}
let cfg = self.cfg_for(inbound);
let mut g = self.inner.lock().unwrap();
let b = g.entry(name.to_string()).or_default();
Expand Down Expand Up @@ -428,7 +478,7 @@ impl BreakerRegistry {

/// 记录中性结果(客户端错误/中断):仅释放半开许可,不计入熔断。
pub fn record_neutral(&self, name: &str, used_permit: bool) {
if !used_permit {
if !self.enabled || !used_permit {
return;
}
let mut g = self.inner.lock().unwrap();
Expand Down Expand Up @@ -473,6 +523,9 @@ impl BreakerRegistry {
_inbound: InboundKind,
now: Instant,
) -> Option<Duration> {
if !self.enabled {
return None;
}
let g = self.inner.lock().unwrap();
let mut soonest: Option<Duration> = None;
let mut any_open = false;
Expand Down Expand Up @@ -529,9 +582,12 @@ pub fn select_candidates(
inbound: InboundKind,
now: Instant,
) -> Vec<Endpoint> {
if !registry.is_enabled() {
return enabled.to_vec();
}
enabled
.iter()
.filter(|e| registry.is_available(&e.name, inbound, now))
.filter(|e| e.circuit_breaker_disabled || registry.is_available(&e.name, inbound, now))
.cloned()
.collect()
}
Expand Down Expand Up @@ -573,6 +629,7 @@ mod tests {
sort_order: 0,
fast: false,
fast_sort_order: 0,
circuit_breaker_disabled: false,
test_status: "unknown".into(),
created_at: "".into(),
updated_at: "".into(),
Expand Down Expand Up @@ -1146,4 +1203,53 @@ mod tests {
assert!(!reg.is_available("c", InboundKind::OpenAi, now + Duration::from_secs(61)));
assert!(reg.is_available("c", InboundKind::OpenAi, now + Duration::from_secs(91)));
}

#[test]
fn disabled_registry_always_allows_and_never_trips() {
let reg = BreakerRegistry::from_options(false, 3, 60);
let now = Instant::now();
for _ in 0..10 {
let changed = reg.record_failure(
"a",
false,
now,
"500",
InboundKind::OpenAi,
FailureKind::Broken,
);
assert!(!changed);
}
assert!(reg.is_available("a", InboundKind::OpenAi, now));
let allow = reg.allow_request("a", InboundKind::OpenAi, now);
assert!(allow.allowed);
assert!(!allow.used_half_open_permit);
}

#[test]
fn select_candidates_honors_disabled_endpoint() {
let reg = BreakerRegistry::new_uniform(cfg());
let now = Instant::now();
for _ in 0..3 {
reg.record_failure(
"a",
false,
now,
"500",
InboundKind::OpenAi,
FailureKind::Broken,
);
}
let mut ep_a = ep("a");
let ep_b = ep("b");
let eps = vec![ep_a.clone(), ep_b.clone()];
let cands = select_candidates(&eps, &reg, InboundKind::OpenAi, now);
assert_eq!(cands.len(), 1);
assert_eq!(cands[0].name, "b");

// 当 a 标记了 circuit_breaker_disabled,即使处于 open 也保留在候选
ep_a.circuit_breaker_disabled = true;
let eps2 = vec![ep_a, ep_b];
let cands2 = select_candidates(&eps2, &reg, InboundKind::OpenAi, now);
assert_eq!(cands2.len(), 2);
}
}
Loading