Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions .github/workflows/docker-smoke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,14 @@ jobs:
mods = [
"integrated_app.vendor.voxcpm",
"integrated_app.engines.voxcpm2.engine",
# 真正 `from einops import rearrange` 的那三个模块。只列包根目录不够:
# 2026-09-21 这一步第一次真跑就抓出 `No module named 'einops'`(run 35618578940),
# 而 einops 只在 funasr/modelscope 的 **extras** 里被声明,核心集装不出来 ——
# 点名到模块,报的错才等于"谁缺的依赖"。由 D5(tests/test_dependency_consistency.py)
# 在跑 pytest 的门禁上先拦一道,这里兜住"只有干净镜像才暴露"的那一半。
"integrated_app.vendor.voxcpm.model.voxcpm",
"integrated_app.vendor.voxcpm.model.voxcpm2",
"integrated_app.vendor.voxcpm.modules.locenc.local_encoder",
]
bad = []
for m in mods:
Expand All @@ -185,9 +193,13 @@ jobs:
bad.append((m, type(exc).__name__, str(exc)[:200]))
print("FAIL", m, type(exc).__name__, str(exc)[:200])
if bad:
print("::error::镜像内的引擎模块导入失败 —— 检查 pyproject/requirements.txt 的 "
"transformers 与 tokenizers 区间(引擎要求 4.52.x,见 "
"docs/SECURITY_DEPENDABOT_TRIAGE.md §2)")
print("::error::镜像内的引擎模块导入失败 —— 两类原因都要排:")
print("::error:: (a) ModuleNotFoundError 指某个包没装:它多半只在 extras 里声明,"
"核心集(pyproject [project].dependencies / requirements.txt)里没有 → "
"补声明并同步 requirements-lock.txt 与 launcher/requirements-small.txt,"
"D5 会替你守住;")
print("::error:: (b) ImportError 指向 transformers/tokenizers:引擎要求 4.52.x,"
"见 docs/SECURITY_DEPENDABOT_TRIAGE.md §2")
sys.exit(1)
PY

Expand Down
34 changes: 32 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,31 @@
# README 推荐版本统一,消除"测的是 3.12、发的是 3.10"漂移。
# Ubuntu 22.04 官方源无 3.12,经 deadsnakes PPA 提供(torch/funasr 等全量依赖均有 cp312 wheel)。
# --no-install-recommends:不拉 idle/lib2to3 等推荐包,减小体积与 CVE 面。
# 索引就绪守卫:apt-get update 对「某个索引没抓下来」只打一行
# W: Some index files failed to download. They have been ignored, or old ones used instead.
# 然后 **返回 0**。deadsnakes 的 PPA 一抖,这一步就静默带着半套索引往下走,真正的报错落在
# 下一行、且长得完全不像网络问题:E: Unable to locate package python3.12。
# 2026-09-21 实测(docker-build.yml run 35598902698 / 35599696780 红,35607107888 绿):
# 红的两次日志里有 Ign:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy/main amd64 Packages,
# 绿的那次同一行是 Get:7 ... Packages [44.3 kB]。同一条 update-alternatives: error:
# alternative path /usr/share/man/man7/bash-builtins.7.gz 在绿色 run 里也出现 → 它是
# apt-get upgrade 期间的良性噪声,不是构建失败的原因(先前误把它当根因,已推翻)。
# 所以:把「PPA 索引里真查得到 python3.12」当作继续装的前置条件,取不到就重试,
# 5 轮仍取不到就地硬停说清原因——不带着半套索引继续构建。
RUN apt-get update && apt-get install -y --no-install-recommends \
software-properties-common git git-lfs ffmpeg ca-certificates \
&& add-apt-repository -y ppa:deadsnakes/ppa \
&& apt-get update && apt-get install -y --no-install-recommends \
&& for i in 1 2 3 4 5; do \
apt-get update -o Acquire::Retries=3 -o Acquire::http::Timeout=30; \
if apt-cache show python3.12 >/dev/null 2>&1; then break; fi; \
if [ "$i" = 5 ]; then \
echo "E: deadsnakes 索引重试 5 轮仍查不到 python3.12 —— 是 PPA 侧或网络故障,不是本仓依赖声明问题。" >&2; \
echo " 可手工核对:curl -sI https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu/dists/jammy/main/binary-amd64/Packages.gz" >&2; \
exit 1; \
fi; \
echo "W: deadsnakes 索引未就绪,15s 后第 $((i+1)) 轮重试" >&2; sleep 15; \
done \
&& apt-get install -y --no-install-recommends \
python3.12 python3.12-venv \
&& rm -rf /var/lib/apt/lists/* \
&& python3.12 -m ensurepip --upgrade
Expand Down Expand Up @@ -57,7 +78,16 @@
RUN apt-get update && apt-get install -y --no-install-recommends \
software-properties-common ca-certificates ffmpeg \
&& add-apt-repository -y ppa:deadsnakes/ppa \
&& apt-get update && apt-get upgrade -y \
&& for i in 1 2 3 4 5; do \
apt-get update -o Acquire::Retries=3 -o Acquire::http::Timeout=30; \
if apt-cache show python3.12 >/dev/null 2>&1; then break; fi; \
if [ "$i" = 5 ]; then \
echo "E: deadsnakes 索引重试 5 轮仍查不到 python3.12(原因与修法见 builder 阶段同一段守卫的注释)。" >&2; \
exit 1; \
fi; \
echo "W: deadsnakes 索引未就绪,15s 后第 $((i+1)) 轮重试" >&2; sleep 15; \
done \
&& apt-get upgrade -y \
&& apt-get install -y --no-install-recommends python3.12 python3.12-venv \
&& rm -rf /var/lib/apt/lists/* \
&& python3.12 -m ensurepip --upgrade \
Expand Down Expand Up @@ -103,7 +133,7 @@

# 容器内默认开启 API Auth:0.0.0.0 监听必须配合认证(run_server 安全网要求)。
# Token 优先取 TTS_API_AUTH_TOKEN;未提供时由应用启动时生成一次性随机 token 并打印到日志。
ENV TTS_API_AUTH_ENABLED=1

Check warning on line 136 in Dockerfile

View workflow job for this annotation

GitHub Actions / Boot hardened container & probe

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ENV "TTS_API_AUTH_ENABLED") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/

Check warning on line 136 in Dockerfile

View workflow job for this annotation

GitHub Actions / Build & Scan Image

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ENV "TTS_API_AUTH_ENABLED") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/

# 自动加载模型(app_server.lifespan 实际消费这两个变量,非死配置)
ENV TTS_AUTO_LOAD_MODEL=1
Expand Down
28 changes: 17 additions & 11 deletions docs/DOD.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,17 +185,23 @@
② 完整冒烟 10 格全绿:`engine_imports` → `ready` → `csrf_ticket` → voxcpm2 321,962 B →
切 2.5 → `openai_tts1hd_contract` 400 → 2.5 = 386,796 B → 切 2.0 → 2.0 = 402,400 B →
版本门负向按预期拒绝;结束显存回落 2,666 MiB。
* **镜像构建今天起在 CI 上确定性失败(与本仓改动无关的基础设施故障,未修)**:
`Dockerfile` 的 `apt-get install` 层报
`update-alternatives: error: alternative path /usr/share/man/man7/bash-builtins.7.gz doesn't exist`
→ buildx 失败,`Build & Scan Image` 与 `Boot hardened container & probe` 两个作业同时红。
判据链:main 上 11:07 的同类构建还是 **success**(`41f5a12`/`1b29d0f`),12:19 与 12:28 两次
PR 构建红在**同一步**,且**各重试一次仍然一模一样** → 不是瞬时网络、也不是本 PR 引入
(本 PR 没碰 `Dockerfile`,失败发生在我的探针步骤之前)。jammy 已进入归档期,
疑点是 `software-properties-common` 一条依赖链带进来的 man-db/manpages 组合。
**我没有改 Dockerfile**:本机没有 docker daemon,任何 apt/dpkg 层的规避手法(
`path-exclude=/usr/share/man/*` 之类)在我这儿都是盲改,而它会改变发版镜像的内容 ——
要改就该在能真构建的环境里改并验,不该靠 CI 试错。交所有者定谁来做。
* **镜像构建的间歇性失败已定位并修掉(PR #111)—— 记一次我自己的误判纠偏**:
先前这里写的是"确定性失败、根因是 `update-alternatives: error: alternative path
/usr/share/man/man7/bash-builtins.7.gz doesn't exist`、jammy 归档期 man-db/manpages 组合问题"。
**那是错的**,两条都错:
① 那条 error 在**成功**的构建里同样出现(run 35607107888 13:42:31 `#11 97.72`),
它是 `apt-get upgrade` 期间的良性噪声,与失败无因果;为验证它而开的探针 PR #110 五步全绿,
等于把自己的前提证伪,故关闭。
② 也不是确定性基础设施故障:同一个 `Dockerfile`,12:19/12:28 红、13:40 绿。
"重试一次仍一模一样"只说明那 9 分钟里 PPA 一直取不到,不说明它不是网络问题。
真 fatal(红 run 的 `--log-failed`):
`Ign:7 https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy/main amd64 Packages`
→ `W: Some index files failed to download. They have been ignored, or old ones used instead.`
→ **`apt-get update` 仍然返回 0** → 两条命令之后才炸
`E: Unable to locate package python3.12` / `python3.12-venv`。
绿色 run 同一行是 `Get:7 ... Packages [44.3 kB]`。
性质上正是"警告后照旧继续":报错文案(包不存在)与真原因(索引没抓下来)完全对不上,
下一次抖动随时会再红一遍。修法与验收(含不等抖动的破坏态复现)见 #111 与 `Dockerfile` 注释。
* **仍未覆盖**:桌面安装包链路(staging → data 7z → NSIS)**无任何 workflow 调用**、本机也无从安装
(`scripts/installer/` 只有一个 4.3 MB `Setup.exe`、无同目录分卷),所以 `unpack_desktop.ps1`
新加的许可/字体落地核对只过了语法层,`release_gate.ps1` 的第 ⑥ 步也只在发版/dispatch 时跑;
Expand Down
2 changes: 2 additions & 0 deletions launcher/requirements-small.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
# 安装(CUDA wheel 体积大、不走 PyPI 默认索引)。
# ============================================================

addict==2.4.0
aiofiles==25.1.0
aliyun-python-sdk-core==2.16.1
aliyun-python-sdk-kms==2.16.5
Expand All @@ -24,6 +25,7 @@ colorama==0.4.6
crcmod==1.7
cryptography==50.0.1 # GOTCHAS #98:WinPython 7z 解 NSIS 后预装 cryptography 丢失 -> Ed25519 验签不可用 -> 自检误报缺少有效签名,便携包必须显式钉装
decorator==5.3.1
einops==0.8.2
fastapi==0.141.1
filelock==3.32.7
fsspec==2026.7.0
Expand Down
10 changes: 10 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,16 @@ dependencies = [
# (_load_voxcpm2_engine 强制 include_denoiser=True)会经 speech_zipenhancer
# 走到它。缺失时降噪会降级关闭、音质下降,故列为生产依赖而非可选。
"addict>=2.4.0",
# vendored VoxCPM 的模型代码在**模块顶层** `from einops import rearrange`
# (vendor/voxcpm/model/voxcpm.py:29、voxcpm2.py:30、modules/locenc/local_encoder.py:4),
# 而 funasr / modelscope 只在 extras 里声明 einops(我们装的是无 extras 的核心集)——
# 于是任何只按 [project].dependencies / requirements.txt 装出来的环境都起不来 tts-1:
# 2026-09-21 docker-smoke 的镜像内导入探针第一次真跑就报
# `FAIL integrated_app.vendor.voxcpm ModuleNotFoundError No module named 'einops'`(run 35618578940)。
# 本机 .venv 从未暴露此事,因为 einops 是从 conformer / vector-quantize-pytorch / indextts
# 那条**训练链路传递**进来的(`pip show einops` 的 Required-by 实测),不是我们自己声明的。
# 它是第一方代码的直接依赖,不该靠运气继承 —— 由 tests/test_dependency_consistency.py D5 守住。
"einops>=0.7.0",
# === Web 框架 ===
"fastapi>=0.110.0",
"uvicorn>=0.29.0",
Expand Down
2 changes: 2 additions & 0 deletions requirements-lock.txt
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#
# pip-compile --no-annotate --output-file=requirements-lock.txt requirements.txt
#
addict==2.4.0
aiofiles==25.1.0
aliyun-python-sdk-core==2.16.1
aliyun-python-sdk-kms==2.16.5
Expand All @@ -19,6 +20,7 @@ colorama==0.4.6
crcmod==1.7
cryptography==50.0.1
decorator==5.3.1
einops==0.8.2
fastapi==0.141.1
filelock==3.32.7
fsspec==2026.7.0
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ tokenizers>=0.21.0,<0.22
funasr>=1.0.0
modelscope>=1.9.0
addict>=2.4.0
einops>=0.7.0
fastapi>=0.110.0
uvicorn>=0.29.0
jinja2>=3.1.0
Expand Down
69 changes: 56 additions & 13 deletions scripts/check_pin_crossconflicts.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@
from __future__ import annotations

import argparse
import concurrent.futures
import http.client
import json
import re
import sys
import urllib.error
import time
import urllib.request
from pathlib import Path

Expand Down Expand Up @@ -64,15 +66,48 @@ def requires_dist(name: str, ver: str) -> list[str] | None:
key = f"{name}=={ver}"
if key in _CACHE:
return _CACHE[key]
out = _fetch_requires_dist(name, ver)
if out is not None:
_CACHE[key] = out
return out


def _fetch_requires_dist(name: str, ver: str) -> list[str] | None:
"""抓一个版本的 requires_dist,带重试。

为什么要重试:这是**pre-commit 的一个 hook**,一次网络抖动就会把整条 `git commit`
打掉。2026-09-21 实测:单次 `RemoteDisconnected` 让 hook 吐 traceback(不是按设计
报 unchecked + 退出码 2),而它顺序抓 ~97+94 个包的元数据,撞上一次丢包几乎是必然。

异常面要写宽:`urllib` 只在 `h.request()` 那一步把 OSError 包成 `URLError`,
`h.getresponse()` 里抛的 `http.client.RemoteDisconnected` 是**裸的** OSError 子类,
原先只捕 `URLError` 所以漏了它。
"""
url = f"https://pypi.org/pypi/{name}/{ver}/json"
try:
with urllib.request.urlopen(url, timeout=25) as resp:
data = json.load(resp)
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, ValueError):
for attempt in range(3):
try:
with urllib.request.urlopen(url, timeout=10) as resp:
data = json.load(resp)
break
except (OSError, http.client.HTTPException, ValueError):
if attempt == 2:
return None
time.sleep(0.6 * (attempt + 1))
else: # pragma: no cover - for 循环带 break,理论到不了
return None
out = data.get("info", {}).get("requires_dist") or []
_CACHE[key] = out
return out
return data.get("info", {}).get("requires_dist") or []


def prefetch(pairs: list[tuple[str, str]], workers: int = 8) -> None:
"""并发把元数据灌进 _CACHE,并汇报**取不到**的包(否则只剩一行行 [SKIP] 看不出是网络问题)。"""
todo = [(n, v) for n, v in pairs if f"{n}=={v}" not in _CACHE]
if not todo:
return
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
results = list(pool.map(lambda p: (p, _fetch_requires_dist(*p)), todo))
for (n, v), out in results:
if out is not None:
_CACHE[f"{n}=={v}"] = out


_SPEC_RE = re.compile(r"^\s*([A-Za-z0-9._-]+)\s*(?:\[[^\]]*\])?\s*([^;]*)")
Expand Down Expand Up @@ -139,12 +174,16 @@ def main(argv: list[str] | None = None) -> int:
ap.add_argument("--quiet", action="store_true")
args = ap.parse_args(argv)

existing = [p for rel in LOCK_FILES if (p := _ROOT / rel).exists()]
pairs = sorted({(n, v) for p in existing for n, (v, _) in read_pins(p).items()})
t0 = time.monotonic()
prefetch(pairs)
if not args.quiet:
print(f"[pin-cross] 并发抓 PyPI 元数据 {len(pairs)} 个包,用时 {time.monotonic() - t0:.1f}s")

all_conflicts: list[str] = []
all_unchecked: list[str] = []
for rel in LOCK_FILES:
p = _ROOT / rel
if not p.exists():
continue
for p in existing:
c, u = check(p)
all_conflicts += c
all_unchecked += u
Expand All @@ -166,7 +205,11 @@ def main(argv: list[str] | None = None) -> int:
)
return 1
if all_unchecked:
print("\n[pin-cross] 有依赖未能核验(网络/版本不存在),不当作通过。", file=sys.stderr)
print(
"\n[pin-cross] 有依赖未能核验(重试 3 次后 PyPI 元数据仍取不到),不当作通过。"
"\n 先确认网络/代理可达 pypi.org,再重跑本脚本;这不是锁冲突,别去改锁。",
file=sys.stderr,
)
return 2
print("[pin-cross] PASS 未发现交叉约束冲突")
return 0
Expand Down
61 changes: 61 additions & 0 deletions tests/test_dependency_consistency.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,3 +321,64 @@ def test_d3_d4_guards_are_not_vacuous():
assert "ignorefile:" not in text, f"{wf.name} 里混进了 trivy-action 不认的输入名"
mutated = text.replace(f"{_TRIVY_INPUT}:", "ignorefile:")
assert f"{_TRIVY_INPUT}:" not in mutated, "判据不是恒真:把输入名换成错的那个之后必须失去匹配"


# ---------------------------------------------------------------------------
# D5 核心声明必须在两份钉版集里齐全(2026-09-21 由镜像内导入失败暴露)
# ---------------------------------------------------------------------------

# 便携包里 torch 家族是独立组件(CUDA wheel 不走 PyPI 默认索引),所以它们**理应**不在
# 钉版集中。allowlist 写死成这三条,是为了让"再加一个豁免"必须过一次评审。
_TORCH_TRIO = {"torch", "torchvision", "torchaudio"}


def _core_declared_names() -> set[str]:
"""requirements.txt(pyproject `[project].dependencies` 的同步产物)里的核心包名。"""
names: set[str] = set()
for line in _REQ.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith(("#", "-")):
continue
m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)", line)
if m:
names.add(_norm(m.group(1)))
return names


def _unpinned_core(pins: dict[str, str], core: set[str]) -> set[str]:
return {n for n in core if n not in pins} - _TORCH_TRIO


def test_core_dependencies_are_pinned_in_both_manifests() -> None:
"""第一方代码 import 的东西不能靠传递依赖碰运气。

`app/integrated_app/vendor/voxcpm/model/voxcpm.py:29`、`voxcpm2.py:30` 在**模块顶层**
`from einops import rearrange`,而 funasr / modelscope 只在 extras 里声明 einops
(我们装的是无 extras 的核心集)—— docker-smoke 的镜像内导入探针第一次真跑就报
`FAIL integrated_app.vendor.voxcpm ModuleNotFoundError No module named 'einops'`
(run 35618578940)。本机 .venv 永远看不见这件事,因为 einops 是从训练链路
(`pip show` 实测:conformer / vector-quantize-pytorch / indextts)传递进来的。
`addict` 是同形状的缺席:已声明为生产依赖(modelscope 运行期要),却两份钉版集里都没有。
"""
core = _core_declared_names()
assert len(core) >= 25, f"只解析到 {len(core)} 个核心声明,本条的解析已失效"
for manifest in (_LOCK, _SMALL):
missing = _unpinned_core(_pins(manifest), core)
assert not missing, (
f"{manifest.name} 缺钉版:{sorted(missing)} —— 声明了却没钉,"
"镜像与便携这类干净环境装不出来(开发 venv 里却因为传递依赖而照常能跑)"
)


def test_d5_guard_is_not_vacuous() -> None:
"""变异自证:把 einops 的钉版摘掉,D5 必须变红。"""
core = _core_declared_names()
assert "einops" in core, "einops 不在核心声明里 —— D5 的前提(我们自己顶层 import 它)已不成立"
pins = _pins(_LOCK)
assert "einops" in pins, "锁里没有 einops,D5 现在本该红"
stripped = {k: v for k, v in pins.items() if k != "einops"}
assert "einops" in _unpinned_core(stripped, core), "摘掉钉版却不被抓到 = D5 在空转"
# 反向:torch 三件套缺席是设计如此,不能因为"缺钉版"把便携包判红
assert _unpinned_core({k: v for k, v in _pins(_SMALL).items() if not k.startswith("torch")}, core) == set(), (
"torch 家族应当被 allowlist 排除,否则便携包永远红"
)
Loading