From 314267e53b24d61c7cbf63fac9794868e8bcd549 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 5 Jun 2026 04:44:14 +0000 Subject: [PATCH 1/2] Add VoiceBridge: real-time AI voice changer to a virtual game mic High-level design + implementation of a real-time neural voice-conversion pipeline that turns a physical mic into a cloned-voice virtual microphone for games on Windows (sub-300ms target, RVC/Seed-VC backends; not a pitch shifter). - docs/DESIGN.md: full architecture, latency budget, streaming algorithm - engine.py: chunking + context window + equal-power crossfade streaming - audio_io.py: non-blocking duplex pipeline (queues + worker thread, VB-CABLE) - backends/: passthrough (works today) + rvc + seedvc integration seams - record.py/train.py: guided voice-clone recording and training orchestration - gui.py/cli.py: control panel + CLI; config.py: typed YAML config - tests/: 16 pure-Python unit tests for the DSP/streaming/config (all passing) https://claude.ai/code/session_01Tic4oPjP417WDdfPXW1bFx --- voicebridge/.github/workflows/ci.yml | 25 ++ voicebridge/.gitignore | 38 +++ voicebridge/LICENSE | 21 ++ voicebridge/README.md | 119 +++++++ voicebridge/config.example.yaml | 60 ++++ voicebridge/docs/DESIGN.md | 317 ++++++++++++++++++ voicebridge/docs/LATENCY.md | 50 +++ voicebridge/docs/SETUP_WINDOWS.md | 139 ++++++++ voicebridge/docs/TRAINING.md | 103 ++++++ voicebridge/docs/sentences.txt | 41 +++ voicebridge/pyproject.toml | 36 ++ voicebridge/requirements.txt | 21 ++ voicebridge/scripts/bootstrap_private_repo.sh | 44 +++ voicebridge/scripts/list_devices.py | 9 + voicebridge/src/voicebridge/__init__.py | 11 + voicebridge/src/voicebridge/__main__.py | 8 + voicebridge/src/voicebridge/audio_io.py | 243 ++++++++++++++ .../src/voicebridge/backends/__init__.py | 1 + .../src/voicebridge/backends/passthrough.py | 27 ++ voicebridge/src/voicebridge/backends/rvc.py | 137 ++++++++ .../src/voicebridge/backends/seedvc.py | 71 ++++ voicebridge/src/voicebridge/cli.py | 126 +++++++ voicebridge/src/voicebridge/config.py | 111 ++++++ voicebridge/src/voicebridge/converter.py | 72 ++++ voicebridge/src/voicebridge/dsp.py | 122 +++++++ voicebridge/src/voicebridge/engine.py | 178 ++++++++++ voicebridge/src/voicebridge/gui.py | 149 ++++++++ voicebridge/src/voicebridge/record.py | 146 ++++++++ voicebridge/src/voicebridge/train.py | 98 ++++++ voicebridge/tests/test_config.py | 26 ++ voicebridge/tests/test_dsp.py | 70 ++++ voicebridge/tests/test_engine.py | 73 ++++ 32 files changed, 2692 insertions(+) create mode 100644 voicebridge/.github/workflows/ci.yml create mode 100644 voicebridge/.gitignore create mode 100644 voicebridge/LICENSE create mode 100644 voicebridge/README.md create mode 100644 voicebridge/config.example.yaml create mode 100644 voicebridge/docs/DESIGN.md create mode 100644 voicebridge/docs/LATENCY.md create mode 100644 voicebridge/docs/SETUP_WINDOWS.md create mode 100644 voicebridge/docs/TRAINING.md create mode 100644 voicebridge/docs/sentences.txt create mode 100644 voicebridge/pyproject.toml create mode 100644 voicebridge/requirements.txt create mode 100755 voicebridge/scripts/bootstrap_private_repo.sh create mode 100644 voicebridge/scripts/list_devices.py create mode 100644 voicebridge/src/voicebridge/__init__.py create mode 100644 voicebridge/src/voicebridge/__main__.py create mode 100644 voicebridge/src/voicebridge/audio_io.py create mode 100644 voicebridge/src/voicebridge/backends/__init__.py create mode 100644 voicebridge/src/voicebridge/backends/passthrough.py create mode 100644 voicebridge/src/voicebridge/backends/rvc.py create mode 100644 voicebridge/src/voicebridge/backends/seedvc.py create mode 100644 voicebridge/src/voicebridge/cli.py create mode 100644 voicebridge/src/voicebridge/config.py create mode 100644 voicebridge/src/voicebridge/converter.py create mode 100644 voicebridge/src/voicebridge/dsp.py create mode 100644 voicebridge/src/voicebridge/engine.py create mode 100644 voicebridge/src/voicebridge/gui.py create mode 100644 voicebridge/src/voicebridge/record.py create mode 100644 voicebridge/src/voicebridge/train.py create mode 100644 voicebridge/tests/test_config.py create mode 100644 voicebridge/tests/test_dsp.py create mode 100644 voicebridge/tests/test_engine.py diff --git a/voicebridge/.github/workflows/ci.yml b/voicebridge/.github/workflows/ci.yml new file mode 100644 index 0000000..6b864e7 --- /dev/null +++ b/voicebridge/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: ci + +on: + push: + branches: [main, master] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install (core + dev, no torch/audio hardware) + run: | + python -m pip install --upgrade pip + pip install numpy PyYAML soxr soundfile pytest + pip install -e . --no-deps + - name: Run unit tests + run: pytest -q diff --git a/voicebridge/.gitignore b/voicebridge/.gitignore new file mode 100644 index 0000000..dd0d562 --- /dev/null +++ b/voicebridge/.gitignore @@ -0,0 +1,38 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ +build/ +dist/ +.venv/ +venv/ +env/ +.python-version + +# VoiceBridge data & models (never commit voice data or weights) +models/ +*.pth +*.index +*.onnx +*.pt +*.ckpt +data/ +recordings/ +datasets/ +logs/ +*.wav +*.flac +*.mp3 +*.npy + +# Local config (may contain device names / personal paths) +config.yaml +config.local.yaml + +# OS / editor +.DS_Store +Thumbs.db +.idea/ +.vscode/ +*.log diff --git a/voicebridge/LICENSE b/voicebridge/LICENSE new file mode 100644 index 0000000..9d79475 --- /dev/null +++ b/voicebridge/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 MRZ-OW + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/voicebridge/README.md b/voicebridge/README.md new file mode 100644 index 0000000..0bf73a7 --- /dev/null +++ b/voicebridge/README.md @@ -0,0 +1,119 @@ +# VoiceBridge + +**Real-time AI voice conversion that turns one person's microphone into a +cloned-voice virtual mic for games on Windows.** Sub-300 ms, neural speaker +conversion (RVC / Seed-VC) — *not* a pitch shifter. + +> Built for a specific use: record yourself, train a model of your voice, and +> let your girlfriend speak with **your** voice in Rust, Discord, or any app — +> live, through a virtual microphone. + +``` + Her real mic ──► VoiceBridge engine ──► your cloned voice ──► VB-CABLE ──► Rust / Discord + (GPU, ~200-300 ms) (RVC model) (virtual mic) +``` + +--- + +## How it works (30-second version) + +Three independent pieces (full detail in **[docs/DESIGN.md](docs/DESIGN.md)**): + +1. **Clone your voice** — record ~10 min of yourself, train an **RVC** model → + `your_voice.pth`. Done once. ([docs/TRAINING.md](docs/TRAINING.md)) +2. **Convert in real time** — this repo's engine captures her mic, runs it + through the model on her GPU, and hides chunk boundaries with a context + window + equal-power crossfade for clean, low-latency audio. +3. **Become a microphone** — output goes to **VB-CABLE**, a virtual audio + device, so any game/app can select it as the mic. Game-agnostic. + +The neural quality comes from the RVC model family (the community standard for +voice cloning); VoiceBridge is the real-time engine, the recording/training +workflow, the routing, and a one-click control panel around it. + +## Requirements + +- **Windows 10/11** on the PC that uses the voice. +- **NVIDIA GPU** (RTX 2060/3060 or better recommended) for real-time. CPU works + only for the no-AI `passthrough` test. +- **Python 3.10/3.11**, and **VB-CABLE** (free virtual mic). + +## Quickstart + +```powershell +# 1. install +python -m venv .venv ; .\.venv\Scripts\Activate.ps1 +pip install -e . + +# 2. see your devices +python -m voicebridge devices + +# 3. configure +copy config.example.yaml config.yaml # then edit input/output devices + +# 4a. PROVE THE ROUTING (no GPU, no model): backend: passthrough +python -m voicebridge run +# -> set the game's mic to "CABLE Output"; you should hear yourself routed. + +# 4b. THE REAL THING: train a voice (docs/TRAINING.md), set backend: rvc, then: +python -m voicebridge gui # friendly control panel, or `run` for headless +``` + +Full step-by-step (VB-CABLE, GPU PyTorch, model deploy): +**[docs/SETUP_WINDOWS.md](docs/SETUP_WINDOWS.md)**. + +## Commands + +| Command | Does | +|---|---| +| `python -m voicebridge devices` | list audio input/output devices | +| `python -m voicebridge record` | guided recording of your training data | +| `python -m voicebridge record --reference` | one ~30s clip for zero-shot Seed-VC | +| `python -m voicebridge train --check` | validate your dataset | +| `python -m voicebridge train` | print the exact training plan | +| `python -m voicebridge run` | start the real-time engine (headless) | +| `python -m voicebridge gui` | open the control panel | + +## Backends + +Set `backend:` in `config.yaml`: + +- **`passthrough`** — not AI; proves mic→cable→game works on any machine today. +- **`rvc`** — your fine-tuned cloned voice. The real product. Needs a GPU + `.pth`. +- **`seedvc`** — zero-shot from a reference clip, no training. Quick first test. + +## Latency + +Target is "feels live in voice chat," ~200–300 ms mouth-to-game on a decent +NVIDIA GPU. Tune it in **[docs/LATENCY.md](docs/LATENCY.md)**. It will never be +literally zero — neural conversion needs a small chunk of audio to work on. + +## Project layout + +See [docs/DESIGN.md §10](docs/DESIGN.md). The streaming algorithm is in +`src/voicebridge/engine.py`; the audio plumbing in `audio_io.py`; backends in +`backends/`. Pure-DSP/streaming logic is unit-tested in `tests/` (no GPU needed): + +```powershell +pip install pytest ; pytest +``` + +## Ethics & fair use + +This exists to let you share **your own** voice, with your consent, with your +partner, for fun in games. Don't: +- impersonate real people who haven't agreed, +- use it for fraud, harassment, scams, or to evade bans/identity checks, +- record or convert anyone without their knowledge. + +VoiceBridge only creates/uses a virtual **audio device** — it does not inject +into or modify any game process, so it isn't a game hack. Still, follow each +game's and platform's rules on voice/streaming software. + +## Status + +v0.1. The real-time engine, audio routing, recording tool, config, GUI, and the +passthrough backend are implemented and unit-tested. The `rvc`/`seedvc` neural +backends are wired as clearly-marked integration seams that must be validated on +a Windows + NVIDIA GPU machine (see notes in `backends/`); the trained model +also runs as-is in w-okada/voice-changer if you want a turnkey runner. License: MIT. diff --git a/voicebridge/config.example.yaml b/voicebridge/config.example.yaml new file mode 100644 index 0000000..20f20eb --- /dev/null +++ b/voicebridge/config.example.yaml @@ -0,0 +1,60 @@ +# ============================================================================ +# VoiceBridge configuration +# ---------------------------------------------------------------------------- +# Copy this file to `config.yaml` and edit it for your machine. +# cp config.example.yaml config.yaml (Windows: copy config.example.yaml config.yaml) +# +# Find your device names/IDs with: python -m voicebridge devices +# ============================================================================ + +audio: + # Input = her real microphone. Output = the virtual cable the game listens to. + # Use the exact substring of the device name (case-insensitive) OR a numeric id. + input_device: "default" # e.g. "Microphone (USB)" -> her physical mic + output_device: "CABLE Input" # VB-CABLE playback device -> game reads "CABLE Output" + monitor_device: null # optional: also play converted audio to headphones to hear herself + + samplerate: 48000 # device sample rate (Hz). 48000 is the safe default on Windows. + channels: 1 # mono in/out + +# Real-time streaming parameters. These trade latency against quality/stability. +# See docs/LATENCY.md for tuning. Total added latency ~= block_ms + crossfade_ms + inference. +stream: + block_ms: 150 # audio captured per inference step. Lower = less latency, more CPU/GPU load. + crossfade_ms: 40 # overlap blended between chunks to hide seams. 25-60 is reasonable. + context_ms: 200 # past audio fed to the model for continuity (not played, just context). + pipeline_samplerate: 16000 # internal SR the engine resamples to for the model (HuBERT-style = 16000). + +# Which conversion backend to use: passthrough | rvc | seedvc +backend: passthrough + +# ---- Backend: passthrough ------------------------------------------------- +# Not AI. Proves the mic -> cable -> game routing works on ANY machine, today. +# Optional toy pitch shift so you can *hear* a change while testing routing. +passthrough: + pitch_semitones: 0 # e.g. -4 to sound deeper. Formants NOT preserved; it's only a test tone. + +# ---- Backend: rvc (your cloned voice) ------------------------------------- +# This is the real product: a model trained on YOUR voice (see docs/TRAINING.md). +rvc: + model_path: "models/your_voice.pth" # the trained RVC generator + index_path: "models/your_voice.index" # retrieval index (improves timbre); optional + index_rate: 0.5 # 0..1 how strongly to pull toward the trained timbre + f0_method: "rmvpe" # pitch estimator: rmvpe (best), crepe, harvest, pm + f0_up_key: 0 # pitch shift in semitones. If she is higher-pitched than you, + # try +0 first; the model already retargets timbre. Tune by ear. + protect: 0.33 # protect unvoiced consonants (reduces artifacts); 0..0.5 + device: "cuda:0" # "cuda:0" for NVIDIA GPU (required for real time), or "cpu" (slow) + +# ---- Backend: seedvc (zero-shot, no training) ----------------------------- +# Point it at a reference clip of your voice; no training step. Lower quality +# than a fine-tuned RVC model but instant. Good for a quick first test. +seedvc: + reference_wav: "recordings/reference.wav" + checkpoint: null # null = use the default pretrained Seed-VC checkpoint + diffusion_steps: 4 # fewer steps = lower latency, rougher quality + device: "cuda:0" + +logging: + level: INFO # DEBUG | INFO | WARNING + show_latency: true # print rolling input->output latency while running diff --git a/voicebridge/docs/DESIGN.md b/voicebridge/docs/DESIGN.md new file mode 100644 index 0000000..313c9f2 --- /dev/null +++ b/voicebridge/docs/DESIGN.md @@ -0,0 +1,317 @@ +# VoiceBridge — High-Level Design + +> Goal: let your girlfriend speak into her own microphone and have games (Rust, +> Discord, anything) hear **your** voice instead, in real time, with no +> perceptible delay. The voice is an **AI voice clone of you**, not a pitch +> shifter. + +This document is the architecture. If you only read one file, read this one. + +--- + +## 1. What we're actually building (and what we're not) + +There are three separable problems. It's important to keep them separate, +because conflating them is how this kind of project dies. + +| # | Problem | Solved by | Runs when | +|---|---------|-----------|-----------| +| A | **Clone your voice** into a model file | RVC training on a recording of you | Once, offline, on your PC/GPU/cloud | +| B | **Convert audio in real time** through that model | The VoiceBridge real-time engine (this repo) + a neural backend | Live, on *her* PC, while she talks | +| C | **Make the game accept it as a microphone** | A virtual audio device (VB-CABLE) | Always, once installed | + +The "magic" the user feels is B+C. The "it sounds like him" quality comes +from A. This repo **owns B and C**, **orchestrates A**, and is honest that the +neural quality lives in a well-established model family (RVC), not in code we +hand-wrote from scratch. + +### Why not "just write the AI"? +Real-time neural voice conversion that is *good* and *low-latency* is a large, +already-solved research/engineering problem. The sane move — the one that +actually ships and sounds good — is to stand on: + +- **RVC (Retrieval-based Voice Conversion)** for the cloned-voice model. It's + the de-facto standard the gaming/VTuber community uses precisely because it + does **speaker conversion** (timbre retargeting) rather than pitch shifting, + trains from ~5–15 min of audio, and runs in real time on a consumer NVIDIA GPU. +- **VB-CABLE** for the virtual microphone. + +What we add on top — and what makes this a *product* your girlfriend can use +instead of a pile of research scripts — is: + +1. A clean **real-time streaming engine** (chunking, context windows, + equal-power crossfade, device routing, latency accounting) with a + **pluggable backend** interface. +2. A **guided recording tool** so capturing your training data is foolproof. +3. A **training playbook + automation** to turn that recording into a model. +4. A **one-button control panel** (GUI) she can run without touching a terminal. +5. A **passthrough backend** that proves the whole mic→cable→game path works on + *any* machine before a GPU/model is in the picture. + +--- + +## 2. Hard constraints & the reality check + +| Requirement | Reality | Design consequence | +|---|---|---| +| "No latency, sub-300 ms" | Achievable with neural VC, but **only on a GPU**. CPU-only is ~seconds. | The engine targets NVIDIA CUDA. Latency is *budgeted* (see §5), not hoped for. | +| "AI voice clone, not a pitch changer" | RVC/Seed-VC do real timbre conversion. | Pitch-shift code exists **only** in the passthrough test backend, clearly labelled. | +| "She uses it in any game" | Games read whatever Windows mic is selected. | We expose a **virtual mic** (VB-CABLE). Game just picks "CABLE Output". Game-agnostic. | +| "Record me, then she uses my voice" | Two phases: train once (you), run live (her). | Model file (`.pth`, ~55 MB) is portable. You train, send it to her, she runs the client. | + +### Where each piece runs +``` + YOU (one time) HER (every gaming session) + ┌───────────────────┐ ┌──────────────────────────────┐ + │ record your voice │ │ VoiceBridge real-time engine │ + │ (record.py) │ │ + your_voice.pth model │ + │ │ │ send the .pth │ │ │ + │ ▼ │ ───────────────► │ ▼ │ + │ train RVC model │ (Discord/USB) │ VB-CABLE virtual mic │ + │ your_voice.pth │ │ │ │ + └───────────────────┘ │ ▼ │ + │ Rust / Discord / etc. │ + └──────────────────────────────┘ +``` +> **Critical hardware note:** the *real-time* model runs on **her** PC, so **her +> PC needs the NVIDIA GPU**, not yours. Training can happen anywhere (your PC, a +> rented cloud GPU, Colab). See §7. + +--- + +## 3. System architecture + +``` + HER MICROPHONE (physical) + │ WASAPI capture (sounddevice) + ▼ + ┌───────────────────────────────┐ + │ AudioPipeline │ src/voicebridge/audio_io.py + │ InputStream callback ──► in_q │ + └───────────────┬───────────────┘ + │ float32 mono blocks @ device SR + ▼ + ┌───────────────────────────────┐ + │ RealtimeEngine │ src/voicebridge/engine.py + │ • resample to pipeline SR │ + │ • maintain context buffer │ + │ • call backend.convert() │ + │ • equal-power crossfade (OLA) │ + │ • resample back to device SR │ + └───────────────┬───────────────┘ + │ calls + ▼ + ┌───────────────────────────────┐ + │ VoiceConverter backend │ src/voicebridge/backends/* + │ passthrough | rvc | seedvc │ + │ (RVC = HuBERT feats → f0 → │ + │ generator → your timbre) │ + └───────────────┬───────────────┘ + │ converted float32 blocks + ▼ + ┌───────────────────────────────┐ + │ AudioPipeline │ + │ out_q ──► OutputStream cb │──► (optional monitor to headphones) + └───────────────┬───────────────┘ + ▼ + VB-CABLE "CABLE Input" (a speaker that is secretly a wire) + │ + ▼ + VB-CABLE "CABLE Output" (appears to Windows as a MICROPHONE) + │ + ▼ + RUST / DISCORD / OBS / anything +``` + +### Threading model +Real-time audio is unforgiving: the audio callbacks **must never block** (no +model inference, no allocation storms, no logging) or you get crackles/dropouts. +So: + +- **Input callback** (PortAudio thread): copy incoming frames into a bounded + `in_q`. Nothing else. +- **Worker thread** (`RealtimeEngine.run`): the only place the model runs. + Pulls from `in_q`, does resample → context assembly → `convert()` → crossfade + → resample, pushes finished blocks to `out_q`. +- **Output callback** (PortAudio thread): pop a block from `out_q` and write it. + On underflow (worker fell behind) it writes silence rather than blocking. + +Bounded queues give us back-pressure and a measurable, fixed latency budget. + +--- + +## 4. The streaming algorithm (chunking + crossfade) + +Neural VC is not sample-by-sample; it processes a *chunk*. Naively chopping the +mic into chunks and converting each independently produces clicks at the seams +and loses continuity. We fix both with a **context window** and **overlap-add +crossfade** — the same idea proven in w-okada's voice changer. + +Per tick we receive a new block of `B` samples (`block_ms`). Let `X` = +`crossfade_ms`, `C` = `context_ms`, all in samples at the pipeline SR. + +``` +1. model_in = concat(context_history, new_block) # length C + B +2. context_history = last C samples of model_in # slide the window +3. model_out = backend.convert(model_in) # length C + B (length-preserving) +4. seg = last (B + X) samples of model_out # the part we'll actually use +5. play[:X] = prev_tail * fade_out + seg[:X] * fade_in # equal-power crossfade + play[X:B] = seg[X:B] # untouched body +6. prev_tail = seg[B:B+X] # save overlap for next tick +7. emit play (B samples) to out_q +``` + +- **Context (C)** gives the model lookback so pitch/timbre stay coherent across + chunk boundaries; it is *not* played, only fed in. +- **Crossfade (X)** blends the end of the previous output with the start of the + current one using an equal-power (sin/cos) curve, eliminating seam clicks. +- The body `play[X:B]` is emitted verbatim. + +This is implemented in `engine.py` (`ChunkProcessor`) and the crossfade math is +unit-tested in `tests/test_dsp.py` (runs without a GPU or audio hardware). + +--- + +## 5. Latency budget (the "sub-300 ms" claim, made concrete) + +End-to-end latency = capture buffering + algorithmic + inference + playback +buffering + the game's own pipeline. We control the first four. + +| Component | Source | Typical | +|---|---|---| +| Capture block | `block_ms` (we hold a block before processing) | 150 ms | +| Crossfade hold | `crossfade_ms` (we delay by the overlap tail) | 40 ms | +| Inference | RVC forward pass on a mid-range NVIDIA GPU | 30–90 ms | +| Output buffer | one block in `out_q` + WASAPI | ~20–40 ms | +| **VoiceBridge subtotal** | | **~240–320 ms** | +| Game/Discord network + jitter | out of our hands | +50–150 ms | + +**Tuning levers** (see `docs/LATENCY.md`): +- Lower `block_ms` (e.g. 96 ms) → less latency, but inference must keep up or you + get dropouts. The dominant lever. +- Lower `crossfade_ms` toward ~25 ms. +- A faster GPU shrinks the inference term directly. +- `f0_method: rmvpe` is the best quality/speed tradeoff for pitch. + +Honest expectation: on a decent NVIDIA card (RTX 3060+), **~200–280 ms +mouth-to-game** is realistic and feels "live" in voice chat. On an older/weaker +GPU you push `block_ms` up and live with ~300–400 ms. CPU-only is not viable for +real time — it's there only so the pipeline runs for testing. + +--- + +## 6. The conversion backends (pluggable) + +All backends implement one interface (`VoiceConverter` in `converter.py`): + +```python +class VoiceConverter(ABC): + sample_rate: int # SR it wants audio at + def warmup(self) -> None: ... # run a dummy forward pass (JIT/CUDA warmup) + def convert(self, audio: np.ndarray) -> np.ndarray: ... # mono float32 in/out, length-preserving + def close(self) -> None: ... +``` + +| Backend | What it is | Needs | Use it for | +|---|---|---|---| +| `passthrough` | Identity (+ optional toy pitch shift). **Not AI.** | nothing | Verifying mic→cable→game routing on any machine, today. | +| `rvc` | Your fine-tuned cloned-voice model. The real product. | NVIDIA GPU + trained `.pth` | Production. Best quality. | +| `seedvc` | Zero-shot conversion from a reference clip — no training. | NVIDIA GPU + reference wav | Instant first test before you commit to training. | + +Swap with one line in `config.yaml` (`backend: rvc`). New backends (e.g. a +future faster model) drop into `backends/` and register themselves — the engine +doesn't change. + +> **Recommended path for "working tonight":** if you want zero risk, the +> battle-tested **w-okada/voice-changer** app can load the very same RVC `.pth` +> this repo helps you train, and route to VB-CABLE. See `docs/SETUP_WINDOWS.md` +> §"Fast path". VoiceBridge's own engine is the integrated, scriptable option; +> w-okada is the safety net. They are interchangeable because the **model file +> is the asset**, not the runner. + +--- + +## 7. Voice cloning workflow (problem A) + +1. **Record** 5–15 minutes of you, clean mic, quiet room, varied sentences. + `python -m voicebridge record` reads prompts aloud-style from + `docs/sentences.txt`, auto-splits clips, normalizes, and saves a dataset. +2. **Train** an RVC model from that dataset (`docs/TRAINING.md`). Outputs + `your_voice.pth` (+ `.index`). ~20–60 min on a GPU; doable on free/cheap + cloud GPUs if you don't have one. `train.py` wraps the steps. +3. **Deploy**: copy `your_voice.pth`/`.index` into her `models/`, set + `backend: rvc`, run. + +Data quality dominates final quality. The recording tool enforces the boring +things that matter: mono, consistent level, no clipping, no long silences, +16 kHz+ source. See `docs/TRAINING.md` for the do's and don'ts. + +--- + +## 8. The girlfriend-facing experience + +She should never see a terminal. `python -m voicebridge gui` (or a desktop +shortcut / packaged `.exe`) opens a small control panel: + +- Pick **microphone** (her mic) and confirm **output** = CABLE Input. +- **Big Start/Stop** button. Live input/output level meters + latency readout. +- A model dropdown (in case you make her several voices). +- "Monitor in headphones" toggle so she can hear herself as you. + +In the game (Rust/Discord) she sets **microphone = "CABLE Output (VB-Audio +Virtual Cable)"** once, and forgets it. + +--- + +## 9. Risks, limitations, and honest caveats + +- **GPU is non-negotiable for real time.** Her PC needs a reasonably modern + NVIDIA GPU. AMD/Intel GPUs are a rough road (DirectML/ONNX) and out of scope + for v1. +- **Quality ≠ perfect.** Cloned RVC voices are convincing but can wobble on + whispers, shouting, laughter, and heavy background noise. A noise gate and a + good mic help a lot. +- **It converts *whatever* it hears.** Background TV, a roommate, keyboard + clatter all get "voiced" as you. Quiet room + push-to-talk in the game helps. +- **Latency is real, just small.** It will never be literally zero; the design + goal is "feels live in voice chat," ~200–300 ms. +- **Consent & ethics.** This clones *your* voice with your consent, for your + partner, in games. Don't use it to impersonate people who didn't agree, for + fraud, or to evade bans. See `README.md` §Ethics. +- **Anti-cheat:** VoiceBridge only touches audio devices (a virtual mic). It + does not inject into or modify any game process, so it's not a game cheat — + but always follow each game's and platform's rules. + +--- + +## 10. Repository map + +``` +voicebridge/ +├── README.md # quickstart + ethics +├── config.example.yaml # all tunables, documented +├── docs/ +│ ├── DESIGN.md # this file +│ ├── SETUP_WINDOWS.md # step-by-step install (VB-CABLE, GPU torch, run) +│ ├── TRAINING.md # record -> train -> deploy your cloned voice +│ ├── LATENCY.md # how to hit sub-300 ms +│ └── sentences.txt # the script you read while recording +├── src/voicebridge/ +│ ├── config.py # typed config + YAML loader +│ ├── dsp.py # crossfade / resample / gate / level helpers (pure, tested) +│ ├── audio_io.py # real-time duplex pipeline (streams + queues + worker) +│ ├── converter.py # VoiceConverter interface +│ ├── backends/ +│ │ ├── passthrough.py # works today; routing test +│ │ ├── rvc.py # cloned-voice inference +│ │ └── seedvc.py # zero-shot inference +│ ├── engine.py # ChunkProcessor + RealtimeEngine (the §4 algorithm) +│ ├── record.py # guided recording tool +│ ├── train.py # RVC training orchestration +│ ├── gui.py # Tkinter control panel +│ └── cli.py # `python -m voicebridge {devices,record,run,gui,train}` +├── scripts/ +│ ├── list_devices.py # standalone device enumerator +│ └── bootstrap_private_repo.sh +└── tests/ # pure-Python unit tests (no GPU/audio needed) +``` diff --git a/voicebridge/docs/LATENCY.md b/voicebridge/docs/LATENCY.md new file mode 100644 index 0000000..8993596 --- /dev/null +++ b/voicebridge/docs/LATENCY.md @@ -0,0 +1,50 @@ +# Hitting sub-300 ms (latency tuning) + +End-to-end delay = how long after she speaks the game hears it. VoiceBridge +controls everything up to the virtual mic; the game/Discord adds its own +network buffer on top. + +## Where the milliseconds go + +``` +block_ms we hold one block before processing (biggest lever) +crossfade_ms we delay by the overlap tail +inference model forward pass on the GPU (GPU speed) +output buffer one block sitting in the output queue + WASAPI +``` +VoiceBridge subtotal ≈ `block_ms + crossfade_ms + inference + ~one block`. + +## The levers (in `config.yaml` → `stream:`) + +| Lever | Lower it to... | Cost | +|---|---|---| +| `block_ms` | cut latency the most | GPU must convert a block faster than its real-time duration, or you get dropouts/crackle | +| `crossfade_ms` | shave a bit more | too low (<20 ms) brings back faint seam clicks | +| `context_ms` | (minor) | too low hurts continuity; doesn't add output latency, only compute | +| GPU | cut the inference term directly | hardware | + +## Recommended starting points + +| Her GPU | block_ms | crossfade_ms | Expect | +|---|---|---|---| +| RTX 4070 / 4080+ | 96 | 25 | ~150–220 ms, very live | +| RTX 3060 / 3070 | 128 | 30 | ~200–280 ms | +| GTX 1660 / older | 192 | 40 | ~300–380 ms (usable) | +| CPU only | — | — | not real-time; testing only | + +## Method +1. Start with the row for her GPU. +2. Run with `logging.show_latency: true`. Watch the `infer ... ms` and + `dropouts` counters it prints every ~2s. +3. **Lower `block_ms` until `dropouts` start climbing**, then back off one step. + `infer_ms` should stay comfortably below `block_ms`. +4. Trim `crossfade_ms` toward 25 ms if you can't hear seams. + +## Other wins +- Close other GPU apps (browsers with lots of tabs, OBS encoding on GPU, other + games). Inference time is the variable that blows the budget. +- In Discord, turn **off** "Echo Cancellation", "Noise Suppression", and + "Automatic Gain Control" on the CABLE Output mic — they add latency and fight + the converted audio. Keep the input sensitivity manual. +- Prefer a wired headset/mic; some Bluetooth mics add 100 ms+ on their own. +- A noise gate (engine-side block gate, or the game's) stops it voicing silence. diff --git a/voicebridge/docs/SETUP_WINDOWS.md b/voicebridge/docs/SETUP_WINDOWS.md new file mode 100644 index 0000000..6bef51f --- /dev/null +++ b/voicebridge/docs/SETUP_WINDOWS.md @@ -0,0 +1,139 @@ +# Windows Setup — from zero to a cloned-voice mic in a game + +This is the install + run guide for the PC that will **use** the voice (your +girlfriend's PC). It needs a reasonably modern **NVIDIA GPU** for real-time +neural conversion. + +> If you just want to prove the plumbing works first (no GPU, no model), do +> Parts 1–3 with `backend: passthrough`. You'll hear your routed audio in the +> game immediately, then come back for the AI model. + +--- + +## Part 1 — Virtual microphone (VB-CABLE) + +The game needs a "microphone" to read from. VB-CABLE creates one. + +1. Download **VB-CABLE** (free) from https://vb-audio.com/Cable/ . +2. Unzip, right-click `VBCABLE_Setup_x64.exe` → **Run as administrator** → Install. +3. **Reboot.** +4. Now Windows has two new devices: + - **CABLE Input** — a "speaker." VoiceBridge plays the converted voice here. + - **CABLE Output** — a "microphone." The game listens to this. + +> Want her to *also* hear game audio + herself naturally? VB-CABLE is enough for +> v1. For fancier routing (hear yourself, mix multiple sources) install +> **VoiceMeeter** later — same idea, more knobs. + +--- + +## Part 2 — Python + VoiceBridge + +1. Install **Python 3.10 or 3.11** (64-bit) from python.org. Tick **"Add Python + to PATH"** during install. +2. Open **PowerShell** in the `voicebridge` folder and create a virtual env: + ```powershell + python -m venv .venv + .\.venv\Scripts\Activate.ps1 + python -m pip install --upgrade pip + pip install -e . + ``` +3. Check audio is visible: + ```powershell + python -m voicebridge devices + ``` + You should see her mic, **CABLE Input**, and **CABLE Output** in the list. + +--- + +## Part 3 — Configure and test the routing (no GPU yet) + +1. Make your config: + ```powershell + copy config.example.yaml config.yaml + ``` +2. Edit `config.yaml`: + - `audio.input_device`: a unique part of her mic's name (e.g. `"Microphone (USB"`). + - `audio.output_device`: `"CABLE Input"`. + - `backend: passthrough` and (optional) `passthrough.pitch_semitones: -4` so + you can clearly hear a change. +3. Run it: + ```powershell + python -m voicebridge run + ``` +4. In **Discord** (or Rust) set **Input Device / Microphone = "CABLE Output + (VB-Audio Virtual Cable)"**. Talk — your voice should come through (pitched + down if you set that). Routing works. Stop with `Ctrl+C`. + +--- + +## Part 4 — GPU PyTorch (for the real AI model) + +1. Confirm the NVIDIA driver is installed (`nvidia-smi` in PowerShell shows the GPU). +2. Install the CUDA build of PyTorch — get the exact command from + https://pytorch.org (pick Stable → Windows → Pip → CUDA). For example: + ```powershell + pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu121 + ``` +3. Verify CUDA is seen by torch: + ```powershell + python -c "import torch; print(torch.cuda.is_available(), torch.cuda.get_device_name(0))" + ``` + It must print `True` and the GPU name. + +--- + +## Part 5 — The cloned voice + +You need a trained model `your_voice.pth` (+ `.index`). Make one with +`docs/TRAINING.md` (record with `python -m voicebridge record`, train, copy the +files into `models/`). Then: + +1. In `config.yaml`: + ```yaml + backend: rvc + rvc: + model_path: models/your_voice.pth + index_path: models/your_voice.index + device: cuda:0 + ``` +2. Install the RVC runtime: + ```powershell + pip install rvc-python + ``` +3. Run: + ```powershell + python -m voicebridge run # or: python -m voicebridge gui + ``` + Game mic = **CABLE Output**. She talks; the game hears you. + +Tune latency in `docs/LATENCY.md` and voice quality in `docs/TRAINING.md`. + +--- + +## Fast path / safety net — w-okada voice changer + +If the integrated `rvc` backend gives you trouble on a specific driver/version, +you can run the **same** `your_voice.pth` in the well-established +**w-okada/voice-changer** app instead: + +1. Download w-okada's prebuilt Windows release (it bundles everything). +2. Load `your_voice.pth` (+ `.index`) as an RVC model. +3. Set its **output device = CABLE Input**, input = her mic, tune chunk/extra. +4. Game mic = **CABLE Output**. + +VoiceBridge and w-okada are interchangeable runners — the trained model is the +asset. Use whichever is smoother on her machine. + +--- + +## Troubleshooting + +| Symptom | Fix | +|---|---| +| Game doesn't list the mic | Reboot after VB-CABLE install; pick "CABLE Output". | +| Robotic/crackly audio | Worker can't keep up: raise `stream.block_ms`, or use a faster GPU. See LATENCY.md. | +| `torch.cuda.is_available()` is False | Wrong torch build or driver. Reinstall the CUDA wheel matching your driver. | +| Too much delay | Lower `block_ms`/`crossfade_ms` (LATENCY.md); close GPU-hungry apps. | +| It voices background noise | Quiet room; use push-to-talk in the game; a noise gate helps. | +| Multiple devices match a name | Use a more specific substring, or the numeric id from `devices`. | diff --git a/voicebridge/docs/TRAINING.md b/voicebridge/docs/TRAINING.md new file mode 100644 index 0000000..66412e5 --- /dev/null +++ b/voicebridge/docs/TRAINING.md @@ -0,0 +1,103 @@ +# Cloning your voice (training the RVC model) + +This produces `your_voice.pth` (+ `your_voice.index`) — the file that makes the +output sound like **you**. You do this once. Then your girlfriend just runs the +real-time engine with that file. + +## TL;DR +1. `python -m voicebridge record` — read the prompts, get 5–15 min of clean audio. +2. `python -m voicebridge train --check` — make sure the dataset is good. +3. Train with the **RVC WebUI** (`python -m voicebridge train` prints exact steps). +4. Copy `your_voice.pth` + `.index` into `models/`, set `backend: rvc`. + +--- + +## 1. Record good data (this matters most) + +Garbage in, garbage out — final quality tracks recording quality more than any +setting. The recorder enforces mono / level / trimming; you handle the room: + +- **Quiet room.** No fan, TV, music, or roommates. Background noise gets cloned. +- **One consistent mic, consistent distance** (a hand-span away). Don't move + closer/farther between clips. +- **Normal speaking energy** — talk like you're in a game, not whispering, not + shouting. Avoid clipping (the recorder normalizes, but don't redline the mic). +- **Variety helps:** statements, questions, callouts, numbers, your actual slang. + Edit `docs/sentences.txt` to add lines you really say in Rust/Discord. +- **5–15 minutes total.** More varied (not just longer) is better. ~10 min is a + sweet spot. + +```powershell +python -m voicebridge record +``` +Clips land in `recordings/dataset/`. Run it again any time to add more. + +Validate: +```powershell +python -m voicebridge train --check +``` +Fix anything it flags (clipped/quiet/too short) by re-recording those bits. + +--- + +## 2. Train (RVC WebUI — easiest) + +Run `python -m voicebridge train` to print the exact, current steps. In short: + +1. Clone the RVC WebUI: + ``` + git clone https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI + ``` + Install its requirements (CUDA torch) and download the pretrained base models + it links (ContentVec/HuBERT + the `rmvpe` pitch model). +2. `python infer-web.py` → opens a browser UI. +3. **Train** tab: + - Experiment name: `your_voice` + - Dataset path: your `recordings/dataset` folder (absolute path) + - Sample rate: **48k** (or 40k), **Pitch guidance: yes**, f0 method **rmvpe** + - Epochs: ~**150–250** to start; save every ~25 + - Run **Process data → Feature extraction → Train model → Train feature index** +4. Outputs: + - generator → `your_voice.pth` + - feature index → `added_*.index` → rename to `your_voice.index` + +**No GPU?** Train on a rented cloud GPU (vast.ai/runpod) or a "RVC training +Colab" notebook, then download the two files. + +--- + +## 3. Deploy + +Copy both files into `voicebridge/models/`, then in `config.yaml`: +```yaml +backend: rvc +rvc: + model_path: models/your_voice.pth + index_path: models/your_voice.index + f0_up_key: 0 + index_rate: 0.5 + protect: 0.33 + device: cuda:0 +``` +Run `python -m voicebridge run` (or `gui`). + +--- + +## 4. Tuning by ear + +| Knob | What it does | Try | +|---|---|---| +| `f0_up_key` | Shifts pitch in semitones. Her natural pitch differs from yours; the model retargets timbre but pitch can sit high/low. | Start `0`. If it sounds too high/squeaky, `-2` to `-5`. Too deep, `+2`. | +| `index_rate` | How hard it pulls toward your trained timbre vs. clarity. | `0.5` default. Higher = more "you" but can smear; lower = clearer but less you. | +| `protect` | Protects unvoiced consonants (s, t, breaths) from artifacts. | `0.33`. Raise toward `0.5` if consonants sound weird. | +| `f0_method` | Pitch tracker. `rmvpe` is the best all-rounder. | Keep `rmvpe`. | + +Change one thing at a time, say the same sentence, compare. A **noise gate** and +a decent mic on her end do more for realism than any of these. + +--- + +## Ethics reminder +Train on **your own** voice, with your consent, for your partner, in games. +Don't clone people who didn't agree, don't use it for fraud/impersonation, and +follow each game's rules. See the README. diff --git a/voicebridge/docs/sentences.txt b/voicebridge/docs/sentences.txt new file mode 100644 index 0000000..2acae81 --- /dev/null +++ b/voicebridge/docs/sentences.txt @@ -0,0 +1,41 @@ +# VoiceBridge recording script. +# Read each line in a natural, relaxed voice — the way you'd actually talk in a +# game. Lines starting with '#' are ignored. Aim for 5-15 minutes total across +# everything; add your own lines to cover how you really speak (callouts, slang). + +The quick brown fox jumps over the lazy dog while the sun sets behind the hills. +I honestly didn't think we'd make it out of there alive, but here we are. +Could you cover the east side? I'll push through the compound from the north. +One, two, three, four, five, six, seven, eight, nine, ten, eleven, twelve. +She sells seashells by the seashore on a bright and breezy summer morning. +Watch out, there's a sniper on the rooftop near the blue water tower. +My favorite part of the day is when everything finally goes quiet. +Pass me the medkit, I'm down to about thirty health and no armor left. +The weather changed so fast this afternoon, it went from sunny to storming. +Honestly, that was the cleanest play I've seen you pull off all week. +How much wood would a woodchuck chuck if a woodchuck could chuck wood? +Let's regroup at the bridge, then we'll hit the base together at midnight. +Red leather, yellow leather, red leather, yellow leather, said it perfectly. +I'm telling you, that loot is way better than anything we found yesterday. +Peter Piper picked a peck of pickled peppers from the patch by the river. +Give me two seconds to reload, then I'll lay down some covering fire for you. +The old lighthouse stood alone against the grey and endless rolling waves. +Are you absolutely sure about this plan, or are we just making it up as we go? +Thirty-three thirsty travelers thought they'd thaw the thick frozen lake. +Keep your voice down, I think I can hear footsteps right above us somewhere. +A big black bug bit a big black bear and made the big black bear bleed badly. +We should build the wall higher before nightfall, otherwise they'll raid us. +I can't believe you actually remembered to bring the keys this time around. +Fresh fried fish, fish fresh fried, fried fish fresh, fresh fish fried fast. +Take the left path, it loops around behind them and gives us the high ground. +The early morning fog rolled gently across the valley and into the open fields. +Whatever you do, do not open that door until I give you the all clear signal. +Six slippery snails slid slowly seaward along the smooth and shining shoreline. +That's a solid trade if you ask me, but it's totally your call in the end. +The library was so silent you could hear a single page turn across the room. +Eleven benevolent elephants elegantly ambled along the avenue at eleven. +Alright everyone, stack up on the door, breach on three, and watch your corners. +I poured the coffee, grabbed my coat, and headed straight out into the cold. +Unique New York, you know you need unique New York, you really really do. +If we hold this position for two more minutes, the reinforcements will arrive. +Sometimes the simplest answer really is the right one, even when it feels wrong. diff --git a/voicebridge/pyproject.toml b/voicebridge/pyproject.toml new file mode 100644 index 0000000..25a6442 --- /dev/null +++ b/voicebridge/pyproject.toml @@ -0,0 +1,36 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "voicebridge" +version = "0.1.0" +description = "Real-time AI voice conversion that turns a physical mic into a cloned-voice virtual microphone for games on Windows." +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.10" +authors = [{ name = "MRZ-OW" }] +dependencies = [ + "numpy>=1.24,<2.0", + "sounddevice>=0.4.6", + "soundfile>=0.12.1", + "PyYAML>=6.0", + "soxr>=0.3.7", +] + +[project.optional-dependencies] +rvc = ["rvc-python>=0.1.5"] +dev = ["pytest>=7.4", "ruff>=0.4"] + +[project.scripts] +voicebridge = "voicebridge.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/voicebridge/requirements.txt b/voicebridge/requirements.txt new file mode 100644 index 0000000..7b26c2c --- /dev/null +++ b/voicebridge/requirements.txt @@ -0,0 +1,21 @@ +# --- Core runtime (needed for the real-time engine on Windows) --- +numpy>=1.24,<2.0 +sounddevice>=0.4.6 # PortAudio bindings; WASAPI low-latency capture/playback on Windows +soundfile>=0.12.1 # read/write wav for recording + reference clips +PyYAML>=6.0 # config files +soxr>=0.3.7 # high-quality, fast resampling for the real-time path + +# --- Neural voice-conversion backends --- +# PyTorch is intentionally NOT pinned here: install the CUDA build that matches +# your GPU/driver from https://pytorch.org (see docs/SETUP_WINDOWS.md). +# torch>=2.1 +# torchaudio>=2.1 + +# RVC backend (cloned-voice inference). Optional; install when you use it. +# rvc-python>=0.1.5 + +# --- GUI (girlfriend-facing control panel) --- +# Tkinter ships with the standard Windows Python installer; no pip package needed. + +# --- Dev / tests (not needed at runtime) --- +# pytest>=7.4 diff --git a/voicebridge/scripts/bootstrap_private_repo.sh b/voicebridge/scripts/bootstrap_private_repo.sh new file mode 100755 index 0000000..d4e5f0d --- /dev/null +++ b/voicebridge/scripts/bootstrap_private_repo.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Create a NEW PRIVATE GitHub repo from this folder and push it. +# +# Why this exists: the automated session that scaffolded VoiceBridge could not +# create a repo for you (the GitHub integration lacked repo-creation +# permission), so it left you this one-shot script. Run it from inside the +# `voicebridge/` folder on your own machine. +# +# Requirements: git, and EITHER the GitHub CLI (`gh`, recommended) OR a remote +# you create yourself in the browser. +# +# Usage: +# ./scripts/bootstrap_private_repo.sh [repo-name] +# Default repo name: voicebridge +set -euo pipefail + +REPO_NAME="${1:-voicebridge}" + +cd "$(dirname "$0")/.." + +if [ ! -d .git ]; then + git init +fi +git add -A +git commit -m "Initial commit: VoiceBridge real-time AI voice changer" || true +git branch -M main + +if command -v gh >/dev/null 2>&1; then + echo "Creating private repo '$REPO_NAME' via GitHub CLI..." + gh repo create "$REPO_NAME" --private --source=. --remote=origin --push + echo "Done: $(gh repo view "$REPO_NAME" --json url -q .url)" +else + cat </$REPO_NAME.git + git push -u origin main + +Or install gh (https://cli.github.com) and re-run this script. +EOF +fi diff --git a/voicebridge/scripts/list_devices.py b/voicebridge/scripts/list_devices.py new file mode 100644 index 0000000..7776c92 --- /dev/null +++ b/voicebridge/scripts/list_devices.py @@ -0,0 +1,9 @@ +"""Standalone audio device lister (no install needed): python scripts/list_devices.py + +Equivalent to `python -m voicebridge devices`, handy before the package is set up. +""" + +import sounddevice as sd + +print(sd.query_devices()) +print("\nDefault (input, output) indices:", sd.default.device) diff --git a/voicebridge/src/voicebridge/__init__.py b/voicebridge/src/voicebridge/__init__.py new file mode 100644 index 0000000..5466c66 --- /dev/null +++ b/voicebridge/src/voicebridge/__init__.py @@ -0,0 +1,11 @@ +"""VoiceBridge: real-time AI voice conversion into a virtual microphone. + +See docs/DESIGN.md for the architecture. Public entry point is the CLI: + + python -m voicebridge devices # list audio devices + python -m voicebridge record # record training data + python -m voicebridge run # start the real-time engine + python -m voicebridge gui # open the control panel +""" + +__version__ = "0.1.0" diff --git a/voicebridge/src/voicebridge/__main__.py b/voicebridge/src/voicebridge/__main__.py new file mode 100644 index 0000000..ee2a248 --- /dev/null +++ b/voicebridge/src/voicebridge/__main__.py @@ -0,0 +1,8 @@ +"""Enable ``python -m voicebridge``.""" + +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/voicebridge/src/voicebridge/audio_io.py b/voicebridge/src/voicebridge/audio_io.py new file mode 100644 index 0000000..73022b7 --- /dev/null +++ b/voicebridge/src/voicebridge/audio_io.py @@ -0,0 +1,243 @@ +"""Real-time duplex audio plumbing built on sounddevice (PortAudio / WASAPI). + +The golden rule of real-time audio: **audio callbacks must never block**. So the +model never runs inside a callback. Instead: + + input callback ──► in_q ──► worker thread (runs the model) ──► out_q ──► output callback + +Bounded queues give back-pressure and a fixed, measurable latency. On overrun +(model can't keep up) or underrun (model fell behind) we drop/feed-silence and +count it, rather than blocking and crackling the whole stream. + +sounddevice is imported lazily so the rest of the package imports fine on +headless machines without PortAudio. +""" + +from __future__ import annotations + +import logging +import queue +import threading +from typing import Callable, Optional + +import numpy as np + +from .config import AudioConfig + +log = logging.getLogger("voicebridge.audio") + +Processor = Callable[[np.ndarray], np.ndarray] + + +def list_devices() -> str: + """Human-readable table of audio devices (for the CLI/GUI and docs).""" + import sounddevice as sd + + return str(sd.query_devices()) + + +def resolve_device(spec, kind: str) -> Optional[int]: + """Resolve a device spec to a PortAudio index. + + ``spec`` may be: "default"/None (-> system default), an int / numeric string + (-> that index), or a case-insensitive substring of the device name. ``kind`` + is "input" or "output" and restricts matches to devices with the right + channel direction. + """ + import sounddevice as sd + + if spec is None or (isinstance(spec, str) and spec.strip().lower() in ("", "default")): + return None + if isinstance(spec, int) or (isinstance(spec, str) and spec.strip().lstrip("-").isdigit()): + return int(spec) + + want = "max_input_channels" if kind == "input" else "max_output_channels" + needle = str(spec).lower() + matches = [] + for idx, dev in enumerate(sd.query_devices()): + if dev[want] > 0 and needle in dev["name"].lower(): + matches.append((idx, dev["name"])) + if not matches: + raise ValueError( + f"No {kind} device matching {spec!r}. Run `python -m voicebridge devices` " + f"to see available devices." + ) + if len(matches) > 1: + log.warning( + "Multiple %s devices match %r: %s — using the first (%s).", + kind, spec, [m[1] for m in matches], matches[0][1], + ) + return matches[0][0] + + +class AudioPipeline: + def __init__( + self, + audio_cfg: AudioConfig, + block_frames: int, + processor: Processor, + on_tick: Optional[Callable[[], None]] = None, + queue_blocks: int = 8, + ) -> None: + self.cfg = audio_cfg + self.block_frames = block_frames + self.processor = processor + self.on_tick = on_tick + + self.in_q: "queue.Queue[np.ndarray]" = queue.Queue(maxsize=queue_blocks) + self.out_q: "queue.Queue[np.ndarray]" = queue.Queue(maxsize=queue_blocks) + self.monitor_q: "queue.Queue[np.ndarray]" = queue.Queue(maxsize=queue_blocks) + + self.dropouts = 0 + self.running = False + self._worker: Optional[threading.Thread] = None + self._in_stream = None + self._out_stream = None + self._mon_stream = None + + # -- lifecycle --------------------------------------------------------- + def start(self) -> None: + import sounddevice as sd + + in_dev = resolve_device(self.cfg.input_device, "input") + out_dev = resolve_device(self.cfg.output_device, "output") + mon_dev = ( + resolve_device(self.cfg.monitor_device, "output") + if self.cfg.monitor_device + else None + ) + log.info( + "Audio devices | in=%s out=%s monitor=%s @ %d Hz", + _name(in_dev), _name(out_dev), _name(mon_dev) if mon_dev is not None else "off", + self.cfg.samplerate, + ) + + self.running = True + self._worker = threading.Thread(target=self._run_worker, name="vb-worker", daemon=True) + self._worker.start() + + self._in_stream = sd.InputStream( + samplerate=self.cfg.samplerate, + blocksize=self.block_frames, + device=in_dev, + channels=1, + dtype="float32", + callback=self._on_input, + ) + self._out_stream = sd.OutputStream( + samplerate=self.cfg.samplerate, + blocksize=self.block_frames, + device=out_dev, + channels=self.cfg.channels, + dtype="float32", + callback=self._on_output, + ) + self._in_stream.start() + self._out_stream.start() + if mon_dev is not None: + self._mon_stream = sd.OutputStream( + samplerate=self.cfg.samplerate, + blocksize=self.block_frames, + device=mon_dev, + channels=self.cfg.channels, + dtype="float32", + callback=self._on_monitor, + ) + self._mon_stream.start() + + def stop(self) -> None: + self.running = False + for stream in (self._in_stream, self._out_stream, self._mon_stream): + if stream is not None: + try: + stream.stop() + stream.close() + except Exception: + pass + self._in_stream = self._out_stream = self._mon_stream = None + if self._worker is not None: + self._worker.join(timeout=1.0) + self._worker = None + + # -- callbacks (run on PortAudio threads; keep them tiny) -------------- + def _on_input(self, indata, frames, time_info, status) -> None: + if status: + log.debug("input status: %s", status) + mono = np.ascontiguousarray(indata[:, 0], dtype=np.float32) + try: + self.in_q.put_nowait(mono.copy()) + except queue.Full: + self.dropouts += 1 # overrun: worker is behind; drop this block + + def _on_output(self, outdata, frames, time_info, status) -> None: + if status: + log.debug("output status: %s", status) + block = self._pop_output() + self._write(outdata, block, frames) + + def _on_monitor(self, outdata, frames, time_info, status) -> None: + try: + block = self.monitor_q.get_nowait() + except queue.Empty: + block = None + self._write(outdata, block, frames) + + def _pop_output(self) -> Optional[np.ndarray]: + try: + return self.out_q.get_nowait() + except queue.Empty: + self.dropouts += 1 # underrun: nothing ready, play silence + return None + + def _write(self, outdata, block: Optional[np.ndarray], frames: int) -> None: + if block is None: + outdata.fill(0.0) + return + if len(block) != frames: + fitted = np.zeros(frames, dtype=np.float32) + n = min(frames, len(block)) + fitted[:n] = block[:n] + block = fitted + for ch in range(outdata.shape[1]): + outdata[:, ch] = block + + # -- worker thread (the only place the model runs) --------------------- + def _run_worker(self) -> None: + while self.running: + try: + block = self.in_q.get(timeout=0.1) + except queue.Empty: + continue + try: + out = self.processor(block) + except Exception: + log.exception("processor error; emitting silence for this block") + out = np.zeros_like(block) + _offer(self.out_q, out) + if self._mon_stream is not None: + _offer(self.monitor_q, out) + if self.on_tick is not None: + self.on_tick() + + +def _offer(q: "queue.Queue[np.ndarray]", item: np.ndarray) -> None: + """Put without blocking; if full, drop the oldest to favour fresh audio.""" + try: + q.put_nowait(item) + except queue.Full: + try: + q.get_nowait() + q.put_nowait(item) + except queue.Empty: + pass + + +def _name(idx: Optional[int]) -> str: + if idx is None: + return "default" + try: + import sounddevice as sd + + return f"[{idx}] {sd.query_devices(idx)['name']}" + except Exception: + return f"[{idx}]" diff --git a/voicebridge/src/voicebridge/backends/__init__.py b/voicebridge/src/voicebridge/backends/__init__.py new file mode 100644 index 0000000..068fdf3 --- /dev/null +++ b/voicebridge/src/voicebridge/backends/__init__.py @@ -0,0 +1 @@ +"""Voice-conversion backends. See converter.build_converter for the factory.""" diff --git a/voicebridge/src/voicebridge/backends/passthrough.py b/voicebridge/src/voicebridge/backends/passthrough.py new file mode 100644 index 0000000..573e8a6 --- /dev/null +++ b/voicebridge/src/voicebridge/backends/passthrough.py @@ -0,0 +1,27 @@ +"""Passthrough backend — the "is the plumbing alive?" backend. + +It is deliberately NOT AI. It returns the input untouched, or, if you set a +non-zero ``pitch_semitones``, applies a crude resample-based pitch shift so you +can *hear* that audio is flowing mic -> engine -> virtual cable -> game before +any GPU or trained model is involved. Use it to validate device routing and +latency on literally any machine. +""" + +from __future__ import annotations + +import numpy as np + +from ..config import PassthroughConfig +from ..converter import VoiceConverter +from ..dsp import pitch_shift_naive + + +class PassthroughConverter(VoiceConverter): + def __init__(self, cfg: PassthroughConfig, sample_rate: int) -> None: + self.sample_rate = sample_rate + self.pitch_semitones = float(cfg.pitch_semitones) + + def convert(self, audio: np.ndarray) -> np.ndarray: + if self.pitch_semitones == 0.0: + return audio + return pitch_shift_naive(audio, self.pitch_semitones) diff --git a/voicebridge/src/voicebridge/backends/rvc.py b/voicebridge/src/voicebridge/backends/rvc.py new file mode 100644 index 0000000..3bf1d0f --- /dev/null +++ b/voicebridge/src/voicebridge/backends/rvc.py @@ -0,0 +1,137 @@ +"""RVC backend — real cloned-voice conversion (the actual product). + +RVC (Retrieval-based Voice Conversion) takes incoming speech, extracts +speaker-independent content features (HuBERT/ContentVec) and a pitch (F0) track, +then a generator trained on YOUR voice resynthesizes that content in your +timbre. It is speaker conversion, not pitch shifting — which is exactly the +requirement. + +----------------------------------------------------------------------------- +INTEGRATION NOTE (read me) +----------------------------------------------------------------------------- +This module wraps the `rvc-python` package (`pip install rvc-python` + a CUDA +build of torch). The package's public API has shifted across versions, so we +probe at runtime for an in-memory (numpy in -> numpy out) inference entry point +rather than hard-coding one, and fail with a clear message if we can't find one. + +Because real-time RVC is hardware-bound (NVIDIA GPU) and version-sensitive, this +backend is written to be correct in shape but MUST be validated on the target +Windows+GPU machine. If you want a zero-risk runner for the very same trained +`.pth`, w-okada/voice-changer loads it directly — see docs/SETUP_WINDOWS.md. +The trained model file is the asset; this is just one way to run it. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +import numpy as np + +from ..config import RVCConfig +from ..converter import VoiceConverter + +log = logging.getLogger("voicebridge.rvc") + + +class RVCConverter(VoiceConverter): + def __init__(self, cfg: RVCConfig) -> None: + self.cfg = cfg + self.sample_rate = 16000 # RVC content features are extracted at 16 kHz + self._impl = None # the loaded rvc-python inference object + self._infer_array = None # resolved numpy->numpy callable + self._load() + + # -- model loading ----------------------------------------------------- + def _load(self) -> None: + model_path = Path(self.cfg.model_path) + if not model_path.exists(): + raise FileNotFoundError( + f"RVC model not found: {model_path}. Train one (docs/TRAINING.md) " + f"or point rvc.model_path at your .pth." + ) + try: + from rvc_python.infer import RVCInference # type: ignore + except Exception as exc: # pragma: no cover - depends on optional install + raise RuntimeError( + "The 'rvc' backend needs the rvc-python package and a CUDA build " + "of torch. Install per docs/SETUP_WINDOWS.md (`pip install rvc-python`). " + f"Underlying import error: {exc}" + ) from exc + + log.info("Loading RVC model %s on %s", model_path, self.cfg.device) + self._impl = RVCInference(device=self.cfg.device) + self._impl.load_model(str(model_path)) + # Common parameter names across rvc-python versions; set what exists. + _try_set_params( + self._impl, + f0method=self.cfg.f0_method, + f0up_key=self.cfg.f0_up_key, + index_path=self.cfg.index_path, + index_rate=self.cfg.index_rate, + protect=self.cfg.protect, + ) + self._infer_array = _resolve_array_infer(self._impl) + if self._infer_array is None: + raise RuntimeError( + "Could not find an in-memory inference method on rvc-python " + "(looked for infer_array / infer_audio / infer_numpy / vc / infer). " + "Your installed version may only support file-based inference, " + "which is unsuitable for real time. Use the w-okada runner with " + "this same .pth instead (docs/SETUP_WINDOWS.md), or pin a " + "compatible rvc-python version." + ) + + # -- inference --------------------------------------------------------- + def convert(self, audio: np.ndarray) -> np.ndarray: + if self._infer_array is None: + raise RuntimeError("RVC backend not initialized") + out = self._infer_array(audio.astype(np.float32)) + out = np.asarray(out, dtype=np.float32).reshape(-1) + # Length-preserve so the engine's crossfade math stays exact. + if len(out) > len(audio): + out = out[: len(audio)] + elif len(out) < len(audio): + out = np.pad(out, (0, len(audio) - len(out))) + return out + + def close(self) -> None: + self._impl = None + self._infer_array = None + try: + import torch # type: ignore + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except Exception: + pass + + +def _try_set_params(impl, **params) -> None: + """Best-effort parameter setting across rvc-python API variants.""" + setter = getattr(impl, "set_params", None) + if callable(setter): + try: + setter(**{k: v for k, v in params.items() if v is not None}) + return + except TypeError: + pass # fall through to attribute assignment + for key, value in params.items(): + if value is None: + continue + for attr in (key, key.replace("_", "")): + if hasattr(impl, attr): + try: + setattr(impl, attr, value) + except Exception: + pass + break + + +def _resolve_array_infer(impl): + """Find a numpy-in/numpy-out inference callable on the impl, or None.""" + for name in ("infer_array", "infer_audio", "infer_numpy", "vc", "infer"): + fn = getattr(impl, name, None) + if callable(fn): + return fn + return None diff --git a/voicebridge/src/voicebridge/backends/seedvc.py b/voicebridge/src/voicebridge/backends/seedvc.py new file mode 100644 index 0000000..29b125f --- /dev/null +++ b/voicebridge/src/voicebridge/backends/seedvc.py @@ -0,0 +1,71 @@ +"""Seed-VC backend — zero-shot conversion, no training step. + +Seed-VC converts your girlfriend's speech toward a *reference clip* of your +voice without any fine-tuning: point it at `recordings/reference.wav` and go. +It is the fastest way to a first "wait, that sounds like him" moment. Quality +and stability are below a properly fine-tuned RVC model, so treat it as the +quick-start / fallback, and graduate to the `rvc` backend for production. + +----------------------------------------------------------------------------- +INTEGRATION NOTE +----------------------------------------------------------------------------- +Seed-VC ships as a GitHub project (Plachtaa/seed-vc) rather than a stable pip +package, and exposes its real-time path through `real-time-gui.py` helpers. This +class defines the seam: load the model + reference embedding once, then convert +chunks. Wire `_load_model` / `_infer` to the Seed-VC version you vendor or +install, and validate on the target GPU. Kept intentionally thin so it's obvious +where to plug in. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +import numpy as np + +from ..config import SeedVCConfig +from ..converter import VoiceConverter + +log = logging.getLogger("voicebridge.seedvc") + + +class SeedVCConverter(VoiceConverter): + def __init__(self, cfg: SeedVCConfig) -> None: + self.cfg = cfg + self.sample_rate = 16000 + ref = Path(cfg.reference_wav) + if not ref.exists(): + raise FileNotFoundError( + f"Seed-VC reference clip not found: {ref}. Record one with " + f"`python -m voicebridge record --reference`." + ) + self._model = self._load_model() + self._ref_embedding = self._embed_reference(ref) + + def _load_model(self): + raise NotImplementedError( + "Seed-VC integration seam: install/vendor Plachtaa/seed-vc and load " + "its real-time model + vocoder here (checkpoint=%r, device=%r, " + "diffusion_steps=%d). See backends/seedvc.py docstring." + % (self.cfg.checkpoint, self.cfg.device, self.cfg.diffusion_steps) + ) + + def _embed_reference(self, ref_path: Path): + # Compute the speaker embedding of your reference clip once, up front. + raise NotImplementedError("Seed-VC integration seam: embed the reference clip.") + + def convert(self, audio: np.ndarray) -> np.ndarray: # pragma: no cover - needs model + out = self._infer(audio.astype(np.float32)) + out = np.asarray(out, dtype=np.float32).reshape(-1) + if len(out) > len(audio): + out = out[: len(audio)] + elif len(out) < len(audio): + out = np.pad(out, (0, len(audio) - len(out))) + return out + + def _infer(self, audio: np.ndarray) -> np.ndarray: + raise NotImplementedError( + "Seed-VC integration seam: run zero-shot conversion of `audio` toward " + "self._ref_embedding and return mono float32 at self.sample_rate." + ) diff --git a/voicebridge/src/voicebridge/cli.py b/voicebridge/src/voicebridge/cli.py new file mode 100644 index 0000000..4538f9a --- /dev/null +++ b/voicebridge/src/voicebridge/cli.py @@ -0,0 +1,126 @@ +"""Command-line entry point: ``python -m voicebridge ``. + +Commands: + devices list audio input/output devices + record [--reference] record training data (or a single reference clip) + train [--check] validate the dataset / print the training plan + run start the real-time engine (headless) + gui open the control panel +""" + +from __future__ import annotations + +import argparse +import logging +import sys +import time + + +def _setup_logging(level: str = "INFO") -> None: + logging.basicConfig( + level=getattr(logging, level.upper(), logging.INFO), + format="%(asctime)s %(levelname)-7s %(name)s: %(message)s", + datefmt="%H:%M:%S", + ) + + +def _load_config(path: str): + from .config import Config + + return Config.load(path) + + +def cmd_devices(_args) -> int: + from .audio_io import list_devices + + print(list_devices()) + return 0 + + +def cmd_record(args) -> int: + from . import record + + if args.reference: + record.record_reference(out_path=args.out or "recordings/reference.wav") + else: + record.record_dataset( + out_dir=args.out or "recordings/dataset", + sentences_file=args.sentences, + ) + return 0 + + +def cmd_train(args) -> int: + from . import train + + if args.check: + train.check_dataset(args.dataset) + else: + train.check_dataset(args.dataset) + train.print_plan(args.dataset) + return 0 + + +def cmd_run(args) -> int: + from .engine import RealtimeEngine + + config = _load_config(args.config) + _setup_logging(config.logging.level) + engine = RealtimeEngine(config) + try: + engine.start() + except Exception as exc: + logging.getLogger("voicebridge").error("Failed to start: %s", exc) + return 1 + print("VoiceBridge is live. In the game, set Microphone = 'CABLE Output'. Ctrl+C to stop.") + try: + while engine.running: + time.sleep(0.2) + except KeyboardInterrupt: + print("\nstopping...") + finally: + engine.stop() + return 0 + + +def cmd_gui(args) -> int: + from .gui import run_gui + + config = _load_config(args.config) + _setup_logging(config.logging.level) + run_gui(config) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser(prog="voicebridge", description="Real-time AI voice changer.") + p.add_argument("--config", default="config.yaml", help="path to config.yaml") + sub = p.add_subparsers(dest="command", required=True) + + sub.add_parser("devices", help="list audio devices").set_defaults(func=cmd_devices) + + pr = sub.add_parser("record", help="record training data") + pr.add_argument("--reference", action="store_true", help="record one reference clip (Seed-VC)") + pr.add_argument("--out", default=None, help="output dir (dataset) or file (reference)") + pr.add_argument("--sentences", default="docs/sentences.txt", help="prompt list") + pr.set_defaults(func=cmd_record) + + pt = sub.add_parser("train", help="validate dataset / print training plan") + pt.add_argument("--check", action="store_true", help="only validate the dataset") + pt.add_argument("--dataset", default="recordings/dataset", help="dataset dir") + pt.set_defaults(func=cmd_train) + + sub.add_parser("run", help="start the real-time engine").set_defaults(func=cmd_run) + sub.add_parser("gui", help="open the control panel").set_defaults(func=cmd_gui) + return p + + +def main(argv: list[str] | None = None) -> int: + _setup_logging() + parser = build_parser() + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/voicebridge/src/voicebridge/config.py b/voicebridge/src/voicebridge/config.py new file mode 100644 index 0000000..f57f98d --- /dev/null +++ b/voicebridge/src/voicebridge/config.py @@ -0,0 +1,111 @@ +"""Typed configuration for VoiceBridge, loaded from a YAML file. + +Keeping config as plain dataclasses (rather than passing dicts around) means the +engine, backends, and GUI all agree on field names and types, and a typo in the +YAML fails loudly at load time instead of deep inside a real-time callback. +""" + +from dataclasses import dataclass, field, fields, is_dataclass +from pathlib import Path +from typing import Any, Optional + +import yaml + + +@dataclass +class AudioConfig: + input_device: str = "default" + output_device: str = "CABLE Input" + monitor_device: Optional[str] = None + samplerate: int = 48000 + channels: int = 1 + + +@dataclass +class StreamConfig: + block_ms: int = 150 + crossfade_ms: int = 40 + context_ms: int = 200 + pipeline_samplerate: int = 16000 + + def __post_init__(self) -> None: + if self.crossfade_ms >= self.block_ms: + raise ValueError( + f"crossfade_ms ({self.crossfade_ms}) must be smaller than " + f"block_ms ({self.block_ms}); the crossfade is an overlap inside a block." + ) + + +@dataclass +class PassthroughConfig: + pitch_semitones: float = 0.0 + + +@dataclass +class RVCConfig: + model_path: str = "models/your_voice.pth" + index_path: Optional[str] = "models/your_voice.index" + index_rate: float = 0.5 + f0_method: str = "rmvpe" + f0_up_key: int = 0 + protect: float = 0.33 + device: str = "cuda:0" + + +@dataclass +class SeedVCConfig: + reference_wav: str = "recordings/reference.wav" + checkpoint: Optional[str] = None + diffusion_steps: int = 4 + device: str = "cuda:0" + + +@dataclass +class LoggingConfig: + level: str = "INFO" + show_latency: bool = True + + +@dataclass +class Config: + audio: AudioConfig = field(default_factory=AudioConfig) + stream: StreamConfig = field(default_factory=StreamConfig) + backend: str = "passthrough" + passthrough: PassthroughConfig = field(default_factory=PassthroughConfig) + rvc: RVCConfig = field(default_factory=RVCConfig) + seedvc: SeedVCConfig = field(default_factory=SeedVCConfig) + logging: LoggingConfig = field(default_factory=LoggingConfig) + + # -- loading ----------------------------------------------------------- + @classmethod + def load(cls, path: str | Path) -> "Config": + path = Path(path) + if not path.exists(): + raise FileNotFoundError( + f"Config file not found: {path}. Copy config.example.yaml to config.yaml first." + ) + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return cls.from_dict(raw) + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> "Config": + return _build(cls, raw) + + +def _build(dc_type: type, raw: dict[str, Any]) -> Any: + """Recursively construct a (possibly nested) dataclass from a dict, + rejecting unknown keys so config typos surface immediately.""" + kwargs: dict[str, Any] = {} + known = {f.name: f for f in fields(dc_type)} + unknown = set(raw) - set(known) + if unknown: + raise ValueError(f"Unknown config keys for {dc_type.__name__}: {sorted(unknown)}") + for name, f in known.items(): + if name not in raw: + continue + value = raw[name] + if is_dataclass(f.type) and isinstance(value, dict): + kwargs[name] = _build(f.type, value) # type: ignore[arg-type] + else: + kwargs[name] = value + return dc_type(**kwargs) diff --git a/voicebridge/src/voicebridge/converter.py b/voicebridge/src/voicebridge/converter.py new file mode 100644 index 0000000..03edcad --- /dev/null +++ b/voicebridge/src/voicebridge/converter.py @@ -0,0 +1,72 @@ +"""The backend interface every voice-conversion engine implements. + +The real-time engine (engine.py) only ever talks to this interface, so swapping +``passthrough`` -> ``rvc`` -> ``seedvc`` (or a future model) changes nothing in +the streaming/audio code. + +Contract: + * ``convert`` receives mono float32 audio at ``self.sample_rate`` and returns + mono float32 audio at the same sample rate. + * Output length SHOULD equal input length (length-preserving). The engine + trims/pads defensively, but backends that drift will hurt latency. + * ``convert`` is called from a single worker thread, never from an audio + callback, so it may use the GPU and block — but it must keep up with + real time (process a block faster than the block's wall-clock duration). +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod + +import numpy as np + + +class VoiceConverter(ABC): + #: sample rate (Hz) this backend wants audio delivered at + sample_rate: int = 16000 + + @abstractmethod + def convert(self, audio: np.ndarray) -> np.ndarray: + """Convert one chunk of mono float32 audio. Length-preserving.""" + raise NotImplementedError + + def warmup(self) -> None: + """Optional: run a dummy forward pass so the first real chunk isn't slow. + + GPU kernels, cuDNN autotuning, and lazy weight loading otherwise make the + first ``convert`` call take hundreds of ms and cause an audible hitch. + """ + try: + self.convert(np.zeros(self.sample_rate // 4, dtype=np.float32)) + except Exception: + # Warmup is best-effort; a failure here will resurface (loudly) on + # the first real chunk where we can report it in context. + pass + + def close(self) -> None: + """Release models / GPU memory. Safe to call multiple times.""" + return None + + +def build_converter(config) -> VoiceConverter: + """Factory: instantiate the backend named in ``config.backend``. + + Imports are done lazily inside each branch so that, e.g., running the + passthrough backend on a machine without torch/CUDA never imports torch. + """ + name = config.backend.lower() + if name == "passthrough": + from .backends.passthrough import PassthroughConverter + + return PassthroughConverter(config.passthrough, config.stream.pipeline_samplerate) + if name == "rvc": + from .backends.rvc import RVCConverter + + return RVCConverter(config.rvc) + if name == "seedvc": + from .backends.seedvc import SeedVCConverter + + return SeedVCConverter(config.seedvc) + raise ValueError( + f"Unknown backend '{config.backend}'. Expected one of: passthrough, rvc, seedvc." + ) diff --git a/voicebridge/src/voicebridge/dsp.py b/voicebridge/src/voicebridge/dsp.py new file mode 100644 index 0000000..b5d127c --- /dev/null +++ b/voicebridge/src/voicebridge/dsp.py @@ -0,0 +1,122 @@ +"""Pure-DSP helpers used by the real-time engine. + +Everything here is plain NumPy with no audio-hardware or GPU dependency, so it +runs and is unit-tested anywhere (see tests/test_dsp.py). Audio is always +float32, mono, in the range [-1, 1] unless noted. +""" + +from __future__ import annotations + +import numpy as np + +try: # soxr is the high-quality real-time resampler; fall back to linear if absent. + import soxr # type: ignore + + _HAVE_SOXR = True +except Exception: # pragma: no cover - optional dependency + _HAVE_SOXR = False + + +def equal_power_fades(n: int) -> tuple[np.ndarray, np.ndarray]: + """Return (fade_in, fade_out) curves of length ``n`` whose squares sum to 1. + + Equal-power (constant-energy) crossfades avoid the volume dip you get from a + naive linear blend, which matters because consecutive model chunks are + correlated. fade_in rises as sin, fade_out falls as cos, and + fade_in**2 + fade_out**2 == 1 everywhere. + """ + if n <= 0: + empty = np.zeros(0, dtype=np.float32) + return empty, empty.copy() + t = np.linspace(0.0, np.pi / 2.0, n, endpoint=True, dtype=np.float32) + fade_in = np.sin(t).astype(np.float32) + fade_out = np.cos(t).astype(np.float32) + return fade_in, fade_out + + +def crossfade(prev_tail: np.ndarray, new_head: np.ndarray) -> np.ndarray: + """Equal-power blend of the previous chunk's tail with the new chunk's head. + + Both inputs must be the same length (the crossfade region). Returns the + blended region. + """ + if prev_tail.shape != new_head.shape: + raise ValueError( + f"crossfade regions must match: {prev_tail.shape} vs {new_head.shape}" + ) + fade_in, fade_out = equal_power_fades(len(new_head)) + return (prev_tail * fade_out + new_head * fade_in).astype(np.float32) + + +def resample(audio: np.ndarray, src_sr: int, dst_sr: int) -> np.ndarray: + """Resample mono float32 audio. Uses soxr when available, else linear interp. + + Linear interpolation is only a fallback for environments without soxr; it is + fine for the passthrough test path but you want soxr installed for quality. + """ + if src_sr == dst_sr or audio.size == 0: + return audio.astype(np.float32, copy=False) + if _HAVE_SOXR: + return soxr.resample(audio, src_sr, dst_sr).astype(np.float32) + # Linear fallback. + duration = audio.shape[0] / float(src_sr) + dst_len = int(round(duration * dst_sr)) + if dst_len <= 0: + return np.zeros(0, dtype=np.float32) + src_idx = np.linspace(0.0, audio.shape[0] - 1, dst_len, dtype=np.float64) + return np.interp(src_idx, np.arange(audio.shape[0]), audio).astype(np.float32) + + +def rms(audio: np.ndarray) -> float: + """Root-mean-square level of a block (0.0 for empty).""" + if audio.size == 0: + return 0.0 + return float(np.sqrt(np.mean(np.square(audio, dtype=np.float64)))) + + +def dbfs(audio: np.ndarray) -> float: + """RMS level in dBFS (decibels relative to full scale). -inf for silence.""" + r = rms(audio) + if r <= 1e-9: + return float("-inf") + return 20.0 * float(np.log10(r)) + + +def noise_gate(audio: np.ndarray, threshold_db: float = -45.0) -> np.ndarray: + """Hard noise gate: silence a whole block whose RMS is below ``threshold_db``. + + Real-time VC happily "voices" room hiss, keyboard noise, and a roommate's TV + as you. Gating sub-threshold blocks keeps the virtual mic quiet when she is + not actually speaking. Block-level (not sample-level) on purpose: it's cheap + and avoids chopping word onsets. + """ + if audio.size == 0: + return audio + if dbfs(audio) < threshold_db: + return np.zeros_like(audio) + return audio + + +def soft_clip(audio: np.ndarray) -> np.ndarray: + """Tanh soft-clip to keep output inside [-1, 1] without harsh digital clipping.""" + return np.tanh(audio).astype(np.float32) + + +def pitch_shift_naive(audio: np.ndarray, semitones: float) -> np.ndarray: + """A *toy* pitch shift (resample-based) for the passthrough test backend ONLY. + + This is the thing the project is explicitly NOT: it does not preserve + formants and will sound like a chipmunk/giant. It exists solely so you can + audibly confirm the mic->cable->game routing is alive before a real model is + installed. The neural backends do proper timbre conversion instead. + """ + if semitones == 0 or audio.size == 0: + return audio.astype(np.float32, copy=False) + ratio = 2.0 ** (semitones / 12.0) + # Resample to change pitch+length, then restore original length. + stretched = resample(audio, src_sr=int(1e6), dst_sr=int(1e6 / ratio)) + if len(stretched) >= len(audio): + return stretched[: len(audio)].astype(np.float32) + out = np.zeros(len(audio), dtype=np.float32) + out[: len(stretched)] = stretched + return out diff --git a/voicebridge/src/voicebridge/engine.py b/voicebridge/src/voicebridge/engine.py new file mode 100644 index 0000000..a465a3c --- /dev/null +++ b/voicebridge/src/voicebridge/engine.py @@ -0,0 +1,178 @@ +"""The real-time conversion engine: chunking, context, crossfade, latency. + +``ChunkProcessor`` is the pure streaming algorithm (NumPy only, unit-testable): +it turns a stream of device-rate audio blocks into a stream of converted blocks, +hiding chunk seams with an equal-power overlap-add crossfade and giving the model +lookback via a context window. It is exactly the algorithm in docs/DESIGN.md §4. + +``RealtimeEngine`` wires a ChunkProcessor to a backend and the audio pipeline, +and exposes live meters for the GUI. +""" + +from __future__ import annotations + +import logging +import time + +import numpy as np + +from . import dsp +from .audio_io import AudioPipeline +from .config import Config +from .converter import VoiceConverter, build_converter + +log = logging.getLogger("voicebridge.engine") + + +class ChunkProcessor: + """Stateful, single-threaded streaming converter. + + All internal buffers live at the converter's sample rate ("pipeline SR"). + Device-rate audio is resampled in on the way through ``process`` and back out. + """ + + def __init__(self, converter: VoiceConverter, stream_cfg, device_sr: int) -> None: + self.converter = converter + self.pipeline_sr = converter.sample_rate + self.device_sr = device_sr + + self.crossfade_n = max(0, int(round(stream_cfg.crossfade_ms / 1000.0 * self.pipeline_sr))) + self.context_n = max(0, int(round(stream_cfg.context_ms / 1000.0 * self.pipeline_sr))) + + self._context = np.zeros(self.context_n, dtype=np.float32) + self._prev_tail = np.zeros(self.crossfade_n, dtype=np.float32) + + # live telemetry (read by the GUI/CLI; written here) + self.last_infer_ms: float = 0.0 + self.input_rms: float = 0.0 + self.output_rms: float = 0.0 + + def process(self, device_block: np.ndarray) -> np.ndarray: + """Convert one device-rate mono block; returns a device-rate mono block + of the same length.""" + out_frames = len(device_block) + self.input_rms = dsp.rms(device_block) + + # 1. device SR -> pipeline SR + x = dsp.resample(device_block, self.device_sr, self.pipeline_sr) + b = len(x) + xf = min(self.crossfade_n, b) # crossfade can't exceed the block + + # 2. prepend context (lookback) and slide the window + model_in = np.concatenate([self._context, x]) if self.context_n else x + if self.context_n: + self._context = model_in[-self.context_n:].astype(np.float32, copy=True) + + # 3. run the model + t0 = time.perf_counter() + y = self.converter.convert(model_in) + self.last_infer_ms = (time.perf_counter() - t0) * 1000.0 + # defensively length-preserve + if len(y) != len(model_in): + y = _fit(y, len(model_in)) + + # 4. take the tail corresponding to (block + crossfade overlap) + seg = y[-(b + xf):] if (b + xf) > 0 else y + seg = _fit(seg, b + xf) + + # 5. equal-power crossfade the head with the previous tail; body verbatim + play = np.empty(b, dtype=np.float32) + if xf > 0: + prev = _fit(self._prev_tail, xf) + play[:xf] = dsp.crossfade(prev, seg[:xf]) + play[xf:] = seg[xf:b] + + # 6. remember the overlap tail for next time + self._prev_tail = seg[b:b + xf].astype(np.float32, copy=True) if xf > 0 else self._prev_tail + + # 7. pipeline SR -> device SR, match the requested block length, protect output + out = dsp.resample(play, self.pipeline_sr, self.device_sr) + out = _fit(out, out_frames) + out = dsp.soft_clip(out) + self.output_rms = dsp.rms(out) + return out + + +def _amp_to_db(amplitude: float) -> float: + """Convert a linear RMS amplitude to dBFS.""" + return 20.0 * float(np.log10(max(amplitude, 1e-9))) + + +def _fit(a: np.ndarray, n: int) -> np.ndarray: + """Trim or zero-pad ``a`` to exactly length ``n`` (real-time safety net).""" + if len(a) == n: + return a.astype(np.float32, copy=False) + if len(a) > n: + return a[:n].astype(np.float32, copy=False) + out = np.zeros(n, dtype=np.float32) + out[: len(a)] = a + return out + + +class RealtimeEngine: + """Owns the converter, the chunk processor, and the audio pipeline.""" + + def __init__(self, config: Config) -> None: + self.config = config + self.converter: VoiceConverter | None = None + self.processor: ChunkProcessor | None = None + self.pipeline: AudioPipeline | None = None + self._last_report = 0.0 + + # -- lifecycle --------------------------------------------------------- + def start(self) -> None: + cfg = self.config + log.info("Building backend: %s", cfg.backend) + self.converter = build_converter(cfg) + log.info("Warming up backend ...") + self.converter.warmup() + + self.processor = ChunkProcessor(self.converter, cfg.stream, cfg.audio.samplerate) + block_frames = int(round(cfg.stream.block_ms / 1000.0 * cfg.audio.samplerate)) + + self.pipeline = AudioPipeline( + audio_cfg=cfg.audio, + block_frames=block_frames, + processor=self.processor.process, + on_tick=self._maybe_report if cfg.logging.show_latency else None, + ) + algo = cfg.stream.block_ms + cfg.stream.crossfade_ms + log.info( + "Starting engine | block=%dms crossfade=%dms context=%dms " + "| pipeline_sr=%d device_sr=%d | est. added latency ~%dms + inference", + cfg.stream.block_ms, cfg.stream.crossfade_ms, cfg.stream.context_ms, + self.processor.pipeline_sr, cfg.audio.samplerate, algo, + ) + self.pipeline.start() + + def stop(self) -> None: + if self.pipeline is not None: + self.pipeline.stop() + self.pipeline = None + if self.converter is not None: + self.converter.close() + self.converter = None + + @property + def running(self) -> bool: + return self.pipeline is not None and self.pipeline.running + + # -- telemetry --------------------------------------------------------- + def levels(self) -> tuple[float, float, float]: + """(input_rms, output_rms, last_infer_ms) for the GUI meters.""" + if self.processor is None: + return 0.0, 0.0, 0.0 + return self.processor.input_rms, self.processor.output_rms, self.processor.last_infer_ms + + def _maybe_report(self) -> None: + now = time.perf_counter() + if now - self._last_report < 2.0 or self.processor is None: + return + self._last_report = now + in_db = _amp_to_db(self.processor.input_rms) + out_db = _amp_to_db(self.processor.output_rms) + drops = self.pipeline.dropouts if self.pipeline else 0 + log.info( + "in %5.1f dBFS | out %5.1f dBFS | infer %5.1f ms | dropouts %d", + in_db, out_db, self.processor.last_infer_ms, drops, + ) diff --git a/voicebridge/src/voicebridge/gui.py b/voicebridge/src/voicebridge/gui.py new file mode 100644 index 0000000..6fd4be4 --- /dev/null +++ b/voicebridge/src/voicebridge/gui.py @@ -0,0 +1,149 @@ +"""A tiny control panel so the engine can be driven without a terminal. + +Design goal: the person using the voice (not the person who built it) opens this, +picks her microphone, clicks Start, and switches the game's mic to "CABLE +Output". Everything else is defaults from config.yaml. + +Tkinter ships with the standard Windows Python install, so there's no extra +dependency. Imported lazily so the rest of the package works on headless boxes. +""" + +from __future__ import annotations + +import logging +import math + +from .config import Config +from .engine import RealtimeEngine + +log = logging.getLogger("voicebridge.gui") + + +def _db_to_meter(db: float) -> int: + """Map dBFS (-60..0) to a 0..100 meter value.""" + if math.isinf(db): + return 0 + return max(0, min(100, int((db + 60.0) / 60.0 * 100.0))) + + +def run_gui(config: Config) -> None: + import tkinter as tk + from tkinter import ttk + + import sounddevice as sd + + engine = RealtimeEngine(config) + + root = tk.Tk() + root.title("VoiceBridge") + root.geometry("440x340") + root.resizable(False, False) + + main = ttk.Frame(root, padding=14) + main.pack(fill="both", expand=True) + + # -- device pickers ---------------------------------------------------- + inputs = [d["name"] for d in sd.query_devices() if d["max_input_channels"] > 0] + outputs = [d["name"] for d in sd.query_devices() if d["max_output_channels"] > 0] + + ttk.Label(main, text="Your microphone").grid(row=0, column=0, sticky="w") + mic_var = tk.StringVar(value=_match(inputs, config.audio.input_device)) + ttk.Combobox(main, textvariable=mic_var, values=inputs, width=42, state="readonly").grid( + row=1, column=0, columnspan=2, pady=(0, 8), sticky="we" + ) + + ttk.Label(main, text="Output to game (virtual cable)").grid(row=2, column=0, sticky="w") + out_var = tk.StringVar(value=_match(outputs, config.audio.output_device)) + ttk.Combobox(main, textvariable=out_var, values=outputs, width=42, state="readonly").grid( + row=3, column=0, columnspan=2, pady=(0, 8), sticky="we" + ) + + monitor_var = tk.BooleanVar(value=bool(config.audio.monitor_device)) + ttk.Checkbutton(main, text="Hear myself in headphones (monitor)", variable=monitor_var).grid( + row=4, column=0, columnspan=2, sticky="w", pady=(0, 8) + ) + + # -- meters ------------------------------------------------------------ + ttk.Label(main, text="Input").grid(row=5, column=0, sticky="w") + in_meter = ttk.Progressbar(main, length=300, maximum=100) + in_meter.grid(row=5, column=1, sticky="we") + ttk.Label(main, text="Output").grid(row=6, column=0, sticky="w") + out_meter = ttk.Progressbar(main, length=300, maximum=100) + out_meter.grid(row=6, column=1, sticky="we") + + status = ttk.Label(main, text=f"Backend: {config.backend} | stopped", foreground="#666") + status.grid(row=7, column=0, columnspan=2, sticky="w", pady=(8, 8)) + + # -- start/stop -------------------------------------------------------- + def apply_selection() -> None: + config.audio.input_device = mic_var.get() or "default" + config.audio.output_device = out_var.get() or config.audio.output_device + # Use the default output for monitoring if the box is ticked and none set. + if monitor_var.get() and not config.audio.monitor_device: + config.audio.monitor_device = "default" + if not monitor_var.get(): + config.audio.monitor_device = None + + def toggle() -> None: + if engine.running: + engine.stop() + start_btn.config(text="Start") + status.config(text=f"Backend: {config.backend} | stopped", foreground="#666") + else: + apply_selection() + try: + engine.start() + except Exception as exc: # surface device/model errors to the user + status.config(text=f"Error: {exc}", foreground="#b00") + log.exception("failed to start engine") + return + start_btn.config(text="Stop") + status.config(text=f"Backend: {config.backend} | LIVE", foreground="#0a0") + + start_btn = ttk.Button(main, text="Start", command=toggle) + start_btn.grid(row=8, column=0, columnspan=2, pady=4, sticky="we") + + ttk.Label( + main, + text='In the game, set Microphone = "CABLE Output".', + foreground="#888", + ).grid(row=9, column=0, columnspan=2, sticky="w", pady=(6, 0)) + + main.columnconfigure(1, weight=1) + + # -- meter refresh loop ------------------------------------------------ + def refresh() -> None: + if engine.running: + in_rms, out_rms, infer_ms = engine.levels() + in_meter["value"] = _db_to_meter(_db(in_rms)) + out_meter["value"] = _db_to_meter(_db(out_rms)) + status.config(text=f"Backend: {config.backend} | LIVE | {infer_ms:.0f} ms/chunk") + else: + in_meter["value"] = 0 + out_meter["value"] = 0 + root.after(100, refresh) + + def on_close() -> None: + engine.stop() + root.destroy() + + root.protocol("WM_DELETE_WINDOW", on_close) + refresh() + root.mainloop() + + +def _db(rms: float) -> float: + return 20.0 * math.log10(rms) if rms > 1e-9 else float("-inf") + + +def _match(names: list[str], spec: str) -> str: + """Pick the device name in ``names`` that best matches the config spec.""" + if not names: + return "" + if not spec or spec.lower() == "default": + return names[0] + needle = spec.lower() + for n in names: + if needle in n.lower(): + return n + return names[0] diff --git a/voicebridge/src/voicebridge/record.py b/voicebridge/src/voicebridge/record.py new file mode 100644 index 0000000..c84419d --- /dev/null +++ b/voicebridge/src/voicebridge/record.py @@ -0,0 +1,146 @@ +"""Guided recording tool — capture clean training data for your voice clone. + +Final model quality is dominated by the quality of *this* recording, not by +clever settings later. So this tool enforces the boring essentials: mono, a +consistent and non-clipping level, light silence trimming, and a known sample +rate. It walks you sentence-by-sentence through docs/sentences.txt. + +Usage: + python -m voicebridge record # full guided dataset + python -m voicebridge record --reference # one ~30s clip for zero-shot Seed-VC +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +import numpy as np + +log = logging.getLogger("voicebridge.record") + +DEFAULT_SR = 48000 # record high; training tools downsample as needed + + +def _record_until_enter(samplerate: int) -> np.ndarray: + """Record mono audio from the default input until the user presses Enter.""" + import sounddevice as sd + + chunks: list[np.ndarray] = [] + + def cb(indata, frames, time_info, status): + if status: + log.debug("rec status: %s", status) + chunks.append(indata[:, 0].copy()) + + with sd.InputStream(samplerate=samplerate, channels=1, dtype="float32", callback=cb): + input() # blocks here while audio accumulates in the callback + if not chunks: + return np.zeros(0, dtype=np.float32) + return np.concatenate(chunks).astype(np.float32) + + +def _postprocess(audio: np.ndarray, peak: float = 0.97, silence_db: float = -45.0, + samplerate: int = DEFAULT_SR) -> np.ndarray: + """Trim leading/trailing silence and peak-normalize without clipping.""" + if audio.size == 0: + return audio + # trim silence based on a short-window envelope + win = max(1, samplerate // 100) # 10 ms + env = np.convolve(np.abs(audio), np.ones(win) / win, mode="same") + thresh = 10.0 ** (silence_db / 20.0) + voiced = np.where(env > thresh)[0] + if voiced.size: + audio = audio[voiced[0]: voiced[-1] + 1] + peak_now = float(np.max(np.abs(audio))) if audio.size else 0.0 + if peak_now > 1e-6: + audio = audio * (peak / peak_now) + return audio.astype(np.float32) + + +def _save(audio: np.ndarray, path: Path, samplerate: int) -> None: + import soundfile as sf + + path.parent.mkdir(parents=True, exist_ok=True) + sf.write(str(path), audio, samplerate) + log.info("saved %s (%.1fs)", path, len(audio) / samplerate) + + +def record_reference(out_path: str = "recordings/reference.wav", samplerate: int = DEFAULT_SR) -> None: + print("\n=== Reference clip (zero-shot Seed-VC) ===") + print("Read ~30 seconds of natural speech. Press Enter to START, Enter again to STOP.") + input("Press Enter to start... ") + print("Recording... (Enter to stop)") + audio = _record_until_enter(samplerate) + audio = _postprocess(audio, samplerate=samplerate) + _save(audio, Path(out_path), samplerate) + print("Done. Set seedvc.reference_wav to this file.\n") + + +def record_dataset( + out_dir: str = "recordings/dataset", + sentences_file: str = "docs/sentences.txt", + samplerate: int = DEFAULT_SR, +) -> None: + sentences = _load_sentences(sentences_file) + out = Path(out_dir) + out.mkdir(parents=True, exist_ok=True) + print("\n=== Voice dataset recording ===") + print("Tips: quiet room, consistent distance from the mic, normal speaking voice.") + print(f"You'll read {len(sentences)} prompts. For each: Enter to start, Enter to stop.") + print("Type 'r' then Enter to redo the previous one, 'q' to quit early.\n") + + saved = 0 + i = 0 + while i < len(sentences): + sentence = sentences[i] + print(f"[{i + 1}/{len(sentences)}] {sentence}") + cmd = input(" Enter=record r=redo last q=quit : ").strip().lower() + if cmd == "q": + break + if cmd == "r" and i > 0: + i -= 1 + continue + print(" recording... (Enter to stop)") + audio = _record_until_enter(samplerate) + audio = _postprocess(audio, samplerate=samplerate) + if audio.size < samplerate // 2: # < 0.5s -> probably a misfire + print(" too short, let's try that one again.") + continue + _save(audio, out / f"clip_{i + 1:03d}.wav", samplerate) + saved += 1 + i += 1 + + total_s = _dataset_seconds(out) + print(f"\nSaved {saved} clips (~{total_s/60:.1f} min) to {out}.") + if total_s < 5 * 60: + print("Heads up: aim for 5-15 minutes total for a good clone. Run again to add more.") + print("Next: docs/TRAINING.md to turn this into your_voice.pth.\n") + + +def _load_sentences(path: str) -> list[str]: + p = Path(path) + if not p.exists(): + # fall back to a tiny built-in set so the tool still works + return [ + "The quick brown fox jumps over the lazy dog near the riverbank.", + "I never expected the weather to change so quickly this afternoon.", + "Could you pass me the controller? I think it is your turn now.", + "Numbers like one, seven, twelve, and ninety-nine are easy to say.", + "She sells seashells by the seashore on a bright summer morning.", + ] + lines = [ln.strip() for ln in p.read_text(encoding="utf-8").splitlines()] + return [ln for ln in lines if ln and not ln.startswith("#")] + + +def _dataset_seconds(out: Path) -> float: + import soundfile as sf + + total = 0.0 + for wav in out.glob("*.wav"): + try: + info = sf.info(str(wav)) + total += info.frames / info.samplerate + except Exception: + pass + return total diff --git a/voicebridge/src/voicebridge/train.py b/voicebridge/src/voicebridge/train.py new file mode 100644 index 0000000..56dd41a --- /dev/null +++ b/voicebridge/src/voicebridge/train.py @@ -0,0 +1,98 @@ +"""Training orchestration for the RVC cloned-voice model. + +Training a good RVC model is an interactive, GPU-bound process that the RVC +project's own WebUI does well, so this module does NOT reimplement training. +Instead it (a) sanity-checks your recorded dataset and (b) prints the exact, +ordered steps to produce ``your_voice.pth`` — either via the RVC WebUI (easiest) +or the RVC CLI. See docs/TRAINING.md for the full walkthrough. + + python -m voicebridge train --check # validate the dataset + python -m voicebridge train # print the step-by-step plan +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +log = logging.getLogger("voicebridge.train") + + +def check_dataset(dataset_dir: str = "recordings/dataset") -> bool: + """Validate the recorded dataset and warn about common quality problems.""" + import numpy as np + import soundfile as sf + + d = Path(dataset_dir) + wavs = sorted(d.glob("*.wav")) + if not wavs: + print(f"No .wav files in {d}. Record first: python -m voicebridge record") + return False + + total_s = 0.0 + clipped = 0 + quiet = 0 + for wav in wavs: + audio, sr = sf.read(str(wav), dtype="float32") + if audio.ndim > 1: + audio = audio[:, 0] + total_s += len(audio) / sr + peak = float(np.max(np.abs(audio))) if audio.size else 0.0 + if peak >= 0.999: + clipped += 1 + if peak < 0.05: + quiet += 1 + + print(f"Dataset: {len(wavs)} clips, ~{total_s/60:.1f} min total.") + ok = True + if total_s < 5 * 60: + print(" ! Only %.1f min. Aim for 5-15 min; more varied speech = better clone." % (total_s / 60)) + ok = False + if clipped: + print(f" ! {clipped} clip(s) appear clipped (peak ~1.0). Re-record those a bit quieter.") + ok = False + if quiet: + print(f" ! {quiet} clip(s) are very quiet. Move closer to the mic / raise input gain.") + if ok: + print(" Looks good. Proceed to training (see the plan below / docs/TRAINING.md).") + return ok + + +def print_plan(dataset_dir: str = "recordings/dataset", model_name: str = "your_voice") -> None: + plan = f""" +=== Training plan: turn your recordings into {model_name}.pth === + +Recommended (easiest): RVC WebUI + 1. Get the WebUI: + git clone https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI + cd Retrieval-based-Voice-Conversion-WebUI + (install torch with CUDA + requirements per its README; download the + pretrained base models it links — hubert/contentvec + rmvpe.) + 2. Launch: python infer-web.py (opens a browser UI) + 3. Train tab: + - Experiment name: {model_name} + - Point the trainer at your dataset folder: + {Path(dataset_dir).resolve()} + - Target sample rate: 48k (or 40k) + - Pitch guidance: YES (needed for speech), f0 method: rmvpe + - Epochs: start ~150-250; save frequency every ~25 epochs + - Run "Process data" -> "Feature extraction" -> "Train model" -> + "Train feature index". + 4. Collect the outputs: + - generator weights -> {model_name}.pth + - feature index -> added_*.index (rename to {model_name}.index) + 5. Copy both into VoiceBridge's models/ folder and set in config.yaml: + backend: rvc + rvc.model_path: models/{model_name}.pth + rvc.index_path: models/{model_name}.index + +No local GPU? Train on a rented cloud GPU or Colab (search "RVC training +colab"), then download the .pth/.index and use them locally. + +Quick test WITHOUT training: set `backend: seedvc` and record a reference clip +(`python -m voicebridge record --reference`). Lower quality, but instant. + +After deploying: `python -m voicebridge run` (or `gui`), then tune +rvc.f0_up_key / index_rate / protect by ear (docs/TRAINING.md §Tuning). +""" + print(plan) diff --git a/voicebridge/tests/test_config.py b/voicebridge/tests/test_config.py new file mode 100644 index 0000000..6f005de --- /dev/null +++ b/voicebridge/tests/test_config.py @@ -0,0 +1,26 @@ +import pytest + +from voicebridge.config import Config, StreamConfig + + +def test_defaults(): + c = Config() + assert c.backend == "passthrough" + assert c.audio.samplerate == 48000 + assert c.stream.block_ms == 150 + + +def test_from_dict_nested_and_unknown_key(): + c = Config.from_dict({"backend": "rvc", "audio": {"samplerate": 44100}}) + assert c.backend == "rvc" + assert c.audio.samplerate == 44100 + assert isinstance(c.stream, StreamConfig) # untouched -> default + + with pytest.raises(ValueError): + Config.from_dict({"audio": {"bogus_key": 1}}) + + +def test_crossfade_must_be_smaller_than_block(): + with pytest.raises(ValueError): + StreamConfig(block_ms=40, crossfade_ms=40) + StreamConfig(block_ms=150, crossfade_ms=40) # ok diff --git a/voicebridge/tests/test_dsp.py b/voicebridge/tests/test_dsp.py new file mode 100644 index 0000000..26af50f --- /dev/null +++ b/voicebridge/tests/test_dsp.py @@ -0,0 +1,70 @@ +import numpy as np + +from voicebridge import dsp + + +def test_equal_power_fades_sum_to_one(): + fi, fo = dsp.equal_power_fades(256) + energy = fi**2 + fo**2 + assert np.allclose(energy, 1.0, atol=1e-5) + assert fi[0] == 0.0 and abs(fi[-1] - 1.0) < 1e-6 + assert abs(fo[0] - 1.0) < 1e-6 and abs(fo[-1]) < 1e-6 + + +def test_equal_power_fades_zero_length(): + fi, fo = dsp.equal_power_fades(0) + assert fi.size == 0 and fo.size == 0 + + +def test_crossfade_endpoints(): + n = 128 + a = np.ones(n, dtype=np.float32) # previous tail + b = np.full(n, 0.5, dtype=np.float32) # new head + out = dsp.crossfade(a, b) + # start ~= prev value, end ~= new value + assert abs(out[0] - 1.0) < 1e-3 + assert abs(out[-1] - 0.5) < 1e-3 + assert out.dtype == np.float32 + + +def test_crossfade_shape_mismatch_raises(): + try: + dsp.crossfade(np.zeros(4), np.zeros(5)) + except ValueError: + return + raise AssertionError("expected ValueError on mismatched crossfade regions") + + +def test_resample_length_and_identity(): + x = np.random.randn(16000).astype(np.float32) + assert np.array_equal(dsp.resample(x, 16000, 16000), x) + y = dsp.resample(x, 16000, 8000) + assert abs(len(y) - 8000) <= 2 + + +def test_rms_and_dbfs(): + assert dsp.rms(np.zeros(100)) == 0.0 + full = np.ones(100, dtype=np.float32) + assert abs(dsp.rms(full) - 1.0) < 1e-6 + assert abs(dsp.dbfs(full) - 0.0) < 1e-6 + assert dsp.dbfs(np.zeros(10)) == float("-inf") + + +def test_noise_gate_silences_quiet_block(): + quiet = np.full(1000, 1e-4, dtype=np.float32) + assert np.all(dsp.noise_gate(quiet, threshold_db=-45.0) == 0.0) + loud = np.full(1000, 0.5, dtype=np.float32) + assert np.allclose(dsp.noise_gate(loud, threshold_db=-45.0), loud) + + +def test_soft_clip_bounds(): + x = np.array([-5.0, 0.0, 5.0], dtype=np.float32) + out = dsp.soft_clip(x) + assert np.all(np.abs(out) < 1.0) + + +def test_pitch_shift_preserves_length(): + x = np.random.randn(4000).astype(np.float32) + out = dsp.pitch_shift_naive(x, -4.0) + assert len(out) == len(x) + assert np.array_equal(dsp.pitch_shift_naive(x, 0.0), x) diff --git a/voicebridge/tests/test_engine.py b/voicebridge/tests/test_engine.py new file mode 100644 index 0000000..dfaece3 --- /dev/null +++ b/voicebridge/tests/test_engine.py @@ -0,0 +1,73 @@ +import numpy as np + +from voicebridge.config import StreamConfig +from voicebridge.converter import VoiceConverter +from voicebridge.engine import ChunkProcessor + + +class IdentityConverter(VoiceConverter): + """Length-preserving identity backend for testing the streaming math.""" + + def __init__(self, sr: int) -> None: + self.sample_rate = sr + + def convert(self, audio: np.ndarray) -> np.ndarray: + return audio + + +class GainConverter(VoiceConverter): + def __init__(self, sr: int, gain: float) -> None: + self.sample_rate = sr + self.gain = gain + + def convert(self, audio: np.ndarray) -> np.ndarray: + return audio * self.gain + + +def _processor(sr=16000, block_ms=150, crossfade_ms=40, context_ms=200, conv=None): + cfg = StreamConfig(block_ms=block_ms, crossfade_ms=crossfade_ms, + context_ms=context_ms, pipeline_samplerate=sr) + conv = conv or IdentityConverter(sr) + return ChunkProcessor(conv, cfg, device_sr=sr) + + +def test_output_block_matches_input_length(): + sr = 16000 + proc = _processor(sr=sr) + block = (np.random.randn(int(0.15 * sr)) * 0.1).astype(np.float32) + for _ in range(5): + out = proc.process(block) + assert len(out) == len(block) + assert out.dtype == np.float32 + + +def test_handles_resampling_between_device_and_pipeline(): + # device 48k, model wants 16k + cfg = StreamConfig(block_ms=120, crossfade_ms=30, context_ms=200, pipeline_samplerate=16000) + proc = ChunkProcessor(IdentityConverter(16000), cfg, device_sr=48000) + block = (np.random.randn(int(0.12 * 48000)) * 0.1).astype(np.float32) + out = proc.process(block) + assert len(out) == len(block) # comes back at device rate, same length + + +def test_steady_state_signal_passes_through_roughly(): + # A constant tone through identity should come out close to itself in the body. + sr = 16000 + proc = _processor(sr=sr, conv=IdentityConverter(sr)) + tone = (0.2 * np.sin(2 * np.pi * 220 * np.arange(int(0.15 * sr)) / sr)).astype(np.float32) + out = None + for _ in range(4): # let context/crossfade reach steady state + out = proc.process(tone) + # soft-clip(tanh) barely changes a 0.2-amplitude signal; energies should be close + assert abs(float(np.sqrt(np.mean(out**2))) - float(np.sqrt(np.mean(tone**2)))) < 0.05 + + +def test_gain_backend_changes_level(): + sr = 16000 + proc = _processor(sr=sr, conv=GainConverter(sr, 0.5)) + block = (0.4 * np.ones(int(0.15 * sr))).astype(np.float32) + out = None + for _ in range(4): + out = proc.process(block) + # halved (then mild tanh) -> clearly below the input level + assert float(np.mean(np.abs(out))) < 0.4 From 1a3398b3e037a18bd948309c2d3e38e73bd0ecd9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 5 Jun 2026 10:01:55 +0000 Subject: [PATCH 2/2] Add Claude Code config: max reasoning effort effortLevel: xhigh (highest persistable level) + CLAUDE_CODE_EFFORT_LEVEL=max env var (highest-priority lever, accepts session-only 'max'). https://claude.ai/code/session_01Tic4oPjP417WDdfPXW1bFx --- .claude/settings.json | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..07122da --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,7 @@ +{ + "$schema": "https://json.schemastore.org/claude-code-settings.json", + "effortLevel": "xhigh", + "env": { + "CLAUDE_CODE_EFFORT_LEVEL": "max" + } +}