diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 00000000..fe18d135 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,46 @@ +name: CodSpeed + +on: + push: + branches: [main] + paths-ignore: + - "docs/**" + - "*.md" + pull_request: + branches: [main] + paths-ignore: + - "docs/**" + - "*.md" + # Lets CodSpeed trigger backtest runs to generate initial baseline data. + workflow_dispatch: + +# Least privilege: read the repository and mint an OIDC token for CodSpeed. +permissions: + contents: read + id-token: write + +concurrency: + group: ${{ github.workflow }}-${{ github.repository }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('run-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + benchmarks: + name: Run benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12" + + - name: Install benchmark dependencies + run: python -m pip install --disable-pip-version-check --require-hashes -r benchmarks/requirements-bench.txt + + - name: Run benchmarks + uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 + with: + mode: simulation + run: python -m pytest benchmarks --codspeed diff --git a/README.md b/README.md index 75f027cb..ad4c28b6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Codec Carver +[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/ContextualWisdomLab/codec-carver?utm_source=badge) + Python CLI for carving long recordings into metadata-preserved FLAC/Opus files. For the long-recording curation contract (TMK/VAD evidence precedence, @@ -24,6 +26,17 @@ This installs the `codec-carver` console command: codec-carver /path/to/recordings --execute --output-dir under_2gb ``` +## Benchmarks + +CPU-bound planning and transcript-processing paths are benchmarked with +[`pytest-codspeed`](https://codspeed.io/docs/reference/pytest-codspeed) and +tracked by CodSpeed on every pull request: + +```bash +pip install --require-hashes -r benchmarks/requirements-bench.txt +python -m pytest benchmarks --codspeed +``` + ## Web service (Docker) ```bash diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py new file mode 100644 index 00000000..fbe47f44 --- /dev/null +++ b/benchmarks/conftest.py @@ -0,0 +1,11 @@ +"""Shared fixtures for the CodSpeed benchmark suite. + +The project ships flat top-level modules rather than a package, so the +repository root is put on ``sys.path`` to make them importable regardless of +where ``pytest`` is invoked from. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) diff --git a/benchmarks/requirements-bench.in b/benchmarks/requirements-bench.in new file mode 100644 index 00000000..7277b7a5 --- /dev/null +++ b/benchmarks/requirements-bench.in @@ -0,0 +1,2 @@ +pytest +pytest-codspeed diff --git a/benchmarks/requirements-bench.txt b/benchmarks/requirements-bench.txt new file mode 100644 index 00000000..6c6855dc --- /dev/null +++ b/benchmarks/requirements-bench.txt @@ -0,0 +1,69 @@ +# Hash-pinned dependencies for the CodSpeed benchmark job. +# Regenerate with: uv pip compile benchmarks/requirements-bench.in --generate-hashes --python-version 3.12 +iniconfig==2.3.0 \ + --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ + --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + # via pytest +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c + # via pytest +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via pytest +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c + # via + # pytest + # rich +pytest==9.1.1 \ + --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ + --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + # via + # -r benchmarks/requirements-bench.in + # pytest-codspeed +pytest-codspeed==5.0.3 \ + --hash=sha256:005348ea52ace3ede2e2f595913912ad2564cca7b124211a88dc78a9cb1fca63 \ + --hash=sha256:0c383c9121deb58a69f174188e9e4488ffc0daced0ed276abf87747182511901 \ + --hash=sha256:20eba63765be9d1b6cacbbfad84b87d49eb04b357a7045a0899880da181f81e3 \ + --hash=sha256:25464363c7f9b9bd5022e969c0addba616fa40ac9b8f0fc9e030c4538863b32d \ + --hash=sha256:2eeb25fb1ac3f73c4de50e739e78fea396b89782bdb740bf2a7cd2df21f8d4ee \ + --hash=sha256:3a2097247985f5d915a94b80c5552d10979ca858c859fc3edef1bf2baa5c9b7a \ + --hash=sha256:4c682f6645d4eb472f3bd95dbda1805e3af4243610572cb7d6bf94a88e8a0b6c \ + --hash=sha256:51e192905a2230f9956e6160732f76577836953a4a1fb2b1e7be74e51ac7b2a0 \ + --hash=sha256:6524c57fec279a22ffef6112af404036afc71b4704758ae9f0abda429b8478d4 \ + --hash=sha256:73c5c9d98a3372a42611989ccfa437cce3842431ac6d6b9ab42c4f0e59c070f7 \ + --hash=sha256:782f9985b6f6b45b8bc20152d206d3a52b56dd088ba81cb70a71f0b39841be9e \ + --hash=sha256:7ac4344f34bbcdd17f6f8c30dbac3da2f80d223dd112e568fd7f7c2cd4cbc693 \ + --hash=sha256:85505c96a3477c346ec2d2b7dced8478f4c651e2b1666ee102d53a832b511853 \ + --hash=sha256:8df77b3409f54f4a268f77f3ff74992fe1d995cdbaf2cecf8ad74d32db217ce7 \ + --hash=sha256:91afef90e6a96b013495e4702ef5d6358614a449e71008cdc194ef668778b92f \ + --hash=sha256:9aa0815b90196f3c20d736ea8691381e97f12bbe8c7d87af10a351e434b452cb \ + --hash=sha256:a2e0ab65df73e837666d12357280ca50ff6d6ac03ea5266703be518b68170edf \ + --hash=sha256:a4bcdb4b6522738152885ef067e0c8524d5699828d780fb6f464cdb3db44369c \ + --hash=sha256:a5d8695a227ea1c3a41d25db5b3fe720bf1b4808bd38862be811a4efd902c792 \ + --hash=sha256:abe793da40f87295d33988673d34f06ea569848b44490b847552cd416816258a \ + --hash=sha256:bce0a6ea93a5b19658f713312bb67554c19283ab15b454a1e3e55a13e78130f8 \ + --hash=sha256:bf4cc4178cbace8f4d2bd240408276bc4da3850ac5fcb5fb5f8a74ab417615bb \ + --hash=sha256:c3a9ed38dfa776443b86f4b49a982e8443d0953db4974bd2673d63cc904ae1ad \ + --hash=sha256:dbe6a4a00b449b6ba2771f644cbc38bdf55acf5c812e60e5659110e19dd9f510 \ + --hash=sha256:ec9fa6f0af0a9feb0e0bd517fb59ef28f806fbd50c0c6900ac26cbb4d080eba5 \ + --hash=sha256:efd43f82ea03ced8488a767ded9473f050791ab7783ea8654107e1e0ac66af40 \ + --hash=sha256:f56d0339cd98d26f6e561987be25bdd2761a5d53d8f73493b1ebe02d0d451093 \ + --hash=sha256:f852bee785a7a124cb1720b1915670c6742af87747dc4d838f3ffdbd365ce9d9 \ + --hash=sha256:fe2ea83c924c2250675b75686c3ee456b8cf0208d83d552e182a195fdf467378 + # via -r benchmarks/requirements-bench.in +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via pytest-codspeed diff --git a/benchmarks/test_bench_media_shrinker.py b/benchmarks/test_bench_media_shrinker.py new file mode 100644 index 00000000..5b1f8768 --- /dev/null +++ b/benchmarks/test_bench_media_shrinker.py @@ -0,0 +1,213 @@ +"""Benchmarks for the CPU-bound planning and parsing paths of ``media_shrinker``. + +Every function exercised here is pure (no ffmpeg/ffprobe subprocess and no +filesystem access), so the results are deterministic under CodSpeed's CPU +simulation instrument. +""" + +from pathlib import Path + +import pytest + +import media_shrinker +from media_shrinker import MediaProbe, SilenceInterval + +# A 12-hour recording with a detected silence roughly every 45 seconds. +_LONG_DURATION_SECONDS = 12 * 60 * 60.0 +_SILENCE_EVERY_SECONDS = 45.0 + + +def _silencedetect_stderr(count: int) -> str: + """Build a realistic ffmpeg ``silencedetect`` stderr log with ``count`` pairs.""" + lines = [ + "ffmpeg version 6.1.1 Copyright (c) 2000-2023 the FFmpeg developers", + "Input #0, wav, from 'recording.wav':", + " Duration: 12:00:00.00, bitrate: 1411 kb/s", + ] + for index in range(count): + start = index * _SILENCE_EVERY_SECONDS + 12.25 + end = start + 2.5 + (index % 7) * 0.125 + lines.append( + f"[silencedetect @ 0x55d0c8a1b2c0] silence_start: {start:.6f}" + ) + lines.append( + f"size=N/A time={start:.2f} bitrate=N/A speed=512x" + ) + lines.append( + f"[silencedetect @ 0x55d0c8a1b2c0] silence_end: {end:.6f} | " + f"silence_duration: {end - start:.6f}" + ) + return "\n".join(lines) + + +def _silence_intervals(count: int) -> list[SilenceInterval]: + """Return ``count`` evenly spread, unsorted silence intervals.""" + intervals = [ + SilenceInterval( + start_seconds=index * _SILENCE_EVERY_SECONDS + 12.25, + end_seconds=index * _SILENCE_EVERY_SECONDS + 15.0, + ) + for index in range(count) + ] + # Reverse so build_segments has to sort its input like real callers do. + intervals.reverse() + return intervals + + +_SILENCE_LOG_SMALL = _silencedetect_stderr(100) +_SILENCE_LOG_LARGE = _silencedetect_stderr(5_000) +_INTERVALS = _silence_intervals(int(_LONG_DURATION_SECONDS // _SILENCE_EVERY_SECONDS)) + + +@pytest.mark.parametrize( + "stderr", + [ + pytest.param(_SILENCE_LOG_SMALL, id="100_silences"), + pytest.param(_SILENCE_LOG_LARGE, id="5000_silences"), + ], +) +def test_parse_silencedetect_intervals(benchmark, stderr): + """Parse ffmpeg silencedetect stderr into silence intervals.""" + intervals = benchmark(media_shrinker.parse_silencedetect_intervals, stderr) + assert intervals + + +@pytest.mark.parametrize( + "max_segment_seconds", + [ + pytest.param(4 * 60 * 60.0, id="4h_windows"), + pytest.param(10 * 60.0, id="10min_windows"), + ], +) +def test_build_segments_with_silences(benchmark, max_segment_seconds): + """Plan split windows for a 12h recording using detected silences.""" + segments = benchmark( + media_shrinker.build_segments, + duration_seconds=_LONG_DURATION_SECONDS, + max_segment_duration_seconds=max_segment_seconds, + silence_intervals=_INTERVALS, + ) + assert len(segments) > 1 + + +def test_build_segments_hard_splits(benchmark): + """Plan split windows when no silence is available (hard splits only).""" + segments = benchmark( + media_shrinker.build_segments, + duration_seconds=_LONG_DURATION_SECONDS, + max_segment_duration_seconds=60.0, + silence_intervals=(), + ) + assert len(segments) > 1 + + +_PROBE_PAYLOAD = { + "streams": [ + {"codec_type": "video", "codec_name": "mjpeg"}, + *({"codec_type": "data", "codec_name": "bin_data"} for _ in range(30)), + { + "codec_type": "audio", + "codec_name": "pcm_s24le", + "duration": "43200.000000", + "bit_rate": "2304000", + }, + {"codec_type": "audio", "codec_name": "aac", "duration": "43200.0"}, + ], + "format": { + "duration": "43200.000000", + "size": "12441600000", + "bit_rate": "2304000", + "format_name": "mov,mp4,m4a,3gp,3g2,mj2", + }, +} + + +def test_parse_probe_payload(benchmark): + """Parse a multi-stream ffprobe JSON payload into a MediaProbe.""" + + def parse_many(): + source = Path("recording.m4a") + for _ in range(200): + media_shrinker._parse_probe_payload(_PROBE_PAYLOAD, source) + + benchmark(parse_many) + + +_LOSSLESS_PROBE = MediaProbe( + duration_seconds=_LONG_DURATION_SECONDS, + size_bytes=12_441_600_000, + audio_codec="pcm_s24le", + audio_bit_rate=2_304_000, + has_video=False, + format_name="wav", +) +_LOSSY_PROBE = MediaProbe( + duration_seconds=_LONG_DURATION_SECONDS, + size_bytes=4_000_000_000, + audio_codec="aac", + audio_bit_rate=256_000, + has_video=False, + format_name="mov,mp4,m4a,3gp,3g2,mj2", +) +_TAGS = { + "title": "Weekly research meeting", + "artist": "Codec Carver", + "album": "Archive", + "date": "2026-01-01", + "comment": "carved from long recording", +} + + +@pytest.mark.parametrize( + ("probe", "output_format"), + [ + pytest.param(_LOSSLESS_PROBE, "auto", id="flac_auto"), + pytest.param(_LOSSY_PROBE, "auto", id="opus_auto"), + pytest.param(_LOSSY_PROBE, "aac", id="aac"), + pytest.param(_LOSSY_PROBE, "mp3", id="mp3"), + ], +) +def test_build_audio_plan_for_segments(benchmark, probe, output_format): + """Build ffmpeg conversion plans for every segment of a long recording.""" + segments = media_shrinker.build_segments( + duration_seconds=probe.duration_seconds, + max_segment_duration_seconds=30 * 60.0, + silence_intervals=_INTERVALS, + ) + source = Path("/archive/recordings/meeting.wav") + output_dir = Path("/archive/under_2gb") + + def build_all(): + return [ + media_shrinker.build_audio_plan( + source, + probe, + target_bytes=media_shrinker.DEFAULT_SIZE_LIMIT_BYTES, + output_dir=output_dir, + ffmpeg_threads=4, + segment=segment, + tags=_TAGS, + normalize=True, + output_format=output_format, + ) + for segment in segments + ] + + plans = benchmark(build_all) + assert len(plans) == len(segments) + + +def test_calculate_audio_bitrate(benchmark): + """Compute target-fitting bitrates across a range of durations.""" + + def compute_many(): + total = 0 + for minutes in range(10, 1_450, 3): + total += media_shrinker.calculate_audio_bitrate( + minutes * 60.0, + media_shrinker.DEFAULT_SIZE_LIMIT_BYTES, + 320_000, + ) + return total + + assert benchmark(compute_many) > 0 diff --git a/benchmarks/test_bench_transcripts.py b/benchmarks/test_bench_transcripts.py new file mode 100644 index 00000000..6ad5a3dc --- /dev/null +++ b/benchmarks/test_bench_transcripts.py @@ -0,0 +1,169 @@ +"""Benchmarks for transcript post-processing: subtitles, search, diarization, +chapters and summarization. + +All inputs are synthetic and generated in memory so the measured work is the +pure-Python processing, not file I/O. +""" + +import random + +import pytest + +import chapters +import diarize +import subtitles +import summarize +import transcript_search +from media_shrinker import SilenceInterval + +_VOCABULARY = ( + "audio recording meeting budget roadmap release codec opus flac silence " + "segment transcript speaker model latency throughput archive metadata " + "quarter planning review decision action customer feedback research " + "experiment result analysis deadline migration pipeline storage" +).split() + + +def _sentence(rng: random.Random) -> str: + """Return a pseudo-random sentence drawn from a fixed vocabulary.""" + words = [rng.choice(_VOCABULARY) for _ in range(rng.randint(6, 18))] + words[0] = words[0].capitalize() + return " ".join(words) + rng.choice((".", "!", "?", ".")) + + +def _transcript(count: int, seed: int = 1234) -> list[tuple[float, float, str]]: + """Return ``count`` contiguous (start, end, text) transcript segments.""" + rng = random.Random(seed) + segments = [] + cursor = 0.0 + for _ in range(count): + length = rng.uniform(1.5, 8.0) + segments.append((cursor, cursor + length, _sentence(rng))) + cursor += length + rng.uniform(0.0, 0.6) + return segments + + +_RAW_SEGMENTS = _transcript(2_000) +_TOTAL_DURATION = _RAW_SEGMENTS[-1][1] + 1.0 + + +@pytest.mark.parametrize("fmt", ["srt", "vtt"]) +def test_subtitles_render(benchmark, fmt): + """Render a 2,000-cue transcript to SRT/WebVTT.""" + cues = [subtitles.Segment(start, end, text) for start, end, text in _RAW_SEGMENTS] + render = subtitles.to_srt if fmt == "srt" else subtitles.to_vtt + output = benchmark(render, cues) + assert output + + +def test_transcript_index_build(benchmark): + """Build an inverted index over several recordings' transcripts.""" + recordings = { + f"recording-{index:02d}.flac": [ + transcript_search.Segment(start, end, text) + for start, end, text in _transcript(500, seed=index) + ] + for index in range(8) + } + + def build(): + index = transcript_search.TranscriptIndex() + for recording_id, segments in recordings.items(): + index.add(recording_id, segments) + return index + + assert len(benchmark(build)) == 4_000 + + +@pytest.mark.parametrize( + "query", + [ + pytest.param("budget", id="single_term"), + pytest.param("release roadmap decision", id="multi_term"), + ], +) +def test_transcript_index_search(benchmark, query): + """Run AND-semantics queries against a populated transcript index.""" + index = transcript_search.TranscriptIndex() + for recording in range(8): + index.add( + f"recording-{recording:02d}.flac", + ( + transcript_search.Segment(start, end, text) + for start, end, text in _transcript(500, seed=recording) + ), + ) + matches = benchmark(index.search, query) + assert matches + + +def test_tokenize(benchmark): + """Tokenize transcript text for indexing.""" + texts = [text for _, _, text in _RAW_SEGMENTS] + + def tokenize_all(): + return sum(len(transcript_search.tokenize(text)) for text in texts) + + assert benchmark(tokenize_all) > 0 + + +def test_diarize_merge_with_transcript(benchmark): + """Attribute transcript segments to speakers by maximum time overlap.""" + rng = random.Random(99) + turns = [] + cursor = 0.0 + while cursor < _TOTAL_DURATION: + length = rng.uniform(3.0, 40.0) + turns.append( + diarize.SpeakerTurn( + start=cursor, + end=cursor + length, + speaker=f"SPEAKER_{rng.randint(0, 3):02d}", + ) + ) + cursor += length + segments = [ + subtitles.Segment(start, end, text) for start, end, text in _RAW_SEGMENTS[:1_000] + ] + merged = benchmark(diarize.merge_with_transcript, turns, segments) + assert len(merged) == len(segments) + + +def test_detect_chapters(benchmark): + """Propose chapters for a 12h recording from ~1,000 silence intervals.""" + rng = random.Random(7) + total = 12 * 60 * 60.0 + silences = [] + cursor = 0.0 + while cursor < total: + cursor += rng.uniform(10.0, 80.0) + silences.append( + SilenceInterval(start_seconds=cursor, end_seconds=cursor + rng.uniform(0.5, 9.0)) + ) + rng.shuffle(silences) + result = benchmark(chapters.detect_chapters, silences, total) + assert result + + +def test_chapters_serialize(benchmark): + """Serialize detected chapters to FFMETADATA and JSON.""" + silences = [ + SilenceInterval(start_seconds=start, end_seconds=start + 5.0) + for start in range(90, 43_200, 90) + ] + detected = chapters.detect_chapters(silences, 43_200.0) + + def serialize(): + return chapters.to_ffmetadata(detected), chapters.to_json(detected) + + ffmetadata, as_json = benchmark(serialize) + assert ffmetadata and as_json + + +def test_summarize_segments(benchmark): + """Produce an extractive summary of a long transcript.""" + segments = [ + subtitles.Segment(start, end, text) for start, end, text in _RAW_SEGMENTS[:600] + ] + summary = benchmark(summarize.summarize_segments, segments, 8) + assert summary diff --git a/tests/test_workflow_security.py b/tests/test_workflow_security.py index 06aaf1a3..d8939e92 100644 --- a/tests/test_workflow_security.py +++ b/tests/test_workflow_security.py @@ -9,6 +9,7 @@ _WORKFLOW_PATHS = ( _REPOSITORY_ROOT / ".github" / "workflows" / "ci.yml", _REPOSITORY_ROOT / ".github" / "workflows" / "fuzz.yml", + _REPOSITORY_ROOT / ".github" / "workflows" / "codspeed.yml", ) _CHECKOUT_STEP = re.compile( r"(?m)^\s*-\s+uses:\s+actions/checkout@[^\n]+\n"