Add frame dedup and Whisper auto-chunking

Frame extraction now runs a perceptual dedup pass (default on, --no-dedup to disable) that drops near-identical frames before the budget cap, so the budget goes to distinct content instead of held slides/static recordings. The Frames report line notes how many near-duplicates were dropped. Whisper transcription splits audio over the 25 MB upload cap into evenly sized chunks, transcribes each, shifts segment timestamps back into source time, and tolerates partial failures (only fails if every chunk fails) — length alone no longer fails transcription. token-burner is exempt from the long-video sparse-scan warning since it keeps every scene-change frame. README/SKILL.md updated. Adds test_dedup.py and test_whisper.py. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-29 23:16:17 +10:00
parent 429f3143e5
commit 304b639b4d
8 changed files with 707 additions and 43 deletions
@@ -0,0 +1,162 @@
+"""Frame-delta dedup: per-pixel difference, greedy de-duplication, integration."""
+from __future__ import annotations
+
+from pathlib import Path
+
+import frames
+
+
+# --- _frame_delta: mean absolute per-pixel difference ------------------------
+
+def test_frame_delta_identical_is_zero():
+    a = bytes([10] * 16)
+    assert frames._frame_delta(a, a) == 0.0
+
+
+def test_frame_delta_is_mean_absolute_difference():
+    a = bytes([0, 0, 0, 0])
+    b = bytes([4, 0, 0, 0])
+    assert frames._frame_delta(a, b) == 1.0  # (4+0+0+0)/4
+
+
+def test_frame_delta_mismatched_length_is_infinite():
+    assert frames._frame_delta(bytes([1, 2]), bytes([1, 2, 3])) == float("inf")
+
+
+# --- _dedupe_by_deltas: greedy drop vs last *kept* thumbnail ------------------
+
+def _touch(dirpath: Path, n: int) -> list[dict]:
+    dirpath.mkdir(parents=True, exist_ok=True)
+    out = []
+    for i in range(n):
+        p = dirpath / f"frame_{i:04d}.jpg"
+        p.write_bytes(b"x")
+        out.append({"index": i, "timestamp_seconds": float(i), "path": str(p), "reason": "scene-change"})
+    return out
+
+
+FLAT0 = bytes([0, 0, 0, 0])
+FLAT255 = bytes([255, 255, 255, 255])
+
+
+def test_dedupe_collapses_identical_run(tmp_path: Path):
+    cands = _touch(tmp_path, 5)
+    thumbs = [FLAT0, FLAT0, FLAT0, FLAT0, FLAT0]
+    survivors, dropped = frames._dedupe_by_deltas(cands, thumbs, threshold=2.0)
+    assert dropped == 4
+    assert len(survivors) == 1
+    assert survivors[0]["index"] == 0
+    assert sorted(p.name for p in tmp_path.glob("frame_*.jpg")) == ["frame_0000.jpg"]
+
+
+def test_dedupe_keeps_all_distinct(tmp_path: Path):
+    cands = _touch(tmp_path, 4)
+    thumbs = [FLAT0, FLAT255, FLAT0, FLAT255]
+    survivors, dropped = frames._dedupe_by_deltas(cands, thumbs, threshold=2.0)
+    assert dropped == 0
+    assert [s["index"] for s in survivors] == [0, 1, 2, 3]
+    assert len(list(tmp_path.glob("frame_*.jpg"))) == 4
+
+
+def test_dedupe_compares_against_last_kept_not_previous(tmp_path: Path):
+    """A,A,B,B,A with A/B far apart -> keep A0, B2, A4 (drops the repeats)."""
+    cands = _touch(tmp_path, 5)
+    survivors, dropped = frames._dedupe_by_deltas(
+        cands, [FLAT0, FLAT0, FLAT255, FLAT255, FLAT0], threshold=2.0
+    )
+    assert [s["index"] for s in survivors] == [0, 1, 2]  # reindexed survivors
+    assert dropped == 2
+
+
+def test_dedupe_threshold_is_inclusive(tmp_path: Path):
+    """Delta exactly == threshold is treated as a duplicate (<=)."""
+    cands = _touch(tmp_path, 2)
+    a = bytes([0, 0, 0, 0])
+    b = bytes([8, 0, 0, 0])  # mean abs diff == 2.0
+    survivors, dropped = frames._dedupe_by_deltas(cands, [a, b], threshold=2.0)
+    assert dropped == 1
+    assert len(survivors) == 1
+
+
+def test_dedupe_empty_and_single_are_noops(tmp_path: Path):
+    assert frames._dedupe_by_deltas([], [], threshold=2.0) == ([], 0)
+    one = _touch(tmp_path, 1)
+    survivors, dropped = frames._dedupe_by_deltas(one, [FLAT0], threshold=2.0)
+    assert dropped == 0
+    assert len(survivors) == 1
+
+
+def test_dedupe_mismatched_thumb_count_is_noop(tmp_path: Path):
+    """Fail open: if thumbs don't line up with candidates, change nothing."""
+    cands = _touch(tmp_path, 3)
+    survivors, dropped = frames._dedupe_by_deltas(cands, [FLAT0], threshold=2.0)
+    assert dropped == 0
+    assert len(survivors) == 3
+
+
+# --- _thumb_frames + dedupe_perceptual: real ffmpeg over extracted JPEGs ------
+
+def test_thumb_frames_match_candidate_count(cut_clip: Path, tmp_path: Path):
+    out = frames.extract_scene_candidates(str(cut_clip), tmp_path / "f", max_frames=None)
+    thumbs = frames._thumb_frames([Path(fr["path"]) for fr in out])
+    assert len(thumbs) == len(out)
+    assert all(len(t) == frames.DEDUP_THUMB * frames.DEDUP_THUMB for t in thumbs)
+
+
+def test_dedupe_perceptual_collapses_static_clip(static_clip: Path, tmp_path: Path):
+    out = frames.extract(str(static_clip), tmp_path / "f", fps=4.0, max_frames=10)
+    n_before = len(out)
+    survivors, dropped = frames.dedupe_perceptual(out)
+    assert n_before > 1
+    assert len(survivors) == 1
+    assert dropped == n_before - 1
+    assert len(list((tmp_path / "f").glob("frame_*.jpg"))) == 1
+
+
+def test_dedupe_perceptual_keeps_distinct_cuts(cut_clip: Path, tmp_path: Path):
+    """Distinct color shots differ in luma, so frame-delta keeps them all."""
+    out = frames.extract_scene_candidates(str(cut_clip), tmp_path / "f", max_frames=None)
+    n_before = len(out)
+    survivors, dropped = frames.dedupe_perceptual(out)
+    assert dropped == 0
+    assert len(survivors) == n_before
+
+
+# --- engine integration: dedup runs before the cap, reports deduped_count -----
+
+def test_scene_engine_reports_zero_dedup_on_distinct(cut_clip: Path, tmp_path: Path):
+    out, meta = frames.extract_scene_or_uniform(
+        str(cut_clip), tmp_path / "f", fps=2.0, target_frames=50, max_frames=100,
+    )
+    assert meta["engine"] == "scene"
+    assert meta["deduped_count"] == 0
+    assert len(out) == len(list((tmp_path / "f").glob("frame_*.jpg")))
+
+
+def test_uniform_fallback_dedupes_static(static_clip: Path, tmp_path: Path):
+    out, meta = frames.extract_scene_or_uniform(
+        str(static_clip), tmp_path / "f", fps=4.0, target_frames=12, max_frames=100,
+    )
+    assert meta["engine"] == "uniform"
+    assert meta["fallback"] is True
+    assert meta["deduped_count"] > 0
+    assert meta["selected_count"] == 1  # identical frames collapse to one
+    assert len(out) == 1
+    assert len(list((tmp_path / "f").glob("frame_*.jpg"))) == 1
+
+
+def test_keyframe_uniform_fallback_dedupes_static(static_clip: Path, tmp_path: Path):
+    out, meta = frames.extract_keyframes(str(static_clip), tmp_path / "f", max_frames=50)
+    assert meta["engine"] == "uniform"
+    assert meta["deduped_count"] > 0
+    assert len(out) == 1
+
+
+def test_dedup_false_disables_collapse(static_clip: Path, tmp_path: Path):
+    out, meta = frames.extract_scene_or_uniform(
+        str(static_clip), tmp_path / "f", fps=4.0, target_frames=12, max_frames=100,
+        dedup=False,
+    )
+    assert meta["deduped_count"] == 0
+    assert meta["selected_count"] > 1  # no collapse without dedup
+    assert len(out) > 1
@@ -67,3 +67,19 @@ def test_timestamps_with_transcript_detail_is_cue_only(cut_clip: Path):
    assert "reason=transcript-cue" in out
    assert "reason=scene-change" not in out
    assert "reason=keyframe" not in out
+
+
+def _frame_lines(out: str) -> int:
+    return sum(1 for line in out.splitlines() if "/frames/frame_" in line and "(t=" in line)
+
+
+def test_dedup_collapses_static_by_default(static_clip: Path):
+    out = _run(static_clip)  # solid blue → identical frames collapse to one
+    assert "near-duplicate" in out
+    assert _frame_lines(out) == 1
+
+
+def test_no_dedup_preserves_static_frames(static_clip: Path):
+    out = _run(static_clip, "--no-dedup")
+    assert "near-duplicate" not in out
+    assert _frame_lines(out) > 1
@@ -0,0 +1,157 @@
+"""Whisper auto-chunking: plan, split, and timestamp stitching."""
+from __future__ import annotations
+
+import math
+import subprocess
+from pathlib import Path
+
+import pytest
+
+import whisper
+
+
+MB = 1024 * 1024
+
+
+class TestPlanChunks:
+    def test_under_limit_is_single_chunk(self):
+        plan = whisper.plan_chunks(total_seconds=600.0, total_bytes=5 * MB, max_bytes=24 * MB)
+        assert plan == [(0.0, 600.0)]
+
+    def test_at_limit_is_single_chunk(self):
+        plan = whisper.plan_chunks(total_seconds=600.0, total_bytes=24 * MB, max_bytes=24 * MB)
+        assert plan == [(0.0, 600.0)]
+
+    def test_over_limit_splits_into_enough_chunks(self):
+        # 71 MB against a 24 MB cap → ceil(71/24) = 3 chunks.
+        plan = whisper.plan_chunks(total_seconds=3600.0, total_bytes=71 * MB, max_bytes=24 * MB)
+        assert len(plan) == 3
+
+    def test_chunks_are_contiguous_and_cover_full_duration(self):
+        total = 3600.0
+        plan = whisper.plan_chunks(total_seconds=total, total_bytes=71 * MB, max_bytes=24 * MB)
+        # Offsets start at 0 and each picks up where the previous ended.
+        assert plan[0][0] == 0.0
+        for (off, dur), (next_off, _) in zip(plan, plan[1:]):
+            assert math.isclose(off + dur, next_off)
+        last_off, last_dur = plan[-1]
+        assert math.isclose(last_off + last_dur, total)
+
+    def test_each_chunk_estimated_under_limit(self):
+        total_seconds, total_bytes, cap = 3600.0, 71 * MB, 24 * MB
+        plan = whisper.plan_chunks(total_seconds, total_bytes, cap)
+        bytes_per_second = total_bytes / total_seconds
+        for _off, dur in plan:
+            assert dur * bytes_per_second <= cap
+
+    def test_zero_duration_is_single_chunk(self):
+        plan = whisper.plan_chunks(total_seconds=0.0, total_bytes=0, max_bytes=24 * MB)
+        assert plan == [(0.0, 0.0)]
+
+
+class TestShiftSegments:
+    def test_adds_offset_to_start_and_end(self):
+        segs = [{"start": 0.0, "end": 2.5, "text": "hi"}, {"start": 2.5, "end": 4.0, "text": "there"}]
+        shifted = whisper.shift_segments(segs, 1800.0)
+        assert shifted == [
+            {"start": 1800.0, "end": 1802.5, "text": "hi"},
+            {"start": 1802.5, "end": 1804.0, "text": "there"},
+        ]
+
+    def test_zero_offset_is_identity(self):
+        segs = [{"start": 1.0, "end": 2.0, "text": "x"}]
+        assert whisper.shift_segments(segs, 0.0) == segs
+
+    def test_does_not_mutate_input(self):
+        segs = [{"start": 0.0, "end": 1.0, "text": "x"}]
+        whisper.shift_segments(segs, 10.0)
+        assert segs[0]["start"] == 0.0
+
+
+def _make_mp3(path: Path, seconds: float) -> None:
+    """Synthesize a mono 16k 64k mp3 of a sine tone — mirrors extract_audio's format."""
+    subprocess.run(
+        [
+            "ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
+            "-f", "lavfi", "-t", str(seconds), "-i", "sine=frequency=440:sample_rate=16000",
+            "-acodec", "libmp3lame", "-ar", "16000", "-ac", "1", "-b:a", "64k",
+            str(path),
+        ],
+        check=True,
+    )
+
+
+class TestSplitAudio:
+    def test_creates_one_file_per_plan_entry(self, tmp_path: Path):
+        full = tmp_path / "audio.mp3"
+        _make_mp3(full, 6.0)
+        plan = [(0.0, 3.0), (3.0, 3.0)]
+
+        chunks = whisper.split_audio(full, tmp_path, plan)
+
+        assert len(chunks) == 2
+        for chunk_path, _offset in chunks:
+            assert chunk_path.exists() and chunk_path.stat().st_size > 0
+
+    def test_returns_plan_offsets(self, tmp_path: Path):
+        full = tmp_path / "audio.mp3"
+        _make_mp3(full, 6.0)
+        plan = [(0.0, 3.0), (3.0, 3.0)]
+
+        chunks = whisper.split_audio(full, tmp_path, plan)
+
+        assert [offset for _path, offset in chunks] == [0.0, 3.0]
+
+    def test_chunks_are_smaller_than_full(self, tmp_path: Path):
+        full = tmp_path / "audio.mp3"
+        _make_mp3(full, 6.0)
+        plan = [(0.0, 3.0), (3.0, 3.0)]
+
+        chunks = whisper.split_audio(full, tmp_path, plan)
+
+        full_size = full.stat().st_size
+        for chunk_path, _offset in chunks:
+            assert chunk_path.stat().st_size < full_size
+
+
+class TestAudioDuration:
+    def test_reads_duration_of_synthesized_clip(self, tmp_path: Path):
+        audio = tmp_path / "audio.mp3"
+        _make_mp3(audio, 5.0)
+        assert whisper.audio_duration(audio) == pytest.approx(5.0, abs=0.5)
+
+
+class TestTranscribeChunks:
+    def test_shifts_and_concatenates_each_chunk(self):
+        chunks = [(Path("a.mp3"), 0.0), (Path("b.mp3"), 100.0)]
+
+        def fake_transcribe(path: Path) -> list[dict]:
+            return [{"start": 0.0, "end": 2.0, "text": path.stem}]
+
+        out = whisper.transcribe_chunks(chunks, fake_transcribe)
+
+        assert out == [
+            {"start": 0.0, "end": 2.0, "text": "a"},
+            {"start": 100.0, "end": 102.0, "text": "b"},
+        ]
+
+    def test_keeps_successful_chunks_when_one_fails(self):
+        chunks = [(Path("a.mp3"), 0.0), (Path("b.mp3"), 100.0)]
+
+        def flaky(path: Path) -> list[dict]:
+            if path.stem == "b":
+                raise SystemExit("chunk b failed")
+            return [{"start": 1.0, "end": 2.0, "text": "a"}]
+
+        out = whisper.transcribe_chunks(chunks, flaky)
+
+        assert out == [{"start": 1.0, "end": 2.0, "text": "a"}]
+
+    def test_raises_when_every_chunk_fails(self):
+        chunks = [(Path("a.mp3"), 0.0), (Path("b.mp3"), 100.0)]
+
+        def always_fail(path: Path) -> list[dict]:
+            raise SystemExit("boom")
+
+        with pytest.raises(SystemExit):
+            whisper.transcribe_chunks(chunks, always_fail)