Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 bradautomates
|
83da59fa78 | ||
|
bradautomates
|
a1e7c4128b | ||
|
bradautomates
|
242ba568ed | ||
|
bradautomates
|
304b639b4d | ||
|
bradautomates
|
429f3143e5 | ||
|
bradautomates
|
c333c2289e | ||
|
bradautomates
|
205af96c19 | ||
|
bradautomates
|
3c488688d1 | ||
|
bradautomates
|
755c157466 |
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "claude-video",
|
||||
"interface": {
|
||||
"displayName": "watch"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "watch",
|
||||
"source": {
|
||||
"source": "url",
|
||||
"url": "https://github.com/bradautomates/claude-video.git"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
"authentication": "ON_INSTALL"
|
||||
},
|
||||
"category": "Productivity"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "watch",
|
||||
"version": "0.1.1",
|
||||
"version": "0.2.0",
|
||||
"description": "Let Claude watch a video. Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls captions or falls back to Whisper, and hands frames + transcript to Claude so it can answer questions about the video.",
|
||||
"author": {
|
||||
"name": "Bradley Bonanno"
|
||||
|
||||
@@ -1,3 +1,43 @@
|
||||
{
|
||||
"name": "watch"
|
||||
"name": "watch",
|
||||
"version": "0.2.0",
|
||||
"description": "Let Claude watch a video. Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls captions or falls back to Whisper, and hands frames + transcript to Claude so it can answer questions about the video.",
|
||||
"author": {
|
||||
"name": "Bradley Bonanno"
|
||||
},
|
||||
"homepage": "https://github.com/bradautomates/claude-video",
|
||||
"repository": "https://github.com/bradautomates/claude-video",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"video",
|
||||
"watch",
|
||||
"youtube",
|
||||
"vimeo",
|
||||
"tiktok",
|
||||
"transcription",
|
||||
"whisper",
|
||||
"yt-dlp",
|
||||
"ffmpeg",
|
||||
"multimodal",
|
||||
"frames"
|
||||
],
|
||||
"skills": "./skills/",
|
||||
"interface": {
|
||||
"displayName": "watch",
|
||||
"shortDescription": "Give Claude a video input — paste a URL or path and ask about it.",
|
||||
"longDescription": "watch adds a skill that lets the agent watch any video: it downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls a timestamped transcript from native captions (or the Whisper API as a fallback), and hands frames + transcript to the model so it can answer questions grounded in what's actually on screen and in the audio.",
|
||||
"developerName": "Bradley Bonanno",
|
||||
"category": "Productivity",
|
||||
"capabilities": [
|
||||
"Interactive",
|
||||
"Read"
|
||||
],
|
||||
"websiteURL": "https://github.com/bradautomates/claude-video",
|
||||
"defaultPrompt": [
|
||||
"/watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark?",
|
||||
"/watch ~/Movies/screen-recording.mp4 when does the UI break?",
|
||||
"/watch https://youtu.be/<long-talk> summarize this"
|
||||
],
|
||||
"brandColor": "#E11D48"
|
||||
}
|
||||
}
|
||||
|
||||
+17
-5
@@ -1,5 +1,6 @@
|
||||
# Exclude non-runtime files from `git archive` output.
|
||||
# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file.
|
||||
# Used by skills/watch/scripts/build-skill.sh to produce the claude.ai .skill file
|
||||
# and by Claude Code's /plugin install (full-repo archive).
|
||||
|
||||
# Anthropic canonical skill-packaging excludes
|
||||
# (mirrors anthropics/skills/skills/skill-creator/scripts/package_skill.py)
|
||||
@@ -15,10 +16,21 @@ fixtures/ export-ignore
|
||||
assets/ export-ignore
|
||||
examples/ export-ignore
|
||||
|
||||
# NOTE: commands/, hooks/, and .claude-plugin/ are NOT export-ignored because
|
||||
# Claude Code's /plugin install fetches this same git archive tarball — they
|
||||
# must be in the archive for the plugin to install correctly. The claude.ai
|
||||
# .skill bundle strips them afterward via `zip -d` in scripts/build-skill.sh.
|
||||
# Dev tooling and planning notes — repo-only, not shipped to any install surface
|
||||
dev-sync.sh export-ignore
|
||||
V2_PLAN.md export-ignore
|
||||
V2_CONCERNS.md export-ignore
|
||||
.agents/ export-ignore
|
||||
|
||||
# Dev-only build script + scanner config inside the skill tree: keep them out of
|
||||
# the claude.ai .skill bundle (built via `git archive HEAD:skills/watch`).
|
||||
skills/watch/scripts/build-skill.sh export-ignore
|
||||
skills/watch/.skillignore export-ignore
|
||||
|
||||
# NOTE: hooks/ and .claude-plugin/ are NOT export-ignored because Claude Code's
|
||||
# /plugin install fetches the full-repo git archive — they must be present for
|
||||
# the plugin to install correctly. The claude.ai .skill bundle is built from the
|
||||
# skills/watch/ subtree only, so it never sees them in the first place.
|
||||
|
||||
# CI workflows — repo-only, not needed at skill runtime
|
||||
.github/ export-ignore
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
- name: Build .skill artifact
|
||||
run: |
|
||||
bash scripts/build-skill.sh
|
||||
bash skills/watch/scripts/build-skill.sh
|
||||
test -f dist/watch.skill
|
||||
|
||||
- name: Create GitHub release
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# Agent Skills install-time scanner exclusions for repository-root scans.
|
||||
# Keep the public bundle focused on the runtime skill under skills/watch/.
|
||||
|
||||
# VCS, local envs, caches, and generated outputs
|
||||
.git/
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.log
|
||||
.DS_Store
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
dist/
|
||||
|
||||
# Repo/dev automation and host-specific package metadata
|
||||
.github/
|
||||
.agents/
|
||||
.claude-plugin/
|
||||
hooks/
|
||||
dev-sync.sh
|
||||
|
||||
# Non-runtime docs, plans, and tests
|
||||
tests/
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
V2_PLAN.md
|
||||
V2_CONCERNS.md
|
||||
|
||||
# Dev-only script shipped inside the skill tree but not needed at runtime
|
||||
skills/watch/scripts/build-skill.sh
|
||||
@@ -0,0 +1,50 @@
|
||||
# claude-video / watch skill
|
||||
|
||||
Agent Skills package that gives an agent a video input. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, and 50+ other [Agent Skills](https://agentskills.io) hosts. Pure-stdlib Python that orchestrates `yt-dlp` + `ffmpeg` and an optional Whisper API.
|
||||
|
||||
## Structure
|
||||
|
||||
- `skills/watch/SKILL.md` — canonical skill contract the model reads when `/watch` fires. Source of truth for behavior across every host.
|
||||
- `skills/watch/scripts/watch.py` — entry point; orchestrates download → frames → transcript.
|
||||
- `skills/watch/scripts/{download,frames,transcribe,whisper,setup,config}.py` — yt-dlp wrapper, ffmpeg frame extraction + auto-fps, caption/Whisper transcription, preflight/installer, shared config.
|
||||
- `skills/watch/scripts/build-skill.sh` — builds `dist/watch.skill` for claude.ai upload (dev-only).
|
||||
- `hooks/` — Claude Code SessionStart setup-status hook (Claude Code only).
|
||||
- `.claude-plugin/` — `plugin.json` + `marketplace.json` (Claude Code plugin + local marketplace).
|
||||
- `.codex-plugin/plugin.json` — Codex/agents manifest; `"skills": "./skills/"` points the Agent Skills CLI at the self-contained skill folder.
|
||||
- `.agents/plugins/marketplace.json` — agents marketplace listing pointing at the repo-root plugin.
|
||||
- `CLAUDE.md` → `@AGENTS.md` — generic-agent entry point.
|
||||
- `tests/` — pytest suite (ffmpeg-synthesized clips; no network).
|
||||
|
||||
## Orientation
|
||||
|
||||
- The product is the slash-command-invoked skill (`/watch <url-or-path> [question]`), not a CLI. `scripts/watch.py` is implementation. Features must work across every harness the skill installs into, not just Claude Code.
|
||||
- **The skill is one self-contained folder: `skills/watch/`.** SKILL.md and `scripts/` are siblings inside it. This is what lets `npx skills add` copy a working skill as a unit — do NOT move SKILL.md or `scripts/` back to the repo root, or non-Claude installers will copy SKILL.md without the scripts.
|
||||
- **Path resolution is harness-agnostic.** SKILL.md resolves `SKILL_DIR` as the directory of the SKILL.md the model just Read, then runs `${SKILL_DIR}/scripts/...`. Do NOT reintroduce `${CLAUDE_SKILL_DIR}` (Claude-Code-only) — it is unset on Codex/Cursor/agents and breaks every script call there.
|
||||
- **No `commands/` wrapper.** `/watch` is derived from SKILL.md frontmatter (`name: watch` + `user-invocable: true`). A separate command file creates a duplicate slash command.
|
||||
|
||||
## Install surfaces
|
||||
|
||||
| Surface | Install |
|
||||
|---------|---------|
|
||||
| Claude Code | `/plugin marketplace add bradautomates/claude-video` then `/plugin install watch@claude-video` |
|
||||
| Codex / Cursor / Copilot / +50 | `npx skills add bradautomates/claude-video -g` |
|
||||
| claude.ai (web) | upload `dist/watch.skill` (built by `skills/watch/scripts/build-skill.sh`) |
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Tests (stdlib + pytest; ffmpeg required for frame tests)
|
||||
.venv/bin/pytest -q # or: python3 -m pytest -q
|
||||
|
||||
# Build the claude.ai upload bundle (archives skills/watch/ as the bundle root)
|
||||
bash skills/watch/scripts/build-skill.sh # → dist/watch.skill
|
||||
|
||||
# Dev: mirror the working tree into the installed Claude Code plugin cache
|
||||
./dev-sync.sh # --dry-run to preview
|
||||
```
|
||||
|
||||
## Rules
|
||||
|
||||
- Keep the version in sync across `skills/watch/SKILL.md` (frontmatter), `.claude-plugin/plugin.json`, and `.codex-plugin/plugin.json` when cutting a release.
|
||||
- Releasing: tag `vX.Y.Z` and push the tag; `.github/workflows/release.yml` builds `dist/watch.skill` and attaches it to the GitHub release.
|
||||
- Never commit real API keys or `.env` contents; keys live in `~/.config/watch/.env` (mode `0600`) at runtime.
|
||||
@@ -2,6 +2,48 @@
|
||||
|
||||
All notable changes to `/watch` are documented here.
|
||||
|
||||
## [0.2.0] — 2026-06-29
|
||||
|
||||
### Added
|
||||
- **`--detail` dial** with four modes — `transcript` (captions only, no frames), `efficient` (fast keyframe pass, cap 50), `balanced` (scene-aware, cap 100, default), and `token-burner` (scene-aware, uncapped). Set the default with `WATCH_DETAIL` in `~/.config/watch/.env`.
|
||||
- **Frame deduplication** (default on; `--no-dedup` to disable). Before the budget cap, a pass downscales each frame to a 16×16 grayscale thumbnail and drops frames whose mean per-pixel difference from the last *kept* frame is within threshold — so the budget goes to distinct content instead of held slides and static recordings. The **Frames** report line shows how many near-duplicates were dropped.
|
||||
- **Whisper auto-chunking.** Audio over the 25 MB upload cap is split into evenly sized chunks, transcribed per chunk, with segment timestamps shifted back into source time. Partial failures are tolerated — transcription only fails if *every* chunk fails, so length alone no longer breaks it.
|
||||
- **`--timestamps T1,T2,…`** — grab a frame at each absolute timestamp; reserved against the cap, and the only frames produced under `--detail transcript`.
|
||||
- **`--no-whisper`** — disable transcription entirely (frames only).
|
||||
- pytest suite covering config, dedup, download, fixtures, frames, setup, timestamps, watch, and whisper (no network; ffmpeg-synthesized clips).
|
||||
|
||||
### Changed
|
||||
- **Restructured into a self-contained `skills/watch/` package** so `SKILL.md` and its `scripts/` runtime are siblings in one folder. This fixes installs on Codex, Cursor, Copilot, and other Agent Skills hosts: `npx skills add` now copies the skill as a working unit instead of grabbing the root `SKILL.md` without its scripts.
|
||||
- **Harness-agnostic path resolution** — `SKILL.md` resolves `$SKILL_DIR` from where it was Read instead of the Claude-Code-only `${CLAUDE_SKILL_DIR}`, so script calls work on every host.
|
||||
- `/watch` is now derived from `SKILL.md` frontmatter; the separate `commands/watch.md` wrapper was dropped to avoid a duplicate slash command.
|
||||
- `balanced` now full-decodes to detect every scene cut across the whole video. The previous early-exit was faster but kept only the first cuts and dropped the tail of long videos.
|
||||
- `token-burner` is exempt from the long-video "sparse scan" warning, since it keeps every scene-change frame.
|
||||
- `--max-frames` is now an override on top of each mode's default cap, rather than a fixed default of 80.
|
||||
|
||||
### Fixed
|
||||
- Non-Claude installs (`npx skills add`) were dead on arrival — the installer copied `SKILL.md` without the `scripts/` it shells out to. The self-contained package layout resolves this.
|
||||
|
||||
### Removed
|
||||
- `V2_PLAN.md` and `V2_CONCERNS.md` planning docs.
|
||||
|
||||
## [0.1.3] — 2026-05-09
|
||||
|
||||
### Fixed
|
||||
- Windows: `video.info.json` is read as UTF-8 (#4). Previously `Path.read_text()` defaulted to cp1252 on Windows and crashed on yt-dlp's UTF-8 output, silently dropping Title/Uploader from the report. Same fix applied to `.env` reads/writes in `whisper.py` and `setup.py`.
|
||||
- `download.py` now logs info.json parse failures to stderr instead of swallowing them.
|
||||
|
||||
### Security
|
||||
- Hardened subprocess argv against option injection (#2): inserted `--` before the URL in the yt-dlp argv, and tightened `is_url` to reject `-`-prefixed sources and require a non-empty netloc. Resolved video/audio paths to absolute via `Path.resolve()` before passing to `ffmpeg`/`ffprobe`, so a relative path starting with `-` can't be misinterpreted as a flag.
|
||||
|
||||
## [0.1.2] — 2026-04-24
|
||||
|
||||
### Fixed
|
||||
- Windows console crash: removed the emoji from the long-video warning in `watch.py`; cp1252 consoles couldn't encode it.
|
||||
- `setup.py` now prints `winget` / `pip` install commands on Windows instead of "unsupported platform" — matches what the README already promised.
|
||||
|
||||
### Changed
|
||||
- `SKILL.md` notes that on Windows the scripts must be invoked with `python`, not `python3` (the latter is the Microsoft Store stub on Windows).
|
||||
|
||||
## [0.1.1] — 2026-04-24
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -2,18 +2,19 @@
|
||||
|
||||
**Give Claude the ability to watch any video.**
|
||||
|
||||
Claude Code:
|
||||
Claude Code (recommended — auto-updates via marketplace):
|
||||
```
|
||||
/plugin marketplace add bradautomates/claude-video
|
||||
/plugin install watch@claude-video
|
||||
```
|
||||
|
||||
claude.ai (web): [download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) and drop it into Settings → Capabilities → Skills.
|
||||
|
||||
Codex / generic skills:
|
||||
Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:
|
||||
```bash
|
||||
git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch
|
||||
npx skills add bradautomates/claude-video -g
|
||||
```
|
||||
(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.)
|
||||
|
||||
More install options (claude.ai web, manual) in the [Install](#install) section below.
|
||||
|
||||
Zero config to start — `yt-dlp` and `ffmpeg` install on first run via `brew` on macOS (Linux/Windows print exact commands). Captions cover most public videos for free. Whisper API key is only needed when a video has no captions.
|
||||
|
||||
@@ -21,20 +22,12 @@ Zero config to start — `yt-dlp` and `ffmpeg` install on first run via `brew` o
|
||||
|
||||
Claude can read a webpage, run a script, browse a repo. What it can't do, out of the box, is *watch a video*. You paste a YouTube link and it has to either guess from the title or pull a transcript that's missing 90% of what's on screen.
|
||||
|
||||
With Claude Video `/watch` you can paste a URL or a local path, ask a question, and Claude downloads the video, extracts frames at an auto-scaled rate, pulls a timestamped transcript (free captions when available, Whisper API as fallback), and `Read`s every frame as an image. By the time it answers, it has *seen* the video and *heard* the audio.
|
||||
With Claude Video `/watch` you can paste a URL or a local path, ask a question, and Claude fetches captions first, downloads only what it needs, extracts frames (scene-aware, or fast keyframes at `efficient` detail), pulls a timestamped transcript (free captions when available, Whisper API as fallback), and `Read`s every frame as an image. By the time it answers, it has *seen* the video and *heard* the audio.
|
||||
|
||||
```
|
||||
/watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark?
|
||||
```
|
||||
|
||||
## Why this exists
|
||||
|
||||
I built this because I'm constantly using video to keep up with content. If I see a YouTube video that's blowing up, I want to know how the creator structured the hook — what's on screen in the first 3 seconds, what they said, why it worked. That used to mean watching it myself with a notepad. Now I just paste the URL and ask.
|
||||
|
||||
The other half is summarization. Most YouTube videos don't deserve 20 minutes of my attention. I hand the URL to Claude, it pulls the transcript, and tells me what actually happened. If the visual matters, frames come along too. If it's a podcast or a talking head, transcript is enough.
|
||||
|
||||
Claude is great at reading and synthesizing — but until now, video was the one input I couldn't hand it. Pasting a YouTube link got you nothing useful. `/watch` closes that gap.
|
||||
|
||||
## What people actually use it for
|
||||
|
||||
**Analyze someone else's content.** `/watch https://youtu.be/<viral-video> what hook did they open with?` Claude looks at the first frames, reads the opening transcript, breaks down the structure. Same for ad creative, competitor launches, podcast intros, anything where the *how* matters as much as the *what*.
|
||||
@@ -43,12 +36,16 @@ Claude is great at reading and synthesizing — but until now, video was the one
|
||||
|
||||
**Summarize a video.** `/watch https://youtu.be/<long-thing> summarize this` does the obvious thing — pulls the structure, the key moments, what was actually said and shown. Faster than watching at 2x.
|
||||
|
||||
**Cut the hype out of an update video.** `/watch https://youtu.be/<launch-video> what's actually new — skip the hype` Strip a "game-changer" feature drop down to the few things that matter, so you get the substance without ten minutes of intro and overselling.
|
||||
|
||||
**Turn a playlist into notes.** `/watch https://youtu.be/<video> summarize this to a note` Run it across a series and file a per-video summary, so a channel or course becomes a searchable set of notes instead of hours you have to sit through.
|
||||
|
||||
## How it works
|
||||
|
||||
1. **You paste a video and a question.** URL (anything yt-dlp supports — YouTube, Loom, TikTok, X, Instagram, plus a few hundred more) or a local path (`.mp4`, `.mov`, `.mkv`, `.webm`).
|
||||
2. **`yt-dlp` downloads it.** For URLs, into a temp working directory. For local files, no download — just probed in place.
|
||||
3. **`ffmpeg` extracts frames at an auto-scaled rate.** The frame budget is duration-aware: ≤30s gets ~30 frames, 30-60s gets ~40, 1-3min gets ~60, 3-10min gets ~80, longer gets 100 sparsely. Hard ceilings: 2 fps, 100 frames. JPEGs at 512px wide by default — bump with `--resolution 1024` if Claude needs to read on-screen text.
|
||||
4. **The transcript comes from one of two places.** First try: `yt-dlp` pulls native captions (manual or auto-generated) from the source. Free, instant, accurate-ish. Fallback: extract a mono 16 kHz audio clip and ship it to Whisper — Groq's `whisper-large-v3` (preferred — cheaper and faster) or OpenAI's `whisper-1`.
|
||||
2. **`yt-dlp` checks captions first.** At `transcript` detail, captioned URLs return without downloading video. Otherwise, or when Whisper needs audio, it downloads only what the run needs.
|
||||
3. **`ffmpeg` extracts frames at the chosen detail.** `efficient` decodes keyframes only (near-instant); `balanced`/`token-burner` prefer scene-change frames and fall back to the duration-aware uniform sampler when they under-produce. JPEGs are 512px wide by default and clamped to 1998px tall for Claude Read compatibility.
|
||||
4. **The transcript comes from one of two places.** First try: `yt-dlp` pulls native captions (manual or auto-generated) from the source. Free, instant, accurate-ish. Fallback: extract a mono 16 kHz 64 kbps mp3 audio clip (~480 kB/min) and ship it to Whisper — Groq's `whisper-large-v3` (preferred — cheaper and faster) or OpenAI's `whisper-1`.
|
||||
5. **Frames + transcript are handed to Claude.** The script prints frame paths with `t=MM:SS` markers and the transcript with timestamps. Claude `Read`s each frame in parallel — JPEGs render directly as images in its context.
|
||||
6. **Claude answers grounded in what's actually on screen and in the audio.** Not "based on the description" or "according to the title." It saw the frames. It heard the transcript. It answers the way someone who watched the video would.
|
||||
7. **Cleanup.** The script prints a working directory at the end. If you're not asking follow-ups, Claude removes it.
|
||||
@@ -63,18 +60,49 @@ Token cost is dominated by frames. Every frame is an image; image tokens add up
|
||||
| 30 s - 1 min | ~40 frames | Still dense |
|
||||
| 1 - 3 min | ~60 frames | Comfortable |
|
||||
| 3 - 10 min | ~80 frames | Sparse but workable |
|
||||
| > 10 min | 100 frames | "Sparse scan" warning — re-run focused |
|
||||
| > 10 min | 100 frames (capped modes) | "Sparse scan" warning — re-run focused, or `--detail token-burner` for full uncapped coverage |
|
||||
|
||||
When the user names a moment ("around 2:30", "the last 30 seconds", "from 0:45 to 1:00"), pass `--start` / `--end`. Focused mode gets denser per-second budgets, capped at 2 fps. Far more useful than a sparse pass over the whole thing.
|
||||
|
||||
## Frame deduplication
|
||||
|
||||
Frame selection — keyframes (`efficient`), scene-change detection (`balanced`/`token-burner`), or the uniform sampler it falls back to — can still surface near-identical frames: a screen recording that holds one slide for 90 seconds produces a dozen, each billed as a separate image. A dedup pass drops them before frames reach Claude. It runs by default on every frame mode (`--no-dedup` turns it off):
|
||||
|
||||
1. One `ffmpeg` call scales each extracted JPEG to a 16×16 grayscale thumbnail. Everything after is pure-stdlib Python — no image libraries.
|
||||
2. For each frame, compute the **mean absolute difference** against the *last frame that was kept* (average per-pixel brightness change, 0–255 scale).
|
||||
3. If that difference is at or below the threshold (`2.0`), the frame is a near-duplicate and is dropped. Otherwise it's kept and becomes the new reference.
|
||||
4. The frame-budget cap applies *after* dedup, so the budget is spent on distinct frames.
|
||||
|
||||
Comparing against the last *kept* frame (not the previous one) catches slow fades that never trip a frame-to-frame threshold. The threshold is deliberately low and measures absolute brightness rather than structure, so a one-line code diff, a terminal scrolling a row, or two differently-colored flat slides all survive.
|
||||
|
||||
The **Frames** line reports what was collapsed, e.g. `6 selected from 14 candidates (… 8 near-duplicates dropped …)`. On always-moving footage nothing is dropped and you pay what you would have anyway.
|
||||
|
||||
## Detail modes — measured
|
||||
|
||||
The `--detail` dial trades speed and token cost for visual fidelity. Numbers below are from a real run against a **49:08** YouTube video (1280×720, English auto-captions) — a long, mostly-static screen recording, the case that stresses the caps hardest. Extraction times are local CPU against a pre-downloaded copy; the one-time download was **~37 s** / 76 MB, shared by the three frame modes.
|
||||
|
||||
| Mode | Engine | Frames | Cap | Extraction time | Temporal coverage | Est. image tokens |
|
||||
|------|--------|--------|-----|-----------------|-------------------|-------------------|
|
||||
| `transcript` | none (captions) | 0 | — | **~4.5 s** (one yt-dlp call, no download) | full (text) | 0 (≈26.6k text tokens) |
|
||||
| `efficient` | keyframe (`-skip_frame nokey`) | 50 | 50 | **~0.5 s** | 0:00 → 49:04 (full) | **~9.8k** |
|
||||
| `balanced` | scene-change | 100 | 100 | **~20.9 s** | 0:00 → 48:38 (full) | **~19.7k** |
|
||||
| `token-burner` | scene-change | 116 | uncapped | **~21.0 s** | 0:00 → 48:38 (full) | **~22.8k** |
|
||||
|
||||
- **Image tokens** use Anthropic's `(width × height) / 750` — at the default 512px width these 720p frames are 512×288, **≈197 tokens/frame**; `--resolution 1024` roughly 4×s that. The transcript is surfaced in every captioned mode and on long videos is often the larger cost.
|
||||
- **One sampling rule across frame modes.** Each detects all candidates across the full range, then even-samples (first + last always kept) down to its cap. The modes differ only in candidate *source* (keyframes vs. scene cuts) and cap, never in how coverage is spread — so the last frame always lands at the end, not partway through.
|
||||
- **`efficient` is the speed tier** (~0.5 s) — it only reconstructs keyframes, so it's ~40× faster than the scene modes, which decode every frame to find cuts. It can also return *more* frames than `balanced` on low-motion footage (keyframes outnumber scene cuts); "efficient" means fast extraction, not fewer frames.
|
||||
- **`token-burner` only diverges from `balanced` past the cap.** This clip had 116 cuts, so `balanced` sampled 100 and `token-burner` kept all 116. On high-motion video with hundreds of cuts, `token-burner` keeps everything (and trips the >250-frame token warning) while `balanced` thins to 100.
|
||||
|
||||
End-to-end from a cold URL, `transcript` is the cheapest mode by far; the frame modes add the shared ~37 s download on top of the extraction times above.
|
||||
|
||||
## Install
|
||||
|
||||
| Surface | Install |
|
||||
|---------|---------|
|
||||
| **Claude Code** | `/plugin marketplace add bradautomates/claude-video` then `/plugin install watch@claude-video` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, +50 more** | `npx skills add bradautomates/claude-video -g` |
|
||||
| **claude.ai** (web) | [Download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) → Settings → Capabilities → Skills → `+` |
|
||||
| **Codex** | `git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch` |
|
||||
| **Manual / dev** | `git clone https://github.com/bradautomates/claude-video.git ~/.claude/skills/watch` |
|
||||
| **Manual / dev** | `git clone` then symlink `skills/watch` into your host's skills dir (see below) |
|
||||
|
||||
### Claude Code
|
||||
|
||||
@@ -85,6 +113,24 @@ When the user names a moment ("around 2:30", "the last 30 seconds", "from 0:45 t
|
||||
|
||||
Update later with `/plugin update watch@claude-video`.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI, and 50+ other hosts
|
||||
|
||||
The [Agent Skills](https://agentskills.io) CLI installs the skill into whatever agents it detects:
|
||||
|
||||
```bash
|
||||
npx skills add bradautomates/claude-video -g
|
||||
```
|
||||
|
||||
`-g` installs globally for your user (`~/.codex/skills`, `~/.cursor/skills`, etc.); drop it to install into the current project instead. Useful flags:
|
||||
|
||||
- `-a, --agent <names…>` — target specific hosts, e.g. `-a codex -a cursor`
|
||||
- `-l, --list` — list the skills in this repo without installing
|
||||
- `--copy` — copy files instead of symlinking (for filesystems without symlink support)
|
||||
|
||||
The CLI discovers the skill from `skills/watch/SKILL.md` and copies the whole folder — `SKILL.md` plus its `scripts/` runtime — as a self-contained unit. `SKILL.md` resolves its own scripts relative to wherever it was installed, so it works the same on every host.
|
||||
|
||||
Update later with `npx skills update watch -g`.
|
||||
|
||||
### claude.ai (web)
|
||||
|
||||
1. [Download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) from the latest release.
|
||||
@@ -93,18 +139,17 @@ Update later with `/plugin update watch@claude-video`.
|
||||
|
||||
Enable "Code execution and file creation" under Capabilities first — the skill shells out to `ffmpeg` and `yt-dlp`, so it won't run without it.
|
||||
|
||||
### Codex
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch
|
||||
```
|
||||
|
||||
### Manual (developer)
|
||||
|
||||
Clone the repo and symlink the self-contained skill folder into your host's skills directory — the symlink keeps the install in sync with your working tree as you edit:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bradautomates/claude-video.git ~/.claude/skills/watch
|
||||
git clone https://github.com/bradautomates/claude-video.git
|
||||
ln -s "$(pwd)/claude-video/skills/watch" ~/.claude/skills/watch # or ~/.codex/skills/watch
|
||||
```
|
||||
|
||||
For claude.ai, build the `.skill` bundle from source: `bash skills/watch/scripts/build-skill.sh` produces `dist/watch.skill`.
|
||||
|
||||
## First run
|
||||
|
||||
On the first `/watch` call, the skill runs `scripts/setup.py --check`. If `ffmpeg` / `yt-dlp` aren't on your PATH, or no Whisper API key is set, it walks you through fixing it:
|
||||
@@ -145,47 +190,56 @@ Focused on a specific section — denser frame budget, lower token cost:
|
||||
|
||||
Other knobs (passed to `scripts/watch.py`):
|
||||
|
||||
- `--detail transcript|efficient|balanced|token-burner` — fidelity/speed dial. `transcript` skips frames (transcript only); `efficient` uses fast keyframes (cap 50); `balanced` uses scene-aware frames (cap 100); `token-burner` is scene-aware and uncapped.
|
||||
- `--timestamps T1,T2,…` — grab a frame at each absolute timestamp (`SS`/`MM:SS`/`HH:MM:SS`). Claude reads the transcript first, then targets the moments the presenter flags ("look here", "as you can see"). Added on top of the detail frames (reserved against the cap); out-of-window cues are dropped in focus mode; with `--detail transcript` these become the only frames.
|
||||
- `--max-frames N` — lower the frame cap for a tighter token budget.
|
||||
- `--resolution W` — bump frame width to 1024 px when Claude needs to read on-screen text (slides, terminals, code).
|
||||
- `--fps F` — override the auto-fps calculation (still capped at 2 fps).
|
||||
- `--whisper groq|openai` — force a specific Whisper backend.
|
||||
- `--no-whisper` — disable transcription entirely; frames only.
|
||||
- `--no-dedup` — keep near-duplicate frames. By default a frame-delta pass drops frames that are visually near-identical to the one before them (held slides, static screen recordings, paused video), so the frame budget is spent on distinct content; this flag turns that off.
|
||||
- `--out-dir DIR` — keep working files somewhere specific (default: auto-generated tmp dir).
|
||||
|
||||
## Limits
|
||||
|
||||
- **Best accuracy: under 10 minutes.** Past that the script prints a "sparse scan" warning — re-run focused on the part you actually care about with `--start`/`--end`.
|
||||
- **Hard caps: 2 fps, 100 frames.** Frame count drives token cost; the script enforces this even when the auto-fps math would imply higher.
|
||||
- **Whisper upload limit: 25 MB.** At mono 16 kHz that's about 50 minutes of audio. Longer videos need either captions or `--start`/`--end` to a smaller window.
|
||||
- **No private platforms.** This skill doesn't log into anything. Public URLs and local files only. If yt-dlp can't reach it without auth, neither can `/watch`.
|
||||
- **Long-video accuracy depends on the detail mode.** On the capped modes (`efficient`, default `balanced`) coverage thins out past ~10 minutes — the frame cap spreads across the whole clip, so the script prints a "sparse scan" warning and you're better off re-running focused with `--start`/`--end`. `token-burner` lifts the cap and keeps *every* scene-change frame across the full video, so it stays complete on longer clips at the cost of more image tokens. The 10-minute mark is guidance for the capped modes, not a hard ceiling.
|
||||
- **Detail is one dial.** Defaults are balanced: scene-aware frames, 2 fps max, 100-frame cap. Use `--detail efficient` for a fast 50-frame keyframe pass, or `--detail token-burner` for uncapped scene candidates. Set `WATCH_DETAIL` in `~/.config/watch/.env` to change the default.
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
.
|
||||
├── SKILL.md # skill contract — loaded by all three surfaces
|
||||
├── scripts/
|
||||
│ ├── watch.py # entry point — orchestrates download → frames → transcript
|
||||
│ ├── download.py # yt-dlp wrapper
|
||||
│ ├── frames.py # ffmpeg frame extraction + auto-fps logic
|
||||
│ ├── transcribe.py # VTT parsing + dedupe + Whisper orchestration
|
||||
│ ├── whisper.py # Groq / OpenAI clients (pure stdlib)
|
||||
│ ├── setup.py # preflight + installer
|
||||
│ └── build-skill.sh # build dist/watch.skill for claude.ai upload
|
||||
├── hooks/ # SessionStart status hook (Claude Code only)
|
||||
├── .claude-plugin/ # plugin.json + marketplace.json (Claude Code)
|
||||
├── .codex-plugin/ # codex packaging
|
||||
└── .github/workflows/ # release.yml — auto-builds watch.skill on tag push
|
||||
├── skills/watch/ # self-contained skill — copied as a unit by every installer
|
||||
│ ├── SKILL.md # skill contract — the source of truth across all surfaces
|
||||
│ └── scripts/
|
||||
│ ├── watch.py # entry point — orchestrates download → frames → transcript
|
||||
│ ├── download.py # yt-dlp wrapper
|
||||
│ ├── frames.py # ffmpeg frame extraction + auto-fps logic
|
||||
│ ├── transcribe.py # VTT parsing + dedupe + Whisper orchestration
|
||||
│ ├── whisper.py # Groq / OpenAI clients (pure stdlib)
|
||||
│ ├── config.py # shared config (~/.config/watch/.env)
|
||||
│ ├── setup.py # preflight + installer
|
||||
│ └── build-skill.sh # build dist/watch.skill for claude.ai upload (dev-only)
|
||||
├── hooks/ # SessionStart status hook (Claude Code only)
|
||||
├── .claude-plugin/ # plugin.json + marketplace.json (Claude Code)
|
||||
├── .codex-plugin/ # plugin.json — Codex/agents manifest ("skills": "./skills/")
|
||||
├── .agents/plugins/ # marketplace.json — Agent Skills marketplace listing
|
||||
├── AGENTS.md → CLAUDE.md # generic-agent entry point
|
||||
├── tests/ # pytest suite (ffmpeg-synthesized clips, no network)
|
||||
└── .github/workflows/ # release.yml — auto-builds watch.skill on tag push
|
||||
```
|
||||
|
||||
## Develop
|
||||
|
||||
```bash
|
||||
# Run the test suite (stdlib + pytest; ffmpeg required for frame tests):
|
||||
python3 -m pytest -q
|
||||
|
||||
# Build the claude.ai upload bundle:
|
||||
bash scripts/build-skill.sh # → dist/watch.skill
|
||||
bash skills/watch/scripts/build-skill.sh # → dist/watch.skill
|
||||
```
|
||||
|
||||
Releasing: tag `vX.Y.Z`, push the tag. The workflow builds `dist/watch.skill` and attaches it to the GitHub release.
|
||||
Releasing: tag `vX.Y.Z`, push the tag. The workflow builds `dist/watch.skill` and attaches it to the GitHub release. Keep the version in sync across `skills/watch/SKILL.md`, `.claude-plugin/plugin.json`, and `.codex-plugin/plugin.json`.
|
||||
|
||||
See [CHANGELOG.md](CHANGELOG.md) for version history.
|
||||
|
||||
@@ -195,6 +249,18 @@ MIT license.
|
||||
|
||||
Built on `yt-dlp`, `ffmpeg`, and Claude's multimodal `Read` tool. Whisper transcription via [Groq](https://groq.com) or [OpenAI](https://openai.com).
|
||||
|
||||
Built by Brad Bonanno — I make content about building with AI on [YouTube (@bradbonanno)](https://www.youtube.com/@bradbonanno), and build AI operating systems for businesses at [Solaris Automation](https://www.solarisautomation.io/). If `/watch` saves you from scrubbing through a video, come say hi on the channel.
|
||||
|
||||
## Star History
|
||||
|
||||
<a href="https://www.star-history.com/?repos=bradautomates%2Fclaude-video&type=date&legend=top-left">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=bradautomates/claude-video&type=date&theme=dark&legend=top-left" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=bradautomates/claude-video&type=date&legend=top-left" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/chart?repos=bradautomates/claude-video&type=date&legend=top-left" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
---
|
||||
|
||||
[github.com/bradautomates/claude-video](https://github.com/bradautomates/claude-video) · [LICENSE](LICENSE)
|
||||
[github.com/bradautomates/claude-video](https://github.com/bradautomates/claude-video) · [@bradbonanno](https://www.youtube.com/@bradbonanno) · [Solaris Automation](https://www.solarisautomation.io/) · [LICENSE](LICENSE)
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
---
|
||||
name: watch
|
||||
description: Watch a video (URL or local path). Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls the transcript from captions (or Whisper API fallback), and hands the result to Claude so it can answer questions about what's in the video.
|
||||
argument-hint: "<video-url-or-path> [question]"
|
||||
allowed-tools: Bash, Read, AskUserQuestion
|
||||
homepage: https://github.com/bradautomates/claude-video
|
||||
repository: https://github.com/bradautomates/claude-video
|
||||
author: bradautomates
|
||||
license: MIT
|
||||
user-invocable: true
|
||||
---
|
||||
|
||||
# /watch — Claude watches a video
|
||||
|
||||
You don't have a video input; this skill gives you one. A Python script downloads the video, extracts frames as JPEGs, gets a timestamped transcript (native captions first, then Whisper API as fallback), and prints frame paths. You then `Read` each frame path to see the images and combine them with the transcript to answer the user.
|
||||
|
||||
## Step 0 — Setup preflight (runs every `/watch` invocation, silent on success)
|
||||
|
||||
Before every `/watch` run, verify that dependencies and an API key are in place:
|
||||
|
||||
```bash
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" --check
|
||||
```
|
||||
|
||||
This is a <100ms lookup. On exit 0, the script emits **nothing** — proceed to Step 1 without comment. **Do NOT announce "setup is complete" to the user** — they don't need a status message on every turn. The only acceptable user-visible output from Step 0 is when remediation is required.
|
||||
|
||||
On non-zero exit, follow the table:
|
||||
|
||||
| Exit | Meaning | Action |
|
||||
|------|---------|--------|
|
||||
| `2` | Missing binaries (`ffmpeg` / `ffprobe` / `yt-dlp`) | Run installer |
|
||||
| `3` | No Whisper API key | Run installer to scaffold `.env`, then ask user for a key |
|
||||
| `4` | Both missing | Run installer, then ask for a key |
|
||||
|
||||
The installer is idempotent — safe to re-run:
|
||||
|
||||
```bash
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py"
|
||||
```
|
||||
|
||||
On macOS with Homebrew, it auto-installs `ffmpeg` and `yt-dlp`. On Linux/Windows, it prints the exact install commands for the user to run. It scaffolds `~/.config/watch/.env` with commented placeholders at `0600` perms, and writes `SETUP_COMPLETE=true` once deps + a key are in place so the next session knows this user has already been through the wizard.
|
||||
|
||||
**If an API key is still missing after install:** use `AskUserQuestion` to ask the user whether they have a Groq API key (preferred — cheaper, faster) or an OpenAI key. Then write it into `~/.config/watch/.env` — set the matching `GROQ_API_KEY=...` or `OPENAI_API_KEY=...` line. If they don't want to set up Whisper, proceed with `--no-whisper` and tell them videos without native captions will come back frames-only.
|
||||
|
||||
**Structured mode (optional):** `python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" --json` emits `{status, first_run, missing_binaries, whisper_backend, has_api_key, config_file, platform}` where `status` is one of `ready | needs_install | needs_key | needs_install_and_key`. Use this when you need to branch on specifics (e.g. "is this the user's very first run?" → `first_run: true`).
|
||||
|
||||
Within a single session, you can skip Step 0 on follow-up `/watch` calls — once `--check` returned 0, nothing about the environment changes between turns.
|
||||
|
||||
## When to use
|
||||
|
||||
- User pastes a video URL (YouTube, Vimeo, X, TikTok, Twitch clip, most yt-dlp-supported sites) and asks about it.
|
||||
- User points at a local video file (`.mp4`, `.mov`, `.mkv`, `.webm`, etc.) and asks about it.
|
||||
- User types `/watch <url-or-path> [question]`.
|
||||
|
||||
## Recommended limits
|
||||
|
||||
- **Best accuracy: videos under 10 minutes.** Frame coverage scales inversely with duration.
|
||||
- **Hard caps: 100 frames total and 2 fps.** Token cost grows with frame count, so the script targets a frame budget by duration (and never exceeds 2 fps even when the budget would imply more):
|
||||
- ≤30s → ~1-2 fps (up to 30 frames)
|
||||
- 30s-1min → ~40 frames
|
||||
- 1-3min → ~60 frames
|
||||
- 3-10min → ~80 frames
|
||||
- \>10min → 100 frames, sparsely spaced (warning printed)
|
||||
- If the user hands you a long video, consider asking whether they want a specific section before burning tokens on a sparse scan.
|
||||
|
||||
## How to invoke
|
||||
|
||||
**Step 1 — parse the user input.** Separate the video source (URL or path) from any question the user asked. Example: `/watch https://youtu.be/abc what language is this in?` → source = `https://youtu.be/abc`, question = `what language is this in?`.
|
||||
|
||||
**Step 2 — run the watch script.** Pass the source verbatim. Do not shell-escape it yourself beyond normal quoting:
|
||||
|
||||
```bash
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "<source>"
|
||||
```
|
||||
|
||||
Optional flags:
|
||||
- `--start T` / `--end T` — focus on a section. Accepts `SS`, `MM:SS`, or `HH:MM:SS`. When either is set, fps auto-scales denser (see "Focusing on a section" below).
|
||||
- `--max-frames N` — lower the cap for tighter token budget (e.g. `--max-frames 40`)
|
||||
- `--resolution W` — change frame width in px (default 512; bump to 1024 only if the user needs to read on-screen text)
|
||||
- `--fps F` — override auto-fps (clamped to 2 fps max)
|
||||
- `--out-dir DIR` — keep working files somewhere specific (default: an auto-generated tmp dir)
|
||||
- `--whisper groq|openai` — force a specific Whisper backend (default: prefer Groq if both keys exist)
|
||||
- `--no-whisper` — disable the Whisper fallback entirely (frames-only if no captions)
|
||||
|
||||
### Focusing on a section (higher frame rate)
|
||||
|
||||
When the user asks about a specific moment — "what happens at the 2 minute mark?", "zoom into 0:45 to 1:00", "the first 10 seconds" — pass `--start` and/or `--end`. The script switches to focused-mode budgets, which are denser than full-video budgets (still capped at 2 fps):
|
||||
|
||||
- ≤5s → 2 fps (up to 10 frames)
|
||||
- 5-15s → 2 fps (up to 30 frames)
|
||||
- 15-30s → ~2 fps (up to 60 frames)
|
||||
- 30-60s → ~1.3 fps (up to 80 frames)
|
||||
- 60-180s → ~0.6 fps (100 frames, capped)
|
||||
|
||||
Focused mode is the right call for:
|
||||
- Any moment/range the user names explicitly ("around 2:30", "the intro", "the last 30 seconds").
|
||||
- Any video longer than ~10 minutes where the user's question is about a specific part — running focused on the relevant section is far more useful than a sparse scan of the whole thing.
|
||||
- Re-runs after a full scan didn't have enough detail in some region.
|
||||
|
||||
Transcript is auto-filtered to the same range. Frame timestamps are absolute (real video timeline, not offset-from-start).
|
||||
|
||||
Examples:
|
||||
```bash
|
||||
# Last 10 seconds of a 1 minute video
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" video.mp4 --start 50 --end 60
|
||||
|
||||
# Zoom into 2:15 → 2:45 at 3 fps (90 frames)
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "$URL" --start 2:15 --end 2:45 --fps 3
|
||||
|
||||
# From 1h12m to the end of the video
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "$URL" --start 1:12:00
|
||||
```
|
||||
|
||||
**Step 3 — Read every frame path the script lists.** The Read tool renders JPEGs directly as images for you. Read all frames in a single message (parallel tool calls) so you see them together. The frames are in chronological order with a `t=MM:SS` timestamp so you can align them to the transcript.
|
||||
|
||||
**Step 4 — answer the user.** You now have two streams of evidence:
|
||||
- **Frames** — what's on screen at each timestamp
|
||||
- **Transcript** — what's said at each timestamp. The report's header shows the source (`captions` = yt-dlp pulled native subs; `whisper (groq)` or `whisper (openai)` = transcribed by API).
|
||||
|
||||
If the user asked a specific question, answer it directly citing timestamps. If they didn't ask anything, summarize what happens in the video — structure, key moments, notable visuals, spoken content.
|
||||
|
||||
**Step 5 — clean up.** The script prints a working directory at the end. If the user isn't going to ask follow-ups about this video, delete it with `rm -rf <dir>`. If they might, leave it in place.
|
||||
|
||||
## Transcription
|
||||
|
||||
The script gets a timestamped transcript in one of two ways:
|
||||
|
||||
1. **Native captions (free, preferred).** yt-dlp pulls manual or auto-generated subtitles from the source platform if available.
|
||||
2. **Whisper API fallback.** If no captions came back (or the source is a local file), the script extracts audio (`ffmpeg -vn -ac 1 -ar 16000 -b:a 64k`, ~0.5 MB/min) and uploads it to whichever Whisper API has a key configured:
|
||||
- **Groq** — `whisper-large-v3`. Preferred default: cheaper, faster. Get a key at console.groq.com/keys.
|
||||
- **OpenAI** — `whisper-1`. Fallback. Get a key at platform.openai.com/api-keys.
|
||||
|
||||
Both keys live in `~/.config/watch/.env`. The script prefers Groq when both are set; override with `--whisper openai` to force OpenAI. Use `--no-whisper` to skip the fallback entirely.
|
||||
|
||||
## Failure modes and handling
|
||||
|
||||
- **Setup preflight failed** → run `python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py"` (auto-installs ffmpeg/yt-dlp via brew on macOS, scaffolds the `.env`). For API key, ask the user via `AskUserQuestion` and write it to `~/.config/watch/.env`.
|
||||
- **No transcript available** → captions missing AND (no Whisper key OR Whisper API failed). Script prints a hint pointing to setup. Proceed frames-only and tell the user.
|
||||
- **Long video warning printed** → acknowledge it in your answer. Offer to re-run focused on a specific section via `--start`/`--end` rather than a sparse full-video scan.
|
||||
- **Download fails** → yt-dlp's error goes to stderr. If it's a login-required or region-locked video, tell the user plainly; do not keep retrying.
|
||||
- **Whisper request fails** → the error is printed to stderr (likely: invalid key, rate limit, or 25 MB upload limit on a very long video). The report will say "none available" for transcript. You can retry with `--whisper openai` if Groq failed (or vice versa).
|
||||
|
||||
## Token efficiency
|
||||
|
||||
This skill burns tokens primarily on frames. Order of magnitude:
|
||||
- 80 frames at 512px wide is roughly 50-80k image tokens depending on aspect ratio.
|
||||
- The transcript is cheap (a few thousand tokens at most for a 10-minute video).
|
||||
- Bumping `--resolution` to 1024 roughly quadruples the image tokens per frame. Only do it when necessary.
|
||||
|
||||
If you already watched a video this session and the user asks a follow-up, do **not** re-run the script — you already have the frames and transcript in context. Just answer from what you have.
|
||||
|
||||
## Security & Permissions
|
||||
|
||||
**What this skill does:**
|
||||
- Runs `yt-dlp` locally to download the video and pull native captions when the source supports them (public data; the request goes directly to whatever host the URL points at)
|
||||
- Runs `ffmpeg` / `ffprobe` locally to extract frames as JPEGs and, when Whisper is needed, a mono 16 kHz audio clip
|
||||
- Sends the extracted audio clip to Groq's Whisper API (`api.groq.com/openai/v1/audio/transcriptions`) when `GROQ_API_KEY` is set (preferred — cheaper, faster)
|
||||
- Sends the extracted audio clip to OpenAI's audio transcription API (`api.openai.com/v1/audio/transcriptions`) when `OPENAI_API_KEY` is set and Groq is not, or when `--whisper openai` is forced
|
||||
- Writes the downloaded video, frames, audio, and an intermediate transcript to a working directory under the system temp dir (or `--out-dir` if specified) so Claude can `Read` them
|
||||
- Reads / creates `~/.config/watch/.env` (mode `0600`) to store the Whisper API key(s) and a `SETUP_COMPLETE` marker. As a fallback, also reads `.env` in the current working directory
|
||||
|
||||
**What this skill does NOT do:**
|
||||
- Does not upload the video itself to any API — only the extracted audio goes out, and only when native captions are missing AND Whisper is not disabled with `--no-whisper`
|
||||
- Does not access any platform account (no login, no session cookies, no posting)
|
||||
- Does not share API keys between providers (Groq key only goes to `api.groq.com`, OpenAI key only goes to `api.openai.com`)
|
||||
- Does not log, cache, or write API keys to stdout, stderr, or output files
|
||||
- Does not persist anything outside the working directory and `~/.config/watch/.env` — clean up the working directory when you're done (Step 5)
|
||||
|
||||
**Bundled scripts:** `scripts/watch.py` (entry point), `scripts/download.py` (yt-dlp wrapper), `scripts/frames.py` (ffmpeg frame extraction), `scripts/transcribe.py` (caption selection + Whisper orchestration), `scripts/whisper.py` (Groq / OpenAI clients), `scripts/setup.py` (preflight + installer)
|
||||
|
||||
Review scripts before first use to verify behavior.
|
||||
@@ -1,9 +0,0 @@
|
||||
---
|
||||
description: Watch a video (URL or local path). Downloads with yt-dlp, extracts frames with ffmpeg, transcribes from captions or Whisper, and answers questions about what's in the video.
|
||||
argument-hint: <video-url-or-path> [question]
|
||||
allowed-tools: [Bash, Read, AskUserQuestion]
|
||||
---
|
||||
|
||||
Invoke the `watch` skill (defined in SKILL.md) with the user's arguments: $ARGUMENTS
|
||||
|
||||
Follow the skill's full pipeline: preflight setup check → download via yt-dlp → extract frames at auto-scaled fps → pull captions or Whisper transcript → Read each frame → answer the user grounded in frames and transcript. If the user provided no arguments, ask them for a video URL or local path before proceeding.
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# dev-sync.sh — copy this working tree into the installed /watch plugin cache so
|
||||
# local edits are picked up by Claude Code without publishing a release.
|
||||
#
|
||||
# The install path is resolved from ~/.claude/plugins/installed_plugins.json, so
|
||||
# it follows version bumps automatically. Override it by passing a path as $1 or
|
||||
# setting WATCH_INSTALL_PATH. Pass --dry-run to preview without writing.
|
||||
#
|
||||
# Usage:
|
||||
# ./dev-sync.sh # sync into the resolved install path
|
||||
# ./dev-sync.sh --dry-run # show what would change
|
||||
# ./dev-sync.sh /some/other/dir # sync into an explicit path
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PLUGIN_KEY="watch@claude-video"
|
||||
INSTALLED_JSON="${HOME}/.claude/plugins/installed_plugins.json"
|
||||
|
||||
DRY_RUN=()
|
||||
DEST=""
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--dry-run) DRY_RUN=(--dry-run --itemize-changes) ;;
|
||||
-*) echo "unknown flag: $arg" >&2; exit 2 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Resolve the destination install path if not given explicitly.
|
||||
if [[ -z "$DEST" ]]; then
|
||||
DEST="${WATCH_INSTALL_PATH:-}"
|
||||
fi
|
||||
if [[ -z "$DEST" ]]; then
|
||||
if [[ ! -f "$INSTALLED_JSON" ]]; then
|
||||
echo "error: $INSTALLED_JSON not found; pass an install path explicitly" >&2
|
||||
exit 1
|
||||
fi
|
||||
DEST="$(PLUGIN_KEY="$PLUGIN_KEY" python3 - "$INSTALLED_JSON" <<'PY'
|
||||
import json, os, sys
|
||||
data = json.load(open(sys.argv[1]))
|
||||
key = os.environ["PLUGIN_KEY"]
|
||||
records = data.get("plugins", {}).get(key, [])
|
||||
# Prefer a record whose installPath actually exists on disk.
|
||||
paths = [r.get("installPath") for r in records if r.get("installPath")]
|
||||
for p in paths:
|
||||
if os.path.isdir(p):
|
||||
print(p); break
|
||||
else:
|
||||
print(paths[0] if paths else "")
|
||||
PY
|
||||
)"
|
||||
fi
|
||||
|
||||
if [[ -z "$DEST" ]]; then
|
||||
echo "error: could not resolve an install path for '$PLUGIN_KEY'" >&2
|
||||
echo " install the plugin first, or pass a path: scripts/dev-sync.sh /path/to/install" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -d "$DEST" ]]; then
|
||||
echo "error: install path does not exist: $DEST" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "source: $REPO_ROOT"
|
||||
echo "dest: $DEST"
|
||||
echo
|
||||
|
||||
# Mirror shipped files only. Dev-only artifacts and runtime state are excluded.
|
||||
# No --delete: the cache holds runtime state (.in_use/) we must not touch.
|
||||
rsync -a ${DRY_RUN[@]+"${DRY_RUN[@]}"} \
|
||||
--exclude '.git/' \
|
||||
--exclude '.venv/' \
|
||||
--exclude '.pytest_cache/' \
|
||||
--exclude '__pycache__/' \
|
||||
--exclude '*.pyc' \
|
||||
--exclude '.DS_Store' \
|
||||
--exclude '.in_use/' \
|
||||
--exclude 'tests/' \
|
||||
--exclude 'docs/' \
|
||||
--exclude 'dist/' \
|
||||
--exclude 'V2_*.md' \
|
||||
--exclude 'dev-sync.sh' \
|
||||
"$REPO_ROOT/" "$DEST/"
|
||||
|
||||
echo "done."
|
||||
@@ -50,7 +50,7 @@ fi
|
||||
|
||||
# First-run / partially-configured → one-line hint.
|
||||
if [[ -z "$HAS_FFMPEG" || -z "$HAS_YTDLP" ]]; then
|
||||
echo "/watch: needs ffmpeg + yt-dlp. Run \`python3 \$CLAUDE_PLUGIN_ROOT/scripts/setup.py\` once to install and scaffold config."
|
||||
echo "/watch: needs ffmpeg + yt-dlp. Run \`python3 \$CLAUDE_PLUGIN_ROOT/skills/watch/scripts/setup.py\` once to install and scaffold config."
|
||||
elif [[ -z "$HAS_GROQ" && -z "$HAS_OPENAI" ]]; then
|
||||
echo "/watch: ready for videos with native captions. Add GROQ_API_KEY (preferred) or OPENAI_API_KEY to ~/.config/watch/.env to unlock Whisper fallback."
|
||||
else
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-skill.sh — package this repo as a claude.ai-upload-ready .skill file.
|
||||
# Usage: bash scripts/build-skill.sh (run from repo root)
|
||||
#
|
||||
# Produces dist/watch.skill, a zip with a single top-level `watch/` directory
|
||||
# containing SKILL.md and the scripts/ runtime. claude.ai's skill upload has a
|
||||
# 200-file cap; `export-ignore` in .gitattributes + the zip -d strips below
|
||||
# keep the bundle lean.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
echo "error: working tree is dirty; commit or stash before building" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p dist
|
||||
OUT="dist/watch.skill"
|
||||
git archive --format=zip --prefix=watch/ --output="$OUT" HEAD
|
||||
|
||||
# claude.ai's .skill bundle needs only SKILL.md + scripts/ runtime. Claude Code
|
||||
# needs hooks/, commands/, and .claude-plugin/ in the git archive (that's why
|
||||
# they are NOT in .gitattributes export-ignore), but the .skill bundle should
|
||||
# strip them to keep a single canonical SKILL.md and stay well under the
|
||||
# 200-file cap.
|
||||
zip -d "$OUT" \
|
||||
"watch/hooks/*" \
|
||||
"watch/commands/*" \
|
||||
"watch/.claude-plugin/*" \
|
||||
> /dev/null 2>&1 || true
|
||||
|
||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||
SIZE=$(du -h "$OUT" | cut -f1)
|
||||
|
||||
if [ "$COUNT" -gt 200 ]; then
|
||||
echo "error: $COUNT files in zip, claude.ai's cap is 200" >&2
|
||||
echo " check .gitattributes export-ignore entries and this script's zip -d excludes" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SKILL_MD_COUNT=$(unzip -l "$OUT" | grep -c "SKILL.md" || true)
|
||||
if [ "$SKILL_MD_COUNT" -ne 1 ]; then
|
||||
echo "error: expected exactly one SKILL.md, found $SKILL_MD_COUNT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "built $OUT ($COUNT files, $SIZE)"
|
||||
echo "upload via the claude.ai skill UI"
|
||||
@@ -1,250 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Probe video metadata and extract frames at an auto-scaled fps.
|
||||
|
||||
Auto-fps targets a frame budget, not a fixed rate. Token cost scales with frame
|
||||
count, so budget-by-duration keeps short videos dense and long videos capped.
|
||||
When a user-specified range is passed, focused-mode budgets denser (they are
|
||||
zooming in for detail).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MAX_FPS = 2.0
|
||||
|
||||
|
||||
def _clamp_fps(fps: float, duration_seconds: float, max_frames: int) -> tuple[float, int]:
|
||||
fps = min(fps, MAX_FPS)
|
||||
target = min(max_frames, max(1, int(round(fps * duration_seconds))))
|
||||
return fps, target
|
||||
|
||||
|
||||
def parse_time(value: str | float | int | None) -> float | None:
|
||||
"""Parse SS, MM:SS, or HH:MM:SS (with optional .ms) into seconds."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
s = str(value).strip()
|
||||
if not s:
|
||||
return None
|
||||
parts = s.split(":")
|
||||
try:
|
||||
if len(parts) == 1:
|
||||
return float(parts[0])
|
||||
if len(parts) == 2:
|
||||
return int(parts[0]) * 60 + float(parts[1])
|
||||
if len(parts) == 3:
|
||||
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
|
||||
except ValueError:
|
||||
pass
|
||||
raise SystemExit(f"Cannot parse time value: {value!r} (expected SS, MM:SS, or HH:MM:SS)")
|
||||
|
||||
|
||||
def format_time(seconds: float) -> str:
|
||||
total = int(round(seconds))
|
||||
hours, rem = divmod(total, 3600)
|
||||
minutes, sec = divmod(rem, 60)
|
||||
if hours:
|
||||
return f"{hours}:{minutes:02d}:{sec:02d}"
|
||||
return f"{minutes:02d}:{sec:02d}"
|
||||
|
||||
|
||||
def get_metadata(video_path: str) -> dict:
|
||||
if shutil.which("ffprobe") is None:
|
||||
raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v", "quiet",
|
||||
"-print_format", "json",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
video_path,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffprobe failed: {result.stderr.strip()}")
|
||||
|
||||
data = json.loads(result.stdout or "{}")
|
||||
streams = data.get("streams", [])
|
||||
fmt = data.get("format", {})
|
||||
video_stream = next((s for s in streams if s.get("codec_type") == "video"), {})
|
||||
audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None)
|
||||
|
||||
duration = float(fmt.get("duration") or video_stream.get("duration") or 0)
|
||||
return {
|
||||
"duration_seconds": duration,
|
||||
"width": video_stream.get("width"),
|
||||
"height": video_stream.get("height"),
|
||||
"codec": video_stream.get("codec_name"),
|
||||
"size_bytes": int(fmt.get("size") or 0),
|
||||
"has_audio": audio_stream is not None,
|
||||
}
|
||||
|
||||
|
||||
def auto_fps(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
|
||||
"""Pick fps that targets a sensible frame budget for full-video scans."""
|
||||
if duration_seconds <= 0:
|
||||
return 1.0, 1
|
||||
|
||||
if duration_seconds <= 30:
|
||||
target = min(max_frames, max(12, int(round(duration_seconds))))
|
||||
elif duration_seconds <= 60:
|
||||
target = min(max_frames, 40)
|
||||
elif duration_seconds <= 180: # 3 min
|
||||
target = min(max_frames, 60)
|
||||
elif duration_seconds <= 600: # 10 min
|
||||
target = min(max_frames, 80)
|
||||
else:
|
||||
target = max_frames
|
||||
|
||||
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
|
||||
|
||||
|
||||
def auto_fps_focus(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
|
||||
"""Denser budget for user-specified ranges — they are zooming in for detail."""
|
||||
if duration_seconds <= 0:
|
||||
return min(MAX_FPS, 2.0), 2
|
||||
|
||||
if duration_seconds <= 5:
|
||||
target = min(max_frames, max(10, int(round(duration_seconds * 6))))
|
||||
elif duration_seconds <= 15:
|
||||
target = min(max_frames, max(30, int(round(duration_seconds * 4))))
|
||||
elif duration_seconds <= 30:
|
||||
target = min(max_frames, 60)
|
||||
elif duration_seconds <= 60:
|
||||
target = min(max_frames, 80)
|
||||
elif duration_seconds <= 180:
|
||||
target = max_frames
|
||||
else:
|
||||
target = max_frames
|
||||
|
||||
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
|
||||
|
||||
|
||||
def extract(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
fps: float,
|
||||
resolution: int = 512,
|
||||
max_frames: int = 100,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
) -> list[dict]:
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("frame_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
output_pattern = str(out_dir / "frame_%04d.jpg")
|
||||
cmd: list[str] = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-y",
|
||||
]
|
||||
|
||||
# -ss before -i = fast seek (keyframe-snap, good enough for preview frames).
|
||||
if start_seconds is not None:
|
||||
cmd += ["-ss", f"{start_seconds:.3f}"]
|
||||
if end_seconds is not None:
|
||||
cmd += ["-to", f"{end_seconds:.3f}"]
|
||||
|
||||
cmd += [
|
||||
"-i", video_path,
|
||||
"-vf", f"fps={fps},scale={resolution}:-2",
|
||||
"-frames:v", str(max_frames),
|
||||
"-q:v", "4",
|
||||
output_pattern,
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffmpeg frame extraction failed: {result.stderr.strip()}")
|
||||
|
||||
offset = start_seconds or 0.0
|
||||
frames = sorted(out_dir.glob("frame_*.jpg"))
|
||||
return [
|
||||
{
|
||||
"index": i,
|
||||
"timestamp_seconds": round(offset + (i / fps if fps > 0 else 0.0), 2),
|
||||
"path": str(p),
|
||||
}
|
||||
for i, p in enumerate(frames)
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
print(
|
||||
"usage: frames.py <video-path> <out-dir> [--fps F] [--resolution W] "
|
||||
"[--max-frames N] [--start T] [--end T]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(2)
|
||||
|
||||
video = sys.argv[1]
|
||||
out = Path(sys.argv[2])
|
||||
args = sys.argv[3:]
|
||||
|
||||
fps_override = None
|
||||
resolution = 512
|
||||
max_frames = 100
|
||||
start_arg = None
|
||||
end_arg = None
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == "--fps":
|
||||
fps_override = float(args[i + 1]); i += 2
|
||||
elif args[i] == "--resolution":
|
||||
resolution = int(args[i + 1]); i += 2
|
||||
elif args[i] == "--max-frames":
|
||||
max_frames = int(args[i + 1]); i += 2
|
||||
elif args[i] == "--start":
|
||||
start_arg = args[i + 1]; i += 2
|
||||
elif args[i] == "--end":
|
||||
end_arg = args[i + 1]; i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
meta = get_metadata(video)
|
||||
start_sec = parse_time(start_arg)
|
||||
end_sec = parse_time(end_arg)
|
||||
full_duration = meta["duration_seconds"]
|
||||
|
||||
effective_start = start_sec if start_sec is not None else 0.0
|
||||
effective_end = end_sec if end_sec is not None else full_duration
|
||||
effective_duration = max(0.0, effective_end - effective_start)
|
||||
|
||||
focused = start_sec is not None or end_sec is not None
|
||||
if focused:
|
||||
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
|
||||
else:
|
||||
fps, target = auto_fps(effective_duration, max_frames=max_frames)
|
||||
if fps_override is not None:
|
||||
fps = fps_override
|
||||
target = max(1, int(round(fps * effective_duration)))
|
||||
|
||||
frames = extract(
|
||||
video, out,
|
||||
fps=fps,
|
||||
resolution=resolution,
|
||||
max_frames=max_frames,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
print(json.dumps(
|
||||
{"meta": meta, "fps": fps, "target": target, "focused": focused, "frames": frames},
|
||||
indent=2,
|
||||
))
|
||||
@@ -1,230 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""/watch entry point: download video, extract frames, parse transcript.
|
||||
|
||||
Prints a markdown report to stdout listing frame paths + transcript. Claude
|
||||
then Reads each frame path to see the video.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from download import download, is_url # noqa: E402
|
||||
from frames import MAX_FPS, auto_fps, auto_fps_focus, extract, format_time, get_metadata, parse_time # noqa: E402
|
||||
from transcribe import filter_range, format_transcript, parse_vtt # noqa: E402
|
||||
from whisper import load_api_key, transcribe_video # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="watch",
|
||||
description="Download a video, extract auto-scaled frames, and surface the transcript.",
|
||||
)
|
||||
ap.add_argument("source", help="Video URL or local file path")
|
||||
ap.add_argument("--max-frames", type=int, default=80, help="Cap on frame count (default 80, hard max 100)")
|
||||
ap.add_argument("--resolution", type=int, default=512, help="Frame width in pixels (default 512)")
|
||||
ap.add_argument("--fps", type=float, default=None, help="Override auto-fps")
|
||||
ap.add_argument("--start", type=str, default=None, help="Range start (SS, MM:SS, or HH:MM:SS)")
|
||||
ap.add_argument("--end", type=str, default=None, help="Range end (SS, MM:SS, or HH:MM:SS)")
|
||||
ap.add_argument("--out-dir", type=str, default=None, help="Working directory (default: tmp)")
|
||||
ap.add_argument(
|
||||
"--no-whisper",
|
||||
action="store_true",
|
||||
help="Disable Whisper fallback. Report frames-only if no captions available.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--whisper",
|
||||
choices=["groq", "openai"],
|
||||
default=None,
|
||||
help="Force a specific Whisper backend. Default: prefer Groq, fall back to OpenAI.",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
max_frames = min(args.max_frames, 100)
|
||||
|
||||
if args.out_dir:
|
||||
work = Path(args.out_dir).expanduser().resolve()
|
||||
else:
|
||||
work = Path(tempfile.mkdtemp(prefix="watch-"))
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
print(f"[watch] working dir: {work}", file=sys.stderr)
|
||||
|
||||
print(
|
||||
"[watch] downloading via yt-dlp…" if is_url(args.source) else "[watch] using local file…",
|
||||
file=sys.stderr,
|
||||
)
|
||||
dl = download(args.source, work / "download")
|
||||
video_path = dl["video_path"]
|
||||
|
||||
meta = get_metadata(video_path)
|
||||
full_duration = meta["duration_seconds"]
|
||||
|
||||
start_sec = parse_time(args.start)
|
||||
end_sec = parse_time(args.end)
|
||||
|
||||
if start_sec is not None and start_sec < 0:
|
||||
raise SystemExit("--start must be non-negative")
|
||||
if end_sec is not None and start_sec is not None and end_sec <= start_sec:
|
||||
raise SystemExit("--end must be greater than --start")
|
||||
if full_duration > 0 and start_sec is not None and start_sec >= full_duration:
|
||||
raise SystemExit(f"--start {start_sec:.1f}s is past end of video ({full_duration:.1f}s)")
|
||||
|
||||
effective_start = start_sec if start_sec is not None else 0.0
|
||||
effective_end = end_sec if end_sec is not None else full_duration
|
||||
effective_duration = max(0.0, effective_end - effective_start)
|
||||
focused = start_sec is not None or end_sec is not None
|
||||
|
||||
if focused:
|
||||
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
|
||||
else:
|
||||
fps, target = auto_fps(effective_duration, max_frames=max_frames)
|
||||
if args.fps is not None:
|
||||
fps = min(args.fps, MAX_FPS)
|
||||
target = max(1, int(round(fps * effective_duration)))
|
||||
|
||||
scope = (
|
||||
f"{format_time(effective_start)}-{format_time(effective_end)} ({effective_duration:.1f}s)"
|
||||
if focused else f"full {effective_duration:.1f}s"
|
||||
)
|
||||
print(f"[watch] extracting ~{target} frames at {fps:.3f} fps over {scope}…", file=sys.stderr)
|
||||
|
||||
frames = extract(
|
||||
video_path,
|
||||
work / "frames",
|
||||
fps=fps,
|
||||
resolution=args.resolution,
|
||||
max_frames=max_frames,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
|
||||
transcript_segments: list[dict] = []
|
||||
transcript_text: str | None = None
|
||||
transcript_source: str | None = None
|
||||
if dl.get("subtitle_path"):
|
||||
try:
|
||||
all_segments = parse_vtt(dl["subtitle_path"])
|
||||
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = "captions"
|
||||
except Exception as exc:
|
||||
print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr)
|
||||
|
||||
if not transcript_segments and not args.no_whisper:
|
||||
backend, api_key = load_api_key(args.whisper)
|
||||
if backend and api_key:
|
||||
try:
|
||||
all_segments, used_backend = transcribe_video(
|
||||
video_path,
|
||||
work / "audio.mp3",
|
||||
backend=backend,
|
||||
api_key=api_key,
|
||||
)
|
||||
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = f"whisper ({used_backend})"
|
||||
except SystemExit as exc:
|
||||
print(f"[watch] whisper fallback failed: {exc}", file=sys.stderr)
|
||||
else:
|
||||
hint = (
|
||||
f"--whisper {args.whisper} was set but the matching API key is missing"
|
||||
if args.whisper else
|
||||
"no subtitles and no Whisper API key found"
|
||||
)
|
||||
setup_py = SCRIPT_DIR / "setup.py"
|
||||
print(
|
||||
f"[watch] {hint} — run `python3 {setup_py}` to enable the Whisper fallback",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
info = dl.get("info") or {}
|
||||
|
||||
print()
|
||||
print("# watch: video report")
|
||||
print()
|
||||
print(f"- **Source:** {args.source}")
|
||||
if info.get("title"):
|
||||
print(f"- **Title:** {info['title']}")
|
||||
if info.get("uploader"):
|
||||
print(f"- **Uploader:** {info['uploader']}")
|
||||
print(f"- **Duration:** {format_time(full_duration)} ({full_duration:.1f}s)")
|
||||
if focused:
|
||||
print(
|
||||
f"- **Focus range:** {format_time(effective_start)} → {format_time(effective_end)} "
|
||||
f"({effective_duration:.1f}s)"
|
||||
)
|
||||
if meta.get("width") and meta.get("height"):
|
||||
print(f"- **Resolution:** {meta['width']}x{meta['height']} ({meta.get('codec') or 'unknown codec'})")
|
||||
mode = "focused" if focused else "full"
|
||||
print(f"- **Frames:** {len(frames)} @ {fps:.3f} fps, {mode} mode (budget {target}, max {max_frames})")
|
||||
print(f"- **Frame size:** {args.resolution}px wide")
|
||||
if transcript_segments:
|
||||
in_range = " in range" if focused else ""
|
||||
print(
|
||||
f"- **Transcript:** {len(transcript_segments)} segments{in_range} "
|
||||
f"(via {transcript_source or 'captions'})"
|
||||
)
|
||||
else:
|
||||
print("- **Transcript:** none available")
|
||||
|
||||
if not focused and full_duration > 600:
|
||||
mins = int(full_duration // 60)
|
||||
print()
|
||||
print(
|
||||
f"> ⚠️ This is a {mins}-minute video. Frame coverage is sparse at this length — "
|
||||
"accuracy degrades noticeably on anything over 10 minutes. For better results, "
|
||||
"re-run with `--start HH:MM:SS --end HH:MM:SS` to zoom into a specific section."
|
||||
)
|
||||
|
||||
print()
|
||||
print("## Frames")
|
||||
print()
|
||||
print(f"Frames live at: `{work / 'frames'}`")
|
||||
print()
|
||||
print(
|
||||
"**Read each frame path below with the Read tool to view the image.** "
|
||||
"Frames are in chronological order; `t=MM:SS` is the absolute timestamp in the source video."
|
||||
)
|
||||
print()
|
||||
for frame in frames:
|
||||
print(f"- `{frame['path']}` (t={format_time(frame['timestamp_seconds'])})")
|
||||
|
||||
print()
|
||||
print("## Transcript")
|
||||
print()
|
||||
if transcript_text:
|
||||
label = transcript_source or "captions"
|
||||
if focused:
|
||||
print(f"_Source: {label}. Filtered to {format_time(effective_start)} → {format_time(effective_end)}:_")
|
||||
else:
|
||||
print(f"_Source: {label}._")
|
||||
print()
|
||||
print("```")
|
||||
print(transcript_text)
|
||||
print("```")
|
||||
elif focused and dl.get("subtitle_path"):
|
||||
print(f"_No transcript lines fell inside {format_time(effective_start)} → {format_time(effective_end)}._")
|
||||
else:
|
||||
setup_py = SCRIPT_DIR / "setup.py"
|
||||
print(
|
||||
"_No transcript available — proceed with frames only. "
|
||||
"Captions were missing and the Whisper fallback was unavailable "
|
||||
"(no API key set, or `--no-whisper` was used). "
|
||||
f"Run `python3 {setup_py}` to enable Whisper, then re-run._"
|
||||
)
|
||||
|
||||
print()
|
||||
print("---")
|
||||
print(f"_Work dir: `{work}` — delete when done._")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,3 @@
|
||||
# Scans that start from this skill directory (not the repo root).
|
||||
# Keep non-runtime packaging/dev artifacts out of install-time security scans.
|
||||
scripts/build-skill.sh
|
||||
@@ -0,0 +1,268 @@
|
||||
---
|
||||
name: watch
|
||||
version: "0.2.0"
|
||||
description: Watch a video (URL or local path). Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls the transcript from captions (or Whisper API fallback), and hands the result to Claude so it can answer questions about what's in the video.
|
||||
argument-hint: "<video-url-or-path> [question]"
|
||||
allowed-tools: Bash, Read, AskUserQuestion
|
||||
homepage: https://github.com/bradautomates/claude-video
|
||||
repository: https://github.com/bradautomates/claude-video
|
||||
author: bradautomates
|
||||
license: MIT
|
||||
user-invocable: true
|
||||
---
|
||||
|
||||
# /watch
|
||||
|
||||
You don't have a video input; this skill gives you one. A Python script gets captions first, optionally downloads the video, extracts frames as JPEGs (scene-aware, or fast keyframes at `efficient` detail), gets a timestamped transcript (native captions first, then Whisper API as fallback), and prints frame paths. You then `Read` each frame path to see the images and combine them with the transcript to answer the user.
|
||||
|
||||
## Resolve `SKILL_DIR` (do this before any command)
|
||||
|
||||
Every `python3 ...` command below runs a bundled script under `SKILL_DIR/scripts/`. Set `SKILL_DIR` to the **absolute path of the directory containing THIS SKILL.md you just Read** — your harness told you that path in the Read result. The scripts are always a direct sibling of this file (`SKILL_DIR/scripts/watch.py`), in every install layout:
|
||||
|
||||
```
|
||||
Read ~/.claude/plugins/cache/claude-video/watch/<ver>/skills/watch/SKILL.md → SKILL_DIR=…/skills/watch
|
||||
Read ~/.codex/skills/watch/SKILL.md → SKILL_DIR=~/.codex/skills/watch
|
||||
Read ~/.agents/skills/watch/SKILL.md → SKILL_DIR=~/.agents/skills/watch
|
||||
```
|
||||
|
||||
Substitute that literal path for `${SKILL_DIR}` in every command. This works on every harness (Claude Code, Codex, Cursor, Gemini CLI, …) without relying on any harness-specific environment variable. Guard once at the start of a run:
|
||||
|
||||
```bash
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
if [ ! -f "$SKILL_DIR/scripts/watch.py" ]; then
|
||||
echo "ERROR: scripts/watch.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR." >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
## Step 0 — Setup preflight (runs every `/watch` invocation, silent on success)
|
||||
|
||||
**Python interpreter:** every `python3 ...` command in this skill is for macOS/Linux. On **Windows**, substitute `python` — the `python3` command on Windows is the Microsoft Store stub and will not run the script.
|
||||
|
||||
On the first `/watch` invocation in a session, use structured preflight so you can detect first-run setup:
|
||||
|
||||
```bash
|
||||
python3 "${SKILL_DIR}/scripts/setup.py" --json
|
||||
```
|
||||
|
||||
Branch on two fields:
|
||||
|
||||
- **`can_proceed: true` and `first_run: false`** → setup is already done (the user may have deliberately skipped a Whisper key — that's allowed). Proceed to Step 1 without comment.
|
||||
- **`first_run: true`** → genuine first-time setup. Do these in order:
|
||||
1. If `missing_binaries` is non-empty, run the installer first (it auto-installs on macOS / prints commands elsewhere — see below) and confirm the binaries land. **Do not skip this and jump to preferences.**
|
||||
2. Run the installer once more if needed so it scaffolds `~/.config/watch/.env` (it only writes the template when the file is absent, so let it create the file *before* you write any values into it).
|
||||
3. Encourage a Whisper API key and ask the watch-preference questions below, then write the selected values into `~/.config/watch/.env` and set `SETUP_COMPLETE=true`.
|
||||
- **`can_proceed: false` and `first_run: false`** → setup was finished before but the environment regressed (e.g. `missing_binaries` after an OS change). Run the installer to remediate, then proceed. Don't re-ask preferences.
|
||||
|
||||
A missing Whisper key is *encouraged to fix, not required*: on a genuine first run `status` will read `needs_key` even when binaries are present — that's your cue to encourage a key, not a blocker.
|
||||
|
||||
On follow-up `/watch` calls in the same session, use the silent check:
|
||||
|
||||
```bash
|
||||
python3 "${SKILL_DIR}/scripts/setup.py" --check
|
||||
```
|
||||
|
||||
This is a <100ms lookup. Exit 0 means /watch can run — this **includes a user who finished setup without a Whisper key** (keyless is allowed). On exit 0 the script emits **nothing** — proceed to Step 1 without comment. **Do NOT announce "setup is complete" to the user** — they don't need a status message on every turn. The only acceptable user-visible output from Step 0 is when remediation is required.
|
||||
|
||||
On non-zero exit, follow the table:
|
||||
|
||||
| Exit | Meaning | Action |
|
||||
|------|---------|--------|
|
||||
| `2` | Missing binaries (`ffmpeg` / `ffprobe` / `yt-dlp`) | Run installer |
|
||||
| `3` | Genuine first run with no Whisper API key | Run installer to scaffold `.env`, then encourage a key (the user may decline — proceed with `--no-whisper`) |
|
||||
| `4` | Both missing | Run installer, then encourage a key |
|
||||
|
||||
Exit `3` only fires before the user has completed setup. Once `SETUP_COMPLETE=true` is written, a keyless install returns exit 0 and is never nagged again.
|
||||
|
||||
The installer is idempotent — safe to re-run:
|
||||
|
||||
```bash
|
||||
python3 "${SKILL_DIR}/scripts/setup.py"
|
||||
```
|
||||
|
||||
On macOS with Homebrew, it auto-installs `ffmpeg` and `yt-dlp`. On Linux/Windows, it prints the exact install commands for the user to run. It scaffolds `~/.config/watch/.env` with commented placeholders and default watch settings at `0600` perms.
|
||||
|
||||
**If an API key is still missing after install:** use `AskUserQuestion` to ask the user whether they have a Groq API key (preferred — cheaper, faster) or an OpenAI key. Then write it into `~/.config/watch/.env` — set the matching `GROQ_API_KEY=...` or `OPENAI_API_KEY=...` line. If they don't want to set up Whisper, proceed with `--no-whisper` and tell them videos without native captions will come back frames-only.
|
||||
|
||||
**First-run watch preference:** after the installer has scaffolded `~/.config/watch/.env`, use `AskUserQuestion` to ask one question:
|
||||
|
||||
- Default detail (one dial). Present these as `AskUserQuestion` options in this exact order — lightest to heaviest — and keep `(recommended)` on `balanced` even though it is not first (do **not** reorder to put the recommended option first):
|
||||
- `transcript` — no frames at all, transcript only (skips video download when captions exist).
|
||||
- `efficient` — fast keyframe pass (cap 50).
|
||||
- `balanced` (recommended) — scene-aware frames (cap 100, default).
|
||||
- `token-burner` — scene-aware, uncapped (maximum fidelity; high token cost).
|
||||
|
||||
Write the answer directly into `~/.config/watch/.env` by setting the bare key on its own line — **no trailing inline comment** (a `# note` after the value can break parsing):
|
||||
|
||||
```bash
|
||||
WATCH_DETAIL=balanced
|
||||
```
|
||||
|
||||
Use the user's selected value. If they skip the question, keep the recommended default. Once dependencies, the API-key choice, and this preference are handled, write or update `SETUP_COMPLETE=true` in the same file. Do not ask this preference question again when `SETUP_COMPLETE=true`.
|
||||
|
||||
**Structured mode (optional):** `python3 "${SKILL_DIR}/scripts/setup.py" --json` emits `{status, can_proceed, first_run, setup_complete, missing_binaries, whisper_backend, has_api_key, config_file, watch_detail, platform}` where `status` is one of `ready | needs_install | needs_key | needs_install_and_key`. `status` describes the *ideal* state (a key is encouraged, so a keyless first run reads `needs_key`); `can_proceed` is the operational gate (binaries present AND a key is set OR setup was already completed). Branch on `can_proceed`/`first_run` to decide whether to run; use `status` to decide what to encourage.
|
||||
|
||||
Within a single session, you can skip Step 0 on follow-up `/watch` calls — once `--check` returned 0, nothing about the environment changes between turns.
|
||||
|
||||
## When to use
|
||||
|
||||
- User pastes a video URL (YouTube, Vimeo, X, TikTok, Twitch clip, most yt-dlp-supported sites) and asks about it.
|
||||
- User points at a local video file (`.mp4`, `.mov`, `.mkv`, `.webm`, etc.) and asks about it.
|
||||
- User types `/watch <url-or-path> [question]`.
|
||||
|
||||
## Recommended limits
|
||||
|
||||
- **Best accuracy: videos under 10 minutes.** Frame coverage scales inversely with duration.
|
||||
- **Universal rate cap: 2 fps.** The script never samples faster than 2 fps, even when a budget or `--fps` would imply more.
|
||||
- **The frame ceiling is set by the detail mode** (`WATCH_DETAIL` in `~/.config/watch/.env`, or `--detail`), not a single global cap:
|
||||
- `transcript` → no frames
|
||||
- `efficient` → up to **50** (keyframes)
|
||||
- `balanced` (default) → up to **100** (scene-aware)
|
||||
- `token-burner` → **uncapped** (scene-aware; a soft warning prints past 250 frames)
|
||||
- `--max-frames N` overrides whichever cap the mode would otherwise use.
|
||||
- **Full-video frame budget by duration.** Token cost grows with frame count, so the script targets a budget by duration. This budget sets the fps and the uniform-sampling fallback; scene-aware selection can fill up to the detail cap above, whichever is lower:
|
||||
- ≤30s → ~12-30 frames
|
||||
- 30s-1min → ~40 frames
|
||||
- 1-3min → ~60 frames
|
||||
- 3-10min → ~80 frames
|
||||
- \>10min → up to the detail cap, sparsely spaced (warning printed)
|
||||
- If the user hands you a long video, consider asking whether they want a specific section before burning tokens on a sparse scan.
|
||||
|
||||
## How to invoke
|
||||
|
||||
**Step 1 — parse the user input.** Separate the video source (URL or path) from any question the user asked. Example: `/watch https://youtu.be/abc what language is this in?` → source = `https://youtu.be/abc`, question = `what language is this in?`.
|
||||
|
||||
**Step 2 — run the watch script.** Pass the source verbatim. Do not shell-escape it yourself beyond normal quoting:
|
||||
|
||||
```bash
|
||||
python3 "${SKILL_DIR}/scripts/watch.py" "<source>"
|
||||
```
|
||||
|
||||
Optional flags:
|
||||
- `--detail transcript|efficient|balanced|token-burner` — fidelity/speed dial. `transcript` = no frames (transcript only, skips video download when captions exist); `efficient` = fast keyframes (cap 50); `balanced` = scene-aware frames (cap 100); `token-burner` = scene-aware, uncapped.
|
||||
- `--start T` / `--end T` — focus on a section. Accepts `SS`, `MM:SS`, or `HH:MM:SS`. When either is set, fps auto-scales denser (see "Focusing on a section" below).
|
||||
- `--timestamps T1,T2,…` — grab a frame at each of these absolute timestamps (`SS`, `MM:SS`, or `HH:MM:SS`). Use this after reading the transcript to capture deictic moments the presenter flags ("look here", "as you can see", "notice this") that visual selection alone may miss. See "Transcript-cue frames" below.
|
||||
- `--max-frames N` — override the preset cap for tighter token budget (e.g. `--max-frames 40`)
|
||||
- `--resolution W` — change frame width in px (default 512; bump to 1024 only if the user needs to read on-screen text)
|
||||
- `--fps F` — override auto-fps (clamped to 2 fps max)
|
||||
- `--out-dir DIR` — keep working files somewhere specific (default: an auto-generated tmp dir)
|
||||
- `--whisper groq|openai` — force a specific Whisper backend (default: prefer Groq if both keys exist)
|
||||
- `--no-whisper` — disable the Whisper fallback entirely (frames-only if no captions)
|
||||
- `--no-dedup` — keep near-duplicate frames. By default a frame-delta pass drops frames that are visually near-identical to the previous kept one (held slides, static screen recordings, paused video) so the frame budget goes to distinct content; the report's **Frames** line notes how many were dropped. Pass this only if the user needs every sampled frame (e.g. judging subtle frame-to-frame motion).
|
||||
|
||||
### Focusing on a section (higher frame rate)
|
||||
|
||||
When the user asks about a specific moment — "what happens at the 2 minute mark?", "zoom into 0:45 to 1:00", "the first 10 seconds" — pass `--start` and/or `--end`. The script switches to focused-mode budgets, which are denser than full-video budgets (still capped at 2 fps, and still bounded by the detail-mode cap — the counts below assume the default `balanced` cap of 100; `efficient` tops out at 50):
|
||||
|
||||
- ≤5s → 2 fps (up to 10 frames)
|
||||
- 5-15s → 2 fps (up to 30 frames)
|
||||
- 15-30s → ~2 fps (up to 60 frames)
|
||||
- 30-60s → ~1.3 fps (up to 80 frames)
|
||||
- 60-180s → ~0.6 fps (100 frames, capped)
|
||||
|
||||
Focused mode is the right call for:
|
||||
- Any moment/range the user names explicitly ("around 2:30", "the intro", "the last 30 seconds").
|
||||
- Any video longer than ~10 minutes where the user's question is about a specific part — running focused on the relevant section is far more useful than a sparse scan of the whole thing.
|
||||
- Re-runs after a full scan didn't have enough detail in some region.
|
||||
|
||||
Transcript is auto-filtered to the same range. Frame timestamps are absolute (real video timeline, not offset-from-start).
|
||||
|
||||
Examples:
|
||||
```bash
|
||||
# Last 10 seconds of a 1 minute video
|
||||
python3 "${SKILL_DIR}/scripts/watch.py" video.mp4 --start 50 --end 60
|
||||
|
||||
# Zoom into 2:15 → 2:45 at 2 fps (60 frames)
|
||||
python3 "${SKILL_DIR}/scripts/watch.py" "$URL" --start 2:15 --end 2:45 --fps 2
|
||||
|
||||
# From 1h12m to the end of the video
|
||||
python3 "${SKILL_DIR}/scripts/watch.py" "$URL" --start 1:12:00
|
||||
```
|
||||
|
||||
**Step 3 — Read every frame path the script lists.** The Read tool renders JPEGs directly as images for you. Read all frames in a single message (parallel tool calls) so you see them together. The frames are in chronological order with a `t=MM:SS` timestamp so you can align them to the transcript.
|
||||
|
||||
**Step 4 — answer the user.** You now have two streams of evidence:
|
||||
- **Frames** — what's on screen at each timestamp
|
||||
- **Transcript** — what's said at each timestamp. The report's header shows the source (`captions` = yt-dlp pulled native subs; `whisper (groq)` or `whisper (openai)` = transcribed by API).
|
||||
|
||||
If the user asked a specific question, answer it directly citing timestamps. If they didn't ask anything, summarize what happens in the video — structure, key moments, notable visuals, spoken content.
|
||||
|
||||
This holds for `transcript` detail too: even with no frames, produce a **summary** like the other modes — do not paste the full transcript into chat. Synthesize structure, key moments, and spoken content with timestamps; quote only the lines that matter. Offer the raw transcript only if the user explicitly asks for it.
|
||||
|
||||
**Step 5 — clean up.** The script prints a working directory at the end. If the user isn't going to ask follow-ups about this video, delete it with `rm -rf <dir>`. If they might, leave it in place.
|
||||
|
||||
## Detail and frames
|
||||
|
||||
Default behavior comes from `~/.config/watch/.env`:
|
||||
|
||||
- `WATCH_DETAIL=transcript|efficient|balanced|token-burner` (default: `balanced`)
|
||||
|
||||
At `transcript` detail, captions are enough to return a report without downloading video. If captions are missing, the script downloads audio only and tries Whisper. If no transcript can be produced, it reports the limitation clearly; re-run with `--detail balanced` for frames.
|
||||
|
||||
At `efficient` detail, the script downloads the video and extracts **keyframes only** (`ffmpeg -skip_frame nokey`) — a near-instant pass that lands frames on scene cuts. If a clip has fewer than 4 keyframes it falls back to uniform sampling.
|
||||
|
||||
At `balanced` / `token-burner` detail, the script extracts **scene-aware** frames: ffmpeg scene-change selection first, falling back to uniform sampling only when the video is effectively static. `balanced` caps at 100 frames; `token-burner` is uncapped. Frame report lines include both timestamp and selection reason. Extracted images are clamped to a maximum 1998px height for Claude Read compatibility.
|
||||
|
||||
## Transcript-cue frames
|
||||
|
||||
Visual frame selection (scene/keyframe) can miss the moments a presenter explicitly flags — "look here", "as you can see", "notice this", "watch what happens" — because pointing at a slide is often a *low* visual change. `--timestamps` lets you force a frame at those exact moments. **You** decide which moments matter, by reading the transcript:
|
||||
|
||||
1. Run once at `--detail transcript` (or any detail) to get the timestamped transcript.
|
||||
2. Scan it for deictic cues — phrases where the speaker directs attention to something on screen. This is a judgment call (ignore rhetorical "look, the point is…"); that's why it's done by you, not a regex.
|
||||
3. Re-run with `--timestamps 4:32,7:10,9:55` (absolute source times). For a URL, point the second run at the **downloaded local file** in the work dir so it doesn't re-download.
|
||||
|
||||
Behavior:
|
||||
- **Additive by default.** Cue frames (`reason=transcript-cue`) are merged into whatever `--detail` already selected, in chronological order.
|
||||
- **Pinned and counted first.** Cue frames are reserved against the frame cap before the detail engine runs, so they're never evicted by even-sampling.
|
||||
- **Honors focus mode.** With `--start/--end`, any cue timestamp outside the window is dropped (reported in the summary). Coordinates are always absolute source time.
|
||||
- **Cue-only frames.** `--detail transcript --timestamps …` skips scene/keyframe sampling and returns *only* the cue frames (it will download the video to do so, since frames need pixels).
|
||||
|
||||
## Transcription
|
||||
|
||||
The script gets a timestamped transcript in one of two ways:
|
||||
|
||||
1. **Native captions (free, preferred).** yt-dlp pulls manual or auto-generated subtitles from the source platform if available.
|
||||
2. **Whisper API fallback.** If no captions came back (or the source is a local file), the script extracts audio (`ffmpeg -vn -ac 1 -ar 16000 -b:a 64k`, ~0.5 MB/min) and uploads it to whichever Whisper API has a key configured:
|
||||
- **Groq** — `whisper-large-v3`. Preferred default: cheaper, faster. Get a key at console.groq.com/keys.
|
||||
- **OpenAI** — `whisper-1`. Fallback. Get a key at platform.openai.com/api-keys.
|
||||
|
||||
Both keys live in `~/.config/watch/.env`. The script prefers Groq when both are set; override with `--whisper openai` to force OpenAI. Use `--no-whisper` to skip the fallback entirely.
|
||||
|
||||
## Failure modes and handling
|
||||
|
||||
- **Setup preflight failed** → run `python3 "${SKILL_DIR}/scripts/setup.py"` (auto-installs ffmpeg/yt-dlp via brew on macOS, scaffolds the `.env`). For API key, ask the user via `AskUserQuestion` and write it to `~/.config/watch/.env`.
|
||||
- **No transcript available** → captions missing AND (no Whisper key OR Whisper API failed). Script prints a hint pointing to setup. Proceed frames-only and tell the user.
|
||||
- **Long video warning printed** → acknowledge it in your answer. Offer to re-run focused on a specific section via `--start`/`--end` rather than a sparse full-video scan.
|
||||
- **Download fails** → yt-dlp's error goes to stderr. If it's a login-required or region-locked video, tell the user plainly; do not keep retrying.
|
||||
- **Whisper request fails** → the error is printed to stderr (likely: invalid key or rate limit). Audio over the API's 25 MB upload cap is split into chunks and transcribed automatically, so length alone won't fail it; if some chunks fail the transcript is partial and the dropped chunks are noted on stderr. The report will say "none available" only if every chunk fails. You can retry with `--whisper openai` if Groq failed (or vice versa).
|
||||
|
||||
## Token efficiency
|
||||
|
||||
This skill burns tokens primarily on frames. Order of magnitude:
|
||||
- 80 frames at 512px wide is roughly 50-80k image tokens depending on aspect ratio.
|
||||
- The transcript is cheap (a few thousand tokens at most for a 10-minute video).
|
||||
- Bumping `--resolution` to 1024 roughly quadruples the image tokens per frame. Only do it when necessary.
|
||||
|
||||
If you already watched a video this session and the user asks a follow-up, do **not** re-run the script — you already have the frames and transcript in context. Just answer from what you have.
|
||||
|
||||
## Security & Permissions
|
||||
|
||||
**What this skill does:**
|
||||
- Runs `yt-dlp` locally to download the video and pull native captions when the source supports them (public data; the request goes directly to whatever host the URL points at)
|
||||
- Runs `ffmpeg` / `ffprobe` locally to extract frames as JPEGs and, when Whisper is needed, a mono 16 kHz audio clip
|
||||
- Sends the extracted audio clip to Groq's Whisper API (`api.groq.com/openai/v1/audio/transcriptions`) when `GROQ_API_KEY` is set (preferred — cheaper, faster)
|
||||
- Sends the extracted audio clip to OpenAI's audio transcription API (`api.openai.com/v1/audio/transcriptions`) when `OPENAI_API_KEY` is set and Groq is not, or when `--whisper openai` is forced
|
||||
- Writes the downloaded video, frames, audio, and an intermediate transcript to a working directory under the system temp dir (or `--out-dir` if specified) so Claude can `Read` them
|
||||
- Reads / creates `~/.config/watch/.env` (mode `0600`) to store the Whisper API key(s) and a `SETUP_COMPLETE` marker. As a fallback, also reads `.env` in the current working directory
|
||||
|
||||
**What this skill does NOT do:**
|
||||
- Does not upload the video itself to any API — only the extracted audio goes out, and only when native captions are missing AND Whisper is not disabled with `--no-whisper`
|
||||
- Does not access any platform account (no login, no session cookies, no posting) — yt-dlp only ever requests public data
|
||||
- Does not share API keys between providers (Groq key only goes to `api.groq.com`, OpenAI key only goes to `api.openai.com`)
|
||||
- Does not log, cache, or write API keys to stdout, stderr, or output files
|
||||
- Does not persist anything outside the working directory and `~/.config/watch/.env` — clean up the working directory when you're done (Step 5)
|
||||
|
||||
**Bundled scripts:** `scripts/watch.py` (entry point), `scripts/download.py` (yt-dlp wrapper), `scripts/frames.py` (ffmpeg frame extraction), `scripts/transcribe.py` (caption selection + Whisper orchestration), `scripts/whisper.py` (Groq / OpenAI clients), `scripts/setup.py` (preflight + installer)
|
||||
|
||||
Review scripts before first use to verify behavior.
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-skill.sh — package the watch skill as a claude.ai-upload-ready .skill file.
|
||||
# Usage: bash skills/watch/scripts/build-skill.sh (run from anywhere)
|
||||
#
|
||||
# Produces dist/watch.skill, a zip with a single top-level `watch/` directory
|
||||
# containing SKILL.md and the scripts/ runtime from skills/watch. Archiving the
|
||||
# skills/watch subtree directly keeps the bundle to exactly one SKILL.md and
|
||||
# well under claude.ai's 200-file cap, with no post-hoc `zip -d` stripping.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
echo "error: working tree is dirty; commit or stash before building" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p dist
|
||||
OUT="dist/watch.skill"
|
||||
git archive --format=zip --prefix=watch/ --output="$OUT" HEAD:skills/watch
|
||||
|
||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||
SIZE=$(du -h "$OUT" | cut -f1)
|
||||
|
||||
if [ "$COUNT" -gt 200 ]; then
|
||||
echo "error: $COUNT files in zip, claude.ai's cap is 200" >&2
|
||||
echo " trim the skills/watch/ tree or add a .gitattributes export-ignore entry" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SKILL_MD_COUNT=$(unzip -l "$OUT" | grep -c "SKILL.md" || true)
|
||||
if [ "$SKILL_MD_COUNT" -ne 1 ]; then
|
||||
echo "error: expected exactly one SKILL.md, found $SKILL_MD_COUNT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "built $OUT ($COUNT files, $SIZE)"
|
||||
echo "upload via the claude.ai skill UI"
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared /watch configuration helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CONFIG_DIR = Path.home() / ".config" / "watch"
|
||||
CONFIG_FILE = CONFIG_DIR / ".env"
|
||||
|
||||
DEFAULT_DETAIL = "balanced"
|
||||
|
||||
DETAILS = {"transcript", "efficient", "balanced", "token-burner"}
|
||||
|
||||
|
||||
def read_env_file(path: Path | None = None) -> dict[str, str]:
|
||||
if path is None:
|
||||
path = CONFIG_FILE
|
||||
values: dict[str, str] = {}
|
||||
if not path.exists():
|
||||
return values
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
except OSError:
|
||||
return values
|
||||
for line in lines:
|
||||
raw = line.strip()
|
||||
if not raw or raw.startswith("#") or "=" not in raw:
|
||||
continue
|
||||
key, _, value = raw.partition("=")
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] in ('"', "'") and value[-1] == value[0]:
|
||||
value = value[1:-1]
|
||||
else:
|
||||
# Strip an inline comment (a '#' preceded by whitespace) from an
|
||||
# unquoted value. Without this, `WATCH_DETAIL=balanced # note`
|
||||
# parses as "balanced # note", fails validation, and silently
|
||||
# falls back to the default. Keeps '#' inside quotes / API keys.
|
||||
for i, ch in enumerate(value):
|
||||
if ch == "#" and i > 0 and value[i - 1] in " \t":
|
||||
value = value[:i].rstrip()
|
||||
break
|
||||
values[key.strip()] = value
|
||||
return values
|
||||
|
||||
|
||||
def get_config() -> dict[str, object]:
|
||||
file_values = read_env_file()
|
||||
|
||||
detail = (
|
||||
os.environ.get("WATCH_DETAIL")
|
||||
or file_values.get("WATCH_DETAIL")
|
||||
or DEFAULT_DETAIL
|
||||
)
|
||||
if detail not in DETAILS:
|
||||
detail = DEFAULT_DETAIL
|
||||
|
||||
return {
|
||||
"detail": detail,
|
||||
"config_file": str(CONFIG_FILE),
|
||||
}
|
||||
|
||||
|
||||
def frame_cap(detail: str) -> int | None:
|
||||
if detail == "efficient":
|
||||
return 50
|
||||
if detail == "balanced":
|
||||
return 100
|
||||
if detail == "token-burner":
|
||||
return None
|
||||
if detail == "transcript":
|
||||
return None
|
||||
return 100
|
||||
@@ -18,8 +18,10 @@ VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".mov", ".m4v", ".avi", ".flv", ".wmv"}
|
||||
|
||||
|
||||
def is_url(source: str) -> bool:
|
||||
if source.startswith("-"):
|
||||
return False
|
||||
parsed = urlparse(source)
|
||||
return parsed.scheme in ("http", "https")
|
||||
return parsed.scheme in ("http", "https") and bool(parsed.netloc)
|
||||
|
||||
|
||||
def resolve_local(path: str) -> dict:
|
||||
@@ -43,12 +45,15 @@ def _pick_subtitle(out_dir: Path) -> Path | None:
|
||||
candidates = sorted(out_dir.glob("video*.vtt"))
|
||||
if not candidates:
|
||||
return None
|
||||
preferred = [c for c in candidates if ".en" in c.name]
|
||||
preferred = [
|
||||
c for c in candidates
|
||||
if any(marker in c.name for marker in (".en.", ".en-US.", ".en-GB.", ".en-orig."))
|
||||
]
|
||||
return preferred[0] if preferred else candidates[0]
|
||||
|
||||
|
||||
def _pick_video(out_dir: Path) -> Path | None:
|
||||
for ext in (".mp4", ".mkv", ".webm", ".mov"):
|
||||
for ext in (".mp4", ".mkv", ".webm", ".mov", ".m4a", ".mp3", ".opus"):
|
||||
for candidate in out_dir.glob(f"video*{ext}"):
|
||||
return candidate
|
||||
for candidate in out_dir.glob("video.*"):
|
||||
@@ -57,27 +62,83 @@ def _pick_video(out_dir: Path) -> Path | None:
|
||||
return None
|
||||
|
||||
|
||||
def download_url(url: str, out_dir: Path) -> dict:
|
||||
def fetch_captions(url: str, out_dir: Path) -> dict:
|
||||
"""Fetch metadata and best available VTT captions without downloading video."""
|
||||
if shutil.which("yt-dlp") is None:
|
||||
raise SystemExit("yt-dlp is not installed. Install with: brew install yt-dlp")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_template = str(out_dir / "video.%(ext)s")
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"--skip-download",
|
||||
"--write-info-json",
|
||||
"--write-subs",
|
||||
"--write-auto-subs",
|
||||
"--sub-langs", "en.*",
|
||||
"--sub-format", "vtt",
|
||||
"--convert-subs", "vtt",
|
||||
"--no-playlist",
|
||||
"--ignore-errors",
|
||||
"-o", output_template,
|
||||
"--",
|
||||
url,
|
||||
]
|
||||
subprocess.run(cmd, stdout=sys.stderr, stderr=sys.stderr)
|
||||
subtitle = _pick_subtitle(out_dir)
|
||||
info = _read_info(out_dir / "video.info.json", url)
|
||||
return {
|
||||
"video_path": None,
|
||||
"subtitle_path": str(subtitle) if subtitle else None,
|
||||
"info": info or {"url": url},
|
||||
"downloaded": False,
|
||||
}
|
||||
|
||||
|
||||
def _read_info(info_path: Path, url: str) -> dict:
|
||||
info: dict = {}
|
||||
if info_path.exists():
|
||||
try:
|
||||
raw = json.loads(info_path.read_text(encoding="utf-8"))
|
||||
info = {
|
||||
"title": raw.get("title"),
|
||||
"uploader": raw.get("uploader") or raw.get("channel"),
|
||||
"duration": raw.get("duration"),
|
||||
"url": raw.get("webpage_url") or url,
|
||||
}
|
||||
except Exception as exc:
|
||||
print(f"[watch] info.json parse failed: {exc}", file=sys.stderr)
|
||||
info = {"url": url}
|
||||
return info
|
||||
|
||||
|
||||
def download_url(
|
||||
url: str,
|
||||
out_dir: Path,
|
||||
audio_only: bool = False,
|
||||
) -> dict:
|
||||
if shutil.which("yt-dlp") is None:
|
||||
raise SystemExit("yt-dlp is not installed. Install with: brew install yt-dlp")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_template = str(out_dir / "video.%(ext)s")
|
||||
|
||||
fmt = "ba/bestaudio" if audio_only else "bv*[height<=720]+ba/b[height<=720]/bv+ba/b"
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"-N", "8",
|
||||
"-f", "bv*[height<=720]+ba/b[height<=720]/bv+ba/b",
|
||||
"-f", fmt,
|
||||
"--merge-output-format", "mp4",
|
||||
"--write-info-json",
|
||||
"--write-subs",
|
||||
"--write-auto-subs",
|
||||
"--sub-langs", "en,en-US,en-GB,en-orig",
|
||||
"--sub-langs", "en.*",
|
||||
"--sub-format", "vtt",
|
||||
"--convert-subs", "vtt",
|
||||
"--no-playlist",
|
||||
"--ignore-errors",
|
||||
"-o", output_template,
|
||||
"--",
|
||||
url,
|
||||
]
|
||||
|
||||
@@ -91,19 +152,7 @@ def download_url(url: str, out_dir: Path) -> dict:
|
||||
)
|
||||
|
||||
subtitle = _pick_subtitle(out_dir)
|
||||
info_path = out_dir / "video.info.json"
|
||||
info: dict = {}
|
||||
if info_path.exists():
|
||||
try:
|
||||
raw = json.loads(info_path.read_text())
|
||||
info = {
|
||||
"title": raw.get("title"),
|
||||
"uploader": raw.get("uploader") or raw.get("channel"),
|
||||
"duration": raw.get("duration"),
|
||||
"url": raw.get("webpage_url") or url,
|
||||
}
|
||||
except Exception:
|
||||
info = {"url": url}
|
||||
info = _read_info(out_dir / "video.info.json", url)
|
||||
|
||||
return {
|
||||
"video_path": str(video),
|
||||
@@ -113,9 +162,13 @@ def download_url(url: str, out_dir: Path) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def download(source: str, out_dir: Path) -> dict:
|
||||
def download(
|
||||
source: str,
|
||||
out_dir: Path,
|
||||
audio_only: bool = False,
|
||||
) -> dict:
|
||||
if is_url(source):
|
||||
return download_url(source, out_dir)
|
||||
return download_url(source, out_dir, audio_only=audio_only)
|
||||
return resolve_local(source)
|
||||
|
||||
|
||||
Executable
+756
@@ -0,0 +1,756 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Probe video metadata and extract frames at an auto-scaled fps.
|
||||
|
||||
Auto-fps targets a frame budget, not a fixed rate. Token cost scales with frame
|
||||
count, so budget-by-duration keeps short videos dense and long videos capped.
|
||||
When a user-specified range is passed, focused-mode budgets denser (they are
|
||||
zooming in for detail).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MAX_FPS = 2.0
|
||||
SCENE_THRESHOLD = 0.20
|
||||
# Keep scene-detection results once we have at least this many distinct shots.
|
||||
# Below this the video is effectively static (screen recording, talking head),
|
||||
# so we fall back to uniform sampling. Matching the reference fork's behaviour,
|
||||
# this is a low floor — NOT the frame budget — so normal videos with cuts use
|
||||
# the (single-pass) scene engine instead of paying for a wasted second decode.
|
||||
SCENE_MIN_FRAMES = 8
|
||||
# Below this many decoded keyframes a clip is too sparse for keyframe coverage
|
||||
# (very short or oddly encoded), so the cheap tier falls back to uniform.
|
||||
KEYFRAME_MIN = 4
|
||||
MAX_READ_DIMENSION = 1998
|
||||
# Frame-delta dedup: downscale each frame to a DEDUP_THUMB x DEDUP_THUMB
|
||||
# grayscale thumbnail and treat two frames as near-identical when their mean
|
||||
# per-pixel difference (0-255) is at or below DEDUP_THRESHOLD. Conservative on
|
||||
# purpose: only collapses frames that are visually the same shot, so a code diff
|
||||
# / scrolling terminal / slide-gaining-a-bullet survives. Unlike a within-frame
|
||||
# perceptual hash, this distinguishes flat frames (solid slides, fades) by luma.
|
||||
DEDUP_THUMB = 16
|
||||
DEDUP_THRESHOLD = 2.0
|
||||
SHOWINFO_TS_RE = re.compile(r"pts_time:([0-9.]+)")
|
||||
|
||||
|
||||
def _scale_filter(resolution: int) -> str:
|
||||
return (
|
||||
f"scale=w='min({resolution},iw)':h='min({MAX_READ_DIMENSION},ih)':"
|
||||
"force_original_aspect_ratio=decrease:force_divisible_by=2"
|
||||
)
|
||||
|
||||
|
||||
def _clamp_fps(fps: float, duration_seconds: float, max_frames: int) -> tuple[float, int]:
|
||||
fps = min(fps, MAX_FPS)
|
||||
target = min(max_frames, max(1, int(round(fps * duration_seconds))))
|
||||
return fps, target
|
||||
|
||||
|
||||
def parse_time(value: str | float | int | None) -> float | None:
|
||||
"""Parse SS, MM:SS, or HH:MM:SS (with optional .ms) into seconds."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
s = str(value).strip()
|
||||
if not s:
|
||||
return None
|
||||
parts = s.split(":")
|
||||
try:
|
||||
if len(parts) == 1:
|
||||
return float(parts[0])
|
||||
if len(parts) == 2:
|
||||
return int(parts[0]) * 60 + float(parts[1])
|
||||
if len(parts) == 3:
|
||||
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
|
||||
except ValueError:
|
||||
pass
|
||||
raise SystemExit(f"Cannot parse time value: {value!r} (expected SS, MM:SS, or HH:MM:SS)")
|
||||
|
||||
|
||||
def format_time(seconds: float) -> str:
|
||||
total = int(round(seconds))
|
||||
hours, rem = divmod(total, 3600)
|
||||
minutes, sec = divmod(rem, 60)
|
||||
if hours:
|
||||
return f"{hours}:{minutes:02d}:{sec:02d}"
|
||||
return f"{minutes:02d}:{sec:02d}"
|
||||
|
||||
|
||||
def get_metadata(video_path: str) -> dict:
|
||||
if shutil.which("ffprobe") is None:
|
||||
raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v", "quiet",
|
||||
"-print_format", "json",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
str(Path(video_path).resolve()),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffprobe failed: {result.stderr.strip()}")
|
||||
|
||||
data = json.loads(result.stdout or "{}")
|
||||
streams = data.get("streams", [])
|
||||
fmt = data.get("format", {})
|
||||
video_stream = next((s for s in streams if s.get("codec_type") == "video"), {})
|
||||
audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None)
|
||||
|
||||
duration = float(fmt.get("duration") or video_stream.get("duration") or 0)
|
||||
return {
|
||||
"duration_seconds": duration,
|
||||
"width": video_stream.get("width"),
|
||||
"height": video_stream.get("height"),
|
||||
"codec": video_stream.get("codec_name"),
|
||||
"size_bytes": int(fmt.get("size") or 0),
|
||||
"has_audio": audio_stream is not None,
|
||||
}
|
||||
|
||||
|
||||
def auto_fps(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
|
||||
"""Pick fps that targets a sensible frame budget for full-video scans."""
|
||||
if duration_seconds <= 0:
|
||||
return 1.0, 1
|
||||
|
||||
if duration_seconds <= 30:
|
||||
target = min(max_frames, max(12, int(round(duration_seconds))))
|
||||
elif duration_seconds <= 60:
|
||||
target = min(max_frames, 40)
|
||||
elif duration_seconds <= 180: # 3 min
|
||||
target = min(max_frames, 60)
|
||||
elif duration_seconds <= 600: # 10 min
|
||||
target = min(max_frames, 80)
|
||||
else:
|
||||
target = max_frames
|
||||
|
||||
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
|
||||
|
||||
|
||||
def auto_fps_focus(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
|
||||
"""Denser budget for user-specified ranges — they are zooming in for detail."""
|
||||
if duration_seconds <= 0:
|
||||
return min(MAX_FPS, 2.0), 2
|
||||
|
||||
if duration_seconds <= 5:
|
||||
target = min(max_frames, max(10, int(round(duration_seconds * 6))))
|
||||
elif duration_seconds <= 15:
|
||||
target = min(max_frames, max(30, int(round(duration_seconds * 4))))
|
||||
elif duration_seconds <= 30:
|
||||
target = min(max_frames, 60)
|
||||
elif duration_seconds <= 60:
|
||||
target = min(max_frames, 80)
|
||||
elif duration_seconds <= 180:
|
||||
target = max_frames
|
||||
else:
|
||||
target = max_frames
|
||||
|
||||
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
|
||||
|
||||
|
||||
def extract(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
fps: float,
|
||||
resolution: int = 512,
|
||||
max_frames: int = 100,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
) -> list[dict]:
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("frame_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
output_pattern = str(out_dir / "frame_%04d.jpg")
|
||||
cmd: list[str] = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-y",
|
||||
]
|
||||
|
||||
# -ss before -i = fast seek (keyframe-snap, good enough for preview frames).
|
||||
if start_seconds is not None:
|
||||
cmd += ["-ss", f"{start_seconds:.3f}"]
|
||||
if end_seconds is not None:
|
||||
cmd += ["-to", f"{end_seconds:.3f}"]
|
||||
|
||||
cmd += [
|
||||
"-i", str(Path(video_path).resolve()),
|
||||
"-vf", f"fps={fps},{_scale_filter(resolution)}",
|
||||
"-frames:v", str(max_frames),
|
||||
"-q:v", "4",
|
||||
output_pattern,
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffmpeg frame extraction failed: {result.stderr.strip()}")
|
||||
|
||||
offset = start_seconds or 0.0
|
||||
frames = sorted(out_dir.glob("frame_*.jpg"))
|
||||
return [
|
||||
{
|
||||
"index": i,
|
||||
"timestamp_seconds": round(offset + (i / fps if fps > 0 else 0.0), 2),
|
||||
"path": str(p),
|
||||
"reason": "uniform",
|
||||
}
|
||||
for i, p in enumerate(frames)
|
||||
]
|
||||
|
||||
|
||||
def extract_scene_candidates(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
resolution: int = 512,
|
||||
max_frames: int | None = 100,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
threshold: float = SCENE_THRESHOLD,
|
||||
) -> list[dict]:
|
||||
"""Extract first frame plus ffmpeg scene-change frames.
|
||||
|
||||
When ``max_frames`` is set, ``-frames:v`` lets ffmpeg stop decoding once it
|
||||
has emitted that many frames (early exit) and avoids writing extras that we
|
||||
would only delete afterwards. ``None`` (uncapped "complete" detail) keeps
|
||||
every detected shot, as the user explicitly opted in.
|
||||
"""
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("frame_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
output_pattern = str(out_dir / "frame_%04d.jpg")
|
||||
cmd: list[str] = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "info",
|
||||
"-y",
|
||||
]
|
||||
if start_seconds is not None:
|
||||
cmd += ["-ss", f"{start_seconds:.3f}"]
|
||||
if end_seconds is not None:
|
||||
cmd += ["-to", f"{end_seconds:.3f}"]
|
||||
|
||||
vf = f"select='eq(n\\,0)+gt(scene\\,{threshold})',{_scale_filter(resolution)},showinfo"
|
||||
cmd += [
|
||||
"-i", str(Path(video_path).resolve()),
|
||||
"-vf", vf,
|
||||
"-vsync", "vfr",
|
||||
]
|
||||
if max_frames is not None:
|
||||
cmd += ["-frames:v", str(max_frames)]
|
||||
cmd += [
|
||||
"-q:v", "4",
|
||||
output_pattern,
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffmpeg scene extraction failed: {result.stderr.strip()}")
|
||||
|
||||
offset = start_seconds or 0.0
|
||||
timestamps = [round(offset + float(match.group(1)), 2) for match in SHOWINFO_TS_RE.finditer(result.stderr)]
|
||||
frames = sorted(out_dir.glob("frame_*.jpg"))
|
||||
out: list[dict] = []
|
||||
for i, path in enumerate(frames):
|
||||
ts = timestamps[i] if i < len(timestamps) else offset
|
||||
out.append({
|
||||
"index": i,
|
||||
"timestamp_seconds": ts,
|
||||
"path": str(path),
|
||||
"reason": "first-frame" if i == 0 else "scene-change",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _even_indices(count: int, n: int) -> list[int]:
|
||||
"""Indices of ``n`` evenly-spaced items out of ``count`` (first + last kept).
|
||||
|
||||
``n >= count`` returns every index; ``n == 1`` returns just the first.
|
||||
"""
|
||||
if n >= count:
|
||||
return list(range(count))
|
||||
if n <= 1:
|
||||
return [0]
|
||||
return [round(i * (count - 1) / (n - 1)) for i in range(n)]
|
||||
|
||||
|
||||
def parse_timestamps(value: str | None) -> list[float]:
|
||||
"""Parse a comma-separated list of times (SS, MM:SS, HH:MM:SS) into a
|
||||
sorted, de-duplicated list of seconds. Empty/blank tokens are skipped;
|
||||
an unparseable token raises (via :func:`parse_time`)."""
|
||||
if not value:
|
||||
return []
|
||||
out: list[float] = []
|
||||
for token in value.split(","):
|
||||
token = token.strip()
|
||||
if not token:
|
||||
continue
|
||||
seconds = parse_time(token)
|
||||
if seconds is not None:
|
||||
out.append(float(seconds))
|
||||
return sorted(set(out))
|
||||
|
||||
|
||||
def merge_frames(primary: list[dict], pinned: list[dict]) -> list[dict]:
|
||||
"""Combine two frame lists into one chronological list and reindex 0..n-1.
|
||||
|
||||
``pinned`` frames (transcript cues) are never dropped — this is a plain
|
||||
union, so the cap is enforced upstream by reserving budget for the cues.
|
||||
"""
|
||||
merged = sorted([*primary, *pinned], key=lambda f: f["timestamp_seconds"])
|
||||
for i, frame in enumerate(merged):
|
||||
frame["index"] = i
|
||||
return merged
|
||||
|
||||
|
||||
def extract_at_timestamps(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
timestamps: list[float],
|
||||
resolution: int = 512,
|
||||
max_frames: int | None = None,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""Grab exactly one frame at each requested timestamp (transcript cues).
|
||||
|
||||
Timestamps are absolute source seconds. Any falling outside an active
|
||||
``[start, end]`` focus window are dropped. Files use a ``cue_*.jpg`` prefix
|
||||
so they sit alongside detail-engine ``frame_*.jpg`` output without either
|
||||
clobbering the other. When more cues than ``max_frames`` survive, they are
|
||||
even-sampled (first + last kept) before extraction.
|
||||
"""
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("cue_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
lo = start_seconds or 0.0
|
||||
hi = end_seconds if end_seconds is not None else float("inf")
|
||||
requested = sorted(set(round(float(t), 2) for t in timestamps))
|
||||
in_window = [t for t in requested if lo <= t <= hi]
|
||||
dropped = len(requested) - len(in_window)
|
||||
|
||||
if max_frames is not None and len(in_window) > max_frames:
|
||||
points = [in_window[i] for i in _even_indices(len(in_window), max_frames)]
|
||||
else:
|
||||
points = in_window
|
||||
|
||||
out: list[dict] = []
|
||||
for t in points:
|
||||
path = out_dir / f"cue_{len(out):04d}.jpg"
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-y",
|
||||
"-ss", f"{t:.3f}",
|
||||
"-i", str(Path(video_path).resolve()),
|
||||
"-frames:v", "1",
|
||||
"-vf", _scale_filter(resolution),
|
||||
"-q:v", "4",
|
||||
str(path),
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode == 0 and path.exists():
|
||||
out.append({
|
||||
"index": len(out),
|
||||
"timestamp_seconds": t,
|
||||
"path": str(path),
|
||||
"reason": "transcript-cue",
|
||||
})
|
||||
|
||||
meta = {
|
||||
"engine": "timestamps",
|
||||
"candidate_count": len(requested),
|
||||
"selected_count": len(out),
|
||||
"dropped_out_of_window": dropped,
|
||||
"fallback": False,
|
||||
}
|
||||
return out, meta
|
||||
|
||||
|
||||
def _even_sample(candidates: list[dict], n: int) -> list[dict]:
|
||||
"""Pick ``n`` evenly-spaced candidates (always including first and last),
|
||||
delete the JPEGs we drop, and reindex the survivors 0..len-1.
|
||||
|
||||
Shared by every capped engine so all detail modes sample the same way:
|
||||
detect all candidates across the full range, then thin down to the cap.
|
||||
``n >= len(candidates)`` keeps everything (the uncapped / under-cap case).
|
||||
"""
|
||||
selected = [candidates[i] for i in _even_indices(len(candidates), n)]
|
||||
|
||||
keep_paths = {sel["path"] for sel in selected}
|
||||
for cand in candidates:
|
||||
if cand["path"] not in keep_paths:
|
||||
try:
|
||||
Path(cand["path"]).unlink()
|
||||
except OSError:
|
||||
pass
|
||||
for i, frame in enumerate(selected):
|
||||
frame["index"] = i
|
||||
return selected
|
||||
|
||||
|
||||
def _frame_delta(a: bytes, b: bytes) -> float:
|
||||
"""Mean absolute per-pixel difference (0-255) between two grayscale
|
||||
thumbnails. Mismatched lengths are treated as maximally different so a
|
||||
decode hiccup never collapses distinct frames."""
|
||||
if not a or len(a) != len(b):
|
||||
return float("inf")
|
||||
return sum(abs(x - y) for x, y in zip(a, b)) / len(a)
|
||||
|
||||
|
||||
def _thumb_frames(paths: list[Path]) -> list[bytes]:
|
||||
"""Decode every frame in ``paths`` to a small grayscale thumbnail via one
|
||||
ffmpeg pass over the JPEG sequence.
|
||||
|
||||
ffmpeg does the pixel decode (keeps us pure-stdlib); we slice the raw
|
||||
grayscale stream into one ``DEDUP_THUMB``-square thumbnail per frame.
|
||||
Fail-open: any ffmpeg error, an unrecognized name, or a byte-count mismatch
|
||||
returns ``[]`` so the caller skips dedup rather than breaking extraction.
|
||||
"""
|
||||
if not paths:
|
||||
return []
|
||||
paths = [Path(p) for p in paths]
|
||||
m = re.match(r"(.*?)(\d+)(\.[A-Za-z0-9]+)$", paths[0].name)
|
||||
if m is None:
|
||||
return []
|
||||
prefix, digits, ext = m.group(1), m.group(2), m.group(3)
|
||||
pattern = str(paths[0].parent / f"{prefix}%0{len(digits)}d{ext}")
|
||||
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-start_number", str(int(digits)),
|
||||
"-i", pattern,
|
||||
"-vf", f"scale={DEDUP_THUMB}:{DEDUP_THUMB},format=gray",
|
||||
"-f", "rawvideo",
|
||||
"-",
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True)
|
||||
if result.returncode != 0:
|
||||
return []
|
||||
|
||||
chunk = DEDUP_THUMB * DEDUP_THUMB
|
||||
data = result.stdout
|
||||
if len(data) != chunk * len(paths):
|
||||
return []
|
||||
return [data[i * chunk:(i + 1) * chunk] for i in range(len(paths))]
|
||||
|
||||
|
||||
def dedupe_perceptual(
|
||||
candidates: list[dict], threshold: float = DEDUP_THRESHOLD
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Drop near-identical frames from a chronological candidate list.
|
||||
|
||||
Thumbnails the extracted JPEGs and greedily removes frames whose mean
|
||||
per-pixel difference from the last kept one is within ``threshold``. Returns
|
||||
``(survivors, dropped_count)``; a no-op (unchanged list) when thumbnails are
|
||||
unavailable or there are fewer than two candidates.
|
||||
"""
|
||||
if len(candidates) <= 1:
|
||||
return candidates, 0
|
||||
thumbs = _thumb_frames([Path(c["path"]) for c in candidates])
|
||||
return _dedupe_by_deltas(candidates, thumbs, threshold)
|
||||
|
||||
|
||||
def _dedupe_by_deltas(
|
||||
candidates: list[dict], thumbs: list[bytes], threshold: float = DEDUP_THRESHOLD
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Greedily drop frames within ``threshold`` mean per-pixel difference of the
|
||||
last *kept* frame. Deletes dropped JPEGs and reindexes survivors 0..n-1 (same
|
||||
cleanup contract as :func:`_even_sample`). Fail-open: if ``thumbs`` does not
|
||||
line up 1:1 with ``candidates``, return them unchanged.
|
||||
"""
|
||||
if len(thumbs) != len(candidates) or len(candidates) <= 1:
|
||||
return candidates, 0
|
||||
|
||||
kept = [candidates[0]]
|
||||
last = thumbs[0]
|
||||
dropped: list[dict] = []
|
||||
for cand, thumb in zip(candidates[1:], thumbs[1:]):
|
||||
if _frame_delta(thumb, last) <= threshold:
|
||||
dropped.append(cand)
|
||||
else:
|
||||
kept.append(cand)
|
||||
last = thumb
|
||||
|
||||
for cand in dropped:
|
||||
try:
|
||||
Path(cand["path"]).unlink()
|
||||
except OSError:
|
||||
pass
|
||||
for i, frame in enumerate(kept):
|
||||
frame["index"] = i
|
||||
return kept, len(dropped)
|
||||
|
||||
|
||||
def extract_scene_or_uniform(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
fps: float,
|
||||
target_frames: int,
|
||||
resolution: int = 512,
|
||||
max_frames: int | None = 100,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
dedup: bool = True,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""Prefer scene selection, falling back to uniform only when the video is
|
||||
effectively static (fewer than ``SCENE_MIN_FRAMES`` detected shots).
|
||||
|
||||
Scene cuts are detected across the *whole* range (uncapped), near-identical
|
||||
frames are dropped (:func:`dedupe_perceptual`, unless ``dedup`` is False),
|
||||
and the survivors are even-sampled down to ``max_frames`` via
|
||||
:func:`_even_sample`, exactly like the keyframe engine. This costs a full
|
||||
decode, but it guarantees coverage spans the entire clip — capping detection
|
||||
with ``-frames:v`` instead would keep only the first ``max_frames`` cuts and
|
||||
drop the tail of long videos (and could even fall below ``SCENE_MIN_FRAMES``
|
||||
and misfire the uniform fallback on a cut-heavy clip).
|
||||
"""
|
||||
scene_frames = extract_scene_candidates(
|
||||
video_path,
|
||||
out_dir,
|
||||
resolution=resolution,
|
||||
max_frames=None,
|
||||
start_seconds=start_seconds,
|
||||
end_seconds=end_seconds,
|
||||
)
|
||||
scene_count = len(scene_frames)
|
||||
if scene_count >= SCENE_MIN_FRAMES:
|
||||
deduped, n_dropped = dedupe_perceptual(scene_frames) if dedup else (scene_frames, 0)
|
||||
cap = len(deduped) if max_frames is None else max_frames
|
||||
selected = _even_sample(deduped, cap)
|
||||
return selected, {
|
||||
"engine": "scene",
|
||||
"candidate_count": scene_count,
|
||||
"deduped_count": n_dropped,
|
||||
"selected_count": len(selected),
|
||||
"fallback": False,
|
||||
}
|
||||
|
||||
fallback_cap = target_frames if max_frames is None else min(max_frames, target_frames)
|
||||
frames = extract(
|
||||
video_path,
|
||||
out_dir,
|
||||
fps=fps,
|
||||
resolution=resolution,
|
||||
max_frames=fallback_cap,
|
||||
start_seconds=start_seconds,
|
||||
end_seconds=end_seconds,
|
||||
)
|
||||
n_dropped = 0
|
||||
if dedup:
|
||||
frames, n_dropped = dedupe_perceptual(frames)
|
||||
return frames, {
|
||||
"engine": "uniform",
|
||||
"candidate_count": scene_count,
|
||||
"deduped_count": n_dropped,
|
||||
"selected_count": len(frames),
|
||||
"fallback": True,
|
||||
}
|
||||
|
||||
|
||||
def extract_keyframes(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
resolution: int = 512,
|
||||
max_frames: int | None = 50,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
dedup: bool = True,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""Decode only keyframes (I-frames) — the cheap, near-instant tier.
|
||||
|
||||
``-skip_frame nokey`` makes ffmpeg reconstruct only keyframes, skipping all
|
||||
P/B frames. Encoders emit keyframes at scene cuts, so these already
|
||||
approximate "distinct moments". Near-identical frames are dropped
|
||||
(:func:`dedupe_perceptual`, unless ``dedup`` is False); over-cap →
|
||||
even-sample first→last; too few keyframes → uniform fallback.
|
||||
"""
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("frame_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
output_pattern = str(out_dir / "frame_%04d.jpg")
|
||||
cmd: list[str] = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "info",
|
||||
"-y",
|
||||
]
|
||||
if start_seconds is not None:
|
||||
cmd += ["-ss", f"{start_seconds:.3f}"]
|
||||
if end_seconds is not None:
|
||||
cmd += ["-to", f"{end_seconds:.3f}"]
|
||||
cmd += [
|
||||
"-skip_frame", "nokey",
|
||||
"-i", str(Path(video_path).resolve()),
|
||||
"-vf", f"{_scale_filter(resolution)},showinfo",
|
||||
"-vsync", "vfr",
|
||||
"-q:v", "4",
|
||||
output_pattern,
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffmpeg keyframe extraction failed: {result.stderr.strip()}")
|
||||
|
||||
offset = start_seconds or 0.0
|
||||
timestamps = [round(offset + float(m.group(1)), 2) for m in SHOWINFO_TS_RE.finditer(result.stderr)]
|
||||
files = sorted(out_dir.glob("frame_*.jpg"))
|
||||
candidates: list[dict] = []
|
||||
for i, path in enumerate(files):
|
||||
ts = timestamps[i] if i < len(timestamps) else offset
|
||||
candidates.append({
|
||||
"index": i,
|
||||
"timestamp_seconds": ts,
|
||||
"path": str(path),
|
||||
"reason": "keyframe",
|
||||
})
|
||||
|
||||
# Too few keyframes → uniform fallback over the same range.
|
||||
if len(candidates) < KEYFRAME_MIN:
|
||||
for cand in candidates:
|
||||
try:
|
||||
Path(cand["path"]).unlink()
|
||||
except OSError:
|
||||
pass
|
||||
meta = get_metadata(video_path)
|
||||
full_duration = meta["duration_seconds"]
|
||||
eff_start = start_seconds or 0.0
|
||||
eff_end = end_seconds if end_seconds is not None else full_duration
|
||||
eff_duration = max(0.0, eff_end - eff_start)
|
||||
budget = max_frames if max_frames is not None else 100
|
||||
fps, _ = auto_fps(eff_duration, max_frames=budget)
|
||||
frames_out = extract(
|
||||
video_path,
|
||||
out_dir,
|
||||
fps=fps,
|
||||
resolution=resolution,
|
||||
max_frames=budget,
|
||||
start_seconds=start_seconds,
|
||||
end_seconds=end_seconds,
|
||||
)
|
||||
n_dropped = 0
|
||||
if dedup:
|
||||
frames_out, n_dropped = dedupe_perceptual(frames_out)
|
||||
return frames_out, {
|
||||
"engine": "uniform",
|
||||
"candidate_count": len(candidates),
|
||||
"deduped_count": n_dropped,
|
||||
"selected_count": len(frames_out),
|
||||
"fallback": True,
|
||||
}
|
||||
|
||||
# Detect-all, drop near-duplicates, then even-sample down to the cap (first +
|
||||
# last always kept). ``max_frames is None`` (uncapped) keeps every keyframe.
|
||||
candidate_count = len(candidates)
|
||||
deduped, n_dropped = dedupe_perceptual(candidates) if dedup else (candidates, 0)
|
||||
cap = len(deduped) if max_frames is None else max_frames
|
||||
selected = _even_sample(deduped, cap)
|
||||
return selected, {
|
||||
"engine": "keyframe",
|
||||
"candidate_count": candidate_count,
|
||||
"deduped_count": n_dropped,
|
||||
"selected_count": len(selected),
|
||||
"fallback": False,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
print(
|
||||
"usage: frames.py <video-path> <out-dir> [--fps F] [--resolution W] "
|
||||
"[--max-frames N] [--start T] [--end T] [--no-dedup]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(2)
|
||||
|
||||
video = sys.argv[1]
|
||||
out = Path(sys.argv[2])
|
||||
args = sys.argv[3:]
|
||||
|
||||
fps_override = None
|
||||
resolution = 512
|
||||
max_frames = 100
|
||||
start_arg = None
|
||||
end_arg = None
|
||||
dedup = True
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == "--fps":
|
||||
fps_override = float(args[i + 1]); i += 2
|
||||
elif args[i] == "--resolution":
|
||||
resolution = int(args[i + 1]); i += 2
|
||||
elif args[i] == "--max-frames":
|
||||
max_frames = int(args[i + 1]); i += 2
|
||||
elif args[i] == "--start":
|
||||
start_arg = args[i + 1]; i += 2
|
||||
elif args[i] == "--end":
|
||||
end_arg = args[i + 1]; i += 2
|
||||
elif args[i] == "--no-dedup":
|
||||
dedup = False; i += 1
|
||||
else:
|
||||
i += 1
|
||||
|
||||
meta = get_metadata(video)
|
||||
start_sec = parse_time(start_arg)
|
||||
end_sec = parse_time(end_arg)
|
||||
full_duration = meta["duration_seconds"]
|
||||
|
||||
effective_start = start_sec if start_sec is not None else 0.0
|
||||
effective_end = end_sec if end_sec is not None else full_duration
|
||||
effective_duration = max(0.0, effective_end - effective_start)
|
||||
|
||||
focused = start_sec is not None or end_sec is not None
|
||||
if focused:
|
||||
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
|
||||
else:
|
||||
fps, target = auto_fps(effective_duration, max_frames=max_frames)
|
||||
if fps_override is not None:
|
||||
fps = fps_override
|
||||
target = max(1, int(round(fps * effective_duration)))
|
||||
|
||||
frames = extract(
|
||||
video, out,
|
||||
fps=fps,
|
||||
resolution=resolution,
|
||||
max_frames=max_frames,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
deduped_count = 0
|
||||
if dedup:
|
||||
frames, deduped_count = dedupe_perceptual(frames)
|
||||
print(json.dumps(
|
||||
{
|
||||
"meta": meta, "fps": fps, "target": target, "focused": focused,
|
||||
"deduped_count": deduped_count, "frames": frames,
|
||||
},
|
||||
indent=2,
|
||||
))
|
||||
@@ -26,6 +26,11 @@ import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
from config import get_config # noqa: E402
|
||||
|
||||
|
||||
REQUIRED_BINARIES = ["ffmpeg", "ffprobe", "yt-dlp"]
|
||||
CONFIG_DIR = Path.home() / ".config" / "watch"
|
||||
@@ -46,6 +51,11 @@ ENV_TEMPLATE = """# /watch API configuration
|
||||
|
||||
GROQ_API_KEY=
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# Default watch behavior (the /watch first-run wizard sets this for you).
|
||||
# Allowed values: transcript | efficient | balanced | token-burner
|
||||
# Keep the value on its own line with no trailing comment.
|
||||
# WATCH_DETAIL=balanced
|
||||
"""
|
||||
|
||||
|
||||
@@ -57,11 +67,19 @@ def _check_binaries() -> list[str]:
|
||||
return [b for b in REQUIRED_BINARIES if not _which(b)]
|
||||
|
||||
|
||||
_PERM_WARNED: set[str] = set()
|
||||
|
||||
|
||||
def _check_file_permissions(path: Path) -> None:
|
||||
"""Warn to stderr if a secrets file is world/group readable."""
|
||||
"""Warn to stderr (once per path per process) if a secrets file is
|
||||
world/group readable."""
|
||||
key = str(path)
|
||||
if key in _PERM_WARNED:
|
||||
return
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
if mode & 0o044:
|
||||
_PERM_WARNED.add(key)
|
||||
sys.stderr.write(
|
||||
f"[watch] WARNING: {path} is readable by other users. "
|
||||
f"Run: chmod 600 {path}\n"
|
||||
@@ -79,7 +97,7 @@ def _read_env_key(name: str) -> str | None:
|
||||
return None
|
||||
_check_file_permissions(CONFIG_FILE)
|
||||
try:
|
||||
for line in CONFIG_FILE.read_text().splitlines():
|
||||
for line in CONFIG_FILE.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
@@ -113,7 +131,7 @@ def _scaffold_env() -> bool:
|
||||
if CONFIG_FILE.exists():
|
||||
return False
|
||||
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
CONFIG_FILE.write_text(ENV_TEMPLATE)
|
||||
CONFIG_FILE.write_text(ENV_TEMPLATE, encoding="utf-8")
|
||||
try:
|
||||
CONFIG_FILE.chmod(0o600)
|
||||
except OSError:
|
||||
@@ -130,15 +148,15 @@ def _write_setup_complete() -> None:
|
||||
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
existing = ""
|
||||
if CONFIG_FILE.exists():
|
||||
existing = CONFIG_FILE.read_text()
|
||||
existing = CONFIG_FILE.read_text(encoding="utf-8")
|
||||
for line in existing.splitlines():
|
||||
if line.strip().startswith("SETUP_COMPLETE="):
|
||||
return
|
||||
if existing and not existing.endswith("\n"):
|
||||
existing += "\n"
|
||||
CONFIG_FILE.write_text(existing + "SETUP_COMPLETE=true\n")
|
||||
CONFIG_FILE.write_text(existing + "SETUP_COMPLETE=true\n", encoding="utf-8")
|
||||
else:
|
||||
CONFIG_FILE.write_text(ENV_TEMPLATE + "\nSETUP_COMPLETE=true\n")
|
||||
CONFIG_FILE.write_text(ENV_TEMPLATE + "\nSETUP_COMPLETE=true\n", encoding="utf-8")
|
||||
try:
|
||||
CONFIG_FILE.chmod(0o600)
|
||||
except OSError:
|
||||
@@ -186,10 +204,31 @@ def _install_hint_linux(missing: list[str]) -> str:
|
||||
return "\n ".join(hints) if hints else "nothing to install"
|
||||
|
||||
|
||||
def _install_hint_windows(missing: list[str]) -> str:
|
||||
pkgs = _brew_pkg(missing)
|
||||
hints = []
|
||||
if "ffmpeg" in pkgs:
|
||||
hints.append("winget: `winget install Gyan.FFmpeg`")
|
||||
if "yt-dlp" in pkgs:
|
||||
hints.append("winget: `winget install yt-dlp.yt-dlp` or pip: `pip install --user yt-dlp`")
|
||||
return "\n ".join(hints) if hints else "nothing to install"
|
||||
|
||||
|
||||
def _status() -> dict:
|
||||
"""Structured preflight snapshot."""
|
||||
"""Structured preflight snapshot.
|
||||
|
||||
`status` describes the *ideal* state (a Whisper key is encouraged), so a
|
||||
keyless install still reports `needs_key` on the very first run — that's
|
||||
the agent's cue to encourage adding one.
|
||||
|
||||
`can_proceed` is the operational gate: /watch can run as long as the
|
||||
binaries are present AND the user has either set a key or already finished
|
||||
setup (consciously opting out of Whisper). A keyless user who completed
|
||||
setup is NOT nagged on every call.
|
||||
"""
|
||||
missing = _check_binaries()
|
||||
has_key, backend = _have_api_key()
|
||||
setup_complete = not is_first_run()
|
||||
|
||||
if not missing and has_key:
|
||||
status = "ready"
|
||||
@@ -200,13 +239,19 @@ def _status() -> dict:
|
||||
else:
|
||||
status = "needs_key"
|
||||
|
||||
can_proceed = (not missing) and (has_key or setup_complete)
|
||||
|
||||
cfg = get_config()
|
||||
return {
|
||||
"status": status,
|
||||
"first_run": is_first_run(),
|
||||
"can_proceed": can_proceed,
|
||||
"first_run": not setup_complete,
|
||||
"setup_complete": setup_complete,
|
||||
"missing_binaries": missing,
|
||||
"whisper_backend": backend,
|
||||
"has_api_key": has_key,
|
||||
"config_file": str(CONFIG_FILE),
|
||||
"watch_detail": cfg["detail"],
|
||||
"platform": platform.system(),
|
||||
}
|
||||
|
||||
@@ -214,20 +259,23 @@ def _status() -> dict:
|
||||
def cmd_check() -> int:
|
||||
"""Silent-on-success preflight.
|
||||
|
||||
Exit 0 with no output when ready. On failure, print one actionable line
|
||||
to stderr and return:
|
||||
Exit 0 with no output when /watch can run. A keyless user who already
|
||||
finished setup (SETUP_COMPLETE=true) counts as ready — Whisper is
|
||||
encouraged, not required — so they are never nagged on follow-up calls.
|
||||
|
||||
On a state that blocks /watch, print one actionable line to stderr:
|
||||
2 → binaries missing
|
||||
3 → API key missing
|
||||
3 → genuine first run with no API key (encourage one)
|
||||
4 → both missing
|
||||
"""
|
||||
s = _status()
|
||||
if s["status"] == "ready":
|
||||
if s["can_proceed"]:
|
||||
return 0
|
||||
|
||||
parts = []
|
||||
if s["missing_binaries"]:
|
||||
parts.append(f"missing binaries: {', '.join(s['missing_binaries'])}")
|
||||
if not s["has_api_key"]:
|
||||
if not s["has_api_key"] and not s["setup_complete"]:
|
||||
parts.append("no Whisper API key (GROQ_API_KEY or OPENAI_API_KEY)")
|
||||
installer = Path(__file__).resolve()
|
||||
sys.stderr.write(
|
||||
@@ -268,6 +316,10 @@ def cmd_install() -> int:
|
||||
print("[setup] dependencies missing on Linux — please install:", file=sys.stderr)
|
||||
print(" " + _install_hint_linux(missing), file=sys.stderr)
|
||||
return 2
|
||||
elif system == "Windows":
|
||||
print("[setup] dependencies missing on Windows — please install:", file=sys.stderr)
|
||||
print(" " + _install_hint_windows(missing), file=sys.stderr)
|
||||
return 2
|
||||
else:
|
||||
print(f"[setup] unsupported platform ({system}) for auto-install. Install manually:", file=sys.stderr)
|
||||
print(f" missing: {', '.join(missing)}", file=sys.stderr)
|
||||
Executable
+393
@@ -0,0 +1,393 @@
|
||||
#!/usr/bin/env python3
|
||||
"""/watch entry point: download video, extract frames, parse transcript.
|
||||
|
||||
Prints a markdown report to stdout listing frame paths + transcript. Claude
|
||||
then Reads each frame path to see the video.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from config import frame_cap, get_config # noqa: E402
|
||||
from download import download, fetch_captions, is_url # noqa: E402
|
||||
from frames import MAX_FPS, auto_fps, auto_fps_focus, extract_at_timestamps, extract_keyframes, extract_scene_or_uniform, format_time, get_metadata, merge_frames, parse_time, parse_timestamps # noqa: E402
|
||||
from transcribe import filter_range, format_transcript, parse_vtt # noqa: E402
|
||||
from whisper import load_api_key, transcribe_video # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="watch",
|
||||
description="Download a video, extract auto-scaled frames, and surface the transcript.",
|
||||
)
|
||||
ap.add_argument("source", help="Video URL or local file path")
|
||||
ap.add_argument("--max-frames", type=int, default=None, help="Override frame cap")
|
||||
ap.add_argument("--resolution", type=int, default=512, help="Frame width in pixels (default 512)")
|
||||
ap.add_argument("--fps", type=float, default=None, help="Override auto-fps")
|
||||
ap.add_argument(
|
||||
"--detail",
|
||||
choices=["transcript", "efficient", "balanced", "token-burner"],
|
||||
default=None,
|
||||
help="Fidelity/speed dial: transcript (no frames), efficient (fast keyframes, cap 50), "
|
||||
"balanced (scene, cap 100), token-burner (scene, uncapped).",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--timestamps",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Comma-separated absolute timestamps (SS, MM:SS, HH:MM:SS) to grab a frame at, "
|
||||
"e.g. transcript-flagged 'look here' moments. Added on top of the detail frames "
|
||||
"(reserved against the cap); with --detail transcript these become the only frames.",
|
||||
)
|
||||
ap.add_argument("--start", type=str, default=None, help="Range start (SS, MM:SS, or HH:MM:SS)")
|
||||
ap.add_argument("--end", type=str, default=None, help="Range end (SS, MM:SS, or HH:MM:SS)")
|
||||
ap.add_argument("--out-dir", type=str, default=None, help="Working directory (default: tmp)")
|
||||
ap.add_argument(
|
||||
"--no-whisper",
|
||||
action="store_true",
|
||||
help="Disable Whisper fallback. Report frames-only if no captions available.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--whisper",
|
||||
choices=["groq", "openai"],
|
||||
default=None,
|
||||
help="Force a specific Whisper backend. Default: prefer Groq, fall back to OpenAI.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--no-dedup",
|
||||
action="store_true",
|
||||
help="Disable near-duplicate frame removal. Keeps visually identical "
|
||||
"frames (static screen recordings, held slides) instead of collapsing them.",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
config = get_config()
|
||||
detail = args.detail or str(config["detail"])
|
||||
configured_cap = frame_cap(detail)
|
||||
if args.max_frames is not None:
|
||||
max_frames = args.max_frames
|
||||
else:
|
||||
max_frames = configured_cap
|
||||
if max_frames is not None and max_frames < 1:
|
||||
raise SystemExit("--max-frames must be greater than zero")
|
||||
budget_cap = max_frames if max_frames is not None else 100
|
||||
cue_timestamps = parse_timestamps(args.timestamps)
|
||||
|
||||
if args.out_dir:
|
||||
work = Path(args.out_dir).expanduser().resolve()
|
||||
else:
|
||||
work = Path(tempfile.mkdtemp(prefix="watch-"))
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
print(f"[watch] working dir: {work}", file=sys.stderr)
|
||||
|
||||
url_source = is_url(args.source)
|
||||
dl: dict = {"subtitle_path": None, "info": {}, "downloaded": False}
|
||||
transcript_segments: list[dict] = []
|
||||
transcript_text: str | None = None
|
||||
transcript_source: str | None = None
|
||||
video_path: str | None = None
|
||||
|
||||
if url_source:
|
||||
print("[watch] checking metadata/captions via yt-dlp…", file=sys.stderr)
|
||||
dl = fetch_captions(args.source, work / "download")
|
||||
if dl.get("subtitle_path"):
|
||||
try:
|
||||
transcript_segments = parse_vtt(dl["subtitle_path"])
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = "captions"
|
||||
except Exception as exc:
|
||||
print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr)
|
||||
transcript_segments = []
|
||||
|
||||
# --timestamps needs the video for frame grabs, so it overrides the
|
||||
# transcript-mode download skip (and forces a full, not audio-only, fetch).
|
||||
audio_only = detail == "transcript" and not cue_timestamps
|
||||
if detail == "transcript" and transcript_segments and not cue_timestamps:
|
||||
video_path = None
|
||||
else:
|
||||
if url_source:
|
||||
print(
|
||||
"[watch] downloading audio via yt-dlp…" if audio_only
|
||||
else "[watch] downloading video via yt-dlp…",
|
||||
file=sys.stderr,
|
||||
)
|
||||
dl = download(
|
||||
args.source,
|
||||
work / "download",
|
||||
audio_only=audio_only,
|
||||
)
|
||||
else:
|
||||
print("[watch] using local file…", file=sys.stderr)
|
||||
dl = download(args.source, work / "download")
|
||||
video_path = dl["video_path"]
|
||||
|
||||
meta = get_metadata(video_path) if video_path else {
|
||||
"duration_seconds": float((dl.get("info") or {}).get("duration") or 0),
|
||||
"width": None,
|
||||
"height": None,
|
||||
"codec": None,
|
||||
"has_audio": False,
|
||||
}
|
||||
full_duration = meta["duration_seconds"]
|
||||
|
||||
start_sec = parse_time(args.start)
|
||||
end_sec = parse_time(args.end)
|
||||
|
||||
if start_sec is not None and start_sec < 0:
|
||||
raise SystemExit("--start must be non-negative")
|
||||
if end_sec is not None and start_sec is not None and end_sec <= start_sec:
|
||||
raise SystemExit("--end must be greater than --start")
|
||||
if full_duration > 0 and start_sec is not None and start_sec >= full_duration:
|
||||
raise SystemExit(f"--start {start_sec:.1f}s is past end of video ({full_duration:.1f}s)")
|
||||
|
||||
effective_start = start_sec if start_sec is not None else 0.0
|
||||
effective_end = end_sec if end_sec is not None else full_duration
|
||||
effective_duration = max(0.0, effective_end - effective_start)
|
||||
focused = start_sec is not None or end_sec is not None
|
||||
|
||||
if focused:
|
||||
fps, target = auto_fps_focus(effective_duration, max_frames=budget_cap)
|
||||
else:
|
||||
fps, target = auto_fps(effective_duration, max_frames=budget_cap)
|
||||
if args.fps is not None:
|
||||
fps = min(args.fps, MAX_FPS)
|
||||
target = max(1, int(round(fps * effective_duration)))
|
||||
|
||||
if transcript_segments and focused:
|
||||
transcript_segments = filter_range(transcript_segments, start_sec, end_sec)
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
|
||||
scope = (
|
||||
f"{format_time(effective_start)}-{format_time(effective_end)} ({effective_duration:.1f}s)"
|
||||
if focused else f"full {effective_duration:.1f}s"
|
||||
)
|
||||
frames: list[dict] = []
|
||||
frame_meta: dict = {"engine": "none", "candidate_count": 0, "selected_count": 0, "fallback": False}
|
||||
cue_frames: list[dict] = []
|
||||
cue_meta: dict = {}
|
||||
|
||||
# Transcript cues are pinned: extracted first and counted against the cap so
|
||||
# the detail engine never evicts the moments the user explicitly asked for.
|
||||
if cue_timestamps and video_path:
|
||||
cue_frames, cue_meta = extract_at_timestamps(
|
||||
video_path,
|
||||
work / "frames",
|
||||
cue_timestamps,
|
||||
resolution=args.resolution,
|
||||
max_frames=max_frames,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
if cue_meta.get("dropped_out_of_window"):
|
||||
print(
|
||||
f"[watch] {cue_meta['dropped_out_of_window']} cue timestamp(s) outside the "
|
||||
"focus range — dropped",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
detail_budget = max_frames if max_frames is None else max(0, max_frames - len(cue_frames))
|
||||
if detail != "transcript" and video_path and detail_budget != 0:
|
||||
cap_label = "unlimited" if detail_budget is None else str(detail_budget)
|
||||
engine_label = "keyframes" if detail == "efficient" else "scene-aware frames"
|
||||
print(
|
||||
f"[watch] extracting {engine_label} over {scope} "
|
||||
f"(target {target}, cap {cap_label})…",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if detail == "efficient":
|
||||
frames, frame_meta = extract_keyframes(
|
||||
video_path,
|
||||
work / "frames",
|
||||
resolution=args.resolution,
|
||||
max_frames=detail_budget,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
dedup=not args.no_dedup,
|
||||
)
|
||||
else: # balanced, token-burner
|
||||
frames, frame_meta = extract_scene_or_uniform(
|
||||
video_path,
|
||||
work / "frames",
|
||||
fps=fps,
|
||||
target_frames=target,
|
||||
resolution=args.resolution,
|
||||
max_frames=detail_budget,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
dedup=not args.no_dedup,
|
||||
)
|
||||
|
||||
if cue_frames:
|
||||
frames = merge_frames(frames, cue_frames)
|
||||
|
||||
if not transcript_segments and dl.get("subtitle_path"):
|
||||
try:
|
||||
all_segments = parse_vtt(dl["subtitle_path"])
|
||||
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = "captions"
|
||||
except Exception as exc:
|
||||
print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr)
|
||||
|
||||
if not transcript_segments and not args.no_whisper and video_path and meta.get("has_audio"):
|
||||
backend, api_key = load_api_key(args.whisper)
|
||||
if backend and api_key:
|
||||
try:
|
||||
all_segments, used_backend = transcribe_video(
|
||||
video_path,
|
||||
work / "audio.mp3",
|
||||
backend=backend,
|
||||
api_key=api_key,
|
||||
)
|
||||
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = f"whisper ({used_backend})"
|
||||
except SystemExit as exc:
|
||||
print(f"[watch] whisper fallback failed: {exc}", file=sys.stderr)
|
||||
else:
|
||||
hint = (
|
||||
f"--whisper {args.whisper} was set but the matching API key is missing"
|
||||
if args.whisper else
|
||||
"no subtitles and no Whisper API key found"
|
||||
)
|
||||
setup_py = SCRIPT_DIR / "setup.py"
|
||||
print(
|
||||
f"[watch] {hint} — run `python3 {setup_py}` to enable the Whisper fallback",
|
||||
file=sys.stderr,
|
||||
)
|
||||
elif not transcript_segments and video_path and not meta.get("has_audio"):
|
||||
print("[watch] no audio stream found — proceeding without transcription", file=sys.stderr)
|
||||
|
||||
info = dl.get("info") or {}
|
||||
|
||||
print()
|
||||
print("# watch: video report")
|
||||
print()
|
||||
print(f"- **Source:** {args.source}")
|
||||
if info.get("title"):
|
||||
print(f"- **Title:** {info['title']}")
|
||||
if info.get("uploader"):
|
||||
print(f"- **Uploader:** {info['uploader']}")
|
||||
print(f"- **Duration:** {format_time(full_duration)} ({full_duration:.1f}s)")
|
||||
if focused:
|
||||
print(
|
||||
f"- **Focus range:** {format_time(effective_start)} → {format_time(effective_end)} "
|
||||
f"({effective_duration:.1f}s)"
|
||||
)
|
||||
if meta.get("width") and meta.get("height"):
|
||||
print(f"- **Resolution:** {meta['width']}x{meta['height']} ({meta.get('codec') or 'unknown codec'})")
|
||||
range_mode = "focused" if focused else "full"
|
||||
print(f"- **Detail:** {detail}")
|
||||
detail_count = frame_meta.get("selected_count", 0)
|
||||
if detail != "transcript":
|
||||
cap_label = "unlimited" if detail_budget is None else str(detail_budget)
|
||||
engine = frame_meta.get("engine", "scene")
|
||||
fallback = " with uniform fallback" if frame_meta.get("fallback") else ""
|
||||
deduped = frame_meta.get("deduped_count", 0)
|
||||
dedup_note = f", {deduped} near-duplicate{'s' if deduped != 1 else ''} dropped" if deduped else ""
|
||||
print(
|
||||
f"- **Frames:** {detail_count} selected from {frame_meta.get('candidate_count', detail_count)} "
|
||||
f"candidates ({engine}{fallback}{dedup_note}, {range_mode} range, budget {target}, cap {cap_label})"
|
||||
)
|
||||
elif not cue_frames:
|
||||
print("- **Frames:** skipped (transcript detail)")
|
||||
if cue_frames:
|
||||
dropped = cue_meta.get("dropped_out_of_window", 0)
|
||||
drop_note = f", {dropped} dropped outside range" if dropped else ""
|
||||
print(
|
||||
f"- **Cue frames:** {len(cue_frames)} at transcript-flagged timestamps "
|
||||
f"(transcript-cue{drop_note})"
|
||||
)
|
||||
if frames:
|
||||
print(f"- **Frame size:** max {args.resolution}px wide, max 1998px tall")
|
||||
if transcript_segments:
|
||||
in_range = " in range" if focused else ""
|
||||
print(
|
||||
f"- **Transcript:** {len(transcript_segments)} segments{in_range} "
|
||||
f"(via {transcript_source or 'captions'})"
|
||||
)
|
||||
else:
|
||||
print("- **Transcript:** none available")
|
||||
|
||||
if detail == "token-burner" and len(frames) > 250:
|
||||
print()
|
||||
print(
|
||||
f"> **Warning:** token-burner detail selected {len(frames)} frames. "
|
||||
"This may use a large number of image tokens."
|
||||
)
|
||||
|
||||
if not focused and full_duration > 600 and detail not in ("transcript", "token-burner"):
|
||||
mins = int(full_duration // 60)
|
||||
print()
|
||||
print(
|
||||
f"> **Warning:** This is a {mins}-minute video. Frame coverage is sparse at this length "
|
||||
f"under `{detail}` detail — its cap spreads thin across the full clip. For better results, "
|
||||
"re-run with `--start HH:MM:SS --end HH:MM:SS` to zoom into a section, or use "
|
||||
"`--detail token-burner` to keep every scene-change frame across the whole video."
|
||||
)
|
||||
|
||||
print()
|
||||
print("## Frames")
|
||||
print()
|
||||
if frames:
|
||||
print(f"Frames live at: `{work / 'frames'}`")
|
||||
print()
|
||||
print(
|
||||
"**Read each frame path below with the Read tool to view the image.** "
|
||||
"Frames are in chronological order; `t=MM:SS` is the absolute timestamp in the source video."
|
||||
)
|
||||
print()
|
||||
for frame in frames:
|
||||
print(
|
||||
f"- `{frame['path']}` "
|
||||
f"(t={format_time(frame['timestamp_seconds'])}, reason={frame.get('reason', 'selected')})"
|
||||
)
|
||||
else:
|
||||
print("_No frames extracted._")
|
||||
|
||||
print()
|
||||
print("## Transcript")
|
||||
print()
|
||||
if transcript_text:
|
||||
label = transcript_source or "captions"
|
||||
if focused:
|
||||
print(f"_Source: {label}. Filtered to {format_time(effective_start)} → {format_time(effective_end)}:_")
|
||||
else:
|
||||
print(f"_Source: {label}._")
|
||||
print()
|
||||
print("```")
|
||||
print(transcript_text)
|
||||
print("```")
|
||||
elif detail == "transcript":
|
||||
print(
|
||||
"_No transcript available at transcript detail. Captions were missing and Whisper was "
|
||||
"unavailable or failed, so there is no visual fallback here. Re-run with "
|
||||
"`--detail balanced` for frames._"
|
||||
)
|
||||
elif focused and dl.get("subtitle_path"):
|
||||
print(f"_No transcript lines fell inside {format_time(effective_start)} → {format_time(effective_end)}._")
|
||||
else:
|
||||
setup_py = SCRIPT_DIR / "setup.py"
|
||||
print(
|
||||
"_No transcript available — proceed with frames only. "
|
||||
"Captions were missing and the Whisper fallback was unavailable "
|
||||
"(no API key set, or `--no-whisper` was used). "
|
||||
f"Run `python3 {setup_py}` to enable Whisper, then re-run._"
|
||||
)
|
||||
|
||||
print()
|
||||
print("---")
|
||||
print(f"_Work dir: `{work}` — delete when done._")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -12,6 +12,7 @@ from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
@@ -31,6 +32,35 @@ GROQ_MODEL = "whisper-large-v3"
|
||||
OPENAI_ENDPOINT = "https://api.openai.com/v1/audio/transcriptions"
|
||||
OPENAI_MODEL = "whisper-1"
|
||||
|
||||
# Both Groq's free tier and OpenAI whisper-1 cap uploads at 25 MB. We target a
|
||||
# margin under that so multipart framing overhead never pushes a chunk over.
|
||||
MAX_UPLOAD_BYTES = 24 * 1024 * 1024
|
||||
|
||||
|
||||
def plan_chunks(
|
||||
total_seconds: float,
|
||||
total_bytes: int,
|
||||
max_bytes: int = MAX_UPLOAD_BYTES,
|
||||
) -> list[tuple[float, float]]:
|
||||
"""Split a duration into contiguous (offset, duration) chunks under max_bytes.
|
||||
|
||||
Size scales linearly with duration (constant-bitrate mono mp3), so an even
|
||||
time split yields evenly-sized chunks. Returns a single full-length chunk
|
||||
when the audio already fits.
|
||||
"""
|
||||
if total_bytes <= max_bytes or total_seconds <= 0:
|
||||
return [(0.0, total_seconds)]
|
||||
|
||||
n = math.ceil(total_bytes / max_bytes)
|
||||
chunk = total_seconds / n
|
||||
plan: list[tuple[float, float]] = []
|
||||
for i in range(n):
|
||||
offset = i * chunk
|
||||
# The last chunk absorbs any rounding remainder so durations sum exactly.
|
||||
duration = (total_seconds - offset) if i == n - 1 else chunk
|
||||
plan.append((round(offset, 3), round(duration, 3)))
|
||||
return plan
|
||||
|
||||
|
||||
def load_api_key(preferred: str | None = None) -> tuple[str, str] | tuple[None, None]:
|
||||
"""Return (backend, api_key). Prefers Groq, falls back to OpenAI.
|
||||
@@ -45,7 +75,7 @@ def load_api_key(preferred: str | None = None) -> tuple[str, str] | tuple[None,
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
for line in path.read_text().splitlines():
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
@@ -93,13 +123,13 @@ def extract_audio(video_path: str, out_path: Path) -> Path:
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-y",
|
||||
"-i", video_path,
|
||||
"-i", str(Path(video_path).resolve()),
|
||||
"-vn",
|
||||
"-acodec", "libmp3lame",
|
||||
"-ar", "16000",
|
||||
"-ac", "1",
|
||||
"-b:a", "64k",
|
||||
str(out_path),
|
||||
str(out_path.resolve()),
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
@@ -109,6 +139,65 @@ def extract_audio(video_path: str, out_path: Path) -> Path:
|
||||
return out_path
|
||||
|
||||
|
||||
def audio_duration(audio_path: Path) -> float:
|
||||
"""Return the duration of an audio file in seconds via ffprobe."""
|
||||
if shutil.which("ffprobe") is None:
|
||||
raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v", "quiet",
|
||||
"-print_format", "json",
|
||||
"-show_format",
|
||||
str(audio_path.resolve()),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffprobe failed: {result.stderr.strip()}")
|
||||
fmt = json.loads(result.stdout or "{}").get("format", {})
|
||||
return float(fmt.get("duration") or 0.0)
|
||||
|
||||
|
||||
def split_audio(
|
||||
full_audio: Path,
|
||||
work_dir: Path,
|
||||
plan: list[tuple[float, float]],
|
||||
) -> list[tuple[Path, float]]:
|
||||
"""Slice full_audio into per-plan chunk files, returning (path, offset) pairs.
|
||||
|
||||
Uses stream copy (`-c copy`) so there is no re-encode and no quality loss;
|
||||
mp3 frame boundaries are close enough for transcription's purposes.
|
||||
"""
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
work_dir.mkdir(parents=True, exist_ok=True)
|
||||
chunks: list[tuple[Path, float]] = []
|
||||
for index, (offset, duration) in enumerate(plan):
|
||||
out_path = work_dir / f"chunk_{index:03d}.mp3"
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-y",
|
||||
"-ss", f"{offset:.3f}",
|
||||
"-i", str(full_audio.resolve()),
|
||||
"-t", f"{duration:.3f}",
|
||||
"-c", "copy",
|
||||
str(out_path.resolve()),
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0 or not out_path.exists() or out_path.stat().st_size == 0:
|
||||
raise SystemExit(
|
||||
f"ffmpeg failed to split audio chunk {index + 1}: {result.stderr.strip()}"
|
||||
)
|
||||
chunks.append((out_path, offset))
|
||||
return chunks
|
||||
|
||||
|
||||
def _build_multipart(fields: dict[str, str], file_path: Path) -> tuple[bytes, str]:
|
||||
"""Assemble a multipart/form-data body the Whisper APIs accept.
|
||||
|
||||
@@ -240,6 +329,24 @@ def _retry_after(exc: urllib.error.HTTPError) -> float | None:
|
||||
return None
|
||||
|
||||
|
||||
def shift_segments(segments: list[dict], offset_seconds: float) -> list[dict]:
|
||||
"""Return a copy of segments with start/end shifted by offset_seconds.
|
||||
|
||||
Each chunk is transcribed in isolation, so Whisper returns 0-based timestamps
|
||||
per chunk; shifting by the chunk's offset stitches them into source time.
|
||||
"""
|
||||
if offset_seconds == 0:
|
||||
return segments
|
||||
return [
|
||||
{
|
||||
"start": round(seg["start"] + offset_seconds, 2),
|
||||
"end": round(seg["end"] + offset_seconds, 2),
|
||||
"text": seg["text"],
|
||||
}
|
||||
for seg in segments
|
||||
]
|
||||
|
||||
|
||||
def _segments_from_response(data: dict) -> list[dict]:
|
||||
"""Convert Whisper verbose_json into our {start, end, text} segment format."""
|
||||
out: list[dict] = []
|
||||
@@ -261,6 +368,49 @@ def _segments_from_response(data: dict) -> list[dict]:
|
||||
return out
|
||||
|
||||
|
||||
def transcribe_chunks(
|
||||
chunks: list[tuple[Path, float]],
|
||||
transcribe_one,
|
||||
) -> list[dict]:
|
||||
"""Transcribe each chunk, shift its segments by the chunk offset, concatenate.
|
||||
|
||||
A chunk that fails after its own retries is logged and skipped so one bad
|
||||
slice doesn't discard the whole transcript. Raises only if every chunk fails.
|
||||
"""
|
||||
segments: list[dict] = []
|
||||
failures = 0
|
||||
for index, (path, offset) in enumerate(chunks):
|
||||
try:
|
||||
chunk_segments = transcribe_one(path)
|
||||
except SystemExit as exc:
|
||||
failures += 1
|
||||
print(
|
||||
f"[watch] chunk {index + 1}/{len(chunks)} failed — skipping ({exc})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
segments.extend(shift_segments(chunk_segments, offset))
|
||||
print(
|
||||
f"[watch] chunk {index + 1}/{len(chunks)} → {len(chunk_segments)} segments",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
if failures == len(chunks):
|
||||
raise SystemExit("Whisper failed on every audio chunk")
|
||||
return segments
|
||||
|
||||
|
||||
def _transcribe_file(backend: str, api_key: str, audio_path: Path) -> list[dict]:
|
||||
"""Upload one audio file and return its 0-based segments."""
|
||||
if backend == "groq":
|
||||
response = _post_whisper(GROQ_ENDPOINT, api_key, GROQ_MODEL, audio_path)
|
||||
elif backend == "openai":
|
||||
response = _post_whisper(OPENAI_ENDPOINT, api_key, OPENAI_MODEL, audio_path)
|
||||
else:
|
||||
raise SystemExit(f"Unknown whisper backend: {backend}")
|
||||
return _segments_from_response(response)
|
||||
|
||||
|
||||
def transcribe_video(
|
||||
video_path: str,
|
||||
audio_out: Path,
|
||||
@@ -286,17 +436,28 @@ def transcribe_video(
|
||||
|
||||
print(f"[watch] extracting audio for Whisper ({backend})…", file=sys.stderr)
|
||||
audio_path = extract_audio(video_path, audio_out)
|
||||
size_kb = audio_path.stat().st_size / 1024
|
||||
print(f"[watch] audio: {size_kb:.0f} kB — uploading to {backend} Whisper…", file=sys.stderr)
|
||||
audio_bytes = audio_path.stat().st_size
|
||||
|
||||
if backend == "groq":
|
||||
response = _post_whisper(GROQ_ENDPOINT, api_key, GROQ_MODEL, audio_path)
|
||||
elif backend == "openai":
|
||||
response = _post_whisper(OPENAI_ENDPOINT, api_key, OPENAI_MODEL, audio_path)
|
||||
def transcribe_one(path: Path) -> list[dict]:
|
||||
return _transcribe_file(backend, api_key, path)
|
||||
|
||||
if audio_bytes <= MAX_UPLOAD_BYTES:
|
||||
print(
|
||||
f"[watch] audio: {audio_bytes / 1024:.0f} kB — uploading to {backend} Whisper…",
|
||||
file=sys.stderr,
|
||||
)
|
||||
segments = transcribe_one(audio_path)
|
||||
else:
|
||||
raise SystemExit(f"Unknown whisper backend: {backend}")
|
||||
duration = audio_duration(audio_path)
|
||||
plan = plan_chunks(duration, audio_bytes, MAX_UPLOAD_BYTES)
|
||||
print(
|
||||
f"[watch] audio: {audio_bytes / (1024 * 1024):.0f} MB exceeds "
|
||||
f"{MAX_UPLOAD_BYTES // (1024 * 1024)} MB — splitting into {len(plan)} chunks…",
|
||||
file=sys.stderr,
|
||||
)
|
||||
chunks = split_audio(audio_path, audio_out.parent / "chunks", plan)
|
||||
segments = transcribe_chunks(chunks, transcribe_one)
|
||||
|
||||
segments = _segments_from_response(response)
|
||||
if not segments:
|
||||
raise SystemExit("Whisper returned no transcript segments")
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Shared pytest fixtures: ffmpeg-synthesized clips and scripts/ on sys.path."""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Make the bundled scripts importable (mirrors watch.py's sys.path insert).
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
# 14 visually distinct fills → 14 abrupt cuts → x264 emits a keyframe per cut.
|
||||
COLORS = [
|
||||
"red", "green", "blue", "white", "black", "yellow", "cyan",
|
||||
"magenta", "gray", "orange", "purple", "brown", "navy", "olive",
|
||||
]
|
||||
|
||||
|
||||
def _run(cmd: list[str]) -> None:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"ffmpeg failed: {' '.join(cmd)}\n{result.stderr}")
|
||||
|
||||
|
||||
def build_cut_clip(
|
||||
path: Path,
|
||||
n: int = 14,
|
||||
seg: float = 0.4,
|
||||
size: str = "320x240",
|
||||
fps: int = 10,
|
||||
) -> None:
|
||||
"""Concatenate ``n`` solid-color segments into one clip with ``n`` cuts.
|
||||
|
||||
Each color change is a hard scene cut, so the scene selector finds ~n-1
|
||||
changes. x264's own scenecut detection is unreliable on flat fills, so we
|
||||
force a keyframe at every ``seg`` boundary — giving ~n real keyframes for
|
||||
the keyframe engine to find.
|
||||
"""
|
||||
inputs: list[str] = []
|
||||
for i in range(n):
|
||||
color = COLORS[i % len(COLORS)]
|
||||
inputs += ["-f", "lavfi", "-t", str(seg), "-i", f"color=c={color}:s={size}:r={fps}"]
|
||||
streams = "".join(f"[{i}:v]" for i in range(n))
|
||||
filt = f"{streams}concat=n={n}:v=1:a=0[out]"
|
||||
_run([
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
*inputs,
|
||||
"-filter_complex", filt, "-map", "[out]",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p",
|
||||
"-force_key_frames", f"expr:gte(t,n_forced*{seg})",
|
||||
str(path),
|
||||
])
|
||||
|
||||
|
||||
def build_static_clip(
|
||||
path: Path,
|
||||
duration: float = 3.0,
|
||||
size: str = "320x240",
|
||||
fps: int = 10,
|
||||
) -> None:
|
||||
"""One solid color: 1 keyframe, no scene changes → triggers both fallbacks."""
|
||||
_run([
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-f", "lavfi", "-t", str(duration), "-i", f"color=c=blue:s={size}:r={fps}",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-g", "600",
|
||||
str(path),
|
||||
])
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def cut_clip(tmp_path_factory: pytest.TempPathFactory) -> Path:
|
||||
path = tmp_path_factory.mktemp("clips") / "cuts.mp4"
|
||||
build_cut_clip(path)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def static_clip(tmp_path_factory: pytest.TempPathFactory) -> Path:
|
||||
path = tmp_path_factory.mktemp("clips") / "static.mp4"
|
||||
build_static_clip(path)
|
||||
return path
|
||||
@@ -0,0 +1,37 @@
|
||||
"""WATCH_DETAIL resolution and frame_cap mapping."""
|
||||
from __future__ import annotations
|
||||
|
||||
import config
|
||||
|
||||
|
||||
def test_default_detail_is_balanced(monkeypatch, tmp_path):
|
||||
monkeypatch.delenv("WATCH_DETAIL", raising=False)
|
||||
monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env")
|
||||
assert config.get_config()["detail"] == "balanced"
|
||||
|
||||
|
||||
def test_env_overrides_detail(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("WATCH_DETAIL", "efficient")
|
||||
monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env")
|
||||
assert config.get_config()["detail"] == "efficient"
|
||||
|
||||
|
||||
def test_invalid_detail_falls_back_to_default(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("WATCH_DETAIL", "bogus")
|
||||
monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env")
|
||||
assert config.get_config()["detail"] == "balanced"
|
||||
|
||||
|
||||
def test_get_config_keys(monkeypatch, tmp_path):
|
||||
monkeypatch.delenv("WATCH_DETAIL", raising=False)
|
||||
monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env")
|
||||
cfg = config.get_config()
|
||||
assert set(cfg) == {"detail", "config_file"}
|
||||
|
||||
|
||||
def test_frame_cap_mapping():
|
||||
assert config.frame_cap("efficient") == 50
|
||||
assert config.frame_cap("balanced") == 100
|
||||
assert config.frame_cap("token-burner") is None
|
||||
assert config.frame_cap("transcript") is None
|
||||
assert config.frame_cap("anything-else") == 100
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Frame-delta dedup: per-pixel difference, greedy de-duplication, integration."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import frames
|
||||
|
||||
|
||||
# --- _frame_delta: mean absolute per-pixel difference ------------------------
|
||||
|
||||
def test_frame_delta_identical_is_zero():
|
||||
a = bytes([10] * 16)
|
||||
assert frames._frame_delta(a, a) == 0.0
|
||||
|
||||
|
||||
def test_frame_delta_is_mean_absolute_difference():
|
||||
a = bytes([0, 0, 0, 0])
|
||||
b = bytes([4, 0, 0, 0])
|
||||
assert frames._frame_delta(a, b) == 1.0 # (4+0+0+0)/4
|
||||
|
||||
|
||||
def test_frame_delta_mismatched_length_is_infinite():
|
||||
assert frames._frame_delta(bytes([1, 2]), bytes([1, 2, 3])) == float("inf")
|
||||
|
||||
|
||||
# --- _dedupe_by_deltas: greedy drop vs last *kept* thumbnail ------------------
|
||||
|
||||
def _touch(dirpath: Path, n: int) -> list[dict]:
|
||||
dirpath.mkdir(parents=True, exist_ok=True)
|
||||
out = []
|
||||
for i in range(n):
|
||||
p = dirpath / f"frame_{i:04d}.jpg"
|
||||
p.write_bytes(b"x")
|
||||
out.append({"index": i, "timestamp_seconds": float(i), "path": str(p), "reason": "scene-change"})
|
||||
return out
|
||||
|
||||
|
||||
FLAT0 = bytes([0, 0, 0, 0])
|
||||
FLAT255 = bytes([255, 255, 255, 255])
|
||||
|
||||
|
||||
def test_dedupe_collapses_identical_run(tmp_path: Path):
|
||||
cands = _touch(tmp_path, 5)
|
||||
thumbs = [FLAT0, FLAT0, FLAT0, FLAT0, FLAT0]
|
||||
survivors, dropped = frames._dedupe_by_deltas(cands, thumbs, threshold=2.0)
|
||||
assert dropped == 4
|
||||
assert len(survivors) == 1
|
||||
assert survivors[0]["index"] == 0
|
||||
assert sorted(p.name for p in tmp_path.glob("frame_*.jpg")) == ["frame_0000.jpg"]
|
||||
|
||||
|
||||
def test_dedupe_keeps_all_distinct(tmp_path: Path):
|
||||
cands = _touch(tmp_path, 4)
|
||||
thumbs = [FLAT0, FLAT255, FLAT0, FLAT255]
|
||||
survivors, dropped = frames._dedupe_by_deltas(cands, thumbs, threshold=2.0)
|
||||
assert dropped == 0
|
||||
assert [s["index"] for s in survivors] == [0, 1, 2, 3]
|
||||
assert len(list(tmp_path.glob("frame_*.jpg"))) == 4
|
||||
|
||||
|
||||
def test_dedupe_compares_against_last_kept_not_previous(tmp_path: Path):
|
||||
"""A,A,B,B,A with A/B far apart -> keep A0, B2, A4 (drops the repeats)."""
|
||||
cands = _touch(tmp_path, 5)
|
||||
survivors, dropped = frames._dedupe_by_deltas(
|
||||
cands, [FLAT0, FLAT0, FLAT255, FLAT255, FLAT0], threshold=2.0
|
||||
)
|
||||
assert [s["index"] for s in survivors] == [0, 1, 2] # reindexed survivors
|
||||
assert dropped == 2
|
||||
|
||||
|
||||
def test_dedupe_threshold_is_inclusive(tmp_path: Path):
|
||||
"""Delta exactly == threshold is treated as a duplicate (<=)."""
|
||||
cands = _touch(tmp_path, 2)
|
||||
a = bytes([0, 0, 0, 0])
|
||||
b = bytes([8, 0, 0, 0]) # mean abs diff == 2.0
|
||||
survivors, dropped = frames._dedupe_by_deltas(cands, [a, b], threshold=2.0)
|
||||
assert dropped == 1
|
||||
assert len(survivors) == 1
|
||||
|
||||
|
||||
def test_dedupe_empty_and_single_are_noops(tmp_path: Path):
|
||||
assert frames._dedupe_by_deltas([], [], threshold=2.0) == ([], 0)
|
||||
one = _touch(tmp_path, 1)
|
||||
survivors, dropped = frames._dedupe_by_deltas(one, [FLAT0], threshold=2.0)
|
||||
assert dropped == 0
|
||||
assert len(survivors) == 1
|
||||
|
||||
|
||||
def test_dedupe_mismatched_thumb_count_is_noop(tmp_path: Path):
|
||||
"""Fail open: if thumbs don't line up with candidates, change nothing."""
|
||||
cands = _touch(tmp_path, 3)
|
||||
survivors, dropped = frames._dedupe_by_deltas(cands, [FLAT0], threshold=2.0)
|
||||
assert dropped == 0
|
||||
assert len(survivors) == 3
|
||||
|
||||
|
||||
# --- _thumb_frames + dedupe_perceptual: real ffmpeg over extracted JPEGs ------
|
||||
|
||||
def test_thumb_frames_match_candidate_count(cut_clip: Path, tmp_path: Path):
|
||||
out = frames.extract_scene_candidates(str(cut_clip), tmp_path / "f", max_frames=None)
|
||||
thumbs = frames._thumb_frames([Path(fr["path"]) for fr in out])
|
||||
assert len(thumbs) == len(out)
|
||||
assert all(len(t) == frames.DEDUP_THUMB * frames.DEDUP_THUMB for t in thumbs)
|
||||
|
||||
|
||||
def test_dedupe_perceptual_collapses_static_clip(static_clip: Path, tmp_path: Path):
|
||||
out = frames.extract(str(static_clip), tmp_path / "f", fps=4.0, max_frames=10)
|
||||
n_before = len(out)
|
||||
survivors, dropped = frames.dedupe_perceptual(out)
|
||||
assert n_before > 1
|
||||
assert len(survivors) == 1
|
||||
assert dropped == n_before - 1
|
||||
assert len(list((tmp_path / "f").glob("frame_*.jpg"))) == 1
|
||||
|
||||
|
||||
def test_dedupe_perceptual_keeps_distinct_cuts(cut_clip: Path, tmp_path: Path):
|
||||
"""Distinct color shots differ in luma, so frame-delta keeps them all."""
|
||||
out = frames.extract_scene_candidates(str(cut_clip), tmp_path / "f", max_frames=None)
|
||||
n_before = len(out)
|
||||
survivors, dropped = frames.dedupe_perceptual(out)
|
||||
assert dropped == 0
|
||||
assert len(survivors) == n_before
|
||||
|
||||
|
||||
# --- engine integration: dedup runs before the cap, reports deduped_count -----
|
||||
|
||||
def test_scene_engine_reports_zero_dedup_on_distinct(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_scene_or_uniform(
|
||||
str(cut_clip), tmp_path / "f", fps=2.0, target_frames=50, max_frames=100,
|
||||
)
|
||||
assert meta["engine"] == "scene"
|
||||
assert meta["deduped_count"] == 0
|
||||
assert len(out) == len(list((tmp_path / "f").glob("frame_*.jpg")))
|
||||
|
||||
|
||||
def test_uniform_fallback_dedupes_static(static_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_scene_or_uniform(
|
||||
str(static_clip), tmp_path / "f", fps=4.0, target_frames=12, max_frames=100,
|
||||
)
|
||||
assert meta["engine"] == "uniform"
|
||||
assert meta["fallback"] is True
|
||||
assert meta["deduped_count"] > 0
|
||||
assert meta["selected_count"] == 1 # identical frames collapse to one
|
||||
assert len(out) == 1
|
||||
assert len(list((tmp_path / "f").glob("frame_*.jpg"))) == 1
|
||||
|
||||
|
||||
def test_keyframe_uniform_fallback_dedupes_static(static_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_keyframes(str(static_clip), tmp_path / "f", max_frames=50)
|
||||
assert meta["engine"] == "uniform"
|
||||
assert meta["deduped_count"] > 0
|
||||
assert len(out) == 1
|
||||
|
||||
|
||||
def test_dedup_false_disables_collapse(static_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_scene_or_uniform(
|
||||
str(static_clip), tmp_path / "f", fps=4.0, target_frames=12, max_frames=100,
|
||||
dedup=False,
|
||||
)
|
||||
assert meta["deduped_count"] == 0
|
||||
assert meta["selected_count"] > 1 # no collapse without dedup
|
||||
assert len(out) > 1
|
||||
@@ -0,0 +1,64 @@
|
||||
"""yt-dlp argv construction for download.py.
|
||||
|
||||
Regression guard: ``--sub-langs all`` makes yt-dlp fetch YouTube's hundreds of
|
||||
auto-translated caption tracks, which can take minutes and stalls before the
|
||||
video download even starts. We only support English, so the request must stay
|
||||
bounded to the English-only pattern.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
import download # noqa: E402
|
||||
|
||||
URL = "https://www.youtube.com/watch?v=rlOpbu3Enkw"
|
||||
|
||||
|
||||
def _capture_argv(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]:
|
||||
"""Stub subprocess.run inside download.py and record every argv."""
|
||||
calls: list[list[str]] = []
|
||||
|
||||
class _Result:
|
||||
returncode = 0
|
||||
stdout = ""
|
||||
stderr = ""
|
||||
|
||||
def fake_run(cmd, *args, **kwargs):
|
||||
calls.append(list(cmd))
|
||||
return _Result()
|
||||
|
||||
monkeypatch.setattr(download.subprocess, "run", fake_run)
|
||||
return calls
|
||||
|
||||
|
||||
def _sub_langs(argv: list[str]) -> str:
|
||||
idx = argv.index("--sub-langs")
|
||||
return argv[idx + 1]
|
||||
|
||||
|
||||
def _assert_english_only(langs: str) -> None:
|
||||
tokens = langs.split(",")
|
||||
assert "all" not in tokens, f"sub-langs must not request all languages, got {langs!r}"
|
||||
assert all(t.startswith("en") for t in tokens), f"sub-langs must be English-only, got {langs!r}"
|
||||
|
||||
|
||||
def test_fetch_captions_requests_english_only(monkeypatch, tmp_path):
|
||||
calls = _capture_argv(monkeypatch)
|
||||
download.fetch_captions(URL, tmp_path / "download")
|
||||
_assert_english_only(_sub_langs(calls[0]))
|
||||
|
||||
|
||||
def test_download_url_requests_english_only(monkeypatch, tmp_path):
|
||||
calls = _capture_argv(monkeypatch)
|
||||
# _pick_video returns None with no real file, which raises SystemExit after
|
||||
# the yt-dlp argv is already built — that's all we need to inspect.
|
||||
with pytest.raises(SystemExit):
|
||||
download.download_url(URL, tmp_path / "download")
|
||||
_assert_english_only(_sub_langs(calls[0]))
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Smoke test: the ffmpeg fixtures actually produce playable clips."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _duration(path: Path) -> float:
|
||||
out = subprocess.run(
|
||||
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", str(path)],
|
||||
capture_output=True, text=True,
|
||||
).stdout
|
||||
return float(json.loads(out)["format"]["duration"])
|
||||
|
||||
|
||||
def test_cut_clip_builds(cut_clip: Path):
|
||||
assert cut_clip.exists() and cut_clip.stat().st_size > 0
|
||||
assert _duration(cut_clip) > 4.0 # 14 * 0.4s ≈ 5.6s
|
||||
|
||||
|
||||
def test_static_clip_builds(static_clip: Path):
|
||||
assert static_clip.exists() and static_clip.stat().st_size > 0
|
||||
assert _duration(static_clip) > 2.0
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Keyframe engine + preserved scene/uniform fallbacks."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import frames
|
||||
|
||||
|
||||
def test_keyframe_engine_on_cut_clip(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_keyframes(str(cut_clip), tmp_path / "f", max_frames=50)
|
||||
assert meta["engine"] == "keyframe"
|
||||
assert meta["fallback"] is False
|
||||
assert len(out) >= frames.KEYFRAME_MIN
|
||||
assert all(fr["reason"] == "keyframe" for fr in out)
|
||||
assert len(out) == len(list((tmp_path / "f").glob("frame_*.jpg")))
|
||||
|
||||
|
||||
def test_keyframe_even_sampling_caps_and_spans(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_keyframes(str(cut_clip), tmp_path / "f", max_frames=5)
|
||||
assert meta["engine"] == "keyframe"
|
||||
assert len(out) == 5
|
||||
assert meta["selected_count"] == 5
|
||||
assert meta["candidate_count"] > 5
|
||||
ts = [fr["timestamp_seconds"] for fr in out]
|
||||
assert ts == sorted(ts)
|
||||
assert ts[0] < ts[-1] # spans first → last keyframe
|
||||
assert [fr["index"] for fr in out] == [0, 1, 2, 3, 4]
|
||||
|
||||
|
||||
def test_keyframe_fallback_on_static_clip(static_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_keyframes(str(static_clip), tmp_path / "f", max_frames=50)
|
||||
assert meta["engine"] == "uniform"
|
||||
assert meta["fallback"] is True
|
||||
assert len(out) > 0
|
||||
assert all(fr["reason"] == "uniform" for fr in out)
|
||||
|
||||
|
||||
def test_scene_engine_on_cut_clip(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_scene_or_uniform(
|
||||
str(cut_clip), tmp_path / "f", fps=2.0, target_frames=50, max_frames=100,
|
||||
)
|
||||
assert meta["engine"] == "scene"
|
||||
assert meta["fallback"] is False
|
||||
assert len(out) >= frames.SCENE_MIN_FRAMES
|
||||
|
||||
|
||||
def test_scene_even_sampling_caps_and_spans(cut_clip: Path, tmp_path: Path):
|
||||
"""Over-cap scene detection must even-sample across the whole clip, not keep
|
||||
the first N cuts and drop the tail (the long-video coverage bug)."""
|
||||
out, meta = frames.extract_scene_or_uniform(
|
||||
str(cut_clip), tmp_path / "f", fps=2.0, target_frames=50, max_frames=5,
|
||||
)
|
||||
assert meta["engine"] == "scene"
|
||||
assert meta["fallback"] is False
|
||||
assert len(out) == 5
|
||||
assert meta["selected_count"] == 5
|
||||
assert meta["candidate_count"] > 5 # all cuts detected, then sampled down
|
||||
ts = [fr["timestamp_seconds"] for fr in out]
|
||||
assert ts == sorted(ts)
|
||||
assert ts[-1] > 4.0 # spans the full ~5.6s clip, not just the first ~1.6s
|
||||
assert len(out) == len(list((tmp_path / "f").glob("frame_*.jpg")))
|
||||
assert [fr["index"] for fr in out] == [0, 1, 2, 3, 4]
|
||||
|
||||
|
||||
def test_scene_fallback_on_static_clip(static_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_scene_or_uniform(
|
||||
str(static_clip), tmp_path / "f", fps=2.0, target_frames=12, max_frames=100,
|
||||
)
|
||||
assert meta["engine"] == "uniform"
|
||||
assert meta["fallback"] is True
|
||||
@@ -0,0 +1,80 @@
|
||||
"""setup.py --json surfaces the resolved watch detail."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SETUP = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts" / "setup.py"
|
||||
|
||||
|
||||
def _run(args, *, home=None, extra_env=None):
|
||||
env = dict(os.environ)
|
||||
env.pop("WATCH_DETAIL", None)
|
||||
# Don't let a real key in the developer's shell env leak into the test.
|
||||
env.pop("GROQ_API_KEY", None)
|
||||
env.pop("OPENAI_API_KEY", None)
|
||||
env.pop("SETUP_COMPLETE", None)
|
||||
if home is not None:
|
||||
env["HOME"] = str(home)
|
||||
env["USERPROFILE"] = str(home) # Windows
|
||||
if extra_env:
|
||||
env.update(extra_env)
|
||||
return subprocess.run(
|
||||
[sys.executable, str(SETUP), *args],
|
||||
capture_output=True, text=True, env=env,
|
||||
)
|
||||
|
||||
|
||||
def _write_env(home: Path, body: str) -> None:
|
||||
cfg = home / ".config" / "watch"
|
||||
cfg.mkdir(parents=True, exist_ok=True)
|
||||
f = cfg / ".env"
|
||||
f.write_text(body, encoding="utf-8")
|
||||
f.chmod(0o600)
|
||||
|
||||
|
||||
def test_json_reports_watch_detail():
|
||||
proc = _run(["--json"])
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
data = json.loads(proc.stdout)
|
||||
assert data["watch_detail"] == "balanced"
|
||||
|
||||
|
||||
def test_keyless_completed_setup_proceeds_silently(tmp_path):
|
||||
"""A user who finished setup without a key must NOT be nagged forever."""
|
||||
_write_env(tmp_path, "GROQ_API_KEY=\nOPENAI_API_KEY=\nSETUP_COMPLETE=true\n")
|
||||
chk = _run(["--check"], home=tmp_path)
|
||||
assert chk.returncode == 0, f"keyless-complete should pass --check; got {chk.returncode}: {chk.stderr}"
|
||||
assert chk.stdout == "" and chk.stderr == ""
|
||||
|
||||
js = json.loads(_run(["--json"], home=tmp_path).stdout)
|
||||
assert js["can_proceed"] is True
|
||||
assert js["first_run"] is False
|
||||
assert js["setup_complete"] is True
|
||||
# status still encourages a key even though we can proceed
|
||||
assert js["status"] == "needs_key"
|
||||
|
||||
|
||||
def test_keyless_first_run_is_encouraged(tmp_path):
|
||||
"""Genuine first run with no key: --check reports exit 3 (encourage a key)."""
|
||||
_write_env(tmp_path, "GROQ_API_KEY=\nOPENAI_API_KEY=\n")
|
||||
chk = _run(["--check"], home=tmp_path)
|
||||
assert chk.returncode == 3, chk.stderr
|
||||
|
||||
js = json.loads(_run(["--json"], home=tmp_path).stdout)
|
||||
assert js["can_proceed"] is False
|
||||
assert js["first_run"] is True
|
||||
|
||||
|
||||
def test_key_present_is_ready(tmp_path):
|
||||
_write_env(tmp_path, "GROQ_API_KEY=sk-test-abc\n")
|
||||
chk = _run(["--check"], home=tmp_path)
|
||||
assert chk.returncode == 0, chk.stderr
|
||||
|
||||
js = json.loads(_run(["--json"], home=tmp_path).stdout)
|
||||
assert js["status"] == "ready"
|
||||
assert js["can_proceed"] is True
|
||||
assert js["whisper_backend"] == "groq"
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Transcript-cue timestamps: parsing, point extraction, and pinned merge."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import frames
|
||||
|
||||
|
||||
def test_parse_timestamps_mixed_formats():
|
||||
assert frames.parse_timestamps("30,1:05,90") == [30.0, 65.0, 90.0]
|
||||
|
||||
|
||||
def test_parse_timestamps_strips_and_dedupes():
|
||||
assert frames.parse_timestamps(" 90 , 30, 30 ") == [30.0, 90.0]
|
||||
|
||||
|
||||
def test_parse_timestamps_empty():
|
||||
assert frames.parse_timestamps("") == []
|
||||
assert frames.parse_timestamps(" , ") == []
|
||||
|
||||
|
||||
def test_parse_timestamps_rejects_garbage():
|
||||
with pytest.raises(SystemExit):
|
||||
frames.parse_timestamps("4:bad")
|
||||
|
||||
|
||||
def test_merge_frames_sorts_and_reindexes():
|
||||
primary = [
|
||||
{"index": 0, "timestamp_seconds": 1.0, "path": "a", "reason": "scene-change"},
|
||||
{"index": 1, "timestamp_seconds": 5.0, "path": "b", "reason": "scene-change"},
|
||||
]
|
||||
pinned = [
|
||||
{"index": 0, "timestamp_seconds": 3.0, "path": "c", "reason": "transcript-cue"},
|
||||
]
|
||||
merged = frames.merge_frames(primary, pinned)
|
||||
assert [f["path"] for f in merged] == ["a", "c", "b"]
|
||||
assert [f["index"] for f in merged] == [0, 1, 2]
|
||||
assert merged[1]["reason"] == "transcript-cue"
|
||||
|
||||
|
||||
def test_merge_frames_keeps_all_pinned():
|
||||
pinned = [{"index": 0, "timestamp_seconds": 2.0, "path": "c", "reason": "transcript-cue"}]
|
||||
merged = frames.merge_frames([], pinned)
|
||||
assert [f["path"] for f in merged] == ["c"]
|
||||
|
||||
|
||||
def test_extract_at_timestamps_one_frame_per_point(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_at_timestamps(str(cut_clip), tmp_path / "f", [0.5, 2.0, 4.0])
|
||||
assert meta["engine"] == "timestamps"
|
||||
assert meta["fallback"] is False
|
||||
assert len(out) == 3
|
||||
assert all(f["reason"] == "transcript-cue" for f in out)
|
||||
ts = [f["timestamp_seconds"] for f in out]
|
||||
assert ts == sorted(ts)
|
||||
assert len(out) == len(list((tmp_path / "f").glob("cue_*.jpg")))
|
||||
|
||||
|
||||
def test_extract_at_timestamps_drops_out_of_window(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_at_timestamps(
|
||||
str(cut_clip), tmp_path / "f", [0.5, 2.0, 4.0],
|
||||
start_seconds=1.0, end_seconds=3.0,
|
||||
)
|
||||
assert [f["timestamp_seconds"] for f in out] == [2.0]
|
||||
assert meta["dropped_out_of_window"] == 2
|
||||
|
||||
|
||||
def test_extract_at_timestamps_caps_and_spans(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_at_timestamps(
|
||||
str(cut_clip), tmp_path / "f", [0.5, 1.5, 2.5, 3.5, 4.5], max_frames=3,
|
||||
)
|
||||
assert len(out) == 3
|
||||
ts = [f["timestamp_seconds"] for f in out]
|
||||
assert ts[0] == 0.5 and ts[-1] == 4.5 # even-sample keeps first + last
|
||||
assert len(out) == len(list((tmp_path / "f").glob("cue_*.jpg")))
|
||||
|
||||
|
||||
def test_extract_at_timestamps_does_not_clobber_detail_frames(cut_clip: Path, tmp_path: Path):
|
||||
"""Cue frames live alongside detail frames in the same dir without deleting them."""
|
||||
d = tmp_path / "f"
|
||||
scene, _ = frames.extract_scene_or_uniform(
|
||||
str(cut_clip), d, fps=2.0, target_frames=50, max_frames=100,
|
||||
)
|
||||
cues, _ = frames.extract_at_timestamps(str(cut_clip), d, [1.0, 3.0])
|
||||
assert len(list(d.glob("frame_*.jpg"))) == len(scene)
|
||||
assert len(list(d.glob("cue_*.jpg"))) == len(cues)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""End-to-end routing of --detail through watch.py on a local clip."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
WATCH = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts" / "watch.py"
|
||||
|
||||
|
||||
def _run(clip: Path, *args: str, env_extra: dict | None = None) -> str:
|
||||
env = dict(os.environ)
|
||||
env.pop("WATCH_DETAIL", None)
|
||||
if env_extra:
|
||||
env.update(env_extra)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(WATCH), str(clip), "--no-whisper", *args],
|
||||
capture_output=True, text=True, env=env,
|
||||
)
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def test_efficient_uses_keyframe_engine(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "efficient")
|
||||
assert "(keyframe" in out
|
||||
assert "**Detail:** efficient" in out
|
||||
|
||||
|
||||
def test_balanced_uses_scene_engine(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "balanced")
|
||||
assert "(scene" in out
|
||||
assert "**Detail:** balanced" in out
|
||||
|
||||
|
||||
def test_token_burner_uses_scene_engine(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "token-burner")
|
||||
assert "(scene" in out
|
||||
|
||||
|
||||
def test_transcript_skips_frames(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "transcript")
|
||||
assert "skipped" in out
|
||||
assert "frame_0000.jpg" not in out
|
||||
|
||||
|
||||
def test_flag_overrides_env(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "efficient", env_extra={"WATCH_DETAIL": "balanced"})
|
||||
assert "(keyframe" in out
|
||||
|
||||
|
||||
def test_default_is_balanced(cut_clip: Path):
|
||||
out = _run(cut_clip) # no flag, WATCH_DETAIL cleared
|
||||
assert "**Detail:** balanced" in out
|
||||
assert "(scene" in out
|
||||
|
||||
|
||||
def test_timestamps_add_cue_frames_to_detail(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "balanced", "--timestamps", "1,3")
|
||||
assert "reason=transcript-cue" in out
|
||||
assert "reason=scene-change" in out # detail frames still present (additive)
|
||||
|
||||
|
||||
def test_timestamps_with_transcript_detail_is_cue_only(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "transcript", "--timestamps", "1,3")
|
||||
assert "reason=transcript-cue" in out
|
||||
assert "reason=scene-change" not in out
|
||||
assert "reason=keyframe" not in out
|
||||
|
||||
|
||||
def _frame_lines(out: str) -> int:
|
||||
return sum(1 for line in out.splitlines() if "/frames/frame_" in line and "(t=" in line)
|
||||
|
||||
|
||||
def test_dedup_collapses_static_by_default(static_clip: Path):
|
||||
out = _run(static_clip) # solid blue → identical frames collapse to one
|
||||
assert "near-duplicate" in out
|
||||
assert _frame_lines(out) == 1
|
||||
|
||||
|
||||
def test_no_dedup_preserves_static_frames(static_clip: Path):
|
||||
out = _run(static_clip, "--no-dedup")
|
||||
assert "near-duplicate" not in out
|
||||
assert _frame_lines(out) > 1
|
||||
@@ -0,0 +1,157 @@
|
||||
"""Whisper auto-chunking: plan, split, and timestamp stitching."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import whisper
|
||||
|
||||
|
||||
MB = 1024 * 1024
|
||||
|
||||
|
||||
class TestPlanChunks:
|
||||
def test_under_limit_is_single_chunk(self):
|
||||
plan = whisper.plan_chunks(total_seconds=600.0, total_bytes=5 * MB, max_bytes=24 * MB)
|
||||
assert plan == [(0.0, 600.0)]
|
||||
|
||||
def test_at_limit_is_single_chunk(self):
|
||||
plan = whisper.plan_chunks(total_seconds=600.0, total_bytes=24 * MB, max_bytes=24 * MB)
|
||||
assert plan == [(0.0, 600.0)]
|
||||
|
||||
def test_over_limit_splits_into_enough_chunks(self):
|
||||
# 71 MB against a 24 MB cap → ceil(71/24) = 3 chunks.
|
||||
plan = whisper.plan_chunks(total_seconds=3600.0, total_bytes=71 * MB, max_bytes=24 * MB)
|
||||
assert len(plan) == 3
|
||||
|
||||
def test_chunks_are_contiguous_and_cover_full_duration(self):
|
||||
total = 3600.0
|
||||
plan = whisper.plan_chunks(total_seconds=total, total_bytes=71 * MB, max_bytes=24 * MB)
|
||||
# Offsets start at 0 and each picks up where the previous ended.
|
||||
assert plan[0][0] == 0.0
|
||||
for (off, dur), (next_off, _) in zip(plan, plan[1:]):
|
||||
assert math.isclose(off + dur, next_off)
|
||||
last_off, last_dur = plan[-1]
|
||||
assert math.isclose(last_off + last_dur, total)
|
||||
|
||||
def test_each_chunk_estimated_under_limit(self):
|
||||
total_seconds, total_bytes, cap = 3600.0, 71 * MB, 24 * MB
|
||||
plan = whisper.plan_chunks(total_seconds, total_bytes, cap)
|
||||
bytes_per_second = total_bytes / total_seconds
|
||||
for _off, dur in plan:
|
||||
assert dur * bytes_per_second <= cap
|
||||
|
||||
def test_zero_duration_is_single_chunk(self):
|
||||
plan = whisper.plan_chunks(total_seconds=0.0, total_bytes=0, max_bytes=24 * MB)
|
||||
assert plan == [(0.0, 0.0)]
|
||||
|
||||
|
||||
class TestShiftSegments:
|
||||
def test_adds_offset_to_start_and_end(self):
|
||||
segs = [{"start": 0.0, "end": 2.5, "text": "hi"}, {"start": 2.5, "end": 4.0, "text": "there"}]
|
||||
shifted = whisper.shift_segments(segs, 1800.0)
|
||||
assert shifted == [
|
||||
{"start": 1800.0, "end": 1802.5, "text": "hi"},
|
||||
{"start": 1802.5, "end": 1804.0, "text": "there"},
|
||||
]
|
||||
|
||||
def test_zero_offset_is_identity(self):
|
||||
segs = [{"start": 1.0, "end": 2.0, "text": "x"}]
|
||||
assert whisper.shift_segments(segs, 0.0) == segs
|
||||
|
||||
def test_does_not_mutate_input(self):
|
||||
segs = [{"start": 0.0, "end": 1.0, "text": "x"}]
|
||||
whisper.shift_segments(segs, 10.0)
|
||||
assert segs[0]["start"] == 0.0
|
||||
|
||||
|
||||
def _make_mp3(path: Path, seconds: float) -> None:
|
||||
"""Synthesize a mono 16k 64k mp3 of a sine tone — mirrors extract_audio's format."""
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-f", "lavfi", "-t", str(seconds), "-i", "sine=frequency=440:sample_rate=16000",
|
||||
"-acodec", "libmp3lame", "-ar", "16000", "-ac", "1", "-b:a", "64k",
|
||||
str(path),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
class TestSplitAudio:
|
||||
def test_creates_one_file_per_plan_entry(self, tmp_path: Path):
|
||||
full = tmp_path / "audio.mp3"
|
||||
_make_mp3(full, 6.0)
|
||||
plan = [(0.0, 3.0), (3.0, 3.0)]
|
||||
|
||||
chunks = whisper.split_audio(full, tmp_path, plan)
|
||||
|
||||
assert len(chunks) == 2
|
||||
for chunk_path, _offset in chunks:
|
||||
assert chunk_path.exists() and chunk_path.stat().st_size > 0
|
||||
|
||||
def test_returns_plan_offsets(self, tmp_path: Path):
|
||||
full = tmp_path / "audio.mp3"
|
||||
_make_mp3(full, 6.0)
|
||||
plan = [(0.0, 3.0), (3.0, 3.0)]
|
||||
|
||||
chunks = whisper.split_audio(full, tmp_path, plan)
|
||||
|
||||
assert [offset for _path, offset in chunks] == [0.0, 3.0]
|
||||
|
||||
def test_chunks_are_smaller_than_full(self, tmp_path: Path):
|
||||
full = tmp_path / "audio.mp3"
|
||||
_make_mp3(full, 6.0)
|
||||
plan = [(0.0, 3.0), (3.0, 3.0)]
|
||||
|
||||
chunks = whisper.split_audio(full, tmp_path, plan)
|
||||
|
||||
full_size = full.stat().st_size
|
||||
for chunk_path, _offset in chunks:
|
||||
assert chunk_path.stat().st_size < full_size
|
||||
|
||||
|
||||
class TestAudioDuration:
|
||||
def test_reads_duration_of_synthesized_clip(self, tmp_path: Path):
|
||||
audio = tmp_path / "audio.mp3"
|
||||
_make_mp3(audio, 5.0)
|
||||
assert whisper.audio_duration(audio) == pytest.approx(5.0, abs=0.5)
|
||||
|
||||
|
||||
class TestTranscribeChunks:
|
||||
def test_shifts_and_concatenates_each_chunk(self):
|
||||
chunks = [(Path("a.mp3"), 0.0), (Path("b.mp3"), 100.0)]
|
||||
|
||||
def fake_transcribe(path: Path) -> list[dict]:
|
||||
return [{"start": 0.0, "end": 2.0, "text": path.stem}]
|
||||
|
||||
out = whisper.transcribe_chunks(chunks, fake_transcribe)
|
||||
|
||||
assert out == [
|
||||
{"start": 0.0, "end": 2.0, "text": "a"},
|
||||
{"start": 100.0, "end": 102.0, "text": "b"},
|
||||
]
|
||||
|
||||
def test_keeps_successful_chunks_when_one_fails(self):
|
||||
chunks = [(Path("a.mp3"), 0.0), (Path("b.mp3"), 100.0)]
|
||||
|
||||
def flaky(path: Path) -> list[dict]:
|
||||
if path.stem == "b":
|
||||
raise SystemExit("chunk b failed")
|
||||
return [{"start": 1.0, "end": 2.0, "text": "a"}]
|
||||
|
||||
out = whisper.transcribe_chunks(chunks, flaky)
|
||||
|
||||
assert out == [{"start": 1.0, "end": 2.0, "text": "a"}]
|
||||
|
||||
def test_raises_when_every_chunk_fails(self):
|
||||
chunks = [(Path("a.mp3"), 0.0), (Path("b.mp3"), 100.0)]
|
||||
|
||||
def always_fail(path: Path) -> list[dict]:
|
||||
raise SystemExit("boom")
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
whisper.transcribe_chunks(chunks, always_fail)
|
||||
Reference in New Issue
Block a user