From 429f3143e54975dbc4ad0a4b5b34163b9ba5639b Mon Sep 17 00:00:00 2001 From: bradautomates Date: Mon, 29 Jun 2026 22:12:54 +1000 Subject: [PATCH] Restructure as a self-contained Agent Skills package (fix Codex install) Move the skill into skills/watch/ so SKILL.md and its scripts/ runtime are siblings inside one folder. `npx skills add` (Codex/Cursor/Copilot/agents) now copies a working skill as a unit; previously it grabbed the root SKILL.md alone and left scripts/ behind, so the skill was dead on arrival on every non-Claude host. Mirrors the layout last30days-skill adopted for the same reason. - skills/watch/{SKILL.md,scripts/}: self-contained skill folder - SKILL.md: resolve a harness-agnostic $SKILL_DIR (the dir it was Read from) instead of the Claude-Code-only ${CLAUDE_SKILL_DIR}; guard + 19 call sites - drop commands/watch.md: /watch derives from frontmatter (name + user-invocable) - .codex-plugin/plugin.json: full manifest with "skills": "./skills/" + interface - add .agents/plugins/marketplace.json, AGENTS.md, CLAUDE.md, .skillignore - build-skill.sh: archive the skills/watch subtree (one SKILL.md, no zip -d) - fix paths in tests, hooks hint, .gitattributes, release.yml - relocate dev-sync.sh to repo root and fix REPO_ROOT - README: content-ideas structure, npx skills install, star history Verified: 37/37 tests pass; npx skills add bundles the full scripts/ runtime; manifests valid; versions synced at 0.1.3. Co-Authored-By: Claude Opus 4.8 (1M context) --- .agents/plugins/marketplace.json | 20 + .codex-plugin/plugin.json | 42 +- .gitattributes | 22 +- .github/workflows/release.yml | 2 +- .skillignore | 33 + AGENTS.md | 50 ++ CLAUDE.md | 1 + README.md | 131 +++- SKILL.md | 173 ----- commands/watch.md | 9 - dev-sync.sh | 87 +++ hooks/scripts/check-setup.sh | 2 +- scripts/build-skill.sh | 50 -- scripts/frames.py | 250 ------- scripts/watch.py | 230 ------- skills/watch/.skillignore | 3 + skills/watch/SKILL.md | 267 ++++++++ skills/watch/scripts/build-skill.sh | 39 ++ skills/watch/scripts/config.py | 65 ++ {scripts => skills/watch/scripts}/download.py | 91 ++- skills/watch/scripts/frames.py | 628 ++++++++++++++++++ {scripts => skills/watch/scripts}/setup.py | 52 +- .../watch/scripts}/transcribe.py | 0 skills/watch/scripts/watch.py | 382 +++++++++++ {scripts => skills/watch/scripts}/whisper.py | 0 tests/conftest.py | 83 +++ tests/test_config.py | 37 ++ tests/test_download.py | 64 ++ tests/test_fixtures.py | 24 + tests/test_frames.py | 70 ++ tests/test_setup.py | 80 +++ tests/test_timestamps.py | 87 +++ tests/test_watch.py | 69 ++ 33 files changed, 2360 insertions(+), 783 deletions(-) create mode 100644 .agents/plugins/marketplace.json create mode 100644 .skillignore create mode 100644 AGENTS.md create mode 100644 CLAUDE.md delete mode 100644 SKILL.md delete mode 100644 commands/watch.md create mode 100755 dev-sync.sh delete mode 100755 scripts/build-skill.sh delete mode 100755 scripts/frames.py delete mode 100755 scripts/watch.py create mode 100644 skills/watch/.skillignore create mode 100644 skills/watch/SKILL.md create mode 100755 skills/watch/scripts/build-skill.sh create mode 100644 skills/watch/scripts/config.py rename {scripts => skills/watch/scripts}/download.py (67%) create mode 100755 skills/watch/scripts/frames.py rename {scripts => skills/watch/scripts}/setup.py (85%) rename {scripts => skills/watch/scripts}/transcribe.py (100%) create mode 100755 skills/watch/scripts/watch.py rename {scripts => skills/watch/scripts}/whisper.py (100%) create mode 100644 tests/conftest.py create mode 100644 tests/test_config.py create mode 100644 tests/test_download.py create mode 100644 tests/test_fixtures.py create mode 100644 tests/test_frames.py create mode 100644 tests/test_setup.py create mode 100644 tests/test_timestamps.py create mode 100644 tests/test_watch.py diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json new file mode 100644 index 0000000..7744c56 --- /dev/null +++ b/.agents/plugins/marketplace.json @@ -0,0 +1,20 @@ +{ + "name": "claude-video", + "interface": { + "displayName": "watch" + }, + "plugins": [ + { + "name": "watch", + "source": { + "source": "url", + "url": "https://github.com/bradautomates/claude-video.git" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "Productivity" + } + ] +} diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index d677e4b..da4f9e7 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,3 +1,43 @@ { - "name": "watch" + "name": "watch", + "version": "0.1.3", + "description": "Let Claude watch a video. Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls captions or falls back to Whisper, and hands frames + transcript to Claude so it can answer questions about the video.", + "author": { + "name": "Bradley Bonanno" + }, + "homepage": "https://github.com/bradautomates/claude-video", + "repository": "https://github.com/bradautomates/claude-video", + "license": "MIT", + "keywords": [ + "video", + "watch", + "youtube", + "vimeo", + "tiktok", + "transcription", + "whisper", + "yt-dlp", + "ffmpeg", + "multimodal", + "frames" + ], + "skills": "./skills/", + "interface": { + "displayName": "watch", + "shortDescription": "Give Claude a video input — paste a URL or path and ask about it.", + "longDescription": "watch adds a skill that lets the agent watch any video: it downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls a timestamped transcript from native captions (or the Whisper API as a fallback), and hands frames + transcript to the model so it can answer questions grounded in what's actually on screen and in the audio.", + "developerName": "Bradley Bonanno", + "category": "Productivity", + "capabilities": [ + "Interactive", + "Read" + ], + "websiteURL": "https://github.com/bradautomates/claude-video", + "defaultPrompt": [ + "/watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark?", + "/watch ~/Movies/screen-recording.mp4 when does the UI break?", + "/watch https://youtu.be/ summarize this" + ], + "brandColor": "#E11D48" + } } diff --git a/.gitattributes b/.gitattributes index fe2ca2c..5119f8d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,6 @@ # Exclude non-runtime files from `git archive` output. -# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file. +# Used by skills/watch/scripts/build-skill.sh to produce the claude.ai .skill file +# and by Claude Code's /plugin install (full-repo archive). # Anthropic canonical skill-packaging excludes # (mirrors anthropics/skills/skills/skill-creator/scripts/package_skill.py) @@ -15,10 +16,21 @@ fixtures/ export-ignore assets/ export-ignore examples/ export-ignore -# NOTE: commands/, hooks/, and .claude-plugin/ are NOT export-ignored because -# Claude Code's /plugin install fetches this same git archive tarball — they -# must be in the archive for the plugin to install correctly. The claude.ai -# .skill bundle strips them afterward via `zip -d` in scripts/build-skill.sh. +# Dev tooling and planning notes — repo-only, not shipped to any install surface +dev-sync.sh export-ignore +V2_PLAN.md export-ignore +V2_CONCERNS.md export-ignore +.agents/ export-ignore + +# Dev-only build script + scanner config inside the skill tree: keep them out of +# the claude.ai .skill bundle (built via `git archive HEAD:skills/watch`). +skills/watch/scripts/build-skill.sh export-ignore +skills/watch/.skillignore export-ignore + +# NOTE: hooks/ and .claude-plugin/ are NOT export-ignored because Claude Code's +# /plugin install fetches the full-repo git archive — they must be present for +# the plugin to install correctly. The claude.ai .skill bundle is built from the +# skills/watch/ subtree only, so it never sees them in the first place. # CI workflows — repo-only, not needed at skill runtime .github/ export-ignore diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b5c26d5..80db5da 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: - name: Build .skill artifact run: | - bash scripts/build-skill.sh + bash skills/watch/scripts/build-skill.sh test -f dist/watch.skill - name: Create GitHub release diff --git a/.skillignore b/.skillignore new file mode 100644 index 0000000..2f96a45 --- /dev/null +++ b/.skillignore @@ -0,0 +1,33 @@ +# Agent Skills install-time scanner exclusions for repository-root scans. +# Keep the public bundle focused on the runtime skill under skills/watch/. + +# VCS, local envs, caches, and generated outputs +.git/ +.venv/ +__pycache__/ +*.pyc +*.log +.DS_Store +.pytest_cache/ +.coverage +htmlcov/ +dist/ + +# Repo/dev automation and host-specific package metadata +.github/ +.agents/ +.claude-plugin/ +hooks/ +dev-sync.sh + +# Non-runtime docs, plans, and tests +tests/ +README.md +CHANGELOG.md +AGENTS.md +CLAUDE.md +V2_PLAN.md +V2_CONCERNS.md + +# Dev-only script shipped inside the skill tree but not needed at runtime +skills/watch/scripts/build-skill.sh diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6068f24 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,50 @@ +# claude-video / watch skill + +Agent Skills package that gives an agent a video input. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, and 50+ other [Agent Skills](https://agentskills.io) hosts. Pure-stdlib Python that orchestrates `yt-dlp` + `ffmpeg` and an optional Whisper API. + +## Structure + +- `skills/watch/SKILL.md` — canonical skill contract the model reads when `/watch` fires. Source of truth for behavior across every host. +- `skills/watch/scripts/watch.py` — entry point; orchestrates download → frames → transcript. +- `skills/watch/scripts/{download,frames,transcribe,whisper,setup,config}.py` — yt-dlp wrapper, ffmpeg frame extraction + auto-fps, caption/Whisper transcription, preflight/installer, shared config. +- `skills/watch/scripts/build-skill.sh` — builds `dist/watch.skill` for claude.ai upload (dev-only). +- `hooks/` — Claude Code SessionStart setup-status hook (Claude Code only). +- `.claude-plugin/` — `plugin.json` + `marketplace.json` (Claude Code plugin + local marketplace). +- `.codex-plugin/plugin.json` — Codex/agents manifest; `"skills": "./skills/"` points the Agent Skills CLI at the self-contained skill folder. +- `.agents/plugins/marketplace.json` — agents marketplace listing pointing at the repo-root plugin. +- `CLAUDE.md` → `@AGENTS.md` — generic-agent entry point. +- `tests/` — pytest suite (ffmpeg-synthesized clips; no network). + +## Orientation + +- The product is the slash-command-invoked skill (`/watch [question]`), not a CLI. `scripts/watch.py` is implementation. Features must work across every harness the skill installs into, not just Claude Code. +- **The skill is one self-contained folder: `skills/watch/`.** SKILL.md and `scripts/` are siblings inside it. This is what lets `npx skills add` copy a working skill as a unit — do NOT move SKILL.md or `scripts/` back to the repo root, or non-Claude installers will copy SKILL.md without the scripts. +- **Path resolution is harness-agnostic.** SKILL.md resolves `SKILL_DIR` as the directory of the SKILL.md the model just Read, then runs `${SKILL_DIR}/scripts/...`. Do NOT reintroduce `${CLAUDE_SKILL_DIR}` (Claude-Code-only) — it is unset on Codex/Cursor/agents and breaks every script call there. +- **No `commands/` wrapper.** `/watch` is derived from SKILL.md frontmatter (`name: watch` + `user-invocable: true`). A separate command file creates a duplicate slash command. + +## Install surfaces + +| Surface | Install | +|---------|---------| +| Claude Code | `/plugin marketplace add bradautomates/claude-video` then `/plugin install watch@claude-video` | +| Codex / Cursor / Copilot / +50 | `npx skills add bradautomates/claude-video -g` | +| claude.ai (web) | upload `dist/watch.skill` (built by `skills/watch/scripts/build-skill.sh`) | + +## Commands + +```bash +# Tests (stdlib + pytest; ffmpeg required for frame tests) +.venv/bin/pytest -q # or: python3 -m pytest -q + +# Build the claude.ai upload bundle (archives skills/watch/ as the bundle root) +bash skills/watch/scripts/build-skill.sh # → dist/watch.skill + +# Dev: mirror the working tree into the installed Claude Code plugin cache +./dev-sync.sh # --dry-run to preview +``` + +## Rules + +- Keep the version in sync across `skills/watch/SKILL.md` (frontmatter), `.claude-plugin/plugin.json`, and `.codex-plugin/plugin.json` when cutting a release. +- Releasing: tag `vX.Y.Z` and push the tag; `.github/workflows/release.yml` builds `dist/watch.skill` and attaches it to the GitHub release. +- Never commit real API keys or `.env` contents; keys live in `~/.config/watch/.env` (mode `0600`) at runtime. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/README.md b/README.md index 659edb1..39e6389 100644 --- a/README.md +++ b/README.md @@ -2,18 +2,19 @@ **Give Claude the ability to watch any video.** -Claude Code: +Claude Code (recommended — auto-updates via marketplace): ``` /plugin marketplace add bradautomates/claude-video /plugin install watch@claude-video ``` -claude.ai (web): [download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) and drop it into Settings → Capabilities → Skills. - -Codex / generic skills: +Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts: ```bash -git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch +npx skills add bradautomates/claude-video -g ``` +(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.) + +More install options (claude.ai web, manual) in the [Install](#install) section below. Zero config to start — `yt-dlp` and `ffmpeg` install on first run via `brew` on macOS (Linux/Windows print exact commands). Captions cover most public videos for free. Whisper API key is only needed when a video has no captions. @@ -21,7 +22,7 @@ Zero config to start — `yt-dlp` and `ffmpeg` install on first run via `brew` o Claude can read a webpage, run a script, browse a repo. What it can't do, out of the box, is *watch a video*. You paste a YouTube link and it has to either guess from the title or pull a transcript that's missing 90% of what's on screen. -With Claude Video `/watch` you can paste a URL or a local path, ask a question, and Claude downloads the video, extracts frames at an auto-scaled rate, pulls a timestamped transcript (free captions when available, Whisper API as fallback), and `Read`s every frame as an image. By the time it answers, it has *seen* the video and *heard* the audio. +With Claude Video `/watch` you can paste a URL or a local path, ask a question, and Claude fetches captions first, downloads only what it needs, extracts frames (scene-aware, or fast keyframes at `efficient` detail), pulls a timestamped transcript (free captions when available, Whisper API as fallback), and `Read`s every frame as an image. By the time it answers, it has *seen* the video and *heard* the audio. ``` /watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark? @@ -46,8 +47,8 @@ Claude is great at reading and synthesizing — but until now, video was the one ## How it works 1. **You paste a video and a question.** URL (anything yt-dlp supports — YouTube, Loom, TikTok, X, Instagram, plus a few hundred more) or a local path (`.mp4`, `.mov`, `.mkv`, `.webm`). -2. **`yt-dlp` downloads it.** For URLs, into a temp working directory. For local files, no download — just probed in place. -3. **`ffmpeg` extracts frames at an auto-scaled rate.** The frame budget is duration-aware: ≤30s gets ~30 frames, 30-60s gets ~40, 1-3min gets ~60, 3-10min gets ~80, longer gets 100 sparsely. Hard ceilings: 2 fps, 100 frames. JPEGs at 512px wide by default — bump with `--resolution 1024` if Claude needs to read on-screen text. +2. **`yt-dlp` checks captions first.** At `transcript` detail, captioned URLs return without downloading video. Otherwise, or when Whisper needs audio, it downloads only what the run needs. +3. **`ffmpeg` extracts frames at the chosen detail.** `efficient` decodes keyframes only (near-instant); `balanced`/`token-burner` prefer scene-change frames and fall back to the duration-aware uniform sampler when they under-produce. JPEGs are 512px wide by default and clamped to 1998px tall for Claude Read compatibility. 4. **The transcript comes from one of two places.** First try: `yt-dlp` pulls native captions (manual or auto-generated) from the source. Free, instant, accurate-ish. Fallback: extract a mono 16 kHz audio clip and ship it to Whisper — Groq's `whisper-large-v3` (preferred — cheaper and faster) or OpenAI's `whisper-1`. 5. **Frames + transcript are handed to Claude.** The script prints frame paths with `t=MM:SS` markers and the transcript with timestamps. Claude `Read`s each frame in parallel — JPEGs render directly as images in its context. 6. **Claude answers grounded in what's actually on screen and in the audio.** Not "based on the description" or "according to the title." It saw the frames. It heard the transcript. It answers the way someone who watched the video would. @@ -67,14 +68,40 @@ Token cost is dominated by frames. Every frame is an image; image tokens add up When the user names a moment ("around 2:30", "the last 30 seconds", "from 0:45 to 1:00"), pass `--start` / `--end`. Focused mode gets denser per-second budgets, capped at 2 fps. Far more useful than a sparse pass over the whole thing. +## Detail modes — measured + +The `--detail` dial trades speed and token cost for visual fidelity. Numbers below are from a real run against a **49:08 (2948 s)** YouTube video (1280×720 source, English auto-captions, 1394 caption segments) — a long, mostly-static screen recording, which is the case that stresses the caps hardest. Frame-extraction times are measured against a pre-downloaded local copy so they reflect the *mode's* CPU cost, not network speed. The one-time video download for this clip was **~37 s** / 76 MB (shared by `efficient` / `balanced` / `token-burner`). + +| Mode | Engine | Frames | Cap | Extraction time | Temporal coverage | Est. image tokens | +|------|--------|--------|-----|-----------------|-------------------|-------------------| +| `transcript` | none (captions) | 0 | — | **~4.5 s** (network-bound — one yt-dlp call, no video download) | full (text) | 0 (transcript ≈26.6k text tokens) | +| `efficient` | keyframe (`-skip_frame nokey`) | 50 | 50 | **~0.5 s** | 0:00 → 49:04 (full) | **~9.8k** | +| `balanced` | scene-change | 100 | 100 | **~20.9 s** | 0:00 → 48:38 (full) | **~19.7k** | +| `token-burner` | scene-change | 116 | uncapped | **~21.0 s** | 0:00 → 48:38 (full) | **~22.8k** | + +Image-token estimate uses Anthropic's `(width × height) / 750` per image. At the default 512px width these 720p frames are 512×288, so **≈197 tokens/frame**; total image tokens ≈ `frames × 197`. Bumping `--resolution` to 1024 roughly **4×s** that. The transcript (~26.6k tokens here) is surfaced in *every* mode that has captions, so on the frame modes it adds to the image tokens above — on long videos the transcript, not the frames, is often the larger cost. + +**Note on the `transcript` time.** It looks slower than `efficient` only because the two numbers measure different things: `transcript`'s ~4.5 s is a network round-trip to the source (its *only* step — no video download, no ffmpeg), while the frame modes' times are local CPU with the shared ~37 s download billed separately. End-to-end from a cold URL, `transcript` is the **cheapest** mode by far; `efficient` from a cold URL would be ~37 s of download + 0.5 s of decode. + +What the numbers show: + +- **One consistent sampling rule across every frame mode.** All three detect *all* candidates across the full range, then even-sample (first + last always kept) down to the cap via a shared `_even_sample` helper — `transcript` excepted. Keyframes (`efficient`) and scene-cuts (`balanced`/`token-burner`) differ only in the candidate *source* and the cap, never in how coverage is spread. +- **`efficient` is the speed tier** — ~0.5 s because it only reconstructs keyframes (P/B frames are skipped). For this clip it decoded **675 keyframes** and evenly sampled down to its 50-frame cap, spread across the whole 49 minutes. +- **Caps are enforced and coverage spans the full clip.** `efficient` (675 → 50) and `balanced` (116 → 100) both sample first→last, so the last frame lands at 48:38–49:04, not partway through. Verified on the full video and on focused ranges (`--start`/`--end` of 30 s → 10 frames, 3 min → 37 frames). The even-sample step (`len(indices) == cap`) cannot return more than the cap. *(This addresses the "sometimes returns too many / drops the tail" concern: the `N selected from 675 candidates` line shows the pre-sample candidate count, not what gets surfaced — only the sampled frames are written and Read.)* +- **`balanced` now full-decodes like `token-burner`** (~21 s vs `efficient`'s ~0.5 s). Detecting every scene cut requires decoding the whole video; the old `-frames:v` early-exit was ~3× faster but kept only the *first* 100 cuts and dropped the tail of long videos — so it was removed in favor of even coverage. +- **`token-burner` only diverges from `balanced` past the cap.** This recording had **116** cuts over 49 min, so `balanced` sampled 100 of them and `token-burner` kept all 116 — both spanning the full video. On a high-motion video with hundreds of cuts, `token-burner` keeps everything (and the >250-frame token warning kicks in) while `balanced` thins to 100. +- **`efficient` can return *more* frames than `balanced`** on low-motion footage (50 keyframes vs. few scene cuts) — the tiers differ by extraction *method* and cap, not by a guaranteed frame-count ordering. "Efficient" means near-instant extraction, not always fewer frames. + +Bottom line on timing: every mode finished its own work in **under 22 s** (plus the shared ~37 s download for the frame modes). `efficient` is ~40× faster than the scene modes here because the scene detector decodes every frame of the full 49 minutes, while `efficient` only reconstructs keyframes. + ## Install | Surface | Install | |---------|---------| | **Claude Code** | `/plugin marketplace add bradautomates/claude-video` then `/plugin install watch@claude-video` | +| **Codex, Cursor, Copilot, Gemini CLI, +50 more** | `npx skills add bradautomates/claude-video -g` | | **claude.ai** (web) | [Download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) → Settings → Capabilities → Skills → `+` | -| **Codex** | `git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch` | -| **Manual / dev** | `git clone https://github.com/bradautomates/claude-video.git ~/.claude/skills/watch` | +| **Manual / dev** | `git clone` then symlink `skills/watch` into your host's skills dir (see below) | ### Claude Code @@ -85,6 +112,24 @@ When the user names a moment ("around 2:30", "the last 30 seconds", "from 0:45 t Update later with `/plugin update watch@claude-video`. +### Codex, Cursor, Copilot, Gemini CLI, and 50+ other hosts + +The [Agent Skills](https://agentskills.io) CLI installs the skill into whatever agents it detects: + +```bash +npx skills add bradautomates/claude-video -g +``` + +`-g` installs globally for your user (`~/.codex/skills`, `~/.cursor/skills`, etc.); drop it to install into the current project instead. Useful flags: + +- `-a, --agent ` — target specific hosts, e.g. `-a codex -a cursor` +- `-l, --list` — list the skills in this repo without installing +- `--copy` — copy files instead of symlinking (for filesystems without symlink support) + +The CLI discovers the skill from `skills/watch/SKILL.md` and copies the whole folder — `SKILL.md` plus its `scripts/` runtime — as a self-contained unit. `SKILL.md` resolves its own scripts relative to wherever it was installed, so it works the same on every host. + +Update later with `npx skills update watch -g`. + ### claude.ai (web) 1. [Download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) from the latest release. @@ -93,18 +138,17 @@ Update later with `/plugin update watch@claude-video`. Enable "Code execution and file creation" under Capabilities first — the skill shells out to `ffmpeg` and `yt-dlp`, so it won't run without it. -### Codex - -```bash -git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch -``` - ### Manual (developer) +Clone the repo and symlink the self-contained skill folder into your host's skills directory — the symlink keeps the install in sync with your working tree as you edit: + ```bash -git clone https://github.com/bradautomates/claude-video.git ~/.claude/skills/watch +git clone https://github.com/bradautomates/claude-video.git +ln -s "$(pwd)/claude-video/skills/watch" ~/.claude/skills/watch # or ~/.codex/skills/watch ``` +For claude.ai, build the `.skill` bundle from source: `bash skills/watch/scripts/build-skill.sh` produces `dist/watch.skill`. + ## First run On the first `/watch` call, the skill runs `scripts/setup.py --check`. If `ffmpeg` / `yt-dlp` aren't on your PATH, or no Whisper API key is set, it walks you through fixing it: @@ -145,6 +189,8 @@ Focused on a specific section — denser frame budget, lower token cost: Other knobs (passed to `scripts/watch.py`): +- `--detail transcript|efficient|balanced|token-burner` — fidelity/speed dial. `transcript` skips frames (transcript only); `efficient` uses fast keyframes (cap 50); `balanced` uses scene-aware frames (cap 100); `token-burner` is scene-aware and uncapped. +- `--timestamps T1,T2,…` — grab a frame at each absolute timestamp (`SS`/`MM:SS`/`HH:MM:SS`). Claude reads the transcript first, then targets the moments the presenter flags ("look here", "as you can see"). Added on top of the detail frames (reserved against the cap); out-of-window cues are dropped in focus mode; with `--detail transcript` these become the only frames. - `--max-frames N` — lower the frame cap for a tighter token budget. - `--resolution W` — bump frame width to 1024 px when Claude needs to read on-screen text (slides, terminals, code). - `--fps F` — override the auto-fps calculation (still capped at 2 fps). @@ -155,37 +201,44 @@ Other knobs (passed to `scripts/watch.py`): ## Limits - **Best accuracy: under 10 minutes.** Past that the script prints a "sparse scan" warning — re-run focused on the part you actually care about with `--start`/`--end`. -- **Hard caps: 2 fps, 100 frames.** Frame count drives token cost; the script enforces this even when the auto-fps math would imply higher. +- **Detail is one dial.** Defaults are balanced: scene-aware frames, 2 fps max, 100-frame cap. Use `--detail efficient` for a fast 50-frame keyframe pass, or `--detail token-burner` for uncapped scene candidates. Set `WATCH_DETAIL` in `~/.config/watch/.env` to change the default. - **Whisper upload limit: 25 MB.** At mono 16 kHz that's about 50 minutes of audio. Longer videos need either captions or `--start`/`--end` to a smaller window. -- **No private platforms.** This skill doesn't log into anything. Public URLs and local files only. If yt-dlp can't reach it without auth, neither can `/watch`. ## Structure ``` . -├── SKILL.md # skill contract — loaded by all three surfaces -├── scripts/ -│ ├── watch.py # entry point — orchestrates download → frames → transcript -│ ├── download.py # yt-dlp wrapper -│ ├── frames.py # ffmpeg frame extraction + auto-fps logic -│ ├── transcribe.py # VTT parsing + dedupe + Whisper orchestration -│ ├── whisper.py # Groq / OpenAI clients (pure stdlib) -│ ├── setup.py # preflight + installer -│ └── build-skill.sh # build dist/watch.skill for claude.ai upload -├── hooks/ # SessionStart status hook (Claude Code only) -├── .claude-plugin/ # plugin.json + marketplace.json (Claude Code) -├── .codex-plugin/ # codex packaging -└── .github/workflows/ # release.yml — auto-builds watch.skill on tag push +├── skills/watch/ # self-contained skill — copied as a unit by every installer +│ ├── SKILL.md # skill contract — the source of truth across all surfaces +│ └── scripts/ +│ ├── watch.py # entry point — orchestrates download → frames → transcript +│ ├── download.py # yt-dlp wrapper +│ ├── frames.py # ffmpeg frame extraction + auto-fps logic +│ ├── transcribe.py # VTT parsing + dedupe + Whisper orchestration +│ ├── whisper.py # Groq / OpenAI clients (pure stdlib) +│ ├── config.py # shared config (~/.config/watch/.env) +│ ├── setup.py # preflight + installer +│ └── build-skill.sh # build dist/watch.skill for claude.ai upload (dev-only) +├── hooks/ # SessionStart status hook (Claude Code only) +├── .claude-plugin/ # plugin.json + marketplace.json (Claude Code) +├── .codex-plugin/ # plugin.json — Codex/agents manifest ("skills": "./skills/") +├── .agents/plugins/ # marketplace.json — Agent Skills marketplace listing +├── AGENTS.md → CLAUDE.md # generic-agent entry point +├── tests/ # pytest suite (ffmpeg-synthesized clips, no network) +└── .github/workflows/ # release.yml — auto-builds watch.skill on tag push ``` ## Develop ```bash +# Run the test suite (stdlib + pytest; ffmpeg required for frame tests): +python3 -m pytest -q + # Build the claude.ai upload bundle: -bash scripts/build-skill.sh # → dist/watch.skill +bash skills/watch/scripts/build-skill.sh # → dist/watch.skill ``` -Releasing: tag `vX.Y.Z`, push the tag. The workflow builds `dist/watch.skill` and attaches it to the GitHub release. +Releasing: tag `vX.Y.Z`, push the tag. The workflow builds `dist/watch.skill` and attaches it to the GitHub release. Keep the version in sync across `skills/watch/SKILL.md`, `.claude-plugin/plugin.json`, and `.codex-plugin/plugin.json`. See [CHANGELOG.md](CHANGELOG.md) for version history. @@ -195,6 +248,16 @@ MIT license. Built on `yt-dlp`, `ffmpeg`, and Claude's multimodal `Read` tool. Whisper transcription via [Groq](https://groq.com) or [OpenAI](https://openai.com). +## Star History + + + + + + Star History Chart + + + --- [github.com/bradautomates/claude-video](https://github.com/bradautomates/claude-video) · [LICENSE](LICENSE) diff --git a/SKILL.md b/SKILL.md deleted file mode 100644 index acd057d..0000000 --- a/SKILL.md +++ /dev/null @@ -1,173 +0,0 @@ ---- -name: watch -description: Watch a video (URL or local path). Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls the transcript from captions (or Whisper API fallback), and hands the result to Claude so it can answer questions about what's in the video. -argument-hint: " [question]" -allowed-tools: Bash, Read, AskUserQuestion -homepage: https://github.com/bradautomates/claude-video -repository: https://github.com/bradautomates/claude-video -author: bradautomates -license: MIT -user-invocable: true ---- - -# /watch — Claude watches a video - -You don't have a video input; this skill gives you one. A Python script downloads the video, extracts frames as JPEGs, gets a timestamped transcript (native captions first, then Whisper API as fallback), and prints frame paths. You then `Read` each frame path to see the images and combine them with the transcript to answer the user. - -## Step 0 — Setup preflight (runs every `/watch` invocation, silent on success) - -**Python interpreter:** every `python3 ...` command in this skill is for macOS/Linux. On **Windows**, substitute `python` — the `python3` command on Windows is the Microsoft Store stub and will not run the script. - -Before every `/watch` run, verify that dependencies and an API key are in place: - -```bash -python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" --check -``` - -This is a <100ms lookup. On exit 0, the script emits **nothing** — proceed to Step 1 without comment. **Do NOT announce "setup is complete" to the user** — they don't need a status message on every turn. The only acceptable user-visible output from Step 0 is when remediation is required. - -On non-zero exit, follow the table: - -| Exit | Meaning | Action | -|------|---------|--------| -| `2` | Missing binaries (`ffmpeg` / `ffprobe` / `yt-dlp`) | Run installer | -| `3` | No Whisper API key | Run installer to scaffold `.env`, then ask user for a key | -| `4` | Both missing | Run installer, then ask for a key | - -The installer is idempotent — safe to re-run: - -```bash -python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" -``` - -On macOS with Homebrew, it auto-installs `ffmpeg` and `yt-dlp`. On Linux/Windows, it prints the exact install commands for the user to run. It scaffolds `~/.config/watch/.env` with commented placeholders at `0600` perms, and writes `SETUP_COMPLETE=true` once deps + a key are in place so the next session knows this user has already been through the wizard. - -**If an API key is still missing after install:** use `AskUserQuestion` to ask the user whether they have a Groq API key (preferred — cheaper, faster) or an OpenAI key. Then write it into `~/.config/watch/.env` — set the matching `GROQ_API_KEY=...` or `OPENAI_API_KEY=...` line. If they don't want to set up Whisper, proceed with `--no-whisper` and tell them videos without native captions will come back frames-only. - -**Structured mode (optional):** `python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" --json` emits `{status, first_run, missing_binaries, whisper_backend, has_api_key, config_file, platform}` where `status` is one of `ready | needs_install | needs_key | needs_install_and_key`. Use this when you need to branch on specifics (e.g. "is this the user's very first run?" → `first_run: true`). - -Within a single session, you can skip Step 0 on follow-up `/watch` calls — once `--check` returned 0, nothing about the environment changes between turns. - -## When to use - -- User pastes a video URL (YouTube, Vimeo, X, TikTok, Twitch clip, most yt-dlp-supported sites) and asks about it. -- User points at a local video file (`.mp4`, `.mov`, `.mkv`, `.webm`, etc.) and asks about it. -- User types `/watch [question]`. - -## Recommended limits - -- **Best accuracy: videos under 10 minutes.** Frame coverage scales inversely with duration. -- **Hard caps: 100 frames total and 2 fps.** Token cost grows with frame count, so the script targets a frame budget by duration (and never exceeds 2 fps even when the budget would imply more): - - ≤30s → ~1-2 fps (up to 30 frames) - - 30s-1min → ~40 frames - - 1-3min → ~60 frames - - 3-10min → ~80 frames - - \>10min → 100 frames, sparsely spaced (warning printed) -- If the user hands you a long video, consider asking whether they want a specific section before burning tokens on a sparse scan. - -## How to invoke - -**Step 1 — parse the user input.** Separate the video source (URL or path) from any question the user asked. Example: `/watch https://youtu.be/abc what language is this in?` → source = `https://youtu.be/abc`, question = `what language is this in?`. - -**Step 2 — run the watch script.** Pass the source verbatim. Do not shell-escape it yourself beyond normal quoting: - -```bash -python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "" -``` - -Optional flags: -- `--start T` / `--end T` — focus on a section. Accepts `SS`, `MM:SS`, or `HH:MM:SS`. When either is set, fps auto-scales denser (see "Focusing on a section" below). -- `--max-frames N` — lower the cap for tighter token budget (e.g. `--max-frames 40`) -- `--resolution W` — change frame width in px (default 512; bump to 1024 only if the user needs to read on-screen text) -- `--fps F` — override auto-fps (clamped to 2 fps max) -- `--out-dir DIR` — keep working files somewhere specific (default: an auto-generated tmp dir) -- `--whisper groq|openai` — force a specific Whisper backend (default: prefer Groq if both keys exist) -- `--no-whisper` — disable the Whisper fallback entirely (frames-only if no captions) - -### Focusing on a section (higher frame rate) - -When the user asks about a specific moment — "what happens at the 2 minute mark?", "zoom into 0:45 to 1:00", "the first 10 seconds" — pass `--start` and/or `--end`. The script switches to focused-mode budgets, which are denser than full-video budgets (still capped at 2 fps): - -- ≤5s → 2 fps (up to 10 frames) -- 5-15s → 2 fps (up to 30 frames) -- 15-30s → ~2 fps (up to 60 frames) -- 30-60s → ~1.3 fps (up to 80 frames) -- 60-180s → ~0.6 fps (100 frames, capped) - -Focused mode is the right call for: -- Any moment/range the user names explicitly ("around 2:30", "the intro", "the last 30 seconds"). -- Any video longer than ~10 minutes where the user's question is about a specific part — running focused on the relevant section is far more useful than a sparse scan of the whole thing. -- Re-runs after a full scan didn't have enough detail in some region. - -Transcript is auto-filtered to the same range. Frame timestamps are absolute (real video timeline, not offset-from-start). - -Examples: -```bash -# Last 10 seconds of a 1 minute video -python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" video.mp4 --start 50 --end 60 - -# Zoom into 2:15 → 2:45 at 3 fps (90 frames) -python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "$URL" --start 2:15 --end 2:45 --fps 3 - -# From 1h12m to the end of the video -python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "$URL" --start 1:12:00 -``` - -**Step 3 — Read every frame path the script lists.** The Read tool renders JPEGs directly as images for you. Read all frames in a single message (parallel tool calls) so you see them together. The frames are in chronological order with a `t=MM:SS` timestamp so you can align them to the transcript. - -**Step 4 — answer the user.** You now have two streams of evidence: -- **Frames** — what's on screen at each timestamp -- **Transcript** — what's said at each timestamp. The report's header shows the source (`captions` = yt-dlp pulled native subs; `whisper (groq)` or `whisper (openai)` = transcribed by API). - -If the user asked a specific question, answer it directly citing timestamps. If they didn't ask anything, summarize what happens in the video — structure, key moments, notable visuals, spoken content. - -**Step 5 — clean up.** The script prints a working directory at the end. If the user isn't going to ask follow-ups about this video, delete it with `rm -rf `. If they might, leave it in place. - -## Transcription - -The script gets a timestamped transcript in one of two ways: - -1. **Native captions (free, preferred).** yt-dlp pulls manual or auto-generated subtitles from the source platform if available. -2. **Whisper API fallback.** If no captions came back (or the source is a local file), the script extracts audio (`ffmpeg -vn -ac 1 -ar 16000 -b:a 64k`, ~0.5 MB/min) and uploads it to whichever Whisper API has a key configured: - - **Groq** — `whisper-large-v3`. Preferred default: cheaper, faster. Get a key at console.groq.com/keys. - - **OpenAI** — `whisper-1`. Fallback. Get a key at platform.openai.com/api-keys. - -Both keys live in `~/.config/watch/.env`. The script prefers Groq when both are set; override with `--whisper openai` to force OpenAI. Use `--no-whisper` to skip the fallback entirely. - -## Failure modes and handling - -- **Setup preflight failed** → run `python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py"` (auto-installs ffmpeg/yt-dlp via brew on macOS, scaffolds the `.env`). For API key, ask the user via `AskUserQuestion` and write it to `~/.config/watch/.env`. -- **No transcript available** → captions missing AND (no Whisper key OR Whisper API failed). Script prints a hint pointing to setup. Proceed frames-only and tell the user. -- **Long video warning printed** → acknowledge it in your answer. Offer to re-run focused on a specific section via `--start`/`--end` rather than a sparse full-video scan. -- **Download fails** → yt-dlp's error goes to stderr. If it's a login-required or region-locked video, tell the user plainly; do not keep retrying. -- **Whisper request fails** → the error is printed to stderr (likely: invalid key, rate limit, or 25 MB upload limit on a very long video). The report will say "none available" for transcript. You can retry with `--whisper openai` if Groq failed (or vice versa). - -## Token efficiency - -This skill burns tokens primarily on frames. Order of magnitude: -- 80 frames at 512px wide is roughly 50-80k image tokens depending on aspect ratio. -- The transcript is cheap (a few thousand tokens at most for a 10-minute video). -- Bumping `--resolution` to 1024 roughly quadruples the image tokens per frame. Only do it when necessary. - -If you already watched a video this session and the user asks a follow-up, do **not** re-run the script — you already have the frames and transcript in context. Just answer from what you have. - -## Security & Permissions - -**What this skill does:** -- Runs `yt-dlp` locally to download the video and pull native captions when the source supports them (public data; the request goes directly to whatever host the URL points at) -- Runs `ffmpeg` / `ffprobe` locally to extract frames as JPEGs and, when Whisper is needed, a mono 16 kHz audio clip -- Sends the extracted audio clip to Groq's Whisper API (`api.groq.com/openai/v1/audio/transcriptions`) when `GROQ_API_KEY` is set (preferred — cheaper, faster) -- Sends the extracted audio clip to OpenAI's audio transcription API (`api.openai.com/v1/audio/transcriptions`) when `OPENAI_API_KEY` is set and Groq is not, or when `--whisper openai` is forced -- Writes the downloaded video, frames, audio, and an intermediate transcript to a working directory under the system temp dir (or `--out-dir` if specified) so Claude can `Read` them -- Reads / creates `~/.config/watch/.env` (mode `0600`) to store the Whisper API key(s) and a `SETUP_COMPLETE` marker. As a fallback, also reads `.env` in the current working directory - -**What this skill does NOT do:** -- Does not upload the video itself to any API — only the extracted audio goes out, and only when native captions are missing AND Whisper is not disabled with `--no-whisper` -- Does not access any platform account (no login, no session cookies, no posting) -- Does not share API keys between providers (Groq key only goes to `api.groq.com`, OpenAI key only goes to `api.openai.com`) -- Does not log, cache, or write API keys to stdout, stderr, or output files -- Does not persist anything outside the working directory and `~/.config/watch/.env` — clean up the working directory when you're done (Step 5) - -**Bundled scripts:** `scripts/watch.py` (entry point), `scripts/download.py` (yt-dlp wrapper), `scripts/frames.py` (ffmpeg frame extraction), `scripts/transcribe.py` (caption selection + Whisper orchestration), `scripts/whisper.py` (Groq / OpenAI clients), `scripts/setup.py` (preflight + installer) - -Review scripts before first use to verify behavior. diff --git a/commands/watch.md b/commands/watch.md deleted file mode 100644 index 3f437e9..0000000 --- a/commands/watch.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -description: Watch a video (URL or local path). Downloads with yt-dlp, extracts frames with ffmpeg, transcribes from captions or Whisper, and answers questions about what's in the video. -argument-hint: [question] -allowed-tools: [Bash, Read, AskUserQuestion] ---- - -Invoke the `watch` skill (defined in SKILL.md) with the user's arguments: $ARGUMENTS - -Follow the skill's full pipeline: preflight setup check → download via yt-dlp → extract frames at auto-scaled fps → pull captions or Whisper transcript → Read each frame → answer the user grounded in frames and transcript. If the user provided no arguments, ask them for a video URL or local path before proceeding. diff --git a/dev-sync.sh b/dev-sync.sh new file mode 100755 index 0000000..e4cb433 --- /dev/null +++ b/dev-sync.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# dev-sync.sh — copy this working tree into the installed /watch plugin cache so +# local edits are picked up by Claude Code without publishing a release. +# +# The install path is resolved from ~/.claude/plugins/installed_plugins.json, so +# it follows version bumps automatically. Override it by passing a path as $1 or +# setting WATCH_INSTALL_PATH. Pass --dry-run to preview without writing. +# +# Usage: +# ./dev-sync.sh # sync into the resolved install path +# ./dev-sync.sh --dry-run # show what would change +# ./dev-sync.sh /some/other/dir # sync into an explicit path +# +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLUGIN_KEY="watch@claude-video" +INSTALLED_JSON="${HOME}/.claude/plugins/installed_plugins.json" + +DRY_RUN=() +DEST="" +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=(--dry-run --itemize-changes) ;; + -*) echo "unknown flag: $arg" >&2; exit 2 ;; + *) DEST="$arg" ;; + esac +done + +# Resolve the destination install path if not given explicitly. +if [[ -z "$DEST" ]]; then + DEST="${WATCH_INSTALL_PATH:-}" +fi +if [[ -z "$DEST" ]]; then + if [[ ! -f "$INSTALLED_JSON" ]]; then + echo "error: $INSTALLED_JSON not found; pass an install path explicitly" >&2 + exit 1 + fi + DEST="$(PLUGIN_KEY="$PLUGIN_KEY" python3 - "$INSTALLED_JSON" <<'PY' +import json, os, sys +data = json.load(open(sys.argv[1])) +key = os.environ["PLUGIN_KEY"] +records = data.get("plugins", {}).get(key, []) +# Prefer a record whose installPath actually exists on disk. +paths = [r.get("installPath") for r in records if r.get("installPath")] +for p in paths: + if os.path.isdir(p): + print(p); break +else: + print(paths[0] if paths else "") +PY +)" +fi + +if [[ -z "$DEST" ]]; then + echo "error: could not resolve an install path for '$PLUGIN_KEY'" >&2 + echo " install the plugin first, or pass a path: scripts/dev-sync.sh /path/to/install" >&2 + exit 1 +fi +if [[ ! -d "$DEST" ]]; then + echo "error: install path does not exist: $DEST" >&2 + exit 1 +fi + +echo "source: $REPO_ROOT" +echo "dest: $DEST" +echo + +# Mirror shipped files only. Dev-only artifacts and runtime state are excluded. +# No --delete: the cache holds runtime state (.in_use/) we must not touch. +rsync -a ${DRY_RUN[@]+"${DRY_RUN[@]}"} \ + --exclude '.git/' \ + --exclude '.venv/' \ + --exclude '.pytest_cache/' \ + --exclude '__pycache__/' \ + --exclude '*.pyc' \ + --exclude '.DS_Store' \ + --exclude '.in_use/' \ + --exclude 'tests/' \ + --exclude 'docs/' \ + --exclude 'dist/' \ + --exclude 'V2_*.md' \ + --exclude 'dev-sync.sh' \ + "$REPO_ROOT/" "$DEST/" + +echo "done." diff --git a/hooks/scripts/check-setup.sh b/hooks/scripts/check-setup.sh index 0363b7a..7106640 100755 --- a/hooks/scripts/check-setup.sh +++ b/hooks/scripts/check-setup.sh @@ -50,7 +50,7 @@ fi # First-run / partially-configured → one-line hint. if [[ -z "$HAS_FFMPEG" || -z "$HAS_YTDLP" ]]; then - echo "/watch: needs ffmpeg + yt-dlp. Run \`python3 \$CLAUDE_PLUGIN_ROOT/scripts/setup.py\` once to install and scaffold config." + echo "/watch: needs ffmpeg + yt-dlp. Run \`python3 \$CLAUDE_PLUGIN_ROOT/skills/watch/scripts/setup.py\` once to install and scaffold config." elif [[ -z "$HAS_GROQ" && -z "$HAS_OPENAI" ]]; then echo "/watch: ready for videos with native captions. Add GROQ_API_KEY (preferred) or OPENAI_API_KEY to ~/.config/watch/.env to unlock Whisper fallback." else diff --git a/scripts/build-skill.sh b/scripts/build-skill.sh deleted file mode 100755 index 6ca9a6a..0000000 --- a/scripts/build-skill.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env bash -# build-skill.sh — package this repo as a claude.ai-upload-ready .skill file. -# Usage: bash scripts/build-skill.sh (run from repo root) -# -# Produces dist/watch.skill, a zip with a single top-level `watch/` directory -# containing SKILL.md and the scripts/ runtime. claude.ai's skill upload has a -# 200-file cap; `export-ignore` in .gitattributes + the zip -d strips below -# keep the bundle lean. -set -euo pipefail - -REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" -cd "$REPO_ROOT" - -if ! git diff --quiet || ! git diff --cached --quiet; then - echo "error: working tree is dirty; commit or stash before building" >&2 - exit 1 -fi - -mkdir -p dist -OUT="dist/watch.skill" -git archive --format=zip --prefix=watch/ --output="$OUT" HEAD - -# claude.ai's .skill bundle needs only SKILL.md + scripts/ runtime. Claude Code -# needs hooks/, commands/, and .claude-plugin/ in the git archive (that's why -# they are NOT in .gitattributes export-ignore), but the .skill bundle should -# strip them to keep a single canonical SKILL.md and stay well under the -# 200-file cap. -zip -d "$OUT" \ - "watch/hooks/*" \ - "watch/commands/*" \ - "watch/.claude-plugin/*" \ - > /dev/null 2>&1 || true - -COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}') -SIZE=$(du -h "$OUT" | cut -f1) - -if [ "$COUNT" -gt 200 ]; then - echo "error: $COUNT files in zip, claude.ai's cap is 200" >&2 - echo " check .gitattributes export-ignore entries and this script's zip -d excludes" >&2 - exit 1 -fi - -SKILL_MD_COUNT=$(unzip -l "$OUT" | grep -c "SKILL.md" || true) -if [ "$SKILL_MD_COUNT" -ne 1 ]; then - echo "error: expected exactly one SKILL.md, found $SKILL_MD_COUNT" >&2 - exit 1 -fi - -echo "built $OUT ($COUNT files, $SIZE)" -echo "upload via the claude.ai skill UI" diff --git a/scripts/frames.py b/scripts/frames.py deleted file mode 100755 index 01d205a..0000000 --- a/scripts/frames.py +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/env python3 -"""Probe video metadata and extract frames at an auto-scaled fps. - -Auto-fps targets a frame budget, not a fixed rate. Token cost scales with frame -count, so budget-by-duration keeps short videos dense and long videos capped. -When a user-specified range is passed, focused-mode budgets denser (they are -zooming in for detail). -""" -from __future__ import annotations - -import json -import shutil -import subprocess -import sys -from pathlib import Path - - -MAX_FPS = 2.0 - - -def _clamp_fps(fps: float, duration_seconds: float, max_frames: int) -> tuple[float, int]: - fps = min(fps, MAX_FPS) - target = min(max_frames, max(1, int(round(fps * duration_seconds)))) - return fps, target - - -def parse_time(value: str | float | int | None) -> float | None: - """Parse SS, MM:SS, or HH:MM:SS (with optional .ms) into seconds.""" - if value is None: - return None - if isinstance(value, (int, float)): - return float(value) - s = str(value).strip() - if not s: - return None - parts = s.split(":") - try: - if len(parts) == 1: - return float(parts[0]) - if len(parts) == 2: - return int(parts[0]) * 60 + float(parts[1]) - if len(parts) == 3: - return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2]) - except ValueError: - pass - raise SystemExit(f"Cannot parse time value: {value!r} (expected SS, MM:SS, or HH:MM:SS)") - - -def format_time(seconds: float) -> str: - total = int(round(seconds)) - hours, rem = divmod(total, 3600) - minutes, sec = divmod(rem, 60) - if hours: - return f"{hours}:{minutes:02d}:{sec:02d}" - return f"{minutes:02d}:{sec:02d}" - - -def get_metadata(video_path: str) -> dict: - if shutil.which("ffprobe") is None: - raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg") - - result = subprocess.run( - [ - "ffprobe", - "-v", "quiet", - "-print_format", "json", - "-show_format", - "-show_streams", - str(Path(video_path).resolve()), - ], - capture_output=True, - text=True, - ) - if result.returncode != 0: - raise SystemExit(f"ffprobe failed: {result.stderr.strip()}") - - data = json.loads(result.stdout or "{}") - streams = data.get("streams", []) - fmt = data.get("format", {}) - video_stream = next((s for s in streams if s.get("codec_type") == "video"), {}) - audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None) - - duration = float(fmt.get("duration") or video_stream.get("duration") or 0) - return { - "duration_seconds": duration, - "width": video_stream.get("width"), - "height": video_stream.get("height"), - "codec": video_stream.get("codec_name"), - "size_bytes": int(fmt.get("size") or 0), - "has_audio": audio_stream is not None, - } - - -def auto_fps(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]: - """Pick fps that targets a sensible frame budget for full-video scans.""" - if duration_seconds <= 0: - return 1.0, 1 - - if duration_seconds <= 30: - target = min(max_frames, max(12, int(round(duration_seconds)))) - elif duration_seconds <= 60: - target = min(max_frames, 40) - elif duration_seconds <= 180: # 3 min - target = min(max_frames, 60) - elif duration_seconds <= 600: # 10 min - target = min(max_frames, 80) - else: - target = max_frames - - return _clamp_fps(target / duration_seconds, duration_seconds, max_frames) - - -def auto_fps_focus(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]: - """Denser budget for user-specified ranges — they are zooming in for detail.""" - if duration_seconds <= 0: - return min(MAX_FPS, 2.0), 2 - - if duration_seconds <= 5: - target = min(max_frames, max(10, int(round(duration_seconds * 6)))) - elif duration_seconds <= 15: - target = min(max_frames, max(30, int(round(duration_seconds * 4)))) - elif duration_seconds <= 30: - target = min(max_frames, 60) - elif duration_seconds <= 60: - target = min(max_frames, 80) - elif duration_seconds <= 180: - target = max_frames - else: - target = max_frames - - return _clamp_fps(target / duration_seconds, duration_seconds, max_frames) - - -def extract( - video_path: str, - out_dir: Path, - fps: float, - resolution: int = 512, - max_frames: int = 100, - start_seconds: float | None = None, - end_seconds: float | None = None, -) -> list[dict]: - if shutil.which("ffmpeg") is None: - raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg") - - out_dir.mkdir(parents=True, exist_ok=True) - for existing in out_dir.glob("frame_*.jpg"): - existing.unlink() - - output_pattern = str(out_dir / "frame_%04d.jpg") - cmd: list[str] = [ - "ffmpeg", - "-hide_banner", - "-loglevel", "error", - "-y", - ] - - # -ss before -i = fast seek (keyframe-snap, good enough for preview frames). - if start_seconds is not None: - cmd += ["-ss", f"{start_seconds:.3f}"] - if end_seconds is not None: - cmd += ["-to", f"{end_seconds:.3f}"] - - cmd += [ - "-i", str(Path(video_path).resolve()), - "-vf", f"fps={fps},scale={resolution}:-2", - "-frames:v", str(max_frames), - "-q:v", "4", - output_pattern, - ] - - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - raise SystemExit(f"ffmpeg frame extraction failed: {result.stderr.strip()}") - - offset = start_seconds or 0.0 - frames = sorted(out_dir.glob("frame_*.jpg")) - return [ - { - "index": i, - "timestamp_seconds": round(offset + (i / fps if fps > 0 else 0.0), 2), - "path": str(p), - } - for i, p in enumerate(frames) - ] - - -if __name__ == "__main__": - if len(sys.argv) < 3: - print( - "usage: frames.py [--fps F] [--resolution W] " - "[--max-frames N] [--start T] [--end T]", - file=sys.stderr, - ) - raise SystemExit(2) - - video = sys.argv[1] - out = Path(sys.argv[2]) - args = sys.argv[3:] - - fps_override = None - resolution = 512 - max_frames = 100 - start_arg = None - end_arg = None - i = 0 - while i < len(args): - if args[i] == "--fps": - fps_override = float(args[i + 1]); i += 2 - elif args[i] == "--resolution": - resolution = int(args[i + 1]); i += 2 - elif args[i] == "--max-frames": - max_frames = int(args[i + 1]); i += 2 - elif args[i] == "--start": - start_arg = args[i + 1]; i += 2 - elif args[i] == "--end": - end_arg = args[i + 1]; i += 2 - else: - i += 1 - - meta = get_metadata(video) - start_sec = parse_time(start_arg) - end_sec = parse_time(end_arg) - full_duration = meta["duration_seconds"] - - effective_start = start_sec if start_sec is not None else 0.0 - effective_end = end_sec if end_sec is not None else full_duration - effective_duration = max(0.0, effective_end - effective_start) - - focused = start_sec is not None or end_sec is not None - if focused: - fps, target = auto_fps_focus(effective_duration, max_frames=max_frames) - else: - fps, target = auto_fps(effective_duration, max_frames=max_frames) - if fps_override is not None: - fps = fps_override - target = max(1, int(round(fps * effective_duration))) - - frames = extract( - video, out, - fps=fps, - resolution=resolution, - max_frames=max_frames, - start_seconds=start_sec, - end_seconds=end_sec, - ) - print(json.dumps( - {"meta": meta, "fps": fps, "target": target, "focused": focused, "frames": frames}, - indent=2, - )) diff --git a/scripts/watch.py b/scripts/watch.py deleted file mode 100755 index 8aa0600..0000000 --- a/scripts/watch.py +++ /dev/null @@ -1,230 +0,0 @@ -#!/usr/bin/env python3 -"""/watch entry point: download video, extract frames, parse transcript. - -Prints a markdown report to stdout listing frame paths + transcript. Claude -then Reads each frame path to see the video. -""" -from __future__ import annotations - -import argparse -import sys -import tempfile -from pathlib import Path - - -SCRIPT_DIR = Path(__file__).parent.resolve() -sys.path.insert(0, str(SCRIPT_DIR)) - -from download import download, is_url # noqa: E402 -from frames import MAX_FPS, auto_fps, auto_fps_focus, extract, format_time, get_metadata, parse_time # noqa: E402 -from transcribe import filter_range, format_transcript, parse_vtt # noqa: E402 -from whisper import load_api_key, transcribe_video # noqa: E402 - - -def main() -> int: - ap = argparse.ArgumentParser( - prog="watch", - description="Download a video, extract auto-scaled frames, and surface the transcript.", - ) - ap.add_argument("source", help="Video URL or local file path") - ap.add_argument("--max-frames", type=int, default=80, help="Cap on frame count (default 80, hard max 100)") - ap.add_argument("--resolution", type=int, default=512, help="Frame width in pixels (default 512)") - ap.add_argument("--fps", type=float, default=None, help="Override auto-fps") - ap.add_argument("--start", type=str, default=None, help="Range start (SS, MM:SS, or HH:MM:SS)") - ap.add_argument("--end", type=str, default=None, help="Range end (SS, MM:SS, or HH:MM:SS)") - ap.add_argument("--out-dir", type=str, default=None, help="Working directory (default: tmp)") - ap.add_argument( - "--no-whisper", - action="store_true", - help="Disable Whisper fallback. Report frames-only if no captions available.", - ) - ap.add_argument( - "--whisper", - choices=["groq", "openai"], - default=None, - help="Force a specific Whisper backend. Default: prefer Groq, fall back to OpenAI.", - ) - args = ap.parse_args() - - max_frames = min(args.max_frames, 100) - - if args.out_dir: - work = Path(args.out_dir).expanduser().resolve() - else: - work = Path(tempfile.mkdtemp(prefix="watch-")) - work.mkdir(parents=True, exist_ok=True) - print(f"[watch] working dir: {work}", file=sys.stderr) - - print( - "[watch] downloading via yt-dlp…" if is_url(args.source) else "[watch] using local file…", - file=sys.stderr, - ) - dl = download(args.source, work / "download") - video_path = dl["video_path"] - - meta = get_metadata(video_path) - full_duration = meta["duration_seconds"] - - start_sec = parse_time(args.start) - end_sec = parse_time(args.end) - - if start_sec is not None and start_sec < 0: - raise SystemExit("--start must be non-negative") - if end_sec is not None and start_sec is not None and end_sec <= start_sec: - raise SystemExit("--end must be greater than --start") - if full_duration > 0 and start_sec is not None and start_sec >= full_duration: - raise SystemExit(f"--start {start_sec:.1f}s is past end of video ({full_duration:.1f}s)") - - effective_start = start_sec if start_sec is not None else 0.0 - effective_end = end_sec if end_sec is not None else full_duration - effective_duration = max(0.0, effective_end - effective_start) - focused = start_sec is not None or end_sec is not None - - if focused: - fps, target = auto_fps_focus(effective_duration, max_frames=max_frames) - else: - fps, target = auto_fps(effective_duration, max_frames=max_frames) - if args.fps is not None: - fps = min(args.fps, MAX_FPS) - target = max(1, int(round(fps * effective_duration))) - - scope = ( - f"{format_time(effective_start)}-{format_time(effective_end)} ({effective_duration:.1f}s)" - if focused else f"full {effective_duration:.1f}s" - ) - print(f"[watch] extracting ~{target} frames at {fps:.3f} fps over {scope}…", file=sys.stderr) - - frames = extract( - video_path, - work / "frames", - fps=fps, - resolution=args.resolution, - max_frames=max_frames, - start_seconds=start_sec, - end_seconds=end_sec, - ) - - transcript_segments: list[dict] = [] - transcript_text: str | None = None - transcript_source: str | None = None - if dl.get("subtitle_path"): - try: - all_segments = parse_vtt(dl["subtitle_path"]) - transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments - transcript_text = format_transcript(transcript_segments) - transcript_source = "captions" - except Exception as exc: - print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr) - - if not transcript_segments and not args.no_whisper: - backend, api_key = load_api_key(args.whisper) - if backend and api_key: - try: - all_segments, used_backend = transcribe_video( - video_path, - work / "audio.mp3", - backend=backend, - api_key=api_key, - ) - transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments - transcript_text = format_transcript(transcript_segments) - transcript_source = f"whisper ({used_backend})" - except SystemExit as exc: - print(f"[watch] whisper fallback failed: {exc}", file=sys.stderr) - else: - hint = ( - f"--whisper {args.whisper} was set but the matching API key is missing" - if args.whisper else - "no subtitles and no Whisper API key found" - ) - setup_py = SCRIPT_DIR / "setup.py" - print( - f"[watch] {hint} — run `python3 {setup_py}` to enable the Whisper fallback", - file=sys.stderr, - ) - - info = dl.get("info") or {} - - print() - print("# watch: video report") - print() - print(f"- **Source:** {args.source}") - if info.get("title"): - print(f"- **Title:** {info['title']}") - if info.get("uploader"): - print(f"- **Uploader:** {info['uploader']}") - print(f"- **Duration:** {format_time(full_duration)} ({full_duration:.1f}s)") - if focused: - print( - f"- **Focus range:** {format_time(effective_start)} → {format_time(effective_end)} " - f"({effective_duration:.1f}s)" - ) - if meta.get("width") and meta.get("height"): - print(f"- **Resolution:** {meta['width']}x{meta['height']} ({meta.get('codec') or 'unknown codec'})") - mode = "focused" if focused else "full" - print(f"- **Frames:** {len(frames)} @ {fps:.3f} fps, {mode} mode (budget {target}, max {max_frames})") - print(f"- **Frame size:** {args.resolution}px wide") - if transcript_segments: - in_range = " in range" if focused else "" - print( - f"- **Transcript:** {len(transcript_segments)} segments{in_range} " - f"(via {transcript_source or 'captions'})" - ) - else: - print("- **Transcript:** none available") - - if not focused and full_duration > 600: - mins = int(full_duration // 60) - print() - print( - f"> **Warning:** This is a {mins}-minute video. Frame coverage is sparse at this length — " - "accuracy degrades noticeably on anything over 10 minutes. For better results, " - "re-run with `--start HH:MM:SS --end HH:MM:SS` to zoom into a specific section." - ) - - print() - print("## Frames") - print() - print(f"Frames live at: `{work / 'frames'}`") - print() - print( - "**Read each frame path below with the Read tool to view the image.** " - "Frames are in chronological order; `t=MM:SS` is the absolute timestamp in the source video." - ) - print() - for frame in frames: - print(f"- `{frame['path']}` (t={format_time(frame['timestamp_seconds'])})") - - print() - print("## Transcript") - print() - if transcript_text: - label = transcript_source or "captions" - if focused: - print(f"_Source: {label}. Filtered to {format_time(effective_start)} → {format_time(effective_end)}:_") - else: - print(f"_Source: {label}._") - print() - print("```") - print(transcript_text) - print("```") - elif focused and dl.get("subtitle_path"): - print(f"_No transcript lines fell inside {format_time(effective_start)} → {format_time(effective_end)}._") - else: - setup_py = SCRIPT_DIR / "setup.py" - print( - "_No transcript available — proceed with frames only. " - "Captions were missing and the Whisper fallback was unavailable " - "(no API key set, or `--no-whisper` was used). " - f"Run `python3 {setup_py}` to enable Whisper, then re-run._" - ) - - print() - print("---") - print(f"_Work dir: `{work}` — delete when done._") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/skills/watch/.skillignore b/skills/watch/.skillignore new file mode 100644 index 0000000..a0bf873 --- /dev/null +++ b/skills/watch/.skillignore @@ -0,0 +1,3 @@ +# Scans that start from this skill directory (not the repo root). +# Keep non-runtime packaging/dev artifacts out of install-time security scans. +scripts/build-skill.sh diff --git a/skills/watch/SKILL.md b/skills/watch/SKILL.md new file mode 100644 index 0000000..4b7e034 --- /dev/null +++ b/skills/watch/SKILL.md @@ -0,0 +1,267 @@ +--- +name: watch +version: "0.1.3" +description: Watch a video (URL or local path). Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls the transcript from captions (or Whisper API fallback), and hands the result to Claude so it can answer questions about what's in the video. +argument-hint: " [question]" +allowed-tools: Bash, Read, AskUserQuestion +homepage: https://github.com/bradautomates/claude-video +repository: https://github.com/bradautomates/claude-video +author: bradautomates +license: MIT +user-invocable: true +--- + +# /watch + +You don't have a video input; this skill gives you one. A Python script gets captions first, optionally downloads the video, extracts frames as JPEGs (scene-aware, or fast keyframes at `efficient` detail), gets a timestamped transcript (native captions first, then Whisper API as fallback), and prints frame paths. You then `Read` each frame path to see the images and combine them with the transcript to answer the user. + +## Resolve `SKILL_DIR` (do this before any command) + +Every `python3 ...` command below runs a bundled script under `SKILL_DIR/scripts/`. Set `SKILL_DIR` to the **absolute path of the directory containing THIS SKILL.md you just Read** — your harness told you that path in the Read result. The scripts are always a direct sibling of this file (`SKILL_DIR/scripts/watch.py`), in every install layout: + +``` +Read ~/.claude/plugins/cache/claude-video/watch//skills/watch/SKILL.md → SKILL_DIR=…/skills/watch +Read ~/.codex/skills/watch/SKILL.md → SKILL_DIR=~/.codex/skills/watch +Read ~/.agents/skills/watch/SKILL.md → SKILL_DIR=~/.agents/skills/watch +``` + +Substitute that literal path for `${SKILL_DIR}` in every command. This works on every harness (Claude Code, Codex, Cursor, Gemini CLI, …) without relying on any harness-specific environment variable. Guard once at the start of a run: + +```bash +SKILL_DIR="" +if [ ! -f "$SKILL_DIR/scripts/watch.py" ]; then + echo "ERROR: scripts/watch.py not found under SKILL_DIR=$SKILL_DIR" >&2 + echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR." >&2 + exit 1 +fi +``` + +## Step 0 — Setup preflight (runs every `/watch` invocation, silent on success) + +**Python interpreter:** every `python3 ...` command in this skill is for macOS/Linux. On **Windows**, substitute `python` — the `python3` command on Windows is the Microsoft Store stub and will not run the script. + +On the first `/watch` invocation in a session, use structured preflight so you can detect first-run setup: + +```bash +python3 "${SKILL_DIR}/scripts/setup.py" --json +``` + +Branch on two fields: + +- **`can_proceed: true` and `first_run: false`** → setup is already done (the user may have deliberately skipped a Whisper key — that's allowed). Proceed to Step 1 without comment. +- **`first_run: true`** → genuine first-time setup. Do these in order: + 1. If `missing_binaries` is non-empty, run the installer first (it auto-installs on macOS / prints commands elsewhere — see below) and confirm the binaries land. **Do not skip this and jump to preferences.** + 2. Run the installer once more if needed so it scaffolds `~/.config/watch/.env` (it only writes the template when the file is absent, so let it create the file *before* you write any values into it). + 3. Encourage a Whisper API key and ask the watch-preference questions below, then write the selected values into `~/.config/watch/.env` and set `SETUP_COMPLETE=true`. +- **`can_proceed: false` and `first_run: false`** → setup was finished before but the environment regressed (e.g. `missing_binaries` after an OS change). Run the installer to remediate, then proceed. Don't re-ask preferences. + +A missing Whisper key is *encouraged to fix, not required*: on a genuine first run `status` will read `needs_key` even when binaries are present — that's your cue to encourage a key, not a blocker. + +On follow-up `/watch` calls in the same session, use the silent check: + +```bash +python3 "${SKILL_DIR}/scripts/setup.py" --check +``` + +This is a <100ms lookup. Exit 0 means /watch can run — this **includes a user who finished setup without a Whisper key** (keyless is allowed). On exit 0 the script emits **nothing** — proceed to Step 1 without comment. **Do NOT announce "setup is complete" to the user** — they don't need a status message on every turn. The only acceptable user-visible output from Step 0 is when remediation is required. + +On non-zero exit, follow the table: + +| Exit | Meaning | Action | +|------|---------|--------| +| `2` | Missing binaries (`ffmpeg` / `ffprobe` / `yt-dlp`) | Run installer | +| `3` | Genuine first run with no Whisper API key | Run installer to scaffold `.env`, then encourage a key (the user may decline — proceed with `--no-whisper`) | +| `4` | Both missing | Run installer, then encourage a key | + +Exit `3` only fires before the user has completed setup. Once `SETUP_COMPLETE=true` is written, a keyless install returns exit 0 and is never nagged again. + +The installer is idempotent — safe to re-run: + +```bash +python3 "${SKILL_DIR}/scripts/setup.py" +``` + +On macOS with Homebrew, it auto-installs `ffmpeg` and `yt-dlp`. On Linux/Windows, it prints the exact install commands for the user to run. It scaffolds `~/.config/watch/.env` with commented placeholders and default watch settings at `0600` perms. + +**If an API key is still missing after install:** use `AskUserQuestion` to ask the user whether they have a Groq API key (preferred — cheaper, faster) or an OpenAI key. Then write it into `~/.config/watch/.env` — set the matching `GROQ_API_KEY=...` or `OPENAI_API_KEY=...` line. If they don't want to set up Whisper, proceed with `--no-whisper` and tell them videos without native captions will come back frames-only. + +**First-run watch preference:** after the installer has scaffolded `~/.config/watch/.env`, use `AskUserQuestion` to ask one question: + +- Default detail (one dial). Present these as `AskUserQuestion` options in this exact order — lightest to heaviest — and keep `(recommended)` on `balanced` even though it is not first (do **not** reorder to put the recommended option first): + - `transcript` — no frames at all, transcript only (skips video download when captions exist). + - `efficient` — fast keyframe pass (cap 50). + - `balanced` (recommended) — scene-aware frames (cap 100, default). + - `token-burner` — scene-aware, uncapped (maximum fidelity; high token cost). + +Write the answer directly into `~/.config/watch/.env` by setting: + +```bash +WATCH_DETAIL=balanced +``` + +Use the user's selected value. If they skip the question, keep the recommended default. Once dependencies, the API-key choice, and this preference are handled, write or update `SETUP_COMPLETE=true` in the same file. Do not ask this preference question again when `SETUP_COMPLETE=true`. + +**Structured mode (optional):** `python3 "${SKILL_DIR}/scripts/setup.py" --json` emits `{status, can_proceed, first_run, setup_complete, missing_binaries, whisper_backend, has_api_key, config_file, watch_detail, platform}` where `status` is one of `ready | needs_install | needs_key | needs_install_and_key`. `status` describes the *ideal* state (a key is encouraged, so a keyless first run reads `needs_key`); `can_proceed` is the operational gate (binaries present AND a key is set OR setup was already completed). Branch on `can_proceed`/`first_run` to decide whether to run; use `status` to decide what to encourage. + +Within a single session, you can skip Step 0 on follow-up `/watch` calls — once `--check` returned 0, nothing about the environment changes between turns. + +## When to use + +- User pastes a video URL (YouTube, Vimeo, X, TikTok, Twitch clip, most yt-dlp-supported sites) and asks about it. +- User points at a local video file (`.mp4`, `.mov`, `.mkv`, `.webm`, etc.) and asks about it. +- User types `/watch [question]`. + +## Recommended limits + +- **Best accuracy: videos under 10 minutes.** Frame coverage scales inversely with duration. +- **Universal rate cap: 2 fps.** The script never samples faster than 2 fps, even when a budget or `--fps` would imply more. +- **The frame ceiling is set by the detail mode** (`WATCH_DETAIL` in `~/.config/watch/.env`, or `--detail`), not a single global cap: + - `transcript` → no frames + - `efficient` → up to **50** (keyframes) + - `balanced` (default) → up to **100** (scene-aware) + - `token-burner` → **uncapped** (scene-aware; a soft warning prints past 250 frames) + - `--max-frames N` overrides whichever cap the mode would otherwise use. +- **Full-video frame budget by duration.** Token cost grows with frame count, so the script targets a budget by duration. This budget sets the fps and the uniform-sampling fallback; scene-aware selection can fill up to the detail cap above, whichever is lower: + - ≤30s → ~12-30 frames + - 30s-1min → ~40 frames + - 1-3min → ~60 frames + - 3-10min → ~80 frames + - \>10min → up to the detail cap, sparsely spaced (warning printed) +- If the user hands you a long video, consider asking whether they want a specific section before burning tokens on a sparse scan. + +## How to invoke + +**Step 1 — parse the user input.** Separate the video source (URL or path) from any question the user asked. Example: `/watch https://youtu.be/abc what language is this in?` → source = `https://youtu.be/abc`, question = `what language is this in?`. + +**Step 2 — run the watch script.** Pass the source verbatim. Do not shell-escape it yourself beyond normal quoting: + +```bash +python3 "${SKILL_DIR}/scripts/watch.py" "" +``` + +Optional flags: +- `--detail transcript|efficient|balanced|token-burner` — fidelity/speed dial. `transcript` = no frames (transcript only, skips video download when captions exist); `efficient` = fast keyframes (cap 50); `balanced` = scene-aware frames (cap 100); `token-burner` = scene-aware, uncapped. +- `--start T` / `--end T` — focus on a section. Accepts `SS`, `MM:SS`, or `HH:MM:SS`. When either is set, fps auto-scales denser (see "Focusing on a section" below). +- `--timestamps T1,T2,…` — grab a frame at each of these absolute timestamps (`SS`, `MM:SS`, or `HH:MM:SS`). Use this after reading the transcript to capture deictic moments the presenter flags ("look here", "as you can see", "notice this") that visual selection alone may miss. See "Transcript-cue frames" below. +- `--max-frames N` — override the preset cap for tighter token budget (e.g. `--max-frames 40`) +- `--resolution W` — change frame width in px (default 512; bump to 1024 only if the user needs to read on-screen text) +- `--fps F` — override auto-fps (clamped to 2 fps max) +- `--out-dir DIR` — keep working files somewhere specific (default: an auto-generated tmp dir) +- `--whisper groq|openai` — force a specific Whisper backend (default: prefer Groq if both keys exist) +- `--no-whisper` — disable the Whisper fallback entirely (frames-only if no captions) + +### Focusing on a section (higher frame rate) + +When the user asks about a specific moment — "what happens at the 2 minute mark?", "zoom into 0:45 to 1:00", "the first 10 seconds" — pass `--start` and/or `--end`. The script switches to focused-mode budgets, which are denser than full-video budgets (still capped at 2 fps, and still bounded by the detail-mode cap — the counts below assume the default `balanced` cap of 100; `efficient` tops out at 50): + +- ≤5s → 2 fps (up to 10 frames) +- 5-15s → 2 fps (up to 30 frames) +- 15-30s → ~2 fps (up to 60 frames) +- 30-60s → ~1.3 fps (up to 80 frames) +- 60-180s → ~0.6 fps (100 frames, capped) + +Focused mode is the right call for: +- Any moment/range the user names explicitly ("around 2:30", "the intro", "the last 30 seconds"). +- Any video longer than ~10 minutes where the user's question is about a specific part — running focused on the relevant section is far more useful than a sparse scan of the whole thing. +- Re-runs after a full scan didn't have enough detail in some region. + +Transcript is auto-filtered to the same range. Frame timestamps are absolute (real video timeline, not offset-from-start). + +Examples: +```bash +# Last 10 seconds of a 1 minute video +python3 "${SKILL_DIR}/scripts/watch.py" video.mp4 --start 50 --end 60 + +# Zoom into 2:15 → 2:45 at 2 fps (60 frames) +python3 "${SKILL_DIR}/scripts/watch.py" "$URL" --start 2:15 --end 2:45 --fps 2 + +# From 1h12m to the end of the video +python3 "${SKILL_DIR}/scripts/watch.py" "$URL" --start 1:12:00 +``` + +**Step 3 — Read every frame path the script lists.** The Read tool renders JPEGs directly as images for you. Read all frames in a single message (parallel tool calls) so you see them together. The frames are in chronological order with a `t=MM:SS` timestamp so you can align them to the transcript. + +**Step 4 — answer the user.** You now have two streams of evidence: +- **Frames** — what's on screen at each timestamp +- **Transcript** — what's said at each timestamp. The report's header shows the source (`captions` = yt-dlp pulled native subs; `whisper (groq)` or `whisper (openai)` = transcribed by API). + +If the user asked a specific question, answer it directly citing timestamps. If they didn't ask anything, summarize what happens in the video — structure, key moments, notable visuals, spoken content. + +This holds for `transcript` detail too: even with no frames, produce a **summary** like the other modes — do not paste the full transcript into chat. Synthesize structure, key moments, and spoken content with timestamps; quote only the lines that matter. Offer the raw transcript only if the user explicitly asks for it. + +**Step 5 — clean up.** The script prints a working directory at the end. If the user isn't going to ask follow-ups about this video, delete it with `rm -rf `. If they might, leave it in place. + +## Detail and frames + +Default behavior comes from `~/.config/watch/.env`: + +- `WATCH_DETAIL=transcript|efficient|balanced|token-burner` (default: `balanced`) + +At `transcript` detail, captions are enough to return a report without downloading video. If captions are missing, the script downloads audio only and tries Whisper. If no transcript can be produced, it reports the limitation clearly; re-run with `--detail balanced` for frames. + +At `efficient` detail, the script downloads the video and extracts **keyframes only** (`ffmpeg -skip_frame nokey`) — a near-instant pass that lands frames on scene cuts. If a clip has fewer than 4 keyframes it falls back to uniform sampling. + +At `balanced` / `token-burner` detail, the script extracts **scene-aware** frames: ffmpeg scene-change selection first, falling back to uniform sampling only when the video is effectively static. `balanced` caps at 100 frames; `token-burner` is uncapped. Frame report lines include both timestamp and selection reason. Extracted images are clamped to a maximum 1998px height for Claude Read compatibility. + +## Transcript-cue frames + +Visual frame selection (scene/keyframe) can miss the moments a presenter explicitly flags — "look here", "as you can see", "notice this", "watch what happens" — because pointing at a slide is often a *low* visual change. `--timestamps` lets you force a frame at those exact moments. **You** decide which moments matter, by reading the transcript: + +1. Run once at `--detail transcript` (or any detail) to get the timestamped transcript. +2. Scan it for deictic cues — phrases where the speaker directs attention to something on screen. This is a judgment call (ignore rhetorical "look, the point is…"); that's why it's done by you, not a regex. +3. Re-run with `--timestamps 4:32,7:10,9:55` (absolute source times). For a URL, point the second run at the **downloaded local file** in the work dir so it doesn't re-download. + +Behavior: +- **Additive by default.** Cue frames (`reason=transcript-cue`) are merged into whatever `--detail` already selected, in chronological order. +- **Pinned and counted first.** Cue frames are reserved against the frame cap before the detail engine runs, so they're never evicted by even-sampling. +- **Honors focus mode.** With `--start/--end`, any cue timestamp outside the window is dropped (reported in the summary). Coordinates are always absolute source time. +- **Cue-only frames.** `--detail transcript --timestamps …` skips scene/keyframe sampling and returns *only* the cue frames (it will download the video to do so, since frames need pixels). + +## Transcription + +The script gets a timestamped transcript in one of two ways: + +1. **Native captions (free, preferred).** yt-dlp pulls manual or auto-generated subtitles from the source platform if available. +2. **Whisper API fallback.** If no captions came back (or the source is a local file), the script extracts audio (`ffmpeg -vn -ac 1 -ar 16000 -b:a 64k`, ~0.5 MB/min) and uploads it to whichever Whisper API has a key configured: + - **Groq** — `whisper-large-v3`. Preferred default: cheaper, faster. Get a key at console.groq.com/keys. + - **OpenAI** — `whisper-1`. Fallback. Get a key at platform.openai.com/api-keys. + +Both keys live in `~/.config/watch/.env`. The script prefers Groq when both are set; override with `--whisper openai` to force OpenAI. Use `--no-whisper` to skip the fallback entirely. + +## Failure modes and handling + +- **Setup preflight failed** → run `python3 "${SKILL_DIR}/scripts/setup.py"` (auto-installs ffmpeg/yt-dlp via brew on macOS, scaffolds the `.env`). For API key, ask the user via `AskUserQuestion` and write it to `~/.config/watch/.env`. +- **No transcript available** → captions missing AND (no Whisper key OR Whisper API failed). Script prints a hint pointing to setup. Proceed frames-only and tell the user. +- **Long video warning printed** → acknowledge it in your answer. Offer to re-run focused on a specific section via `--start`/`--end` rather than a sparse full-video scan. +- **Download fails** → yt-dlp's error goes to stderr. If it's a login-required or region-locked video, tell the user plainly; do not keep retrying. +- **Whisper request fails** → the error is printed to stderr (likely: invalid key, rate limit, or 25 MB upload limit on a very long video). The report will say "none available" for transcript. You can retry with `--whisper openai` if Groq failed (or vice versa). + +## Token efficiency + +This skill burns tokens primarily on frames. Order of magnitude: +- 80 frames at 512px wide is roughly 50-80k image tokens depending on aspect ratio. +- The transcript is cheap (a few thousand tokens at most for a 10-minute video). +- Bumping `--resolution` to 1024 roughly quadruples the image tokens per frame. Only do it when necessary. + +If you already watched a video this session and the user asks a follow-up, do **not** re-run the script — you already have the frames and transcript in context. Just answer from what you have. + +## Security & Permissions + +**What this skill does:** +- Runs `yt-dlp` locally to download the video and pull native captions when the source supports them (public data; the request goes directly to whatever host the URL points at) +- Runs `ffmpeg` / `ffprobe` locally to extract frames as JPEGs and, when Whisper is needed, a mono 16 kHz audio clip +- Sends the extracted audio clip to Groq's Whisper API (`api.groq.com/openai/v1/audio/transcriptions`) when `GROQ_API_KEY` is set (preferred — cheaper, faster) +- Sends the extracted audio clip to OpenAI's audio transcription API (`api.openai.com/v1/audio/transcriptions`) when `OPENAI_API_KEY` is set and Groq is not, or when `--whisper openai` is forced +- Writes the downloaded video, frames, audio, and an intermediate transcript to a working directory under the system temp dir (or `--out-dir` if specified) so Claude can `Read` them +- Reads / creates `~/.config/watch/.env` (mode `0600`) to store the Whisper API key(s) and a `SETUP_COMPLETE` marker. As a fallback, also reads `.env` in the current working directory + +**What this skill does NOT do:** +- Does not upload the video itself to any API — only the extracted audio goes out, and only when native captions are missing AND Whisper is not disabled with `--no-whisper` +- Does not access any platform account (no login, no session cookies, no posting) — yt-dlp only ever requests public data +- Does not share API keys between providers (Groq key only goes to `api.groq.com`, OpenAI key only goes to `api.openai.com`) +- Does not log, cache, or write API keys to stdout, stderr, or output files +- Does not persist anything outside the working directory and `~/.config/watch/.env` — clean up the working directory when you're done (Step 5) + +**Bundled scripts:** `scripts/watch.py` (entry point), `scripts/download.py` (yt-dlp wrapper), `scripts/frames.py` (ffmpeg frame extraction), `scripts/transcribe.py` (caption selection + Whisper orchestration), `scripts/whisper.py` (Groq / OpenAI clients), `scripts/setup.py` (preflight + installer) + +Review scripts before first use to verify behavior. diff --git a/skills/watch/scripts/build-skill.sh b/skills/watch/scripts/build-skill.sh new file mode 100755 index 0000000..a65f8f9 --- /dev/null +++ b/skills/watch/scripts/build-skill.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# build-skill.sh — package the watch skill as a claude.ai-upload-ready .skill file. +# Usage: bash skills/watch/scripts/build-skill.sh (run from anywhere) +# +# Produces dist/watch.skill, a zip with a single top-level `watch/` directory +# containing SKILL.md and the scripts/ runtime from skills/watch. Archiving the +# skills/watch subtree directly keeps the bundle to exactly one SKILL.md and +# well under claude.ai's 200-file cap, with no post-hoc `zip -d` stripping. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +cd "$REPO_ROOT" + +if ! git diff --quiet || ! git diff --cached --quiet; then + echo "error: working tree is dirty; commit or stash before building" >&2 + exit 1 +fi + +mkdir -p dist +OUT="dist/watch.skill" +git archive --format=zip --prefix=watch/ --output="$OUT" HEAD:skills/watch + +COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}') +SIZE=$(du -h "$OUT" | cut -f1) + +if [ "$COUNT" -gt 200 ]; then + echo "error: $COUNT files in zip, claude.ai's cap is 200" >&2 + echo " trim the skills/watch/ tree or add a .gitattributes export-ignore entry" >&2 + exit 1 +fi + +SKILL_MD_COUNT=$(unzip -l "$OUT" | grep -c "SKILL.md" || true) +if [ "$SKILL_MD_COUNT" -ne 1 ]; then + echo "error: expected exactly one SKILL.md, found $SKILL_MD_COUNT" >&2 + exit 1 +fi + +echo "built $OUT ($COUNT files, $SIZE)" +echo "upload via the claude.ai skill UI" diff --git a/skills/watch/scripts/config.py b/skills/watch/scripts/config.py new file mode 100644 index 0000000..93f70d1 --- /dev/null +++ b/skills/watch/scripts/config.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Shared /watch configuration helpers.""" +from __future__ import annotations + +import os +from pathlib import Path + + +CONFIG_DIR = Path.home() / ".config" / "watch" +CONFIG_FILE = CONFIG_DIR / ".env" + +DEFAULT_DETAIL = "balanced" + +DETAILS = {"transcript", "efficient", "balanced", "token-burner"} + + +def read_env_file(path: Path | None = None) -> dict[str, str]: + if path is None: + path = CONFIG_FILE + values: dict[str, str] = {} + if not path.exists(): + return values + try: + lines = path.read_text(encoding="utf-8").splitlines() + except OSError: + return values + for line in lines: + raw = line.strip() + if not raw or raw.startswith("#") or "=" not in raw: + continue + key, _, value = raw.partition("=") + value = value.strip() + if len(value) >= 2 and value[0] in ('"', "'") and value[-1] == value[0]: + value = value[1:-1] + values[key.strip()] = value + return values + + +def get_config() -> dict[str, object]: + file_values = read_env_file() + + detail = ( + os.environ.get("WATCH_DETAIL") + or file_values.get("WATCH_DETAIL") + or DEFAULT_DETAIL + ) + if detail not in DETAILS: + detail = DEFAULT_DETAIL + + return { + "detail": detail, + "config_file": str(CONFIG_FILE), + } + + +def frame_cap(detail: str) -> int | None: + if detail == "efficient": + return 50 + if detail == "balanced": + return 100 + if detail == "token-burner": + return None + if detail == "transcript": + return None + return 100 diff --git a/scripts/download.py b/skills/watch/scripts/download.py similarity index 67% rename from scripts/download.py rename to skills/watch/scripts/download.py index afa1ef4..f188b4f 100755 --- a/scripts/download.py +++ b/skills/watch/scripts/download.py @@ -45,12 +45,15 @@ def _pick_subtitle(out_dir: Path) -> Path | None: candidates = sorted(out_dir.glob("video*.vtt")) if not candidates: return None - preferred = [c for c in candidates if ".en" in c.name] + preferred = [ + c for c in candidates + if any(marker in c.name for marker in (".en.", ".en-US.", ".en-GB.", ".en-orig.")) + ] return preferred[0] if preferred else candidates[0] def _pick_video(out_dir: Path) -> Path | None: - for ext in (".mp4", ".mkv", ".webm", ".mov"): + for ext in (".mp4", ".mkv", ".webm", ".mov", ".m4a", ".mp3", ".opus"): for candidate in out_dir.glob(f"video*{ext}"): return candidate for candidate in out_dir.glob("video.*"): @@ -59,22 +62,77 @@ def _pick_video(out_dir: Path) -> Path | None: return None -def download_url(url: str, out_dir: Path) -> dict: +def fetch_captions(url: str, out_dir: Path) -> dict: + """Fetch metadata and best available VTT captions without downloading video.""" + if shutil.which("yt-dlp") is None: + raise SystemExit("yt-dlp is not installed. Install with: brew install yt-dlp") + + out_dir.mkdir(parents=True, exist_ok=True) + output_template = str(out_dir / "video.%(ext)s") + cmd = [ + "yt-dlp", + "--skip-download", + "--write-info-json", + "--write-subs", + "--write-auto-subs", + "--sub-langs", "en.*", + "--sub-format", "vtt", + "--convert-subs", "vtt", + "--no-playlist", + "--ignore-errors", + "-o", output_template, + "--", + url, + ] + subprocess.run(cmd, stdout=sys.stderr, stderr=sys.stderr) + subtitle = _pick_subtitle(out_dir) + info = _read_info(out_dir / "video.info.json", url) + return { + "video_path": None, + "subtitle_path": str(subtitle) if subtitle else None, + "info": info or {"url": url}, + "downloaded": False, + } + + +def _read_info(info_path: Path, url: str) -> dict: + info: dict = {} + if info_path.exists(): + try: + raw = json.loads(info_path.read_text(encoding="utf-8")) + info = { + "title": raw.get("title"), + "uploader": raw.get("uploader") or raw.get("channel"), + "duration": raw.get("duration"), + "url": raw.get("webpage_url") or url, + } + except Exception as exc: + print(f"[watch] info.json parse failed: {exc}", file=sys.stderr) + info = {"url": url} + return info + + +def download_url( + url: str, + out_dir: Path, + audio_only: bool = False, +) -> dict: if shutil.which("yt-dlp") is None: raise SystemExit("yt-dlp is not installed. Install with: brew install yt-dlp") out_dir.mkdir(parents=True, exist_ok=True) output_template = str(out_dir / "video.%(ext)s") + fmt = "ba/bestaudio" if audio_only else "bv*[height<=720]+ba/b[height<=720]/bv+ba/b" cmd = [ "yt-dlp", "-N", "8", - "-f", "bv*[height<=720]+ba/b[height<=720]/bv+ba/b", + "-f", fmt, "--merge-output-format", "mp4", "--write-info-json", "--write-subs", "--write-auto-subs", - "--sub-langs", "en,en-US,en-GB,en-orig", + "--sub-langs", "en.*", "--sub-format", "vtt", "--convert-subs", "vtt", "--no-playlist", @@ -94,20 +152,7 @@ def download_url(url: str, out_dir: Path) -> dict: ) subtitle = _pick_subtitle(out_dir) - info_path = out_dir / "video.info.json" - info: dict = {} - if info_path.exists(): - try: - raw = json.loads(info_path.read_text(encoding="utf-8")) - info = { - "title": raw.get("title"), - "uploader": raw.get("uploader") or raw.get("channel"), - "duration": raw.get("duration"), - "url": raw.get("webpage_url") or url, - } - except Exception as exc: - print(f"[watch] info.json parse failed: {exc}", file=sys.stderr) - info = {"url": url} + info = _read_info(out_dir / "video.info.json", url) return { "video_path": str(video), @@ -117,9 +162,13 @@ def download_url(url: str, out_dir: Path) -> dict: } -def download(source: str, out_dir: Path) -> dict: +def download( + source: str, + out_dir: Path, + audio_only: bool = False, +) -> dict: if is_url(source): - return download_url(source, out_dir) + return download_url(source, out_dir, audio_only=audio_only) return resolve_local(source) diff --git a/skills/watch/scripts/frames.py b/skills/watch/scripts/frames.py new file mode 100755 index 0000000..6f891e4 --- /dev/null +++ b/skills/watch/scripts/frames.py @@ -0,0 +1,628 @@ +#!/usr/bin/env python3 +"""Probe video metadata and extract frames at an auto-scaled fps. + +Auto-fps targets a frame budget, not a fixed rate. Token cost scales with frame +count, so budget-by-duration keeps short videos dense and long videos capped. +When a user-specified range is passed, focused-mode budgets denser (they are +zooming in for detail). +""" +from __future__ import annotations + +import json +import re +import shutil +import subprocess +import sys +from pathlib import Path + + +MAX_FPS = 2.0 +SCENE_THRESHOLD = 0.20 +# Keep scene-detection results once we have at least this many distinct shots. +# Below this the video is effectively static (screen recording, talking head), +# so we fall back to uniform sampling. Matching the reference fork's behaviour, +# this is a low floor — NOT the frame budget — so normal videos with cuts use +# the (single-pass) scene engine instead of paying for a wasted second decode. +SCENE_MIN_FRAMES = 8 +# Below this many decoded keyframes a clip is too sparse for keyframe coverage +# (very short or oddly encoded), so the cheap tier falls back to uniform. +KEYFRAME_MIN = 4 +MAX_READ_DIMENSION = 1998 +SHOWINFO_TS_RE = re.compile(r"pts_time:([0-9.]+)") + + +def _scale_filter(resolution: int) -> str: + return ( + f"scale=w='min({resolution},iw)':h='min({MAX_READ_DIMENSION},ih)':" + "force_original_aspect_ratio=decrease:force_divisible_by=2" + ) + + +def _clamp_fps(fps: float, duration_seconds: float, max_frames: int) -> tuple[float, int]: + fps = min(fps, MAX_FPS) + target = min(max_frames, max(1, int(round(fps * duration_seconds)))) + return fps, target + + +def parse_time(value: str | float | int | None) -> float | None: + """Parse SS, MM:SS, or HH:MM:SS (with optional .ms) into seconds.""" + if value is None: + return None + if isinstance(value, (int, float)): + return float(value) + s = str(value).strip() + if not s: + return None + parts = s.split(":") + try: + if len(parts) == 1: + return float(parts[0]) + if len(parts) == 2: + return int(parts[0]) * 60 + float(parts[1]) + if len(parts) == 3: + return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2]) + except ValueError: + pass + raise SystemExit(f"Cannot parse time value: {value!r} (expected SS, MM:SS, or HH:MM:SS)") + + +def format_time(seconds: float) -> str: + total = int(round(seconds)) + hours, rem = divmod(total, 3600) + minutes, sec = divmod(rem, 60) + if hours: + return f"{hours}:{minutes:02d}:{sec:02d}" + return f"{minutes:02d}:{sec:02d}" + + +def get_metadata(video_path: str) -> dict: + if shutil.which("ffprobe") is None: + raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg") + + result = subprocess.run( + [ + "ffprobe", + "-v", "quiet", + "-print_format", "json", + "-show_format", + "-show_streams", + str(Path(video_path).resolve()), + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise SystemExit(f"ffprobe failed: {result.stderr.strip()}") + + data = json.loads(result.stdout or "{}") + streams = data.get("streams", []) + fmt = data.get("format", {}) + video_stream = next((s for s in streams if s.get("codec_type") == "video"), {}) + audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None) + + duration = float(fmt.get("duration") or video_stream.get("duration") or 0) + return { + "duration_seconds": duration, + "width": video_stream.get("width"), + "height": video_stream.get("height"), + "codec": video_stream.get("codec_name"), + "size_bytes": int(fmt.get("size") or 0), + "has_audio": audio_stream is not None, + } + + +def auto_fps(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]: + """Pick fps that targets a sensible frame budget for full-video scans.""" + if duration_seconds <= 0: + return 1.0, 1 + + if duration_seconds <= 30: + target = min(max_frames, max(12, int(round(duration_seconds)))) + elif duration_seconds <= 60: + target = min(max_frames, 40) + elif duration_seconds <= 180: # 3 min + target = min(max_frames, 60) + elif duration_seconds <= 600: # 10 min + target = min(max_frames, 80) + else: + target = max_frames + + return _clamp_fps(target / duration_seconds, duration_seconds, max_frames) + + +def auto_fps_focus(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]: + """Denser budget for user-specified ranges — they are zooming in for detail.""" + if duration_seconds <= 0: + return min(MAX_FPS, 2.0), 2 + + if duration_seconds <= 5: + target = min(max_frames, max(10, int(round(duration_seconds * 6)))) + elif duration_seconds <= 15: + target = min(max_frames, max(30, int(round(duration_seconds * 4)))) + elif duration_seconds <= 30: + target = min(max_frames, 60) + elif duration_seconds <= 60: + target = min(max_frames, 80) + elif duration_seconds <= 180: + target = max_frames + else: + target = max_frames + + return _clamp_fps(target / duration_seconds, duration_seconds, max_frames) + + +def extract( + video_path: str, + out_dir: Path, + fps: float, + resolution: int = 512, + max_frames: int = 100, + start_seconds: float | None = None, + end_seconds: float | None = None, +) -> list[dict]: + if shutil.which("ffmpeg") is None: + raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg") + + out_dir.mkdir(parents=True, exist_ok=True) + for existing in out_dir.glob("frame_*.jpg"): + existing.unlink() + + output_pattern = str(out_dir / "frame_%04d.jpg") + cmd: list[str] = [ + "ffmpeg", + "-hide_banner", + "-loglevel", "error", + "-y", + ] + + # -ss before -i = fast seek (keyframe-snap, good enough for preview frames). + if start_seconds is not None: + cmd += ["-ss", f"{start_seconds:.3f}"] + if end_seconds is not None: + cmd += ["-to", f"{end_seconds:.3f}"] + + cmd += [ + "-i", str(Path(video_path).resolve()), + "-vf", f"fps={fps},{_scale_filter(resolution)}", + "-frames:v", str(max_frames), + "-q:v", "4", + output_pattern, + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise SystemExit(f"ffmpeg frame extraction failed: {result.stderr.strip()}") + + offset = start_seconds or 0.0 + frames = sorted(out_dir.glob("frame_*.jpg")) + return [ + { + "index": i, + "timestamp_seconds": round(offset + (i / fps if fps > 0 else 0.0), 2), + "path": str(p), + "reason": "uniform", + } + for i, p in enumerate(frames) + ] + + +def extract_scene_candidates( + video_path: str, + out_dir: Path, + resolution: int = 512, + max_frames: int | None = 100, + start_seconds: float | None = None, + end_seconds: float | None = None, + threshold: float = SCENE_THRESHOLD, +) -> list[dict]: + """Extract first frame plus ffmpeg scene-change frames. + + When ``max_frames`` is set, ``-frames:v`` lets ffmpeg stop decoding once it + has emitted that many frames (early exit) and avoids writing extras that we + would only delete afterwards. ``None`` (uncapped "complete" detail) keeps + every detected shot, as the user explicitly opted in. + """ + if shutil.which("ffmpeg") is None: + raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg") + + out_dir.mkdir(parents=True, exist_ok=True) + for existing in out_dir.glob("frame_*.jpg"): + existing.unlink() + + output_pattern = str(out_dir / "frame_%04d.jpg") + cmd: list[str] = [ + "ffmpeg", + "-hide_banner", + "-loglevel", "info", + "-y", + ] + if start_seconds is not None: + cmd += ["-ss", f"{start_seconds:.3f}"] + if end_seconds is not None: + cmd += ["-to", f"{end_seconds:.3f}"] + + vf = f"select='eq(n\\,0)+gt(scene\\,{threshold})',{_scale_filter(resolution)},showinfo" + cmd += [ + "-i", str(Path(video_path).resolve()), + "-vf", vf, + "-vsync", "vfr", + ] + if max_frames is not None: + cmd += ["-frames:v", str(max_frames)] + cmd += [ + "-q:v", "4", + output_pattern, + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise SystemExit(f"ffmpeg scene extraction failed: {result.stderr.strip()}") + + offset = start_seconds or 0.0 + timestamps = [round(offset + float(match.group(1)), 2) for match in SHOWINFO_TS_RE.finditer(result.stderr)] + frames = sorted(out_dir.glob("frame_*.jpg")) + out: list[dict] = [] + for i, path in enumerate(frames): + ts = timestamps[i] if i < len(timestamps) else offset + out.append({ + "index": i, + "timestamp_seconds": ts, + "path": str(path), + "reason": "first-frame" if i == 0 else "scene-change", + }) + return out + + +def _even_indices(count: int, n: int) -> list[int]: + """Indices of ``n`` evenly-spaced items out of ``count`` (first + last kept). + + ``n >= count`` returns every index; ``n == 1`` returns just the first. + """ + if n >= count: + return list(range(count)) + if n <= 1: + return [0] + return [round(i * (count - 1) / (n - 1)) for i in range(n)] + + +def parse_timestamps(value: str | None) -> list[float]: + """Parse a comma-separated list of times (SS, MM:SS, HH:MM:SS) into a + sorted, de-duplicated list of seconds. Empty/blank tokens are skipped; + an unparseable token raises (via :func:`parse_time`).""" + if not value: + return [] + out: list[float] = [] + for token in value.split(","): + token = token.strip() + if not token: + continue + seconds = parse_time(token) + if seconds is not None: + out.append(float(seconds)) + return sorted(set(out)) + + +def merge_frames(primary: list[dict], pinned: list[dict]) -> list[dict]: + """Combine two frame lists into one chronological list and reindex 0..n-1. + + ``pinned`` frames (transcript cues) are never dropped — this is a plain + union, so the cap is enforced upstream by reserving budget for the cues. + """ + merged = sorted([*primary, *pinned], key=lambda f: f["timestamp_seconds"]) + for i, frame in enumerate(merged): + frame["index"] = i + return merged + + +def extract_at_timestamps( + video_path: str, + out_dir: Path, + timestamps: list[float], + resolution: int = 512, + max_frames: int | None = None, + start_seconds: float | None = None, + end_seconds: float | None = None, +) -> tuple[list[dict], dict]: + """Grab exactly one frame at each requested timestamp (transcript cues). + + Timestamps are absolute source seconds. Any falling outside an active + ``[start, end]`` focus window are dropped. Files use a ``cue_*.jpg`` prefix + so they sit alongside detail-engine ``frame_*.jpg`` output without either + clobbering the other. When more cues than ``max_frames`` survive, they are + even-sampled (first + last kept) before extraction. + """ + if shutil.which("ffmpeg") is None: + raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg") + + out_dir.mkdir(parents=True, exist_ok=True) + for existing in out_dir.glob("cue_*.jpg"): + existing.unlink() + + lo = start_seconds or 0.0 + hi = end_seconds if end_seconds is not None else float("inf") + requested = sorted(set(round(float(t), 2) for t in timestamps)) + in_window = [t for t in requested if lo <= t <= hi] + dropped = len(requested) - len(in_window) + + if max_frames is not None and len(in_window) > max_frames: + points = [in_window[i] for i in _even_indices(len(in_window), max_frames)] + else: + points = in_window + + out: list[dict] = [] + for t in points: + path = out_dir / f"cue_{len(out):04d}.jpg" + cmd = [ + "ffmpeg", + "-hide_banner", + "-loglevel", "error", + "-y", + "-ss", f"{t:.3f}", + "-i", str(Path(video_path).resolve()), + "-frames:v", "1", + "-vf", _scale_filter(resolution), + "-q:v", "4", + str(path), + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode == 0 and path.exists(): + out.append({ + "index": len(out), + "timestamp_seconds": t, + "path": str(path), + "reason": "transcript-cue", + }) + + meta = { + "engine": "timestamps", + "candidate_count": len(requested), + "selected_count": len(out), + "dropped_out_of_window": dropped, + "fallback": False, + } + return out, meta + + +def _even_sample(candidates: list[dict], n: int) -> list[dict]: + """Pick ``n`` evenly-spaced candidates (always including first and last), + delete the JPEGs we drop, and reindex the survivors 0..len-1. + + Shared by every capped engine so all detail modes sample the same way: + detect all candidates across the full range, then thin down to the cap. + ``n >= len(candidates)`` keeps everything (the uncapped / under-cap case). + """ + selected = [candidates[i] for i in _even_indices(len(candidates), n)] + + keep_paths = {sel["path"] for sel in selected} + for cand in candidates: + if cand["path"] not in keep_paths: + try: + Path(cand["path"]).unlink() + except OSError: + pass + for i, frame in enumerate(selected): + frame["index"] = i + return selected + + +def extract_scene_or_uniform( + video_path: str, + out_dir: Path, + fps: float, + target_frames: int, + resolution: int = 512, + max_frames: int | None = 100, + start_seconds: float | None = None, + end_seconds: float | None = None, +) -> tuple[list[dict], dict]: + """Prefer scene selection, falling back to uniform only when the video is + effectively static (fewer than ``SCENE_MIN_FRAMES`` detected shots). + + Scene cuts are detected across the *whole* range (uncapped) and then + even-sampled down to ``max_frames`` via :func:`_even_sample`, exactly like + the keyframe engine. This costs a full decode, but it guarantees coverage + spans the entire clip — capping detection with ``-frames:v`` instead would + keep only the first ``max_frames`` cuts and drop the tail of long videos + (and could even fall below ``SCENE_MIN_FRAMES`` and misfire the uniform + fallback on a cut-heavy clip). + """ + scene_frames = extract_scene_candidates( + video_path, + out_dir, + resolution=resolution, + max_frames=None, + start_seconds=start_seconds, + end_seconds=end_seconds, + ) + scene_count = len(scene_frames) + if scene_count >= SCENE_MIN_FRAMES: + cap = scene_count if max_frames is None else max_frames + selected = _even_sample(scene_frames, cap) + return selected, { + "engine": "scene", + "candidate_count": scene_count, + "selected_count": len(selected), + "fallback": False, + } + + fallback_cap = target_frames if max_frames is None else min(max_frames, target_frames) + frames = extract( + video_path, + out_dir, + fps=fps, + resolution=resolution, + max_frames=fallback_cap, + start_seconds=start_seconds, + end_seconds=end_seconds, + ) + return frames, { + "engine": "uniform", + "candidate_count": scene_count, + "selected_count": len(frames), + "fallback": True, + } + + +def extract_keyframes( + video_path: str, + out_dir: Path, + resolution: int = 512, + max_frames: int | None = 50, + start_seconds: float | None = None, + end_seconds: float | None = None, +) -> tuple[list[dict], dict]: + """Decode only keyframes (I-frames) — the cheap, near-instant tier. + + ``-skip_frame nokey`` makes ffmpeg reconstruct only keyframes, skipping all + P/B frames. Encoders emit keyframes at scene cuts, so these already + approximate "distinct moments". Over-cap → even-sample first→last; too few + keyframes → uniform fallback. + """ + if shutil.which("ffmpeg") is None: + raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg") + + out_dir.mkdir(parents=True, exist_ok=True) + for existing in out_dir.glob("frame_*.jpg"): + existing.unlink() + + output_pattern = str(out_dir / "frame_%04d.jpg") + cmd: list[str] = [ + "ffmpeg", + "-hide_banner", + "-loglevel", "info", + "-y", + ] + if start_seconds is not None: + cmd += ["-ss", f"{start_seconds:.3f}"] + if end_seconds is not None: + cmd += ["-to", f"{end_seconds:.3f}"] + cmd += [ + "-skip_frame", "nokey", + "-i", str(Path(video_path).resolve()), + "-vf", f"{_scale_filter(resolution)},showinfo", + "-vsync", "vfr", + "-q:v", "4", + output_pattern, + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise SystemExit(f"ffmpeg keyframe extraction failed: {result.stderr.strip()}") + + offset = start_seconds or 0.0 + timestamps = [round(offset + float(m.group(1)), 2) for m in SHOWINFO_TS_RE.finditer(result.stderr)] + files = sorted(out_dir.glob("frame_*.jpg")) + candidates: list[dict] = [] + for i, path in enumerate(files): + ts = timestamps[i] if i < len(timestamps) else offset + candidates.append({ + "index": i, + "timestamp_seconds": ts, + "path": str(path), + "reason": "keyframe", + }) + + # Too few keyframes → uniform fallback over the same range. + if len(candidates) < KEYFRAME_MIN: + for cand in candidates: + try: + Path(cand["path"]).unlink() + except OSError: + pass + meta = get_metadata(video_path) + full_duration = meta["duration_seconds"] + eff_start = start_seconds or 0.0 + eff_end = end_seconds if end_seconds is not None else full_duration + eff_duration = max(0.0, eff_end - eff_start) + budget = max_frames if max_frames is not None else 100 + fps, _ = auto_fps(eff_duration, max_frames=budget) + frames_out = extract( + video_path, + out_dir, + fps=fps, + resolution=resolution, + max_frames=budget, + start_seconds=start_seconds, + end_seconds=end_seconds, + ) + return frames_out, { + "engine": "uniform", + "candidate_count": len(candidates), + "selected_count": len(frames_out), + "fallback": True, + } + + # Detect-all then even-sample down to the cap (first + last always kept). + # ``max_frames is None`` (uncapped) keeps every keyframe. + candidate_count = len(candidates) + cap = candidate_count if max_frames is None else max_frames + selected = _even_sample(candidates, cap) + return selected, { + "engine": "keyframe", + "candidate_count": candidate_count, + "selected_count": len(selected), + "fallback": False, + } + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print( + "usage: frames.py [--fps F] [--resolution W] " + "[--max-frames N] [--start T] [--end T]", + file=sys.stderr, + ) + raise SystemExit(2) + + video = sys.argv[1] + out = Path(sys.argv[2]) + args = sys.argv[3:] + + fps_override = None + resolution = 512 + max_frames = 100 + start_arg = None + end_arg = None + i = 0 + while i < len(args): + if args[i] == "--fps": + fps_override = float(args[i + 1]); i += 2 + elif args[i] == "--resolution": + resolution = int(args[i + 1]); i += 2 + elif args[i] == "--max-frames": + max_frames = int(args[i + 1]); i += 2 + elif args[i] == "--start": + start_arg = args[i + 1]; i += 2 + elif args[i] == "--end": + end_arg = args[i + 1]; i += 2 + else: + i += 1 + + meta = get_metadata(video) + start_sec = parse_time(start_arg) + end_sec = parse_time(end_arg) + full_duration = meta["duration_seconds"] + + effective_start = start_sec if start_sec is not None else 0.0 + effective_end = end_sec if end_sec is not None else full_duration + effective_duration = max(0.0, effective_end - effective_start) + + focused = start_sec is not None or end_sec is not None + if focused: + fps, target = auto_fps_focus(effective_duration, max_frames=max_frames) + else: + fps, target = auto_fps(effective_duration, max_frames=max_frames) + if fps_override is not None: + fps = fps_override + target = max(1, int(round(fps * effective_duration))) + + frames = extract( + video, out, + fps=fps, + resolution=resolution, + max_frames=max_frames, + start_seconds=start_sec, + end_seconds=end_sec, + ) + print(json.dumps( + {"meta": meta, "fps": fps, "target": target, "focused": focused, "frames": frames}, + indent=2, + )) diff --git a/scripts/setup.py b/skills/watch/scripts/setup.py similarity index 85% rename from scripts/setup.py rename to skills/watch/scripts/setup.py index b82d4c1..5582eb4 100755 --- a/scripts/setup.py +++ b/skills/watch/scripts/setup.py @@ -26,6 +26,11 @@ import subprocess import sys from pathlib import Path +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) +from config import get_config # noqa: E402 + REQUIRED_BINARIES = ["ffmpeg", "ffprobe", "yt-dlp"] CONFIG_DIR = Path.home() / ".config" / "watch" @@ -46,6 +51,9 @@ ENV_TEMPLATE = """# /watch API configuration GROQ_API_KEY= OPENAI_API_KEY= + +# Default watch behavior (the /watch first-run wizard sets this for you): +# WATCH_DETAIL=balanced # transcript | efficient | balanced | token-burner """ @@ -57,11 +65,19 @@ def _check_binaries() -> list[str]: return [b for b in REQUIRED_BINARIES if not _which(b)] +_PERM_WARNED: set[str] = set() + + def _check_file_permissions(path: Path) -> None: - """Warn to stderr if a secrets file is world/group readable.""" + """Warn to stderr (once per path per process) if a secrets file is + world/group readable.""" + key = str(path) + if key in _PERM_WARNED: + return try: mode = path.stat().st_mode if mode & 0o044: + _PERM_WARNED.add(key) sys.stderr.write( f"[watch] WARNING: {path} is readable by other users. " f"Run: chmod 600 {path}\n" @@ -197,9 +213,20 @@ def _install_hint_windows(missing: list[str]) -> str: def _status() -> dict: - """Structured preflight snapshot.""" + """Structured preflight snapshot. + + `status` describes the *ideal* state (a Whisper key is encouraged), so a + keyless install still reports `needs_key` on the very first run — that's + the agent's cue to encourage adding one. + + `can_proceed` is the operational gate: /watch can run as long as the + binaries are present AND the user has either set a key or already finished + setup (consciously opting out of Whisper). A keyless user who completed + setup is NOT nagged on every call. + """ missing = _check_binaries() has_key, backend = _have_api_key() + setup_complete = not is_first_run() if not missing and has_key: status = "ready" @@ -210,13 +237,19 @@ def _status() -> dict: else: status = "needs_key" + can_proceed = (not missing) and (has_key or setup_complete) + + cfg = get_config() return { "status": status, - "first_run": is_first_run(), + "can_proceed": can_proceed, + "first_run": not setup_complete, + "setup_complete": setup_complete, "missing_binaries": missing, "whisper_backend": backend, "has_api_key": has_key, "config_file": str(CONFIG_FILE), + "watch_detail": cfg["detail"], "platform": platform.system(), } @@ -224,20 +257,23 @@ def _status() -> dict: def cmd_check() -> int: """Silent-on-success preflight. - Exit 0 with no output when ready. On failure, print one actionable line - to stderr and return: + Exit 0 with no output when /watch can run. A keyless user who already + finished setup (SETUP_COMPLETE=true) counts as ready — Whisper is + encouraged, not required — so they are never nagged on follow-up calls. + + On a state that blocks /watch, print one actionable line to stderr: 2 → binaries missing - 3 → API key missing + 3 → genuine first run with no API key (encourage one) 4 → both missing """ s = _status() - if s["status"] == "ready": + if s["can_proceed"]: return 0 parts = [] if s["missing_binaries"]: parts.append(f"missing binaries: {', '.join(s['missing_binaries'])}") - if not s["has_api_key"]: + if not s["has_api_key"] and not s["setup_complete"]: parts.append("no Whisper API key (GROQ_API_KEY or OPENAI_API_KEY)") installer = Path(__file__).resolve() sys.stderr.write( diff --git a/scripts/transcribe.py b/skills/watch/scripts/transcribe.py similarity index 100% rename from scripts/transcribe.py rename to skills/watch/scripts/transcribe.py diff --git a/skills/watch/scripts/watch.py b/skills/watch/scripts/watch.py new file mode 100755 index 0000000..20dc294 --- /dev/null +++ b/skills/watch/scripts/watch.py @@ -0,0 +1,382 @@ +#!/usr/bin/env python3 +"""/watch entry point: download video, extract frames, parse transcript. + +Prints a markdown report to stdout listing frame paths + transcript. Claude +then Reads each frame path to see the video. +""" +from __future__ import annotations + +import argparse +import sys +import tempfile +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).parent.resolve() +sys.path.insert(0, str(SCRIPT_DIR)) + +from config import frame_cap, get_config # noqa: E402 +from download import download, fetch_captions, is_url # noqa: E402 +from frames import MAX_FPS, auto_fps, auto_fps_focus, extract_at_timestamps, extract_keyframes, extract_scene_or_uniform, format_time, get_metadata, merge_frames, parse_time, parse_timestamps # noqa: E402 +from transcribe import filter_range, format_transcript, parse_vtt # noqa: E402 +from whisper import load_api_key, transcribe_video # noqa: E402 + + +def main() -> int: + ap = argparse.ArgumentParser( + prog="watch", + description="Download a video, extract auto-scaled frames, and surface the transcript.", + ) + ap.add_argument("source", help="Video URL or local file path") + ap.add_argument("--max-frames", type=int, default=None, help="Override frame cap") + ap.add_argument("--resolution", type=int, default=512, help="Frame width in pixels (default 512)") + ap.add_argument("--fps", type=float, default=None, help="Override auto-fps") + ap.add_argument( + "--detail", + choices=["transcript", "efficient", "balanced", "token-burner"], + default=None, + help="Fidelity/speed dial: transcript (no frames), efficient (fast keyframes, cap 50), " + "balanced (scene, cap 100), token-burner (scene, uncapped).", + ) + ap.add_argument( + "--timestamps", + type=str, + default=None, + help="Comma-separated absolute timestamps (SS, MM:SS, HH:MM:SS) to grab a frame at, " + "e.g. transcript-flagged 'look here' moments. Added on top of the detail frames " + "(reserved against the cap); with --detail transcript these become the only frames.", + ) + ap.add_argument("--start", type=str, default=None, help="Range start (SS, MM:SS, or HH:MM:SS)") + ap.add_argument("--end", type=str, default=None, help="Range end (SS, MM:SS, or HH:MM:SS)") + ap.add_argument("--out-dir", type=str, default=None, help="Working directory (default: tmp)") + ap.add_argument( + "--no-whisper", + action="store_true", + help="Disable Whisper fallback. Report frames-only if no captions available.", + ) + ap.add_argument( + "--whisper", + choices=["groq", "openai"], + default=None, + help="Force a specific Whisper backend. Default: prefer Groq, fall back to OpenAI.", + ) + args = ap.parse_args() + + config = get_config() + detail = args.detail or str(config["detail"]) + configured_cap = frame_cap(detail) + if args.max_frames is not None: + max_frames = args.max_frames + else: + max_frames = configured_cap + if max_frames is not None and max_frames < 1: + raise SystemExit("--max-frames must be greater than zero") + budget_cap = max_frames if max_frames is not None else 100 + cue_timestamps = parse_timestamps(args.timestamps) + + if args.out_dir: + work = Path(args.out_dir).expanduser().resolve() + else: + work = Path(tempfile.mkdtemp(prefix="watch-")) + work.mkdir(parents=True, exist_ok=True) + print(f"[watch] working dir: {work}", file=sys.stderr) + + url_source = is_url(args.source) + dl: dict = {"subtitle_path": None, "info": {}, "downloaded": False} + transcript_segments: list[dict] = [] + transcript_text: str | None = None + transcript_source: str | None = None + video_path: str | None = None + + if url_source: + print("[watch] checking metadata/captions via yt-dlp…", file=sys.stderr) + dl = fetch_captions(args.source, work / "download") + if dl.get("subtitle_path"): + try: + transcript_segments = parse_vtt(dl["subtitle_path"]) + transcript_text = format_transcript(transcript_segments) + transcript_source = "captions" + except Exception as exc: + print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr) + transcript_segments = [] + + # --timestamps needs the video for frame grabs, so it overrides the + # transcript-mode download skip (and forces a full, not audio-only, fetch). + audio_only = detail == "transcript" and not cue_timestamps + if detail == "transcript" and transcript_segments and not cue_timestamps: + video_path = None + else: + if url_source: + print( + "[watch] downloading audio via yt-dlp…" if audio_only + else "[watch] downloading video via yt-dlp…", + file=sys.stderr, + ) + dl = download( + args.source, + work / "download", + audio_only=audio_only, + ) + else: + print("[watch] using local file…", file=sys.stderr) + dl = download(args.source, work / "download") + video_path = dl["video_path"] + + meta = get_metadata(video_path) if video_path else { + "duration_seconds": float((dl.get("info") or {}).get("duration") or 0), + "width": None, + "height": None, + "codec": None, + "has_audio": False, + } + full_duration = meta["duration_seconds"] + + start_sec = parse_time(args.start) + end_sec = parse_time(args.end) + + if start_sec is not None and start_sec < 0: + raise SystemExit("--start must be non-negative") + if end_sec is not None and start_sec is not None and end_sec <= start_sec: + raise SystemExit("--end must be greater than --start") + if full_duration > 0 and start_sec is not None and start_sec >= full_duration: + raise SystemExit(f"--start {start_sec:.1f}s is past end of video ({full_duration:.1f}s)") + + effective_start = start_sec if start_sec is not None else 0.0 + effective_end = end_sec if end_sec is not None else full_duration + effective_duration = max(0.0, effective_end - effective_start) + focused = start_sec is not None or end_sec is not None + + if focused: + fps, target = auto_fps_focus(effective_duration, max_frames=budget_cap) + else: + fps, target = auto_fps(effective_duration, max_frames=budget_cap) + if args.fps is not None: + fps = min(args.fps, MAX_FPS) + target = max(1, int(round(fps * effective_duration))) + + if transcript_segments and focused: + transcript_segments = filter_range(transcript_segments, start_sec, end_sec) + transcript_text = format_transcript(transcript_segments) + + scope = ( + f"{format_time(effective_start)}-{format_time(effective_end)} ({effective_duration:.1f}s)" + if focused else f"full {effective_duration:.1f}s" + ) + frames: list[dict] = [] + frame_meta: dict = {"engine": "none", "candidate_count": 0, "selected_count": 0, "fallback": False} + cue_frames: list[dict] = [] + cue_meta: dict = {} + + # Transcript cues are pinned: extracted first and counted against the cap so + # the detail engine never evicts the moments the user explicitly asked for. + if cue_timestamps and video_path: + cue_frames, cue_meta = extract_at_timestamps( + video_path, + work / "frames", + cue_timestamps, + resolution=args.resolution, + max_frames=max_frames, + start_seconds=start_sec, + end_seconds=end_sec, + ) + if cue_meta.get("dropped_out_of_window"): + print( + f"[watch] {cue_meta['dropped_out_of_window']} cue timestamp(s) outside the " + "focus range — dropped", + file=sys.stderr, + ) + + detail_budget = max_frames if max_frames is None else max(0, max_frames - len(cue_frames)) + if detail != "transcript" and video_path and detail_budget != 0: + cap_label = "unlimited" if detail_budget is None else str(detail_budget) + engine_label = "keyframes" if detail == "efficient" else "scene-aware frames" + print( + f"[watch] extracting {engine_label} over {scope} " + f"(target {target}, cap {cap_label})…", + file=sys.stderr, + ) + if detail == "efficient": + frames, frame_meta = extract_keyframes( + video_path, + work / "frames", + resolution=args.resolution, + max_frames=detail_budget, + start_seconds=start_sec, + end_seconds=end_sec, + ) + else: # balanced, token-burner + frames, frame_meta = extract_scene_or_uniform( + video_path, + work / "frames", + fps=fps, + target_frames=target, + resolution=args.resolution, + max_frames=detail_budget, + start_seconds=start_sec, + end_seconds=end_sec, + ) + + if cue_frames: + frames = merge_frames(frames, cue_frames) + + if not transcript_segments and dl.get("subtitle_path"): + try: + all_segments = parse_vtt(dl["subtitle_path"]) + transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments + transcript_text = format_transcript(transcript_segments) + transcript_source = "captions" + except Exception as exc: + print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr) + + if not transcript_segments and not args.no_whisper and video_path and meta.get("has_audio"): + backend, api_key = load_api_key(args.whisper) + if backend and api_key: + try: + all_segments, used_backend = transcribe_video( + video_path, + work / "audio.mp3", + backend=backend, + api_key=api_key, + ) + transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments + transcript_text = format_transcript(transcript_segments) + transcript_source = f"whisper ({used_backend})" + except SystemExit as exc: + print(f"[watch] whisper fallback failed: {exc}", file=sys.stderr) + else: + hint = ( + f"--whisper {args.whisper} was set but the matching API key is missing" + if args.whisper else + "no subtitles and no Whisper API key found" + ) + setup_py = SCRIPT_DIR / "setup.py" + print( + f"[watch] {hint} — run `python3 {setup_py}` to enable the Whisper fallback", + file=sys.stderr, + ) + elif not transcript_segments and video_path and not meta.get("has_audio"): + print("[watch] no audio stream found — proceeding without transcription", file=sys.stderr) + + info = dl.get("info") or {} + + print() + print("# watch: video report") + print() + print(f"- **Source:** {args.source}") + if info.get("title"): + print(f"- **Title:** {info['title']}") + if info.get("uploader"): + print(f"- **Uploader:** {info['uploader']}") + print(f"- **Duration:** {format_time(full_duration)} ({full_duration:.1f}s)") + if focused: + print( + f"- **Focus range:** {format_time(effective_start)} → {format_time(effective_end)} " + f"({effective_duration:.1f}s)" + ) + if meta.get("width") and meta.get("height"): + print(f"- **Resolution:** {meta['width']}x{meta['height']} ({meta.get('codec') or 'unknown codec'})") + range_mode = "focused" if focused else "full" + print(f"- **Detail:** {detail}") + detail_count = frame_meta.get("selected_count", 0) + if detail != "transcript": + cap_label = "unlimited" if detail_budget is None else str(detail_budget) + engine = frame_meta.get("engine", "scene") + fallback = " with uniform fallback" if frame_meta.get("fallback") else "" + print( + f"- **Frames:** {detail_count} selected from {frame_meta.get('candidate_count', detail_count)} " + f"candidates ({engine}{fallback}, {range_mode} range, budget {target}, cap {cap_label})" + ) + elif not cue_frames: + print("- **Frames:** skipped (transcript detail)") + if cue_frames: + dropped = cue_meta.get("dropped_out_of_window", 0) + drop_note = f", {dropped} dropped outside range" if dropped else "" + print( + f"- **Cue frames:** {len(cue_frames)} at transcript-flagged timestamps " + f"(transcript-cue{drop_note})" + ) + if frames: + print(f"- **Frame size:** max {args.resolution}px wide, max 1998px tall") + if transcript_segments: + in_range = " in range" if focused else "" + print( + f"- **Transcript:** {len(transcript_segments)} segments{in_range} " + f"(via {transcript_source or 'captions'})" + ) + else: + print("- **Transcript:** none available") + + if detail == "token-burner" and len(frames) > 250: + print() + print( + f"> **Warning:** token-burner detail selected {len(frames)} frames. " + "This may use a large number of image tokens." + ) + + if not focused and full_duration > 600 and detail != "transcript": + mins = int(full_duration // 60) + print() + print( + f"> **Warning:** This is a {mins}-minute video. Frame coverage is sparse at this length — " + "accuracy degrades noticeably on anything over 10 minutes. For better results, " + "re-run with `--start HH:MM:SS --end HH:MM:SS` to zoom into a specific section." + ) + + print() + print("## Frames") + print() + if frames: + print(f"Frames live at: `{work / 'frames'}`") + print() + print( + "**Read each frame path below with the Read tool to view the image.** " + "Frames are in chronological order; `t=MM:SS` is the absolute timestamp in the source video." + ) + print() + for frame in frames: + print( + f"- `{frame['path']}` " + f"(t={format_time(frame['timestamp_seconds'])}, reason={frame.get('reason', 'selected')})" + ) + else: + print("_No frames extracted._") + + print() + print("## Transcript") + print() + if transcript_text: + label = transcript_source or "captions" + if focused: + print(f"_Source: {label}. Filtered to {format_time(effective_start)} → {format_time(effective_end)}:_") + else: + print(f"_Source: {label}._") + print() + print("```") + print(transcript_text) + print("```") + elif detail == "transcript": + print( + "_No transcript available at transcript detail. Captions were missing and Whisper was " + "unavailable or failed, so there is no visual fallback here. Re-run with " + "`--detail balanced` for frames._" + ) + elif focused and dl.get("subtitle_path"): + print(f"_No transcript lines fell inside {format_time(effective_start)} → {format_time(effective_end)}._") + else: + setup_py = SCRIPT_DIR / "setup.py" + print( + "_No transcript available — proceed with frames only. " + "Captions were missing and the Whisper fallback was unavailable " + "(no API key set, or `--no-whisper` was used). " + f"Run `python3 {setup_py}` to enable Whisper, then re-run._" + ) + + print() + print("---") + print(f"_Work dir: `{work}` — delete when done._") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/whisper.py b/skills/watch/scripts/whisper.py similarity index 100% rename from scripts/whisper.py rename to skills/watch/scripts/whisper.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..068b179 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,83 @@ +"""Shared pytest fixtures: ffmpeg-synthesized clips and scripts/ on sys.path.""" +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +# Make the bundled scripts importable (mirrors watch.py's sys.path insert). +SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) + +# 14 visually distinct fills → 14 abrupt cuts → x264 emits a keyframe per cut. +COLORS = [ + "red", "green", "blue", "white", "black", "yellow", "cyan", + "magenta", "gray", "orange", "purple", "brown", "navy", "olive", +] + + +def _run(cmd: list[str]) -> None: + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"ffmpeg failed: {' '.join(cmd)}\n{result.stderr}") + + +def build_cut_clip( + path: Path, + n: int = 14, + seg: float = 0.4, + size: str = "320x240", + fps: int = 10, +) -> None: + """Concatenate ``n`` solid-color segments into one clip with ``n`` cuts. + + Each color change is a hard scene cut, so the scene selector finds ~n-1 + changes. x264's own scenecut detection is unreliable on flat fills, so we + force a keyframe at every ``seg`` boundary — giving ~n real keyframes for + the keyframe engine to find. + """ + inputs: list[str] = [] + for i in range(n): + color = COLORS[i % len(COLORS)] + inputs += ["-f", "lavfi", "-t", str(seg), "-i", f"color=c={color}:s={size}:r={fps}"] + streams = "".join(f"[{i}:v]" for i in range(n)) + filt = f"{streams}concat=n={n}:v=1:a=0[out]" + _run([ + "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + *inputs, + "-filter_complex", filt, "-map", "[out]", + "-c:v", "libx264", "-pix_fmt", "yuv420p", + "-force_key_frames", f"expr:gte(t,n_forced*{seg})", + str(path), + ]) + + +def build_static_clip( + path: Path, + duration: float = 3.0, + size: str = "320x240", + fps: int = 10, +) -> None: + """One solid color: 1 keyframe, no scene changes → triggers both fallbacks.""" + _run([ + "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-t", str(duration), "-i", f"color=c=blue:s={size}:r={fps}", + "-c:v", "libx264", "-pix_fmt", "yuv420p", "-g", "600", + str(path), + ]) + + +@pytest.fixture(scope="session") +def cut_clip(tmp_path_factory: pytest.TempPathFactory) -> Path: + path = tmp_path_factory.mktemp("clips") / "cuts.mp4" + build_cut_clip(path) + return path + + +@pytest.fixture(scope="session") +def static_clip(tmp_path_factory: pytest.TempPathFactory) -> Path: + path = tmp_path_factory.mktemp("clips") / "static.mp4" + build_static_clip(path) + return path diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..6bb86a6 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,37 @@ +"""WATCH_DETAIL resolution and frame_cap mapping.""" +from __future__ import annotations + +import config + + +def test_default_detail_is_balanced(monkeypatch, tmp_path): + monkeypatch.delenv("WATCH_DETAIL", raising=False) + monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env") + assert config.get_config()["detail"] == "balanced" + + +def test_env_overrides_detail(monkeypatch, tmp_path): + monkeypatch.setenv("WATCH_DETAIL", "efficient") + monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env") + assert config.get_config()["detail"] == "efficient" + + +def test_invalid_detail_falls_back_to_default(monkeypatch, tmp_path): + monkeypatch.setenv("WATCH_DETAIL", "bogus") + monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env") + assert config.get_config()["detail"] == "balanced" + + +def test_get_config_keys(monkeypatch, tmp_path): + monkeypatch.delenv("WATCH_DETAIL", raising=False) + monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env") + cfg = config.get_config() + assert set(cfg) == {"detail", "config_file"} + + +def test_frame_cap_mapping(): + assert config.frame_cap("efficient") == 50 + assert config.frame_cap("balanced") == 100 + assert config.frame_cap("token-burner") is None + assert config.frame_cap("transcript") is None + assert config.frame_cap("anything-else") == 100 diff --git a/tests/test_download.py b/tests/test_download.py new file mode 100644 index 0000000..3e0e41a --- /dev/null +++ b/tests/test_download.py @@ -0,0 +1,64 @@ +"""yt-dlp argv construction for download.py. + +Regression guard: ``--sub-langs all`` makes yt-dlp fetch YouTube's hundreds of +auto-translated caption tracks, which can take minutes and stalls before the +video download even starts. We only support English, so the request must stay +bounded to the English-only pattern. +""" +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) + +import download # noqa: E402 + +URL = "https://www.youtube.com/watch?v=rlOpbu3Enkw" + + +def _capture_argv(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]: + """Stub subprocess.run inside download.py and record every argv.""" + calls: list[list[str]] = [] + + class _Result: + returncode = 0 + stdout = "" + stderr = "" + + def fake_run(cmd, *args, **kwargs): + calls.append(list(cmd)) + return _Result() + + monkeypatch.setattr(download.subprocess, "run", fake_run) + return calls + + +def _sub_langs(argv: list[str]) -> str: + idx = argv.index("--sub-langs") + return argv[idx + 1] + + +def _assert_english_only(langs: str) -> None: + tokens = langs.split(",") + assert "all" not in tokens, f"sub-langs must not request all languages, got {langs!r}" + assert all(t.startswith("en") for t in tokens), f"sub-langs must be English-only, got {langs!r}" + + +def test_fetch_captions_requests_english_only(monkeypatch, tmp_path): + calls = _capture_argv(monkeypatch) + download.fetch_captions(URL, tmp_path / "download") + _assert_english_only(_sub_langs(calls[0])) + + +def test_download_url_requests_english_only(monkeypatch, tmp_path): + calls = _capture_argv(monkeypatch) + # _pick_video returns None with no real file, which raises SystemExit after + # the yt-dlp argv is already built — that's all we need to inspect. + with pytest.raises(SystemExit): + download.download_url(URL, tmp_path / "download") + _assert_english_only(_sub_langs(calls[0])) diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py new file mode 100644 index 0000000..599e370 --- /dev/null +++ b/tests/test_fixtures.py @@ -0,0 +1,24 @@ +"""Smoke test: the ffmpeg fixtures actually produce playable clips.""" +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + + +def _duration(path: Path) -> float: + out = subprocess.run( + ["ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", str(path)], + capture_output=True, text=True, + ).stdout + return float(json.loads(out)["format"]["duration"]) + + +def test_cut_clip_builds(cut_clip: Path): + assert cut_clip.exists() and cut_clip.stat().st_size > 0 + assert _duration(cut_clip) > 4.0 # 14 * 0.4s ≈ 5.6s + + +def test_static_clip_builds(static_clip: Path): + assert static_clip.exists() and static_clip.stat().st_size > 0 + assert _duration(static_clip) > 2.0 diff --git a/tests/test_frames.py b/tests/test_frames.py new file mode 100644 index 0000000..e53a907 --- /dev/null +++ b/tests/test_frames.py @@ -0,0 +1,70 @@ +"""Keyframe engine + preserved scene/uniform fallbacks.""" +from __future__ import annotations + +from pathlib import Path + +import frames + + +def test_keyframe_engine_on_cut_clip(cut_clip: Path, tmp_path: Path): + out, meta = frames.extract_keyframes(str(cut_clip), tmp_path / "f", max_frames=50) + assert meta["engine"] == "keyframe" + assert meta["fallback"] is False + assert len(out) >= frames.KEYFRAME_MIN + assert all(fr["reason"] == "keyframe" for fr in out) + assert len(out) == len(list((tmp_path / "f").glob("frame_*.jpg"))) + + +def test_keyframe_even_sampling_caps_and_spans(cut_clip: Path, tmp_path: Path): + out, meta = frames.extract_keyframes(str(cut_clip), tmp_path / "f", max_frames=5) + assert meta["engine"] == "keyframe" + assert len(out) == 5 + assert meta["selected_count"] == 5 + assert meta["candidate_count"] > 5 + ts = [fr["timestamp_seconds"] for fr in out] + assert ts == sorted(ts) + assert ts[0] < ts[-1] # spans first → last keyframe + assert [fr["index"] for fr in out] == [0, 1, 2, 3, 4] + + +def test_keyframe_fallback_on_static_clip(static_clip: Path, tmp_path: Path): + out, meta = frames.extract_keyframes(str(static_clip), tmp_path / "f", max_frames=50) + assert meta["engine"] == "uniform" + assert meta["fallback"] is True + assert len(out) > 0 + assert all(fr["reason"] == "uniform" for fr in out) + + +def test_scene_engine_on_cut_clip(cut_clip: Path, tmp_path: Path): + out, meta = frames.extract_scene_or_uniform( + str(cut_clip), tmp_path / "f", fps=2.0, target_frames=50, max_frames=100, + ) + assert meta["engine"] == "scene" + assert meta["fallback"] is False + assert len(out) >= frames.SCENE_MIN_FRAMES + + +def test_scene_even_sampling_caps_and_spans(cut_clip: Path, tmp_path: Path): + """Over-cap scene detection must even-sample across the whole clip, not keep + the first N cuts and drop the tail (the long-video coverage bug).""" + out, meta = frames.extract_scene_or_uniform( + str(cut_clip), tmp_path / "f", fps=2.0, target_frames=50, max_frames=5, + ) + assert meta["engine"] == "scene" + assert meta["fallback"] is False + assert len(out) == 5 + assert meta["selected_count"] == 5 + assert meta["candidate_count"] > 5 # all cuts detected, then sampled down + ts = [fr["timestamp_seconds"] for fr in out] + assert ts == sorted(ts) + assert ts[-1] > 4.0 # spans the full ~5.6s clip, not just the first ~1.6s + assert len(out) == len(list((tmp_path / "f").glob("frame_*.jpg"))) + assert [fr["index"] for fr in out] == [0, 1, 2, 3, 4] + + +def test_scene_fallback_on_static_clip(static_clip: Path, tmp_path: Path): + out, meta = frames.extract_scene_or_uniform( + str(static_clip), tmp_path / "f", fps=2.0, target_frames=12, max_frames=100, + ) + assert meta["engine"] == "uniform" + assert meta["fallback"] is True diff --git a/tests/test_setup.py b/tests/test_setup.py new file mode 100644 index 0000000..f326025 --- /dev/null +++ b/tests/test_setup.py @@ -0,0 +1,80 @@ +"""setup.py --json surfaces the resolved watch detail.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +SETUP = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts" / "setup.py" + + +def _run(args, *, home=None, extra_env=None): + env = dict(os.environ) + env.pop("WATCH_DETAIL", None) + # Don't let a real key in the developer's shell env leak into the test. + env.pop("GROQ_API_KEY", None) + env.pop("OPENAI_API_KEY", None) + env.pop("SETUP_COMPLETE", None) + if home is not None: + env["HOME"] = str(home) + env["USERPROFILE"] = str(home) # Windows + if extra_env: + env.update(extra_env) + return subprocess.run( + [sys.executable, str(SETUP), *args], + capture_output=True, text=True, env=env, + ) + + +def _write_env(home: Path, body: str) -> None: + cfg = home / ".config" / "watch" + cfg.mkdir(parents=True, exist_ok=True) + f = cfg / ".env" + f.write_text(body, encoding="utf-8") + f.chmod(0o600) + + +def test_json_reports_watch_detail(): + proc = _run(["--json"]) + assert proc.returncode == 0, proc.stderr + data = json.loads(proc.stdout) + assert data["watch_detail"] == "balanced" + + +def test_keyless_completed_setup_proceeds_silently(tmp_path): + """A user who finished setup without a key must NOT be nagged forever.""" + _write_env(tmp_path, "GROQ_API_KEY=\nOPENAI_API_KEY=\nSETUP_COMPLETE=true\n") + chk = _run(["--check"], home=tmp_path) + assert chk.returncode == 0, f"keyless-complete should pass --check; got {chk.returncode}: {chk.stderr}" + assert chk.stdout == "" and chk.stderr == "" + + js = json.loads(_run(["--json"], home=tmp_path).stdout) + assert js["can_proceed"] is True + assert js["first_run"] is False + assert js["setup_complete"] is True + # status still encourages a key even though we can proceed + assert js["status"] == "needs_key" + + +def test_keyless_first_run_is_encouraged(tmp_path): + """Genuine first run with no key: --check reports exit 3 (encourage a key).""" + _write_env(tmp_path, "GROQ_API_KEY=\nOPENAI_API_KEY=\n") + chk = _run(["--check"], home=tmp_path) + assert chk.returncode == 3, chk.stderr + + js = json.loads(_run(["--json"], home=tmp_path).stdout) + assert js["can_proceed"] is False + assert js["first_run"] is True + + +def test_key_present_is_ready(tmp_path): + _write_env(tmp_path, "GROQ_API_KEY=sk-test-abc\n") + chk = _run(["--check"], home=tmp_path) + assert chk.returncode == 0, chk.stderr + + js = json.loads(_run(["--json"], home=tmp_path).stdout) + assert js["status"] == "ready" + assert js["can_proceed"] is True + assert js["whisper_backend"] == "groq" diff --git a/tests/test_timestamps.py b/tests/test_timestamps.py new file mode 100644 index 0000000..1a4a172 --- /dev/null +++ b/tests/test_timestamps.py @@ -0,0 +1,87 @@ +"""Transcript-cue timestamps: parsing, point extraction, and pinned merge.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +import frames + + +def test_parse_timestamps_mixed_formats(): + assert frames.parse_timestamps("30,1:05,90") == [30.0, 65.0, 90.0] + + +def test_parse_timestamps_strips_and_dedupes(): + assert frames.parse_timestamps(" 90 , 30, 30 ") == [30.0, 90.0] + + +def test_parse_timestamps_empty(): + assert frames.parse_timestamps("") == [] + assert frames.parse_timestamps(" , ") == [] + + +def test_parse_timestamps_rejects_garbage(): + with pytest.raises(SystemExit): + frames.parse_timestamps("4:bad") + + +def test_merge_frames_sorts_and_reindexes(): + primary = [ + {"index": 0, "timestamp_seconds": 1.0, "path": "a", "reason": "scene-change"}, + {"index": 1, "timestamp_seconds": 5.0, "path": "b", "reason": "scene-change"}, + ] + pinned = [ + {"index": 0, "timestamp_seconds": 3.0, "path": "c", "reason": "transcript-cue"}, + ] + merged = frames.merge_frames(primary, pinned) + assert [f["path"] for f in merged] == ["a", "c", "b"] + assert [f["index"] for f in merged] == [0, 1, 2] + assert merged[1]["reason"] == "transcript-cue" + + +def test_merge_frames_keeps_all_pinned(): + pinned = [{"index": 0, "timestamp_seconds": 2.0, "path": "c", "reason": "transcript-cue"}] + merged = frames.merge_frames([], pinned) + assert [f["path"] for f in merged] == ["c"] + + +def test_extract_at_timestamps_one_frame_per_point(cut_clip: Path, tmp_path: Path): + out, meta = frames.extract_at_timestamps(str(cut_clip), tmp_path / "f", [0.5, 2.0, 4.0]) + assert meta["engine"] == "timestamps" + assert meta["fallback"] is False + assert len(out) == 3 + assert all(f["reason"] == "transcript-cue" for f in out) + ts = [f["timestamp_seconds"] for f in out] + assert ts == sorted(ts) + assert len(out) == len(list((tmp_path / "f").glob("cue_*.jpg"))) + + +def test_extract_at_timestamps_drops_out_of_window(cut_clip: Path, tmp_path: Path): + out, meta = frames.extract_at_timestamps( + str(cut_clip), tmp_path / "f", [0.5, 2.0, 4.0], + start_seconds=1.0, end_seconds=3.0, + ) + assert [f["timestamp_seconds"] for f in out] == [2.0] + assert meta["dropped_out_of_window"] == 2 + + +def test_extract_at_timestamps_caps_and_spans(cut_clip: Path, tmp_path: Path): + out, meta = frames.extract_at_timestamps( + str(cut_clip), tmp_path / "f", [0.5, 1.5, 2.5, 3.5, 4.5], max_frames=3, + ) + assert len(out) == 3 + ts = [f["timestamp_seconds"] for f in out] + assert ts[0] == 0.5 and ts[-1] == 4.5 # even-sample keeps first + last + assert len(out) == len(list((tmp_path / "f").glob("cue_*.jpg"))) + + +def test_extract_at_timestamps_does_not_clobber_detail_frames(cut_clip: Path, tmp_path: Path): + """Cue frames live alongside detail frames in the same dir without deleting them.""" + d = tmp_path / "f" + scene, _ = frames.extract_scene_or_uniform( + str(cut_clip), d, fps=2.0, target_frames=50, max_frames=100, + ) + cues, _ = frames.extract_at_timestamps(str(cut_clip), d, [1.0, 3.0]) + assert len(list(d.glob("frame_*.jpg"))) == len(scene) + assert len(list(d.glob("cue_*.jpg"))) == len(cues) diff --git a/tests/test_watch.py b/tests/test_watch.py new file mode 100644 index 0000000..67ecf8f --- /dev/null +++ b/tests/test_watch.py @@ -0,0 +1,69 @@ +"""End-to-end routing of --detail through watch.py on a local clip.""" +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +WATCH = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts" / "watch.py" + + +def _run(clip: Path, *args: str, env_extra: dict | None = None) -> str: + env = dict(os.environ) + env.pop("WATCH_DETAIL", None) + if env_extra: + env.update(env_extra) + proc = subprocess.run( + [sys.executable, str(WATCH), str(clip), "--no-whisper", *args], + capture_output=True, text=True, env=env, + ) + assert proc.returncode == 0, proc.stderr + return proc.stdout + + +def test_efficient_uses_keyframe_engine(cut_clip: Path): + out = _run(cut_clip, "--detail", "efficient") + assert "(keyframe" in out + assert "**Detail:** efficient" in out + + +def test_balanced_uses_scene_engine(cut_clip: Path): + out = _run(cut_clip, "--detail", "balanced") + assert "(scene" in out + assert "**Detail:** balanced" in out + + +def test_token_burner_uses_scene_engine(cut_clip: Path): + out = _run(cut_clip, "--detail", "token-burner") + assert "(scene" in out + + +def test_transcript_skips_frames(cut_clip: Path): + out = _run(cut_clip, "--detail", "transcript") + assert "skipped" in out + assert "frame_0000.jpg" not in out + + +def test_flag_overrides_env(cut_clip: Path): + out = _run(cut_clip, "--detail", "efficient", env_extra={"WATCH_DETAIL": "balanced"}) + assert "(keyframe" in out + + +def test_default_is_balanced(cut_clip: Path): + out = _run(cut_clip) # no flag, WATCH_DETAIL cleared + assert "**Detail:** balanced" in out + assert "(scene" in out + + +def test_timestamps_add_cue_frames_to_detail(cut_clip: Path): + out = _run(cut_clip, "--detail", "balanced", "--timestamps", "1,3") + assert "reason=transcript-cue" in out + assert "reason=scene-change" in out # detail frames still present (additive) + + +def test_timestamps_with_transcript_detail_is_cue_only(cut_clip: Path): + out = _run(cut_clip, "--detail", "transcript", "--timestamps", "1,3") + assert "reason=transcript-cue" in out + assert "reason=scene-change" not in out + assert "reason=keyframe" not in out