Restructure as a self-contained Agent Skills package (fix Codex install)
Move the skill into skills/watch/ so SKILL.md and its scripts/ runtime are
siblings inside one folder. `npx skills add` (Codex/Cursor/Copilot/agents) now
copies a working skill as a unit; previously it grabbed the root SKILL.md alone
and left scripts/ behind, so the skill was dead on arrival on every non-Claude
host. Mirrors the layout last30days-skill adopted for the same reason.
- skills/watch/{SKILL.md,scripts/}: self-contained skill folder
- SKILL.md: resolve a harness-agnostic $SKILL_DIR (the dir it was Read from)
instead of the Claude-Code-only ${CLAUDE_SKILL_DIR}; guard + 19 call sites
- drop commands/watch.md: /watch derives from frontmatter (name + user-invocable)
- .codex-plugin/plugin.json: full manifest with "skills": "./skills/" + interface
- add .agents/plugins/marketplace.json, AGENTS.md, CLAUDE.md, .skillignore
- build-skill.sh: archive the skills/watch subtree (one SKILL.md, no zip -d)
- fix paths in tests, hooks hint, .gitattributes, release.yml
- relocate dev-sync.sh to repo root and fix REPO_ROOT
- README: content-ideas structure, npx skills install, star history
Verified: 37/37 tests pass; npx skills add bundles the full scripts/ runtime;
manifests valid; versions synced at 0.1.3.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
bradautomates
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "claude-video",
|
||||
"interface": {
|
||||
"displayName": "watch"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "watch",
|
||||
"source": {
|
||||
"source": "url",
|
||||
"url": "https://github.com/bradautomates/claude-video.git"
|
||||
},
|
||||
"policy": {
|
||||
"installation": "AVAILABLE",
|
||||
"authentication": "ON_INSTALL"
|
||||
},
|
||||
"category": "Productivity"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,3 +1,43 @@
|
||||
{
|
||||
"name": "watch"
|
||||
"name": "watch",
|
||||
"version": "0.1.3",
|
||||
"description": "Let Claude watch a video. Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls captions or falls back to Whisper, and hands frames + transcript to Claude so it can answer questions about the video.",
|
||||
"author": {
|
||||
"name": "Bradley Bonanno"
|
||||
},
|
||||
"homepage": "https://github.com/bradautomates/claude-video",
|
||||
"repository": "https://github.com/bradautomates/claude-video",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"video",
|
||||
"watch",
|
||||
"youtube",
|
||||
"vimeo",
|
||||
"tiktok",
|
||||
"transcription",
|
||||
"whisper",
|
||||
"yt-dlp",
|
||||
"ffmpeg",
|
||||
"multimodal",
|
||||
"frames"
|
||||
],
|
||||
"skills": "./skills/",
|
||||
"interface": {
|
||||
"displayName": "watch",
|
||||
"shortDescription": "Give Claude a video input — paste a URL or path and ask about it.",
|
||||
"longDescription": "watch adds a skill that lets the agent watch any video: it downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls a timestamped transcript from native captions (or the Whisper API as a fallback), and hands frames + transcript to the model so it can answer questions grounded in what's actually on screen and in the audio.",
|
||||
"developerName": "Bradley Bonanno",
|
||||
"category": "Productivity",
|
||||
"capabilities": [
|
||||
"Interactive",
|
||||
"Read"
|
||||
],
|
||||
"websiteURL": "https://github.com/bradautomates/claude-video",
|
||||
"defaultPrompt": [
|
||||
"/watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark?",
|
||||
"/watch ~/Movies/screen-recording.mp4 when does the UI break?",
|
||||
"/watch https://youtu.be/<long-talk> summarize this"
|
||||
],
|
||||
"brandColor": "#E11D48"
|
||||
}
|
||||
}
|
||||
|
||||
+17
-5
@@ -1,5 +1,6 @@
|
||||
# Exclude non-runtime files from `git archive` output.
|
||||
# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file.
|
||||
# Used by skills/watch/scripts/build-skill.sh to produce the claude.ai .skill file
|
||||
# and by Claude Code's /plugin install (full-repo archive).
|
||||
|
||||
# Anthropic canonical skill-packaging excludes
|
||||
# (mirrors anthropics/skills/skills/skill-creator/scripts/package_skill.py)
|
||||
@@ -15,10 +16,21 @@ fixtures/ export-ignore
|
||||
assets/ export-ignore
|
||||
examples/ export-ignore
|
||||
|
||||
# NOTE: commands/, hooks/, and .claude-plugin/ are NOT export-ignored because
|
||||
# Claude Code's /plugin install fetches this same git archive tarball — they
|
||||
# must be in the archive for the plugin to install correctly. The claude.ai
|
||||
# .skill bundle strips them afterward via `zip -d` in scripts/build-skill.sh.
|
||||
# Dev tooling and planning notes — repo-only, not shipped to any install surface
|
||||
dev-sync.sh export-ignore
|
||||
V2_PLAN.md export-ignore
|
||||
V2_CONCERNS.md export-ignore
|
||||
.agents/ export-ignore
|
||||
|
||||
# Dev-only build script + scanner config inside the skill tree: keep them out of
|
||||
# the claude.ai .skill bundle (built via `git archive HEAD:skills/watch`).
|
||||
skills/watch/scripts/build-skill.sh export-ignore
|
||||
skills/watch/.skillignore export-ignore
|
||||
|
||||
# NOTE: hooks/ and .claude-plugin/ are NOT export-ignored because Claude Code's
|
||||
# /plugin install fetches the full-repo git archive — they must be present for
|
||||
# the plugin to install correctly. The claude.ai .skill bundle is built from the
|
||||
# skills/watch/ subtree only, so it never sees them in the first place.
|
||||
|
||||
# CI workflows — repo-only, not needed at skill runtime
|
||||
.github/ export-ignore
|
||||
|
||||
@@ -19,7 +19,7 @@ jobs:
|
||||
|
||||
- name: Build .skill artifact
|
||||
run: |
|
||||
bash scripts/build-skill.sh
|
||||
bash skills/watch/scripts/build-skill.sh
|
||||
test -f dist/watch.skill
|
||||
|
||||
- name: Create GitHub release
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# Agent Skills install-time scanner exclusions for repository-root scans.
|
||||
# Keep the public bundle focused on the runtime skill under skills/watch/.
|
||||
|
||||
# VCS, local envs, caches, and generated outputs
|
||||
.git/
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.log
|
||||
.DS_Store
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
dist/
|
||||
|
||||
# Repo/dev automation and host-specific package metadata
|
||||
.github/
|
||||
.agents/
|
||||
.claude-plugin/
|
||||
hooks/
|
||||
dev-sync.sh
|
||||
|
||||
# Non-runtime docs, plans, and tests
|
||||
tests/
|
||||
README.md
|
||||
CHANGELOG.md
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
V2_PLAN.md
|
||||
V2_CONCERNS.md
|
||||
|
||||
# Dev-only script shipped inside the skill tree but not needed at runtime
|
||||
skills/watch/scripts/build-skill.sh
|
||||
@@ -0,0 +1,50 @@
|
||||
# claude-video / watch skill
|
||||
|
||||
Agent Skills package that gives an agent a video input. Installable across Claude Code (most common host), Codex, Cursor, GitHub Copilot, and 50+ other [Agent Skills](https://agentskills.io) hosts. Pure-stdlib Python that orchestrates `yt-dlp` + `ffmpeg` and an optional Whisper API.
|
||||
|
||||
## Structure
|
||||
|
||||
- `skills/watch/SKILL.md` — canonical skill contract the model reads when `/watch` fires. Source of truth for behavior across every host.
|
||||
- `skills/watch/scripts/watch.py` — entry point; orchestrates download → frames → transcript.
|
||||
- `skills/watch/scripts/{download,frames,transcribe,whisper,setup,config}.py` — yt-dlp wrapper, ffmpeg frame extraction + auto-fps, caption/Whisper transcription, preflight/installer, shared config.
|
||||
- `skills/watch/scripts/build-skill.sh` — builds `dist/watch.skill` for claude.ai upload (dev-only).
|
||||
- `hooks/` — Claude Code SessionStart setup-status hook (Claude Code only).
|
||||
- `.claude-plugin/` — `plugin.json` + `marketplace.json` (Claude Code plugin + local marketplace).
|
||||
- `.codex-plugin/plugin.json` — Codex/agents manifest; `"skills": "./skills/"` points the Agent Skills CLI at the self-contained skill folder.
|
||||
- `.agents/plugins/marketplace.json` — agents marketplace listing pointing at the repo-root plugin.
|
||||
- `CLAUDE.md` → `@AGENTS.md` — generic-agent entry point.
|
||||
- `tests/` — pytest suite (ffmpeg-synthesized clips; no network).
|
||||
|
||||
## Orientation
|
||||
|
||||
- The product is the slash-command-invoked skill (`/watch <url-or-path> [question]`), not a CLI. `scripts/watch.py` is implementation. Features must work across every harness the skill installs into, not just Claude Code.
|
||||
- **The skill is one self-contained folder: `skills/watch/`.** SKILL.md and `scripts/` are siblings inside it. This is what lets `npx skills add` copy a working skill as a unit — do NOT move SKILL.md or `scripts/` back to the repo root, or non-Claude installers will copy SKILL.md without the scripts.
|
||||
- **Path resolution is harness-agnostic.** SKILL.md resolves `SKILL_DIR` as the directory of the SKILL.md the model just Read, then runs `${SKILL_DIR}/scripts/...`. Do NOT reintroduce `${CLAUDE_SKILL_DIR}` (Claude-Code-only) — it is unset on Codex/Cursor/agents and breaks every script call there.
|
||||
- **No `commands/` wrapper.** `/watch` is derived from SKILL.md frontmatter (`name: watch` + `user-invocable: true`). A separate command file creates a duplicate slash command.
|
||||
|
||||
## Install surfaces
|
||||
|
||||
| Surface | Install |
|
||||
|---------|---------|
|
||||
| Claude Code | `/plugin marketplace add bradautomates/claude-video` then `/plugin install watch@claude-video` |
|
||||
| Codex / Cursor / Copilot / +50 | `npx skills add bradautomates/claude-video -g` |
|
||||
| claude.ai (web) | upload `dist/watch.skill` (built by `skills/watch/scripts/build-skill.sh`) |
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
# Tests (stdlib + pytest; ffmpeg required for frame tests)
|
||||
.venv/bin/pytest -q # or: python3 -m pytest -q
|
||||
|
||||
# Build the claude.ai upload bundle (archives skills/watch/ as the bundle root)
|
||||
bash skills/watch/scripts/build-skill.sh # → dist/watch.skill
|
||||
|
||||
# Dev: mirror the working tree into the installed Claude Code plugin cache
|
||||
./dev-sync.sh # --dry-run to preview
|
||||
```
|
||||
|
||||
## Rules
|
||||
|
||||
- Keep the version in sync across `skills/watch/SKILL.md` (frontmatter), `.claude-plugin/plugin.json`, and `.codex-plugin/plugin.json` when cutting a release.
|
||||
- Releasing: tag `vX.Y.Z` and push the tag; `.github/workflows/release.yml` builds `dist/watch.skill` and attaches it to the GitHub release.
|
||||
- Never commit real API keys or `.env` contents; keys live in `~/.config/watch/.env` (mode `0600`) at runtime.
|
||||
@@ -2,18 +2,19 @@
|
||||
|
||||
**Give Claude the ability to watch any video.**
|
||||
|
||||
Claude Code:
|
||||
Claude Code (recommended — auto-updates via marketplace):
|
||||
```
|
||||
/plugin marketplace add bradautomates/claude-video
|
||||
/plugin install watch@claude-video
|
||||
```
|
||||
|
||||
claude.ai (web): [download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) and drop it into Settings → Capabilities → Skills.
|
||||
|
||||
Codex / generic skills:
|
||||
Codex, Cursor, Copilot, Gemini CLI, or any of 50+ [Agent Skills](https://agentskills.io) hosts:
|
||||
```bash
|
||||
git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch
|
||||
npx skills add bradautomates/claude-video -g
|
||||
```
|
||||
(`-g` installs globally for your user, available across all projects. Drop it to scope per-project.)
|
||||
|
||||
More install options (claude.ai web, manual) in the [Install](#install) section below.
|
||||
|
||||
Zero config to start — `yt-dlp` and `ffmpeg` install on first run via `brew` on macOS (Linux/Windows print exact commands). Captions cover most public videos for free. Whisper API key is only needed when a video has no captions.
|
||||
|
||||
@@ -21,7 +22,7 @@ Zero config to start — `yt-dlp` and `ffmpeg` install on first run via `brew` o
|
||||
|
||||
Claude can read a webpage, run a script, browse a repo. What it can't do, out of the box, is *watch a video*. You paste a YouTube link and it has to either guess from the title or pull a transcript that's missing 90% of what's on screen.
|
||||
|
||||
With Claude Video `/watch` you can paste a URL or a local path, ask a question, and Claude downloads the video, extracts frames at an auto-scaled rate, pulls a timestamped transcript (free captions when available, Whisper API as fallback), and `Read`s every frame as an image. By the time it answers, it has *seen* the video and *heard* the audio.
|
||||
With Claude Video `/watch` you can paste a URL or a local path, ask a question, and Claude fetches captions first, downloads only what it needs, extracts frames (scene-aware, or fast keyframes at `efficient` detail), pulls a timestamped transcript (free captions when available, Whisper API as fallback), and `Read`s every frame as an image. By the time it answers, it has *seen* the video and *heard* the audio.
|
||||
|
||||
```
|
||||
/watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark?
|
||||
@@ -46,8 +47,8 @@ Claude is great at reading and synthesizing — but until now, video was the one
|
||||
## How it works
|
||||
|
||||
1. **You paste a video and a question.** URL (anything yt-dlp supports — YouTube, Loom, TikTok, X, Instagram, plus a few hundred more) or a local path (`.mp4`, `.mov`, `.mkv`, `.webm`).
|
||||
2. **`yt-dlp` downloads it.** For URLs, into a temp working directory. For local files, no download — just probed in place.
|
||||
3. **`ffmpeg` extracts frames at an auto-scaled rate.** The frame budget is duration-aware: ≤30s gets ~30 frames, 30-60s gets ~40, 1-3min gets ~60, 3-10min gets ~80, longer gets 100 sparsely. Hard ceilings: 2 fps, 100 frames. JPEGs at 512px wide by default — bump with `--resolution 1024` if Claude needs to read on-screen text.
|
||||
2. **`yt-dlp` checks captions first.** At `transcript` detail, captioned URLs return without downloading video. Otherwise, or when Whisper needs audio, it downloads only what the run needs.
|
||||
3. **`ffmpeg` extracts frames at the chosen detail.** `efficient` decodes keyframes only (near-instant); `balanced`/`token-burner` prefer scene-change frames and fall back to the duration-aware uniform sampler when they under-produce. JPEGs are 512px wide by default and clamped to 1998px tall for Claude Read compatibility.
|
||||
4. **The transcript comes from one of two places.** First try: `yt-dlp` pulls native captions (manual or auto-generated) from the source. Free, instant, accurate-ish. Fallback: extract a mono 16 kHz audio clip and ship it to Whisper — Groq's `whisper-large-v3` (preferred — cheaper and faster) or OpenAI's `whisper-1`.
|
||||
5. **Frames + transcript are handed to Claude.** The script prints frame paths with `t=MM:SS` markers and the transcript with timestamps. Claude `Read`s each frame in parallel — JPEGs render directly as images in its context.
|
||||
6. **Claude answers grounded in what's actually on screen and in the audio.** Not "based on the description" or "according to the title." It saw the frames. It heard the transcript. It answers the way someone who watched the video would.
|
||||
@@ -67,14 +68,40 @@ Token cost is dominated by frames. Every frame is an image; image tokens add up
|
||||
|
||||
When the user names a moment ("around 2:30", "the last 30 seconds", "from 0:45 to 1:00"), pass `--start` / `--end`. Focused mode gets denser per-second budgets, capped at 2 fps. Far more useful than a sparse pass over the whole thing.
|
||||
|
||||
## Detail modes — measured
|
||||
|
||||
The `--detail` dial trades speed and token cost for visual fidelity. Numbers below are from a real run against a **49:08 (2948 s)** YouTube video (1280×720 source, English auto-captions, 1394 caption segments) — a long, mostly-static screen recording, which is the case that stresses the caps hardest. Frame-extraction times are measured against a pre-downloaded local copy so they reflect the *mode's* CPU cost, not network speed. The one-time video download for this clip was **~37 s** / 76 MB (shared by `efficient` / `balanced` / `token-burner`).
|
||||
|
||||
| Mode | Engine | Frames | Cap | Extraction time | Temporal coverage | Est. image tokens |
|
||||
|------|--------|--------|-----|-----------------|-------------------|-------------------|
|
||||
| `transcript` | none (captions) | 0 | — | **~4.5 s** (network-bound — one yt-dlp call, no video download) | full (text) | 0 (transcript ≈26.6k text tokens) |
|
||||
| `efficient` | keyframe (`-skip_frame nokey`) | 50 | 50 | **~0.5 s** | 0:00 → 49:04 (full) | **~9.8k** |
|
||||
| `balanced` | scene-change | 100 | 100 | **~20.9 s** | 0:00 → 48:38 (full) | **~19.7k** |
|
||||
| `token-burner` | scene-change | 116 | uncapped | **~21.0 s** | 0:00 → 48:38 (full) | **~22.8k** |
|
||||
|
||||
Image-token estimate uses Anthropic's `(width × height) / 750` per image. At the default 512px width these 720p frames are 512×288, so **≈197 tokens/frame**; total image tokens ≈ `frames × 197`. Bumping `--resolution` to 1024 roughly **4×s** that. The transcript (~26.6k tokens here) is surfaced in *every* mode that has captions, so on the frame modes it adds to the image tokens above — on long videos the transcript, not the frames, is often the larger cost.
|
||||
|
||||
**Note on the `transcript` time.** It looks slower than `efficient` only because the two numbers measure different things: `transcript`'s ~4.5 s is a network round-trip to the source (its *only* step — no video download, no ffmpeg), while the frame modes' times are local CPU with the shared ~37 s download billed separately. End-to-end from a cold URL, `transcript` is the **cheapest** mode by far; `efficient` from a cold URL would be ~37 s of download + 0.5 s of decode.
|
||||
|
||||
What the numbers show:
|
||||
|
||||
- **One consistent sampling rule across every frame mode.** All three detect *all* candidates across the full range, then even-sample (first + last always kept) down to the cap via a shared `_even_sample` helper — `transcript` excepted. Keyframes (`efficient`) and scene-cuts (`balanced`/`token-burner`) differ only in the candidate *source* and the cap, never in how coverage is spread.
|
||||
- **`efficient` is the speed tier** — ~0.5 s because it only reconstructs keyframes (P/B frames are skipped). For this clip it decoded **675 keyframes** and evenly sampled down to its 50-frame cap, spread across the whole 49 minutes.
|
||||
- **Caps are enforced and coverage spans the full clip.** `efficient` (675 → 50) and `balanced` (116 → 100) both sample first→last, so the last frame lands at 48:38–49:04, not partway through. Verified on the full video and on focused ranges (`--start`/`--end` of 30 s → 10 frames, 3 min → 37 frames). The even-sample step (`len(indices) == cap`) cannot return more than the cap. *(This addresses the "sometimes returns too many / drops the tail" concern: the `N selected from 675 candidates` line shows the pre-sample candidate count, not what gets surfaced — only the sampled frames are written and Read.)*
|
||||
- **`balanced` now full-decodes like `token-burner`** (~21 s vs `efficient`'s ~0.5 s). Detecting every scene cut requires decoding the whole video; the old `-frames:v` early-exit was ~3× faster but kept only the *first* 100 cuts and dropped the tail of long videos — so it was removed in favor of even coverage.
|
||||
- **`token-burner` only diverges from `balanced` past the cap.** This recording had **116** cuts over 49 min, so `balanced` sampled 100 of them and `token-burner` kept all 116 — both spanning the full video. On a high-motion video with hundreds of cuts, `token-burner` keeps everything (and the >250-frame token warning kicks in) while `balanced` thins to 100.
|
||||
- **`efficient` can return *more* frames than `balanced`** on low-motion footage (50 keyframes vs. few scene cuts) — the tiers differ by extraction *method* and cap, not by a guaranteed frame-count ordering. "Efficient" means near-instant extraction, not always fewer frames.
|
||||
|
||||
Bottom line on timing: every mode finished its own work in **under 22 s** (plus the shared ~37 s download for the frame modes). `efficient` is ~40× faster than the scene modes here because the scene detector decodes every frame of the full 49 minutes, while `efficient` only reconstructs keyframes.
|
||||
|
||||
## Install
|
||||
|
||||
| Surface | Install |
|
||||
|---------|---------|
|
||||
| **Claude Code** | `/plugin marketplace add bradautomates/claude-video` then `/plugin install watch@claude-video` |
|
||||
| **Codex, Cursor, Copilot, Gemini CLI, +50 more** | `npx skills add bradautomates/claude-video -g` |
|
||||
| **claude.ai** (web) | [Download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) → Settings → Capabilities → Skills → `+` |
|
||||
| **Codex** | `git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch` |
|
||||
| **Manual / dev** | `git clone https://github.com/bradautomates/claude-video.git ~/.claude/skills/watch` |
|
||||
| **Manual / dev** | `git clone` then symlink `skills/watch` into your host's skills dir (see below) |
|
||||
|
||||
### Claude Code
|
||||
|
||||
@@ -85,6 +112,24 @@ When the user names a moment ("around 2:30", "the last 30 seconds", "from 0:45 t
|
||||
|
||||
Update later with `/plugin update watch@claude-video`.
|
||||
|
||||
### Codex, Cursor, Copilot, Gemini CLI, and 50+ other hosts
|
||||
|
||||
The [Agent Skills](https://agentskills.io) CLI installs the skill into whatever agents it detects:
|
||||
|
||||
```bash
|
||||
npx skills add bradautomates/claude-video -g
|
||||
```
|
||||
|
||||
`-g` installs globally for your user (`~/.codex/skills`, `~/.cursor/skills`, etc.); drop it to install into the current project instead. Useful flags:
|
||||
|
||||
- `-a, --agent <names…>` — target specific hosts, e.g. `-a codex -a cursor`
|
||||
- `-l, --list` — list the skills in this repo without installing
|
||||
- `--copy` — copy files instead of symlinking (for filesystems without symlink support)
|
||||
|
||||
The CLI discovers the skill from `skills/watch/SKILL.md` and copies the whole folder — `SKILL.md` plus its `scripts/` runtime — as a self-contained unit. `SKILL.md` resolves its own scripts relative to wherever it was installed, so it works the same on every host.
|
||||
|
||||
Update later with `npx skills update watch -g`.
|
||||
|
||||
### claude.ai (web)
|
||||
|
||||
1. [Download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) from the latest release.
|
||||
@@ -93,18 +138,17 @@ Update later with `/plugin update watch@claude-video`.
|
||||
|
||||
Enable "Code execution and file creation" under Capabilities first — the skill shells out to `ffmpeg` and `yt-dlp`, so it won't run without it.
|
||||
|
||||
### Codex
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch
|
||||
```
|
||||
|
||||
### Manual (developer)
|
||||
|
||||
Clone the repo and symlink the self-contained skill folder into your host's skills directory — the symlink keeps the install in sync with your working tree as you edit:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bradautomates/claude-video.git ~/.claude/skills/watch
|
||||
git clone https://github.com/bradautomates/claude-video.git
|
||||
ln -s "$(pwd)/claude-video/skills/watch" ~/.claude/skills/watch # or ~/.codex/skills/watch
|
||||
```
|
||||
|
||||
For claude.ai, build the `.skill` bundle from source: `bash skills/watch/scripts/build-skill.sh` produces `dist/watch.skill`.
|
||||
|
||||
## First run
|
||||
|
||||
On the first `/watch` call, the skill runs `scripts/setup.py --check`. If `ffmpeg` / `yt-dlp` aren't on your PATH, or no Whisper API key is set, it walks you through fixing it:
|
||||
@@ -145,6 +189,8 @@ Focused on a specific section — denser frame budget, lower token cost:
|
||||
|
||||
Other knobs (passed to `scripts/watch.py`):
|
||||
|
||||
- `--detail transcript|efficient|balanced|token-burner` — fidelity/speed dial. `transcript` skips frames (transcript only); `efficient` uses fast keyframes (cap 50); `balanced` uses scene-aware frames (cap 100); `token-burner` is scene-aware and uncapped.
|
||||
- `--timestamps T1,T2,…` — grab a frame at each absolute timestamp (`SS`/`MM:SS`/`HH:MM:SS`). Claude reads the transcript first, then targets the moments the presenter flags ("look here", "as you can see"). Added on top of the detail frames (reserved against the cap); out-of-window cues are dropped in focus mode; with `--detail transcript` these become the only frames.
|
||||
- `--max-frames N` — lower the frame cap for a tighter token budget.
|
||||
- `--resolution W` — bump frame width to 1024 px when Claude needs to read on-screen text (slides, terminals, code).
|
||||
- `--fps F` — override the auto-fps calculation (still capped at 2 fps).
|
||||
@@ -155,37 +201,44 @@ Other knobs (passed to `scripts/watch.py`):
|
||||
## Limits
|
||||
|
||||
- **Best accuracy: under 10 minutes.** Past that the script prints a "sparse scan" warning — re-run focused on the part you actually care about with `--start`/`--end`.
|
||||
- **Hard caps: 2 fps, 100 frames.** Frame count drives token cost; the script enforces this even when the auto-fps math would imply higher.
|
||||
- **Detail is one dial.** Defaults are balanced: scene-aware frames, 2 fps max, 100-frame cap. Use `--detail efficient` for a fast 50-frame keyframe pass, or `--detail token-burner` for uncapped scene candidates. Set `WATCH_DETAIL` in `~/.config/watch/.env` to change the default.
|
||||
- **Whisper upload limit: 25 MB.** At mono 16 kHz that's about 50 minutes of audio. Longer videos need either captions or `--start`/`--end` to a smaller window.
|
||||
- **No private platforms.** This skill doesn't log into anything. Public URLs and local files only. If yt-dlp can't reach it without auth, neither can `/watch`.
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
.
|
||||
├── SKILL.md # skill contract — loaded by all three surfaces
|
||||
├── scripts/
|
||||
│ ├── watch.py # entry point — orchestrates download → frames → transcript
|
||||
│ ├── download.py # yt-dlp wrapper
|
||||
│ ├── frames.py # ffmpeg frame extraction + auto-fps logic
|
||||
│ ├── transcribe.py # VTT parsing + dedupe + Whisper orchestration
|
||||
│ ├── whisper.py # Groq / OpenAI clients (pure stdlib)
|
||||
│ ├── setup.py # preflight + installer
|
||||
│ └── build-skill.sh # build dist/watch.skill for claude.ai upload
|
||||
├── hooks/ # SessionStart status hook (Claude Code only)
|
||||
├── .claude-plugin/ # plugin.json + marketplace.json (Claude Code)
|
||||
├── .codex-plugin/ # codex packaging
|
||||
└── .github/workflows/ # release.yml — auto-builds watch.skill on tag push
|
||||
├── skills/watch/ # self-contained skill — copied as a unit by every installer
|
||||
│ ├── SKILL.md # skill contract — the source of truth across all surfaces
|
||||
│ └── scripts/
|
||||
│ ├── watch.py # entry point — orchestrates download → frames → transcript
|
||||
│ ├── download.py # yt-dlp wrapper
|
||||
│ ├── frames.py # ffmpeg frame extraction + auto-fps logic
|
||||
│ ├── transcribe.py # VTT parsing + dedupe + Whisper orchestration
|
||||
│ ├── whisper.py # Groq / OpenAI clients (pure stdlib)
|
||||
│ ├── config.py # shared config (~/.config/watch/.env)
|
||||
│ ├── setup.py # preflight + installer
|
||||
│ └── build-skill.sh # build dist/watch.skill for claude.ai upload (dev-only)
|
||||
├── hooks/ # SessionStart status hook (Claude Code only)
|
||||
├── .claude-plugin/ # plugin.json + marketplace.json (Claude Code)
|
||||
├── .codex-plugin/ # plugin.json — Codex/agents manifest ("skills": "./skills/")
|
||||
├── .agents/plugins/ # marketplace.json — Agent Skills marketplace listing
|
||||
├── AGENTS.md → CLAUDE.md # generic-agent entry point
|
||||
├── tests/ # pytest suite (ffmpeg-synthesized clips, no network)
|
||||
└── .github/workflows/ # release.yml — auto-builds watch.skill on tag push
|
||||
```
|
||||
|
||||
## Develop
|
||||
|
||||
```bash
|
||||
# Run the test suite (stdlib + pytest; ffmpeg required for frame tests):
|
||||
python3 -m pytest -q
|
||||
|
||||
# Build the claude.ai upload bundle:
|
||||
bash scripts/build-skill.sh # → dist/watch.skill
|
||||
bash skills/watch/scripts/build-skill.sh # → dist/watch.skill
|
||||
```
|
||||
|
||||
Releasing: tag `vX.Y.Z`, push the tag. The workflow builds `dist/watch.skill` and attaches it to the GitHub release.
|
||||
Releasing: tag `vX.Y.Z`, push the tag. The workflow builds `dist/watch.skill` and attaches it to the GitHub release. Keep the version in sync across `skills/watch/SKILL.md`, `.claude-plugin/plugin.json`, and `.codex-plugin/plugin.json`.
|
||||
|
||||
See [CHANGELOG.md](CHANGELOG.md) for version history.
|
||||
|
||||
@@ -195,6 +248,16 @@ MIT license.
|
||||
|
||||
Built on `yt-dlp`, `ffmpeg`, and Claude's multimodal `Read` tool. Whisper transcription via [Groq](https://groq.com) or [OpenAI](https://openai.com).
|
||||
|
||||
## Star History
|
||||
|
||||
<a href="https://star-history.com/#bradautomates/claude-video&Date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/svg?repos=bradautomates/claude-video&type=Date&theme=dark" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/svg?repos=bradautomates/claude-video&type=Date" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/svg?repos=bradautomates/claude-video&type=Date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
---
|
||||
|
||||
[github.com/bradautomates/claude-video](https://github.com/bradautomates/claude-video) · [LICENSE](LICENSE)
|
||||
|
||||
@@ -1,173 +0,0 @@
|
||||
---
|
||||
name: watch
|
||||
description: Watch a video (URL or local path). Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls the transcript from captions (or Whisper API fallback), and hands the result to Claude so it can answer questions about what's in the video.
|
||||
argument-hint: "<video-url-or-path> [question]"
|
||||
allowed-tools: Bash, Read, AskUserQuestion
|
||||
homepage: https://github.com/bradautomates/claude-video
|
||||
repository: https://github.com/bradautomates/claude-video
|
||||
author: bradautomates
|
||||
license: MIT
|
||||
user-invocable: true
|
||||
---
|
||||
|
||||
# /watch — Claude watches a video
|
||||
|
||||
You don't have a video input; this skill gives you one. A Python script downloads the video, extracts frames as JPEGs, gets a timestamped transcript (native captions first, then Whisper API as fallback), and prints frame paths. You then `Read` each frame path to see the images and combine them with the transcript to answer the user.
|
||||
|
||||
## Step 0 — Setup preflight (runs every `/watch` invocation, silent on success)
|
||||
|
||||
**Python interpreter:** every `python3 ...` command in this skill is for macOS/Linux. On **Windows**, substitute `python` — the `python3` command on Windows is the Microsoft Store stub and will not run the script.
|
||||
|
||||
Before every `/watch` run, verify that dependencies and an API key are in place:
|
||||
|
||||
```bash
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" --check
|
||||
```
|
||||
|
||||
This is a <100ms lookup. On exit 0, the script emits **nothing** — proceed to Step 1 without comment. **Do NOT announce "setup is complete" to the user** — they don't need a status message on every turn. The only acceptable user-visible output from Step 0 is when remediation is required.
|
||||
|
||||
On non-zero exit, follow the table:
|
||||
|
||||
| Exit | Meaning | Action |
|
||||
|------|---------|--------|
|
||||
| `2` | Missing binaries (`ffmpeg` / `ffprobe` / `yt-dlp`) | Run installer |
|
||||
| `3` | No Whisper API key | Run installer to scaffold `.env`, then ask user for a key |
|
||||
| `4` | Both missing | Run installer, then ask for a key |
|
||||
|
||||
The installer is idempotent — safe to re-run:
|
||||
|
||||
```bash
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py"
|
||||
```
|
||||
|
||||
On macOS with Homebrew, it auto-installs `ffmpeg` and `yt-dlp`. On Linux/Windows, it prints the exact install commands for the user to run. It scaffolds `~/.config/watch/.env` with commented placeholders at `0600` perms, and writes `SETUP_COMPLETE=true` once deps + a key are in place so the next session knows this user has already been through the wizard.
|
||||
|
||||
**If an API key is still missing after install:** use `AskUserQuestion` to ask the user whether they have a Groq API key (preferred — cheaper, faster) or an OpenAI key. Then write it into `~/.config/watch/.env` — set the matching `GROQ_API_KEY=...` or `OPENAI_API_KEY=...` line. If they don't want to set up Whisper, proceed with `--no-whisper` and tell them videos without native captions will come back frames-only.
|
||||
|
||||
**Structured mode (optional):** `python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" --json` emits `{status, first_run, missing_binaries, whisper_backend, has_api_key, config_file, platform}` where `status` is one of `ready | needs_install | needs_key | needs_install_and_key`. Use this when you need to branch on specifics (e.g. "is this the user's very first run?" → `first_run: true`).
|
||||
|
||||
Within a single session, you can skip Step 0 on follow-up `/watch` calls — once `--check` returned 0, nothing about the environment changes between turns.
|
||||
|
||||
## When to use
|
||||
|
||||
- User pastes a video URL (YouTube, Vimeo, X, TikTok, Twitch clip, most yt-dlp-supported sites) and asks about it.
|
||||
- User points at a local video file (`.mp4`, `.mov`, `.mkv`, `.webm`, etc.) and asks about it.
|
||||
- User types `/watch <url-or-path> [question]`.
|
||||
|
||||
## Recommended limits
|
||||
|
||||
- **Best accuracy: videos under 10 minutes.** Frame coverage scales inversely with duration.
|
||||
- **Hard caps: 100 frames total and 2 fps.** Token cost grows with frame count, so the script targets a frame budget by duration (and never exceeds 2 fps even when the budget would imply more):
|
||||
- ≤30s → ~1-2 fps (up to 30 frames)
|
||||
- 30s-1min → ~40 frames
|
||||
- 1-3min → ~60 frames
|
||||
- 3-10min → ~80 frames
|
||||
- \>10min → 100 frames, sparsely spaced (warning printed)
|
||||
- If the user hands you a long video, consider asking whether they want a specific section before burning tokens on a sparse scan.
|
||||
|
||||
## How to invoke
|
||||
|
||||
**Step 1 — parse the user input.** Separate the video source (URL or path) from any question the user asked. Example: `/watch https://youtu.be/abc what language is this in?` → source = `https://youtu.be/abc`, question = `what language is this in?`.
|
||||
|
||||
**Step 2 — run the watch script.** Pass the source verbatim. Do not shell-escape it yourself beyond normal quoting:
|
||||
|
||||
```bash
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "<source>"
|
||||
```
|
||||
|
||||
Optional flags:
|
||||
- `--start T` / `--end T` — focus on a section. Accepts `SS`, `MM:SS`, or `HH:MM:SS`. When either is set, fps auto-scales denser (see "Focusing on a section" below).
|
||||
- `--max-frames N` — lower the cap for tighter token budget (e.g. `--max-frames 40`)
|
||||
- `--resolution W` — change frame width in px (default 512; bump to 1024 only if the user needs to read on-screen text)
|
||||
- `--fps F` — override auto-fps (clamped to 2 fps max)
|
||||
- `--out-dir DIR` — keep working files somewhere specific (default: an auto-generated tmp dir)
|
||||
- `--whisper groq|openai` — force a specific Whisper backend (default: prefer Groq if both keys exist)
|
||||
- `--no-whisper` — disable the Whisper fallback entirely (frames-only if no captions)
|
||||
|
||||
### Focusing on a section (higher frame rate)
|
||||
|
||||
When the user asks about a specific moment — "what happens at the 2 minute mark?", "zoom into 0:45 to 1:00", "the first 10 seconds" — pass `--start` and/or `--end`. The script switches to focused-mode budgets, which are denser than full-video budgets (still capped at 2 fps):
|
||||
|
||||
- ≤5s → 2 fps (up to 10 frames)
|
||||
- 5-15s → 2 fps (up to 30 frames)
|
||||
- 15-30s → ~2 fps (up to 60 frames)
|
||||
- 30-60s → ~1.3 fps (up to 80 frames)
|
||||
- 60-180s → ~0.6 fps (100 frames, capped)
|
||||
|
||||
Focused mode is the right call for:
|
||||
- Any moment/range the user names explicitly ("around 2:30", "the intro", "the last 30 seconds").
|
||||
- Any video longer than ~10 minutes where the user's question is about a specific part — running focused on the relevant section is far more useful than a sparse scan of the whole thing.
|
||||
- Re-runs after a full scan didn't have enough detail in some region.
|
||||
|
||||
Transcript is auto-filtered to the same range. Frame timestamps are absolute (real video timeline, not offset-from-start).
|
||||
|
||||
Examples:
|
||||
```bash
|
||||
# Last 10 seconds of a 1 minute video
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" video.mp4 --start 50 --end 60
|
||||
|
||||
# Zoom into 2:15 → 2:45 at 3 fps (90 frames)
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "$URL" --start 2:15 --end 2:45 --fps 3
|
||||
|
||||
# From 1h12m to the end of the video
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "$URL" --start 1:12:00
|
||||
```
|
||||
|
||||
**Step 3 — Read every frame path the script lists.** The Read tool renders JPEGs directly as images for you. Read all frames in a single message (parallel tool calls) so you see them together. The frames are in chronological order with a `t=MM:SS` timestamp so you can align them to the transcript.
|
||||
|
||||
**Step 4 — answer the user.** You now have two streams of evidence:
|
||||
- **Frames** — what's on screen at each timestamp
|
||||
- **Transcript** — what's said at each timestamp. The report's header shows the source (`captions` = yt-dlp pulled native subs; `whisper (groq)` or `whisper (openai)` = transcribed by API).
|
||||
|
||||
If the user asked a specific question, answer it directly citing timestamps. If they didn't ask anything, summarize what happens in the video — structure, key moments, notable visuals, spoken content.
|
||||
|
||||
**Step 5 — clean up.** The script prints a working directory at the end. If the user isn't going to ask follow-ups about this video, delete it with `rm -rf <dir>`. If they might, leave it in place.
|
||||
|
||||
## Transcription
|
||||
|
||||
The script gets a timestamped transcript in one of two ways:
|
||||
|
||||
1. **Native captions (free, preferred).** yt-dlp pulls manual or auto-generated subtitles from the source platform if available.
|
||||
2. **Whisper API fallback.** If no captions came back (or the source is a local file), the script extracts audio (`ffmpeg -vn -ac 1 -ar 16000 -b:a 64k`, ~0.5 MB/min) and uploads it to whichever Whisper API has a key configured:
|
||||
- **Groq** — `whisper-large-v3`. Preferred default: cheaper, faster. Get a key at console.groq.com/keys.
|
||||
- **OpenAI** — `whisper-1`. Fallback. Get a key at platform.openai.com/api-keys.
|
||||
|
||||
Both keys live in `~/.config/watch/.env`. The script prefers Groq when both are set; override with `--whisper openai` to force OpenAI. Use `--no-whisper` to skip the fallback entirely.
|
||||
|
||||
## Failure modes and handling
|
||||
|
||||
- **Setup preflight failed** → run `python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py"` (auto-installs ffmpeg/yt-dlp via brew on macOS, scaffolds the `.env`). For API key, ask the user via `AskUserQuestion` and write it to `~/.config/watch/.env`.
|
||||
- **No transcript available** → captions missing AND (no Whisper key OR Whisper API failed). Script prints a hint pointing to setup. Proceed frames-only and tell the user.
|
||||
- **Long video warning printed** → acknowledge it in your answer. Offer to re-run focused on a specific section via `--start`/`--end` rather than a sparse full-video scan.
|
||||
- **Download fails** → yt-dlp's error goes to stderr. If it's a login-required or region-locked video, tell the user plainly; do not keep retrying.
|
||||
- **Whisper request fails** → the error is printed to stderr (likely: invalid key, rate limit, or 25 MB upload limit on a very long video). The report will say "none available" for transcript. You can retry with `--whisper openai` if Groq failed (or vice versa).
|
||||
|
||||
## Token efficiency
|
||||
|
||||
This skill burns tokens primarily on frames. Order of magnitude:
|
||||
- 80 frames at 512px wide is roughly 50-80k image tokens depending on aspect ratio.
|
||||
- The transcript is cheap (a few thousand tokens at most for a 10-minute video).
|
||||
- Bumping `--resolution` to 1024 roughly quadruples the image tokens per frame. Only do it when necessary.
|
||||
|
||||
If you already watched a video this session and the user asks a follow-up, do **not** re-run the script — you already have the frames and transcript in context. Just answer from what you have.
|
||||
|
||||
## Security & Permissions
|
||||
|
||||
**What this skill does:**
|
||||
- Runs `yt-dlp` locally to download the video and pull native captions when the source supports them (public data; the request goes directly to whatever host the URL points at)
|
||||
- Runs `ffmpeg` / `ffprobe` locally to extract frames as JPEGs and, when Whisper is needed, a mono 16 kHz audio clip
|
||||
- Sends the extracted audio clip to Groq's Whisper API (`api.groq.com/openai/v1/audio/transcriptions`) when `GROQ_API_KEY` is set (preferred — cheaper, faster)
|
||||
- Sends the extracted audio clip to OpenAI's audio transcription API (`api.openai.com/v1/audio/transcriptions`) when `OPENAI_API_KEY` is set and Groq is not, or when `--whisper openai` is forced
|
||||
- Writes the downloaded video, frames, audio, and an intermediate transcript to a working directory under the system temp dir (or `--out-dir` if specified) so Claude can `Read` them
|
||||
- Reads / creates `~/.config/watch/.env` (mode `0600`) to store the Whisper API key(s) and a `SETUP_COMPLETE` marker. As a fallback, also reads `.env` in the current working directory
|
||||
|
||||
**What this skill does NOT do:**
|
||||
- Does not upload the video itself to any API — only the extracted audio goes out, and only when native captions are missing AND Whisper is not disabled with `--no-whisper`
|
||||
- Does not access any platform account (no login, no session cookies, no posting)
|
||||
- Does not share API keys between providers (Groq key only goes to `api.groq.com`, OpenAI key only goes to `api.openai.com`)
|
||||
- Does not log, cache, or write API keys to stdout, stderr, or output files
|
||||
- Does not persist anything outside the working directory and `~/.config/watch/.env` — clean up the working directory when you're done (Step 5)
|
||||
|
||||
**Bundled scripts:** `scripts/watch.py` (entry point), `scripts/download.py` (yt-dlp wrapper), `scripts/frames.py` (ffmpeg frame extraction), `scripts/transcribe.py` (caption selection + Whisper orchestration), `scripts/whisper.py` (Groq / OpenAI clients), `scripts/setup.py` (preflight + installer)
|
||||
|
||||
Review scripts before first use to verify behavior.
|
||||
@@ -1,9 +0,0 @@
|
||||
---
|
||||
description: Watch a video (URL or local path). Downloads with yt-dlp, extracts frames with ffmpeg, transcribes from captions or Whisper, and answers questions about what's in the video.
|
||||
argument-hint: <video-url-or-path> [question]
|
||||
allowed-tools: [Bash, Read, AskUserQuestion]
|
||||
---
|
||||
|
||||
Invoke the `watch` skill (defined in SKILL.md) with the user's arguments: $ARGUMENTS
|
||||
|
||||
Follow the skill's full pipeline: preflight setup check → download via yt-dlp → extract frames at auto-scaled fps → pull captions or Whisper transcript → Read each frame → answer the user grounded in frames and transcript. If the user provided no arguments, ask them for a video URL or local path before proceeding.
|
||||
Executable
+87
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# dev-sync.sh — copy this working tree into the installed /watch plugin cache so
|
||||
# local edits are picked up by Claude Code without publishing a release.
|
||||
#
|
||||
# The install path is resolved from ~/.claude/plugins/installed_plugins.json, so
|
||||
# it follows version bumps automatically. Override it by passing a path as $1 or
|
||||
# setting WATCH_INSTALL_PATH. Pass --dry-run to preview without writing.
|
||||
#
|
||||
# Usage:
|
||||
# ./dev-sync.sh # sync into the resolved install path
|
||||
# ./dev-sync.sh --dry-run # show what would change
|
||||
# ./dev-sync.sh /some/other/dir # sync into an explicit path
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PLUGIN_KEY="watch@claude-video"
|
||||
INSTALLED_JSON="${HOME}/.claude/plugins/installed_plugins.json"
|
||||
|
||||
DRY_RUN=()
|
||||
DEST=""
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--dry-run) DRY_RUN=(--dry-run --itemize-changes) ;;
|
||||
-*) echo "unknown flag: $arg" >&2; exit 2 ;;
|
||||
*) DEST="$arg" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Resolve the destination install path if not given explicitly.
|
||||
if [[ -z "$DEST" ]]; then
|
||||
DEST="${WATCH_INSTALL_PATH:-}"
|
||||
fi
|
||||
if [[ -z "$DEST" ]]; then
|
||||
if [[ ! -f "$INSTALLED_JSON" ]]; then
|
||||
echo "error: $INSTALLED_JSON not found; pass an install path explicitly" >&2
|
||||
exit 1
|
||||
fi
|
||||
DEST="$(PLUGIN_KEY="$PLUGIN_KEY" python3 - "$INSTALLED_JSON" <<'PY'
|
||||
import json, os, sys
|
||||
data = json.load(open(sys.argv[1]))
|
||||
key = os.environ["PLUGIN_KEY"]
|
||||
records = data.get("plugins", {}).get(key, [])
|
||||
# Prefer a record whose installPath actually exists on disk.
|
||||
paths = [r.get("installPath") for r in records if r.get("installPath")]
|
||||
for p in paths:
|
||||
if os.path.isdir(p):
|
||||
print(p); break
|
||||
else:
|
||||
print(paths[0] if paths else "")
|
||||
PY
|
||||
)"
|
||||
fi
|
||||
|
||||
if [[ -z "$DEST" ]]; then
|
||||
echo "error: could not resolve an install path for '$PLUGIN_KEY'" >&2
|
||||
echo " install the plugin first, or pass a path: scripts/dev-sync.sh /path/to/install" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -d "$DEST" ]]; then
|
||||
echo "error: install path does not exist: $DEST" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "source: $REPO_ROOT"
|
||||
echo "dest: $DEST"
|
||||
echo
|
||||
|
||||
# Mirror shipped files only. Dev-only artifacts and runtime state are excluded.
|
||||
# No --delete: the cache holds runtime state (.in_use/) we must not touch.
|
||||
rsync -a ${DRY_RUN[@]+"${DRY_RUN[@]}"} \
|
||||
--exclude '.git/' \
|
||||
--exclude '.venv/' \
|
||||
--exclude '.pytest_cache/' \
|
||||
--exclude '__pycache__/' \
|
||||
--exclude '*.pyc' \
|
||||
--exclude '.DS_Store' \
|
||||
--exclude '.in_use/' \
|
||||
--exclude 'tests/' \
|
||||
--exclude 'docs/' \
|
||||
--exclude 'dist/' \
|
||||
--exclude 'V2_*.md' \
|
||||
--exclude 'dev-sync.sh' \
|
||||
"$REPO_ROOT/" "$DEST/"
|
||||
|
||||
echo "done."
|
||||
@@ -50,7 +50,7 @@ fi
|
||||
|
||||
# First-run / partially-configured → one-line hint.
|
||||
if [[ -z "$HAS_FFMPEG" || -z "$HAS_YTDLP" ]]; then
|
||||
echo "/watch: needs ffmpeg + yt-dlp. Run \`python3 \$CLAUDE_PLUGIN_ROOT/scripts/setup.py\` once to install and scaffold config."
|
||||
echo "/watch: needs ffmpeg + yt-dlp. Run \`python3 \$CLAUDE_PLUGIN_ROOT/skills/watch/scripts/setup.py\` once to install and scaffold config."
|
||||
elif [[ -z "$HAS_GROQ" && -z "$HAS_OPENAI" ]]; then
|
||||
echo "/watch: ready for videos with native captions. Add GROQ_API_KEY (preferred) or OPENAI_API_KEY to ~/.config/watch/.env to unlock Whisper fallback."
|
||||
else
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-skill.sh — package this repo as a claude.ai-upload-ready .skill file.
|
||||
# Usage: bash scripts/build-skill.sh (run from repo root)
|
||||
#
|
||||
# Produces dist/watch.skill, a zip with a single top-level `watch/` directory
|
||||
# containing SKILL.md and the scripts/ runtime. claude.ai's skill upload has a
|
||||
# 200-file cap; `export-ignore` in .gitattributes + the zip -d strips below
|
||||
# keep the bundle lean.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
echo "error: working tree is dirty; commit or stash before building" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p dist
|
||||
OUT="dist/watch.skill"
|
||||
git archive --format=zip --prefix=watch/ --output="$OUT" HEAD
|
||||
|
||||
# claude.ai's .skill bundle needs only SKILL.md + scripts/ runtime. Claude Code
|
||||
# needs hooks/, commands/, and .claude-plugin/ in the git archive (that's why
|
||||
# they are NOT in .gitattributes export-ignore), but the .skill bundle should
|
||||
# strip them to keep a single canonical SKILL.md and stay well under the
|
||||
# 200-file cap.
|
||||
zip -d "$OUT" \
|
||||
"watch/hooks/*" \
|
||||
"watch/commands/*" \
|
||||
"watch/.claude-plugin/*" \
|
||||
> /dev/null 2>&1 || true
|
||||
|
||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||
SIZE=$(du -h "$OUT" | cut -f1)
|
||||
|
||||
if [ "$COUNT" -gt 200 ]; then
|
||||
echo "error: $COUNT files in zip, claude.ai's cap is 200" >&2
|
||||
echo " check .gitattributes export-ignore entries and this script's zip -d excludes" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SKILL_MD_COUNT=$(unzip -l "$OUT" | grep -c "SKILL.md" || true)
|
||||
if [ "$SKILL_MD_COUNT" -ne 1 ]; then
|
||||
echo "error: expected exactly one SKILL.md, found $SKILL_MD_COUNT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "built $OUT ($COUNT files, $SIZE)"
|
||||
echo "upload via the claude.ai skill UI"
|
||||
@@ -1,250 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Probe video metadata and extract frames at an auto-scaled fps.
|
||||
|
||||
Auto-fps targets a frame budget, not a fixed rate. Token cost scales with frame
|
||||
count, so budget-by-duration keeps short videos dense and long videos capped.
|
||||
When a user-specified range is passed, focused-mode budgets denser (they are
|
||||
zooming in for detail).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MAX_FPS = 2.0
|
||||
|
||||
|
||||
def _clamp_fps(fps: float, duration_seconds: float, max_frames: int) -> tuple[float, int]:
|
||||
fps = min(fps, MAX_FPS)
|
||||
target = min(max_frames, max(1, int(round(fps * duration_seconds))))
|
||||
return fps, target
|
||||
|
||||
|
||||
def parse_time(value: str | float | int | None) -> float | None:
|
||||
"""Parse SS, MM:SS, or HH:MM:SS (with optional .ms) into seconds."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
s = str(value).strip()
|
||||
if not s:
|
||||
return None
|
||||
parts = s.split(":")
|
||||
try:
|
||||
if len(parts) == 1:
|
||||
return float(parts[0])
|
||||
if len(parts) == 2:
|
||||
return int(parts[0]) * 60 + float(parts[1])
|
||||
if len(parts) == 3:
|
||||
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
|
||||
except ValueError:
|
||||
pass
|
||||
raise SystemExit(f"Cannot parse time value: {value!r} (expected SS, MM:SS, or HH:MM:SS)")
|
||||
|
||||
|
||||
def format_time(seconds: float) -> str:
|
||||
total = int(round(seconds))
|
||||
hours, rem = divmod(total, 3600)
|
||||
minutes, sec = divmod(rem, 60)
|
||||
if hours:
|
||||
return f"{hours}:{minutes:02d}:{sec:02d}"
|
||||
return f"{minutes:02d}:{sec:02d}"
|
||||
|
||||
|
||||
def get_metadata(video_path: str) -> dict:
|
||||
if shutil.which("ffprobe") is None:
|
||||
raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v", "quiet",
|
||||
"-print_format", "json",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
str(Path(video_path).resolve()),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffprobe failed: {result.stderr.strip()}")
|
||||
|
||||
data = json.loads(result.stdout or "{}")
|
||||
streams = data.get("streams", [])
|
||||
fmt = data.get("format", {})
|
||||
video_stream = next((s for s in streams if s.get("codec_type") == "video"), {})
|
||||
audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None)
|
||||
|
||||
duration = float(fmt.get("duration") or video_stream.get("duration") or 0)
|
||||
return {
|
||||
"duration_seconds": duration,
|
||||
"width": video_stream.get("width"),
|
||||
"height": video_stream.get("height"),
|
||||
"codec": video_stream.get("codec_name"),
|
||||
"size_bytes": int(fmt.get("size") or 0),
|
||||
"has_audio": audio_stream is not None,
|
||||
}
|
||||
|
||||
|
||||
def auto_fps(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
|
||||
"""Pick fps that targets a sensible frame budget for full-video scans."""
|
||||
if duration_seconds <= 0:
|
||||
return 1.0, 1
|
||||
|
||||
if duration_seconds <= 30:
|
||||
target = min(max_frames, max(12, int(round(duration_seconds))))
|
||||
elif duration_seconds <= 60:
|
||||
target = min(max_frames, 40)
|
||||
elif duration_seconds <= 180: # 3 min
|
||||
target = min(max_frames, 60)
|
||||
elif duration_seconds <= 600: # 10 min
|
||||
target = min(max_frames, 80)
|
||||
else:
|
||||
target = max_frames
|
||||
|
||||
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
|
||||
|
||||
|
||||
def auto_fps_focus(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
|
||||
"""Denser budget for user-specified ranges — they are zooming in for detail."""
|
||||
if duration_seconds <= 0:
|
||||
return min(MAX_FPS, 2.0), 2
|
||||
|
||||
if duration_seconds <= 5:
|
||||
target = min(max_frames, max(10, int(round(duration_seconds * 6))))
|
||||
elif duration_seconds <= 15:
|
||||
target = min(max_frames, max(30, int(round(duration_seconds * 4))))
|
||||
elif duration_seconds <= 30:
|
||||
target = min(max_frames, 60)
|
||||
elif duration_seconds <= 60:
|
||||
target = min(max_frames, 80)
|
||||
elif duration_seconds <= 180:
|
||||
target = max_frames
|
||||
else:
|
||||
target = max_frames
|
||||
|
||||
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
|
||||
|
||||
|
||||
def extract(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
fps: float,
|
||||
resolution: int = 512,
|
||||
max_frames: int = 100,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
) -> list[dict]:
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("frame_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
output_pattern = str(out_dir / "frame_%04d.jpg")
|
||||
cmd: list[str] = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-y",
|
||||
]
|
||||
|
||||
# -ss before -i = fast seek (keyframe-snap, good enough for preview frames).
|
||||
if start_seconds is not None:
|
||||
cmd += ["-ss", f"{start_seconds:.3f}"]
|
||||
if end_seconds is not None:
|
||||
cmd += ["-to", f"{end_seconds:.3f}"]
|
||||
|
||||
cmd += [
|
||||
"-i", str(Path(video_path).resolve()),
|
||||
"-vf", f"fps={fps},scale={resolution}:-2",
|
||||
"-frames:v", str(max_frames),
|
||||
"-q:v", "4",
|
||||
output_pattern,
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffmpeg frame extraction failed: {result.stderr.strip()}")
|
||||
|
||||
offset = start_seconds or 0.0
|
||||
frames = sorted(out_dir.glob("frame_*.jpg"))
|
||||
return [
|
||||
{
|
||||
"index": i,
|
||||
"timestamp_seconds": round(offset + (i / fps if fps > 0 else 0.0), 2),
|
||||
"path": str(p),
|
||||
}
|
||||
for i, p in enumerate(frames)
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
print(
|
||||
"usage: frames.py <video-path> <out-dir> [--fps F] [--resolution W] "
|
||||
"[--max-frames N] [--start T] [--end T]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(2)
|
||||
|
||||
video = sys.argv[1]
|
||||
out = Path(sys.argv[2])
|
||||
args = sys.argv[3:]
|
||||
|
||||
fps_override = None
|
||||
resolution = 512
|
||||
max_frames = 100
|
||||
start_arg = None
|
||||
end_arg = None
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == "--fps":
|
||||
fps_override = float(args[i + 1]); i += 2
|
||||
elif args[i] == "--resolution":
|
||||
resolution = int(args[i + 1]); i += 2
|
||||
elif args[i] == "--max-frames":
|
||||
max_frames = int(args[i + 1]); i += 2
|
||||
elif args[i] == "--start":
|
||||
start_arg = args[i + 1]; i += 2
|
||||
elif args[i] == "--end":
|
||||
end_arg = args[i + 1]; i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
meta = get_metadata(video)
|
||||
start_sec = parse_time(start_arg)
|
||||
end_sec = parse_time(end_arg)
|
||||
full_duration = meta["duration_seconds"]
|
||||
|
||||
effective_start = start_sec if start_sec is not None else 0.0
|
||||
effective_end = end_sec if end_sec is not None else full_duration
|
||||
effective_duration = max(0.0, effective_end - effective_start)
|
||||
|
||||
focused = start_sec is not None or end_sec is not None
|
||||
if focused:
|
||||
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
|
||||
else:
|
||||
fps, target = auto_fps(effective_duration, max_frames=max_frames)
|
||||
if fps_override is not None:
|
||||
fps = fps_override
|
||||
target = max(1, int(round(fps * effective_duration)))
|
||||
|
||||
frames = extract(
|
||||
video, out,
|
||||
fps=fps,
|
||||
resolution=resolution,
|
||||
max_frames=max_frames,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
print(json.dumps(
|
||||
{"meta": meta, "fps": fps, "target": target, "focused": focused, "frames": frames},
|
||||
indent=2,
|
||||
))
|
||||
@@ -1,230 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""/watch entry point: download video, extract frames, parse transcript.
|
||||
|
||||
Prints a markdown report to stdout listing frame paths + transcript. Claude
|
||||
then Reads each frame path to see the video.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from download import download, is_url # noqa: E402
|
||||
from frames import MAX_FPS, auto_fps, auto_fps_focus, extract, format_time, get_metadata, parse_time # noqa: E402
|
||||
from transcribe import filter_range, format_transcript, parse_vtt # noqa: E402
|
||||
from whisper import load_api_key, transcribe_video # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="watch",
|
||||
description="Download a video, extract auto-scaled frames, and surface the transcript.",
|
||||
)
|
||||
ap.add_argument("source", help="Video URL or local file path")
|
||||
ap.add_argument("--max-frames", type=int, default=80, help="Cap on frame count (default 80, hard max 100)")
|
||||
ap.add_argument("--resolution", type=int, default=512, help="Frame width in pixels (default 512)")
|
||||
ap.add_argument("--fps", type=float, default=None, help="Override auto-fps")
|
||||
ap.add_argument("--start", type=str, default=None, help="Range start (SS, MM:SS, or HH:MM:SS)")
|
||||
ap.add_argument("--end", type=str, default=None, help="Range end (SS, MM:SS, or HH:MM:SS)")
|
||||
ap.add_argument("--out-dir", type=str, default=None, help="Working directory (default: tmp)")
|
||||
ap.add_argument(
|
||||
"--no-whisper",
|
||||
action="store_true",
|
||||
help="Disable Whisper fallback. Report frames-only if no captions available.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--whisper",
|
||||
choices=["groq", "openai"],
|
||||
default=None,
|
||||
help="Force a specific Whisper backend. Default: prefer Groq, fall back to OpenAI.",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
max_frames = min(args.max_frames, 100)
|
||||
|
||||
if args.out_dir:
|
||||
work = Path(args.out_dir).expanduser().resolve()
|
||||
else:
|
||||
work = Path(tempfile.mkdtemp(prefix="watch-"))
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
print(f"[watch] working dir: {work}", file=sys.stderr)
|
||||
|
||||
print(
|
||||
"[watch] downloading via yt-dlp…" if is_url(args.source) else "[watch] using local file…",
|
||||
file=sys.stderr,
|
||||
)
|
||||
dl = download(args.source, work / "download")
|
||||
video_path = dl["video_path"]
|
||||
|
||||
meta = get_metadata(video_path)
|
||||
full_duration = meta["duration_seconds"]
|
||||
|
||||
start_sec = parse_time(args.start)
|
||||
end_sec = parse_time(args.end)
|
||||
|
||||
if start_sec is not None and start_sec < 0:
|
||||
raise SystemExit("--start must be non-negative")
|
||||
if end_sec is not None and start_sec is not None and end_sec <= start_sec:
|
||||
raise SystemExit("--end must be greater than --start")
|
||||
if full_duration > 0 and start_sec is not None and start_sec >= full_duration:
|
||||
raise SystemExit(f"--start {start_sec:.1f}s is past end of video ({full_duration:.1f}s)")
|
||||
|
||||
effective_start = start_sec if start_sec is not None else 0.0
|
||||
effective_end = end_sec if end_sec is not None else full_duration
|
||||
effective_duration = max(0.0, effective_end - effective_start)
|
||||
focused = start_sec is not None or end_sec is not None
|
||||
|
||||
if focused:
|
||||
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
|
||||
else:
|
||||
fps, target = auto_fps(effective_duration, max_frames=max_frames)
|
||||
if args.fps is not None:
|
||||
fps = min(args.fps, MAX_FPS)
|
||||
target = max(1, int(round(fps * effective_duration)))
|
||||
|
||||
scope = (
|
||||
f"{format_time(effective_start)}-{format_time(effective_end)} ({effective_duration:.1f}s)"
|
||||
if focused else f"full {effective_duration:.1f}s"
|
||||
)
|
||||
print(f"[watch] extracting ~{target} frames at {fps:.3f} fps over {scope}…", file=sys.stderr)
|
||||
|
||||
frames = extract(
|
||||
video_path,
|
||||
work / "frames",
|
||||
fps=fps,
|
||||
resolution=args.resolution,
|
||||
max_frames=max_frames,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
|
||||
transcript_segments: list[dict] = []
|
||||
transcript_text: str | None = None
|
||||
transcript_source: str | None = None
|
||||
if dl.get("subtitle_path"):
|
||||
try:
|
||||
all_segments = parse_vtt(dl["subtitle_path"])
|
||||
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = "captions"
|
||||
except Exception as exc:
|
||||
print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr)
|
||||
|
||||
if not transcript_segments and not args.no_whisper:
|
||||
backend, api_key = load_api_key(args.whisper)
|
||||
if backend and api_key:
|
||||
try:
|
||||
all_segments, used_backend = transcribe_video(
|
||||
video_path,
|
||||
work / "audio.mp3",
|
||||
backend=backend,
|
||||
api_key=api_key,
|
||||
)
|
||||
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = f"whisper ({used_backend})"
|
||||
except SystemExit as exc:
|
||||
print(f"[watch] whisper fallback failed: {exc}", file=sys.stderr)
|
||||
else:
|
||||
hint = (
|
||||
f"--whisper {args.whisper} was set but the matching API key is missing"
|
||||
if args.whisper else
|
||||
"no subtitles and no Whisper API key found"
|
||||
)
|
||||
setup_py = SCRIPT_DIR / "setup.py"
|
||||
print(
|
||||
f"[watch] {hint} — run `python3 {setup_py}` to enable the Whisper fallback",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
info = dl.get("info") or {}
|
||||
|
||||
print()
|
||||
print("# watch: video report")
|
||||
print()
|
||||
print(f"- **Source:** {args.source}")
|
||||
if info.get("title"):
|
||||
print(f"- **Title:** {info['title']}")
|
||||
if info.get("uploader"):
|
||||
print(f"- **Uploader:** {info['uploader']}")
|
||||
print(f"- **Duration:** {format_time(full_duration)} ({full_duration:.1f}s)")
|
||||
if focused:
|
||||
print(
|
||||
f"- **Focus range:** {format_time(effective_start)} → {format_time(effective_end)} "
|
||||
f"({effective_duration:.1f}s)"
|
||||
)
|
||||
if meta.get("width") and meta.get("height"):
|
||||
print(f"- **Resolution:** {meta['width']}x{meta['height']} ({meta.get('codec') or 'unknown codec'})")
|
||||
mode = "focused" if focused else "full"
|
||||
print(f"- **Frames:** {len(frames)} @ {fps:.3f} fps, {mode} mode (budget {target}, max {max_frames})")
|
||||
print(f"- **Frame size:** {args.resolution}px wide")
|
||||
if transcript_segments:
|
||||
in_range = " in range" if focused else ""
|
||||
print(
|
||||
f"- **Transcript:** {len(transcript_segments)} segments{in_range} "
|
||||
f"(via {transcript_source or 'captions'})"
|
||||
)
|
||||
else:
|
||||
print("- **Transcript:** none available")
|
||||
|
||||
if not focused and full_duration > 600:
|
||||
mins = int(full_duration // 60)
|
||||
print()
|
||||
print(
|
||||
f"> **Warning:** This is a {mins}-minute video. Frame coverage is sparse at this length — "
|
||||
"accuracy degrades noticeably on anything over 10 minutes. For better results, "
|
||||
"re-run with `--start HH:MM:SS --end HH:MM:SS` to zoom into a specific section."
|
||||
)
|
||||
|
||||
print()
|
||||
print("## Frames")
|
||||
print()
|
||||
print(f"Frames live at: `{work / 'frames'}`")
|
||||
print()
|
||||
print(
|
||||
"**Read each frame path below with the Read tool to view the image.** "
|
||||
"Frames are in chronological order; `t=MM:SS` is the absolute timestamp in the source video."
|
||||
)
|
||||
print()
|
||||
for frame in frames:
|
||||
print(f"- `{frame['path']}` (t={format_time(frame['timestamp_seconds'])})")
|
||||
|
||||
print()
|
||||
print("## Transcript")
|
||||
print()
|
||||
if transcript_text:
|
||||
label = transcript_source or "captions"
|
||||
if focused:
|
||||
print(f"_Source: {label}. Filtered to {format_time(effective_start)} → {format_time(effective_end)}:_")
|
||||
else:
|
||||
print(f"_Source: {label}._")
|
||||
print()
|
||||
print("```")
|
||||
print(transcript_text)
|
||||
print("```")
|
||||
elif focused and dl.get("subtitle_path"):
|
||||
print(f"_No transcript lines fell inside {format_time(effective_start)} → {format_time(effective_end)}._")
|
||||
else:
|
||||
setup_py = SCRIPT_DIR / "setup.py"
|
||||
print(
|
||||
"_No transcript available — proceed with frames only. "
|
||||
"Captions were missing and the Whisper fallback was unavailable "
|
||||
"(no API key set, or `--no-whisper` was used). "
|
||||
f"Run `python3 {setup_py}` to enable Whisper, then re-run._"
|
||||
)
|
||||
|
||||
print()
|
||||
print("---")
|
||||
print(f"_Work dir: `{work}` — delete when done._")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,3 @@
|
||||
# Scans that start from this skill directory (not the repo root).
|
||||
# Keep non-runtime packaging/dev artifacts out of install-time security scans.
|
||||
scripts/build-skill.sh
|
||||
@@ -0,0 +1,267 @@
|
||||
---
|
||||
name: watch
|
||||
version: "0.1.3"
|
||||
description: Watch a video (URL or local path). Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls the transcript from captions (or Whisper API fallback), and hands the result to Claude so it can answer questions about what's in the video.
|
||||
argument-hint: "<video-url-or-path> [question]"
|
||||
allowed-tools: Bash, Read, AskUserQuestion
|
||||
homepage: https://github.com/bradautomates/claude-video
|
||||
repository: https://github.com/bradautomates/claude-video
|
||||
author: bradautomates
|
||||
license: MIT
|
||||
user-invocable: true
|
||||
---
|
||||
|
||||
# /watch
|
||||
|
||||
You don't have a video input; this skill gives you one. A Python script gets captions first, optionally downloads the video, extracts frames as JPEGs (scene-aware, or fast keyframes at `efficient` detail), gets a timestamped transcript (native captions first, then Whisper API as fallback), and prints frame paths. You then `Read` each frame path to see the images and combine them with the transcript to answer the user.
|
||||
|
||||
## Resolve `SKILL_DIR` (do this before any command)
|
||||
|
||||
Every `python3 ...` command below runs a bundled script under `SKILL_DIR/scripts/`. Set `SKILL_DIR` to the **absolute path of the directory containing THIS SKILL.md you just Read** — your harness told you that path in the Read result. The scripts are always a direct sibling of this file (`SKILL_DIR/scripts/watch.py`), in every install layout:
|
||||
|
||||
```
|
||||
Read ~/.claude/plugins/cache/claude-video/watch/<ver>/skills/watch/SKILL.md → SKILL_DIR=…/skills/watch
|
||||
Read ~/.codex/skills/watch/SKILL.md → SKILL_DIR=~/.codex/skills/watch
|
||||
Read ~/.agents/skills/watch/SKILL.md → SKILL_DIR=~/.agents/skills/watch
|
||||
```
|
||||
|
||||
Substitute that literal path for `${SKILL_DIR}` in every command. This works on every harness (Claude Code, Codex, Cursor, Gemini CLI, …) without relying on any harness-specific environment variable. Guard once at the start of a run:
|
||||
|
||||
```bash
|
||||
SKILL_DIR="<absolute path of the directory containing the SKILL.md you Read>"
|
||||
if [ ! -f "$SKILL_DIR/scripts/watch.py" ]; then
|
||||
echo "ERROR: scripts/watch.py not found under SKILL_DIR=$SKILL_DIR" >&2
|
||||
echo "Re-check the directory of the SKILL.md you Read and substitute it as SKILL_DIR." >&2
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
## Step 0 — Setup preflight (runs every `/watch` invocation, silent on success)
|
||||
|
||||
**Python interpreter:** every `python3 ...` command in this skill is for macOS/Linux. On **Windows**, substitute `python` — the `python3` command on Windows is the Microsoft Store stub and will not run the script.
|
||||
|
||||
On the first `/watch` invocation in a session, use structured preflight so you can detect first-run setup:
|
||||
|
||||
```bash
|
||||
python3 "${SKILL_DIR}/scripts/setup.py" --json
|
||||
```
|
||||
|
||||
Branch on two fields:
|
||||
|
||||
- **`can_proceed: true` and `first_run: false`** → setup is already done (the user may have deliberately skipped a Whisper key — that's allowed). Proceed to Step 1 without comment.
|
||||
- **`first_run: true`** → genuine first-time setup. Do these in order:
|
||||
1. If `missing_binaries` is non-empty, run the installer first (it auto-installs on macOS / prints commands elsewhere — see below) and confirm the binaries land. **Do not skip this and jump to preferences.**
|
||||
2. Run the installer once more if needed so it scaffolds `~/.config/watch/.env` (it only writes the template when the file is absent, so let it create the file *before* you write any values into it).
|
||||
3. Encourage a Whisper API key and ask the watch-preference questions below, then write the selected values into `~/.config/watch/.env` and set `SETUP_COMPLETE=true`.
|
||||
- **`can_proceed: false` and `first_run: false`** → setup was finished before but the environment regressed (e.g. `missing_binaries` after an OS change). Run the installer to remediate, then proceed. Don't re-ask preferences.
|
||||
|
||||
A missing Whisper key is *encouraged to fix, not required*: on a genuine first run `status` will read `needs_key` even when binaries are present — that's your cue to encourage a key, not a blocker.
|
||||
|
||||
On follow-up `/watch` calls in the same session, use the silent check:
|
||||
|
||||
```bash
|
||||
python3 "${SKILL_DIR}/scripts/setup.py" --check
|
||||
```
|
||||
|
||||
This is a <100ms lookup. Exit 0 means /watch can run — this **includes a user who finished setup without a Whisper key** (keyless is allowed). On exit 0 the script emits **nothing** — proceed to Step 1 without comment. **Do NOT announce "setup is complete" to the user** — they don't need a status message on every turn. The only acceptable user-visible output from Step 0 is when remediation is required.
|
||||
|
||||
On non-zero exit, follow the table:
|
||||
|
||||
| Exit | Meaning | Action |
|
||||
|------|---------|--------|
|
||||
| `2` | Missing binaries (`ffmpeg` / `ffprobe` / `yt-dlp`) | Run installer |
|
||||
| `3` | Genuine first run with no Whisper API key | Run installer to scaffold `.env`, then encourage a key (the user may decline — proceed with `--no-whisper`) |
|
||||
| `4` | Both missing | Run installer, then encourage a key |
|
||||
|
||||
Exit `3` only fires before the user has completed setup. Once `SETUP_COMPLETE=true` is written, a keyless install returns exit 0 and is never nagged again.
|
||||
|
||||
The installer is idempotent — safe to re-run:
|
||||
|
||||
```bash
|
||||
python3 "${SKILL_DIR}/scripts/setup.py"
|
||||
```
|
||||
|
||||
On macOS with Homebrew, it auto-installs `ffmpeg` and `yt-dlp`. On Linux/Windows, it prints the exact install commands for the user to run. It scaffolds `~/.config/watch/.env` with commented placeholders and default watch settings at `0600` perms.
|
||||
|
||||
**If an API key is still missing after install:** use `AskUserQuestion` to ask the user whether they have a Groq API key (preferred — cheaper, faster) or an OpenAI key. Then write it into `~/.config/watch/.env` — set the matching `GROQ_API_KEY=...` or `OPENAI_API_KEY=...` line. If they don't want to set up Whisper, proceed with `--no-whisper` and tell them videos without native captions will come back frames-only.
|
||||
|
||||
**First-run watch preference:** after the installer has scaffolded `~/.config/watch/.env`, use `AskUserQuestion` to ask one question:
|
||||
|
||||
- Default detail (one dial). Present these as `AskUserQuestion` options in this exact order — lightest to heaviest — and keep `(recommended)` on `balanced` even though it is not first (do **not** reorder to put the recommended option first):
|
||||
- `transcript` — no frames at all, transcript only (skips video download when captions exist).
|
||||
- `efficient` — fast keyframe pass (cap 50).
|
||||
- `balanced` (recommended) — scene-aware frames (cap 100, default).
|
||||
- `token-burner` — scene-aware, uncapped (maximum fidelity; high token cost).
|
||||
|
||||
Write the answer directly into `~/.config/watch/.env` by setting:
|
||||
|
||||
```bash
|
||||
WATCH_DETAIL=balanced
|
||||
```
|
||||
|
||||
Use the user's selected value. If they skip the question, keep the recommended default. Once dependencies, the API-key choice, and this preference are handled, write or update `SETUP_COMPLETE=true` in the same file. Do not ask this preference question again when `SETUP_COMPLETE=true`.
|
||||
|
||||
**Structured mode (optional):** `python3 "${SKILL_DIR}/scripts/setup.py" --json` emits `{status, can_proceed, first_run, setup_complete, missing_binaries, whisper_backend, has_api_key, config_file, watch_detail, platform}` where `status` is one of `ready | needs_install | needs_key | needs_install_and_key`. `status` describes the *ideal* state (a key is encouraged, so a keyless first run reads `needs_key`); `can_proceed` is the operational gate (binaries present AND a key is set OR setup was already completed). Branch on `can_proceed`/`first_run` to decide whether to run; use `status` to decide what to encourage.
|
||||
|
||||
Within a single session, you can skip Step 0 on follow-up `/watch` calls — once `--check` returned 0, nothing about the environment changes between turns.
|
||||
|
||||
## When to use
|
||||
|
||||
- User pastes a video URL (YouTube, Vimeo, X, TikTok, Twitch clip, most yt-dlp-supported sites) and asks about it.
|
||||
- User points at a local video file (`.mp4`, `.mov`, `.mkv`, `.webm`, etc.) and asks about it.
|
||||
- User types `/watch <url-or-path> [question]`.
|
||||
|
||||
## Recommended limits
|
||||
|
||||
- **Best accuracy: videos under 10 minutes.** Frame coverage scales inversely with duration.
|
||||
- **Universal rate cap: 2 fps.** The script never samples faster than 2 fps, even when a budget or `--fps` would imply more.
|
||||
- **The frame ceiling is set by the detail mode** (`WATCH_DETAIL` in `~/.config/watch/.env`, or `--detail`), not a single global cap:
|
||||
- `transcript` → no frames
|
||||
- `efficient` → up to **50** (keyframes)
|
||||
- `balanced` (default) → up to **100** (scene-aware)
|
||||
- `token-burner` → **uncapped** (scene-aware; a soft warning prints past 250 frames)
|
||||
- `--max-frames N` overrides whichever cap the mode would otherwise use.
|
||||
- **Full-video frame budget by duration.** Token cost grows with frame count, so the script targets a budget by duration. This budget sets the fps and the uniform-sampling fallback; scene-aware selection can fill up to the detail cap above, whichever is lower:
|
||||
- ≤30s → ~12-30 frames
|
||||
- 30s-1min → ~40 frames
|
||||
- 1-3min → ~60 frames
|
||||
- 3-10min → ~80 frames
|
||||
- \>10min → up to the detail cap, sparsely spaced (warning printed)
|
||||
- If the user hands you a long video, consider asking whether they want a specific section before burning tokens on a sparse scan.
|
||||
|
||||
## How to invoke
|
||||
|
||||
**Step 1 — parse the user input.** Separate the video source (URL or path) from any question the user asked. Example: `/watch https://youtu.be/abc what language is this in?` → source = `https://youtu.be/abc`, question = `what language is this in?`.
|
||||
|
||||
**Step 2 — run the watch script.** Pass the source verbatim. Do not shell-escape it yourself beyond normal quoting:
|
||||
|
||||
```bash
|
||||
python3 "${SKILL_DIR}/scripts/watch.py" "<source>"
|
||||
```
|
||||
|
||||
Optional flags:
|
||||
- `--detail transcript|efficient|balanced|token-burner` — fidelity/speed dial. `transcript` = no frames (transcript only, skips video download when captions exist); `efficient` = fast keyframes (cap 50); `balanced` = scene-aware frames (cap 100); `token-burner` = scene-aware, uncapped.
|
||||
- `--start T` / `--end T` — focus on a section. Accepts `SS`, `MM:SS`, or `HH:MM:SS`. When either is set, fps auto-scales denser (see "Focusing on a section" below).
|
||||
- `--timestamps T1,T2,…` — grab a frame at each of these absolute timestamps (`SS`, `MM:SS`, or `HH:MM:SS`). Use this after reading the transcript to capture deictic moments the presenter flags ("look here", "as you can see", "notice this") that visual selection alone may miss. See "Transcript-cue frames" below.
|
||||
- `--max-frames N` — override the preset cap for tighter token budget (e.g. `--max-frames 40`)
|
||||
- `--resolution W` — change frame width in px (default 512; bump to 1024 only if the user needs to read on-screen text)
|
||||
- `--fps F` — override auto-fps (clamped to 2 fps max)
|
||||
- `--out-dir DIR` — keep working files somewhere specific (default: an auto-generated tmp dir)
|
||||
- `--whisper groq|openai` — force a specific Whisper backend (default: prefer Groq if both keys exist)
|
||||
- `--no-whisper` — disable the Whisper fallback entirely (frames-only if no captions)
|
||||
|
||||
### Focusing on a section (higher frame rate)
|
||||
|
||||
When the user asks about a specific moment — "what happens at the 2 minute mark?", "zoom into 0:45 to 1:00", "the first 10 seconds" — pass `--start` and/or `--end`. The script switches to focused-mode budgets, which are denser than full-video budgets (still capped at 2 fps, and still bounded by the detail-mode cap — the counts below assume the default `balanced` cap of 100; `efficient` tops out at 50):
|
||||
|
||||
- ≤5s → 2 fps (up to 10 frames)
|
||||
- 5-15s → 2 fps (up to 30 frames)
|
||||
- 15-30s → ~2 fps (up to 60 frames)
|
||||
- 30-60s → ~1.3 fps (up to 80 frames)
|
||||
- 60-180s → ~0.6 fps (100 frames, capped)
|
||||
|
||||
Focused mode is the right call for:
|
||||
- Any moment/range the user names explicitly ("around 2:30", "the intro", "the last 30 seconds").
|
||||
- Any video longer than ~10 minutes where the user's question is about a specific part — running focused on the relevant section is far more useful than a sparse scan of the whole thing.
|
||||
- Re-runs after a full scan didn't have enough detail in some region.
|
||||
|
||||
Transcript is auto-filtered to the same range. Frame timestamps are absolute (real video timeline, not offset-from-start).
|
||||
|
||||
Examples:
|
||||
```bash
|
||||
# Last 10 seconds of a 1 minute video
|
||||
python3 "${SKILL_DIR}/scripts/watch.py" video.mp4 --start 50 --end 60
|
||||
|
||||
# Zoom into 2:15 → 2:45 at 2 fps (60 frames)
|
||||
python3 "${SKILL_DIR}/scripts/watch.py" "$URL" --start 2:15 --end 2:45 --fps 2
|
||||
|
||||
# From 1h12m to the end of the video
|
||||
python3 "${SKILL_DIR}/scripts/watch.py" "$URL" --start 1:12:00
|
||||
```
|
||||
|
||||
**Step 3 — Read every frame path the script lists.** The Read tool renders JPEGs directly as images for you. Read all frames in a single message (parallel tool calls) so you see them together. The frames are in chronological order with a `t=MM:SS` timestamp so you can align them to the transcript.
|
||||
|
||||
**Step 4 — answer the user.** You now have two streams of evidence:
|
||||
- **Frames** — what's on screen at each timestamp
|
||||
- **Transcript** — what's said at each timestamp. The report's header shows the source (`captions` = yt-dlp pulled native subs; `whisper (groq)` or `whisper (openai)` = transcribed by API).
|
||||
|
||||
If the user asked a specific question, answer it directly citing timestamps. If they didn't ask anything, summarize what happens in the video — structure, key moments, notable visuals, spoken content.
|
||||
|
||||
This holds for `transcript` detail too: even with no frames, produce a **summary** like the other modes — do not paste the full transcript into chat. Synthesize structure, key moments, and spoken content with timestamps; quote only the lines that matter. Offer the raw transcript only if the user explicitly asks for it.
|
||||
|
||||
**Step 5 — clean up.** The script prints a working directory at the end. If the user isn't going to ask follow-ups about this video, delete it with `rm -rf <dir>`. If they might, leave it in place.
|
||||
|
||||
## Detail and frames
|
||||
|
||||
Default behavior comes from `~/.config/watch/.env`:
|
||||
|
||||
- `WATCH_DETAIL=transcript|efficient|balanced|token-burner` (default: `balanced`)
|
||||
|
||||
At `transcript` detail, captions are enough to return a report without downloading video. If captions are missing, the script downloads audio only and tries Whisper. If no transcript can be produced, it reports the limitation clearly; re-run with `--detail balanced` for frames.
|
||||
|
||||
At `efficient` detail, the script downloads the video and extracts **keyframes only** (`ffmpeg -skip_frame nokey`) — a near-instant pass that lands frames on scene cuts. If a clip has fewer than 4 keyframes it falls back to uniform sampling.
|
||||
|
||||
At `balanced` / `token-burner` detail, the script extracts **scene-aware** frames: ffmpeg scene-change selection first, falling back to uniform sampling only when the video is effectively static. `balanced` caps at 100 frames; `token-burner` is uncapped. Frame report lines include both timestamp and selection reason. Extracted images are clamped to a maximum 1998px height for Claude Read compatibility.
|
||||
|
||||
## Transcript-cue frames
|
||||
|
||||
Visual frame selection (scene/keyframe) can miss the moments a presenter explicitly flags — "look here", "as you can see", "notice this", "watch what happens" — because pointing at a slide is often a *low* visual change. `--timestamps` lets you force a frame at those exact moments. **You** decide which moments matter, by reading the transcript:
|
||||
|
||||
1. Run once at `--detail transcript` (or any detail) to get the timestamped transcript.
|
||||
2. Scan it for deictic cues — phrases where the speaker directs attention to something on screen. This is a judgment call (ignore rhetorical "look, the point is…"); that's why it's done by you, not a regex.
|
||||
3. Re-run with `--timestamps 4:32,7:10,9:55` (absolute source times). For a URL, point the second run at the **downloaded local file** in the work dir so it doesn't re-download.
|
||||
|
||||
Behavior:
|
||||
- **Additive by default.** Cue frames (`reason=transcript-cue`) are merged into whatever `--detail` already selected, in chronological order.
|
||||
- **Pinned and counted first.** Cue frames are reserved against the frame cap before the detail engine runs, so they're never evicted by even-sampling.
|
||||
- **Honors focus mode.** With `--start/--end`, any cue timestamp outside the window is dropped (reported in the summary). Coordinates are always absolute source time.
|
||||
- **Cue-only frames.** `--detail transcript --timestamps …` skips scene/keyframe sampling and returns *only* the cue frames (it will download the video to do so, since frames need pixels).
|
||||
|
||||
## Transcription
|
||||
|
||||
The script gets a timestamped transcript in one of two ways:
|
||||
|
||||
1. **Native captions (free, preferred).** yt-dlp pulls manual or auto-generated subtitles from the source platform if available.
|
||||
2. **Whisper API fallback.** If no captions came back (or the source is a local file), the script extracts audio (`ffmpeg -vn -ac 1 -ar 16000 -b:a 64k`, ~0.5 MB/min) and uploads it to whichever Whisper API has a key configured:
|
||||
- **Groq** — `whisper-large-v3`. Preferred default: cheaper, faster. Get a key at console.groq.com/keys.
|
||||
- **OpenAI** — `whisper-1`. Fallback. Get a key at platform.openai.com/api-keys.
|
||||
|
||||
Both keys live in `~/.config/watch/.env`. The script prefers Groq when both are set; override with `--whisper openai` to force OpenAI. Use `--no-whisper` to skip the fallback entirely.
|
||||
|
||||
## Failure modes and handling
|
||||
|
||||
- **Setup preflight failed** → run `python3 "${SKILL_DIR}/scripts/setup.py"` (auto-installs ffmpeg/yt-dlp via brew on macOS, scaffolds the `.env`). For API key, ask the user via `AskUserQuestion` and write it to `~/.config/watch/.env`.
|
||||
- **No transcript available** → captions missing AND (no Whisper key OR Whisper API failed). Script prints a hint pointing to setup. Proceed frames-only and tell the user.
|
||||
- **Long video warning printed** → acknowledge it in your answer. Offer to re-run focused on a specific section via `--start`/`--end` rather than a sparse full-video scan.
|
||||
- **Download fails** → yt-dlp's error goes to stderr. If it's a login-required or region-locked video, tell the user plainly; do not keep retrying.
|
||||
- **Whisper request fails** → the error is printed to stderr (likely: invalid key, rate limit, or 25 MB upload limit on a very long video). The report will say "none available" for transcript. You can retry with `--whisper openai` if Groq failed (or vice versa).
|
||||
|
||||
## Token efficiency
|
||||
|
||||
This skill burns tokens primarily on frames. Order of magnitude:
|
||||
- 80 frames at 512px wide is roughly 50-80k image tokens depending on aspect ratio.
|
||||
- The transcript is cheap (a few thousand tokens at most for a 10-minute video).
|
||||
- Bumping `--resolution` to 1024 roughly quadruples the image tokens per frame. Only do it when necessary.
|
||||
|
||||
If you already watched a video this session and the user asks a follow-up, do **not** re-run the script — you already have the frames and transcript in context. Just answer from what you have.
|
||||
|
||||
## Security & Permissions
|
||||
|
||||
**What this skill does:**
|
||||
- Runs `yt-dlp` locally to download the video and pull native captions when the source supports them (public data; the request goes directly to whatever host the URL points at)
|
||||
- Runs `ffmpeg` / `ffprobe` locally to extract frames as JPEGs and, when Whisper is needed, a mono 16 kHz audio clip
|
||||
- Sends the extracted audio clip to Groq's Whisper API (`api.groq.com/openai/v1/audio/transcriptions`) when `GROQ_API_KEY` is set (preferred — cheaper, faster)
|
||||
- Sends the extracted audio clip to OpenAI's audio transcription API (`api.openai.com/v1/audio/transcriptions`) when `OPENAI_API_KEY` is set and Groq is not, or when `--whisper openai` is forced
|
||||
- Writes the downloaded video, frames, audio, and an intermediate transcript to a working directory under the system temp dir (or `--out-dir` if specified) so Claude can `Read` them
|
||||
- Reads / creates `~/.config/watch/.env` (mode `0600`) to store the Whisper API key(s) and a `SETUP_COMPLETE` marker. As a fallback, also reads `.env` in the current working directory
|
||||
|
||||
**What this skill does NOT do:**
|
||||
- Does not upload the video itself to any API — only the extracted audio goes out, and only when native captions are missing AND Whisper is not disabled with `--no-whisper`
|
||||
- Does not access any platform account (no login, no session cookies, no posting) — yt-dlp only ever requests public data
|
||||
- Does not share API keys between providers (Groq key only goes to `api.groq.com`, OpenAI key only goes to `api.openai.com`)
|
||||
- Does not log, cache, or write API keys to stdout, stderr, or output files
|
||||
- Does not persist anything outside the working directory and `~/.config/watch/.env` — clean up the working directory when you're done (Step 5)
|
||||
|
||||
**Bundled scripts:** `scripts/watch.py` (entry point), `scripts/download.py` (yt-dlp wrapper), `scripts/frames.py` (ffmpeg frame extraction), `scripts/transcribe.py` (caption selection + Whisper orchestration), `scripts/whisper.py` (Groq / OpenAI clients), `scripts/setup.py` (preflight + installer)
|
||||
|
||||
Review scripts before first use to verify behavior.
|
||||
Executable
+39
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-skill.sh — package the watch skill as a claude.ai-upload-ready .skill file.
|
||||
# Usage: bash skills/watch/scripts/build-skill.sh (run from anywhere)
|
||||
#
|
||||
# Produces dist/watch.skill, a zip with a single top-level `watch/` directory
|
||||
# containing SKILL.md and the scripts/ runtime from skills/watch. Archiving the
|
||||
# skills/watch subtree directly keeps the bundle to exactly one SKILL.md and
|
||||
# well under claude.ai's 200-file cap, with no post-hoc `zip -d` stripping.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
echo "error: working tree is dirty; commit or stash before building" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p dist
|
||||
OUT="dist/watch.skill"
|
||||
git archive --format=zip --prefix=watch/ --output="$OUT" HEAD:skills/watch
|
||||
|
||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||
SIZE=$(du -h "$OUT" | cut -f1)
|
||||
|
||||
if [ "$COUNT" -gt 200 ]; then
|
||||
echo "error: $COUNT files in zip, claude.ai's cap is 200" >&2
|
||||
echo " trim the skills/watch/ tree or add a .gitattributes export-ignore entry" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SKILL_MD_COUNT=$(unzip -l "$OUT" | grep -c "SKILL.md" || true)
|
||||
if [ "$SKILL_MD_COUNT" -ne 1 ]; then
|
||||
echo "error: expected exactly one SKILL.md, found $SKILL_MD_COUNT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "built $OUT ($COUNT files, $SIZE)"
|
||||
echo "upload via the claude.ai skill UI"
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared /watch configuration helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
CONFIG_DIR = Path.home() / ".config" / "watch"
|
||||
CONFIG_FILE = CONFIG_DIR / ".env"
|
||||
|
||||
DEFAULT_DETAIL = "balanced"
|
||||
|
||||
DETAILS = {"transcript", "efficient", "balanced", "token-burner"}
|
||||
|
||||
|
||||
def read_env_file(path: Path | None = None) -> dict[str, str]:
|
||||
if path is None:
|
||||
path = CONFIG_FILE
|
||||
values: dict[str, str] = {}
|
||||
if not path.exists():
|
||||
return values
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
except OSError:
|
||||
return values
|
||||
for line in lines:
|
||||
raw = line.strip()
|
||||
if not raw or raw.startswith("#") or "=" not in raw:
|
||||
continue
|
||||
key, _, value = raw.partition("=")
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] in ('"', "'") and value[-1] == value[0]:
|
||||
value = value[1:-1]
|
||||
values[key.strip()] = value
|
||||
return values
|
||||
|
||||
|
||||
def get_config() -> dict[str, object]:
|
||||
file_values = read_env_file()
|
||||
|
||||
detail = (
|
||||
os.environ.get("WATCH_DETAIL")
|
||||
or file_values.get("WATCH_DETAIL")
|
||||
or DEFAULT_DETAIL
|
||||
)
|
||||
if detail not in DETAILS:
|
||||
detail = DEFAULT_DETAIL
|
||||
|
||||
return {
|
||||
"detail": detail,
|
||||
"config_file": str(CONFIG_FILE),
|
||||
}
|
||||
|
||||
|
||||
def frame_cap(detail: str) -> int | None:
|
||||
if detail == "efficient":
|
||||
return 50
|
||||
if detail == "balanced":
|
||||
return 100
|
||||
if detail == "token-burner":
|
||||
return None
|
||||
if detail == "transcript":
|
||||
return None
|
||||
return 100
|
||||
@@ -45,12 +45,15 @@ def _pick_subtitle(out_dir: Path) -> Path | None:
|
||||
candidates = sorted(out_dir.glob("video*.vtt"))
|
||||
if not candidates:
|
||||
return None
|
||||
preferred = [c for c in candidates if ".en" in c.name]
|
||||
preferred = [
|
||||
c for c in candidates
|
||||
if any(marker in c.name for marker in (".en.", ".en-US.", ".en-GB.", ".en-orig."))
|
||||
]
|
||||
return preferred[0] if preferred else candidates[0]
|
||||
|
||||
|
||||
def _pick_video(out_dir: Path) -> Path | None:
|
||||
for ext in (".mp4", ".mkv", ".webm", ".mov"):
|
||||
for ext in (".mp4", ".mkv", ".webm", ".mov", ".m4a", ".mp3", ".opus"):
|
||||
for candidate in out_dir.glob(f"video*{ext}"):
|
||||
return candidate
|
||||
for candidate in out_dir.glob("video.*"):
|
||||
@@ -59,22 +62,77 @@ def _pick_video(out_dir: Path) -> Path | None:
|
||||
return None
|
||||
|
||||
|
||||
def download_url(url: str, out_dir: Path) -> dict:
|
||||
def fetch_captions(url: str, out_dir: Path) -> dict:
|
||||
"""Fetch metadata and best available VTT captions without downloading video."""
|
||||
if shutil.which("yt-dlp") is None:
|
||||
raise SystemExit("yt-dlp is not installed. Install with: brew install yt-dlp")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_template = str(out_dir / "video.%(ext)s")
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"--skip-download",
|
||||
"--write-info-json",
|
||||
"--write-subs",
|
||||
"--write-auto-subs",
|
||||
"--sub-langs", "en.*",
|
||||
"--sub-format", "vtt",
|
||||
"--convert-subs", "vtt",
|
||||
"--no-playlist",
|
||||
"--ignore-errors",
|
||||
"-o", output_template,
|
||||
"--",
|
||||
url,
|
||||
]
|
||||
subprocess.run(cmd, stdout=sys.stderr, stderr=sys.stderr)
|
||||
subtitle = _pick_subtitle(out_dir)
|
||||
info = _read_info(out_dir / "video.info.json", url)
|
||||
return {
|
||||
"video_path": None,
|
||||
"subtitle_path": str(subtitle) if subtitle else None,
|
||||
"info": info or {"url": url},
|
||||
"downloaded": False,
|
||||
}
|
||||
|
||||
|
||||
def _read_info(info_path: Path, url: str) -> dict:
|
||||
info: dict = {}
|
||||
if info_path.exists():
|
||||
try:
|
||||
raw = json.loads(info_path.read_text(encoding="utf-8"))
|
||||
info = {
|
||||
"title": raw.get("title"),
|
||||
"uploader": raw.get("uploader") or raw.get("channel"),
|
||||
"duration": raw.get("duration"),
|
||||
"url": raw.get("webpage_url") or url,
|
||||
}
|
||||
except Exception as exc:
|
||||
print(f"[watch] info.json parse failed: {exc}", file=sys.stderr)
|
||||
info = {"url": url}
|
||||
return info
|
||||
|
||||
|
||||
def download_url(
|
||||
url: str,
|
||||
out_dir: Path,
|
||||
audio_only: bool = False,
|
||||
) -> dict:
|
||||
if shutil.which("yt-dlp") is None:
|
||||
raise SystemExit("yt-dlp is not installed. Install with: brew install yt-dlp")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_template = str(out_dir / "video.%(ext)s")
|
||||
|
||||
fmt = "ba/bestaudio" if audio_only else "bv*[height<=720]+ba/b[height<=720]/bv+ba/b"
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"-N", "8",
|
||||
"-f", "bv*[height<=720]+ba/b[height<=720]/bv+ba/b",
|
||||
"-f", fmt,
|
||||
"--merge-output-format", "mp4",
|
||||
"--write-info-json",
|
||||
"--write-subs",
|
||||
"--write-auto-subs",
|
||||
"--sub-langs", "en,en-US,en-GB,en-orig",
|
||||
"--sub-langs", "en.*",
|
||||
"--sub-format", "vtt",
|
||||
"--convert-subs", "vtt",
|
||||
"--no-playlist",
|
||||
@@ -94,20 +152,7 @@ def download_url(url: str, out_dir: Path) -> dict:
|
||||
)
|
||||
|
||||
subtitle = _pick_subtitle(out_dir)
|
||||
info_path = out_dir / "video.info.json"
|
||||
info: dict = {}
|
||||
if info_path.exists():
|
||||
try:
|
||||
raw = json.loads(info_path.read_text(encoding="utf-8"))
|
||||
info = {
|
||||
"title": raw.get("title"),
|
||||
"uploader": raw.get("uploader") or raw.get("channel"),
|
||||
"duration": raw.get("duration"),
|
||||
"url": raw.get("webpage_url") or url,
|
||||
}
|
||||
except Exception as exc:
|
||||
print(f"[watch] info.json parse failed: {exc}", file=sys.stderr)
|
||||
info = {"url": url}
|
||||
info = _read_info(out_dir / "video.info.json", url)
|
||||
|
||||
return {
|
||||
"video_path": str(video),
|
||||
@@ -117,9 +162,13 @@ def download_url(url: str, out_dir: Path) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def download(source: str, out_dir: Path) -> dict:
|
||||
def download(
|
||||
source: str,
|
||||
out_dir: Path,
|
||||
audio_only: bool = False,
|
||||
) -> dict:
|
||||
if is_url(source):
|
||||
return download_url(source, out_dir)
|
||||
return download_url(source, out_dir, audio_only=audio_only)
|
||||
return resolve_local(source)
|
||||
|
||||
|
||||
Executable
+628
@@ -0,0 +1,628 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Probe video metadata and extract frames at an auto-scaled fps.
|
||||
|
||||
Auto-fps targets a frame budget, not a fixed rate. Token cost scales with frame
|
||||
count, so budget-by-duration keeps short videos dense and long videos capped.
|
||||
When a user-specified range is passed, focused-mode budgets denser (they are
|
||||
zooming in for detail).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MAX_FPS = 2.0
|
||||
SCENE_THRESHOLD = 0.20
|
||||
# Keep scene-detection results once we have at least this many distinct shots.
|
||||
# Below this the video is effectively static (screen recording, talking head),
|
||||
# so we fall back to uniform sampling. Matching the reference fork's behaviour,
|
||||
# this is a low floor — NOT the frame budget — so normal videos with cuts use
|
||||
# the (single-pass) scene engine instead of paying for a wasted second decode.
|
||||
SCENE_MIN_FRAMES = 8
|
||||
# Below this many decoded keyframes a clip is too sparse for keyframe coverage
|
||||
# (very short or oddly encoded), so the cheap tier falls back to uniform.
|
||||
KEYFRAME_MIN = 4
|
||||
MAX_READ_DIMENSION = 1998
|
||||
SHOWINFO_TS_RE = re.compile(r"pts_time:([0-9.]+)")
|
||||
|
||||
|
||||
def _scale_filter(resolution: int) -> str:
|
||||
return (
|
||||
f"scale=w='min({resolution},iw)':h='min({MAX_READ_DIMENSION},ih)':"
|
||||
"force_original_aspect_ratio=decrease:force_divisible_by=2"
|
||||
)
|
||||
|
||||
|
||||
def _clamp_fps(fps: float, duration_seconds: float, max_frames: int) -> tuple[float, int]:
|
||||
fps = min(fps, MAX_FPS)
|
||||
target = min(max_frames, max(1, int(round(fps * duration_seconds))))
|
||||
return fps, target
|
||||
|
||||
|
||||
def parse_time(value: str | float | int | None) -> float | None:
|
||||
"""Parse SS, MM:SS, or HH:MM:SS (with optional .ms) into seconds."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
s = str(value).strip()
|
||||
if not s:
|
||||
return None
|
||||
parts = s.split(":")
|
||||
try:
|
||||
if len(parts) == 1:
|
||||
return float(parts[0])
|
||||
if len(parts) == 2:
|
||||
return int(parts[0]) * 60 + float(parts[1])
|
||||
if len(parts) == 3:
|
||||
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
|
||||
except ValueError:
|
||||
pass
|
||||
raise SystemExit(f"Cannot parse time value: {value!r} (expected SS, MM:SS, or HH:MM:SS)")
|
||||
|
||||
|
||||
def format_time(seconds: float) -> str:
|
||||
total = int(round(seconds))
|
||||
hours, rem = divmod(total, 3600)
|
||||
minutes, sec = divmod(rem, 60)
|
||||
if hours:
|
||||
return f"{hours}:{minutes:02d}:{sec:02d}"
|
||||
return f"{minutes:02d}:{sec:02d}"
|
||||
|
||||
|
||||
def get_metadata(video_path: str) -> dict:
|
||||
if shutil.which("ffprobe") is None:
|
||||
raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v", "quiet",
|
||||
"-print_format", "json",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
str(Path(video_path).resolve()),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffprobe failed: {result.stderr.strip()}")
|
||||
|
||||
data = json.loads(result.stdout or "{}")
|
||||
streams = data.get("streams", [])
|
||||
fmt = data.get("format", {})
|
||||
video_stream = next((s for s in streams if s.get("codec_type") == "video"), {})
|
||||
audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None)
|
||||
|
||||
duration = float(fmt.get("duration") or video_stream.get("duration") or 0)
|
||||
return {
|
||||
"duration_seconds": duration,
|
||||
"width": video_stream.get("width"),
|
||||
"height": video_stream.get("height"),
|
||||
"codec": video_stream.get("codec_name"),
|
||||
"size_bytes": int(fmt.get("size") or 0),
|
||||
"has_audio": audio_stream is not None,
|
||||
}
|
||||
|
||||
|
||||
def auto_fps(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
|
||||
"""Pick fps that targets a sensible frame budget for full-video scans."""
|
||||
if duration_seconds <= 0:
|
||||
return 1.0, 1
|
||||
|
||||
if duration_seconds <= 30:
|
||||
target = min(max_frames, max(12, int(round(duration_seconds))))
|
||||
elif duration_seconds <= 60:
|
||||
target = min(max_frames, 40)
|
||||
elif duration_seconds <= 180: # 3 min
|
||||
target = min(max_frames, 60)
|
||||
elif duration_seconds <= 600: # 10 min
|
||||
target = min(max_frames, 80)
|
||||
else:
|
||||
target = max_frames
|
||||
|
||||
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
|
||||
|
||||
|
||||
def auto_fps_focus(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
|
||||
"""Denser budget for user-specified ranges — they are zooming in for detail."""
|
||||
if duration_seconds <= 0:
|
||||
return min(MAX_FPS, 2.0), 2
|
||||
|
||||
if duration_seconds <= 5:
|
||||
target = min(max_frames, max(10, int(round(duration_seconds * 6))))
|
||||
elif duration_seconds <= 15:
|
||||
target = min(max_frames, max(30, int(round(duration_seconds * 4))))
|
||||
elif duration_seconds <= 30:
|
||||
target = min(max_frames, 60)
|
||||
elif duration_seconds <= 60:
|
||||
target = min(max_frames, 80)
|
||||
elif duration_seconds <= 180:
|
||||
target = max_frames
|
||||
else:
|
||||
target = max_frames
|
||||
|
||||
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
|
||||
|
||||
|
||||
def extract(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
fps: float,
|
||||
resolution: int = 512,
|
||||
max_frames: int = 100,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
) -> list[dict]:
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("frame_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
output_pattern = str(out_dir / "frame_%04d.jpg")
|
||||
cmd: list[str] = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-y",
|
||||
]
|
||||
|
||||
# -ss before -i = fast seek (keyframe-snap, good enough for preview frames).
|
||||
if start_seconds is not None:
|
||||
cmd += ["-ss", f"{start_seconds:.3f}"]
|
||||
if end_seconds is not None:
|
||||
cmd += ["-to", f"{end_seconds:.3f}"]
|
||||
|
||||
cmd += [
|
||||
"-i", str(Path(video_path).resolve()),
|
||||
"-vf", f"fps={fps},{_scale_filter(resolution)}",
|
||||
"-frames:v", str(max_frames),
|
||||
"-q:v", "4",
|
||||
output_pattern,
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffmpeg frame extraction failed: {result.stderr.strip()}")
|
||||
|
||||
offset = start_seconds or 0.0
|
||||
frames = sorted(out_dir.glob("frame_*.jpg"))
|
||||
return [
|
||||
{
|
||||
"index": i,
|
||||
"timestamp_seconds": round(offset + (i / fps if fps > 0 else 0.0), 2),
|
||||
"path": str(p),
|
||||
"reason": "uniform",
|
||||
}
|
||||
for i, p in enumerate(frames)
|
||||
]
|
||||
|
||||
|
||||
def extract_scene_candidates(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
resolution: int = 512,
|
||||
max_frames: int | None = 100,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
threshold: float = SCENE_THRESHOLD,
|
||||
) -> list[dict]:
|
||||
"""Extract first frame plus ffmpeg scene-change frames.
|
||||
|
||||
When ``max_frames`` is set, ``-frames:v`` lets ffmpeg stop decoding once it
|
||||
has emitted that many frames (early exit) and avoids writing extras that we
|
||||
would only delete afterwards. ``None`` (uncapped "complete" detail) keeps
|
||||
every detected shot, as the user explicitly opted in.
|
||||
"""
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("frame_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
output_pattern = str(out_dir / "frame_%04d.jpg")
|
||||
cmd: list[str] = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "info",
|
||||
"-y",
|
||||
]
|
||||
if start_seconds is not None:
|
||||
cmd += ["-ss", f"{start_seconds:.3f}"]
|
||||
if end_seconds is not None:
|
||||
cmd += ["-to", f"{end_seconds:.3f}"]
|
||||
|
||||
vf = f"select='eq(n\\,0)+gt(scene\\,{threshold})',{_scale_filter(resolution)},showinfo"
|
||||
cmd += [
|
||||
"-i", str(Path(video_path).resolve()),
|
||||
"-vf", vf,
|
||||
"-vsync", "vfr",
|
||||
]
|
||||
if max_frames is not None:
|
||||
cmd += ["-frames:v", str(max_frames)]
|
||||
cmd += [
|
||||
"-q:v", "4",
|
||||
output_pattern,
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffmpeg scene extraction failed: {result.stderr.strip()}")
|
||||
|
||||
offset = start_seconds or 0.0
|
||||
timestamps = [round(offset + float(match.group(1)), 2) for match in SHOWINFO_TS_RE.finditer(result.stderr)]
|
||||
frames = sorted(out_dir.glob("frame_*.jpg"))
|
||||
out: list[dict] = []
|
||||
for i, path in enumerate(frames):
|
||||
ts = timestamps[i] if i < len(timestamps) else offset
|
||||
out.append({
|
||||
"index": i,
|
||||
"timestamp_seconds": ts,
|
||||
"path": str(path),
|
||||
"reason": "first-frame" if i == 0 else "scene-change",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def _even_indices(count: int, n: int) -> list[int]:
|
||||
"""Indices of ``n`` evenly-spaced items out of ``count`` (first + last kept).
|
||||
|
||||
``n >= count`` returns every index; ``n == 1`` returns just the first.
|
||||
"""
|
||||
if n >= count:
|
||||
return list(range(count))
|
||||
if n <= 1:
|
||||
return [0]
|
||||
return [round(i * (count - 1) / (n - 1)) for i in range(n)]
|
||||
|
||||
|
||||
def parse_timestamps(value: str | None) -> list[float]:
|
||||
"""Parse a comma-separated list of times (SS, MM:SS, HH:MM:SS) into a
|
||||
sorted, de-duplicated list of seconds. Empty/blank tokens are skipped;
|
||||
an unparseable token raises (via :func:`parse_time`)."""
|
||||
if not value:
|
||||
return []
|
||||
out: list[float] = []
|
||||
for token in value.split(","):
|
||||
token = token.strip()
|
||||
if not token:
|
||||
continue
|
||||
seconds = parse_time(token)
|
||||
if seconds is not None:
|
||||
out.append(float(seconds))
|
||||
return sorted(set(out))
|
||||
|
||||
|
||||
def merge_frames(primary: list[dict], pinned: list[dict]) -> list[dict]:
|
||||
"""Combine two frame lists into one chronological list and reindex 0..n-1.
|
||||
|
||||
``pinned`` frames (transcript cues) are never dropped — this is a plain
|
||||
union, so the cap is enforced upstream by reserving budget for the cues.
|
||||
"""
|
||||
merged = sorted([*primary, *pinned], key=lambda f: f["timestamp_seconds"])
|
||||
for i, frame in enumerate(merged):
|
||||
frame["index"] = i
|
||||
return merged
|
||||
|
||||
|
||||
def extract_at_timestamps(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
timestamps: list[float],
|
||||
resolution: int = 512,
|
||||
max_frames: int | None = None,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""Grab exactly one frame at each requested timestamp (transcript cues).
|
||||
|
||||
Timestamps are absolute source seconds. Any falling outside an active
|
||||
``[start, end]`` focus window are dropped. Files use a ``cue_*.jpg`` prefix
|
||||
so they sit alongside detail-engine ``frame_*.jpg`` output without either
|
||||
clobbering the other. When more cues than ``max_frames`` survive, they are
|
||||
even-sampled (first + last kept) before extraction.
|
||||
"""
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("cue_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
lo = start_seconds or 0.0
|
||||
hi = end_seconds if end_seconds is not None else float("inf")
|
||||
requested = sorted(set(round(float(t), 2) for t in timestamps))
|
||||
in_window = [t for t in requested if lo <= t <= hi]
|
||||
dropped = len(requested) - len(in_window)
|
||||
|
||||
if max_frames is not None and len(in_window) > max_frames:
|
||||
points = [in_window[i] for i in _even_indices(len(in_window), max_frames)]
|
||||
else:
|
||||
points = in_window
|
||||
|
||||
out: list[dict] = []
|
||||
for t in points:
|
||||
path = out_dir / f"cue_{len(out):04d}.jpg"
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-y",
|
||||
"-ss", f"{t:.3f}",
|
||||
"-i", str(Path(video_path).resolve()),
|
||||
"-frames:v", "1",
|
||||
"-vf", _scale_filter(resolution),
|
||||
"-q:v", "4",
|
||||
str(path),
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode == 0 and path.exists():
|
||||
out.append({
|
||||
"index": len(out),
|
||||
"timestamp_seconds": t,
|
||||
"path": str(path),
|
||||
"reason": "transcript-cue",
|
||||
})
|
||||
|
||||
meta = {
|
||||
"engine": "timestamps",
|
||||
"candidate_count": len(requested),
|
||||
"selected_count": len(out),
|
||||
"dropped_out_of_window": dropped,
|
||||
"fallback": False,
|
||||
}
|
||||
return out, meta
|
||||
|
||||
|
||||
def _even_sample(candidates: list[dict], n: int) -> list[dict]:
|
||||
"""Pick ``n`` evenly-spaced candidates (always including first and last),
|
||||
delete the JPEGs we drop, and reindex the survivors 0..len-1.
|
||||
|
||||
Shared by every capped engine so all detail modes sample the same way:
|
||||
detect all candidates across the full range, then thin down to the cap.
|
||||
``n >= len(candidates)`` keeps everything (the uncapped / under-cap case).
|
||||
"""
|
||||
selected = [candidates[i] for i in _even_indices(len(candidates), n)]
|
||||
|
||||
keep_paths = {sel["path"] for sel in selected}
|
||||
for cand in candidates:
|
||||
if cand["path"] not in keep_paths:
|
||||
try:
|
||||
Path(cand["path"]).unlink()
|
||||
except OSError:
|
||||
pass
|
||||
for i, frame in enumerate(selected):
|
||||
frame["index"] = i
|
||||
return selected
|
||||
|
||||
|
||||
def extract_scene_or_uniform(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
fps: float,
|
||||
target_frames: int,
|
||||
resolution: int = 512,
|
||||
max_frames: int | None = 100,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""Prefer scene selection, falling back to uniform only when the video is
|
||||
effectively static (fewer than ``SCENE_MIN_FRAMES`` detected shots).
|
||||
|
||||
Scene cuts are detected across the *whole* range (uncapped) and then
|
||||
even-sampled down to ``max_frames`` via :func:`_even_sample`, exactly like
|
||||
the keyframe engine. This costs a full decode, but it guarantees coverage
|
||||
spans the entire clip — capping detection with ``-frames:v`` instead would
|
||||
keep only the first ``max_frames`` cuts and drop the tail of long videos
|
||||
(and could even fall below ``SCENE_MIN_FRAMES`` and misfire the uniform
|
||||
fallback on a cut-heavy clip).
|
||||
"""
|
||||
scene_frames = extract_scene_candidates(
|
||||
video_path,
|
||||
out_dir,
|
||||
resolution=resolution,
|
||||
max_frames=None,
|
||||
start_seconds=start_seconds,
|
||||
end_seconds=end_seconds,
|
||||
)
|
||||
scene_count = len(scene_frames)
|
||||
if scene_count >= SCENE_MIN_FRAMES:
|
||||
cap = scene_count if max_frames is None else max_frames
|
||||
selected = _even_sample(scene_frames, cap)
|
||||
return selected, {
|
||||
"engine": "scene",
|
||||
"candidate_count": scene_count,
|
||||
"selected_count": len(selected),
|
||||
"fallback": False,
|
||||
}
|
||||
|
||||
fallback_cap = target_frames if max_frames is None else min(max_frames, target_frames)
|
||||
frames = extract(
|
||||
video_path,
|
||||
out_dir,
|
||||
fps=fps,
|
||||
resolution=resolution,
|
||||
max_frames=fallback_cap,
|
||||
start_seconds=start_seconds,
|
||||
end_seconds=end_seconds,
|
||||
)
|
||||
return frames, {
|
||||
"engine": "uniform",
|
||||
"candidate_count": scene_count,
|
||||
"selected_count": len(frames),
|
||||
"fallback": True,
|
||||
}
|
||||
|
||||
|
||||
def extract_keyframes(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
resolution: int = 512,
|
||||
max_frames: int | None = 50,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""Decode only keyframes (I-frames) — the cheap, near-instant tier.
|
||||
|
||||
``-skip_frame nokey`` makes ffmpeg reconstruct only keyframes, skipping all
|
||||
P/B frames. Encoders emit keyframes at scene cuts, so these already
|
||||
approximate "distinct moments". Over-cap → even-sample first→last; too few
|
||||
keyframes → uniform fallback.
|
||||
"""
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("frame_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
output_pattern = str(out_dir / "frame_%04d.jpg")
|
||||
cmd: list[str] = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "info",
|
||||
"-y",
|
||||
]
|
||||
if start_seconds is not None:
|
||||
cmd += ["-ss", f"{start_seconds:.3f}"]
|
||||
if end_seconds is not None:
|
||||
cmd += ["-to", f"{end_seconds:.3f}"]
|
||||
cmd += [
|
||||
"-skip_frame", "nokey",
|
||||
"-i", str(Path(video_path).resolve()),
|
||||
"-vf", f"{_scale_filter(resolution)},showinfo",
|
||||
"-vsync", "vfr",
|
||||
"-q:v", "4",
|
||||
output_pattern,
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffmpeg keyframe extraction failed: {result.stderr.strip()}")
|
||||
|
||||
offset = start_seconds or 0.0
|
||||
timestamps = [round(offset + float(m.group(1)), 2) for m in SHOWINFO_TS_RE.finditer(result.stderr)]
|
||||
files = sorted(out_dir.glob("frame_*.jpg"))
|
||||
candidates: list[dict] = []
|
||||
for i, path in enumerate(files):
|
||||
ts = timestamps[i] if i < len(timestamps) else offset
|
||||
candidates.append({
|
||||
"index": i,
|
||||
"timestamp_seconds": ts,
|
||||
"path": str(path),
|
||||
"reason": "keyframe",
|
||||
})
|
||||
|
||||
# Too few keyframes → uniform fallback over the same range.
|
||||
if len(candidates) < KEYFRAME_MIN:
|
||||
for cand in candidates:
|
||||
try:
|
||||
Path(cand["path"]).unlink()
|
||||
except OSError:
|
||||
pass
|
||||
meta = get_metadata(video_path)
|
||||
full_duration = meta["duration_seconds"]
|
||||
eff_start = start_seconds or 0.0
|
||||
eff_end = end_seconds if end_seconds is not None else full_duration
|
||||
eff_duration = max(0.0, eff_end - eff_start)
|
||||
budget = max_frames if max_frames is not None else 100
|
||||
fps, _ = auto_fps(eff_duration, max_frames=budget)
|
||||
frames_out = extract(
|
||||
video_path,
|
||||
out_dir,
|
||||
fps=fps,
|
||||
resolution=resolution,
|
||||
max_frames=budget,
|
||||
start_seconds=start_seconds,
|
||||
end_seconds=end_seconds,
|
||||
)
|
||||
return frames_out, {
|
||||
"engine": "uniform",
|
||||
"candidate_count": len(candidates),
|
||||
"selected_count": len(frames_out),
|
||||
"fallback": True,
|
||||
}
|
||||
|
||||
# Detect-all then even-sample down to the cap (first + last always kept).
|
||||
# ``max_frames is None`` (uncapped) keeps every keyframe.
|
||||
candidate_count = len(candidates)
|
||||
cap = candidate_count if max_frames is None else max_frames
|
||||
selected = _even_sample(candidates, cap)
|
||||
return selected, {
|
||||
"engine": "keyframe",
|
||||
"candidate_count": candidate_count,
|
||||
"selected_count": len(selected),
|
||||
"fallback": False,
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
print(
|
||||
"usage: frames.py <video-path> <out-dir> [--fps F] [--resolution W] "
|
||||
"[--max-frames N] [--start T] [--end T]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(2)
|
||||
|
||||
video = sys.argv[1]
|
||||
out = Path(sys.argv[2])
|
||||
args = sys.argv[3:]
|
||||
|
||||
fps_override = None
|
||||
resolution = 512
|
||||
max_frames = 100
|
||||
start_arg = None
|
||||
end_arg = None
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == "--fps":
|
||||
fps_override = float(args[i + 1]); i += 2
|
||||
elif args[i] == "--resolution":
|
||||
resolution = int(args[i + 1]); i += 2
|
||||
elif args[i] == "--max-frames":
|
||||
max_frames = int(args[i + 1]); i += 2
|
||||
elif args[i] == "--start":
|
||||
start_arg = args[i + 1]; i += 2
|
||||
elif args[i] == "--end":
|
||||
end_arg = args[i + 1]; i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
meta = get_metadata(video)
|
||||
start_sec = parse_time(start_arg)
|
||||
end_sec = parse_time(end_arg)
|
||||
full_duration = meta["duration_seconds"]
|
||||
|
||||
effective_start = start_sec if start_sec is not None else 0.0
|
||||
effective_end = end_sec if end_sec is not None else full_duration
|
||||
effective_duration = max(0.0, effective_end - effective_start)
|
||||
|
||||
focused = start_sec is not None or end_sec is not None
|
||||
if focused:
|
||||
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
|
||||
else:
|
||||
fps, target = auto_fps(effective_duration, max_frames=max_frames)
|
||||
if fps_override is not None:
|
||||
fps = fps_override
|
||||
target = max(1, int(round(fps * effective_duration)))
|
||||
|
||||
frames = extract(
|
||||
video, out,
|
||||
fps=fps,
|
||||
resolution=resolution,
|
||||
max_frames=max_frames,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
print(json.dumps(
|
||||
{"meta": meta, "fps": fps, "target": target, "focused": focused, "frames": frames},
|
||||
indent=2,
|
||||
))
|
||||
@@ -26,6 +26,11 @@ import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
from config import get_config # noqa: E402
|
||||
|
||||
|
||||
REQUIRED_BINARIES = ["ffmpeg", "ffprobe", "yt-dlp"]
|
||||
CONFIG_DIR = Path.home() / ".config" / "watch"
|
||||
@@ -46,6 +51,9 @@ ENV_TEMPLATE = """# /watch API configuration
|
||||
|
||||
GROQ_API_KEY=
|
||||
OPENAI_API_KEY=
|
||||
|
||||
# Default watch behavior (the /watch first-run wizard sets this for you):
|
||||
# WATCH_DETAIL=balanced # transcript | efficient | balanced | token-burner
|
||||
"""
|
||||
|
||||
|
||||
@@ -57,11 +65,19 @@ def _check_binaries() -> list[str]:
|
||||
return [b for b in REQUIRED_BINARIES if not _which(b)]
|
||||
|
||||
|
||||
_PERM_WARNED: set[str] = set()
|
||||
|
||||
|
||||
def _check_file_permissions(path: Path) -> None:
|
||||
"""Warn to stderr if a secrets file is world/group readable."""
|
||||
"""Warn to stderr (once per path per process) if a secrets file is
|
||||
world/group readable."""
|
||||
key = str(path)
|
||||
if key in _PERM_WARNED:
|
||||
return
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
if mode & 0o044:
|
||||
_PERM_WARNED.add(key)
|
||||
sys.stderr.write(
|
||||
f"[watch] WARNING: {path} is readable by other users. "
|
||||
f"Run: chmod 600 {path}\n"
|
||||
@@ -197,9 +213,20 @@ def _install_hint_windows(missing: list[str]) -> str:
|
||||
|
||||
|
||||
def _status() -> dict:
|
||||
"""Structured preflight snapshot."""
|
||||
"""Structured preflight snapshot.
|
||||
|
||||
`status` describes the *ideal* state (a Whisper key is encouraged), so a
|
||||
keyless install still reports `needs_key` on the very first run — that's
|
||||
the agent's cue to encourage adding one.
|
||||
|
||||
`can_proceed` is the operational gate: /watch can run as long as the
|
||||
binaries are present AND the user has either set a key or already finished
|
||||
setup (consciously opting out of Whisper). A keyless user who completed
|
||||
setup is NOT nagged on every call.
|
||||
"""
|
||||
missing = _check_binaries()
|
||||
has_key, backend = _have_api_key()
|
||||
setup_complete = not is_first_run()
|
||||
|
||||
if not missing and has_key:
|
||||
status = "ready"
|
||||
@@ -210,13 +237,19 @@ def _status() -> dict:
|
||||
else:
|
||||
status = "needs_key"
|
||||
|
||||
can_proceed = (not missing) and (has_key or setup_complete)
|
||||
|
||||
cfg = get_config()
|
||||
return {
|
||||
"status": status,
|
||||
"first_run": is_first_run(),
|
||||
"can_proceed": can_proceed,
|
||||
"first_run": not setup_complete,
|
||||
"setup_complete": setup_complete,
|
||||
"missing_binaries": missing,
|
||||
"whisper_backend": backend,
|
||||
"has_api_key": has_key,
|
||||
"config_file": str(CONFIG_FILE),
|
||||
"watch_detail": cfg["detail"],
|
||||
"platform": platform.system(),
|
||||
}
|
||||
|
||||
@@ -224,20 +257,23 @@ def _status() -> dict:
|
||||
def cmd_check() -> int:
|
||||
"""Silent-on-success preflight.
|
||||
|
||||
Exit 0 with no output when ready. On failure, print one actionable line
|
||||
to stderr and return:
|
||||
Exit 0 with no output when /watch can run. A keyless user who already
|
||||
finished setup (SETUP_COMPLETE=true) counts as ready — Whisper is
|
||||
encouraged, not required — so they are never nagged on follow-up calls.
|
||||
|
||||
On a state that blocks /watch, print one actionable line to stderr:
|
||||
2 → binaries missing
|
||||
3 → API key missing
|
||||
3 → genuine first run with no API key (encourage one)
|
||||
4 → both missing
|
||||
"""
|
||||
s = _status()
|
||||
if s["status"] == "ready":
|
||||
if s["can_proceed"]:
|
||||
return 0
|
||||
|
||||
parts = []
|
||||
if s["missing_binaries"]:
|
||||
parts.append(f"missing binaries: {', '.join(s['missing_binaries'])}")
|
||||
if not s["has_api_key"]:
|
||||
if not s["has_api_key"] and not s["setup_complete"]:
|
||||
parts.append("no Whisper API key (GROQ_API_KEY or OPENAI_API_KEY)")
|
||||
installer = Path(__file__).resolve()
|
||||
sys.stderr.write(
|
||||
Executable
+382
@@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env python3
|
||||
"""/watch entry point: download video, extract frames, parse transcript.
|
||||
|
||||
Prints a markdown report to stdout listing frame paths + transcript. Claude
|
||||
then Reads each frame path to see the video.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from config import frame_cap, get_config # noqa: E402
|
||||
from download import download, fetch_captions, is_url # noqa: E402
|
||||
from frames import MAX_FPS, auto_fps, auto_fps_focus, extract_at_timestamps, extract_keyframes, extract_scene_or_uniform, format_time, get_metadata, merge_frames, parse_time, parse_timestamps # noqa: E402
|
||||
from transcribe import filter_range, format_transcript, parse_vtt # noqa: E402
|
||||
from whisper import load_api_key, transcribe_video # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="watch",
|
||||
description="Download a video, extract auto-scaled frames, and surface the transcript.",
|
||||
)
|
||||
ap.add_argument("source", help="Video URL or local file path")
|
||||
ap.add_argument("--max-frames", type=int, default=None, help="Override frame cap")
|
||||
ap.add_argument("--resolution", type=int, default=512, help="Frame width in pixels (default 512)")
|
||||
ap.add_argument("--fps", type=float, default=None, help="Override auto-fps")
|
||||
ap.add_argument(
|
||||
"--detail",
|
||||
choices=["transcript", "efficient", "balanced", "token-burner"],
|
||||
default=None,
|
||||
help="Fidelity/speed dial: transcript (no frames), efficient (fast keyframes, cap 50), "
|
||||
"balanced (scene, cap 100), token-burner (scene, uncapped).",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--timestamps",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Comma-separated absolute timestamps (SS, MM:SS, HH:MM:SS) to grab a frame at, "
|
||||
"e.g. transcript-flagged 'look here' moments. Added on top of the detail frames "
|
||||
"(reserved against the cap); with --detail transcript these become the only frames.",
|
||||
)
|
||||
ap.add_argument("--start", type=str, default=None, help="Range start (SS, MM:SS, or HH:MM:SS)")
|
||||
ap.add_argument("--end", type=str, default=None, help="Range end (SS, MM:SS, or HH:MM:SS)")
|
||||
ap.add_argument("--out-dir", type=str, default=None, help="Working directory (default: tmp)")
|
||||
ap.add_argument(
|
||||
"--no-whisper",
|
||||
action="store_true",
|
||||
help="Disable Whisper fallback. Report frames-only if no captions available.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--whisper",
|
||||
choices=["groq", "openai"],
|
||||
default=None,
|
||||
help="Force a specific Whisper backend. Default: prefer Groq, fall back to OpenAI.",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
config = get_config()
|
||||
detail = args.detail or str(config["detail"])
|
||||
configured_cap = frame_cap(detail)
|
||||
if args.max_frames is not None:
|
||||
max_frames = args.max_frames
|
||||
else:
|
||||
max_frames = configured_cap
|
||||
if max_frames is not None and max_frames < 1:
|
||||
raise SystemExit("--max-frames must be greater than zero")
|
||||
budget_cap = max_frames if max_frames is not None else 100
|
||||
cue_timestamps = parse_timestamps(args.timestamps)
|
||||
|
||||
if args.out_dir:
|
||||
work = Path(args.out_dir).expanduser().resolve()
|
||||
else:
|
||||
work = Path(tempfile.mkdtemp(prefix="watch-"))
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
print(f"[watch] working dir: {work}", file=sys.stderr)
|
||||
|
||||
url_source = is_url(args.source)
|
||||
dl: dict = {"subtitle_path": None, "info": {}, "downloaded": False}
|
||||
transcript_segments: list[dict] = []
|
||||
transcript_text: str | None = None
|
||||
transcript_source: str | None = None
|
||||
video_path: str | None = None
|
||||
|
||||
if url_source:
|
||||
print("[watch] checking metadata/captions via yt-dlp…", file=sys.stderr)
|
||||
dl = fetch_captions(args.source, work / "download")
|
||||
if dl.get("subtitle_path"):
|
||||
try:
|
||||
transcript_segments = parse_vtt(dl["subtitle_path"])
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = "captions"
|
||||
except Exception as exc:
|
||||
print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr)
|
||||
transcript_segments = []
|
||||
|
||||
# --timestamps needs the video for frame grabs, so it overrides the
|
||||
# transcript-mode download skip (and forces a full, not audio-only, fetch).
|
||||
audio_only = detail == "transcript" and not cue_timestamps
|
||||
if detail == "transcript" and transcript_segments and not cue_timestamps:
|
||||
video_path = None
|
||||
else:
|
||||
if url_source:
|
||||
print(
|
||||
"[watch] downloading audio via yt-dlp…" if audio_only
|
||||
else "[watch] downloading video via yt-dlp…",
|
||||
file=sys.stderr,
|
||||
)
|
||||
dl = download(
|
||||
args.source,
|
||||
work / "download",
|
||||
audio_only=audio_only,
|
||||
)
|
||||
else:
|
||||
print("[watch] using local file…", file=sys.stderr)
|
||||
dl = download(args.source, work / "download")
|
||||
video_path = dl["video_path"]
|
||||
|
||||
meta = get_metadata(video_path) if video_path else {
|
||||
"duration_seconds": float((dl.get("info") or {}).get("duration") or 0),
|
||||
"width": None,
|
||||
"height": None,
|
||||
"codec": None,
|
||||
"has_audio": False,
|
||||
}
|
||||
full_duration = meta["duration_seconds"]
|
||||
|
||||
start_sec = parse_time(args.start)
|
||||
end_sec = parse_time(args.end)
|
||||
|
||||
if start_sec is not None and start_sec < 0:
|
||||
raise SystemExit("--start must be non-negative")
|
||||
if end_sec is not None and start_sec is not None and end_sec <= start_sec:
|
||||
raise SystemExit("--end must be greater than --start")
|
||||
if full_duration > 0 and start_sec is not None and start_sec >= full_duration:
|
||||
raise SystemExit(f"--start {start_sec:.1f}s is past end of video ({full_duration:.1f}s)")
|
||||
|
||||
effective_start = start_sec if start_sec is not None else 0.0
|
||||
effective_end = end_sec if end_sec is not None else full_duration
|
||||
effective_duration = max(0.0, effective_end - effective_start)
|
||||
focused = start_sec is not None or end_sec is not None
|
||||
|
||||
if focused:
|
||||
fps, target = auto_fps_focus(effective_duration, max_frames=budget_cap)
|
||||
else:
|
||||
fps, target = auto_fps(effective_duration, max_frames=budget_cap)
|
||||
if args.fps is not None:
|
||||
fps = min(args.fps, MAX_FPS)
|
||||
target = max(1, int(round(fps * effective_duration)))
|
||||
|
||||
if transcript_segments and focused:
|
||||
transcript_segments = filter_range(transcript_segments, start_sec, end_sec)
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
|
||||
scope = (
|
||||
f"{format_time(effective_start)}-{format_time(effective_end)} ({effective_duration:.1f}s)"
|
||||
if focused else f"full {effective_duration:.1f}s"
|
||||
)
|
||||
frames: list[dict] = []
|
||||
frame_meta: dict = {"engine": "none", "candidate_count": 0, "selected_count": 0, "fallback": False}
|
||||
cue_frames: list[dict] = []
|
||||
cue_meta: dict = {}
|
||||
|
||||
# Transcript cues are pinned: extracted first and counted against the cap so
|
||||
# the detail engine never evicts the moments the user explicitly asked for.
|
||||
if cue_timestamps and video_path:
|
||||
cue_frames, cue_meta = extract_at_timestamps(
|
||||
video_path,
|
||||
work / "frames",
|
||||
cue_timestamps,
|
||||
resolution=args.resolution,
|
||||
max_frames=max_frames,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
if cue_meta.get("dropped_out_of_window"):
|
||||
print(
|
||||
f"[watch] {cue_meta['dropped_out_of_window']} cue timestamp(s) outside the "
|
||||
"focus range — dropped",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
detail_budget = max_frames if max_frames is None else max(0, max_frames - len(cue_frames))
|
||||
if detail != "transcript" and video_path and detail_budget != 0:
|
||||
cap_label = "unlimited" if detail_budget is None else str(detail_budget)
|
||||
engine_label = "keyframes" if detail == "efficient" else "scene-aware frames"
|
||||
print(
|
||||
f"[watch] extracting {engine_label} over {scope} "
|
||||
f"(target {target}, cap {cap_label})…",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if detail == "efficient":
|
||||
frames, frame_meta = extract_keyframes(
|
||||
video_path,
|
||||
work / "frames",
|
||||
resolution=args.resolution,
|
||||
max_frames=detail_budget,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
else: # balanced, token-burner
|
||||
frames, frame_meta = extract_scene_or_uniform(
|
||||
video_path,
|
||||
work / "frames",
|
||||
fps=fps,
|
||||
target_frames=target,
|
||||
resolution=args.resolution,
|
||||
max_frames=detail_budget,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
|
||||
if cue_frames:
|
||||
frames = merge_frames(frames, cue_frames)
|
||||
|
||||
if not transcript_segments and dl.get("subtitle_path"):
|
||||
try:
|
||||
all_segments = parse_vtt(dl["subtitle_path"])
|
||||
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = "captions"
|
||||
except Exception as exc:
|
||||
print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr)
|
||||
|
||||
if not transcript_segments and not args.no_whisper and video_path and meta.get("has_audio"):
|
||||
backend, api_key = load_api_key(args.whisper)
|
||||
if backend and api_key:
|
||||
try:
|
||||
all_segments, used_backend = transcribe_video(
|
||||
video_path,
|
||||
work / "audio.mp3",
|
||||
backend=backend,
|
||||
api_key=api_key,
|
||||
)
|
||||
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = f"whisper ({used_backend})"
|
||||
except SystemExit as exc:
|
||||
print(f"[watch] whisper fallback failed: {exc}", file=sys.stderr)
|
||||
else:
|
||||
hint = (
|
||||
f"--whisper {args.whisper} was set but the matching API key is missing"
|
||||
if args.whisper else
|
||||
"no subtitles and no Whisper API key found"
|
||||
)
|
||||
setup_py = SCRIPT_DIR / "setup.py"
|
||||
print(
|
||||
f"[watch] {hint} — run `python3 {setup_py}` to enable the Whisper fallback",
|
||||
file=sys.stderr,
|
||||
)
|
||||
elif not transcript_segments and video_path and not meta.get("has_audio"):
|
||||
print("[watch] no audio stream found — proceeding without transcription", file=sys.stderr)
|
||||
|
||||
info = dl.get("info") or {}
|
||||
|
||||
print()
|
||||
print("# watch: video report")
|
||||
print()
|
||||
print(f"- **Source:** {args.source}")
|
||||
if info.get("title"):
|
||||
print(f"- **Title:** {info['title']}")
|
||||
if info.get("uploader"):
|
||||
print(f"- **Uploader:** {info['uploader']}")
|
||||
print(f"- **Duration:** {format_time(full_duration)} ({full_duration:.1f}s)")
|
||||
if focused:
|
||||
print(
|
||||
f"- **Focus range:** {format_time(effective_start)} → {format_time(effective_end)} "
|
||||
f"({effective_duration:.1f}s)"
|
||||
)
|
||||
if meta.get("width") and meta.get("height"):
|
||||
print(f"- **Resolution:** {meta['width']}x{meta['height']} ({meta.get('codec') or 'unknown codec'})")
|
||||
range_mode = "focused" if focused else "full"
|
||||
print(f"- **Detail:** {detail}")
|
||||
detail_count = frame_meta.get("selected_count", 0)
|
||||
if detail != "transcript":
|
||||
cap_label = "unlimited" if detail_budget is None else str(detail_budget)
|
||||
engine = frame_meta.get("engine", "scene")
|
||||
fallback = " with uniform fallback" if frame_meta.get("fallback") else ""
|
||||
print(
|
||||
f"- **Frames:** {detail_count} selected from {frame_meta.get('candidate_count', detail_count)} "
|
||||
f"candidates ({engine}{fallback}, {range_mode} range, budget {target}, cap {cap_label})"
|
||||
)
|
||||
elif not cue_frames:
|
||||
print("- **Frames:** skipped (transcript detail)")
|
||||
if cue_frames:
|
||||
dropped = cue_meta.get("dropped_out_of_window", 0)
|
||||
drop_note = f", {dropped} dropped outside range" if dropped else ""
|
||||
print(
|
||||
f"- **Cue frames:** {len(cue_frames)} at transcript-flagged timestamps "
|
||||
f"(transcript-cue{drop_note})"
|
||||
)
|
||||
if frames:
|
||||
print(f"- **Frame size:** max {args.resolution}px wide, max 1998px tall")
|
||||
if transcript_segments:
|
||||
in_range = " in range" if focused else ""
|
||||
print(
|
||||
f"- **Transcript:** {len(transcript_segments)} segments{in_range} "
|
||||
f"(via {transcript_source or 'captions'})"
|
||||
)
|
||||
else:
|
||||
print("- **Transcript:** none available")
|
||||
|
||||
if detail == "token-burner" and len(frames) > 250:
|
||||
print()
|
||||
print(
|
||||
f"> **Warning:** token-burner detail selected {len(frames)} frames. "
|
||||
"This may use a large number of image tokens."
|
||||
)
|
||||
|
||||
if not focused and full_duration > 600 and detail != "transcript":
|
||||
mins = int(full_duration // 60)
|
||||
print()
|
||||
print(
|
||||
f"> **Warning:** This is a {mins}-minute video. Frame coverage is sparse at this length — "
|
||||
"accuracy degrades noticeably on anything over 10 minutes. For better results, "
|
||||
"re-run with `--start HH:MM:SS --end HH:MM:SS` to zoom into a specific section."
|
||||
)
|
||||
|
||||
print()
|
||||
print("## Frames")
|
||||
print()
|
||||
if frames:
|
||||
print(f"Frames live at: `{work / 'frames'}`")
|
||||
print()
|
||||
print(
|
||||
"**Read each frame path below with the Read tool to view the image.** "
|
||||
"Frames are in chronological order; `t=MM:SS` is the absolute timestamp in the source video."
|
||||
)
|
||||
print()
|
||||
for frame in frames:
|
||||
print(
|
||||
f"- `{frame['path']}` "
|
||||
f"(t={format_time(frame['timestamp_seconds'])}, reason={frame.get('reason', 'selected')})"
|
||||
)
|
||||
else:
|
||||
print("_No frames extracted._")
|
||||
|
||||
print()
|
||||
print("## Transcript")
|
||||
print()
|
||||
if transcript_text:
|
||||
label = transcript_source or "captions"
|
||||
if focused:
|
||||
print(f"_Source: {label}. Filtered to {format_time(effective_start)} → {format_time(effective_end)}:_")
|
||||
else:
|
||||
print(f"_Source: {label}._")
|
||||
print()
|
||||
print("```")
|
||||
print(transcript_text)
|
||||
print("```")
|
||||
elif detail == "transcript":
|
||||
print(
|
||||
"_No transcript available at transcript detail. Captions were missing and Whisper was "
|
||||
"unavailable or failed, so there is no visual fallback here. Re-run with "
|
||||
"`--detail balanced` for frames._"
|
||||
)
|
||||
elif focused and dl.get("subtitle_path"):
|
||||
print(f"_No transcript lines fell inside {format_time(effective_start)} → {format_time(effective_end)}._")
|
||||
else:
|
||||
setup_py = SCRIPT_DIR / "setup.py"
|
||||
print(
|
||||
"_No transcript available — proceed with frames only. "
|
||||
"Captions were missing and the Whisper fallback was unavailable "
|
||||
"(no API key set, or `--no-whisper` was used). "
|
||||
f"Run `python3 {setup_py}` to enable Whisper, then re-run._"
|
||||
)
|
||||
|
||||
print()
|
||||
print("---")
|
||||
print(f"_Work dir: `{work}` — delete when done._")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Shared pytest fixtures: ffmpeg-synthesized clips and scripts/ on sys.path."""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Make the bundled scripts importable (mirrors watch.py's sys.path insert).
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
# 14 visually distinct fills → 14 abrupt cuts → x264 emits a keyframe per cut.
|
||||
COLORS = [
|
||||
"red", "green", "blue", "white", "black", "yellow", "cyan",
|
||||
"magenta", "gray", "orange", "purple", "brown", "navy", "olive",
|
||||
]
|
||||
|
||||
|
||||
def _run(cmd: list[str]) -> None:
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"ffmpeg failed: {' '.join(cmd)}\n{result.stderr}")
|
||||
|
||||
|
||||
def build_cut_clip(
|
||||
path: Path,
|
||||
n: int = 14,
|
||||
seg: float = 0.4,
|
||||
size: str = "320x240",
|
||||
fps: int = 10,
|
||||
) -> None:
|
||||
"""Concatenate ``n`` solid-color segments into one clip with ``n`` cuts.
|
||||
|
||||
Each color change is a hard scene cut, so the scene selector finds ~n-1
|
||||
changes. x264's own scenecut detection is unreliable on flat fills, so we
|
||||
force a keyframe at every ``seg`` boundary — giving ~n real keyframes for
|
||||
the keyframe engine to find.
|
||||
"""
|
||||
inputs: list[str] = []
|
||||
for i in range(n):
|
||||
color = COLORS[i % len(COLORS)]
|
||||
inputs += ["-f", "lavfi", "-t", str(seg), "-i", f"color=c={color}:s={size}:r={fps}"]
|
||||
streams = "".join(f"[{i}:v]" for i in range(n))
|
||||
filt = f"{streams}concat=n={n}:v=1:a=0[out]"
|
||||
_run([
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
*inputs,
|
||||
"-filter_complex", filt, "-map", "[out]",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p",
|
||||
"-force_key_frames", f"expr:gte(t,n_forced*{seg})",
|
||||
str(path),
|
||||
])
|
||||
|
||||
|
||||
def build_static_clip(
|
||||
path: Path,
|
||||
duration: float = 3.0,
|
||||
size: str = "320x240",
|
||||
fps: int = 10,
|
||||
) -> None:
|
||||
"""One solid color: 1 keyframe, no scene changes → triggers both fallbacks."""
|
||||
_run([
|
||||
"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
|
||||
"-f", "lavfi", "-t", str(duration), "-i", f"color=c=blue:s={size}:r={fps}",
|
||||
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-g", "600",
|
||||
str(path),
|
||||
])
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def cut_clip(tmp_path_factory: pytest.TempPathFactory) -> Path:
|
||||
path = tmp_path_factory.mktemp("clips") / "cuts.mp4"
|
||||
build_cut_clip(path)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def static_clip(tmp_path_factory: pytest.TempPathFactory) -> Path:
|
||||
path = tmp_path_factory.mktemp("clips") / "static.mp4"
|
||||
build_static_clip(path)
|
||||
return path
|
||||
@@ -0,0 +1,37 @@
|
||||
"""WATCH_DETAIL resolution and frame_cap mapping."""
|
||||
from __future__ import annotations
|
||||
|
||||
import config
|
||||
|
||||
|
||||
def test_default_detail_is_balanced(monkeypatch, tmp_path):
|
||||
monkeypatch.delenv("WATCH_DETAIL", raising=False)
|
||||
monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env")
|
||||
assert config.get_config()["detail"] == "balanced"
|
||||
|
||||
|
||||
def test_env_overrides_detail(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("WATCH_DETAIL", "efficient")
|
||||
monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env")
|
||||
assert config.get_config()["detail"] == "efficient"
|
||||
|
||||
|
||||
def test_invalid_detail_falls_back_to_default(monkeypatch, tmp_path):
|
||||
monkeypatch.setenv("WATCH_DETAIL", "bogus")
|
||||
monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env")
|
||||
assert config.get_config()["detail"] == "balanced"
|
||||
|
||||
|
||||
def test_get_config_keys(monkeypatch, tmp_path):
|
||||
monkeypatch.delenv("WATCH_DETAIL", raising=False)
|
||||
monkeypatch.setattr(config, "CONFIG_FILE", tmp_path / "missing.env")
|
||||
cfg = config.get_config()
|
||||
assert set(cfg) == {"detail", "config_file"}
|
||||
|
||||
|
||||
def test_frame_cap_mapping():
|
||||
assert config.frame_cap("efficient") == 50
|
||||
assert config.frame_cap("balanced") == 100
|
||||
assert config.frame_cap("token-burner") is None
|
||||
assert config.frame_cap("transcript") is None
|
||||
assert config.frame_cap("anything-else") == 100
|
||||
@@ -0,0 +1,64 @@
|
||||
"""yt-dlp argv construction for download.py.
|
||||
|
||||
Regression guard: ``--sub-langs all`` makes yt-dlp fetch YouTube's hundreds of
|
||||
auto-translated caption tracks, which can take minutes and stalls before the
|
||||
video download even starts. We only support English, so the request must stay
|
||||
bounded to the English-only pattern.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts"
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
import download # noqa: E402
|
||||
|
||||
URL = "https://www.youtube.com/watch?v=rlOpbu3Enkw"
|
||||
|
||||
|
||||
def _capture_argv(monkeypatch: pytest.MonkeyPatch) -> list[list[str]]:
|
||||
"""Stub subprocess.run inside download.py and record every argv."""
|
||||
calls: list[list[str]] = []
|
||||
|
||||
class _Result:
|
||||
returncode = 0
|
||||
stdout = ""
|
||||
stderr = ""
|
||||
|
||||
def fake_run(cmd, *args, **kwargs):
|
||||
calls.append(list(cmd))
|
||||
return _Result()
|
||||
|
||||
monkeypatch.setattr(download.subprocess, "run", fake_run)
|
||||
return calls
|
||||
|
||||
|
||||
def _sub_langs(argv: list[str]) -> str:
|
||||
idx = argv.index("--sub-langs")
|
||||
return argv[idx + 1]
|
||||
|
||||
|
||||
def _assert_english_only(langs: str) -> None:
|
||||
tokens = langs.split(",")
|
||||
assert "all" not in tokens, f"sub-langs must not request all languages, got {langs!r}"
|
||||
assert all(t.startswith("en") for t in tokens), f"sub-langs must be English-only, got {langs!r}"
|
||||
|
||||
|
||||
def test_fetch_captions_requests_english_only(monkeypatch, tmp_path):
|
||||
calls = _capture_argv(monkeypatch)
|
||||
download.fetch_captions(URL, tmp_path / "download")
|
||||
_assert_english_only(_sub_langs(calls[0]))
|
||||
|
||||
|
||||
def test_download_url_requests_english_only(monkeypatch, tmp_path):
|
||||
calls = _capture_argv(monkeypatch)
|
||||
# _pick_video returns None with no real file, which raises SystemExit after
|
||||
# the yt-dlp argv is already built — that's all we need to inspect.
|
||||
with pytest.raises(SystemExit):
|
||||
download.download_url(URL, tmp_path / "download")
|
||||
_assert_english_only(_sub_langs(calls[0]))
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Smoke test: the ffmpeg fixtures actually produce playable clips."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _duration(path: Path) -> float:
|
||||
out = subprocess.run(
|
||||
["ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", str(path)],
|
||||
capture_output=True, text=True,
|
||||
).stdout
|
||||
return float(json.loads(out)["format"]["duration"])
|
||||
|
||||
|
||||
def test_cut_clip_builds(cut_clip: Path):
|
||||
assert cut_clip.exists() and cut_clip.stat().st_size > 0
|
||||
assert _duration(cut_clip) > 4.0 # 14 * 0.4s ≈ 5.6s
|
||||
|
||||
|
||||
def test_static_clip_builds(static_clip: Path):
|
||||
assert static_clip.exists() and static_clip.stat().st_size > 0
|
||||
assert _duration(static_clip) > 2.0
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Keyframe engine + preserved scene/uniform fallbacks."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import frames
|
||||
|
||||
|
||||
def test_keyframe_engine_on_cut_clip(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_keyframes(str(cut_clip), tmp_path / "f", max_frames=50)
|
||||
assert meta["engine"] == "keyframe"
|
||||
assert meta["fallback"] is False
|
||||
assert len(out) >= frames.KEYFRAME_MIN
|
||||
assert all(fr["reason"] == "keyframe" for fr in out)
|
||||
assert len(out) == len(list((tmp_path / "f").glob("frame_*.jpg")))
|
||||
|
||||
|
||||
def test_keyframe_even_sampling_caps_and_spans(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_keyframes(str(cut_clip), tmp_path / "f", max_frames=5)
|
||||
assert meta["engine"] == "keyframe"
|
||||
assert len(out) == 5
|
||||
assert meta["selected_count"] == 5
|
||||
assert meta["candidate_count"] > 5
|
||||
ts = [fr["timestamp_seconds"] for fr in out]
|
||||
assert ts == sorted(ts)
|
||||
assert ts[0] < ts[-1] # spans first → last keyframe
|
||||
assert [fr["index"] for fr in out] == [0, 1, 2, 3, 4]
|
||||
|
||||
|
||||
def test_keyframe_fallback_on_static_clip(static_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_keyframes(str(static_clip), tmp_path / "f", max_frames=50)
|
||||
assert meta["engine"] == "uniform"
|
||||
assert meta["fallback"] is True
|
||||
assert len(out) > 0
|
||||
assert all(fr["reason"] == "uniform" for fr in out)
|
||||
|
||||
|
||||
def test_scene_engine_on_cut_clip(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_scene_or_uniform(
|
||||
str(cut_clip), tmp_path / "f", fps=2.0, target_frames=50, max_frames=100,
|
||||
)
|
||||
assert meta["engine"] == "scene"
|
||||
assert meta["fallback"] is False
|
||||
assert len(out) >= frames.SCENE_MIN_FRAMES
|
||||
|
||||
|
||||
def test_scene_even_sampling_caps_and_spans(cut_clip: Path, tmp_path: Path):
|
||||
"""Over-cap scene detection must even-sample across the whole clip, not keep
|
||||
the first N cuts and drop the tail (the long-video coverage bug)."""
|
||||
out, meta = frames.extract_scene_or_uniform(
|
||||
str(cut_clip), tmp_path / "f", fps=2.0, target_frames=50, max_frames=5,
|
||||
)
|
||||
assert meta["engine"] == "scene"
|
||||
assert meta["fallback"] is False
|
||||
assert len(out) == 5
|
||||
assert meta["selected_count"] == 5
|
||||
assert meta["candidate_count"] > 5 # all cuts detected, then sampled down
|
||||
ts = [fr["timestamp_seconds"] for fr in out]
|
||||
assert ts == sorted(ts)
|
||||
assert ts[-1] > 4.0 # spans the full ~5.6s clip, not just the first ~1.6s
|
||||
assert len(out) == len(list((tmp_path / "f").glob("frame_*.jpg")))
|
||||
assert [fr["index"] for fr in out] == [0, 1, 2, 3, 4]
|
||||
|
||||
|
||||
def test_scene_fallback_on_static_clip(static_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_scene_or_uniform(
|
||||
str(static_clip), tmp_path / "f", fps=2.0, target_frames=12, max_frames=100,
|
||||
)
|
||||
assert meta["engine"] == "uniform"
|
||||
assert meta["fallback"] is True
|
||||
@@ -0,0 +1,80 @@
|
||||
"""setup.py --json surfaces the resolved watch detail."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SETUP = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts" / "setup.py"
|
||||
|
||||
|
||||
def _run(args, *, home=None, extra_env=None):
|
||||
env = dict(os.environ)
|
||||
env.pop("WATCH_DETAIL", None)
|
||||
# Don't let a real key in the developer's shell env leak into the test.
|
||||
env.pop("GROQ_API_KEY", None)
|
||||
env.pop("OPENAI_API_KEY", None)
|
||||
env.pop("SETUP_COMPLETE", None)
|
||||
if home is not None:
|
||||
env["HOME"] = str(home)
|
||||
env["USERPROFILE"] = str(home) # Windows
|
||||
if extra_env:
|
||||
env.update(extra_env)
|
||||
return subprocess.run(
|
||||
[sys.executable, str(SETUP), *args],
|
||||
capture_output=True, text=True, env=env,
|
||||
)
|
||||
|
||||
|
||||
def _write_env(home: Path, body: str) -> None:
|
||||
cfg = home / ".config" / "watch"
|
||||
cfg.mkdir(parents=True, exist_ok=True)
|
||||
f = cfg / ".env"
|
||||
f.write_text(body, encoding="utf-8")
|
||||
f.chmod(0o600)
|
||||
|
||||
|
||||
def test_json_reports_watch_detail():
|
||||
proc = _run(["--json"])
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
data = json.loads(proc.stdout)
|
||||
assert data["watch_detail"] == "balanced"
|
||||
|
||||
|
||||
def test_keyless_completed_setup_proceeds_silently(tmp_path):
|
||||
"""A user who finished setup without a key must NOT be nagged forever."""
|
||||
_write_env(tmp_path, "GROQ_API_KEY=\nOPENAI_API_KEY=\nSETUP_COMPLETE=true\n")
|
||||
chk = _run(["--check"], home=tmp_path)
|
||||
assert chk.returncode == 0, f"keyless-complete should pass --check; got {chk.returncode}: {chk.stderr}"
|
||||
assert chk.stdout == "" and chk.stderr == ""
|
||||
|
||||
js = json.loads(_run(["--json"], home=tmp_path).stdout)
|
||||
assert js["can_proceed"] is True
|
||||
assert js["first_run"] is False
|
||||
assert js["setup_complete"] is True
|
||||
# status still encourages a key even though we can proceed
|
||||
assert js["status"] == "needs_key"
|
||||
|
||||
|
||||
def test_keyless_first_run_is_encouraged(tmp_path):
|
||||
"""Genuine first run with no key: --check reports exit 3 (encourage a key)."""
|
||||
_write_env(tmp_path, "GROQ_API_KEY=\nOPENAI_API_KEY=\n")
|
||||
chk = _run(["--check"], home=tmp_path)
|
||||
assert chk.returncode == 3, chk.stderr
|
||||
|
||||
js = json.loads(_run(["--json"], home=tmp_path).stdout)
|
||||
assert js["can_proceed"] is False
|
||||
assert js["first_run"] is True
|
||||
|
||||
|
||||
def test_key_present_is_ready(tmp_path):
|
||||
_write_env(tmp_path, "GROQ_API_KEY=sk-test-abc\n")
|
||||
chk = _run(["--check"], home=tmp_path)
|
||||
assert chk.returncode == 0, chk.stderr
|
||||
|
||||
js = json.loads(_run(["--json"], home=tmp_path).stdout)
|
||||
assert js["status"] == "ready"
|
||||
assert js["can_proceed"] is True
|
||||
assert js["whisper_backend"] == "groq"
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Transcript-cue timestamps: parsing, point extraction, and pinned merge."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import frames
|
||||
|
||||
|
||||
def test_parse_timestamps_mixed_formats():
|
||||
assert frames.parse_timestamps("30,1:05,90") == [30.0, 65.0, 90.0]
|
||||
|
||||
|
||||
def test_parse_timestamps_strips_and_dedupes():
|
||||
assert frames.parse_timestamps(" 90 , 30, 30 ") == [30.0, 90.0]
|
||||
|
||||
|
||||
def test_parse_timestamps_empty():
|
||||
assert frames.parse_timestamps("") == []
|
||||
assert frames.parse_timestamps(" , ") == []
|
||||
|
||||
|
||||
def test_parse_timestamps_rejects_garbage():
|
||||
with pytest.raises(SystemExit):
|
||||
frames.parse_timestamps("4:bad")
|
||||
|
||||
|
||||
def test_merge_frames_sorts_and_reindexes():
|
||||
primary = [
|
||||
{"index": 0, "timestamp_seconds": 1.0, "path": "a", "reason": "scene-change"},
|
||||
{"index": 1, "timestamp_seconds": 5.0, "path": "b", "reason": "scene-change"},
|
||||
]
|
||||
pinned = [
|
||||
{"index": 0, "timestamp_seconds": 3.0, "path": "c", "reason": "transcript-cue"},
|
||||
]
|
||||
merged = frames.merge_frames(primary, pinned)
|
||||
assert [f["path"] for f in merged] == ["a", "c", "b"]
|
||||
assert [f["index"] for f in merged] == [0, 1, 2]
|
||||
assert merged[1]["reason"] == "transcript-cue"
|
||||
|
||||
|
||||
def test_merge_frames_keeps_all_pinned():
|
||||
pinned = [{"index": 0, "timestamp_seconds": 2.0, "path": "c", "reason": "transcript-cue"}]
|
||||
merged = frames.merge_frames([], pinned)
|
||||
assert [f["path"] for f in merged] == ["c"]
|
||||
|
||||
|
||||
def test_extract_at_timestamps_one_frame_per_point(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_at_timestamps(str(cut_clip), tmp_path / "f", [0.5, 2.0, 4.0])
|
||||
assert meta["engine"] == "timestamps"
|
||||
assert meta["fallback"] is False
|
||||
assert len(out) == 3
|
||||
assert all(f["reason"] == "transcript-cue" for f in out)
|
||||
ts = [f["timestamp_seconds"] for f in out]
|
||||
assert ts == sorted(ts)
|
||||
assert len(out) == len(list((tmp_path / "f").glob("cue_*.jpg")))
|
||||
|
||||
|
||||
def test_extract_at_timestamps_drops_out_of_window(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_at_timestamps(
|
||||
str(cut_clip), tmp_path / "f", [0.5, 2.0, 4.0],
|
||||
start_seconds=1.0, end_seconds=3.0,
|
||||
)
|
||||
assert [f["timestamp_seconds"] for f in out] == [2.0]
|
||||
assert meta["dropped_out_of_window"] == 2
|
||||
|
||||
|
||||
def test_extract_at_timestamps_caps_and_spans(cut_clip: Path, tmp_path: Path):
|
||||
out, meta = frames.extract_at_timestamps(
|
||||
str(cut_clip), tmp_path / "f", [0.5, 1.5, 2.5, 3.5, 4.5], max_frames=3,
|
||||
)
|
||||
assert len(out) == 3
|
||||
ts = [f["timestamp_seconds"] for f in out]
|
||||
assert ts[0] == 0.5 and ts[-1] == 4.5 # even-sample keeps first + last
|
||||
assert len(out) == len(list((tmp_path / "f").glob("cue_*.jpg")))
|
||||
|
||||
|
||||
def test_extract_at_timestamps_does_not_clobber_detail_frames(cut_clip: Path, tmp_path: Path):
|
||||
"""Cue frames live alongside detail frames in the same dir without deleting them."""
|
||||
d = tmp_path / "f"
|
||||
scene, _ = frames.extract_scene_or_uniform(
|
||||
str(cut_clip), d, fps=2.0, target_frames=50, max_frames=100,
|
||||
)
|
||||
cues, _ = frames.extract_at_timestamps(str(cut_clip), d, [1.0, 3.0])
|
||||
assert len(list(d.glob("frame_*.jpg"))) == len(scene)
|
||||
assert len(list(d.glob("cue_*.jpg"))) == len(cues)
|
||||
@@ -0,0 +1,69 @@
|
||||
"""End-to-end routing of --detail through watch.py on a local clip."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
WATCH = Path(__file__).resolve().parent.parent / "skills" / "watch" / "scripts" / "watch.py"
|
||||
|
||||
|
||||
def _run(clip: Path, *args: str, env_extra: dict | None = None) -> str:
|
||||
env = dict(os.environ)
|
||||
env.pop("WATCH_DETAIL", None)
|
||||
if env_extra:
|
||||
env.update(env_extra)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(WATCH), str(clip), "--no-whisper", *args],
|
||||
capture_output=True, text=True, env=env,
|
||||
)
|
||||
assert proc.returncode == 0, proc.stderr
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def test_efficient_uses_keyframe_engine(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "efficient")
|
||||
assert "(keyframe" in out
|
||||
assert "**Detail:** efficient" in out
|
||||
|
||||
|
||||
def test_balanced_uses_scene_engine(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "balanced")
|
||||
assert "(scene" in out
|
||||
assert "**Detail:** balanced" in out
|
||||
|
||||
|
||||
def test_token_burner_uses_scene_engine(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "token-burner")
|
||||
assert "(scene" in out
|
||||
|
||||
|
||||
def test_transcript_skips_frames(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "transcript")
|
||||
assert "skipped" in out
|
||||
assert "frame_0000.jpg" not in out
|
||||
|
||||
|
||||
def test_flag_overrides_env(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "efficient", env_extra={"WATCH_DETAIL": "balanced"})
|
||||
assert "(keyframe" in out
|
||||
|
||||
|
||||
def test_default_is_balanced(cut_clip: Path):
|
||||
out = _run(cut_clip) # no flag, WATCH_DETAIL cleared
|
||||
assert "**Detail:** balanced" in out
|
||||
assert "(scene" in out
|
||||
|
||||
|
||||
def test_timestamps_add_cue_frames_to_detail(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "balanced", "--timestamps", "1,3")
|
||||
assert "reason=transcript-cue" in out
|
||||
assert "reason=scene-change" in out # detail frames still present (additive)
|
||||
|
||||
|
||||
def test_timestamps_with_transcript_detail_is_cue_only(cut_clip: Path):
|
||||
out = _run(cut_clip, "--detail", "transcript", "--timestamps", "1,3")
|
||||
assert "reason=transcript-cue" in out
|
||||
assert "reason=scene-change" not in out
|
||||
assert "reason=keyframe" not in out
|
||||
Reference in New Issue
Block a user