commit 29b8a2988ac07c8ab32aa3583b358d1d77380261 Author: bradautomates Date: Fri Apr 24 14:40:34 2026 +1000 Initial commit: /watch skill v0.1.0 Give Claude the ability to watch any video — yt-dlp download, ffmpeg frame extraction with auto-scaled fps, native-caption transcript with Whisper (Groq/OpenAI) fallback. Co-Authored-By: Claude Opus 4.7 (1M context) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..e0476a6 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,21 @@ +{ + "name": "claude-video", + "metadata": { + "description": "Give Claude a video input. /watch downloads, extracts frames, transcribes, and hands it all to Claude." + }, + "owner": { + "name": "Bradley Bonanno" + }, + "plugins": [ + { + "name": "watch", + "description": "Watch a video (URL or local path). Downloads with yt-dlp, extracts frames with ffmpeg, transcribes via captions or Whisper.", + "author": { + "name": "Bradley Bonanno" + }, + "source": "./", + "category": "productivity", + "homepage": "https://github.com/bradautomates/claude-video" + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..af75ab6 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,12 @@ +{ + "name": "watch", + "version": "0.1.0", + "description": "Let Claude watch a video. Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls captions or falls back to Whisper, and hands frames + transcript to Claude so it can answer questions about the video.", + "author": { + "name": "Bradley Bonanno" + }, + "homepage": "https://github.com/bradautomates/claude-video", + "repository": "https://github.com/bradautomates/claude-video", + "license": "MIT", + "keywords": ["video", "watch", "youtube", "vimeo", "tiktok", "transcription", "whisper", "yt-dlp", "ffmpeg", "multimodal", "frames"] +} diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..d677e4b --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,3 @@ +{ + "name": "watch" +} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fe2ca2c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,28 @@ +# Exclude non-runtime files from `git archive` output. +# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file. + +# Anthropic canonical skill-packaging excludes +# (mirrors anthropics/skills/skills/skill-creator/scripts/package_skill.py) +__pycache__/ export-ignore +node_modules/ export-ignore +*.pyc export-ignore +.DS_Store export-ignore + +# Dev, docs, test, and media — not needed at skill runtime +tests/ export-ignore +docs/ export-ignore +fixtures/ export-ignore +assets/ export-ignore +examples/ export-ignore + +# NOTE: commands/, hooks/, and .claude-plugin/ are NOT export-ignored because +# Claude Code's /plugin install fetches this same git archive tarball — they +# must be in the archive for the plugin to install correctly. The claude.ai +# .skill bundle strips them afterward via `zip -d` in scripts/build-skill.sh. + +# CI workflows — repo-only, not needed at skill runtime +.github/ export-ignore + +# Build config itself +.gitignore export-ignore +.gitattributes export-ignore diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b5c26d5 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,31 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + build-and-release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Build .skill artifact + run: | + bash scripts/build-skill.sh + test -f dist/watch.skill + + - name: Create GitHub release + uses: softprops/action-gh-release@v2 + with: + files: dist/watch.skill + generate_release_notes: true + draft: false + prerelease: false diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea7614e --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# OS / editor / tooling +.DS_Store +.entire/ +__pycache__/ +*.pyc +.venv/ +.coverage +htmlcov/ +.idea/ +.vscode/ + +# build artifact from scripts/build-skill.sh +/dist/ + +# local Claude Code settings +.claude/settings.local.json diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..36228b6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog + +All notable changes to `/watch` are documented here. + +## [0.1.0] — 2026-04-24 + +Initial marketplace release. + +### Added +- `/watch [question]` slash command. +- yt-dlp download with native caption extraction (manual + auto-subs). +- ffmpeg frame extraction with auto-scaled fps (≤2 fps, ≤100 frames, duration-aware budget). +- `--start` / `--end` focused mode with denser frame budget and transcript range filtering. +- Whisper fallback (Groq preferred, OpenAI secondary) for videos without captions. +- `setup.py` preflight: silent `--check`, structured `--json`, and installer that auto-runs `brew install` on macOS. +- Session-start hook that prints a one-line status on first run / partial config. +- `.skill` bundle packaging for claude.ai upload via `scripts/build-skill.sh`. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e23e30f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Bradley Bonanno + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..659edb1 --- /dev/null +++ b/README.md @@ -0,0 +1,200 @@ +# /watch + +**Give Claude the ability to watch any video.** + +Claude Code: +``` +/plugin marketplace add bradautomates/claude-video +/plugin install watch@claude-video +``` + +claude.ai (web): [download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) and drop it into Settings → Capabilities → Skills. + +Codex / generic skills: +```bash +git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch +``` + +Zero config to start — `yt-dlp` and `ffmpeg` install on first run via `brew` on macOS (Linux/Windows print exact commands). Captions cover most public videos for free. Whisper API key is only needed when a video has no captions. + +--- + +Claude can read a webpage, run a script, browse a repo. What it can't do, out of the box, is *watch a video*. You paste a YouTube link and it has to either guess from the title or pull a transcript that's missing 90% of what's on screen. + +With Claude Video `/watch` you can paste a URL or a local path, ask a question, and Claude downloads the video, extracts frames at an auto-scaled rate, pulls a timestamped transcript (free captions when available, Whisper API as fallback), and `Read`s every frame as an image. By the time it answers, it has *seen* the video and *heard* the audio. + +``` +/watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark? +``` + +## Why this exists + +I built this because I'm constantly using video to keep up with content. If I see a YouTube video that's blowing up, I want to know how the creator structured the hook — what's on screen in the first 3 seconds, what they said, why it worked. That used to mean watching it myself with a notepad. Now I just paste the URL and ask. + +The other half is summarization. Most YouTube videos don't deserve 20 minutes of my attention. I hand the URL to Claude, it pulls the transcript, and tells me what actually happened. If the visual matters, frames come along too. If it's a podcast or a talking head, transcript is enough. + +Claude is great at reading and synthesizing — but until now, video was the one input I couldn't hand it. Pasting a YouTube link got you nothing useful. `/watch` closes that gap. + +## What people actually use it for + +**Analyze someone else's content.** `/watch https://youtu.be/ what hook did they open with?` Claude looks at the first frames, reads the opening transcript, breaks down the structure. Same for ad creative, competitor launches, podcast intros, anything where the *how* matters as much as the *what*. + +**Diagnose a bug from a video.** Someone sends you a screen recording of something broken. `/watch bug-repro.mov what's going wrong?` Claude watches the recording, finds the frame where the issue appears, describes what's on screen, often catches the cause without you ever opening the file. + +**Summarize a video.** `/watch https://youtu.be/ summarize this` does the obvious thing — pulls the structure, the key moments, what was actually said and shown. Faster than watching at 2x. + +## How it works + +1. **You paste a video and a question.** URL (anything yt-dlp supports — YouTube, Loom, TikTok, X, Instagram, plus a few hundred more) or a local path (`.mp4`, `.mov`, `.mkv`, `.webm`). +2. **`yt-dlp` downloads it.** For URLs, into a temp working directory. For local files, no download — just probed in place. +3. **`ffmpeg` extracts frames at an auto-scaled rate.** The frame budget is duration-aware: ≤30s gets ~30 frames, 30-60s gets ~40, 1-3min gets ~60, 3-10min gets ~80, longer gets 100 sparsely. Hard ceilings: 2 fps, 100 frames. JPEGs at 512px wide by default — bump with `--resolution 1024` if Claude needs to read on-screen text. +4. **The transcript comes from one of two places.** First try: `yt-dlp` pulls native captions (manual or auto-generated) from the source. Free, instant, accurate-ish. Fallback: extract a mono 16 kHz audio clip and ship it to Whisper — Groq's `whisper-large-v3` (preferred — cheaper and faster) or OpenAI's `whisper-1`. +5. **Frames + transcript are handed to Claude.** The script prints frame paths with `t=MM:SS` markers and the transcript with timestamps. Claude `Read`s each frame in parallel — JPEGs render directly as images in its context. +6. **Claude answers grounded in what's actually on screen and in the audio.** Not "based on the description" or "according to the title." It saw the frames. It heard the transcript. It answers the way someone who watched the video would. +7. **Cleanup.** The script prints a working directory at the end. If you're not asking follow-ups, Claude removes it. + +## Frame budget — why it matters + +Token cost is dominated by frames. Every frame is an image; image tokens add up fast. The script's auto-fps logic exists so you don't blow your context budget on a sparse scan of a 30-minute video that would have been better answered by a focused 30-second window. + +| Duration | Default frame budget | What you get | +|----------|---------------------|--------------| +| ≤30 s | ~30 frames | Dense — basically every key moment | +| 30 s - 1 min | ~40 frames | Still dense | +| 1 - 3 min | ~60 frames | Comfortable | +| 3 - 10 min | ~80 frames | Sparse but workable | +| > 10 min | 100 frames | "Sparse scan" warning — re-run focused | + +When the user names a moment ("around 2:30", "the last 30 seconds", "from 0:45 to 1:00"), pass `--start` / `--end`. Focused mode gets denser per-second budgets, capped at 2 fps. Far more useful than a sparse pass over the whole thing. + +## Install + +| Surface | Install | +|---------|---------| +| **Claude Code** | `/plugin marketplace add bradautomates/claude-video` then `/plugin install watch@claude-video` | +| **claude.ai** (web) | [Download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) → Settings → Capabilities → Skills → `+` | +| **Codex** | `git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch` | +| **Manual / dev** | `git clone https://github.com/bradautomates/claude-video.git ~/.claude/skills/watch` | + +### Claude Code + +``` +/plugin marketplace add bradautomates/claude-video +/plugin install watch@claude-video +``` + +Update later with `/plugin update watch@claude-video`. + +### claude.ai (web) + +1. [Download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) from the latest release. +2. Go to Settings → Capabilities → Skills. +3. Click `+` and drop the file in. + +Enable "Code execution and file creation" under Capabilities first — the skill shells out to `ffmpeg` and `yt-dlp`, so it won't run without it. + +### Codex + +```bash +git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch +``` + +### Manual (developer) + +```bash +git clone https://github.com/bradautomates/claude-video.git ~/.claude/skills/watch +``` + +## First run + +On the first `/watch` call, the skill runs `scripts/setup.py --check`. If `ffmpeg` / `yt-dlp` aren't on your PATH, or no Whisper API key is set, it walks you through fixing it: + +- **macOS** — auto-runs `brew install ffmpeg yt-dlp`. +- **Linux** — prints the exact `apt` / `dnf` / `pipx` commands. +- **Windows** — prints the `winget` / `pip` commands. +- **API key** — scaffolds `~/.config/watch/.env` (mode `0600`) with commented placeholders for `GROQ_API_KEY` (preferred) and `OPENAI_API_KEY`. + +After setup, preflight is silent and `/watch` just works. The check is a sub-100ms lookup, so it doesn't slow you down on subsequent runs. + +## Bring your own keys + +Captions cover the majority of public videos for free. The Whisper fallback only kicks in when a video genuinely has no caption track — typically local files, TikToks, some Vimeos, and the occasional caption-less YouTube upload. + +| Capability | What you need | Cost | +|------------|---------------|------| +| Download + native captions | `yt-dlp` + `ffmpeg` | Free | +| Whisper fallback (preferred) | [Groq API key](https://console.groq.com/keys) — `whisper-large-v3` | Cheap, fast | +| Whisper fallback (alt) | [OpenAI API key](https://platform.openai.com/api-keys) — `whisper-1` | Standard pricing | +| Disable Whisper entirely | `--no-whisper` | Free, frames-only when no captions | + +## Usage + +``` +/watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark? +/watch https://www.tiktok.com/@user/video/123 summarize this +/watch ~/Movies/screen-recording.mp4 when does the UI break? +/watch https://vimeo.com/123 what tools does she mention? +``` + +Focused on a specific section — denser frame budget, lower token cost: +``` +/watch https://youtu.be/abc --start 2:15 --end 2:45 +/watch video.mp4 --start 50 --end 60 +/watch "$URL" --start 1:12:00 # from 1h12m to end +``` + +Other knobs (passed to `scripts/watch.py`): + +- `--max-frames N` — lower the frame cap for a tighter token budget. +- `--resolution W` — bump frame width to 1024 px when Claude needs to read on-screen text (slides, terminals, code). +- `--fps F` — override the auto-fps calculation (still capped at 2 fps). +- `--whisper groq|openai` — force a specific Whisper backend. +- `--no-whisper` — disable transcription entirely; frames only. +- `--out-dir DIR` — keep working files somewhere specific (default: auto-generated tmp dir). + +## Limits + +- **Best accuracy: under 10 minutes.** Past that the script prints a "sparse scan" warning — re-run focused on the part you actually care about with `--start`/`--end`. +- **Hard caps: 2 fps, 100 frames.** Frame count drives token cost; the script enforces this even when the auto-fps math would imply higher. +- **Whisper upload limit: 25 MB.** At mono 16 kHz that's about 50 minutes of audio. Longer videos need either captions or `--start`/`--end` to a smaller window. +- **No private platforms.** This skill doesn't log into anything. Public URLs and local files only. If yt-dlp can't reach it without auth, neither can `/watch`. + +## Structure + +``` +. +├── SKILL.md # skill contract — loaded by all three surfaces +├── scripts/ +│ ├── watch.py # entry point — orchestrates download → frames → transcript +│ ├── download.py # yt-dlp wrapper +│ ├── frames.py # ffmpeg frame extraction + auto-fps logic +│ ├── transcribe.py # VTT parsing + dedupe + Whisper orchestration +│ ├── whisper.py # Groq / OpenAI clients (pure stdlib) +│ ├── setup.py # preflight + installer +│ └── build-skill.sh # build dist/watch.skill for claude.ai upload +├── hooks/ # SessionStart status hook (Claude Code only) +├── .claude-plugin/ # plugin.json + marketplace.json (Claude Code) +├── .codex-plugin/ # codex packaging +└── .github/workflows/ # release.yml — auto-builds watch.skill on tag push +``` + +## Develop + +```bash +# Build the claude.ai upload bundle: +bash scripts/build-skill.sh # → dist/watch.skill +``` + +Releasing: tag `vX.Y.Z`, push the tag. The workflow builds `dist/watch.skill` and attaches it to the GitHub release. + +See [CHANGELOG.md](CHANGELOG.md) for version history. + +## Open source + +MIT license. + +Built on `yt-dlp`, `ffmpeg`, and Claude's multimodal `Read` tool. Whisper transcription via [Groq](https://groq.com) or [OpenAI](https://openai.com). + +--- + +[github.com/bradautomates/claude-video](https://github.com/bradautomates/claude-video) · [LICENSE](LICENSE) diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 0000000..34acefd --- /dev/null +++ b/SKILL.md @@ -0,0 +1,171 @@ +--- +name: watch +description: Watch a video (URL or local path). Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls the transcript from captions (or Whisper API fallback), and hands the result to Claude so it can answer questions about what's in the video. +argument-hint: " [question]" +allowed-tools: Bash, Read, AskUserQuestion +homepage: https://github.com/bradautomates/claude-video +repository: https://github.com/bradautomates/claude-video +author: bradautomates +license: MIT +user-invocable: true +--- + +# /watch — Claude watches a video + +You don't have a video input; this skill gives you one. A Python script downloads the video, extracts frames as JPEGs, gets a timestamped transcript (native captions first, then Whisper API as fallback), and prints frame paths. You then `Read` each frame path to see the images and combine them with the transcript to answer the user. + +## Step 0 — Setup preflight (runs every `/watch` invocation, silent on success) + +Before every `/watch` run, verify that dependencies and an API key are in place: + +```bash +python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" --check +``` + +This is a <100ms lookup. On exit 0, the script emits **nothing** — proceed to Step 1 without comment. **Do NOT announce "setup is complete" to the user** — they don't need a status message on every turn. The only acceptable user-visible output from Step 0 is when remediation is required. + +On non-zero exit, follow the table: + +| Exit | Meaning | Action | +|------|---------|--------| +| `2` | Missing binaries (`ffmpeg` / `ffprobe` / `yt-dlp`) | Run installer | +| `3` | No Whisper API key | Run installer to scaffold `.env`, then ask user for a key | +| `4` | Both missing | Run installer, then ask for a key | + +The installer is idempotent — safe to re-run: + +```bash +python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" +``` + +On macOS with Homebrew, it auto-installs `ffmpeg` and `yt-dlp`. On Linux/Windows, it prints the exact install commands for the user to run. It scaffolds `~/.config/watch/.env` with commented placeholders at `0600` perms, and writes `SETUP_COMPLETE=true` once deps + a key are in place so the next session knows this user has already been through the wizard. + +**If an API key is still missing after install:** use `AskUserQuestion` to ask the user whether they have a Groq API key (preferred — cheaper, faster) or an OpenAI key. Then write it into `~/.config/watch/.env` — set the matching `GROQ_API_KEY=...` or `OPENAI_API_KEY=...` line. If they don't want to set up Whisper, proceed with `--no-whisper` and tell them videos without native captions will come back frames-only. + +**Structured mode (optional):** `python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" --json` emits `{status, first_run, missing_binaries, whisper_backend, has_api_key, config_file, platform}` where `status` is one of `ready | needs_install | needs_key | needs_install_and_key`. Use this when you need to branch on specifics (e.g. "is this the user's very first run?" → `first_run: true`). + +Within a single session, you can skip Step 0 on follow-up `/watch` calls — once `--check` returned 0, nothing about the environment changes between turns. + +## When to use + +- User pastes a video URL (YouTube, Vimeo, X, TikTok, Twitch clip, most yt-dlp-supported sites) and asks about it. +- User points at a local video file (`.mp4`, `.mov`, `.mkv`, `.webm`, etc.) and asks about it. +- User types `/watch [question]`. + +## Recommended limits + +- **Best accuracy: videos under 10 minutes.** Frame coverage scales inversely with duration. +- **Hard caps: 100 frames total and 2 fps.** Token cost grows with frame count, so the script targets a frame budget by duration (and never exceeds 2 fps even when the budget would imply more): + - ≤30s → ~1-2 fps (up to 30 frames) + - 30s-1min → ~40 frames + - 1-3min → ~60 frames + - 3-10min → ~80 frames + - \>10min → 100 frames, sparsely spaced (warning printed) +- If the user hands you a long video, consider asking whether they want a specific section before burning tokens on a sparse scan. + +## How to invoke + +**Step 1 — parse the user input.** Separate the video source (URL or path) from any question the user asked. Example: `/watch https://youtu.be/abc what language is this in?` → source = `https://youtu.be/abc`, question = `what language is this in?`. + +**Step 2 — run the watch script.** Pass the source verbatim. Do not shell-escape it yourself beyond normal quoting: + +```bash +python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "" +``` + +Optional flags: +- `--start T` / `--end T` — focus on a section. Accepts `SS`, `MM:SS`, or `HH:MM:SS`. When either is set, fps auto-scales denser (see "Focusing on a section" below). +- `--max-frames N` — lower the cap for tighter token budget (e.g. `--max-frames 40`) +- `--resolution W` — change frame width in px (default 512; bump to 1024 only if the user needs to read on-screen text) +- `--fps F` — override auto-fps (clamped to 2 fps max) +- `--out-dir DIR` — keep working files somewhere specific (default: an auto-generated tmp dir) +- `--whisper groq|openai` — force a specific Whisper backend (default: prefer Groq if both keys exist) +- `--no-whisper` — disable the Whisper fallback entirely (frames-only if no captions) + +### Focusing on a section (higher frame rate) + +When the user asks about a specific moment — "what happens at the 2 minute mark?", "zoom into 0:45 to 1:00", "the first 10 seconds" — pass `--start` and/or `--end`. The script switches to focused-mode budgets, which are denser than full-video budgets (still capped at 2 fps): + +- ≤5s → 2 fps (up to 10 frames) +- 5-15s → 2 fps (up to 30 frames) +- 15-30s → ~2 fps (up to 60 frames) +- 30-60s → ~1.3 fps (up to 80 frames) +- 60-180s → ~0.6 fps (100 frames, capped) + +Focused mode is the right call for: +- Any moment/range the user names explicitly ("around 2:30", "the intro", "the last 30 seconds"). +- Any video longer than ~10 minutes where the user's question is about a specific part — running focused on the relevant section is far more useful than a sparse scan of the whole thing. +- Re-runs after a full scan didn't have enough detail in some region. + +Transcript is auto-filtered to the same range. Frame timestamps are absolute (real video timeline, not offset-from-start). + +Examples: +```bash +# Last 10 seconds of a 1 minute video +python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" video.mp4 --start 50 --end 60 + +# Zoom into 2:15 → 2:45 at 3 fps (90 frames) +python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "$URL" --start 2:15 --end 2:45 --fps 3 + +# From 1h12m to the end of the video +python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "$URL" --start 1:12:00 +``` + +**Step 3 — Read every frame path the script lists.** The Read tool renders JPEGs directly as images for you. Read all frames in a single message (parallel tool calls) so you see them together. The frames are in chronological order with a `t=MM:SS` timestamp so you can align them to the transcript. + +**Step 4 — answer the user.** You now have two streams of evidence: +- **Frames** — what's on screen at each timestamp +- **Transcript** — what's said at each timestamp. The report's header shows the source (`captions` = yt-dlp pulled native subs; `whisper (groq)` or `whisper (openai)` = transcribed by API). + +If the user asked a specific question, answer it directly citing timestamps. If they didn't ask anything, summarize what happens in the video — structure, key moments, notable visuals, spoken content. + +**Step 5 — clean up.** The script prints a working directory at the end. If the user isn't going to ask follow-ups about this video, delete it with `rm -rf `. If they might, leave it in place. + +## Transcription + +The script gets a timestamped transcript in one of two ways: + +1. **Native captions (free, preferred).** yt-dlp pulls manual or auto-generated subtitles from the source platform if available. +2. **Whisper API fallback.** If no captions came back (or the source is a local file), the script extracts audio (`ffmpeg -vn -ac 1 -ar 16000 -b:a 64k`, ~0.5 MB/min) and uploads it to whichever Whisper API has a key configured: + - **Groq** — `whisper-large-v3`. Preferred default: cheaper, faster. Get a key at console.groq.com/keys. + - **OpenAI** — `whisper-1`. Fallback. Get a key at platform.openai.com/api-keys. + +Both keys live in `~/.config/watch/.env`. The script prefers Groq when both are set; override with `--whisper openai` to force OpenAI. Use `--no-whisper` to skip the fallback entirely. + +## Failure modes and handling + +- **Setup preflight failed** → run `python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py"` (auto-installs ffmpeg/yt-dlp via brew on macOS, scaffolds the `.env`). For API key, ask the user via `AskUserQuestion` and write it to `~/.config/watch/.env`. +- **No transcript available** → captions missing AND (no Whisper key OR Whisper API failed). Script prints a hint pointing to setup. Proceed frames-only and tell the user. +- **Long video warning printed** → acknowledge it in your answer. Offer to re-run focused on a specific section via `--start`/`--end` rather than a sparse full-video scan. +- **Download fails** → yt-dlp's error goes to stderr. If it's a login-required or region-locked video, tell the user plainly; do not keep retrying. +- **Whisper request fails** → the error is printed to stderr (likely: invalid key, rate limit, or 25 MB upload limit on a very long video). The report will say "none available" for transcript. You can retry with `--whisper openai` if Groq failed (or vice versa). + +## Token efficiency + +This skill burns tokens primarily on frames. Order of magnitude: +- 80 frames at 512px wide is roughly 50-80k image tokens depending on aspect ratio. +- The transcript is cheap (a few thousand tokens at most for a 10-minute video). +- Bumping `--resolution` to 1024 roughly quadruples the image tokens per frame. Only do it when necessary. + +If you already watched a video this session and the user asks a follow-up, do **not** re-run the script — you already have the frames and transcript in context. Just answer from what you have. + +## Security & Permissions + +**What this skill does:** +- Runs `yt-dlp` locally to download the video and pull native captions when the source supports them (public data; the request goes directly to whatever host the URL points at) +- Runs `ffmpeg` / `ffprobe` locally to extract frames as JPEGs and, when Whisper is needed, a mono 16 kHz audio clip +- Sends the extracted audio clip to Groq's Whisper API (`api.groq.com/openai/v1/audio/transcriptions`) when `GROQ_API_KEY` is set (preferred — cheaper, faster) +- Sends the extracted audio clip to OpenAI's audio transcription API (`api.openai.com/v1/audio/transcriptions`) when `OPENAI_API_KEY` is set and Groq is not, or when `--whisper openai` is forced +- Writes the downloaded video, frames, audio, and an intermediate transcript to a working directory under the system temp dir (or `--out-dir` if specified) so Claude can `Read` them +- Reads / creates `~/.config/watch/.env` (mode `0600`) to store the Whisper API key(s) and a `SETUP_COMPLETE` marker. As a fallback, also reads `.env` in the current working directory + +**What this skill does NOT do:** +- Does not upload the video itself to any API — only the extracted audio goes out, and only when native captions are missing AND Whisper is not disabled with `--no-whisper` +- Does not access any platform account (no login, no session cookies, no posting) +- Does not share API keys between providers (Groq key only goes to `api.groq.com`, OpenAI key only goes to `api.openai.com`) +- Does not log, cache, or write API keys to stdout, stderr, or output files +- Does not persist anything outside the working directory and `~/.config/watch/.env` — clean up the working directory when you're done (Step 5) + +**Bundled scripts:** `scripts/watch.py` (entry point), `scripts/download.py` (yt-dlp wrapper), `scripts/frames.py` (ffmpeg frame extraction), `scripts/transcribe.py` (caption selection + Whisper orchestration), `scripts/whisper.py` (Groq / OpenAI clients), `scripts/setup.py` (preflight + installer) + +Review scripts before first use to verify behavior. diff --git a/hooks/hooks.json b/hooks/hooks.json new file mode 100644 index 0000000..07df686 --- /dev/null +++ b/hooks/hooks.json @@ -0,0 +1,16 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/check-setup.sh", + "timeout": 5 + } + ] + } + ] + } +} diff --git a/hooks/scripts/check-setup.sh b/hooks/scripts/check-setup.sh new file mode 100755 index 0000000..0363b7a --- /dev/null +++ b/hooks/scripts/check-setup.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# SessionStart hook for /watch — one-line status so users know what's wired up. +# Silent on ready state to avoid spam. Points at the installer when something +# is missing. +set -euo pipefail + +CONFIG_FILE="$HOME/.config/watch/.env" + +# Warn if the secrets file has loose permissions. +if [[ -f "$CONFIG_FILE" ]]; then + perms=$(stat -c '%a' "$CONFIG_FILE" 2>/dev/null || stat -f '%Lp' "$CONFIG_FILE" 2>/dev/null || echo "") + if [[ -n "$perms" && "$perms" != "600" && "$perms" != "400" ]]; then + echo "/watch: WARNING — $CONFIG_FILE has permissions $perms (should be 600)." + echo " Fix: chmod 600 $CONFIG_FILE" + fi +fi + +# Load API keys from the config file without exporting them. +read_key() { + local name="$1" + if [[ -n "${!name:-}" ]]; then + echo "${!name}" + return + fi + if [[ -f "$CONFIG_FILE" ]]; then + awk -F= -v k="$name" ' + /^[[:space:]]*#/ { next } + $1 == k { + sub(/^[[:space:]]*/, "", $2); sub(/[[:space:]]*$/, "", $2); + gsub(/^["'\'']|["'\'']$/, "", $2); + print $2; exit + } + ' "$CONFIG_FILE" + fi +} + +HAS_FFMPEG="" +HAS_YTDLP="" +command -v ffmpeg >/dev/null 2>&1 && HAS_FFMPEG="yes" +command -v yt-dlp >/dev/null 2>&1 && HAS_YTDLP="yes" + +HAS_GROQ="$(read_key GROQ_API_KEY)" +HAS_OPENAI="$(read_key OPENAI_API_KEY)" +SETUP_COMPLETE="$(read_key SETUP_COMPLETE)" + +# Fully configured → silent (Claude can surface status on demand via --check). +if [[ "$SETUP_COMPLETE" == "true" && -n "$HAS_FFMPEG" && -n "$HAS_YTDLP" ]]; then + exit 0 +fi + +# First-run / partially-configured → one-line hint. +if [[ -z "$HAS_FFMPEG" || -z "$HAS_YTDLP" ]]; then + echo "/watch: needs ffmpeg + yt-dlp. Run \`python3 \$CLAUDE_PLUGIN_ROOT/scripts/setup.py\` once to install and scaffold config." +elif [[ -z "$HAS_GROQ" && -z "$HAS_OPENAI" ]]; then + echo "/watch: ready for videos with native captions. Add GROQ_API_KEY (preferred) or OPENAI_API_KEY to ~/.config/watch/.env to unlock Whisper fallback." +else + echo "/watch: ready." +fi diff --git a/scripts/build-skill.sh b/scripts/build-skill.sh new file mode 100755 index 0000000..7a6582f --- /dev/null +++ b/scripts/build-skill.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# build-skill.sh — package this repo as a claude.ai-upload-ready .skill file. +# Usage: bash scripts/build-skill.sh (run from repo root) +# +# Produces dist/watch.skill, a zip with a single top-level `watch/` directory +# containing SKILL.md and the scripts/ runtime. claude.ai's skill upload has a +# 200-file cap; `export-ignore` in .gitattributes + the zip -d strips below +# keep the bundle lean. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REPO_ROOT" + +if ! git diff --quiet || ! git diff --cached --quiet; then + echo "error: working tree is dirty; commit or stash before building" >&2 + exit 1 +fi + +mkdir -p dist +OUT="dist/watch.skill" +git archive --format=zip --prefix=watch/ --output="$OUT" HEAD + +# claude.ai's .skill bundle needs only SKILL.md + scripts/ runtime. Claude Code +# needs hooks/ and .claude-plugin/ in the git archive (that's why they are NOT +# in .gitattributes export-ignore), but the .skill bundle should strip them to +# keep a single canonical SKILL.md and stay well under the 200-file cap. +zip -d "$OUT" \ + "watch/hooks/*" \ + "watch/.claude-plugin/*" \ + > /dev/null 2>&1 || true + +COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}') +SIZE=$(du -h "$OUT" | cut -f1) + +if [ "$COUNT" -gt 200 ]; then + echo "error: $COUNT files in zip, claude.ai's cap is 200" >&2 + echo " check .gitattributes export-ignore entries and this script's zip -d excludes" >&2 + exit 1 +fi + +SKILL_MD_COUNT=$(unzip -l "$OUT" | grep -c "SKILL.md" || true) +if [ "$SKILL_MD_COUNT" -ne 1 ]; then + echo "error: expected exactly one SKILL.md, found $SKILL_MD_COUNT" >&2 + exit 1 +fi + +echo "built $OUT ($COUNT files, $SIZE)" +echo "upload via the claude.ai skill UI" diff --git a/scripts/download.py b/scripts/download.py new file mode 100755 index 0000000..4d3429a --- /dev/null +++ b/scripts/download.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Download a video via yt-dlp, or resolve a local file path. + +Also fetches subtitles (manual first, then auto-generated) in VTT format so +transcribe.py can parse them without needing Whisper. +""" +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path +from urllib.parse import urlparse + + +VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".mov", ".m4v", ".avi", ".flv", ".wmv"} + + +def is_url(source: str) -> bool: + parsed = urlparse(source) + return parsed.scheme in ("http", "https") + + +def resolve_local(path: str) -> dict: + p = Path(path).expanduser().resolve() + if not p.exists(): + raise SystemExit(f"File not found: {p}") + if p.suffix.lower() not in VIDEO_EXTS: + print( + f"[watch] warning: {p.suffix} is not a known video extension, proceeding anyway", + file=sys.stderr, + ) + return { + "video_path": str(p), + "subtitle_path": None, + "info": {"title": p.name, "url": str(p)}, + "downloaded": False, + } + + +def _pick_subtitle(out_dir: Path) -> Path | None: + candidates = sorted(out_dir.glob("video*.vtt")) + if not candidates: + return None + preferred = [c for c in candidates if ".en" in c.name] + return preferred[0] if preferred else candidates[0] + + +def _pick_video(out_dir: Path) -> Path | None: + for ext in (".mp4", ".mkv", ".webm", ".mov"): + for candidate in out_dir.glob(f"video*{ext}"): + return candidate + for candidate in out_dir.glob("video.*"): + if candidate.suffix.lower() in VIDEO_EXTS: + return candidate + return None + + +def download_url(url: str, out_dir: Path) -> dict: + if shutil.which("yt-dlp") is None: + raise SystemExit("yt-dlp is not installed. Install with: brew install yt-dlp") + + out_dir.mkdir(parents=True, exist_ok=True) + output_template = str(out_dir / "video.%(ext)s") + + cmd = [ + "yt-dlp", + "-N", "8", + "-f", "bv*[height<=720]+ba/b[height<=720]/bv+ba/b", + "--merge-output-format", "mp4", + "--write-info-json", + "--write-subs", + "--write-auto-subs", + "--sub-langs", "en,en-US,en-GB,en-orig", + "--sub-format", "vtt", + "--convert-subs", "vtt", + "--no-playlist", + "--ignore-errors", + "-o", output_template, + url, + ] + + # yt-dlp may exit non-zero if a subtitle variant fails (e.g. 429) even when + # the video itself downloaded fine. Treat "video file present" as success. + result = subprocess.run(cmd, stdout=sys.stderr, stderr=sys.stderr) + video = _pick_video(out_dir) + if video is None: + raise SystemExit( + f"yt-dlp did not produce a video file in {out_dir} (exit {result.returncode})" + ) + + subtitle = _pick_subtitle(out_dir) + info_path = out_dir / "video.info.json" + info: dict = {} + if info_path.exists(): + try: + raw = json.loads(info_path.read_text()) + info = { + "title": raw.get("title"), + "uploader": raw.get("uploader") or raw.get("channel"), + "duration": raw.get("duration"), + "url": raw.get("webpage_url") or url, + } + except Exception: + info = {"url": url} + + return { + "video_path": str(video), + "subtitle_path": str(subtitle) if subtitle else None, + "info": info or {"url": url}, + "downloaded": True, + } + + +def download(source: str, out_dir: Path) -> dict: + if is_url(source): + return download_url(source, out_dir) + return resolve_local(source) + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("usage: download.py ", file=sys.stderr) + raise SystemExit(2) + result = download(sys.argv[1], Path(sys.argv[2])) + print(json.dumps(result, indent=2)) diff --git a/scripts/frames.py b/scripts/frames.py new file mode 100755 index 0000000..04e7933 --- /dev/null +++ b/scripts/frames.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Probe video metadata and extract frames at an auto-scaled fps. + +Auto-fps targets a frame budget, not a fixed rate. Token cost scales with frame +count, so budget-by-duration keeps short videos dense and long videos capped. +When a user-specified range is passed, focused-mode budgets denser (they are +zooming in for detail). +""" +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +MAX_FPS = 2.0 + + +def _clamp_fps(fps: float, duration_seconds: float, max_frames: int) -> tuple[float, int]: + fps = min(fps, MAX_FPS) + target = min(max_frames, max(1, int(round(fps * duration_seconds)))) + return fps, target + + +def parse_time(value: str | float | int | None) -> float | None: + """Parse SS, MM:SS, or HH:MM:SS (with optional .ms) into seconds.""" + if value is None: + return None + if isinstance(value, (int, float)): + return float(value) + s = str(value).strip() + if not s: + return None + parts = s.split(":") + try: + if len(parts) == 1: + return float(parts[0]) + if len(parts) == 2: + return int(parts[0]) * 60 + float(parts[1]) + if len(parts) == 3: + return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2]) + except ValueError: + pass + raise SystemExit(f"Cannot parse time value: {value!r} (expected SS, MM:SS, or HH:MM:SS)") + + +def format_time(seconds: float) -> str: + total = int(round(seconds)) + hours, rem = divmod(total, 3600) + minutes, sec = divmod(rem, 60) + if hours: + return f"{hours}:{minutes:02d}:{sec:02d}" + return f"{minutes:02d}:{sec:02d}" + + +def get_metadata(video_path: str) -> dict: + if shutil.which("ffprobe") is None: + raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg") + + result = subprocess.run( + [ + "ffprobe", + "-v", "quiet", + "-print_format", "json", + "-show_format", + "-show_streams", + video_path, + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise SystemExit(f"ffprobe failed: {result.stderr.strip()}") + + data = json.loads(result.stdout or "{}") + streams = data.get("streams", []) + fmt = data.get("format", {}) + video_stream = next((s for s in streams if s.get("codec_type") == "video"), {}) + audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None) + + duration = float(fmt.get("duration") or video_stream.get("duration") or 0) + return { + "duration_seconds": duration, + "width": video_stream.get("width"), + "height": video_stream.get("height"), + "codec": video_stream.get("codec_name"), + "size_bytes": int(fmt.get("size") or 0), + "has_audio": audio_stream is not None, + } + + +def auto_fps(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]: + """Pick fps that targets a sensible frame budget for full-video scans.""" + if duration_seconds <= 0: + return 1.0, 1 + + if duration_seconds <= 30: + target = min(max_frames, max(12, int(round(duration_seconds)))) + elif duration_seconds <= 60: + target = min(max_frames, 40) + elif duration_seconds <= 180: # 3 min + target = min(max_frames, 60) + elif duration_seconds <= 600: # 10 min + target = min(max_frames, 80) + else: + target = max_frames + + return _clamp_fps(target / duration_seconds, duration_seconds, max_frames) + + +def auto_fps_focus(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]: + """Denser budget for user-specified ranges — they are zooming in for detail.""" + if duration_seconds <= 0: + return min(MAX_FPS, 2.0), 2 + + if duration_seconds <= 5: + target = min(max_frames, max(10, int(round(duration_seconds * 6)))) + elif duration_seconds <= 15: + target = min(max_frames, max(30, int(round(duration_seconds * 4)))) + elif duration_seconds <= 30: + target = min(max_frames, 60) + elif duration_seconds <= 60: + target = min(max_frames, 80) + elif duration_seconds <= 180: + target = max_frames + else: + target = max_frames + + return _clamp_fps(target / duration_seconds, duration_seconds, max_frames) + + +def extract( + video_path: str, + out_dir: Path, + fps: float, + resolution: int = 512, + max_frames: int = 100, + start_seconds: float | None = None, + end_seconds: float | None = None, +) -> list[dict]: + if shutil.which("ffmpeg") is None: + raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg") + + out_dir.mkdir(parents=True, exist_ok=True) + for existing in out_dir.glob("frame_*.jpg"): + existing.unlink() + + output_pattern = str(out_dir / "frame_%04d.jpg") + cmd: list[str] = [ + "ffmpeg", + "-hide_banner", + "-loglevel", "error", + "-y", + ] + + # -ss before -i = fast seek (keyframe-snap, good enough for preview frames). + if start_seconds is not None: + cmd += ["-ss", f"{start_seconds:.3f}"] + if end_seconds is not None: + cmd += ["-to", f"{end_seconds:.3f}"] + + cmd += [ + "-i", video_path, + "-vf", f"fps={fps},scale={resolution}:-2", + "-frames:v", str(max_frames), + "-q:v", "4", + output_pattern, + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise SystemExit(f"ffmpeg frame extraction failed: {result.stderr.strip()}") + + offset = start_seconds or 0.0 + frames = sorted(out_dir.glob("frame_*.jpg")) + return [ + { + "index": i, + "timestamp_seconds": round(offset + (i / fps if fps > 0 else 0.0), 2), + "path": str(p), + } + for i, p in enumerate(frames) + ] + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print( + "usage: frames.py [--fps F] [--resolution W] " + "[--max-frames N] [--start T] [--end T]", + file=sys.stderr, + ) + raise SystemExit(2) + + video = sys.argv[1] + out = Path(sys.argv[2]) + args = sys.argv[3:] + + fps_override = None + resolution = 512 + max_frames = 100 + start_arg = None + end_arg = None + i = 0 + while i < len(args): + if args[i] == "--fps": + fps_override = float(args[i + 1]); i += 2 + elif args[i] == "--resolution": + resolution = int(args[i + 1]); i += 2 + elif args[i] == "--max-frames": + max_frames = int(args[i + 1]); i += 2 + elif args[i] == "--start": + start_arg = args[i + 1]; i += 2 + elif args[i] == "--end": + end_arg = args[i + 1]; i += 2 + else: + i += 1 + + meta = get_metadata(video) + start_sec = parse_time(start_arg) + end_sec = parse_time(end_arg) + full_duration = meta["duration_seconds"] + + effective_start = start_sec if start_sec is not None else 0.0 + effective_end = end_sec if end_sec is not None else full_duration + effective_duration = max(0.0, effective_end - effective_start) + + focused = start_sec is not None or end_sec is not None + if focused: + fps, target = auto_fps_focus(effective_duration, max_frames=max_frames) + else: + fps, target = auto_fps(effective_duration, max_frames=max_frames) + if fps_override is not None: + fps = fps_override + target = max(1, int(round(fps * effective_duration))) + + frames = extract( + video, out, + fps=fps, + resolution=resolution, + max_frames=max_frames, + start_seconds=start_sec, + end_seconds=end_sec, + ) + print(json.dumps( + {"meta": meta, "fps": fps, "target": target, "focused": focused, "frames": frames}, + indent=2, + )) diff --git a/scripts/setup.py b/scripts/setup.py new file mode 100755 index 0000000..a82cc1f --- /dev/null +++ b/scripts/setup.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +"""Setup / preflight for /watch. + +Modes: + setup.py --check Silent preflight. Exit 0 if ready, 2/3/4 on failure. + setup.py --json Machine-readable status for Claude to parse. + setup.py Installer. Auto-installs deps, scaffolds .env, marks SETUP_COMPLETE. + +Design: +- Silent on success: --check exits 0 with no output when everything's ready so + that /watch doesn't spam "setup is complete" on every turn. +- Idempotent: re-running the installer is safe — it never clobbers existing + keys and only appends missing ones. +- SETUP_COMPLETE=true in ~/.config/watch/.env tells us the user has been + through a successful installer run at least once. +- Never sudo. On macOS, auto-install via brew. Elsewhere, print exact commands. +- Never write an API key to disk automatically — only scaffold placeholders. +""" +from __future__ import annotations + +import json +import os +import platform +import shutil +import subprocess +import sys +from pathlib import Path + + +REQUIRED_BINARIES = ["ffmpeg", "ffprobe", "yt-dlp"] +CONFIG_DIR = Path.home() / ".config" / "watch" +CONFIG_FILE = CONFIG_DIR / ".env" +ENV_TEMPLATE = """# /watch API configuration +# +# Whisper transcription fallback — used only when yt-dlp cannot get captions +# (or when you point /watch at a local file with no subtitles). +# +# Groq is preferred: it runs whisper-large-v3 at a fraction of OpenAI's price +# and is faster in practice. OpenAI is the compatible fallback. +# +# Get a Groq key: https://console.groq.com/keys +# Get an OpenAI key: https://platform.openai.com/api-keys +# +# Leave both blank to disable Whisper — /watch will still work, but videos +# without native captions will come back frames-only. + +GROQ_API_KEY= +OPENAI_API_KEY= +""" + + +def _which(name: str) -> str | None: + return shutil.which(name) + + +def _check_binaries() -> list[str]: + return [b for b in REQUIRED_BINARIES if not _which(b)] + + +def _check_file_permissions(path: Path) -> None: + """Warn to stderr if a secrets file is world/group readable.""" + try: + mode = path.stat().st_mode + if mode & 0o044: + sys.stderr.write( + f"[watch] WARNING: {path} is readable by other users. " + f"Run: chmod 600 {path}\n" + ) + sys.stderr.flush() + except OSError: + pass + + +def _read_env_key(name: str) -> str | None: + value = os.environ.get(name) + if value and value.strip(): + return value.strip() + if not CONFIG_FILE.exists(): + return None + _check_file_permissions(CONFIG_FILE) + try: + for line in CONFIG_FILE.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, raw = line.partition("=") + if key.strip() != name: + continue + raw = raw.strip() + if len(raw) >= 2 and raw[0] in ('"', "'") and raw[-1] == raw[0]: + raw = raw[1:-1] + return raw or None + except OSError: + return None + return None + + +def _have_api_key() -> tuple[bool, str | None]: + if _read_env_key("GROQ_API_KEY"): + return True, "groq" + if _read_env_key("OPENAI_API_KEY"): + return True, "openai" + return False, None + + +def is_first_run() -> bool: + """True if the installer hasn't completed successfully yet.""" + return _read_env_key("SETUP_COMPLETE") != "true" + + +def _scaffold_env() -> bool: + """Create ~/.config/watch/.env with placeholders if missing.""" + if CONFIG_FILE.exists(): + return False + CONFIG_DIR.mkdir(parents=True, exist_ok=True) + CONFIG_FILE.write_text(ENV_TEMPLATE) + try: + CONFIG_FILE.chmod(0o600) + except OSError: + pass + return True + + +def _write_setup_complete() -> None: + """Idempotently append SETUP_COMPLETE=true to .env. + + Used only after a fully successful install (deps + key). Future sessions + detect this marker to skip wizard-style UI and stay silent. + """ + CONFIG_DIR.mkdir(parents=True, exist_ok=True) + existing = "" + if CONFIG_FILE.exists(): + existing = CONFIG_FILE.read_text() + for line in existing.splitlines(): + if line.strip().startswith("SETUP_COMPLETE="): + return + if existing and not existing.endswith("\n"): + existing += "\n" + CONFIG_FILE.write_text(existing + "SETUP_COMPLETE=true\n") + else: + CONFIG_FILE.write_text(ENV_TEMPLATE + "\nSETUP_COMPLETE=true\n") + try: + CONFIG_FILE.chmod(0o600) + except OSError: + pass + + +def _brew_pkg(missing: list[str]) -> list[str]: + pkgs: list[str] = [] + for bin_name in missing: + if bin_name in ("ffmpeg", "ffprobe"): + if "ffmpeg" not in pkgs: + pkgs.append("ffmpeg") + elif bin_name == "yt-dlp": + if "yt-dlp" not in pkgs: + pkgs.append("yt-dlp") + else: + pkgs.append(bin_name) + return pkgs + + +def _install_macos(missing: list[str]) -> tuple[bool, str]: + if _which("brew") is None: + return False, ( + "Homebrew is not installed. Install it from https://brew.sh, then re-run setup. " + "Or install manually: `brew install " + " ".join(_brew_pkg(missing)) + "`" + ) + pkgs = _brew_pkg(missing) + if not pkgs: + return True, "nothing to install" + cmd = ["brew", "install", *pkgs] + print(f"[setup] running: {' '.join(cmd)}", file=sys.stderr) + result = subprocess.run(cmd) + if result.returncode != 0: + return False, f"brew install failed with exit code {result.returncode}" + return True, f"installed via brew: {', '.join(pkgs)}" + + +def _install_hint_linux(missing: list[str]) -> str: + pkgs = _brew_pkg(missing) + hints = [] + if "ffmpeg" in pkgs: + hints.append("apt: `sudo apt install ffmpeg` or dnf: `sudo dnf install ffmpeg`") + if "yt-dlp" in pkgs: + hints.append("`pipx install yt-dlp` (recommended) or `pip install --user yt-dlp`") + return "\n ".join(hints) if hints else "nothing to install" + + +def _status() -> dict: + """Structured preflight snapshot.""" + missing = _check_binaries() + has_key, backend = _have_api_key() + + if not missing and has_key: + status = "ready" + elif missing and not has_key: + status = "needs_install_and_key" + elif missing: + status = "needs_install" + else: + status = "needs_key" + + return { + "status": status, + "first_run": is_first_run(), + "missing_binaries": missing, + "whisper_backend": backend, + "has_api_key": has_key, + "config_file": str(CONFIG_FILE), + "platform": platform.system(), + } + + +def cmd_check() -> int: + """Silent-on-success preflight. + + Exit 0 with no output when ready. On failure, print one actionable line + to stderr and return: + 2 → binaries missing + 3 → API key missing + 4 → both missing + """ + s = _status() + if s["status"] == "ready": + return 0 + + parts = [] + if s["missing_binaries"]: + parts.append(f"missing binaries: {', '.join(s['missing_binaries'])}") + if not s["has_api_key"]: + parts.append("no Whisper API key (GROQ_API_KEY or OPENAI_API_KEY)") + installer = Path(__file__).resolve() + sys.stderr.write( + f"[watch] setup incomplete ({'; '.join(parts)}). " + f"Run: python3 {installer}\n" + ) + sys.stderr.flush() + + if s["missing_binaries"] and not s["has_api_key"]: + return 4 + if s["missing_binaries"]: + return 2 + return 3 + + +def cmd_json() -> int: + json.dump(_status(), sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + +def cmd_install() -> int: + missing = _check_binaries() + installed_deps = False + if missing: + system = platform.system() + if system == "Darwin": + ok, msg = _install_macos(missing) + print(f"[setup] {msg}", file=sys.stderr) + if not ok: + return 2 + still_missing = _check_binaries() + if still_missing: + print(f"[setup] still missing after install: {', '.join(still_missing)}", file=sys.stderr) + return 2 + installed_deps = True + elif system == "Linux": + print("[setup] dependencies missing on Linux — please install:", file=sys.stderr) + print(" " + _install_hint_linux(missing), file=sys.stderr) + return 2 + else: + print(f"[setup] unsupported platform ({system}) for auto-install. Install manually:", file=sys.stderr) + print(f" missing: {', '.join(missing)}", file=sys.stderr) + return 2 + + created = _scaffold_env() + if created: + print(f"[setup] created config: {CONFIG_FILE}") + else: + print(f"[setup] config exists: {CONFIG_FILE}") + + has_key, backend = _have_api_key() + if has_key: + _write_setup_complete() + print(f"[setup] ready. whisper backend: {backend}") + if installed_deps: + print("[setup] installed dependencies; /watch is fully set up.") + return 0 + + print("") + print("[setup] one step left: add a Whisper API key.") + print("") + print(f" Edit {CONFIG_FILE} and set either:") + print(" GROQ_API_KEY=... (preferred — cheaper, faster; get one at console.groq.com/keys)") + print(" OPENAI_API_KEY=... (fallback; get one at platform.openai.com/api-keys)") + print("") + print(" Without a key, /watch still works but videos without captions come back frames-only.") + return 3 + + +def main() -> int: + if len(sys.argv) > 1: + arg = sys.argv[1] + if arg == "--check": + return cmd_check() + if arg == "--json": + return cmd_json() + return cmd_install() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/transcribe.py b/scripts/transcribe.py new file mode 100755 index 0000000..7596cba --- /dev/null +++ b/scripts/transcribe.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Parse a WebVTT subtitle file into a clean, timestamped transcript. + +YouTube auto-subs emit rolling-duplicate cues (each line appears 2-3 times as it +scrolls). We dedupe consecutive identical cues and merge their time ranges. +""" +from __future__ import annotations + +import re +import sys +from pathlib import Path + + +TS_RE = re.compile( + r"(\d{2}):(\d{2}):(\d{2})[.,](\d{3})\s+-->\s+(\d{2}):(\d{2}):(\d{2})[.,](\d{3})" +) +TAG_RE = re.compile(r"<[^>]+>") + + +def _to_seconds(h: str, m: str, s: str, ms: str) -> float: + return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000.0 + + +def parse_vtt(path: str) -> list[dict]: + text = Path(path).read_text(encoding="utf-8", errors="ignore") + lines = text.splitlines() + + segments: list[dict] = [] + i = 0 + while i < len(lines): + match = TS_RE.match(lines[i]) + if not match: + i += 1 + continue + + start = _to_seconds(*match.groups()[:4]) + end = _to_seconds(*match.groups()[4:]) + i += 1 + + cue_lines: list[str] = [] + while i < len(lines) and lines[i].strip(): + cleaned = TAG_RE.sub("", lines[i]).strip() + if cleaned: + cue_lines.append(cleaned) + i += 1 + + cue_text = " ".join(cue_lines).strip() + if cue_text: + segments.append({"start": round(start, 2), "end": round(end, 2), "text": cue_text}) + i += 1 + + return _dedupe(segments) + + +def _dedupe(segments: list[dict]) -> list[dict]: + """Collapse rolling duplicates common in YouTube auto-subs.""" + out: list[dict] = [] + for seg in segments: + if out and seg["text"] == out[-1]["text"]: + out[-1]["end"] = seg["end"] + continue + if out and seg["text"].startswith(out[-1]["text"] + " "): + out[-1]["text"] = seg["text"] + out[-1]["end"] = seg["end"] + continue + out.append(seg) + return out + + +def filter_range( + segments: list[dict], + start_seconds: float | None, + end_seconds: float | None, +) -> list[dict]: + """Return segments whose time range overlaps [start, end].""" + if start_seconds is None and end_seconds is None: + return segments + lo = start_seconds if start_seconds is not None else float("-inf") + hi = end_seconds if end_seconds is not None else float("inf") + return [seg for seg in segments if seg["end"] >= lo and seg["start"] <= hi] + + +def format_transcript(segments: list[dict]) -> str: + lines = [] + for seg in segments: + start = int(seg["start"]) + stamp = f"[{start // 60:02d}:{start % 60:02d}]" + lines.append(f"{stamp} {seg['text']}") + return "\n".join(lines) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("usage: transcribe.py ", file=sys.stderr) + raise SystemExit(2) + print(format_transcript(parse_vtt(sys.argv[1]))) diff --git a/scripts/watch.py b/scripts/watch.py new file mode 100755 index 0000000..d802279 --- /dev/null +++ b/scripts/watch.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""/watch entry point: download video, extract frames, parse transcript. + +Prints a markdown report to stdout listing frame paths + transcript. Claude +then Reads each frame path to see the video. +""" +from __future__ import annotations + +import argparse +import sys +import tempfile +from pathlib import Path + + +SCRIPT_DIR = Path(__file__).parent.resolve() +sys.path.insert(0, str(SCRIPT_DIR)) + +from download import download, is_url # noqa: E402 +from frames import MAX_FPS, auto_fps, auto_fps_focus, extract, format_time, get_metadata, parse_time # noqa: E402 +from transcribe import filter_range, format_transcript, parse_vtt # noqa: E402 +from whisper import load_api_key, transcribe_video # noqa: E402 + + +def main() -> int: + ap = argparse.ArgumentParser( + prog="watch", + description="Download a video, extract auto-scaled frames, and surface the transcript.", + ) + ap.add_argument("source", help="Video URL or local file path") + ap.add_argument("--max-frames", type=int, default=80, help="Cap on frame count (default 80, hard max 100)") + ap.add_argument("--resolution", type=int, default=512, help="Frame width in pixels (default 512)") + ap.add_argument("--fps", type=float, default=None, help="Override auto-fps") + ap.add_argument("--start", type=str, default=None, help="Range start (SS, MM:SS, or HH:MM:SS)") + ap.add_argument("--end", type=str, default=None, help="Range end (SS, MM:SS, or HH:MM:SS)") + ap.add_argument("--out-dir", type=str, default=None, help="Working directory (default: tmp)") + ap.add_argument( + "--no-whisper", + action="store_true", + help="Disable Whisper fallback. Report frames-only if no captions available.", + ) + ap.add_argument( + "--whisper", + choices=["groq", "openai"], + default=None, + help="Force a specific Whisper backend. Default: prefer Groq, fall back to OpenAI.", + ) + args = ap.parse_args() + + max_frames = min(args.max_frames, 100) + + if args.out_dir: + work = Path(args.out_dir).expanduser().resolve() + else: + work = Path(tempfile.mkdtemp(prefix="watch-")) + work.mkdir(parents=True, exist_ok=True) + print(f"[watch] working dir: {work}", file=sys.stderr) + + print( + "[watch] downloading via yt-dlp…" if is_url(args.source) else "[watch] using local file…", + file=sys.stderr, + ) + dl = download(args.source, work / "download") + video_path = dl["video_path"] + + meta = get_metadata(video_path) + full_duration = meta["duration_seconds"] + + start_sec = parse_time(args.start) + end_sec = parse_time(args.end) + + if start_sec is not None and start_sec < 0: + raise SystemExit("--start must be non-negative") + if end_sec is not None and start_sec is not None and end_sec <= start_sec: + raise SystemExit("--end must be greater than --start") + if full_duration > 0 and start_sec is not None and start_sec >= full_duration: + raise SystemExit(f"--start {start_sec:.1f}s is past end of video ({full_duration:.1f}s)") + + effective_start = start_sec if start_sec is not None else 0.0 + effective_end = end_sec if end_sec is not None else full_duration + effective_duration = max(0.0, effective_end - effective_start) + focused = start_sec is not None or end_sec is not None + + if focused: + fps, target = auto_fps_focus(effective_duration, max_frames=max_frames) + else: + fps, target = auto_fps(effective_duration, max_frames=max_frames) + if args.fps is not None: + fps = min(args.fps, MAX_FPS) + target = max(1, int(round(fps * effective_duration))) + + scope = ( + f"{format_time(effective_start)}-{format_time(effective_end)} ({effective_duration:.1f}s)" + if focused else f"full {effective_duration:.1f}s" + ) + print(f"[watch] extracting ~{target} frames at {fps:.3f} fps over {scope}…", file=sys.stderr) + + frames = extract( + video_path, + work / "frames", + fps=fps, + resolution=args.resolution, + max_frames=max_frames, + start_seconds=start_sec, + end_seconds=end_sec, + ) + + transcript_segments: list[dict] = [] + transcript_text: str | None = None + transcript_source: str | None = None + if dl.get("subtitle_path"): + try: + all_segments = parse_vtt(dl["subtitle_path"]) + transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments + transcript_text = format_transcript(transcript_segments) + transcript_source = "captions" + except Exception as exc: + print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr) + + if not transcript_segments and not args.no_whisper: + backend, api_key = load_api_key(args.whisper) + if backend and api_key: + try: + all_segments, used_backend = transcribe_video( + video_path, + work / "audio.mp3", + backend=backend, + api_key=api_key, + ) + transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments + transcript_text = format_transcript(transcript_segments) + transcript_source = f"whisper ({used_backend})" + except SystemExit as exc: + print(f"[watch] whisper fallback failed: {exc}", file=sys.stderr) + else: + hint = ( + f"--whisper {args.whisper} was set but the matching API key is missing" + if args.whisper else + "no subtitles and no Whisper API key found" + ) + setup_py = SCRIPT_DIR / "setup.py" + print( + f"[watch] {hint} — run `python3 {setup_py}` to enable the Whisper fallback", + file=sys.stderr, + ) + + info = dl.get("info") or {} + + print() + print("# watch: video report") + print() + print(f"- **Source:** {args.source}") + if info.get("title"): + print(f"- **Title:** {info['title']}") + if info.get("uploader"): + print(f"- **Uploader:** {info['uploader']}") + print(f"- **Duration:** {format_time(full_duration)} ({full_duration:.1f}s)") + if focused: + print( + f"- **Focus range:** {format_time(effective_start)} → {format_time(effective_end)} " + f"({effective_duration:.1f}s)" + ) + if meta.get("width") and meta.get("height"): + print(f"- **Resolution:** {meta['width']}x{meta['height']} ({meta.get('codec') or 'unknown codec'})") + mode = "focused" if focused else "full" + print(f"- **Frames:** {len(frames)} @ {fps:.3f} fps, {mode} mode (budget {target}, max {max_frames})") + print(f"- **Frame size:** {args.resolution}px wide") + if transcript_segments: + in_range = " in range" if focused else "" + print( + f"- **Transcript:** {len(transcript_segments)} segments{in_range} " + f"(via {transcript_source or 'captions'})" + ) + else: + print("- **Transcript:** none available") + + if not focused and full_duration > 600: + mins = int(full_duration // 60) + print() + print( + f"> ⚠️ This is a {mins}-minute video. Frame coverage is sparse at this length — " + "accuracy degrades noticeably on anything over 10 minutes. For better results, " + "re-run with `--start HH:MM:SS --end HH:MM:SS` to zoom into a specific section." + ) + + print() + print("## Frames") + print() + print(f"Frames live at: `{work / 'frames'}`") + print() + print( + "**Read each frame path below with the Read tool to view the image.** " + "Frames are in chronological order; `t=MM:SS` is the absolute timestamp in the source video." + ) + print() + for frame in frames: + print(f"- `{frame['path']}` (t={format_time(frame['timestamp_seconds'])})") + + print() + print("## Transcript") + print() + if transcript_text: + label = transcript_source or "captions" + if focused: + print(f"_Source: {label}. Filtered to {format_time(effective_start)} → {format_time(effective_end)}:_") + else: + print(f"_Source: {label}._") + print() + print("```") + print(transcript_text) + print("```") + elif focused and dl.get("subtitle_path"): + print(f"_No transcript lines fell inside {format_time(effective_start)} → {format_time(effective_end)}._") + else: + setup_py = SCRIPT_DIR / "setup.py" + print( + "_No transcript available — proceed with frames only. " + "Captions were missing and the Whisper fallback was unavailable " + "(no API key set, or `--no-whisper` was used). " + f"Run `python3 {setup_py}` to enable Whisper, then re-run._" + ) + + print() + print("---") + print(f"_Work dir: `{work}` — delete when done._") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/whisper.py b/scripts/whisper.py new file mode 100755 index 0000000..565994f --- /dev/null +++ b/scripts/whisper.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +"""Transcribe a video via Groq or OpenAI Whisper API. + +Strategy: extract audio (mono 16kHz mp3, tiny payload), upload to whichever +API has a key. Returns segments in the same shape as transcribe.parse_vtt so +the rest of the pipeline (filter_range, format_transcript) doesn't care where +the transcript came from. + +Pure stdlib — no `pip install groq` or `pip install openai` needed. +""" +from __future__ import annotations + +import io +import json +import mimetypes +import os +import shutil +import ssl +import subprocess +import sys +import time +import urllib.error +import uuid +from pathlib import Path +from urllib.request import Request, urlopen + + +GROQ_ENDPOINT = "https://api.groq.com/openai/v1/audio/transcriptions" +GROQ_MODEL = "whisper-large-v3" + +OPENAI_ENDPOINT = "https://api.openai.com/v1/audio/transcriptions" +OPENAI_MODEL = "whisper-1" + + +def load_api_key(preferred: str | None = None) -> tuple[str, str] | tuple[None, None]: + """Return (backend, api_key). Prefers Groq, falls back to OpenAI. + + If `preferred` is "groq" or "openai", only that backend's key is considered. + """ + def _from_env(name: str) -> str | None: + value = os.environ.get(name) + return value.strip() if value else None + + def _from_dotenv(path: Path, name: str) -> str | None: + if not path.exists(): + return None + try: + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + if key.strip() != name: + continue + value = value.strip() + if len(value) >= 2 and value[0] in ('"', "'") and value[-1] == value[0]: + value = value[1:-1] + return value or None + except OSError: + return None + return None + + dotenv_paths = [ + Path.home() / ".config" / "watch" / ".env", + Path.cwd() / ".env", + ] + + candidates = (("GROQ_API_KEY", "groq"), ("OPENAI_API_KEY", "openai")) + if preferred is not None: + candidates = tuple(c for c in candidates if c[1] == preferred) + + for key_name, backend in candidates: + value = _from_env(key_name) + if not value: + for candidate in dotenv_paths: + value = _from_dotenv(candidate, key_name) + if value: + break + if value: + return backend, value + + return None, None + + +def extract_audio(video_path: str, out_path: Path) -> Path: + """Extract mono 16kHz 64kbps mp3 — ~480 kB/min, fits any Whisper limit.""" + if shutil.which("ffmpeg") is None: + raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg") + + out_path.parent.mkdir(parents=True, exist_ok=True) + cmd = [ + "ffmpeg", + "-hide_banner", + "-loglevel", "error", + "-y", + "-i", video_path, + "-vn", + "-acodec", "libmp3lame", + "-ar", "16000", + "-ac", "1", + "-b:a", "64k", + str(out_path), + ] + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise SystemExit(f"ffmpeg audio extraction failed: {result.stderr.strip()}") + if not out_path.exists() or out_path.stat().st_size == 0: + raise SystemExit("ffmpeg produced no audio — video may have no audio track") + return out_path + + +def _build_multipart(fields: dict[str, str], file_path: Path) -> tuple[bytes, str]: + """Assemble a multipart/form-data body the Whisper APIs accept. + + Whisper's multipart upload is small and predictable — doing it by hand + keeps us on pure stdlib instead of pulling requests/groq/openai SDKs. + """ + boundary = f"----WatchBoundary{uuid.uuid4().hex}" + eol = b"\r\n" + buf = io.BytesIO() + + for name, value in fields.items(): + buf.write(f"--{boundary}".encode()); buf.write(eol) + buf.write(f'Content-Disposition: form-data; name="{name}"'.encode()); buf.write(eol) + buf.write(eol) + buf.write(str(value).encode()); buf.write(eol) + + mimetype = mimetypes.guess_type(file_path.name)[0] or "application/octet-stream" + buf.write(f"--{boundary}".encode()); buf.write(eol) + buf.write( + f'Content-Disposition: form-data; name="file"; filename="{file_path.name}"'.encode() + ) + buf.write(eol) + buf.write(f"Content-Type: {mimetype}".encode()); buf.write(eol) + buf.write(eol) + buf.write(file_path.read_bytes()) + buf.write(eol) + buf.write(f"--{boundary}--".encode()); buf.write(eol) + + return buf.getvalue(), boundary + + +MAX_ATTEMPTS = 4 # initial + 3 retries +MAX_429_RETRIES = 2 +RETRY_BASE_DELAY = 2.0 + + +def _post_whisper(endpoint: str, api_key: str, model: str, audio_path: Path) -> dict: + fields = { + "model": model, + "response_format": "verbose_json", + "temperature": "0", + } + body, boundary = _build_multipart(fields, audio_path) + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": f"multipart/form-data; boundary={boundary}", + # Groq sits behind Cloudflare — the default `Python-urllib/3.x` UA + # trips WAF rule 1010 (403) before auth even runs. Any non-default + # UA clears it; we identify honestly. + "User-Agent": "watch-skill/1.0 (+claude-code; python-urllib)", + } + + context = ssl.create_default_context() + rate_limit_hits = 0 + last_exc: Exception | None = None + last_detail = "" + + for attempt in range(MAX_ATTEMPTS): + request = Request(endpoint, data=body, headers=headers, method="POST") + try: + with urlopen(request, timeout=300, context=context) as response: + payload = response.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as exc: + detail = _read_error_body(exc) + last_exc, last_detail = exc, detail + + # 4xx other than 429 are client errors — no retry will fix them. + if 400 <= exc.code < 500 and exc.code != 429: + raise SystemExit(f"Whisper request failed: {exc}{detail}") + + if exc.code == 429: + rate_limit_hits += 1 + if rate_limit_hits >= MAX_429_RETRIES: + raise SystemExit(f"Whisper request failed: {exc}{detail}") + delay = _retry_after(exc) or RETRY_BASE_DELAY * (2 ** attempt) + 1 + else: + delay = RETRY_BASE_DELAY * (2 ** attempt) + + if attempt < MAX_ATTEMPTS - 1: + print( + f"[watch] whisper HTTP {exc.code} — retrying in {delay:.1f}s " + f"(attempt {attempt + 2}/{MAX_ATTEMPTS})", + file=sys.stderr, + ) + time.sleep(delay) + continue + except (urllib.error.URLError, TimeoutError, ConnectionResetError, OSError) as exc: + last_exc, last_detail = exc, "" + if attempt < MAX_ATTEMPTS - 1: + delay = RETRY_BASE_DELAY * (attempt + 1) + print( + f"[watch] whisper network error ({type(exc).__name__}: {exc}) — " + f"retrying in {delay:.1f}s (attempt {attempt + 2}/{MAX_ATTEMPTS})", + file=sys.stderr, + ) + time.sleep(delay) + continue + + try: + return json.loads(payload) + except json.JSONDecodeError as exc: + raise SystemExit(f"Whisper returned non-JSON response: {exc}: {payload[:200]}") + + raise SystemExit( + f"Whisper request failed after {MAX_ATTEMPTS} attempts: {last_exc}{last_detail}" + ) + + +def _read_error_body(exc: urllib.error.HTTPError) -> str: + try: + body = exc.read() + except Exception: + return "" + if not body: + return "" + try: + return f" — {body.decode('utf-8', errors='replace')[:400]}" + except Exception: + return "" + + +def _retry_after(exc: urllib.error.HTTPError) -> float | None: + header = exc.headers.get("Retry-After") if getattr(exc, "headers", None) else None + if not header: + return None + try: + return float(header) + except ValueError: + return None + + +def _segments_from_response(data: dict) -> list[dict]: + """Convert Whisper verbose_json into our {start, end, text} segment format.""" + out: list[dict] = [] + for seg in data.get("segments") or []: + text = (seg.get("text") or "").strip() + if not text: + continue + out.append({ + "start": round(float(seg.get("start") or 0.0), 2), + "end": round(float(seg.get("end") or 0.0), 2), + "text": text, + }) + + if not out: + full = (data.get("text") or "").strip() + if full: + out.append({"start": 0.0, "end": 0.0, "text": full}) + + return out + + +def transcribe_video( + video_path: str, + audio_out: Path, + backend: str | None = None, + api_key: str | None = None, +) -> tuple[list[dict], str]: + """Run the full flow: extract audio → upload → parse segments. + + Returns (segments, backend_used). Raises SystemExit on any failure. + """ + if backend is None or api_key is None: + detected_backend, detected_key = load_api_key() + backend = backend or detected_backend + api_key = api_key or detected_key + + if not backend or not api_key: + setup_py = Path(__file__).resolve().parent / "setup.py" + raise SystemExit( + "No Whisper API key available. Set GROQ_API_KEY (preferred) or OPENAI_API_KEY " + "in the environment or in ~/.config/watch/.env. " + f"Run `python3 {setup_py}` to configure." + ) + + print(f"[watch] extracting audio for Whisper ({backend})…", file=sys.stderr) + audio_path = extract_audio(video_path, audio_out) + size_kb = audio_path.stat().st_size / 1024 + print(f"[watch] audio: {size_kb:.0f} kB — uploading to {backend} Whisper…", file=sys.stderr) + + if backend == "groq": + response = _post_whisper(GROQ_ENDPOINT, api_key, GROQ_MODEL, audio_path) + elif backend == "openai": + response = _post_whisper(OPENAI_ENDPOINT, api_key, OPENAI_MODEL, audio_path) + else: + raise SystemExit(f"Unknown whisper backend: {backend}") + + segments = _segments_from_response(response) + if not segments: + raise SystemExit("Whisper returned no transcript segments") + + print(f"[watch] transcribed {len(segments)} segments via {backend}", file=sys.stderr) + return segments, backend + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("usage: whisper.py [] [--backend groq|openai]", file=sys.stderr) + raise SystemExit(2) + + video = sys.argv[1] + audio_out = Path(sys.argv[2]) if len(sys.argv) > 2 and not sys.argv[2].startswith("--") else Path("audio.mp3") + backend_override = None + if "--backend" in sys.argv: + backend_override = sys.argv[sys.argv.index("--backend") + 1] + + segments, backend = transcribe_video(video, audio_out, backend=backend_override) + print(json.dumps({"backend": backend, "segments": segments}, indent=2))