Initial commit: /watch skill v0.1.0
Give Claude the ability to watch any video — yt-dlp download, ffmpeg frame extraction with auto-scaled fps, native-caption transcript with Whisper (Groq/OpenAI) fallback. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
bradautomates
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"name": "claude-video",
|
||||
"metadata": {
|
||||
"description": "Give Claude a video input. /watch downloads, extracts frames, transcribes, and hands it all to Claude."
|
||||
},
|
||||
"owner": {
|
||||
"name": "Bradley Bonanno"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "watch",
|
||||
"description": "Watch a video (URL or local path). Downloads with yt-dlp, extracts frames with ffmpeg, transcribes via captions or Whisper.",
|
||||
"author": {
|
||||
"name": "Bradley Bonanno"
|
||||
},
|
||||
"source": "./",
|
||||
"category": "productivity",
|
||||
"homepage": "https://github.com/bradautomates/claude-video"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"name": "watch",
|
||||
"version": "0.1.0",
|
||||
"description": "Let Claude watch a video. Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls captions or falls back to Whisper, and hands frames + transcript to Claude so it can answer questions about the video.",
|
||||
"author": {
|
||||
"name": "Bradley Bonanno"
|
||||
},
|
||||
"homepage": "https://github.com/bradautomates/claude-video",
|
||||
"repository": "https://github.com/bradautomates/claude-video",
|
||||
"license": "MIT",
|
||||
"keywords": ["video", "watch", "youtube", "vimeo", "tiktok", "transcription", "whisper", "yt-dlp", "ffmpeg", "multimodal", "frames"]
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"name": "watch"
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
# Exclude non-runtime files from `git archive` output.
|
||||
# Used by scripts/build-skill.sh to produce a claude.ai-upload-ready .skill file.
|
||||
|
||||
# Anthropic canonical skill-packaging excludes
|
||||
# (mirrors anthropics/skills/skills/skill-creator/scripts/package_skill.py)
|
||||
__pycache__/ export-ignore
|
||||
node_modules/ export-ignore
|
||||
*.pyc export-ignore
|
||||
.DS_Store export-ignore
|
||||
|
||||
# Dev, docs, test, and media — not needed at skill runtime
|
||||
tests/ export-ignore
|
||||
docs/ export-ignore
|
||||
fixtures/ export-ignore
|
||||
assets/ export-ignore
|
||||
examples/ export-ignore
|
||||
|
||||
# NOTE: commands/, hooks/, and .claude-plugin/ are NOT export-ignored because
|
||||
# Claude Code's /plugin install fetches this same git archive tarball — they
|
||||
# must be in the archive for the plugin to install correctly. The claude.ai
|
||||
# .skill bundle strips them afterward via `zip -d` in scripts/build-skill.sh.
|
||||
|
||||
# CI workflows — repo-only, not needed at skill runtime
|
||||
.github/ export-ignore
|
||||
|
||||
# Build config itself
|
||||
.gitignore export-ignore
|
||||
.gitattributes export-ignore
|
||||
@@ -0,0 +1,31 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
build-and-release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build .skill artifact
|
||||
run: |
|
||||
bash scripts/build-skill.sh
|
||||
test -f dist/watch.skill
|
||||
|
||||
- name: Create GitHub release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: dist/watch.skill
|
||||
generate_release_notes: true
|
||||
draft: false
|
||||
prerelease: false
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
# OS / editor / tooling
|
||||
.DS_Store
|
||||
.entire/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.venv/
|
||||
.coverage
|
||||
htmlcov/
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# build artifact from scripts/build-skill.sh
|
||||
/dist/
|
||||
|
||||
# local Claude Code settings
|
||||
.claude/settings.local.json
|
||||
@@ -0,0 +1,17 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to `/watch` are documented here.
|
||||
|
||||
## [0.1.0] — 2026-04-24
|
||||
|
||||
Initial marketplace release.
|
||||
|
||||
### Added
|
||||
- `/watch <url-or-path> [question]` slash command.
|
||||
- yt-dlp download with native caption extraction (manual + auto-subs).
|
||||
- ffmpeg frame extraction with auto-scaled fps (≤2 fps, ≤100 frames, duration-aware budget).
|
||||
- `--start` / `--end` focused mode with denser frame budget and transcript range filtering.
|
||||
- Whisper fallback (Groq preferred, OpenAI secondary) for videos without captions.
|
||||
- `setup.py` preflight: silent `--check`, structured `--json`, and installer that auto-runs `brew install` on macOS.
|
||||
- Session-start hook that prints a one-line status on first run / partial config.
|
||||
- `.skill` bundle packaging for claude.ai upload via `scripts/build-skill.sh`.
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Bradley Bonanno
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,200 @@
|
||||
# /watch
|
||||
|
||||
**Give Claude the ability to watch any video.**
|
||||
|
||||
Claude Code:
|
||||
```
|
||||
/plugin marketplace add bradautomates/claude-video
|
||||
/plugin install watch@claude-video
|
||||
```
|
||||
|
||||
claude.ai (web): [download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) and drop it into Settings → Capabilities → Skills.
|
||||
|
||||
Codex / generic skills:
|
||||
```bash
|
||||
git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch
|
||||
```
|
||||
|
||||
Zero config to start — `yt-dlp` and `ffmpeg` install on first run via `brew` on macOS (Linux/Windows print exact commands). Captions cover most public videos for free. Whisper API key is only needed when a video has no captions.
|
||||
|
||||
---
|
||||
|
||||
Claude can read a webpage, run a script, browse a repo. What it can't do, out of the box, is *watch a video*. You paste a YouTube link and it has to either guess from the title or pull a transcript that's missing 90% of what's on screen.
|
||||
|
||||
With Claude Video `/watch` you can paste a URL or a local path, ask a question, and Claude downloads the video, extracts frames at an auto-scaled rate, pulls a timestamped transcript (free captions when available, Whisper API as fallback), and `Read`s every frame as an image. By the time it answers, it has *seen* the video and *heard* the audio.
|
||||
|
||||
```
|
||||
/watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark?
|
||||
```
|
||||
|
||||
## Why this exists
|
||||
|
||||
I built this because I'm constantly using video to keep up with content. If I see a YouTube video that's blowing up, I want to know how the creator structured the hook — what's on screen in the first 3 seconds, what they said, why it worked. That used to mean watching it myself with a notepad. Now I just paste the URL and ask.
|
||||
|
||||
The other half is summarization. Most YouTube videos don't deserve 20 minutes of my attention. I hand the URL to Claude, it pulls the transcript, and tells me what actually happened. If the visual matters, frames come along too. If it's a podcast or a talking head, transcript is enough.
|
||||
|
||||
Claude is great at reading and synthesizing — but until now, video was the one input I couldn't hand it. Pasting a YouTube link got you nothing useful. `/watch` closes that gap.
|
||||
|
||||
## What people actually use it for
|
||||
|
||||
**Analyze someone else's content.** `/watch https://youtu.be/<viral-video> what hook did they open with?` Claude looks at the first frames, reads the opening transcript, breaks down the structure. Same for ad creative, competitor launches, podcast intros, anything where the *how* matters as much as the *what*.
|
||||
|
||||
**Diagnose a bug from a video.** Someone sends you a screen recording of something broken. `/watch bug-repro.mov what's going wrong?` Claude watches the recording, finds the frame where the issue appears, describes what's on screen, often catches the cause without you ever opening the file.
|
||||
|
||||
**Summarize a video.** `/watch https://youtu.be/<long-thing> summarize this` does the obvious thing — pulls the structure, the key moments, what was actually said and shown. Faster than watching at 2x.
|
||||
|
||||
## How it works
|
||||
|
||||
1. **You paste a video and a question.** URL (anything yt-dlp supports — YouTube, Loom, TikTok, X, Instagram, plus a few hundred more) or a local path (`.mp4`, `.mov`, `.mkv`, `.webm`).
|
||||
2. **`yt-dlp` downloads it.** For URLs, into a temp working directory. For local files, no download — just probed in place.
|
||||
3. **`ffmpeg` extracts frames at an auto-scaled rate.** The frame budget is duration-aware: ≤30s gets ~30 frames, 30-60s gets ~40, 1-3min gets ~60, 3-10min gets ~80, longer gets 100 sparsely. Hard ceilings: 2 fps, 100 frames. JPEGs at 512px wide by default — bump with `--resolution 1024` if Claude needs to read on-screen text.
|
||||
4. **The transcript comes from one of two places.** First try: `yt-dlp` pulls native captions (manual or auto-generated) from the source. Free, instant, accurate-ish. Fallback: extract a mono 16 kHz audio clip and ship it to Whisper — Groq's `whisper-large-v3` (preferred — cheaper and faster) or OpenAI's `whisper-1`.
|
||||
5. **Frames + transcript are handed to Claude.** The script prints frame paths with `t=MM:SS` markers and the transcript with timestamps. Claude `Read`s each frame in parallel — JPEGs render directly as images in its context.
|
||||
6. **Claude answers grounded in what's actually on screen and in the audio.** Not "based on the description" or "according to the title." It saw the frames. It heard the transcript. It answers the way someone who watched the video would.
|
||||
7. **Cleanup.** The script prints a working directory at the end. If you're not asking follow-ups, Claude removes it.
|
||||
|
||||
## Frame budget — why it matters
|
||||
|
||||
Token cost is dominated by frames. Every frame is an image; image tokens add up fast. The script's auto-fps logic exists so you don't blow your context budget on a sparse scan of a 30-minute video that would have been better answered by a focused 30-second window.
|
||||
|
||||
| Duration | Default frame budget | What you get |
|
||||
|----------|---------------------|--------------|
|
||||
| ≤30 s | ~30 frames | Dense — basically every key moment |
|
||||
| 30 s - 1 min | ~40 frames | Still dense |
|
||||
| 1 - 3 min | ~60 frames | Comfortable |
|
||||
| 3 - 10 min | ~80 frames | Sparse but workable |
|
||||
| > 10 min | 100 frames | "Sparse scan" warning — re-run focused |
|
||||
|
||||
When the user names a moment ("around 2:30", "the last 30 seconds", "from 0:45 to 1:00"), pass `--start` / `--end`. Focused mode gets denser per-second budgets, capped at 2 fps. Far more useful than a sparse pass over the whole thing.
|
||||
|
||||
## Install
|
||||
|
||||
| Surface | Install |
|
||||
|---------|---------|
|
||||
| **Claude Code** | `/plugin marketplace add bradautomates/claude-video` then `/plugin install watch@claude-video` |
|
||||
| **claude.ai** (web) | [Download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) → Settings → Capabilities → Skills → `+` |
|
||||
| **Codex** | `git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch` |
|
||||
| **Manual / dev** | `git clone https://github.com/bradautomates/claude-video.git ~/.claude/skills/watch` |
|
||||
|
||||
### Claude Code
|
||||
|
||||
```
|
||||
/plugin marketplace add bradautomates/claude-video
|
||||
/plugin install watch@claude-video
|
||||
```
|
||||
|
||||
Update later with `/plugin update watch@claude-video`.
|
||||
|
||||
### claude.ai (web)
|
||||
|
||||
1. [Download `watch.skill`](https://github.com/bradautomates/claude-video/releases/latest) from the latest release.
|
||||
2. Go to Settings → Capabilities → Skills.
|
||||
3. Click `+` and drop the file in.
|
||||
|
||||
Enable "Code execution and file creation" under Capabilities first — the skill shells out to `ffmpeg` and `yt-dlp`, so it won't run without it.
|
||||
|
||||
### Codex
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bradautomates/claude-video.git ~/.codex/skills/watch
|
||||
```
|
||||
|
||||
### Manual (developer)
|
||||
|
||||
```bash
|
||||
git clone https://github.com/bradautomates/claude-video.git ~/.claude/skills/watch
|
||||
```
|
||||
|
||||
## First run
|
||||
|
||||
On the first `/watch` call, the skill runs `scripts/setup.py --check`. If `ffmpeg` / `yt-dlp` aren't on your PATH, or no Whisper API key is set, it walks you through fixing it:
|
||||
|
||||
- **macOS** — auto-runs `brew install ffmpeg yt-dlp`.
|
||||
- **Linux** — prints the exact `apt` / `dnf` / `pipx` commands.
|
||||
- **Windows** — prints the `winget` / `pip` commands.
|
||||
- **API key** — scaffolds `~/.config/watch/.env` (mode `0600`) with commented placeholders for `GROQ_API_KEY` (preferred) and `OPENAI_API_KEY`.
|
||||
|
||||
After setup, preflight is silent and `/watch` just works. The check is a sub-100ms lookup, so it doesn't slow you down on subsequent runs.
|
||||
|
||||
## Bring your own keys
|
||||
|
||||
Captions cover the majority of public videos for free. The Whisper fallback only kicks in when a video genuinely has no caption track — typically local files, TikToks, some Vimeos, and the occasional caption-less YouTube upload.
|
||||
|
||||
| Capability | What you need | Cost |
|
||||
|------------|---------------|------|
|
||||
| Download + native captions | `yt-dlp` + `ffmpeg` | Free |
|
||||
| Whisper fallback (preferred) | [Groq API key](https://console.groq.com/keys) — `whisper-large-v3` | Cheap, fast |
|
||||
| Whisper fallback (alt) | [OpenAI API key](https://platform.openai.com/api-keys) — `whisper-1` | Standard pricing |
|
||||
| Disable Whisper entirely | `--no-whisper` | Free, frames-only when no captions |
|
||||
|
||||
## Usage
|
||||
|
||||
```
|
||||
/watch https://youtu.be/dQw4w9WgXcQ what happens at the 30 second mark?
|
||||
/watch https://www.tiktok.com/@user/video/123 summarize this
|
||||
/watch ~/Movies/screen-recording.mp4 when does the UI break?
|
||||
/watch https://vimeo.com/123 what tools does she mention?
|
||||
```
|
||||
|
||||
Focused on a specific section — denser frame budget, lower token cost:
|
||||
```
|
||||
/watch https://youtu.be/abc --start 2:15 --end 2:45
|
||||
/watch video.mp4 --start 50 --end 60
|
||||
/watch "$URL" --start 1:12:00 # from 1h12m to end
|
||||
```
|
||||
|
||||
Other knobs (passed to `scripts/watch.py`):
|
||||
|
||||
- `--max-frames N` — lower the frame cap for a tighter token budget.
|
||||
- `--resolution W` — bump frame width to 1024 px when Claude needs to read on-screen text (slides, terminals, code).
|
||||
- `--fps F` — override the auto-fps calculation (still capped at 2 fps).
|
||||
- `--whisper groq|openai` — force a specific Whisper backend.
|
||||
- `--no-whisper` — disable transcription entirely; frames only.
|
||||
- `--out-dir DIR` — keep working files somewhere specific (default: auto-generated tmp dir).
|
||||
|
||||
## Limits
|
||||
|
||||
- **Best accuracy: under 10 minutes.** Past that the script prints a "sparse scan" warning — re-run focused on the part you actually care about with `--start`/`--end`.
|
||||
- **Hard caps: 2 fps, 100 frames.** Frame count drives token cost; the script enforces this even when the auto-fps math would imply higher.
|
||||
- **Whisper upload limit: 25 MB.** At mono 16 kHz that's about 50 minutes of audio. Longer videos need either captions or `--start`/`--end` to a smaller window.
|
||||
- **No private platforms.** This skill doesn't log into anything. Public URLs and local files only. If yt-dlp can't reach it without auth, neither can `/watch`.
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
.
|
||||
├── SKILL.md # skill contract — loaded by all three surfaces
|
||||
├── scripts/
|
||||
│ ├── watch.py # entry point — orchestrates download → frames → transcript
|
||||
│ ├── download.py # yt-dlp wrapper
|
||||
│ ├── frames.py # ffmpeg frame extraction + auto-fps logic
|
||||
│ ├── transcribe.py # VTT parsing + dedupe + Whisper orchestration
|
||||
│ ├── whisper.py # Groq / OpenAI clients (pure stdlib)
|
||||
│ ├── setup.py # preflight + installer
|
||||
│ └── build-skill.sh # build dist/watch.skill for claude.ai upload
|
||||
├── hooks/ # SessionStart status hook (Claude Code only)
|
||||
├── .claude-plugin/ # plugin.json + marketplace.json (Claude Code)
|
||||
├── .codex-plugin/ # codex packaging
|
||||
└── .github/workflows/ # release.yml — auto-builds watch.skill on tag push
|
||||
```
|
||||
|
||||
## Develop
|
||||
|
||||
```bash
|
||||
# Build the claude.ai upload bundle:
|
||||
bash scripts/build-skill.sh # → dist/watch.skill
|
||||
```
|
||||
|
||||
Releasing: tag `vX.Y.Z`, push the tag. The workflow builds `dist/watch.skill` and attaches it to the GitHub release.
|
||||
|
||||
See [CHANGELOG.md](CHANGELOG.md) for version history.
|
||||
|
||||
## Open source
|
||||
|
||||
MIT license.
|
||||
|
||||
Built on `yt-dlp`, `ffmpeg`, and Claude's multimodal `Read` tool. Whisper transcription via [Groq](https://groq.com) or [OpenAI](https://openai.com).
|
||||
|
||||
---
|
||||
|
||||
[github.com/bradautomates/claude-video](https://github.com/bradautomates/claude-video) · [LICENSE](LICENSE)
|
||||
@@ -0,0 +1,171 @@
|
||||
---
|
||||
name: watch
|
||||
description: Watch a video (URL or local path). Downloads with yt-dlp, extracts auto-scaled frames with ffmpeg, pulls the transcript from captions (or Whisper API fallback), and hands the result to Claude so it can answer questions about what's in the video.
|
||||
argument-hint: "<video-url-or-path> [question]"
|
||||
allowed-tools: Bash, Read, AskUserQuestion
|
||||
homepage: https://github.com/bradautomates/claude-video
|
||||
repository: https://github.com/bradautomates/claude-video
|
||||
author: bradautomates
|
||||
license: MIT
|
||||
user-invocable: true
|
||||
---
|
||||
|
||||
# /watch — Claude watches a video
|
||||
|
||||
You don't have a video input; this skill gives you one. A Python script downloads the video, extracts frames as JPEGs, gets a timestamped transcript (native captions first, then Whisper API as fallback), and prints frame paths. You then `Read` each frame path to see the images and combine them with the transcript to answer the user.
|
||||
|
||||
## Step 0 — Setup preflight (runs every `/watch` invocation, silent on success)
|
||||
|
||||
Before every `/watch` run, verify that dependencies and an API key are in place:
|
||||
|
||||
```bash
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" --check
|
||||
```
|
||||
|
||||
This is a <100ms lookup. On exit 0, the script emits **nothing** — proceed to Step 1 without comment. **Do NOT announce "setup is complete" to the user** — they don't need a status message on every turn. The only acceptable user-visible output from Step 0 is when remediation is required.
|
||||
|
||||
On non-zero exit, follow the table:
|
||||
|
||||
| Exit | Meaning | Action |
|
||||
|------|---------|--------|
|
||||
| `2` | Missing binaries (`ffmpeg` / `ffprobe` / `yt-dlp`) | Run installer |
|
||||
| `3` | No Whisper API key | Run installer to scaffold `.env`, then ask user for a key |
|
||||
| `4` | Both missing | Run installer, then ask for a key |
|
||||
|
||||
The installer is idempotent — safe to re-run:
|
||||
|
||||
```bash
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py"
|
||||
```
|
||||
|
||||
On macOS with Homebrew, it auto-installs `ffmpeg` and `yt-dlp`. On Linux/Windows, it prints the exact install commands for the user to run. It scaffolds `~/.config/watch/.env` with commented placeholders at `0600` perms, and writes `SETUP_COMPLETE=true` once deps + a key are in place so the next session knows this user has already been through the wizard.
|
||||
|
||||
**If an API key is still missing after install:** use `AskUserQuestion` to ask the user whether they have a Groq API key (preferred — cheaper, faster) or an OpenAI key. Then write it into `~/.config/watch/.env` — set the matching `GROQ_API_KEY=...` or `OPENAI_API_KEY=...` line. If they don't want to set up Whisper, proceed with `--no-whisper` and tell them videos without native captions will come back frames-only.
|
||||
|
||||
**Structured mode (optional):** `python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py" --json` emits `{status, first_run, missing_binaries, whisper_backend, has_api_key, config_file, platform}` where `status` is one of `ready | needs_install | needs_key | needs_install_and_key`. Use this when you need to branch on specifics (e.g. "is this the user's very first run?" → `first_run: true`).
|
||||
|
||||
Within a single session, you can skip Step 0 on follow-up `/watch` calls — once `--check` returned 0, nothing about the environment changes between turns.
|
||||
|
||||
## When to use
|
||||
|
||||
- User pastes a video URL (YouTube, Vimeo, X, TikTok, Twitch clip, most yt-dlp-supported sites) and asks about it.
|
||||
- User points at a local video file (`.mp4`, `.mov`, `.mkv`, `.webm`, etc.) and asks about it.
|
||||
- User types `/watch <url-or-path> [question]`.
|
||||
|
||||
## Recommended limits
|
||||
|
||||
- **Best accuracy: videos under 10 minutes.** Frame coverage scales inversely with duration.
|
||||
- **Hard caps: 100 frames total and 2 fps.** Token cost grows with frame count, so the script targets a frame budget by duration (and never exceeds 2 fps even when the budget would imply more):
|
||||
- ≤30s → ~1-2 fps (up to 30 frames)
|
||||
- 30s-1min → ~40 frames
|
||||
- 1-3min → ~60 frames
|
||||
- 3-10min → ~80 frames
|
||||
- \>10min → 100 frames, sparsely spaced (warning printed)
|
||||
- If the user hands you a long video, consider asking whether they want a specific section before burning tokens on a sparse scan.
|
||||
|
||||
## How to invoke
|
||||
|
||||
**Step 1 — parse the user input.** Separate the video source (URL or path) from any question the user asked. Example: `/watch https://youtu.be/abc what language is this in?` → source = `https://youtu.be/abc`, question = `what language is this in?`.
|
||||
|
||||
**Step 2 — run the watch script.** Pass the source verbatim. Do not shell-escape it yourself beyond normal quoting:
|
||||
|
||||
```bash
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "<source>"
|
||||
```
|
||||
|
||||
Optional flags:
|
||||
- `--start T` / `--end T` — focus on a section. Accepts `SS`, `MM:SS`, or `HH:MM:SS`. When either is set, fps auto-scales denser (see "Focusing on a section" below).
|
||||
- `--max-frames N` — lower the cap for tighter token budget (e.g. `--max-frames 40`)
|
||||
- `--resolution W` — change frame width in px (default 512; bump to 1024 only if the user needs to read on-screen text)
|
||||
- `--fps F` — override auto-fps (clamped to 2 fps max)
|
||||
- `--out-dir DIR` — keep working files somewhere specific (default: an auto-generated tmp dir)
|
||||
- `--whisper groq|openai` — force a specific Whisper backend (default: prefer Groq if both keys exist)
|
||||
- `--no-whisper` — disable the Whisper fallback entirely (frames-only if no captions)
|
||||
|
||||
### Focusing on a section (higher frame rate)
|
||||
|
||||
When the user asks about a specific moment — "what happens at the 2 minute mark?", "zoom into 0:45 to 1:00", "the first 10 seconds" — pass `--start` and/or `--end`. The script switches to focused-mode budgets, which are denser than full-video budgets (still capped at 2 fps):
|
||||
|
||||
- ≤5s → 2 fps (up to 10 frames)
|
||||
- 5-15s → 2 fps (up to 30 frames)
|
||||
- 15-30s → ~2 fps (up to 60 frames)
|
||||
- 30-60s → ~1.3 fps (up to 80 frames)
|
||||
- 60-180s → ~0.6 fps (100 frames, capped)
|
||||
|
||||
Focused mode is the right call for:
|
||||
- Any moment/range the user names explicitly ("around 2:30", "the intro", "the last 30 seconds").
|
||||
- Any video longer than ~10 minutes where the user's question is about a specific part — running focused on the relevant section is far more useful than a sparse scan of the whole thing.
|
||||
- Re-runs after a full scan didn't have enough detail in some region.
|
||||
|
||||
Transcript is auto-filtered to the same range. Frame timestamps are absolute (real video timeline, not offset-from-start).
|
||||
|
||||
Examples:
|
||||
```bash
|
||||
# Last 10 seconds of a 1 minute video
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" video.mp4 --start 50 --end 60
|
||||
|
||||
# Zoom into 2:15 → 2:45 at 3 fps (90 frames)
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "$URL" --start 2:15 --end 2:45 --fps 3
|
||||
|
||||
# From 1h12m to the end of the video
|
||||
python3 "${CLAUDE_SKILL_DIR}/scripts/watch.py" "$URL" --start 1:12:00
|
||||
```
|
||||
|
||||
**Step 3 — Read every frame path the script lists.** The Read tool renders JPEGs directly as images for you. Read all frames in a single message (parallel tool calls) so you see them together. The frames are in chronological order with a `t=MM:SS` timestamp so you can align them to the transcript.
|
||||
|
||||
**Step 4 — answer the user.** You now have two streams of evidence:
|
||||
- **Frames** — what's on screen at each timestamp
|
||||
- **Transcript** — what's said at each timestamp. The report's header shows the source (`captions` = yt-dlp pulled native subs; `whisper (groq)` or `whisper (openai)` = transcribed by API).
|
||||
|
||||
If the user asked a specific question, answer it directly citing timestamps. If they didn't ask anything, summarize what happens in the video — structure, key moments, notable visuals, spoken content.
|
||||
|
||||
**Step 5 — clean up.** The script prints a working directory at the end. If the user isn't going to ask follow-ups about this video, delete it with `rm -rf <dir>`. If they might, leave it in place.
|
||||
|
||||
## Transcription
|
||||
|
||||
The script gets a timestamped transcript in one of two ways:
|
||||
|
||||
1. **Native captions (free, preferred).** yt-dlp pulls manual or auto-generated subtitles from the source platform if available.
|
||||
2. **Whisper API fallback.** If no captions came back (or the source is a local file), the script extracts audio (`ffmpeg -vn -ac 1 -ar 16000 -b:a 64k`, ~0.5 MB/min) and uploads it to whichever Whisper API has a key configured:
|
||||
- **Groq** — `whisper-large-v3`. Preferred default: cheaper, faster. Get a key at console.groq.com/keys.
|
||||
- **OpenAI** — `whisper-1`. Fallback. Get a key at platform.openai.com/api-keys.
|
||||
|
||||
Both keys live in `~/.config/watch/.env`. The script prefers Groq when both are set; override with `--whisper openai` to force OpenAI. Use `--no-whisper` to skip the fallback entirely.
|
||||
|
||||
## Failure modes and handling
|
||||
|
||||
- **Setup preflight failed** → run `python3 "${CLAUDE_SKILL_DIR}/scripts/setup.py"` (auto-installs ffmpeg/yt-dlp via brew on macOS, scaffolds the `.env`). For API key, ask the user via `AskUserQuestion` and write it to `~/.config/watch/.env`.
|
||||
- **No transcript available** → captions missing AND (no Whisper key OR Whisper API failed). Script prints a hint pointing to setup. Proceed frames-only and tell the user.
|
||||
- **Long video warning printed** → acknowledge it in your answer. Offer to re-run focused on a specific section via `--start`/`--end` rather than a sparse full-video scan.
|
||||
- **Download fails** → yt-dlp's error goes to stderr. If it's a login-required or region-locked video, tell the user plainly; do not keep retrying.
|
||||
- **Whisper request fails** → the error is printed to stderr (likely: invalid key, rate limit, or 25 MB upload limit on a very long video). The report will say "none available" for transcript. You can retry with `--whisper openai` if Groq failed (or vice versa).
|
||||
|
||||
## Token efficiency
|
||||
|
||||
This skill burns tokens primarily on frames. Order of magnitude:
|
||||
- 80 frames at 512px wide is roughly 50-80k image tokens depending on aspect ratio.
|
||||
- The transcript is cheap (a few thousand tokens at most for a 10-minute video).
|
||||
- Bumping `--resolution` to 1024 roughly quadruples the image tokens per frame. Only do it when necessary.
|
||||
|
||||
If you already watched a video this session and the user asks a follow-up, do **not** re-run the script — you already have the frames and transcript in context. Just answer from what you have.
|
||||
|
||||
## Security & Permissions
|
||||
|
||||
**What this skill does:**
|
||||
- Runs `yt-dlp` locally to download the video and pull native captions when the source supports them (public data; the request goes directly to whatever host the URL points at)
|
||||
- Runs `ffmpeg` / `ffprobe` locally to extract frames as JPEGs and, when Whisper is needed, a mono 16 kHz audio clip
|
||||
- Sends the extracted audio clip to Groq's Whisper API (`api.groq.com/openai/v1/audio/transcriptions`) when `GROQ_API_KEY` is set (preferred — cheaper, faster)
|
||||
- Sends the extracted audio clip to OpenAI's audio transcription API (`api.openai.com/v1/audio/transcriptions`) when `OPENAI_API_KEY` is set and Groq is not, or when `--whisper openai` is forced
|
||||
- Writes the downloaded video, frames, audio, and an intermediate transcript to a working directory under the system temp dir (or `--out-dir` if specified) so Claude can `Read` them
|
||||
- Reads / creates `~/.config/watch/.env` (mode `0600`) to store the Whisper API key(s) and a `SETUP_COMPLETE` marker. As a fallback, also reads `.env` in the current working directory
|
||||
|
||||
**What this skill does NOT do:**
|
||||
- Does not upload the video itself to any API — only the extracted audio goes out, and only when native captions are missing AND Whisper is not disabled with `--no-whisper`
|
||||
- Does not access any platform account (no login, no session cookies, no posting)
|
||||
- Does not share API keys between providers (Groq key only goes to `api.groq.com`, OpenAI key only goes to `api.openai.com`)
|
||||
- Does not log, cache, or write API keys to stdout, stderr, or output files
|
||||
- Does not persist anything outside the working directory and `~/.config/watch/.env` — clean up the working directory when you're done (Step 5)
|
||||
|
||||
**Bundled scripts:** `scripts/watch.py` (entry point), `scripts/download.py` (yt-dlp wrapper), `scripts/frames.py` (ffmpeg frame extraction), `scripts/transcribe.py` (caption selection + Whisper orchestration), `scripts/whisper.py` (Groq / OpenAI clients), `scripts/setup.py` (preflight + installer)
|
||||
|
||||
Review scripts before first use to verify behavior.
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"hooks": {
|
||||
"SessionStart": [
|
||||
{
|
||||
"matcher": "",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "bash ${CLAUDE_PLUGIN_ROOT}/hooks/scripts/check-setup.sh",
|
||||
"timeout": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
# SessionStart hook for /watch — one-line status so users know what's wired up.
|
||||
# Silent on ready state to avoid spam. Points at the installer when something
|
||||
# is missing.
|
||||
set -euo pipefail
|
||||
|
||||
CONFIG_FILE="$HOME/.config/watch/.env"
|
||||
|
||||
# Warn if the secrets file has loose permissions.
|
||||
if [[ -f "$CONFIG_FILE" ]]; then
|
||||
perms=$(stat -c '%a' "$CONFIG_FILE" 2>/dev/null || stat -f '%Lp' "$CONFIG_FILE" 2>/dev/null || echo "")
|
||||
if [[ -n "$perms" && "$perms" != "600" && "$perms" != "400" ]]; then
|
||||
echo "/watch: WARNING — $CONFIG_FILE has permissions $perms (should be 600)."
|
||||
echo " Fix: chmod 600 $CONFIG_FILE"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Load API keys from the config file without exporting them.
|
||||
read_key() {
|
||||
local name="$1"
|
||||
if [[ -n "${!name:-}" ]]; then
|
||||
echo "${!name}"
|
||||
return
|
||||
fi
|
||||
if [[ -f "$CONFIG_FILE" ]]; then
|
||||
awk -F= -v k="$name" '
|
||||
/^[[:space:]]*#/ { next }
|
||||
$1 == k {
|
||||
sub(/^[[:space:]]*/, "", $2); sub(/[[:space:]]*$/, "", $2);
|
||||
gsub(/^["'\'']|["'\'']$/, "", $2);
|
||||
print $2; exit
|
||||
}
|
||||
' "$CONFIG_FILE"
|
||||
fi
|
||||
}
|
||||
|
||||
HAS_FFMPEG=""
|
||||
HAS_YTDLP=""
|
||||
command -v ffmpeg >/dev/null 2>&1 && HAS_FFMPEG="yes"
|
||||
command -v yt-dlp >/dev/null 2>&1 && HAS_YTDLP="yes"
|
||||
|
||||
HAS_GROQ="$(read_key GROQ_API_KEY)"
|
||||
HAS_OPENAI="$(read_key OPENAI_API_KEY)"
|
||||
SETUP_COMPLETE="$(read_key SETUP_COMPLETE)"
|
||||
|
||||
# Fully configured → silent (Claude can surface status on demand via --check).
|
||||
if [[ "$SETUP_COMPLETE" == "true" && -n "$HAS_FFMPEG" && -n "$HAS_YTDLP" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# First-run / partially-configured → one-line hint.
|
||||
if [[ -z "$HAS_FFMPEG" || -z "$HAS_YTDLP" ]]; then
|
||||
echo "/watch: needs ffmpeg + yt-dlp. Run \`python3 \$CLAUDE_PLUGIN_ROOT/scripts/setup.py\` once to install and scaffold config."
|
||||
elif [[ -z "$HAS_GROQ" && -z "$HAS_OPENAI" ]]; then
|
||||
echo "/watch: ready for videos with native captions. Add GROQ_API_KEY (preferred) or OPENAI_API_KEY to ~/.config/watch/.env to unlock Whisper fallback."
|
||||
else
|
||||
echo "/watch: ready."
|
||||
fi
|
||||
Executable
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
# build-skill.sh — package this repo as a claude.ai-upload-ready .skill file.
|
||||
# Usage: bash scripts/build-skill.sh (run from repo root)
|
||||
#
|
||||
# Produces dist/watch.skill, a zip with a single top-level `watch/` directory
|
||||
# containing SKILL.md and the scripts/ runtime. claude.ai's skill upload has a
|
||||
# 200-file cap; `export-ignore` in .gitattributes + the zip -d strips below
|
||||
# keep the bundle lean.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
echo "error: working tree is dirty; commit or stash before building" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p dist
|
||||
OUT="dist/watch.skill"
|
||||
git archive --format=zip --prefix=watch/ --output="$OUT" HEAD
|
||||
|
||||
# claude.ai's .skill bundle needs only SKILL.md + scripts/ runtime. Claude Code
|
||||
# needs hooks/ and .claude-plugin/ in the git archive (that's why they are NOT
|
||||
# in .gitattributes export-ignore), but the .skill bundle should strip them to
|
||||
# keep a single canonical SKILL.md and stay well under the 200-file cap.
|
||||
zip -d "$OUT" \
|
||||
"watch/hooks/*" \
|
||||
"watch/.claude-plugin/*" \
|
||||
> /dev/null 2>&1 || true
|
||||
|
||||
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
|
||||
SIZE=$(du -h "$OUT" | cut -f1)
|
||||
|
||||
if [ "$COUNT" -gt 200 ]; then
|
||||
echo "error: $COUNT files in zip, claude.ai's cap is 200" >&2
|
||||
echo " check .gitattributes export-ignore entries and this script's zip -d excludes" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SKILL_MD_COUNT=$(unzip -l "$OUT" | grep -c "SKILL.md" || true)
|
||||
if [ "$SKILL_MD_COUNT" -ne 1 ]; then
|
||||
echo "error: expected exactly one SKILL.md, found $SKILL_MD_COUNT" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "built $OUT ($COUNT files, $SIZE)"
|
||||
echo "upload via the claude.ai skill UI"
|
||||
Executable
+127
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download a video via yt-dlp, or resolve a local file path.
|
||||
|
||||
Also fetches subtitles (manual first, then auto-generated) in VTT format so
|
||||
transcribe.py can parse them without needing Whisper.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".mov", ".m4v", ".avi", ".flv", ".wmv"}
|
||||
|
||||
|
||||
def is_url(source: str) -> bool:
|
||||
parsed = urlparse(source)
|
||||
return parsed.scheme in ("http", "https")
|
||||
|
||||
|
||||
def resolve_local(path: str) -> dict:
|
||||
p = Path(path).expanduser().resolve()
|
||||
if not p.exists():
|
||||
raise SystemExit(f"File not found: {p}")
|
||||
if p.suffix.lower() not in VIDEO_EXTS:
|
||||
print(
|
||||
f"[watch] warning: {p.suffix} is not a known video extension, proceeding anyway",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return {
|
||||
"video_path": str(p),
|
||||
"subtitle_path": None,
|
||||
"info": {"title": p.name, "url": str(p)},
|
||||
"downloaded": False,
|
||||
}
|
||||
|
||||
|
||||
def _pick_subtitle(out_dir: Path) -> Path | None:
|
||||
candidates = sorted(out_dir.glob("video*.vtt"))
|
||||
if not candidates:
|
||||
return None
|
||||
preferred = [c for c in candidates if ".en" in c.name]
|
||||
return preferred[0] if preferred else candidates[0]
|
||||
|
||||
|
||||
def _pick_video(out_dir: Path) -> Path | None:
|
||||
for ext in (".mp4", ".mkv", ".webm", ".mov"):
|
||||
for candidate in out_dir.glob(f"video*{ext}"):
|
||||
return candidate
|
||||
for candidate in out_dir.glob("video.*"):
|
||||
if candidate.suffix.lower() in VIDEO_EXTS:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def download_url(url: str, out_dir: Path) -> dict:
|
||||
if shutil.which("yt-dlp") is None:
|
||||
raise SystemExit("yt-dlp is not installed. Install with: brew install yt-dlp")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_template = str(out_dir / "video.%(ext)s")
|
||||
|
||||
cmd = [
|
||||
"yt-dlp",
|
||||
"-N", "8",
|
||||
"-f", "bv*[height<=720]+ba/b[height<=720]/bv+ba/b",
|
||||
"--merge-output-format", "mp4",
|
||||
"--write-info-json",
|
||||
"--write-subs",
|
||||
"--write-auto-subs",
|
||||
"--sub-langs", "en,en-US,en-GB,en-orig",
|
||||
"--sub-format", "vtt",
|
||||
"--convert-subs", "vtt",
|
||||
"--no-playlist",
|
||||
"--ignore-errors",
|
||||
"-o", output_template,
|
||||
url,
|
||||
]
|
||||
|
||||
# yt-dlp may exit non-zero if a subtitle variant fails (e.g. 429) even when
|
||||
# the video itself downloaded fine. Treat "video file present" as success.
|
||||
result = subprocess.run(cmd, stdout=sys.stderr, stderr=sys.stderr)
|
||||
video = _pick_video(out_dir)
|
||||
if video is None:
|
||||
raise SystemExit(
|
||||
f"yt-dlp did not produce a video file in {out_dir} (exit {result.returncode})"
|
||||
)
|
||||
|
||||
subtitle = _pick_subtitle(out_dir)
|
||||
info_path = out_dir / "video.info.json"
|
||||
info: dict = {}
|
||||
if info_path.exists():
|
||||
try:
|
||||
raw = json.loads(info_path.read_text())
|
||||
info = {
|
||||
"title": raw.get("title"),
|
||||
"uploader": raw.get("uploader") or raw.get("channel"),
|
||||
"duration": raw.get("duration"),
|
||||
"url": raw.get("webpage_url") or url,
|
||||
}
|
||||
except Exception:
|
||||
info = {"url": url}
|
||||
|
||||
return {
|
||||
"video_path": str(video),
|
||||
"subtitle_path": str(subtitle) if subtitle else None,
|
||||
"info": info or {"url": url},
|
||||
"downloaded": True,
|
||||
}
|
||||
|
||||
|
||||
def download(source: str, out_dir: Path) -> dict:
|
||||
if is_url(source):
|
||||
return download_url(source, out_dir)
|
||||
return resolve_local(source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
print("usage: download.py <url-or-path> <out-dir>", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
result = download(sys.argv[1], Path(sys.argv[2]))
|
||||
print(json.dumps(result, indent=2))
|
||||
Executable
+250
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Probe video metadata and extract frames at an auto-scaled fps.
|
||||
|
||||
Auto-fps targets a frame budget, not a fixed rate. Token cost scales with frame
|
||||
count, so budget-by-duration keeps short videos dense and long videos capped.
|
||||
When a user-specified range is passed, focused-mode budgets denser (they are
|
||||
zooming in for detail).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MAX_FPS = 2.0
|
||||
|
||||
|
||||
def _clamp_fps(fps: float, duration_seconds: float, max_frames: int) -> tuple[float, int]:
|
||||
fps = min(fps, MAX_FPS)
|
||||
target = min(max_frames, max(1, int(round(fps * duration_seconds))))
|
||||
return fps, target
|
||||
|
||||
|
||||
def parse_time(value: str | float | int | None) -> float | None:
|
||||
"""Parse SS, MM:SS, or HH:MM:SS (with optional .ms) into seconds."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
s = str(value).strip()
|
||||
if not s:
|
||||
return None
|
||||
parts = s.split(":")
|
||||
try:
|
||||
if len(parts) == 1:
|
||||
return float(parts[0])
|
||||
if len(parts) == 2:
|
||||
return int(parts[0]) * 60 + float(parts[1])
|
||||
if len(parts) == 3:
|
||||
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
|
||||
except ValueError:
|
||||
pass
|
||||
raise SystemExit(f"Cannot parse time value: {value!r} (expected SS, MM:SS, or HH:MM:SS)")
|
||||
|
||||
|
||||
def format_time(seconds: float) -> str:
|
||||
total = int(round(seconds))
|
||||
hours, rem = divmod(total, 3600)
|
||||
minutes, sec = divmod(rem, 60)
|
||||
if hours:
|
||||
return f"{hours}:{minutes:02d}:{sec:02d}"
|
||||
return f"{minutes:02d}:{sec:02d}"
|
||||
|
||||
|
||||
def get_metadata(video_path: str) -> dict:
|
||||
if shutil.which("ffprobe") is None:
|
||||
raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v", "quiet",
|
||||
"-print_format", "json",
|
||||
"-show_format",
|
||||
"-show_streams",
|
||||
video_path,
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffprobe failed: {result.stderr.strip()}")
|
||||
|
||||
data = json.loads(result.stdout or "{}")
|
||||
streams = data.get("streams", [])
|
||||
fmt = data.get("format", {})
|
||||
video_stream = next((s for s in streams if s.get("codec_type") == "video"), {})
|
||||
audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None)
|
||||
|
||||
duration = float(fmt.get("duration") or video_stream.get("duration") or 0)
|
||||
return {
|
||||
"duration_seconds": duration,
|
||||
"width": video_stream.get("width"),
|
||||
"height": video_stream.get("height"),
|
||||
"codec": video_stream.get("codec_name"),
|
||||
"size_bytes": int(fmt.get("size") or 0),
|
||||
"has_audio": audio_stream is not None,
|
||||
}
|
||||
|
||||
|
||||
def auto_fps(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
|
||||
"""Pick fps that targets a sensible frame budget for full-video scans."""
|
||||
if duration_seconds <= 0:
|
||||
return 1.0, 1
|
||||
|
||||
if duration_seconds <= 30:
|
||||
target = min(max_frames, max(12, int(round(duration_seconds))))
|
||||
elif duration_seconds <= 60:
|
||||
target = min(max_frames, 40)
|
||||
elif duration_seconds <= 180: # 3 min
|
||||
target = min(max_frames, 60)
|
||||
elif duration_seconds <= 600: # 10 min
|
||||
target = min(max_frames, 80)
|
||||
else:
|
||||
target = max_frames
|
||||
|
||||
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
|
||||
|
||||
|
||||
def auto_fps_focus(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
|
||||
"""Denser budget for user-specified ranges — they are zooming in for detail."""
|
||||
if duration_seconds <= 0:
|
||||
return min(MAX_FPS, 2.0), 2
|
||||
|
||||
if duration_seconds <= 5:
|
||||
target = min(max_frames, max(10, int(round(duration_seconds * 6))))
|
||||
elif duration_seconds <= 15:
|
||||
target = min(max_frames, max(30, int(round(duration_seconds * 4))))
|
||||
elif duration_seconds <= 30:
|
||||
target = min(max_frames, 60)
|
||||
elif duration_seconds <= 60:
|
||||
target = min(max_frames, 80)
|
||||
elif duration_seconds <= 180:
|
||||
target = max_frames
|
||||
else:
|
||||
target = max_frames
|
||||
|
||||
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
|
||||
|
||||
|
||||
def extract(
|
||||
video_path: str,
|
||||
out_dir: Path,
|
||||
fps: float,
|
||||
resolution: int = 512,
|
||||
max_frames: int = 100,
|
||||
start_seconds: float | None = None,
|
||||
end_seconds: float | None = None,
|
||||
) -> list[dict]:
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for existing in out_dir.glob("frame_*.jpg"):
|
||||
existing.unlink()
|
||||
|
||||
output_pattern = str(out_dir / "frame_%04d.jpg")
|
||||
cmd: list[str] = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-y",
|
||||
]
|
||||
|
||||
# -ss before -i = fast seek (keyframe-snap, good enough for preview frames).
|
||||
if start_seconds is not None:
|
||||
cmd += ["-ss", f"{start_seconds:.3f}"]
|
||||
if end_seconds is not None:
|
||||
cmd += ["-to", f"{end_seconds:.3f}"]
|
||||
|
||||
cmd += [
|
||||
"-i", video_path,
|
||||
"-vf", f"fps={fps},scale={resolution}:-2",
|
||||
"-frames:v", str(max_frames),
|
||||
"-q:v", "4",
|
||||
output_pattern,
|
||||
]
|
||||
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffmpeg frame extraction failed: {result.stderr.strip()}")
|
||||
|
||||
offset = start_seconds or 0.0
|
||||
frames = sorted(out_dir.glob("frame_*.jpg"))
|
||||
return [
|
||||
{
|
||||
"index": i,
|
||||
"timestamp_seconds": round(offset + (i / fps if fps > 0 else 0.0), 2),
|
||||
"path": str(p),
|
||||
}
|
||||
for i, p in enumerate(frames)
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 3:
|
||||
print(
|
||||
"usage: frames.py <video-path> <out-dir> [--fps F] [--resolution W] "
|
||||
"[--max-frames N] [--start T] [--end T]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(2)
|
||||
|
||||
video = sys.argv[1]
|
||||
out = Path(sys.argv[2])
|
||||
args = sys.argv[3:]
|
||||
|
||||
fps_override = None
|
||||
resolution = 512
|
||||
max_frames = 100
|
||||
start_arg = None
|
||||
end_arg = None
|
||||
i = 0
|
||||
while i < len(args):
|
||||
if args[i] == "--fps":
|
||||
fps_override = float(args[i + 1]); i += 2
|
||||
elif args[i] == "--resolution":
|
||||
resolution = int(args[i + 1]); i += 2
|
||||
elif args[i] == "--max-frames":
|
||||
max_frames = int(args[i + 1]); i += 2
|
||||
elif args[i] == "--start":
|
||||
start_arg = args[i + 1]; i += 2
|
||||
elif args[i] == "--end":
|
||||
end_arg = args[i + 1]; i += 2
|
||||
else:
|
||||
i += 1
|
||||
|
||||
meta = get_metadata(video)
|
||||
start_sec = parse_time(start_arg)
|
||||
end_sec = parse_time(end_arg)
|
||||
full_duration = meta["duration_seconds"]
|
||||
|
||||
effective_start = start_sec if start_sec is not None else 0.0
|
||||
effective_end = end_sec if end_sec is not None else full_duration
|
||||
effective_duration = max(0.0, effective_end - effective_start)
|
||||
|
||||
focused = start_sec is not None or end_sec is not None
|
||||
if focused:
|
||||
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
|
||||
else:
|
||||
fps, target = auto_fps(effective_duration, max_frames=max_frames)
|
||||
if fps_override is not None:
|
||||
fps = fps_override
|
||||
target = max(1, int(round(fps * effective_duration)))
|
||||
|
||||
frames = extract(
|
||||
video, out,
|
||||
fps=fps,
|
||||
resolution=resolution,
|
||||
max_frames=max_frames,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
print(json.dumps(
|
||||
{"meta": meta, "fps": fps, "target": target, "focused": focused, "frames": frames},
|
||||
indent=2,
|
||||
))
|
||||
Executable
+312
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Setup / preflight for /watch.
|
||||
|
||||
Modes:
|
||||
setup.py --check Silent preflight. Exit 0 if ready, 2/3/4 on failure.
|
||||
setup.py --json Machine-readable status for Claude to parse.
|
||||
setup.py Installer. Auto-installs deps, scaffolds .env, marks SETUP_COMPLETE.
|
||||
|
||||
Design:
|
||||
- Silent on success: --check exits 0 with no output when everything's ready so
|
||||
that /watch doesn't spam "setup is complete" on every turn.
|
||||
- Idempotent: re-running the installer is safe — it never clobbers existing
|
||||
keys and only appends missing ones.
|
||||
- SETUP_COMPLETE=true in ~/.config/watch/.env tells us the user has been
|
||||
through a successful installer run at least once.
|
||||
- Never sudo. On macOS, auto-install via brew. Elsewhere, print exact commands.
|
||||
- Never write an API key to disk automatically — only scaffold placeholders.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REQUIRED_BINARIES = ["ffmpeg", "ffprobe", "yt-dlp"]
|
||||
CONFIG_DIR = Path.home() / ".config" / "watch"
|
||||
CONFIG_FILE = CONFIG_DIR / ".env"
|
||||
ENV_TEMPLATE = """# /watch API configuration
|
||||
#
|
||||
# Whisper transcription fallback — used only when yt-dlp cannot get captions
|
||||
# (or when you point /watch at a local file with no subtitles).
|
||||
#
|
||||
# Groq is preferred: it runs whisper-large-v3 at a fraction of OpenAI's price
|
||||
# and is faster in practice. OpenAI is the compatible fallback.
|
||||
#
|
||||
# Get a Groq key: https://console.groq.com/keys
|
||||
# Get an OpenAI key: https://platform.openai.com/api-keys
|
||||
#
|
||||
# Leave both blank to disable Whisper — /watch will still work, but videos
|
||||
# without native captions will come back frames-only.
|
||||
|
||||
GROQ_API_KEY=
|
||||
OPENAI_API_KEY=
|
||||
"""
|
||||
|
||||
|
||||
def _which(name: str) -> str | None:
|
||||
return shutil.which(name)
|
||||
|
||||
|
||||
def _check_binaries() -> list[str]:
|
||||
return [b for b in REQUIRED_BINARIES if not _which(b)]
|
||||
|
||||
|
||||
def _check_file_permissions(path: Path) -> None:
|
||||
"""Warn to stderr if a secrets file is world/group readable."""
|
||||
try:
|
||||
mode = path.stat().st_mode
|
||||
if mode & 0o044:
|
||||
sys.stderr.write(
|
||||
f"[watch] WARNING: {path} is readable by other users. "
|
||||
f"Run: chmod 600 {path}\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _read_env_key(name: str) -> str | None:
|
||||
value = os.environ.get(name)
|
||||
if value and value.strip():
|
||||
return value.strip()
|
||||
if not CONFIG_FILE.exists():
|
||||
return None
|
||||
_check_file_permissions(CONFIG_FILE)
|
||||
try:
|
||||
for line in CONFIG_FILE.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, raw = line.partition("=")
|
||||
if key.strip() != name:
|
||||
continue
|
||||
raw = raw.strip()
|
||||
if len(raw) >= 2 and raw[0] in ('"', "'") and raw[-1] == raw[0]:
|
||||
raw = raw[1:-1]
|
||||
return raw or None
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _have_api_key() -> tuple[bool, str | None]:
|
||||
if _read_env_key("GROQ_API_KEY"):
|
||||
return True, "groq"
|
||||
if _read_env_key("OPENAI_API_KEY"):
|
||||
return True, "openai"
|
||||
return False, None
|
||||
|
||||
|
||||
def is_first_run() -> bool:
|
||||
"""True if the installer hasn't completed successfully yet."""
|
||||
return _read_env_key("SETUP_COMPLETE") != "true"
|
||||
|
||||
|
||||
def _scaffold_env() -> bool:
|
||||
"""Create ~/.config/watch/.env with placeholders if missing."""
|
||||
if CONFIG_FILE.exists():
|
||||
return False
|
||||
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
CONFIG_FILE.write_text(ENV_TEMPLATE)
|
||||
try:
|
||||
CONFIG_FILE.chmod(0o600)
|
||||
except OSError:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def _write_setup_complete() -> None:
|
||||
"""Idempotently append SETUP_COMPLETE=true to .env.
|
||||
|
||||
Used only after a fully successful install (deps + key). Future sessions
|
||||
detect this marker to skip wizard-style UI and stay silent.
|
||||
"""
|
||||
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
existing = ""
|
||||
if CONFIG_FILE.exists():
|
||||
existing = CONFIG_FILE.read_text()
|
||||
for line in existing.splitlines():
|
||||
if line.strip().startswith("SETUP_COMPLETE="):
|
||||
return
|
||||
if existing and not existing.endswith("\n"):
|
||||
existing += "\n"
|
||||
CONFIG_FILE.write_text(existing + "SETUP_COMPLETE=true\n")
|
||||
else:
|
||||
CONFIG_FILE.write_text(ENV_TEMPLATE + "\nSETUP_COMPLETE=true\n")
|
||||
try:
|
||||
CONFIG_FILE.chmod(0o600)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _brew_pkg(missing: list[str]) -> list[str]:
|
||||
pkgs: list[str] = []
|
||||
for bin_name in missing:
|
||||
if bin_name in ("ffmpeg", "ffprobe"):
|
||||
if "ffmpeg" not in pkgs:
|
||||
pkgs.append("ffmpeg")
|
||||
elif bin_name == "yt-dlp":
|
||||
if "yt-dlp" not in pkgs:
|
||||
pkgs.append("yt-dlp")
|
||||
else:
|
||||
pkgs.append(bin_name)
|
||||
return pkgs
|
||||
|
||||
|
||||
def _install_macos(missing: list[str]) -> tuple[bool, str]:
|
||||
if _which("brew") is None:
|
||||
return False, (
|
||||
"Homebrew is not installed. Install it from https://brew.sh, then re-run setup. "
|
||||
"Or install manually: `brew install " + " ".join(_brew_pkg(missing)) + "`"
|
||||
)
|
||||
pkgs = _brew_pkg(missing)
|
||||
if not pkgs:
|
||||
return True, "nothing to install"
|
||||
cmd = ["brew", "install", *pkgs]
|
||||
print(f"[setup] running: {' '.join(cmd)}", file=sys.stderr)
|
||||
result = subprocess.run(cmd)
|
||||
if result.returncode != 0:
|
||||
return False, f"brew install failed with exit code {result.returncode}"
|
||||
return True, f"installed via brew: {', '.join(pkgs)}"
|
||||
|
||||
|
||||
def _install_hint_linux(missing: list[str]) -> str:
|
||||
pkgs = _brew_pkg(missing)
|
||||
hints = []
|
||||
if "ffmpeg" in pkgs:
|
||||
hints.append("apt: `sudo apt install ffmpeg` or dnf: `sudo dnf install ffmpeg`")
|
||||
if "yt-dlp" in pkgs:
|
||||
hints.append("`pipx install yt-dlp` (recommended) or `pip install --user yt-dlp`")
|
||||
return "\n ".join(hints) if hints else "nothing to install"
|
||||
|
||||
|
||||
def _status() -> dict:
|
||||
"""Structured preflight snapshot."""
|
||||
missing = _check_binaries()
|
||||
has_key, backend = _have_api_key()
|
||||
|
||||
if not missing and has_key:
|
||||
status = "ready"
|
||||
elif missing and not has_key:
|
||||
status = "needs_install_and_key"
|
||||
elif missing:
|
||||
status = "needs_install"
|
||||
else:
|
||||
status = "needs_key"
|
||||
|
||||
return {
|
||||
"status": status,
|
||||
"first_run": is_first_run(),
|
||||
"missing_binaries": missing,
|
||||
"whisper_backend": backend,
|
||||
"has_api_key": has_key,
|
||||
"config_file": str(CONFIG_FILE),
|
||||
"platform": platform.system(),
|
||||
}
|
||||
|
||||
|
||||
def cmd_check() -> int:
|
||||
"""Silent-on-success preflight.
|
||||
|
||||
Exit 0 with no output when ready. On failure, print one actionable line
|
||||
to stderr and return:
|
||||
2 → binaries missing
|
||||
3 → API key missing
|
||||
4 → both missing
|
||||
"""
|
||||
s = _status()
|
||||
if s["status"] == "ready":
|
||||
return 0
|
||||
|
||||
parts = []
|
||||
if s["missing_binaries"]:
|
||||
parts.append(f"missing binaries: {', '.join(s['missing_binaries'])}")
|
||||
if not s["has_api_key"]:
|
||||
parts.append("no Whisper API key (GROQ_API_KEY or OPENAI_API_KEY)")
|
||||
installer = Path(__file__).resolve()
|
||||
sys.stderr.write(
|
||||
f"[watch] setup incomplete ({'; '.join(parts)}). "
|
||||
f"Run: python3 {installer}\n"
|
||||
)
|
||||
sys.stderr.flush()
|
||||
|
||||
if s["missing_binaries"] and not s["has_api_key"]:
|
||||
return 4
|
||||
if s["missing_binaries"]:
|
||||
return 2
|
||||
return 3
|
||||
|
||||
|
||||
def cmd_json() -> int:
|
||||
json.dump(_status(), sys.stdout, indent=2)
|
||||
sys.stdout.write("\n")
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_install() -> int:
|
||||
missing = _check_binaries()
|
||||
installed_deps = False
|
||||
if missing:
|
||||
system = platform.system()
|
||||
if system == "Darwin":
|
||||
ok, msg = _install_macos(missing)
|
||||
print(f"[setup] {msg}", file=sys.stderr)
|
||||
if not ok:
|
||||
return 2
|
||||
still_missing = _check_binaries()
|
||||
if still_missing:
|
||||
print(f"[setup] still missing after install: {', '.join(still_missing)}", file=sys.stderr)
|
||||
return 2
|
||||
installed_deps = True
|
||||
elif system == "Linux":
|
||||
print("[setup] dependencies missing on Linux — please install:", file=sys.stderr)
|
||||
print(" " + _install_hint_linux(missing), file=sys.stderr)
|
||||
return 2
|
||||
else:
|
||||
print(f"[setup] unsupported platform ({system}) for auto-install. Install manually:", file=sys.stderr)
|
||||
print(f" missing: {', '.join(missing)}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
created = _scaffold_env()
|
||||
if created:
|
||||
print(f"[setup] created config: {CONFIG_FILE}")
|
||||
else:
|
||||
print(f"[setup] config exists: {CONFIG_FILE}")
|
||||
|
||||
has_key, backend = _have_api_key()
|
||||
if has_key:
|
||||
_write_setup_complete()
|
||||
print(f"[setup] ready. whisper backend: {backend}")
|
||||
if installed_deps:
|
||||
print("[setup] installed dependencies; /watch is fully set up.")
|
||||
return 0
|
||||
|
||||
print("")
|
||||
print("[setup] one step left: add a Whisper API key.")
|
||||
print("")
|
||||
print(f" Edit {CONFIG_FILE} and set either:")
|
||||
print(" GROQ_API_KEY=... (preferred — cheaper, faster; get one at console.groq.com/keys)")
|
||||
print(" OPENAI_API_KEY=... (fallback; get one at platform.openai.com/api-keys)")
|
||||
print("")
|
||||
print(" Without a key, /watch still works but videos without captions come back frames-only.")
|
||||
return 3
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) > 1:
|
||||
arg = sys.argv[1]
|
||||
if arg == "--check":
|
||||
return cmd_check()
|
||||
if arg == "--json":
|
||||
return cmd_json()
|
||||
return cmd_install()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+96
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse a WebVTT subtitle file into a clean, timestamped transcript.
|
||||
|
||||
YouTube auto-subs emit rolling-duplicate cues (each line appears 2-3 times as it
|
||||
scrolls). We dedupe consecutive identical cues and merge their time ranges.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
TS_RE = re.compile(
|
||||
r"(\d{2}):(\d{2}):(\d{2})[.,](\d{3})\s+-->\s+(\d{2}):(\d{2}):(\d{2})[.,](\d{3})"
|
||||
)
|
||||
TAG_RE = re.compile(r"<[^>]+>")
|
||||
|
||||
|
||||
def _to_seconds(h: str, m: str, s: str, ms: str) -> float:
|
||||
return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000.0
|
||||
|
||||
|
||||
def parse_vtt(path: str) -> list[dict]:
|
||||
text = Path(path).read_text(encoding="utf-8", errors="ignore")
|
||||
lines = text.splitlines()
|
||||
|
||||
segments: list[dict] = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
match = TS_RE.match(lines[i])
|
||||
if not match:
|
||||
i += 1
|
||||
continue
|
||||
|
||||
start = _to_seconds(*match.groups()[:4])
|
||||
end = _to_seconds(*match.groups()[4:])
|
||||
i += 1
|
||||
|
||||
cue_lines: list[str] = []
|
||||
while i < len(lines) and lines[i].strip():
|
||||
cleaned = TAG_RE.sub("", lines[i]).strip()
|
||||
if cleaned:
|
||||
cue_lines.append(cleaned)
|
||||
i += 1
|
||||
|
||||
cue_text = " ".join(cue_lines).strip()
|
||||
if cue_text:
|
||||
segments.append({"start": round(start, 2), "end": round(end, 2), "text": cue_text})
|
||||
i += 1
|
||||
|
||||
return _dedupe(segments)
|
||||
|
||||
|
||||
def _dedupe(segments: list[dict]) -> list[dict]:
|
||||
"""Collapse rolling duplicates common in YouTube auto-subs."""
|
||||
out: list[dict] = []
|
||||
for seg in segments:
|
||||
if out and seg["text"] == out[-1]["text"]:
|
||||
out[-1]["end"] = seg["end"]
|
||||
continue
|
||||
if out and seg["text"].startswith(out[-1]["text"] + " "):
|
||||
out[-1]["text"] = seg["text"]
|
||||
out[-1]["end"] = seg["end"]
|
||||
continue
|
||||
out.append(seg)
|
||||
return out
|
||||
|
||||
|
||||
def filter_range(
|
||||
segments: list[dict],
|
||||
start_seconds: float | None,
|
||||
end_seconds: float | None,
|
||||
) -> list[dict]:
|
||||
"""Return segments whose time range overlaps [start, end]."""
|
||||
if start_seconds is None and end_seconds is None:
|
||||
return segments
|
||||
lo = start_seconds if start_seconds is not None else float("-inf")
|
||||
hi = end_seconds if end_seconds is not None else float("inf")
|
||||
return [seg for seg in segments if seg["end"] >= lo and seg["start"] <= hi]
|
||||
|
||||
|
||||
def format_transcript(segments: list[dict]) -> str:
|
||||
lines = []
|
||||
for seg in segments:
|
||||
start = int(seg["start"])
|
||||
stamp = f"[{start // 60:02d}:{start % 60:02d}]"
|
||||
lines.append(f"{stamp} {seg['text']}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("usage: transcribe.py <vtt-path>", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
print(format_transcript(parse_vtt(sys.argv[1])))
|
||||
Executable
+230
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
"""/watch entry point: download video, extract frames, parse transcript.
|
||||
|
||||
Prints a markdown report to stdout listing frame paths + transcript. Claude
|
||||
then Reads each frame path to see the video.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from download import download, is_url # noqa: E402
|
||||
from frames import MAX_FPS, auto_fps, auto_fps_focus, extract, format_time, get_metadata, parse_time # noqa: E402
|
||||
from transcribe import filter_range, format_transcript, parse_vtt # noqa: E402
|
||||
from whisper import load_api_key, transcribe_video # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(
|
||||
prog="watch",
|
||||
description="Download a video, extract auto-scaled frames, and surface the transcript.",
|
||||
)
|
||||
ap.add_argument("source", help="Video URL or local file path")
|
||||
ap.add_argument("--max-frames", type=int, default=80, help="Cap on frame count (default 80, hard max 100)")
|
||||
ap.add_argument("--resolution", type=int, default=512, help="Frame width in pixels (default 512)")
|
||||
ap.add_argument("--fps", type=float, default=None, help="Override auto-fps")
|
||||
ap.add_argument("--start", type=str, default=None, help="Range start (SS, MM:SS, or HH:MM:SS)")
|
||||
ap.add_argument("--end", type=str, default=None, help="Range end (SS, MM:SS, or HH:MM:SS)")
|
||||
ap.add_argument("--out-dir", type=str, default=None, help="Working directory (default: tmp)")
|
||||
ap.add_argument(
|
||||
"--no-whisper",
|
||||
action="store_true",
|
||||
help="Disable Whisper fallback. Report frames-only if no captions available.",
|
||||
)
|
||||
ap.add_argument(
|
||||
"--whisper",
|
||||
choices=["groq", "openai"],
|
||||
default=None,
|
||||
help="Force a specific Whisper backend. Default: prefer Groq, fall back to OpenAI.",
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
max_frames = min(args.max_frames, 100)
|
||||
|
||||
if args.out_dir:
|
||||
work = Path(args.out_dir).expanduser().resolve()
|
||||
else:
|
||||
work = Path(tempfile.mkdtemp(prefix="watch-"))
|
||||
work.mkdir(parents=True, exist_ok=True)
|
||||
print(f"[watch] working dir: {work}", file=sys.stderr)
|
||||
|
||||
print(
|
||||
"[watch] downloading via yt-dlp…" if is_url(args.source) else "[watch] using local file…",
|
||||
file=sys.stderr,
|
||||
)
|
||||
dl = download(args.source, work / "download")
|
||||
video_path = dl["video_path"]
|
||||
|
||||
meta = get_metadata(video_path)
|
||||
full_duration = meta["duration_seconds"]
|
||||
|
||||
start_sec = parse_time(args.start)
|
||||
end_sec = parse_time(args.end)
|
||||
|
||||
if start_sec is not None and start_sec < 0:
|
||||
raise SystemExit("--start must be non-negative")
|
||||
if end_sec is not None and start_sec is not None and end_sec <= start_sec:
|
||||
raise SystemExit("--end must be greater than --start")
|
||||
if full_duration > 0 and start_sec is not None and start_sec >= full_duration:
|
||||
raise SystemExit(f"--start {start_sec:.1f}s is past end of video ({full_duration:.1f}s)")
|
||||
|
||||
effective_start = start_sec if start_sec is not None else 0.0
|
||||
effective_end = end_sec if end_sec is not None else full_duration
|
||||
effective_duration = max(0.0, effective_end - effective_start)
|
||||
focused = start_sec is not None or end_sec is not None
|
||||
|
||||
if focused:
|
||||
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
|
||||
else:
|
||||
fps, target = auto_fps(effective_duration, max_frames=max_frames)
|
||||
if args.fps is not None:
|
||||
fps = min(args.fps, MAX_FPS)
|
||||
target = max(1, int(round(fps * effective_duration)))
|
||||
|
||||
scope = (
|
||||
f"{format_time(effective_start)}-{format_time(effective_end)} ({effective_duration:.1f}s)"
|
||||
if focused else f"full {effective_duration:.1f}s"
|
||||
)
|
||||
print(f"[watch] extracting ~{target} frames at {fps:.3f} fps over {scope}…", file=sys.stderr)
|
||||
|
||||
frames = extract(
|
||||
video_path,
|
||||
work / "frames",
|
||||
fps=fps,
|
||||
resolution=args.resolution,
|
||||
max_frames=max_frames,
|
||||
start_seconds=start_sec,
|
||||
end_seconds=end_sec,
|
||||
)
|
||||
|
||||
transcript_segments: list[dict] = []
|
||||
transcript_text: str | None = None
|
||||
transcript_source: str | None = None
|
||||
if dl.get("subtitle_path"):
|
||||
try:
|
||||
all_segments = parse_vtt(dl["subtitle_path"])
|
||||
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = "captions"
|
||||
except Exception as exc:
|
||||
print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr)
|
||||
|
||||
if not transcript_segments and not args.no_whisper:
|
||||
backend, api_key = load_api_key(args.whisper)
|
||||
if backend and api_key:
|
||||
try:
|
||||
all_segments, used_backend = transcribe_video(
|
||||
video_path,
|
||||
work / "audio.mp3",
|
||||
backend=backend,
|
||||
api_key=api_key,
|
||||
)
|
||||
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
|
||||
transcript_text = format_transcript(transcript_segments)
|
||||
transcript_source = f"whisper ({used_backend})"
|
||||
except SystemExit as exc:
|
||||
print(f"[watch] whisper fallback failed: {exc}", file=sys.stderr)
|
||||
else:
|
||||
hint = (
|
||||
f"--whisper {args.whisper} was set but the matching API key is missing"
|
||||
if args.whisper else
|
||||
"no subtitles and no Whisper API key found"
|
||||
)
|
||||
setup_py = SCRIPT_DIR / "setup.py"
|
||||
print(
|
||||
f"[watch] {hint} — run `python3 {setup_py}` to enable the Whisper fallback",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
info = dl.get("info") or {}
|
||||
|
||||
print()
|
||||
print("# watch: video report")
|
||||
print()
|
||||
print(f"- **Source:** {args.source}")
|
||||
if info.get("title"):
|
||||
print(f"- **Title:** {info['title']}")
|
||||
if info.get("uploader"):
|
||||
print(f"- **Uploader:** {info['uploader']}")
|
||||
print(f"- **Duration:** {format_time(full_duration)} ({full_duration:.1f}s)")
|
||||
if focused:
|
||||
print(
|
||||
f"- **Focus range:** {format_time(effective_start)} → {format_time(effective_end)} "
|
||||
f"({effective_duration:.1f}s)"
|
||||
)
|
||||
if meta.get("width") and meta.get("height"):
|
||||
print(f"- **Resolution:** {meta['width']}x{meta['height']} ({meta.get('codec') or 'unknown codec'})")
|
||||
mode = "focused" if focused else "full"
|
||||
print(f"- **Frames:** {len(frames)} @ {fps:.3f} fps, {mode} mode (budget {target}, max {max_frames})")
|
||||
print(f"- **Frame size:** {args.resolution}px wide")
|
||||
if transcript_segments:
|
||||
in_range = " in range" if focused else ""
|
||||
print(
|
||||
f"- **Transcript:** {len(transcript_segments)} segments{in_range} "
|
||||
f"(via {transcript_source or 'captions'})"
|
||||
)
|
||||
else:
|
||||
print("- **Transcript:** none available")
|
||||
|
||||
if not focused and full_duration > 600:
|
||||
mins = int(full_duration // 60)
|
||||
print()
|
||||
print(
|
||||
f"> ⚠️ This is a {mins}-minute video. Frame coverage is sparse at this length — "
|
||||
"accuracy degrades noticeably on anything over 10 minutes. For better results, "
|
||||
"re-run with `--start HH:MM:SS --end HH:MM:SS` to zoom into a specific section."
|
||||
)
|
||||
|
||||
print()
|
||||
print("## Frames")
|
||||
print()
|
||||
print(f"Frames live at: `{work / 'frames'}`")
|
||||
print()
|
||||
print(
|
||||
"**Read each frame path below with the Read tool to view the image.** "
|
||||
"Frames are in chronological order; `t=MM:SS` is the absolute timestamp in the source video."
|
||||
)
|
||||
print()
|
||||
for frame in frames:
|
||||
print(f"- `{frame['path']}` (t={format_time(frame['timestamp_seconds'])})")
|
||||
|
||||
print()
|
||||
print("## Transcript")
|
||||
print()
|
||||
if transcript_text:
|
||||
label = transcript_source or "captions"
|
||||
if focused:
|
||||
print(f"_Source: {label}. Filtered to {format_time(effective_start)} → {format_time(effective_end)}:_")
|
||||
else:
|
||||
print(f"_Source: {label}._")
|
||||
print()
|
||||
print("```")
|
||||
print(transcript_text)
|
||||
print("```")
|
||||
elif focused and dl.get("subtitle_path"):
|
||||
print(f"_No transcript lines fell inside {format_time(effective_start)} → {format_time(effective_end)}._")
|
||||
else:
|
||||
setup_py = SCRIPT_DIR / "setup.py"
|
||||
print(
|
||||
"_No transcript available — proceed with frames only. "
|
||||
"Captions were missing and the Whisper fallback was unavailable "
|
||||
"(no API key set, or `--no-whisper` was used). "
|
||||
f"Run `python3 {setup_py}` to enable Whisper, then re-run._"
|
||||
)
|
||||
|
||||
print()
|
||||
print("---")
|
||||
print(f"_Work dir: `{work}` — delete when done._")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+319
@@ -0,0 +1,319 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Transcribe a video via Groq or OpenAI Whisper API.
|
||||
|
||||
Strategy: extract audio (mono 16kHz mp3, tiny payload), upload to whichever
|
||||
API has a key. Returns segments in the same shape as transcribe.parse_vtt so
|
||||
the rest of the pipeline (filter_range, format_transcript) doesn't care where
|
||||
the transcript came from.
|
||||
|
||||
Pure stdlib — no `pip install groq` or `pip install openai` needed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import shutil
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
GROQ_ENDPOINT = "https://api.groq.com/openai/v1/audio/transcriptions"
|
||||
GROQ_MODEL = "whisper-large-v3"
|
||||
|
||||
OPENAI_ENDPOINT = "https://api.openai.com/v1/audio/transcriptions"
|
||||
OPENAI_MODEL = "whisper-1"
|
||||
|
||||
|
||||
def load_api_key(preferred: str | None = None) -> tuple[str, str] | tuple[None, None]:
|
||||
"""Return (backend, api_key). Prefers Groq, falls back to OpenAI.
|
||||
|
||||
If `preferred` is "groq" or "openai", only that backend's key is considered.
|
||||
"""
|
||||
def _from_env(name: str) -> str | None:
|
||||
value = os.environ.get(name)
|
||||
return value.strip() if value else None
|
||||
|
||||
def _from_dotenv(path: Path, name: str) -> str | None:
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, _, value = line.partition("=")
|
||||
if key.strip() != name:
|
||||
continue
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] in ('"', "'") and value[-1] == value[0]:
|
||||
value = value[1:-1]
|
||||
return value or None
|
||||
except OSError:
|
||||
return None
|
||||
return None
|
||||
|
||||
dotenv_paths = [
|
||||
Path.home() / ".config" / "watch" / ".env",
|
||||
Path.cwd() / ".env",
|
||||
]
|
||||
|
||||
candidates = (("GROQ_API_KEY", "groq"), ("OPENAI_API_KEY", "openai"))
|
||||
if preferred is not None:
|
||||
candidates = tuple(c for c in candidates if c[1] == preferred)
|
||||
|
||||
for key_name, backend in candidates:
|
||||
value = _from_env(key_name)
|
||||
if not value:
|
||||
for candidate in dotenv_paths:
|
||||
value = _from_dotenv(candidate, key_name)
|
||||
if value:
|
||||
break
|
||||
if value:
|
||||
return backend, value
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def extract_audio(video_path: str, out_path: Path) -> Path:
|
||||
"""Extract mono 16kHz 64kbps mp3 — ~480 kB/min, fits any Whisper limit."""
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-hide_banner",
|
||||
"-loglevel", "error",
|
||||
"-y",
|
||||
"-i", video_path,
|
||||
"-vn",
|
||||
"-acodec", "libmp3lame",
|
||||
"-ar", "16000",
|
||||
"-ac", "1",
|
||||
"-b:a", "64k",
|
||||
str(out_path),
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise SystemExit(f"ffmpeg audio extraction failed: {result.stderr.strip()}")
|
||||
if not out_path.exists() or out_path.stat().st_size == 0:
|
||||
raise SystemExit("ffmpeg produced no audio — video may have no audio track")
|
||||
return out_path
|
||||
|
||||
|
||||
def _build_multipart(fields: dict[str, str], file_path: Path) -> tuple[bytes, str]:
|
||||
"""Assemble a multipart/form-data body the Whisper APIs accept.
|
||||
|
||||
Whisper's multipart upload is small and predictable — doing it by hand
|
||||
keeps us on pure stdlib instead of pulling requests/groq/openai SDKs.
|
||||
"""
|
||||
boundary = f"----WatchBoundary{uuid.uuid4().hex}"
|
||||
eol = b"\r\n"
|
||||
buf = io.BytesIO()
|
||||
|
||||
for name, value in fields.items():
|
||||
buf.write(f"--{boundary}".encode()); buf.write(eol)
|
||||
buf.write(f'Content-Disposition: form-data; name="{name}"'.encode()); buf.write(eol)
|
||||
buf.write(eol)
|
||||
buf.write(str(value).encode()); buf.write(eol)
|
||||
|
||||
mimetype = mimetypes.guess_type(file_path.name)[0] or "application/octet-stream"
|
||||
buf.write(f"--{boundary}".encode()); buf.write(eol)
|
||||
buf.write(
|
||||
f'Content-Disposition: form-data; name="file"; filename="{file_path.name}"'.encode()
|
||||
)
|
||||
buf.write(eol)
|
||||
buf.write(f"Content-Type: {mimetype}".encode()); buf.write(eol)
|
||||
buf.write(eol)
|
||||
buf.write(file_path.read_bytes())
|
||||
buf.write(eol)
|
||||
buf.write(f"--{boundary}--".encode()); buf.write(eol)
|
||||
|
||||
return buf.getvalue(), boundary
|
||||
|
||||
|
||||
MAX_ATTEMPTS = 4 # initial + 3 retries
|
||||
MAX_429_RETRIES = 2
|
||||
RETRY_BASE_DELAY = 2.0
|
||||
|
||||
|
||||
def _post_whisper(endpoint: str, api_key: str, model: str, audio_path: Path) -> dict:
|
||||
fields = {
|
||||
"model": model,
|
||||
"response_format": "verbose_json",
|
||||
"temperature": "0",
|
||||
}
|
||||
body, boundary = _build_multipart(fields, audio_path)
|
||||
headers = {
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": f"multipart/form-data; boundary={boundary}",
|
||||
# Groq sits behind Cloudflare — the default `Python-urllib/3.x` UA
|
||||
# trips WAF rule 1010 (403) before auth even runs. Any non-default
|
||||
# UA clears it; we identify honestly.
|
||||
"User-Agent": "watch-skill/1.0 (+claude-code; python-urllib)",
|
||||
}
|
||||
|
||||
context = ssl.create_default_context()
|
||||
rate_limit_hits = 0
|
||||
last_exc: Exception | None = None
|
||||
last_detail = ""
|
||||
|
||||
for attempt in range(MAX_ATTEMPTS):
|
||||
request = Request(endpoint, data=body, headers=headers, method="POST")
|
||||
try:
|
||||
with urlopen(request, timeout=300, context=context) as response:
|
||||
payload = response.read().decode("utf-8", errors="replace")
|
||||
except urllib.error.HTTPError as exc:
|
||||
detail = _read_error_body(exc)
|
||||
last_exc, last_detail = exc, detail
|
||||
|
||||
# 4xx other than 429 are client errors — no retry will fix them.
|
||||
if 400 <= exc.code < 500 and exc.code != 429:
|
||||
raise SystemExit(f"Whisper request failed: {exc}{detail}")
|
||||
|
||||
if exc.code == 429:
|
||||
rate_limit_hits += 1
|
||||
if rate_limit_hits >= MAX_429_RETRIES:
|
||||
raise SystemExit(f"Whisper request failed: {exc}{detail}")
|
||||
delay = _retry_after(exc) or RETRY_BASE_DELAY * (2 ** attempt) + 1
|
||||
else:
|
||||
delay = RETRY_BASE_DELAY * (2 ** attempt)
|
||||
|
||||
if attempt < MAX_ATTEMPTS - 1:
|
||||
print(
|
||||
f"[watch] whisper HTTP {exc.code} — retrying in {delay:.1f}s "
|
||||
f"(attempt {attempt + 2}/{MAX_ATTEMPTS})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
except (urllib.error.URLError, TimeoutError, ConnectionResetError, OSError) as exc:
|
||||
last_exc, last_detail = exc, ""
|
||||
if attempt < MAX_ATTEMPTS - 1:
|
||||
delay = RETRY_BASE_DELAY * (attempt + 1)
|
||||
print(
|
||||
f"[watch] whisper network error ({type(exc).__name__}: {exc}) — "
|
||||
f"retrying in {delay:.1f}s (attempt {attempt + 2}/{MAX_ATTEMPTS})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
|
||||
try:
|
||||
return json.loads(payload)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise SystemExit(f"Whisper returned non-JSON response: {exc}: {payload[:200]}")
|
||||
|
||||
raise SystemExit(
|
||||
f"Whisper request failed after {MAX_ATTEMPTS} attempts: {last_exc}{last_detail}"
|
||||
)
|
||||
|
||||
|
||||
def _read_error_body(exc: urllib.error.HTTPError) -> str:
|
||||
try:
|
||||
body = exc.read()
|
||||
except Exception:
|
||||
return ""
|
||||
if not body:
|
||||
return ""
|
||||
try:
|
||||
return f" — {body.decode('utf-8', errors='replace')[:400]}"
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _retry_after(exc: urllib.error.HTTPError) -> float | None:
|
||||
header = exc.headers.get("Retry-After") if getattr(exc, "headers", None) else None
|
||||
if not header:
|
||||
return None
|
||||
try:
|
||||
return float(header)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _segments_from_response(data: dict) -> list[dict]:
|
||||
"""Convert Whisper verbose_json into our {start, end, text} segment format."""
|
||||
out: list[dict] = []
|
||||
for seg in data.get("segments") or []:
|
||||
text = (seg.get("text") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
out.append({
|
||||
"start": round(float(seg.get("start") or 0.0), 2),
|
||||
"end": round(float(seg.get("end") or 0.0), 2),
|
||||
"text": text,
|
||||
})
|
||||
|
||||
if not out:
|
||||
full = (data.get("text") or "").strip()
|
||||
if full:
|
||||
out.append({"start": 0.0, "end": 0.0, "text": full})
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def transcribe_video(
|
||||
video_path: str,
|
||||
audio_out: Path,
|
||||
backend: str | None = None,
|
||||
api_key: str | None = None,
|
||||
) -> tuple[list[dict], str]:
|
||||
"""Run the full flow: extract audio → upload → parse segments.
|
||||
|
||||
Returns (segments, backend_used). Raises SystemExit on any failure.
|
||||
"""
|
||||
if backend is None or api_key is None:
|
||||
detected_backend, detected_key = load_api_key()
|
||||
backend = backend or detected_backend
|
||||
api_key = api_key or detected_key
|
||||
|
||||
if not backend or not api_key:
|
||||
setup_py = Path(__file__).resolve().parent / "setup.py"
|
||||
raise SystemExit(
|
||||
"No Whisper API key available. Set GROQ_API_KEY (preferred) or OPENAI_API_KEY "
|
||||
"in the environment or in ~/.config/watch/.env. "
|
||||
f"Run `python3 {setup_py}` to configure."
|
||||
)
|
||||
|
||||
print(f"[watch] extracting audio for Whisper ({backend})…", file=sys.stderr)
|
||||
audio_path = extract_audio(video_path, audio_out)
|
||||
size_kb = audio_path.stat().st_size / 1024
|
||||
print(f"[watch] audio: {size_kb:.0f} kB — uploading to {backend} Whisper…", file=sys.stderr)
|
||||
|
||||
if backend == "groq":
|
||||
response = _post_whisper(GROQ_ENDPOINT, api_key, GROQ_MODEL, audio_path)
|
||||
elif backend == "openai":
|
||||
response = _post_whisper(OPENAI_ENDPOINT, api_key, OPENAI_MODEL, audio_path)
|
||||
else:
|
||||
raise SystemExit(f"Unknown whisper backend: {backend}")
|
||||
|
||||
segments = _segments_from_response(response)
|
||||
if not segments:
|
||||
raise SystemExit("Whisper returned no transcript segments")
|
||||
|
||||
print(f"[watch] transcribed {len(segments)} segments via {backend}", file=sys.stderr)
|
||||
return segments, backend
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) < 2:
|
||||
print("usage: whisper.py <video-path> [<audio-out.mp3>] [--backend groq|openai]", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
|
||||
video = sys.argv[1]
|
||||
audio_out = Path(sys.argv[2]) if len(sys.argv) > 2 and not sys.argv[2].startswith("--") else Path("audio.mp3")
|
||||
backend_override = None
|
||||
if "--backend" in sys.argv:
|
||||
backend_override = sys.argv[sys.argv.index("--backend") + 1]
|
||||
|
||||
segments, backend = transcribe_video(video, audio_out, backend=backend_override)
|
||||
print(json.dumps({"backend": backend, "segments": segments}, indent=2))
|
||||
Reference in New Issue
Block a user