Initial commit: /watch skill v0.1.0

Give Claude the ability to watch any video — yt-dlp download, ffmpeg
frame extraction with auto-scaled fps, native-caption transcript with
Whisper (Groq/OpenAI) fallback.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
bradautomates
2026-04-24 14:40:34 +10:00
commit 29b8a2988a
19 changed files with 1976 additions and 0 deletions
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env bash
# build-skill.sh — package this repo as a claude.ai-upload-ready .skill file.
# Usage: bash scripts/build-skill.sh (run from repo root)
#
# Produces dist/watch.skill, a zip with a single top-level `watch/` directory
# containing SKILL.md and the scripts/ runtime. claude.ai's skill upload has a
# 200-file cap; `export-ignore` in .gitattributes + the zip -d strips below
# keep the bundle lean.
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
cd "$REPO_ROOT"
if ! git diff --quiet || ! git diff --cached --quiet; then
echo "error: working tree is dirty; commit or stash before building" >&2
exit 1
fi
mkdir -p dist
OUT="dist/watch.skill"
git archive --format=zip --prefix=watch/ --output="$OUT" HEAD
# claude.ai's .skill bundle needs only SKILL.md + scripts/ runtime. Claude Code
# needs hooks/ and .claude-plugin/ in the git archive (that's why they are NOT
# in .gitattributes export-ignore), but the .skill bundle should strip them to
# keep a single canonical SKILL.md and stay well under the 200-file cap.
zip -d "$OUT" \
"watch/hooks/*" \
"watch/.claude-plugin/*" \
> /dev/null 2>&1 || true
COUNT=$(unzip -l "$OUT" | tail -1 | awk '{print $2}')
SIZE=$(du -h "$OUT" | cut -f1)
if [ "$COUNT" -gt 200 ]; then
echo "error: $COUNT files in zip, claude.ai's cap is 200" >&2
echo " check .gitattributes export-ignore entries and this script's zip -d excludes" >&2
exit 1
fi
SKILL_MD_COUNT=$(unzip -l "$OUT" | grep -c "SKILL.md" || true)
if [ "$SKILL_MD_COUNT" -ne 1 ]; then
echo "error: expected exactly one SKILL.md, found $SKILL_MD_COUNT" >&2
exit 1
fi
echo "built $OUT ($COUNT files, $SIZE)"
echo "upload via the claude.ai skill UI"
+127
View File
@@ -0,0 +1,127 @@
#!/usr/bin/env python3
"""Download a video via yt-dlp, or resolve a local file path.
Also fetches subtitles (manual first, then auto-generated) in VTT format so
transcribe.py can parse them without needing Whisper.
"""
from __future__ import annotations
import json
import shutil
import subprocess
import sys
from pathlib import Path
from urllib.parse import urlparse
VIDEO_EXTS = {".mp4", ".mkv", ".webm", ".mov", ".m4v", ".avi", ".flv", ".wmv"}
def is_url(source: str) -> bool:
parsed = urlparse(source)
return parsed.scheme in ("http", "https")
def resolve_local(path: str) -> dict:
p = Path(path).expanduser().resolve()
if not p.exists():
raise SystemExit(f"File not found: {p}")
if p.suffix.lower() not in VIDEO_EXTS:
print(
f"[watch] warning: {p.suffix} is not a known video extension, proceeding anyway",
file=sys.stderr,
)
return {
"video_path": str(p),
"subtitle_path": None,
"info": {"title": p.name, "url": str(p)},
"downloaded": False,
}
def _pick_subtitle(out_dir: Path) -> Path | None:
candidates = sorted(out_dir.glob("video*.vtt"))
if not candidates:
return None
preferred = [c for c in candidates if ".en" in c.name]
return preferred[0] if preferred else candidates[0]
def _pick_video(out_dir: Path) -> Path | None:
for ext in (".mp4", ".mkv", ".webm", ".mov"):
for candidate in out_dir.glob(f"video*{ext}"):
return candidate
for candidate in out_dir.glob("video.*"):
if candidate.suffix.lower() in VIDEO_EXTS:
return candidate
return None
def download_url(url: str, out_dir: Path) -> dict:
if shutil.which("yt-dlp") is None:
raise SystemExit("yt-dlp is not installed. Install with: brew install yt-dlp")
out_dir.mkdir(parents=True, exist_ok=True)
output_template = str(out_dir / "video.%(ext)s")
cmd = [
"yt-dlp",
"-N", "8",
"-f", "bv*[height<=720]+ba/b[height<=720]/bv+ba/b",
"--merge-output-format", "mp4",
"--write-info-json",
"--write-subs",
"--write-auto-subs",
"--sub-langs", "en,en-US,en-GB,en-orig",
"--sub-format", "vtt",
"--convert-subs", "vtt",
"--no-playlist",
"--ignore-errors",
"-o", output_template,
url,
]
# yt-dlp may exit non-zero if a subtitle variant fails (e.g. 429) even when
# the video itself downloaded fine. Treat "video file present" as success.
result = subprocess.run(cmd, stdout=sys.stderr, stderr=sys.stderr)
video = _pick_video(out_dir)
if video is None:
raise SystemExit(
f"yt-dlp did not produce a video file in {out_dir} (exit {result.returncode})"
)
subtitle = _pick_subtitle(out_dir)
info_path = out_dir / "video.info.json"
info: dict = {}
if info_path.exists():
try:
raw = json.loads(info_path.read_text())
info = {
"title": raw.get("title"),
"uploader": raw.get("uploader") or raw.get("channel"),
"duration": raw.get("duration"),
"url": raw.get("webpage_url") or url,
}
except Exception:
info = {"url": url}
return {
"video_path": str(video),
"subtitle_path": str(subtitle) if subtitle else None,
"info": info or {"url": url},
"downloaded": True,
}
def download(source: str, out_dir: Path) -> dict:
if is_url(source):
return download_url(source, out_dir)
return resolve_local(source)
if __name__ == "__main__":
if len(sys.argv) < 3:
print("usage: download.py <url-or-path> <out-dir>", file=sys.stderr)
raise SystemExit(2)
result = download(sys.argv[1], Path(sys.argv[2]))
print(json.dumps(result, indent=2))
+250
View File
@@ -0,0 +1,250 @@
#!/usr/bin/env python3
"""Probe video metadata and extract frames at an auto-scaled fps.
Auto-fps targets a frame budget, not a fixed rate. Token cost scales with frame
count, so budget-by-duration keeps short videos dense and long videos capped.
When a user-specified range is passed, focused-mode budgets denser (they are
zooming in for detail).
"""
from __future__ import annotations
import json
import shutil
import subprocess
import sys
from pathlib import Path
MAX_FPS = 2.0
def _clamp_fps(fps: float, duration_seconds: float, max_frames: int) -> tuple[float, int]:
fps = min(fps, MAX_FPS)
target = min(max_frames, max(1, int(round(fps * duration_seconds))))
return fps, target
def parse_time(value: str | float | int | None) -> float | None:
"""Parse SS, MM:SS, or HH:MM:SS (with optional .ms) into seconds."""
if value is None:
return None
if isinstance(value, (int, float)):
return float(value)
s = str(value).strip()
if not s:
return None
parts = s.split(":")
try:
if len(parts) == 1:
return float(parts[0])
if len(parts) == 2:
return int(parts[0]) * 60 + float(parts[1])
if len(parts) == 3:
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
except ValueError:
pass
raise SystemExit(f"Cannot parse time value: {value!r} (expected SS, MM:SS, or HH:MM:SS)")
def format_time(seconds: float) -> str:
total = int(round(seconds))
hours, rem = divmod(total, 3600)
minutes, sec = divmod(rem, 60)
if hours:
return f"{hours}:{minutes:02d}:{sec:02d}"
return f"{minutes:02d}:{sec:02d}"
def get_metadata(video_path: str) -> dict:
if shutil.which("ffprobe") is None:
raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg")
result = subprocess.run(
[
"ffprobe",
"-v", "quiet",
"-print_format", "json",
"-show_format",
"-show_streams",
video_path,
],
capture_output=True,
text=True,
)
if result.returncode != 0:
raise SystemExit(f"ffprobe failed: {result.stderr.strip()}")
data = json.loads(result.stdout or "{}")
streams = data.get("streams", [])
fmt = data.get("format", {})
video_stream = next((s for s in streams if s.get("codec_type") == "video"), {})
audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None)
duration = float(fmt.get("duration") or video_stream.get("duration") or 0)
return {
"duration_seconds": duration,
"width": video_stream.get("width"),
"height": video_stream.get("height"),
"codec": video_stream.get("codec_name"),
"size_bytes": int(fmt.get("size") or 0),
"has_audio": audio_stream is not None,
}
def auto_fps(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
"""Pick fps that targets a sensible frame budget for full-video scans."""
if duration_seconds <= 0:
return 1.0, 1
if duration_seconds <= 30:
target = min(max_frames, max(12, int(round(duration_seconds))))
elif duration_seconds <= 60:
target = min(max_frames, 40)
elif duration_seconds <= 180: # 3 min
target = min(max_frames, 60)
elif duration_seconds <= 600: # 10 min
target = min(max_frames, 80)
else:
target = max_frames
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
def auto_fps_focus(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
"""Denser budget for user-specified ranges — they are zooming in for detail."""
if duration_seconds <= 0:
return min(MAX_FPS, 2.0), 2
if duration_seconds <= 5:
target = min(max_frames, max(10, int(round(duration_seconds * 6))))
elif duration_seconds <= 15:
target = min(max_frames, max(30, int(round(duration_seconds * 4))))
elif duration_seconds <= 30:
target = min(max_frames, 60)
elif duration_seconds <= 60:
target = min(max_frames, 80)
elif duration_seconds <= 180:
target = max_frames
else:
target = max_frames
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
def extract(
video_path: str,
out_dir: Path,
fps: float,
resolution: int = 512,
max_frames: int = 100,
start_seconds: float | None = None,
end_seconds: float | None = None,
) -> list[dict]:
if shutil.which("ffmpeg") is None:
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
out_dir.mkdir(parents=True, exist_ok=True)
for existing in out_dir.glob("frame_*.jpg"):
existing.unlink()
output_pattern = str(out_dir / "frame_%04d.jpg")
cmd: list[str] = [
"ffmpeg",
"-hide_banner",
"-loglevel", "error",
"-y",
]
# -ss before -i = fast seek (keyframe-snap, good enough for preview frames).
if start_seconds is not None:
cmd += ["-ss", f"{start_seconds:.3f}"]
if end_seconds is not None:
cmd += ["-to", f"{end_seconds:.3f}"]
cmd += [
"-i", video_path,
"-vf", f"fps={fps},scale={resolution}:-2",
"-frames:v", str(max_frames),
"-q:v", "4",
output_pattern,
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
raise SystemExit(f"ffmpeg frame extraction failed: {result.stderr.strip()}")
offset = start_seconds or 0.0
frames = sorted(out_dir.glob("frame_*.jpg"))
return [
{
"index": i,
"timestamp_seconds": round(offset + (i / fps if fps > 0 else 0.0), 2),
"path": str(p),
}
for i, p in enumerate(frames)
]
if __name__ == "__main__":
if len(sys.argv) < 3:
print(
"usage: frames.py <video-path> <out-dir> [--fps F] [--resolution W] "
"[--max-frames N] [--start T] [--end T]",
file=sys.stderr,
)
raise SystemExit(2)
video = sys.argv[1]
out = Path(sys.argv[2])
args = sys.argv[3:]
fps_override = None
resolution = 512
max_frames = 100
start_arg = None
end_arg = None
i = 0
while i < len(args):
if args[i] == "--fps":
fps_override = float(args[i + 1]); i += 2
elif args[i] == "--resolution":
resolution = int(args[i + 1]); i += 2
elif args[i] == "--max-frames":
max_frames = int(args[i + 1]); i += 2
elif args[i] == "--start":
start_arg = args[i + 1]; i += 2
elif args[i] == "--end":
end_arg = args[i + 1]; i += 2
else:
i += 1
meta = get_metadata(video)
start_sec = parse_time(start_arg)
end_sec = parse_time(end_arg)
full_duration = meta["duration_seconds"]
effective_start = start_sec if start_sec is not None else 0.0
effective_end = end_sec if end_sec is not None else full_duration
effective_duration = max(0.0, effective_end - effective_start)
focused = start_sec is not None or end_sec is not None
if focused:
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
else:
fps, target = auto_fps(effective_duration, max_frames=max_frames)
if fps_override is not None:
fps = fps_override
target = max(1, int(round(fps * effective_duration)))
frames = extract(
video, out,
fps=fps,
resolution=resolution,
max_frames=max_frames,
start_seconds=start_sec,
end_seconds=end_sec,
)
print(json.dumps(
{"meta": meta, "fps": fps, "target": target, "focused": focused, "frames": frames},
indent=2,
))
+312
View File
@@ -0,0 +1,312 @@
#!/usr/bin/env python3
"""Setup / preflight for /watch.
Modes:
setup.py --check Silent preflight. Exit 0 if ready, 2/3/4 on failure.
setup.py --json Machine-readable status for Claude to parse.
setup.py Installer. Auto-installs deps, scaffolds .env, marks SETUP_COMPLETE.
Design:
- Silent on success: --check exits 0 with no output when everything's ready so
that /watch doesn't spam "setup is complete" on every turn.
- Idempotent: re-running the installer is safe — it never clobbers existing
keys and only appends missing ones.
- SETUP_COMPLETE=true in ~/.config/watch/.env tells us the user has been
through a successful installer run at least once.
- Never sudo. On macOS, auto-install via brew. Elsewhere, print exact commands.
- Never write an API key to disk automatically — only scaffold placeholders.
"""
from __future__ import annotations
import json
import os
import platform
import shutil
import subprocess
import sys
from pathlib import Path
REQUIRED_BINARIES = ["ffmpeg", "ffprobe", "yt-dlp"]
CONFIG_DIR = Path.home() / ".config" / "watch"
CONFIG_FILE = CONFIG_DIR / ".env"
ENV_TEMPLATE = """# /watch API configuration
#
# Whisper transcription fallback — used only when yt-dlp cannot get captions
# (or when you point /watch at a local file with no subtitles).
#
# Groq is preferred: it runs whisper-large-v3 at a fraction of OpenAI's price
# and is faster in practice. OpenAI is the compatible fallback.
#
# Get a Groq key: https://console.groq.com/keys
# Get an OpenAI key: https://platform.openai.com/api-keys
#
# Leave both blank to disable Whisper — /watch will still work, but videos
# without native captions will come back frames-only.
GROQ_API_KEY=
OPENAI_API_KEY=
"""
def _which(name: str) -> str | None:
return shutil.which(name)
def _check_binaries() -> list[str]:
return [b for b in REQUIRED_BINARIES if not _which(b)]
def _check_file_permissions(path: Path) -> None:
"""Warn to stderr if a secrets file is world/group readable."""
try:
mode = path.stat().st_mode
if mode & 0o044:
sys.stderr.write(
f"[watch] WARNING: {path} is readable by other users. "
f"Run: chmod 600 {path}\n"
)
sys.stderr.flush()
except OSError:
pass
def _read_env_key(name: str) -> str | None:
value = os.environ.get(name)
if value and value.strip():
return value.strip()
if not CONFIG_FILE.exists():
return None
_check_file_permissions(CONFIG_FILE)
try:
for line in CONFIG_FILE.read_text().splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, raw = line.partition("=")
if key.strip() != name:
continue
raw = raw.strip()
if len(raw) >= 2 and raw[0] in ('"', "'") and raw[-1] == raw[0]:
raw = raw[1:-1]
return raw or None
except OSError:
return None
return None
def _have_api_key() -> tuple[bool, str | None]:
if _read_env_key("GROQ_API_KEY"):
return True, "groq"
if _read_env_key("OPENAI_API_KEY"):
return True, "openai"
return False, None
def is_first_run() -> bool:
"""True if the installer hasn't completed successfully yet."""
return _read_env_key("SETUP_COMPLETE") != "true"
def _scaffold_env() -> bool:
"""Create ~/.config/watch/.env with placeholders if missing."""
if CONFIG_FILE.exists():
return False
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
CONFIG_FILE.write_text(ENV_TEMPLATE)
try:
CONFIG_FILE.chmod(0o600)
except OSError:
pass
return True
def _write_setup_complete() -> None:
"""Idempotently append SETUP_COMPLETE=true to .env.
Used only after a fully successful install (deps + key). Future sessions
detect this marker to skip wizard-style UI and stay silent.
"""
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
existing = ""
if CONFIG_FILE.exists():
existing = CONFIG_FILE.read_text()
for line in existing.splitlines():
if line.strip().startswith("SETUP_COMPLETE="):
return
if existing and not existing.endswith("\n"):
existing += "\n"
CONFIG_FILE.write_text(existing + "SETUP_COMPLETE=true\n")
else:
CONFIG_FILE.write_text(ENV_TEMPLATE + "\nSETUP_COMPLETE=true\n")
try:
CONFIG_FILE.chmod(0o600)
except OSError:
pass
def _brew_pkg(missing: list[str]) -> list[str]:
pkgs: list[str] = []
for bin_name in missing:
if bin_name in ("ffmpeg", "ffprobe"):
if "ffmpeg" not in pkgs:
pkgs.append("ffmpeg")
elif bin_name == "yt-dlp":
if "yt-dlp" not in pkgs:
pkgs.append("yt-dlp")
else:
pkgs.append(bin_name)
return pkgs
def _install_macos(missing: list[str]) -> tuple[bool, str]:
if _which("brew") is None:
return False, (
"Homebrew is not installed. Install it from https://brew.sh, then re-run setup. "
"Or install manually: `brew install " + " ".join(_brew_pkg(missing)) + "`"
)
pkgs = _brew_pkg(missing)
if not pkgs:
return True, "nothing to install"
cmd = ["brew", "install", *pkgs]
print(f"[setup] running: {' '.join(cmd)}", file=sys.stderr)
result = subprocess.run(cmd)
if result.returncode != 0:
return False, f"brew install failed with exit code {result.returncode}"
return True, f"installed via brew: {', '.join(pkgs)}"
def _install_hint_linux(missing: list[str]) -> str:
pkgs = _brew_pkg(missing)
hints = []
if "ffmpeg" in pkgs:
hints.append("apt: `sudo apt install ffmpeg` or dnf: `sudo dnf install ffmpeg`")
if "yt-dlp" in pkgs:
hints.append("`pipx install yt-dlp` (recommended) or `pip install --user yt-dlp`")
return "\n ".join(hints) if hints else "nothing to install"
def _status() -> dict:
"""Structured preflight snapshot."""
missing = _check_binaries()
has_key, backend = _have_api_key()
if not missing and has_key:
status = "ready"
elif missing and not has_key:
status = "needs_install_and_key"
elif missing:
status = "needs_install"
else:
status = "needs_key"
return {
"status": status,
"first_run": is_first_run(),
"missing_binaries": missing,
"whisper_backend": backend,
"has_api_key": has_key,
"config_file": str(CONFIG_FILE),
"platform": platform.system(),
}
def cmd_check() -> int:
"""Silent-on-success preflight.
Exit 0 with no output when ready. On failure, print one actionable line
to stderr and return:
2 → binaries missing
3 → API key missing
4 → both missing
"""
s = _status()
if s["status"] == "ready":
return 0
parts = []
if s["missing_binaries"]:
parts.append(f"missing binaries: {', '.join(s['missing_binaries'])}")
if not s["has_api_key"]:
parts.append("no Whisper API key (GROQ_API_KEY or OPENAI_API_KEY)")
installer = Path(__file__).resolve()
sys.stderr.write(
f"[watch] setup incomplete ({'; '.join(parts)}). "
f"Run: python3 {installer}\n"
)
sys.stderr.flush()
if s["missing_binaries"] and not s["has_api_key"]:
return 4
if s["missing_binaries"]:
return 2
return 3
def cmd_json() -> int:
json.dump(_status(), sys.stdout, indent=2)
sys.stdout.write("\n")
return 0
def cmd_install() -> int:
missing = _check_binaries()
installed_deps = False
if missing:
system = platform.system()
if system == "Darwin":
ok, msg = _install_macos(missing)
print(f"[setup] {msg}", file=sys.stderr)
if not ok:
return 2
still_missing = _check_binaries()
if still_missing:
print(f"[setup] still missing after install: {', '.join(still_missing)}", file=sys.stderr)
return 2
installed_deps = True
elif system == "Linux":
print("[setup] dependencies missing on Linux — please install:", file=sys.stderr)
print(" " + _install_hint_linux(missing), file=sys.stderr)
return 2
else:
print(f"[setup] unsupported platform ({system}) for auto-install. Install manually:", file=sys.stderr)
print(f" missing: {', '.join(missing)}", file=sys.stderr)
return 2
created = _scaffold_env()
if created:
print(f"[setup] created config: {CONFIG_FILE}")
else:
print(f"[setup] config exists: {CONFIG_FILE}")
has_key, backend = _have_api_key()
if has_key:
_write_setup_complete()
print(f"[setup] ready. whisper backend: {backend}")
if installed_deps:
print("[setup] installed dependencies; /watch is fully set up.")
return 0
print("")
print("[setup] one step left: add a Whisper API key.")
print("")
print(f" Edit {CONFIG_FILE} and set either:")
print(" GROQ_API_KEY=... (preferred — cheaper, faster; get one at console.groq.com/keys)")
print(" OPENAI_API_KEY=... (fallback; get one at platform.openai.com/api-keys)")
print("")
print(" Without a key, /watch still works but videos without captions come back frames-only.")
return 3
def main() -> int:
if len(sys.argv) > 1:
arg = sys.argv[1]
if arg == "--check":
return cmd_check()
if arg == "--json":
return cmd_json()
return cmd_install()
if __name__ == "__main__":
raise SystemExit(main())
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""Parse a WebVTT subtitle file into a clean, timestamped transcript.
YouTube auto-subs emit rolling-duplicate cues (each line appears 2-3 times as it
scrolls). We dedupe consecutive identical cues and merge their time ranges.
"""
from __future__ import annotations
import re
import sys
from pathlib import Path
TS_RE = re.compile(
r"(\d{2}):(\d{2}):(\d{2})[.,](\d{3})\s+-->\s+(\d{2}):(\d{2}):(\d{2})[.,](\d{3})"
)
TAG_RE = re.compile(r"<[^>]+>")
def _to_seconds(h: str, m: str, s: str, ms: str) -> float:
return int(h) * 3600 + int(m) * 60 + int(s) + int(ms) / 1000.0
def parse_vtt(path: str) -> list[dict]:
text = Path(path).read_text(encoding="utf-8", errors="ignore")
lines = text.splitlines()
segments: list[dict] = []
i = 0
while i < len(lines):
match = TS_RE.match(lines[i])
if not match:
i += 1
continue
start = _to_seconds(*match.groups()[:4])
end = _to_seconds(*match.groups()[4:])
i += 1
cue_lines: list[str] = []
while i < len(lines) and lines[i].strip():
cleaned = TAG_RE.sub("", lines[i]).strip()
if cleaned:
cue_lines.append(cleaned)
i += 1
cue_text = " ".join(cue_lines).strip()
if cue_text:
segments.append({"start": round(start, 2), "end": round(end, 2), "text": cue_text})
i += 1
return _dedupe(segments)
def _dedupe(segments: list[dict]) -> list[dict]:
"""Collapse rolling duplicates common in YouTube auto-subs."""
out: list[dict] = []
for seg in segments:
if out and seg["text"] == out[-1]["text"]:
out[-1]["end"] = seg["end"]
continue
if out and seg["text"].startswith(out[-1]["text"] + " "):
out[-1]["text"] = seg["text"]
out[-1]["end"] = seg["end"]
continue
out.append(seg)
return out
def filter_range(
segments: list[dict],
start_seconds: float | None,
end_seconds: float | None,
) -> list[dict]:
"""Return segments whose time range overlaps [start, end]."""
if start_seconds is None and end_seconds is None:
return segments
lo = start_seconds if start_seconds is not None else float("-inf")
hi = end_seconds if end_seconds is not None else float("inf")
return [seg for seg in segments if seg["end"] >= lo and seg["start"] <= hi]
def format_transcript(segments: list[dict]) -> str:
lines = []
for seg in segments:
start = int(seg["start"])
stamp = f"[{start // 60:02d}:{start % 60:02d}]"
lines.append(f"{stamp} {seg['text']}")
return "\n".join(lines)
if __name__ == "__main__":
if len(sys.argv) < 2:
print("usage: transcribe.py <vtt-path>", file=sys.stderr)
raise SystemExit(2)
print(format_transcript(parse_vtt(sys.argv[1])))
+230
View File
@@ -0,0 +1,230 @@
#!/usr/bin/env python3
"""/watch entry point: download video, extract frames, parse transcript.
Prints a markdown report to stdout listing frame paths + transcript. Claude
then Reads each frame path to see the video.
"""
from __future__ import annotations
import argparse
import sys
import tempfile
from pathlib import Path
SCRIPT_DIR = Path(__file__).parent.resolve()
sys.path.insert(0, str(SCRIPT_DIR))
from download import download, is_url # noqa: E402
from frames import MAX_FPS, auto_fps, auto_fps_focus, extract, format_time, get_metadata, parse_time # noqa: E402
from transcribe import filter_range, format_transcript, parse_vtt # noqa: E402
from whisper import load_api_key, transcribe_video # noqa: E402
def main() -> int:
ap = argparse.ArgumentParser(
prog="watch",
description="Download a video, extract auto-scaled frames, and surface the transcript.",
)
ap.add_argument("source", help="Video URL or local file path")
ap.add_argument("--max-frames", type=int, default=80, help="Cap on frame count (default 80, hard max 100)")
ap.add_argument("--resolution", type=int, default=512, help="Frame width in pixels (default 512)")
ap.add_argument("--fps", type=float, default=None, help="Override auto-fps")
ap.add_argument("--start", type=str, default=None, help="Range start (SS, MM:SS, or HH:MM:SS)")
ap.add_argument("--end", type=str, default=None, help="Range end (SS, MM:SS, or HH:MM:SS)")
ap.add_argument("--out-dir", type=str, default=None, help="Working directory (default: tmp)")
ap.add_argument(
"--no-whisper",
action="store_true",
help="Disable Whisper fallback. Report frames-only if no captions available.",
)
ap.add_argument(
"--whisper",
choices=["groq", "openai"],
default=None,
help="Force a specific Whisper backend. Default: prefer Groq, fall back to OpenAI.",
)
args = ap.parse_args()
max_frames = min(args.max_frames, 100)
if args.out_dir:
work = Path(args.out_dir).expanduser().resolve()
else:
work = Path(tempfile.mkdtemp(prefix="watch-"))
work.mkdir(parents=True, exist_ok=True)
print(f"[watch] working dir: {work}", file=sys.stderr)
print(
"[watch] downloading via yt-dlp…" if is_url(args.source) else "[watch] using local file…",
file=sys.stderr,
)
dl = download(args.source, work / "download")
video_path = dl["video_path"]
meta = get_metadata(video_path)
full_duration = meta["duration_seconds"]
start_sec = parse_time(args.start)
end_sec = parse_time(args.end)
if start_sec is not None and start_sec < 0:
raise SystemExit("--start must be non-negative")
if end_sec is not None and start_sec is not None and end_sec <= start_sec:
raise SystemExit("--end must be greater than --start")
if full_duration > 0 and start_sec is not None and start_sec >= full_duration:
raise SystemExit(f"--start {start_sec:.1f}s is past end of video ({full_duration:.1f}s)")
effective_start = start_sec if start_sec is not None else 0.0
effective_end = end_sec if end_sec is not None else full_duration
effective_duration = max(0.0, effective_end - effective_start)
focused = start_sec is not None or end_sec is not None
if focused:
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
else:
fps, target = auto_fps(effective_duration, max_frames=max_frames)
if args.fps is not None:
fps = min(args.fps, MAX_FPS)
target = max(1, int(round(fps * effective_duration)))
scope = (
f"{format_time(effective_start)}-{format_time(effective_end)} ({effective_duration:.1f}s)"
if focused else f"full {effective_duration:.1f}s"
)
print(f"[watch] extracting ~{target} frames at {fps:.3f} fps over {scope}", file=sys.stderr)
frames = extract(
video_path,
work / "frames",
fps=fps,
resolution=args.resolution,
max_frames=max_frames,
start_seconds=start_sec,
end_seconds=end_sec,
)
transcript_segments: list[dict] = []
transcript_text: str | None = None
transcript_source: str | None = None
if dl.get("subtitle_path"):
try:
all_segments = parse_vtt(dl["subtitle_path"])
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
transcript_text = format_transcript(transcript_segments)
transcript_source = "captions"
except Exception as exc:
print(f"[watch] subtitle parse failed: {exc}", file=sys.stderr)
if not transcript_segments and not args.no_whisper:
backend, api_key = load_api_key(args.whisper)
if backend and api_key:
try:
all_segments, used_backend = transcribe_video(
video_path,
work / "audio.mp3",
backend=backend,
api_key=api_key,
)
transcript_segments = filter_range(all_segments, start_sec, end_sec) if focused else all_segments
transcript_text = format_transcript(transcript_segments)
transcript_source = f"whisper ({used_backend})"
except SystemExit as exc:
print(f"[watch] whisper fallback failed: {exc}", file=sys.stderr)
else:
hint = (
f"--whisper {args.whisper} was set but the matching API key is missing"
if args.whisper else
"no subtitles and no Whisper API key found"
)
setup_py = SCRIPT_DIR / "setup.py"
print(
f"[watch] {hint} — run `python3 {setup_py}` to enable the Whisper fallback",
file=sys.stderr,
)
info = dl.get("info") or {}
print()
print("# watch: video report")
print()
print(f"- **Source:** {args.source}")
if info.get("title"):
print(f"- **Title:** {info['title']}")
if info.get("uploader"):
print(f"- **Uploader:** {info['uploader']}")
print(f"- **Duration:** {format_time(full_duration)} ({full_duration:.1f}s)")
if focused:
print(
f"- **Focus range:** {format_time(effective_start)}{format_time(effective_end)} "
f"({effective_duration:.1f}s)"
)
if meta.get("width") and meta.get("height"):
print(f"- **Resolution:** {meta['width']}x{meta['height']} ({meta.get('codec') or 'unknown codec'})")
mode = "focused" if focused else "full"
print(f"- **Frames:** {len(frames)} @ {fps:.3f} fps, {mode} mode (budget {target}, max {max_frames})")
print(f"- **Frame size:** {args.resolution}px wide")
if transcript_segments:
in_range = " in range" if focused else ""
print(
f"- **Transcript:** {len(transcript_segments)} segments{in_range} "
f"(via {transcript_source or 'captions'})"
)
else:
print("- **Transcript:** none available")
if not focused and full_duration > 600:
mins = int(full_duration // 60)
print()
print(
f"> ⚠️ This is a {mins}-minute video. Frame coverage is sparse at this length — "
"accuracy degrades noticeably on anything over 10 minutes. For better results, "
"re-run with `--start HH:MM:SS --end HH:MM:SS` to zoom into a specific section."
)
print()
print("## Frames")
print()
print(f"Frames live at: `{work / 'frames'}`")
print()
print(
"**Read each frame path below with the Read tool to view the image.** "
"Frames are in chronological order; `t=MM:SS` is the absolute timestamp in the source video."
)
print()
for frame in frames:
print(f"- `{frame['path']}` (t={format_time(frame['timestamp_seconds'])})")
print()
print("## Transcript")
print()
if transcript_text:
label = transcript_source or "captions"
if focused:
print(f"_Source: {label}. Filtered to {format_time(effective_start)}{format_time(effective_end)}:_")
else:
print(f"_Source: {label}._")
print()
print("```")
print(transcript_text)
print("```")
elif focused and dl.get("subtitle_path"):
print(f"_No transcript lines fell inside {format_time(effective_start)}{format_time(effective_end)}._")
else:
setup_py = SCRIPT_DIR / "setup.py"
print(
"_No transcript available — proceed with frames only. "
"Captions were missing and the Whisper fallback was unavailable "
"(no API key set, or `--no-whisper` was used). "
f"Run `python3 {setup_py}` to enable Whisper, then re-run._"
)
print()
print("---")
print(f"_Work dir: `{work}` — delete when done._")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+319
View File
@@ -0,0 +1,319 @@
#!/usr/bin/env python3
"""Transcribe a video via Groq or OpenAI Whisper API.
Strategy: extract audio (mono 16kHz mp3, tiny payload), upload to whichever
API has a key. Returns segments in the same shape as transcribe.parse_vtt so
the rest of the pipeline (filter_range, format_transcript) doesn't care where
the transcript came from.
Pure stdlib — no `pip install groq` or `pip install openai` needed.
"""
from __future__ import annotations
import io
import json
import mimetypes
import os
import shutil
import ssl
import subprocess
import sys
import time
import urllib.error
import uuid
from pathlib import Path
from urllib.request import Request, urlopen
GROQ_ENDPOINT = "https://api.groq.com/openai/v1/audio/transcriptions"
GROQ_MODEL = "whisper-large-v3"
OPENAI_ENDPOINT = "https://api.openai.com/v1/audio/transcriptions"
OPENAI_MODEL = "whisper-1"
def load_api_key(preferred: str | None = None) -> tuple[str, str] | tuple[None, None]:
"""Return (backend, api_key). Prefers Groq, falls back to OpenAI.
If `preferred` is "groq" or "openai", only that backend's key is considered.
"""
def _from_env(name: str) -> str | None:
value = os.environ.get(name)
return value.strip() if value else None
def _from_dotenv(path: Path, name: str) -> str | None:
if not path.exists():
return None
try:
for line in path.read_text().splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, value = line.partition("=")
if key.strip() != name:
continue
value = value.strip()
if len(value) >= 2 and value[0] in ('"', "'") and value[-1] == value[0]:
value = value[1:-1]
return value or None
except OSError:
return None
return None
dotenv_paths = [
Path.home() / ".config" / "watch" / ".env",
Path.cwd() / ".env",
]
candidates = (("GROQ_API_KEY", "groq"), ("OPENAI_API_KEY", "openai"))
if preferred is not None:
candidates = tuple(c for c in candidates if c[1] == preferred)
for key_name, backend in candidates:
value = _from_env(key_name)
if not value:
for candidate in dotenv_paths:
value = _from_dotenv(candidate, key_name)
if value:
break
if value:
return backend, value
return None, None
def extract_audio(video_path: str, out_path: Path) -> Path:
"""Extract mono 16kHz 64kbps mp3 — ~480 kB/min, fits any Whisper limit."""
if shutil.which("ffmpeg") is None:
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
out_path.parent.mkdir(parents=True, exist_ok=True)
cmd = [
"ffmpeg",
"-hide_banner",
"-loglevel", "error",
"-y",
"-i", video_path,
"-vn",
"-acodec", "libmp3lame",
"-ar", "16000",
"-ac", "1",
"-b:a", "64k",
str(out_path),
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
raise SystemExit(f"ffmpeg audio extraction failed: {result.stderr.strip()}")
if not out_path.exists() or out_path.stat().st_size == 0:
raise SystemExit("ffmpeg produced no audio — video may have no audio track")
return out_path
def _build_multipart(fields: dict[str, str], file_path: Path) -> tuple[bytes, str]:
"""Assemble a multipart/form-data body the Whisper APIs accept.
Whisper's multipart upload is small and predictable — doing it by hand
keeps us on pure stdlib instead of pulling requests/groq/openai SDKs.
"""
boundary = f"----WatchBoundary{uuid.uuid4().hex}"
eol = b"\r\n"
buf = io.BytesIO()
for name, value in fields.items():
buf.write(f"--{boundary}".encode()); buf.write(eol)
buf.write(f'Content-Disposition: form-data; name="{name}"'.encode()); buf.write(eol)
buf.write(eol)
buf.write(str(value).encode()); buf.write(eol)
mimetype = mimetypes.guess_type(file_path.name)[0] or "application/octet-stream"
buf.write(f"--{boundary}".encode()); buf.write(eol)
buf.write(
f'Content-Disposition: form-data; name="file"; filename="{file_path.name}"'.encode()
)
buf.write(eol)
buf.write(f"Content-Type: {mimetype}".encode()); buf.write(eol)
buf.write(eol)
buf.write(file_path.read_bytes())
buf.write(eol)
buf.write(f"--{boundary}--".encode()); buf.write(eol)
return buf.getvalue(), boundary
MAX_ATTEMPTS = 4 # initial + 3 retries
MAX_429_RETRIES = 2
RETRY_BASE_DELAY = 2.0
def _post_whisper(endpoint: str, api_key: str, model: str, audio_path: Path) -> dict:
fields = {
"model": model,
"response_format": "verbose_json",
"temperature": "0",
}
body, boundary = _build_multipart(fields, audio_path)
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": f"multipart/form-data; boundary={boundary}",
# Groq sits behind Cloudflare — the default `Python-urllib/3.x` UA
# trips WAF rule 1010 (403) before auth even runs. Any non-default
# UA clears it; we identify honestly.
"User-Agent": "watch-skill/1.0 (+claude-code; python-urllib)",
}
context = ssl.create_default_context()
rate_limit_hits = 0
last_exc: Exception | None = None
last_detail = ""
for attempt in range(MAX_ATTEMPTS):
request = Request(endpoint, data=body, headers=headers, method="POST")
try:
with urlopen(request, timeout=300, context=context) as response:
payload = response.read().decode("utf-8", errors="replace")
except urllib.error.HTTPError as exc:
detail = _read_error_body(exc)
last_exc, last_detail = exc, detail
# 4xx other than 429 are client errors — no retry will fix them.
if 400 <= exc.code < 500 and exc.code != 429:
raise SystemExit(f"Whisper request failed: {exc}{detail}")
if exc.code == 429:
rate_limit_hits += 1
if rate_limit_hits >= MAX_429_RETRIES:
raise SystemExit(f"Whisper request failed: {exc}{detail}")
delay = _retry_after(exc) or RETRY_BASE_DELAY * (2 ** attempt) + 1
else:
delay = RETRY_BASE_DELAY * (2 ** attempt)
if attempt < MAX_ATTEMPTS - 1:
print(
f"[watch] whisper HTTP {exc.code} — retrying in {delay:.1f}s "
f"(attempt {attempt + 2}/{MAX_ATTEMPTS})",
file=sys.stderr,
)
time.sleep(delay)
continue
except (urllib.error.URLError, TimeoutError, ConnectionResetError, OSError) as exc:
last_exc, last_detail = exc, ""
if attempt < MAX_ATTEMPTS - 1:
delay = RETRY_BASE_DELAY * (attempt + 1)
print(
f"[watch] whisper network error ({type(exc).__name__}: {exc}) — "
f"retrying in {delay:.1f}s (attempt {attempt + 2}/{MAX_ATTEMPTS})",
file=sys.stderr,
)
time.sleep(delay)
continue
try:
return json.loads(payload)
except json.JSONDecodeError as exc:
raise SystemExit(f"Whisper returned non-JSON response: {exc}: {payload[:200]}")
raise SystemExit(
f"Whisper request failed after {MAX_ATTEMPTS} attempts: {last_exc}{last_detail}"
)
def _read_error_body(exc: urllib.error.HTTPError) -> str:
try:
body = exc.read()
except Exception:
return ""
if not body:
return ""
try:
return f"{body.decode('utf-8', errors='replace')[:400]}"
except Exception:
return ""
def _retry_after(exc: urllib.error.HTTPError) -> float | None:
header = exc.headers.get("Retry-After") if getattr(exc, "headers", None) else None
if not header:
return None
try:
return float(header)
except ValueError:
return None
def _segments_from_response(data: dict) -> list[dict]:
"""Convert Whisper verbose_json into our {start, end, text} segment format."""
out: list[dict] = []
for seg in data.get("segments") or []:
text = (seg.get("text") or "").strip()
if not text:
continue
out.append({
"start": round(float(seg.get("start") or 0.0), 2),
"end": round(float(seg.get("end") or 0.0), 2),
"text": text,
})
if not out:
full = (data.get("text") or "").strip()
if full:
out.append({"start": 0.0, "end": 0.0, "text": full})
return out
def transcribe_video(
video_path: str,
audio_out: Path,
backend: str | None = None,
api_key: str | None = None,
) -> tuple[list[dict], str]:
"""Run the full flow: extract audio → upload → parse segments.
Returns (segments, backend_used). Raises SystemExit on any failure.
"""
if backend is None or api_key is None:
detected_backend, detected_key = load_api_key()
backend = backend or detected_backend
api_key = api_key or detected_key
if not backend or not api_key:
setup_py = Path(__file__).resolve().parent / "setup.py"
raise SystemExit(
"No Whisper API key available. Set GROQ_API_KEY (preferred) or OPENAI_API_KEY "
"in the environment or in ~/.config/watch/.env. "
f"Run `python3 {setup_py}` to configure."
)
print(f"[watch] extracting audio for Whisper ({backend})…", file=sys.stderr)
audio_path = extract_audio(video_path, audio_out)
size_kb = audio_path.stat().st_size / 1024
print(f"[watch] audio: {size_kb:.0f} kB — uploading to {backend} Whisper…", file=sys.stderr)
if backend == "groq":
response = _post_whisper(GROQ_ENDPOINT, api_key, GROQ_MODEL, audio_path)
elif backend == "openai":
response = _post_whisper(OPENAI_ENDPOINT, api_key, OPENAI_MODEL, audio_path)
else:
raise SystemExit(f"Unknown whisper backend: {backend}")
segments = _segments_from_response(response)
if not segments:
raise SystemExit("Whisper returned no transcript segments")
print(f"[watch] transcribed {len(segments)} segments via {backend}", file=sys.stderr)
return segments, backend
if __name__ == "__main__":
if len(sys.argv) < 2:
print("usage: whisper.py <video-path> [<audio-out.mp3>] [--backend groq|openai]", file=sys.stderr)
raise SystemExit(2)
video = sys.argv[1]
audio_out = Path(sys.argv[2]) if len(sys.argv) > 2 and not sys.argv[2].startswith("--") else Path("audio.mp3")
backend_override = None
if "--backend" in sys.argv:
backend_override = sys.argv[sys.argv.index("--backend") + 1]
segments, backend = transcribe_video(video, audio_out, backend=backend_override)
print(json.dumps({"backend": backend, "segments": segments}, indent=2))