mortal/claude-video
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restructure as a self-contained Agent Skills package (fix Codex install)

Browse Source 
Move the skill into skills/watch/ so SKILL.md and its scripts/ runtime are
siblings inside one folder. `npx skills add` (Codex/Cursor/Copilot/agents) now
copies a working skill as a unit; previously it grabbed the root SKILL.md alone
and left scripts/ behind, so the skill was dead on arrival on every non-Claude
host. Mirrors the layout last30days-skill adopted for the same reason.

- skills/watch/{SKILL.md,scripts/}: self-contained skill folder
- SKILL.md: resolve a harness-agnostic $SKILL_DIR (the dir it was Read from)
  instead of the Claude-Code-only ${CLAUDE_SKILL_DIR}; guard + 19 call sites
- drop commands/watch.md: /watch derives from frontmatter (name + user-invocable)
- .codex-plugin/plugin.json: full manifest with "skills": "./skills/" + interface
- add .agents/plugins/marketplace.json, AGENTS.md, CLAUDE.md, .skillignore
- build-skill.sh: archive the skills/watch subtree (one SKILL.md, no zip -d)
- fix paths in tests, hooks hint, .gitattributes, release.yml
- relocate dev-sync.sh to repo root and fix REPO_ROOT
- README: content-ideas structure, npx skills install, star history

Verified: 37/37 tests pass; npx skills add bundles the full scripts/ runtime;
manifests valid; versions synced at 0.1.3.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
bradautomates
2026-06-29 22:12:54 +10:00
parent c333c2289e
commit 429f3143e5
33 changed files with 2360 additions and 783 deletions
Download Patch File Download Diff File Expand all files Collapse all files
skills/watch/scripts/frames.py
Executable
+628
View File
@@ -0,0 +1,628 @@
#!/usr/bin/env python3
"""Probe video metadata and extract frames at an auto-scaled fps.
Auto-fps targets a frame budget, not a fixed rate. Token cost scales with frame
count, so budget-by-duration keeps short videos dense and long videos capped.
When a user-specified range is passed, focused-mode budgets denser (they are
zooming in for detail).
"""
from __future__ import annotations
import json
import re
import shutil
import subprocess
import sys
from pathlib import Path
MAX_FPS = 2.0
SCENE_THRESHOLD = 0.20
# Keep scene-detection results once we have at least this many distinct shots.
# Below this the video is effectively static (screen recording, talking head),
# so we fall back to uniform sampling. Matching the reference fork's behaviour,
# this is a low floor — NOT the frame budget — so normal videos with cuts use
# the (single-pass) scene engine instead of paying for a wasted second decode.
SCENE_MIN_FRAMES = 8
# Below this many decoded keyframes a clip is too sparse for keyframe coverage
# (very short or oddly encoded), so the cheap tier falls back to uniform.
KEYFRAME_MIN = 4
MAX_READ_DIMENSION = 1998
SHOWINFO_TS_RE = re.compile(r"pts_time:([0-9.]+)")
def _scale_filter(resolution: int) -> str:
return (
f"scale=w='min({resolution},iw)':h='min({MAX_READ_DIMENSION},ih)':"
"force_original_aspect_ratio=decrease:force_divisible_by=2"
)
def _clamp_fps(fps: float, duration_seconds: float, max_frames: int) -> tuple[float, int]:
fps = min(fps, MAX_FPS)
target = min(max_frames, max(1, int(round(fps * duration_seconds))))
return fps, target
def parse_time(value: str | float | int | None) -> float | None:
"""Parse SS, MM:SS, or HH:MM:SS (with optional .ms) into seconds."""
if value is None:
return None
if isinstance(value, (int, float)):
return float(value)
s = str(value).strip()
if not s:
return None
parts = s.split(":")
try:
if len(parts) == 1:
return float(parts[0])
if len(parts) == 2:
return int(parts[0]) * 60 + float(parts[1])
if len(parts) == 3:
return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2])
except ValueError:
pass
raise SystemExit(f"Cannot parse time value: {value!r} (expected SS, MM:SS, or HH:MM:SS)")
def format_time(seconds: float) -> str:
total = int(round(seconds))
hours, rem = divmod(total, 3600)
minutes, sec = divmod(rem, 60)
if hours:
return f"{hours}:{minutes:02d}:{sec:02d}"
return f"{minutes:02d}:{sec:02d}"
def get_metadata(video_path: str) -> dict:
if shutil.which("ffprobe") is None:
raise SystemExit("ffprobe is not installed. Install with: brew install ffmpeg")
result = subprocess.run(
[
"ffprobe",
"-v", "quiet",
"-print_format", "json",
"-show_format",
"-show_streams",
str(Path(video_path).resolve()),
],
capture_output=True,
text=True,
)
if result.returncode != 0:
raise SystemExit(f"ffprobe failed: {result.stderr.strip()}")
data = json.loads(result.stdout or "{}")
streams = data.get("streams", [])
fmt = data.get("format", {})
video_stream = next((s for s in streams if s.get("codec_type") == "video"), {})
audio_stream = next((s for s in streams if s.get("codec_type") == "audio"), None)
duration = float(fmt.get("duration") or video_stream.get("duration") or 0)
return {
"duration_seconds": duration,
"width": video_stream.get("width"),
"height": video_stream.get("height"),
"codec": video_stream.get("codec_name"),
"size_bytes": int(fmt.get("size") or 0),
"has_audio": audio_stream is not None,
}
def auto_fps(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
"""Pick fps that targets a sensible frame budget for full-video scans."""
if duration_seconds <= 0:
return 1.0, 1
if duration_seconds <= 30:
target = min(max_frames, max(12, int(round(duration_seconds))))
elif duration_seconds <= 60:
target = min(max_frames, 40)
elif duration_seconds <= 180: # 3 min
target = min(max_frames, 60)
elif duration_seconds <= 600: # 10 min
target = min(max_frames, 80)
else:
target = max_frames
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
def auto_fps_focus(duration_seconds: float, max_frames: int = 100) -> tuple[float, int]:
"""Denser budget for user-specified ranges — they are zooming in for detail."""
if duration_seconds <= 0:
return min(MAX_FPS, 2.0), 2
if duration_seconds <= 5:
target = min(max_frames, max(10, int(round(duration_seconds * 6))))
elif duration_seconds <= 15:
target = min(max_frames, max(30, int(round(duration_seconds * 4))))
elif duration_seconds <= 30:
target = min(max_frames, 60)
elif duration_seconds <= 60:
target = min(max_frames, 80)
elif duration_seconds <= 180:
target = max_frames
else:
target = max_frames
return _clamp_fps(target / duration_seconds, duration_seconds, max_frames)
def extract(
video_path: str,
out_dir: Path,
fps: float,
resolution: int = 512,
max_frames: int = 100,
start_seconds: float | None = None,
end_seconds: float | None = None,
) -> list[dict]:
if shutil.which("ffmpeg") is None:
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
out_dir.mkdir(parents=True, exist_ok=True)
for existing in out_dir.glob("frame_*.jpg"):
existing.unlink()
output_pattern = str(out_dir / "frame_%04d.jpg")
cmd: list[str] = [
"ffmpeg",
"-hide_banner",
"-loglevel", "error",
"-y",
]
# -ss before -i = fast seek (keyframe-snap, good enough for preview frames).
if start_seconds is not None:
cmd += ["-ss", f"{start_seconds:.3f}"]
if end_seconds is not None:
cmd += ["-to", f"{end_seconds:.3f}"]
cmd += [
"-i", str(Path(video_path).resolve()),
"-vf", f"fps={fps},{_scale_filter(resolution)}",
"-frames:v", str(max_frames),
"-q:v", "4",
output_pattern,
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
raise SystemExit(f"ffmpeg frame extraction failed: {result.stderr.strip()}")
offset = start_seconds or 0.0
frames = sorted(out_dir.glob("frame_*.jpg"))
return [
{
"index": i,
"timestamp_seconds": round(offset + (i / fps if fps > 0 else 0.0), 2),
"path": str(p),
"reason": "uniform",
}
for i, p in enumerate(frames)
]
def extract_scene_candidates(
video_path: str,
out_dir: Path,
resolution: int = 512,
max_frames: int | None = 100,
start_seconds: float | None = None,
end_seconds: float | None = None,
threshold: float = SCENE_THRESHOLD,
) -> list[dict]:
"""Extract first frame plus ffmpeg scene-change frames.
When ``max_frames`` is set, ``-frames:v`` lets ffmpeg stop decoding once it
has emitted that many frames (early exit) and avoids writing extras that we
would only delete afterwards. ``None`` (uncapped "complete" detail) keeps
every detected shot, as the user explicitly opted in.
"""
if shutil.which("ffmpeg") is None:
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
out_dir.mkdir(parents=True, exist_ok=True)
for existing in out_dir.glob("frame_*.jpg"):
existing.unlink()
output_pattern = str(out_dir / "frame_%04d.jpg")
cmd: list[str] = [
"ffmpeg",
"-hide_banner",
"-loglevel", "info",
"-y",
]
if start_seconds is not None:
cmd += ["-ss", f"{start_seconds:.3f}"]
if end_seconds is not None:
cmd += ["-to", f"{end_seconds:.3f}"]
vf = f"select='eq(n\\,0)+gt(scene\\,{threshold})',{_scale_filter(resolution)},showinfo"
cmd += [
"-i", str(Path(video_path).resolve()),
"-vf", vf,
"-vsync", "vfr",
]
if max_frames is not None:
cmd += ["-frames:v", str(max_frames)]
cmd += [
"-q:v", "4",
output_pattern,
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
raise SystemExit(f"ffmpeg scene extraction failed: {result.stderr.strip()}")
offset = start_seconds or 0.0
timestamps = [round(offset + float(match.group(1)), 2) for match in SHOWINFO_TS_RE.finditer(result.stderr)]
frames = sorted(out_dir.glob("frame_*.jpg"))
out: list[dict] = []
for i, path in enumerate(frames):
ts = timestamps[i] if i < len(timestamps) else offset
out.append({
"index": i,
"timestamp_seconds": ts,
"path": str(path),
"reason": "first-frame" if i == 0 else "scene-change",
})
return out
def _even_indices(count: int, n: int) -> list[int]:
"""Indices of ``n`` evenly-spaced items out of ``count`` (first + last kept).
``n >= count`` returns every index; ``n == 1`` returns just the first.
"""
if n >= count:
return list(range(count))
if n <= 1:
return [0]
return [round(i * (count - 1) / (n - 1)) for i in range(n)]
def parse_timestamps(value: str | None) -> list[float]:
"""Parse a comma-separated list of times (SS, MM:SS, HH:MM:SS) into a
sorted, de-duplicated list of seconds. Empty/blank tokens are skipped;
an unparseable token raises (via :func:`parse_time`)."""
if not value:
return []
out: list[float] = []
for token in value.split(","):
token = token.strip()
if not token:
continue
seconds = parse_time(token)
if seconds is not None:
out.append(float(seconds))
return sorted(set(out))
def merge_frames(primary: list[dict], pinned: list[dict]) -> list[dict]:
"""Combine two frame lists into one chronological list and reindex 0..n-1.
``pinned`` frames (transcript cues) are never dropped — this is a plain
union, so the cap is enforced upstream by reserving budget for the cues.
"""
merged = sorted([*primary, *pinned], key=lambda f: f["timestamp_seconds"])
for i, frame in enumerate(merged):
frame["index"] = i
return merged
def extract_at_timestamps(
video_path: str,
out_dir: Path,
timestamps: list[float],
resolution: int = 512,
max_frames: int | None = None,
start_seconds: float | None = None,
end_seconds: float | None = None,
) -> tuple[list[dict], dict]:
"""Grab exactly one frame at each requested timestamp (transcript cues).
Timestamps are absolute source seconds. Any falling outside an active
``[start, end]`` focus window are dropped. Files use a ``cue_*.jpg`` prefix
so they sit alongside detail-engine ``frame_*.jpg`` output without either
clobbering the other. When more cues than ``max_frames`` survive, they are
even-sampled (first + last kept) before extraction.
"""
if shutil.which("ffmpeg") is None:
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
out_dir.mkdir(parents=True, exist_ok=True)
for existing in out_dir.glob("cue_*.jpg"):
existing.unlink()
lo = start_seconds or 0.0
hi = end_seconds if end_seconds is not None else float("inf")
requested = sorted(set(round(float(t), 2) for t in timestamps))
in_window = [t for t in requested if lo <= t <= hi]
dropped = len(requested) - len(in_window)
if max_frames is not None and len(in_window) > max_frames:
points = [in_window[i] for i in _even_indices(len(in_window), max_frames)]
else:
points = in_window
out: list[dict] = []
for t in points:
path = out_dir / f"cue_{len(out):04d}.jpg"
cmd = [
"ffmpeg",
"-hide_banner",
"-loglevel", "error",
"-y",
"-ss", f"{t:.3f}",
"-i", str(Path(video_path).resolve()),
"-frames:v", "1",
"-vf", _scale_filter(resolution),
"-q:v", "4",
str(path),
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode == 0 and path.exists():
out.append({
"index": len(out),
"timestamp_seconds": t,
"path": str(path),
"reason": "transcript-cue",
})
meta = {
"engine": "timestamps",
"candidate_count": len(requested),
"selected_count": len(out),
"dropped_out_of_window": dropped,
"fallback": False,
}
return out, meta
def _even_sample(candidates: list[dict], n: int) -> list[dict]:
"""Pick ``n`` evenly-spaced candidates (always including first and last),
delete the JPEGs we drop, and reindex the survivors 0..len-1.
Shared by every capped engine so all detail modes sample the same way:
detect all candidates across the full range, then thin down to the cap.
``n >= len(candidates)`` keeps everything (the uncapped / under-cap case).
"""
selected = [candidates[i] for i in _even_indices(len(candidates), n)]
keep_paths = {sel["path"] for sel in selected}
for cand in candidates:
if cand["path"] not in keep_paths:
try:
Path(cand["path"]).unlink()
except OSError:
pass
for i, frame in enumerate(selected):
frame["index"] = i
return selected
def extract_scene_or_uniform(
video_path: str,
out_dir: Path,
fps: float,
target_frames: int,
resolution: int = 512,
max_frames: int | None = 100,
start_seconds: float | None = None,
end_seconds: float | None = None,
) -> tuple[list[dict], dict]:
"""Prefer scene selection, falling back to uniform only when the video is
effectively static (fewer than ``SCENE_MIN_FRAMES`` detected shots).
Scene cuts are detected across the *whole* range (uncapped) and then
even-sampled down to ``max_frames`` via :func:`_even_sample`, exactly like
the keyframe engine. This costs a full decode, but it guarantees coverage
spans the entire clip — capping detection with ``-frames:v`` instead would
keep only the first ``max_frames`` cuts and drop the tail of long videos
(and could even fall below ``SCENE_MIN_FRAMES`` and misfire the uniform
fallback on a cut-heavy clip).
"""
scene_frames = extract_scene_candidates(
video_path,
out_dir,
resolution=resolution,
max_frames=None,
start_seconds=start_seconds,
end_seconds=end_seconds,
)
scene_count = len(scene_frames)
if scene_count >= SCENE_MIN_FRAMES:
cap = scene_count if max_frames is None else max_frames
selected = _even_sample(scene_frames, cap)
return selected, {
"engine": "scene",
"candidate_count": scene_count,
"selected_count": len(selected),
"fallback": False,
}
fallback_cap = target_frames if max_frames is None else min(max_frames, target_frames)
frames = extract(
video_path,
out_dir,
fps=fps,
resolution=resolution,
max_frames=fallback_cap,
start_seconds=start_seconds,
end_seconds=end_seconds,
)
return frames, {
"engine": "uniform",
"candidate_count": scene_count,
"selected_count": len(frames),
"fallback": True,
}
def extract_keyframes(
video_path: str,
out_dir: Path,
resolution: int = 512,
max_frames: int | None = 50,
start_seconds: float | None = None,
end_seconds: float | None = None,
) -> tuple[list[dict], dict]:
"""Decode only keyframes (I-frames) — the cheap, near-instant tier.
``-skip_frame nokey`` makes ffmpeg reconstruct only keyframes, skipping all
P/B frames. Encoders emit keyframes at scene cuts, so these already
approximate "distinct moments". Over-cap → even-sample first→last; too few
keyframes → uniform fallback.
"""
if shutil.which("ffmpeg") is None:
raise SystemExit("ffmpeg is not installed. Install with: brew install ffmpeg")
out_dir.mkdir(parents=True, exist_ok=True)
for existing in out_dir.glob("frame_*.jpg"):
existing.unlink()
output_pattern = str(out_dir / "frame_%04d.jpg")
cmd: list[str] = [
"ffmpeg",
"-hide_banner",
"-loglevel", "info",
"-y",
]
if start_seconds is not None:
cmd += ["-ss", f"{start_seconds:.3f}"]
if end_seconds is not None:
cmd += ["-to", f"{end_seconds:.3f}"]
cmd += [
"-skip_frame", "nokey",
"-i", str(Path(video_path).resolve()),
"-vf", f"{_scale_filter(resolution)},showinfo",
"-vsync", "vfr",
"-q:v", "4",
output_pattern,
]
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
raise SystemExit(f"ffmpeg keyframe extraction failed: {result.stderr.strip()}")
offset = start_seconds or 0.0
timestamps = [round(offset + float(m.group(1)), 2) for m in SHOWINFO_TS_RE.finditer(result.stderr)]
files = sorted(out_dir.glob("frame_*.jpg"))
candidates: list[dict] = []
for i, path in enumerate(files):
ts = timestamps[i] if i < len(timestamps) else offset
candidates.append({
"index": i,
"timestamp_seconds": ts,
"path": str(path),
"reason": "keyframe",
})
# Too few keyframes → uniform fallback over the same range.
if len(candidates) < KEYFRAME_MIN:
for cand in candidates:
try:
Path(cand["path"]).unlink()
except OSError:
pass
meta = get_metadata(video_path)
full_duration = meta["duration_seconds"]
eff_start = start_seconds or 0.0
eff_end = end_seconds if end_seconds is not None else full_duration
eff_duration = max(0.0, eff_end - eff_start)
budget = max_frames if max_frames is not None else 100
fps, _ = auto_fps(eff_duration, max_frames=budget)
frames_out = extract(
video_path,
out_dir,
fps=fps,
resolution=resolution,
max_frames=budget,
start_seconds=start_seconds,
end_seconds=end_seconds,
)
return frames_out, {
"engine": "uniform",
"candidate_count": len(candidates),
"selected_count": len(frames_out),
"fallback": True,
}
# Detect-all then even-sample down to the cap (first + last always kept).
# ``max_frames is None`` (uncapped) keeps every keyframe.
candidate_count = len(candidates)
cap = candidate_count if max_frames is None else max_frames
selected = _even_sample(candidates, cap)
return selected, {
"engine": "keyframe",
"candidate_count": candidate_count,
"selected_count": len(selected),
"fallback": False,
}
if __name__ == "__main__":
if len(sys.argv) < 3:
print(
"usage: frames.py <video-path> <out-dir> [--fps F] [--resolution W] "
"[--max-frames N] [--start T] [--end T]",
file=sys.stderr,
)
raise SystemExit(2)
video = sys.argv[1]
out = Path(sys.argv[2])
args = sys.argv[3:]
fps_override = None
resolution = 512
max_frames = 100
start_arg = None
end_arg = None
i = 0
while i < len(args):
if args[i] == "--fps":
fps_override = float(args[i + 1]); i += 2
elif args[i] == "--resolution":
resolution = int(args[i + 1]); i += 2
elif args[i] == "--max-frames":
max_frames = int(args[i + 1]); i += 2
elif args[i] == "--start":
start_arg = args[i + 1]; i += 2
elif args[i] == "--end":
end_arg = args[i + 1]; i += 2
else:
i += 1
meta = get_metadata(video)
start_sec = parse_time(start_arg)
end_sec = parse_time(end_arg)
full_duration = meta["duration_seconds"]
effective_start = start_sec if start_sec is not None else 0.0
effective_end = end_sec if end_sec is not None else full_duration
effective_duration = max(0.0, effective_end - effective_start)
focused = start_sec is not None or end_sec is not None
if focused:
fps, target = auto_fps_focus(effective_duration, max_frames=max_frames)
else:
fps, target = auto_fps(effective_duration, max_frames=max_frames)
if fps_override is not None:
fps = fps_override
target = max(1, int(round(fps * effective_duration)))
frames = extract(
video, out,
fps=fps,
resolution=resolution,
max_frames=max_frames,
start_seconds=start_sec,
end_seconds=end_sec,
)
print(json.dumps(
{"meta": meta, "fps": fps, "target": target, "focused": focused, "frames": frames},
indent=2,
))