Files
superpowers/skills/agentic-end-to-end-testing/scripts/check-cards-against-spec
T
2026-08-06 16:35:34 -07:00

224 lines
6.8 KiB
Bash
Executable File

#!/usr/bin/env bash
# check-cards-against-spec — verify scenario cards carry their spec table's
# falsification lines verbatim. See authoring-cards-from-a-spec.md.
set -euo pipefail
usage() {
cat <<'EOF'
Usage: check-cards-against-spec <spec.md> <cards-dir>
Verifies the spec's "E2E scenario cards" table against the cards directory:
1. table parses (>=1 row; non-empty Card and Falsification cells)
2. every row has <cards-dir>/<card>.md
3. every card contains its Falsification line verbatim
(whitespace-normalized, fixed-string, case-sensitive)
4. every card has **What this covers** (bold inline) and ## headings
Pre-state, Steps, Expected, Cleanup (Sharp edges not required)
5. extra cards in <cards-dir> are reported as warnings, not failures
Exit: 0 all pass; 1 check failed; 2 no "E2E scenario cards" table; 64 usage.
EOF
}
[ "${1:-}" = "--help" ] && { usage; exit 0; }
[ $# -eq 2 ] || { usage >&2; exit 64; }
SPEC="$1"; CARDS="$2"
[ -f "$SPEC" ] || { echo "error: spec not found: $SPEC" >&2; exit 64; }
[ -d "$CARDS" ] || { echo "error: cards dir not found: $CARDS" >&2; exit 64; }
FAILURES=0
fail() { echo "FAIL: $1"; FAILURES=$((FAILURES + 1)); }
warn() { echo "warn: $1"; }
# Collapse every whitespace run to one space; trim ends. (Normative per the
# design spec: markdown re-wrapping must not defeat the verbatim check.)
normalize() { tr -s '[:space:]' ' ' | sed -e 's/^ //' -e 's/ $//'; }
# Exclude fenced examples from structural matching. This intentionally models
# only backtick/tilde fences; it is not a general Markdown parser.
without_fenced_code() {
awk '
function read_fence(line, first, count) {
marker_family = ""
marker_width = 0
sub(/^[[:space:]]*/, "", line)
first = substr(line, 1, 1)
if (first != "`" && first != "~") return
count = 0
while (substr(line, count + 1, 1) == first) count++
if (count >= 3) {
marker_family = first
marker_width = count
}
}
{
read_fence($0)
if (!in_fence && marker_family != "") {
in_fence = 1
family = marker_family
opening_width = marker_width
next
}
if (in_fence && marker_family == family && marker_width >= opening_width) {
in_fence = 0
family = ""
opening_width = 0
next
}
if (!in_fence) print
}
' "$1"
}
# Text of the card's Expected section only (case-insensitive heading match,
# any ##+ level; section ends at the next heading or EOF).
expected_section() {
awk '
/^#{1,6}[[:space:]]/ {
low = tolower($0)
if (low ~ /^#+[[:space:]]*expected[[:space:]]*$/) { insec = 1; next }
if (insec) exit
}
insec { print }
'
}
# --- extract the first table under the (case-insensitive) heading ----------
TABLE="$(awk '
/^#{1,6}[[:space:]]/ {
h = $0; sub(/^#+[[:space:]]*/, "", h); sub(/[[:space:]]+$/, "", h)
if (tolower(h) == "e2e scenario cards") { insec = 1; next }
if (insec) exit
}
insec && /^[[:space:]]*\|/ { intable = 1; print; next }
insec && intable { exit }
' < <(without_fenced_code "$SPEC"))"
if [ -z "$TABLE" ]; then
echo "no scenario table: $SPEC has no \"E2E scenario cards\" heading with a table under it" >&2
echo "(heading must be exactly \"E2E scenario cards\" — no numbering or extra words)" >&2
echo "(scenario table rows must use leading and trailing outer pipes)" >&2
exit 2
fi
# --- parse: protect escaped pipes, split rows into cells -------------------
US=$'\x1f'
CARD_COL=-1; FALS_COL=-1; ROWS=0
declare -a ROW_CARD ROW_FALS
HEADER_COLS=0; TABLE_VALID=1
lineno=0
while IFS= read -r line; do
lineno=$((lineno + 1))
row="$(printf '%s' "$line" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
case "$row" in
\|*\|) ;;
*)
fail "row $lineno: canonical table rows require leading and trailing pipes"
TABLE_VALID=0
break
;;
esac
row="${row#|}"
row="${row%|}"
esc="${row//\\|/$US}"
IFS='|' read -r -a cells <<< "$esc"
trimmed=()
for c in "${cells[@]}"; do
c="${c//$US/|}"
c="$(printf '%s' "$c" | normalize)"
trimmed+=("$c")
done
if [ "$lineno" -eq 1 ]; then
for i in "${!trimmed[@]}"; do
low="$(printf '%s' "${trimmed[$i]}" | tr '[:upper:]' '[:lower:]')"
[ "$low" = "card" ] && CARD_COL=$i
[ "$low" = "falsification" ] && FALS_COL=$i
done
HEADER_COLS=${#trimmed[@]}
if [ "$CARD_COL" -lt 0 ] || [ "$FALS_COL" -lt 0 ]; then
fail "table header must name Card and Falsification columns"
TABLE_VALID=0
break
fi
continue
fi
if [ "$lineno" -eq 2 ]; then
delimiter_ok=1
[ "${#trimmed[@]}" -eq "$HEADER_COLS" ] || delimiter_ok=0
for c in "${trimmed[@]}"; do
if ! printf '%s\n' "$c" | grep -Eq '^:?-{3,}:?$'; then
delimiter_ok=0
fi
done
if [ "$delimiter_ok" -ne 1 ]; then
fail "row 2: malformed table delimiter"
TABLE_VALID=0
break
fi
continue
fi
card="${trimmed[$CARD_COL]:-}"
falsif="${trimmed[$FALS_COL]:-}"
if [ -z "$card" ] || [ -z "$falsif" ]; then
fail "row $lineno: empty Card or Falsification cell"
continue
fi
case "$card" in
\`*\`)
card="${card#\`}"
card="${card%\`}"
;;
esac
if ! [[ "$card" =~ ^[a-z0-9]+(-[a-z0-9]+)*$ ]]; then
fail "row $lineno: invalid Card value: $card"
continue
fi
ROW_CARD[$ROWS]="$card"; ROW_FALS[$ROWS]="$falsif"; ROWS=$((ROWS + 1))
done <<< "$TABLE"
if [ "$TABLE_VALID" -eq 1 ] && [ "$ROWS" -lt 1 ]; then
fail "scenario table has no data rows"
fi
# --- checks 2-4 per row -----------------------------------------------------
i=0
while [ "$i" -lt "$ROWS" ]; do
card="${ROW_CARD[$i]}"; falsif="${ROW_FALS[$i]}"
f="$CARDS/$card.md"
if [ ! -f "$f" ]; then
fail "missing card file: $f"
i=$((i + 1)); continue
fi
card_text="$(without_fenced_code "$f")"
hay="$(printf '%s\n' "$card_text" | expected_section | normalize)"
case "$hay" in
*"$falsif"*) : ;;
*) fail "$f: falsification line not present verbatim in the ## Expected section.
expected (normalized): $falsif" ;;
esac
grep -q '\*\*What this covers\*\*' <<< "$card_text" || fail "$f: missing **What this covers**"
for sec in Pre-state Steps Expected Cleanup; do
grep -Eiq "^#{2,}[[:space:]]*${sec}[[:space:]]*$" <<< "$card_text" || fail "$f: missing ## ${sec} section"
done
i=$((i + 1))
done
# --- check 5: extra cards are warnings --------------------------------------
for f in "$CARDS"/*.md; do
[ -e "$f" ] || continue
base="$(basename "$f" .md)"
known=0; i=0
while [ "$i" -lt "$ROWS" ]; do
[ "${ROW_CARD[$i]}" = "$base" ] && known=1
i=$((i + 1))
done
[ "$known" -eq 1 ] || warn "extra card not in spec table: $base"
done
if [ "$FAILURES" -gt 0 ]; then
echo "$FAILURES check(s) failed"
exit 1
fi
echo "all checks passed ($ROWS card(s))"