mortal/superpowers
1
0
Fork 0
mirror of https://github.com/obra/superpowers synced 2026-07-16 07:34:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(tests): stop the SDD skill test flaking on timing and prose case

Browse Source 
tests/claude-code/test-subagent-driven-development.sh failed
intermittently for two independent reasons:

- Budget mismatch: the file runs 9 prompts with a 90s timeout each
  (810s worst case) inside the runner's 600s per-file ceiling, so slow
  backend days produced spurious timeouts. Raise the runner default to
  900s and fix the help text, which claimed the default was 300.
- Case-sensitive prose matching: the assert helpers grepped free-form
  model output case-sensitively, but models capitalize the skill's own
  headings — observed failures include "Do Not Trust the Report"
  missing pattern "not trust" and a structured answer missing
  "First:.*spec.*compliance". Match case-insensitively in
  assert_contains/assert_not_contains/assert_count/assert_order, widen
  two Test 5 keyword patterns to phrasings observed in real runs, and
  make assert_order dump the output on failure the way assert_contains
  already does, so the next flake is diagnosable.

Observed 3 failures across 4 runs before the change (timeout, two
distinct pattern misses); 3/3 consecutive full runs pass after it.
This commit is contained in:
Jesse Vincent
2026-07-16 04:00:15 +00:00
parent 93b250b155
commit cda4b15fd6
3 changed files with 16 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/claude-code/run-skill-tests.sh
+3 -2
View File
@@ -25,7 +25,8 @@ fi
# Parse command line arguments
VERBOSE=false
SPECIFIC_TEST=""
TIMEOUT=600 # Default 10 minute timeout per test
TIMEOUT=900 # Per-test-file budget; must exceed the file's worst case
# (test-subagent-driven-development.sh: 9 prompts x 90s each)
RUN_INTEGRATION=false
while [[ $# -gt 0 ]]; do
@@ -52,7 +53,7 @@ while [[ $# -gt 0 ]]; do
echo "Options:"
echo " --verbose, -v Show verbose output"
echo " --test, -t NAME Run only the specified test"
echo " --timeout SECONDS Set timeout per test (default: 300)"
echo " --timeout SECONDS Set timeout per test (default: 900)"
echo " --integration, -i Run integration tests (slow, 10-30 min)"
echo " --help, -h Show this help"
echo ""
tests/claude-code/test-helpers.sh
+11 -5
View File
@@ -30,12 +30,14 @@ run_claude() {
# Check if output contains a pattern
# Usage: assert_contains "output" "pattern" "test name"
# Matching is case-insensitive: patterns are prose keywords, and models
# freely capitalize skill terms ("Do Not Trust", "Spec Compliance").
assert_contains() {
local output="$1"
local pattern="$2"
local test_name="${3:-test}"
if echo "$output" | grep -q "$pattern"; then
if echo "$output" | grep -qi "$pattern"; then
echo " [PASS] $test_name"
return 0
else
@@ -54,7 +56,7 @@ assert_not_contains() {
local pattern="$2"
local test_name="${3:-test}"
if echo "$output" | grep -q "$pattern"; then
if echo "$output" | grep -qi "$pattern"; then
echo " [FAIL] $test_name"
echo " Did not expect to find: $pattern"
echo " In output:"
@@ -74,7 +76,7 @@ assert_count() {
local expected="$3"
local test_name="${4:-test}"
local actual=$(echo "$output" | grep -c "$pattern" || echo "0")
local actual=$(echo "$output" | grep -ci "$pattern" || echo "0")
if [ "$actual" -eq "$expected" ]; then
echo " [PASS] $test_name (found $actual instances)"
@@ -98,16 +100,20 @@ assert_order() {
local test_name="${4:-test}"
# Get line numbers where patterns appear
local line_a=$(echo "$output" | grep -n "$pattern_a" | head -1 | cut -d: -f1)
local line_b=$(echo "$output" | grep -n "$pattern_b" | head -1 | cut -d: -f1)
local line_a=$(echo "$output" | grep -ni "$pattern_a" | head -1 | cut -d: -f1)
local line_b=$(echo "$output" | grep -ni "$pattern_b" | head -1 | cut -d: -f1)
if [ -z "$line_a" ]; then
echo " [FAIL] $test_name: pattern A not found: $pattern_a"
echo " In output:"
echo "$output" | sed 's/^/ /'
return 1
fi
if [ -z "$line_b" ]; then
echo " [FAIL] $test_name: pattern B not found: $pattern_b"
echo " In output:"
echo "$output" | sed 's/^/ /'
return 1
fi
tests/claude-code/test-subagent-driven-development.sh
+2 -2
View File
@@ -96,13 +96,13 @@ echo "Test 5: Spec compliance reviewer mindset..."
output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" "$CLAUDE_PROMPT_TIMEOUT")
if assert_contains "$output" "not trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then
if assert_contains "$output" "not.*trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then
: # pass
else
exit 1
fi
if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code" "Reviewer reads code"; then
if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code\|read.*diff\|trust.*diff" "Reviewer reads code"; then
: # pass
else
exit 1