From cda4b15fd68cec5991ab46c1bc8d5bc2b8ec2677 Mon Sep 17 00:00:00 2001 From: Jesse Vincent Date: Thu, 16 Jul 2026 04:00:15 +0000 Subject: [PATCH] fix(tests): stop the SDD skill test flaking on timing and prose case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/claude-code/test-subagent-driven-development.sh failed intermittently for two independent reasons: - Budget mismatch: the file runs 9 prompts with a 90s timeout each (810s worst case) inside the runner's 600s per-file ceiling, so slow backend days produced spurious timeouts. Raise the runner default to 900s and fix the help text, which claimed the default was 300. - Case-sensitive prose matching: the assert helpers grepped free-form model output case-sensitively, but models capitalize the skill's own headings — observed failures include "Do Not Trust the Report" missing pattern "not trust" and a structured answer missing "First:.*spec.*compliance". Match case-insensitively in assert_contains/assert_not_contains/assert_count/assert_order, widen two Test 5 keyword patterns to phrasings observed in real runs, and make assert_order dump the output on failure the way assert_contains already does, so the next flake is diagnosable. Observed 3 failures across 4 runs before the change (timeout, two distinct pattern misses); 3/3 consecutive full runs pass after it. --- tests/claude-code/run-skill-tests.sh | 5 +++-- tests/claude-code/test-helpers.sh | 16 +++++++++++----- .../test-subagent-driven-development.sh | 4 ++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/tests/claude-code/run-skill-tests.sh b/tests/claude-code/run-skill-tests.sh index c94cfec9..83217cda 100755 --- a/tests/claude-code/run-skill-tests.sh +++ b/tests/claude-code/run-skill-tests.sh @@ -25,7 +25,8 @@ fi # Parse command line arguments VERBOSE=false SPECIFIC_TEST="" -TIMEOUT=600 # Default 10 minute timeout per test +TIMEOUT=900 # Per-test-file budget; must exceed the file's worst case + # (test-subagent-driven-development.sh: 9 prompts x 90s each) RUN_INTEGRATION=false while [[ $# -gt 0 ]]; do @@ -52,7 +53,7 @@ while [[ $# -gt 0 ]]; do echo "Options:" echo " --verbose, -v Show verbose output" echo " --test, -t NAME Run only the specified test" - echo " --timeout SECONDS Set timeout per test (default: 300)" + echo " --timeout SECONDS Set timeout per test (default: 900)" echo " --integration, -i Run integration tests (slow, 10-30 min)" echo " --help, -h Show this help" echo "" diff --git a/tests/claude-code/test-helpers.sh b/tests/claude-code/test-helpers.sh index 1b5ead3b..9e187610 100755 --- a/tests/claude-code/test-helpers.sh +++ b/tests/claude-code/test-helpers.sh @@ -30,12 +30,14 @@ run_claude() { # Check if output contains a pattern # Usage: assert_contains "output" "pattern" "test name" +# Matching is case-insensitive: patterns are prose keywords, and models +# freely capitalize skill terms ("Do Not Trust", "Spec Compliance"). assert_contains() { local output="$1" local pattern="$2" local test_name="${3:-test}" - if echo "$output" | grep -q "$pattern"; then + if echo "$output" | grep -qi "$pattern"; then echo " [PASS] $test_name" return 0 else @@ -54,7 +56,7 @@ assert_not_contains() { local pattern="$2" local test_name="${3:-test}" - if echo "$output" | grep -q "$pattern"; then + if echo "$output" | grep -qi "$pattern"; then echo " [FAIL] $test_name" echo " Did not expect to find: $pattern" echo " In output:" @@ -74,7 +76,7 @@ assert_count() { local expected="$3" local test_name="${4:-test}" - local actual=$(echo "$output" | grep -c "$pattern" || echo "0") + local actual=$(echo "$output" | grep -ci "$pattern" || echo "0") if [ "$actual" -eq "$expected" ]; then echo " [PASS] $test_name (found $actual instances)" @@ -98,16 +100,20 @@ assert_order() { local test_name="${4:-test}" # Get line numbers where patterns appear - local line_a=$(echo "$output" | grep -n "$pattern_a" | head -1 | cut -d: -f1) - local line_b=$(echo "$output" | grep -n "$pattern_b" | head -1 | cut -d: -f1) + local line_a=$(echo "$output" | grep -ni "$pattern_a" | head -1 | cut -d: -f1) + local line_b=$(echo "$output" | grep -ni "$pattern_b" | head -1 | cut -d: -f1) if [ -z "$line_a" ]; then echo " [FAIL] $test_name: pattern A not found: $pattern_a" + echo " In output:" + echo "$output" | sed 's/^/ /' return 1 fi if [ -z "$line_b" ]; then echo " [FAIL] $test_name: pattern B not found: $pattern_b" + echo " In output:" + echo "$output" | sed 's/^/ /' return 1 fi diff --git a/tests/claude-code/test-subagent-driven-development.sh b/tests/claude-code/test-subagent-driven-development.sh index d8f3e10c..151fc64d 100755 --- a/tests/claude-code/test-subagent-driven-development.sh +++ b/tests/claude-code/test-subagent-driven-development.sh @@ -96,13 +96,13 @@ echo "Test 5: Spec compliance reviewer mindset..." output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" "$CLAUDE_PROMPT_TIMEOUT") -if assert_contains "$output" "not trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then +if assert_contains "$output" "not.*trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then : # pass else exit 1 fi -if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code" "Reviewer reads code"; then +if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code\|read.*diff\|trust.*diff" "Reviewer reads code"; then : # pass else exit 1