1#!/usr/bin/env bash
2#
3# Test docs-suggest on multiple PRs and generate a summary report.
4#
5# Usage:
6# script/test-docs-suggest-batch [--limit N] [--output FILE]
7#
8# This script runs docs-suggest in dry-run mode on recent merged PRs
9# to validate the context assembly and help tune the prompt.
10
11set -euo pipefail
12
13LIMIT=50
14OUTPUT="docs-suggest-batch-results.md"
15SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
16
17while [[ $# -gt 0 ]]; do
18 case $1 in
19 --limit)
20 LIMIT="$2"
21 shift 2
22 ;;
23 --output)
24 OUTPUT="$2"
25 shift 2
26 ;;
27 *)
28 echo "Unknown option: $1"
29 exit 1
30 ;;
31 esac
32done
33
34echo "Testing docs-suggest on $LIMIT recent merged PRs..."
35echo "Output: $OUTPUT"
36echo ""
37
38# Get list of PRs
39PRS=$(gh pr list --state merged --limit "$LIMIT" --json number,title --jq '.[] | "\(.number)|\(.title)"')
40
41# Initialize output file
42cat > "$OUTPUT" << HEADER
43# docs-suggest Batch Test Results
44
45**Date**: $(date +%Y-%m-%d)
46**PRs tested**: $LIMIT
47
48## Summary
49
50| PR | Title | Result | Source Files | Notes |
51|----|-------|--------|--------------|-------|
52HEADER
53
54# Track stats
55total=0
56has_source=0
57no_source=0
58errors=0
59
60while IFS='|' read -r pr_num title; do
61 total=$((total + 1))
62 echo -n "[$total/$LIMIT] PR #$pr_num: "
63
64 # Run dry-run and capture output
65 tmpfile=$(mktemp)
66 if "$SCRIPT_DIR/docs-suggest" --pr "$pr_num" --dry-run 2>"$tmpfile.err" >"$tmpfile.out"; then
67 # Check if it found source files
68 if grep -q "No documentation-relevant changes" "$tmpfile.out"; then
69 result="No source changes"
70 no_source=$((no_source + 1))
71 source_count="0"
72 echo "skipped (no source)"
73 else
74 # Extract source file count from verbose output
75 source_count=$(grep -oE '[0-9]+ source' "$tmpfile.err" 2>/dev/null | grep -oE '[0-9]+' || echo "?")
76 result="Has source changes"
77 has_source=$((has_source + 1))
78 echo "has $source_count source files"
79 fi
80 notes=""
81 else
82 result="Error"
83 errors=$((errors + 1))
84 source_count="-"
85 notes=$(head -1 "$tmpfile.err" 2>/dev/null || echo "unknown error")
86 echo "error"
87 fi
88
89 # Escape title for markdown table
90 title_escaped=$(echo "$title" | sed 's/|/\\|/g' | cut -c1-60)
91
92 # Add row to table
93 echo "| [#$pr_num](https://github.com/zed-industries/zed/pull/$pr_num) | $title_escaped | $result | $source_count | $notes |" >> "$OUTPUT"
94
95 rm -f "$tmpfile" "$tmpfile.out" "$tmpfile.err"
96done <<< "$PRS"
97
98# Add summary stats
99cat >> "$OUTPUT" << STATS
100
101## Statistics
102
103- **Total PRs**: $total
104- **With source changes**: $has_source ($(( has_source * 100 / total ))%)
105- **No source changes**: $no_source ($(( no_source * 100 / total ))%)
106- **Errors**: $errors
107
108## Observations
109
110_Add manual observations here after reviewing results._
111
112## Sample Contexts
113
114STATS
115
116# Add 3 sample contexts from PRs with source changes
117echo "" >> "$OUTPUT"
118echo "### Sample 1: PR with source changes" >> "$OUTPUT"
119echo "" >> "$OUTPUT"
120
121sample_pr=$(gh pr list --state merged --limit 20 --json number --jq '.[].number' | while read pr; do
122 if "$SCRIPT_DIR/docs-suggest" --pr "$pr" --dry-run 2>/dev/null | grep -q "## Code Diff"; then
123 echo "$pr"
124 break
125 fi
126done)
127
128if [[ -n "$sample_pr" ]]; then
129 echo "PR #$sample_pr:" >> "$OUTPUT"
130 echo "" >> "$OUTPUT"
131 echo '```' >> "$OUTPUT"
132 "$SCRIPT_DIR/docs-suggest" --pr "$sample_pr" --dry-run 2>/dev/null | head -100 >> "$OUTPUT"
133 echo '```' >> "$OUTPUT"
134fi
135
136echo ""
137echo "Done! Results written to: $OUTPUT"
138echo ""
139echo "Stats: $has_source with source changes, $no_source without, $errors errors"