1#!/usr/bin/env bash
2set -euo pipefail
3
4SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
5REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
6PROMPTS_DIR="$REPO_ROOT/.factory/prompts/docs-automation"
7OUTPUT_DIR="${TMPDIR:-/tmp}/docs-automation-test"
8
9# Default values
10BASE_BRANCH="main"
11# Use fast model for analysis, powerful model for writing
12ANALYSIS_MODEL="${ANALYSIS_MODEL:-gemini-3-flash-preview}"
13WRITING_MODEL="${WRITING_MODEL:-claude-opus-4-5-20251101}"
14DRY_RUN=false
15VERBOSE=false
16PR_NUMBER=""
17SOURCE_BRANCH=""
18
19# Patterns for files that could affect documentation
20DOCS_RELEVANT_PATTERNS=(
21 "crates/.*/src/.*\.rs" # Rust source files
22 "assets/settings/.*" # Settings schemas
23 "assets/keymaps/.*" # Keymaps
24 "extensions/.*" # Extensions
25 "docs/.*" # Docs themselves
26)
27
28usage() {
29 cat << EOF
30Usage: $(basename "$0") [OPTIONS]
31
32Test the documentation automation workflow locally.
33
34OPTIONS:
35 -p, --pr NUMBER PR number to analyze (uses gh pr diff)
36 -r, --branch BRANCH Remote branch to compare (e.g., origin/feature-branch)
37 -b, --base BRANCH Base branch to compare against (default: main)
38 -d, --dry-run Preview changes without modifying files
39 -s, --skip-apply Alias for --dry-run
40 -v, --verbose Show full output from each phase
41 -o, --output DIR Output directory for phase artifacts (default: $OUTPUT_DIR)
42 -h, --help Show this help message
43
44EXAMPLES:
45 # Analyze a PR (most common use case)
46 $(basename "$0") --pr 12345
47
48 # Analyze a PR with dry run (no file changes)
49 $(basename "$0") --pr 12345 --dry-run
50
51 # Analyze a remote branch against main
52 $(basename "$0") --branch origin/feature-branch
53
54ENVIRONMENT:
55 FACTORY_API_KEY Required: Your Factory API key
56 ANALYSIS_MODEL Model for analysis (default: gemini-2.0-flash)
57 WRITING_MODEL Model for writing (default: claude-opus-4-5-20251101)
58 GH_TOKEN Required for --pr option (or gh auth login)
59
60EOF
61 exit 0
62}
63
64while [[ $# -gt 0 ]]; do
65 case $1 in
66 -p|--pr)
67 PR_NUMBER="$2"
68 shift 2
69 ;;
70 -r|--branch)
71 SOURCE_BRANCH="$2"
72 shift 2
73 ;;
74 -b|--base)
75 BASE_BRANCH="$2"
76 shift 2
77 ;;
78 -d|--dry-run|-s|--skip-apply)
79 DRY_RUN=true
80 shift
81 ;;
82 -v|--verbose)
83 VERBOSE=true
84 shift
85 ;;
86 -o|--output)
87 OUTPUT_DIR="$2"
88 shift 2
89 ;;
90 -h|--help)
91 usage
92 ;;
93 *)
94 echo "Unknown option: $1"
95 usage
96 ;;
97 esac
98done
99
100# Cleanup function for restoring original branch
101cleanup_on_exit() {
102 if [[ -f "$OUTPUT_DIR/original-branch.txt" ]]; then
103 ORIGINAL_BRANCH=$(cat "$OUTPUT_DIR/original-branch.txt")
104 CURRENT=$(git -C "$REPO_ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo "")
105 if [[ "$CURRENT" != "$ORIGINAL_BRANCH" && -n "$ORIGINAL_BRANCH" ]]; then
106 echo ""
107 echo "Restoring original branch: $ORIGINAL_BRANCH"
108 git -C "$REPO_ROOT" checkout "$ORIGINAL_BRANCH" 2>/dev/null || true
109 if [[ "$CURRENT" == temp-analysis-* ]]; then
110 git -C "$REPO_ROOT" branch -D "$CURRENT" 2>/dev/null || true
111 fi
112 fi
113 fi
114}
115trap cleanup_on_exit EXIT
116
117# Check for required tools
118if ! command -v droid &> /dev/null; then
119 echo "Error: droid CLI not found. Install from https://app.factory.ai/cli"
120 exit 1
121fi
122
123if [[ -z "${FACTORY_API_KEY:-}" ]]; then
124 echo "Error: FACTORY_API_KEY environment variable is not set"
125 exit 1
126fi
127
128# Check gh CLI if PR mode
129if [[ -n "$PR_NUMBER" ]]; then
130 if ! command -v gh &> /dev/null; then
131 echo "Error: gh CLI not found. Install from https://cli.github.com/"
132 echo "Required for --pr option"
133 exit 1
134 fi
135fi
136
137# Create output directory
138mkdir -p "$OUTPUT_DIR"
139echo "========================================"
140echo "Documentation Automation Test"
141echo "========================================"
142echo "Output directory: $OUTPUT_DIR"
143echo "Analysis model: $ANALYSIS_MODEL"
144echo "Writing model: $WRITING_MODEL"
145echo "Started at: $(date '+%Y-%m-%d %H:%M:%S')"
146echo ""
147
148cd "$REPO_ROOT"
149
150# Get changed files based on mode
151echo "=== Getting changed files ==="
152
153if [[ -n "$PR_NUMBER" ]]; then
154 # PR mode: use gh pr diff like the workflow does
155 echo "Analyzing PR #$PR_NUMBER"
156
157 # Get PR info for context
158 echo "Fetching PR details..."
159 gh pr view "$PR_NUMBER" --json title,headRefName,baseRefName,state > "$OUTPUT_DIR/pr-info.json" 2>/dev/null || true
160 if [[ -f "$OUTPUT_DIR/pr-info.json" ]]; then
161 PR_TITLE=$(jq -r '.title // "Unknown"' "$OUTPUT_DIR/pr-info.json")
162 PR_HEAD=$(jq -r '.headRefName // "Unknown"' "$OUTPUT_DIR/pr-info.json")
163 PR_BASE=$(jq -r '.baseRefName // "Unknown"' "$OUTPUT_DIR/pr-info.json")
164 PR_STATE=$(jq -r '.state // "Unknown"' "$OUTPUT_DIR/pr-info.json")
165 echo " Title: $PR_TITLE"
166 echo " Branch: $PR_HEAD -> $PR_BASE"
167 echo " State: $PR_STATE"
168 fi
169 echo ""
170
171 # Get the list of changed files
172 gh pr diff "$PR_NUMBER" --name-only > "$OUTPUT_DIR/changed_files.txt"
173
174 # Also save the full diff for analysis
175 gh pr diff "$PR_NUMBER" > "$OUTPUT_DIR/pr-diff.patch" 2>/dev/null || true
176
177 # Checkout the PR branch to have the code available for analysis
178 echo "Checking out PR branch for analysis..."
179 ORIGINAL_BRANCH=$(git rev-parse --abbrev-ref HEAD)
180 echo "$ORIGINAL_BRANCH" > "$OUTPUT_DIR/original-branch.txt"
181
182 gh pr checkout "$PR_NUMBER" --force 2>/dev/null || {
183 echo "Warning: Could not checkout PR branch. Analysis will use current branch state."
184 }
185
186elif [[ -n "$SOURCE_BRANCH" ]]; then
187 # Remote branch mode
188 echo "Analyzing branch: $SOURCE_BRANCH"
189 echo "Base branch: $BASE_BRANCH"
190
191 # Fetch the branches
192 git fetch origin 2>/dev/null || true
193
194 # Resolve branch refs
195 SOURCE_REF="$SOURCE_BRANCH"
196 BASE_REF="origin/$BASE_BRANCH"
197
198 # Get merge base
199 MERGE_BASE=$(git merge-base "$BASE_REF" "$SOURCE_REF" 2>/dev/null) || {
200 echo "Error: Could not find merge base between $BASE_REF and $SOURCE_REF"
201 exit 1
202 }
203 echo "Merge base: $MERGE_BASE"
204
205 # Get changed files
206 git diff --name-only "$MERGE_BASE" "$SOURCE_REF" > "$OUTPUT_DIR/changed_files.txt"
207
208 # Checkout the source branch for analysis
209 echo "Checking out $SOURCE_BRANCH for analysis..."
210 ORIGINAL_BRANCH=$(git rev-parse --abbrev-ref HEAD)
211 echo "$ORIGINAL_BRANCH" > "$OUTPUT_DIR/original-branch.txt"
212
213 git checkout "$SOURCE_BRANCH" 2>/dev/null || git checkout -b "temp-analysis-$$" "$SOURCE_REF" || {
214 echo "Warning: Could not checkout branch. Analysis will use current branch state."
215 }
216
217else
218 # Current branch mode (original behavior)
219 CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
220 echo "Analyzing current branch: $CURRENT_BRANCH"
221 echo "Base branch: $BASE_BRANCH"
222
223 # Fetch the base branch
224 git fetch origin "$BASE_BRANCH" 2>/dev/null || true
225
226 # Get merge base
227 MERGE_BASE=$(git merge-base "origin/$BASE_BRANCH" HEAD 2>/dev/null || git merge-base "$BASE_BRANCH" HEAD)
228 echo "Merge base: $MERGE_BASE"
229
230 git diff --name-only "$MERGE_BASE" HEAD > "$OUTPUT_DIR/changed_files.txt"
231fi
232
233if [[ ! -s "$OUTPUT_DIR/changed_files.txt" ]]; then
234 echo "No changed files found."
235 exit 0
236fi
237
238echo ""
239echo "Changed files ($(wc -l < "$OUTPUT_DIR/changed_files.txt" | tr -d ' ') files):"
240cat "$OUTPUT_DIR/changed_files.txt"
241echo ""
242
243# Early exit: Filter for docs-relevant files only
244echo "=== Filtering for docs-relevant files ==="
245DOCS_RELEVANT_FILES=""
246while IFS= read -r file; do
247 for pattern in "${DOCS_RELEVANT_PATTERNS[@]}"; do
248 if [[ "$file" =~ $pattern ]]; then
249 DOCS_RELEVANT_FILES="$DOCS_RELEVANT_FILES $file"
250 break
251 fi
252 done
253done < "$OUTPUT_DIR/changed_files.txt"
254
255# Trim leading space
256DOCS_RELEVANT_FILES="${DOCS_RELEVANT_FILES# }"
257
258if [[ -z "$DOCS_RELEVANT_FILES" ]]; then
259 echo "No docs-relevant files changed (only tests, configs, CI, etc.)"
260 echo "Skipping documentation analysis."
261 exit 0
262fi
263
264echo "Docs-relevant files: $(echo "$DOCS_RELEVANT_FILES" | wc -w | tr -d ' ')"
265echo "$DOCS_RELEVANT_FILES" | tr ' ' '\n' | head -20
266echo ""
267
268# Combined Phase: Analyze + Plan (using fast model)
269echo "=== Analyzing Changes & Planning Documentation Impact ==="
270echo "Model: $ANALYSIS_MODEL"
271echo "Started at: $(date '+%H:%M:%S')"
272echo ""
273
274CHANGED_FILES=$(tr '\n' ' ' < "$OUTPUT_DIR/changed_files.txt")
275
276# Write prompt to temp file to avoid escaping issues
277cat > "$OUTPUT_DIR/analysis-prompt.txt" << 'PROMPT_EOF'
278Analyze these code changes and determine if documentation updates are needed.
279
280## Documentation Guidelines
281
282### Requires Documentation Update
283- New user-facing features or commands
284- Changed keybindings or default behaviors
285- Modified settings schema or options
286- Deprecated or removed functionality
287
288### Does NOT Require Documentation Update
289- Internal refactoring without behavioral changes
290- Performance optimizations (unless user-visible)
291- Bug fixes that restore documented behavior
292- Test changes, CI/CD changes
293
294### In-Scope: docs/src/**/*.md
295### Out-of-Scope: CHANGELOG.md, README.md, code comments, rustdoc
296
297### Output Format Required
298You MUST output a JSON object with this exact structure:
299{
300 "updates_required": true or false,
301 "summary": "Brief description of changes",
302 "planned_changes": [
303 {
304 "file": "docs/src/path/to/file.md",
305 "section": "Section name",
306 "change_type": "update or add or deprecate",
307 "description": "What to change"
308 }
309 ],
310 "skipped_files": ["reason1", "reason2"]
311}
312
313Be conservative - only flag documentation updates for user-visible changes.
314
315## Changed Files
316PROMPT_EOF
317
318echo "$CHANGED_FILES" >> "$OUTPUT_DIR/analysis-prompt.txt"
319
320ANALYSIS_START=$(date +%s)
321
322droid exec \
323 -m "$ANALYSIS_MODEL" \
324 --auto low \
325 -f "$OUTPUT_DIR/analysis-prompt.txt" \
326 > "$OUTPUT_DIR/analysis.json" 2>&1 || true
327
328ANALYSIS_END=$(date +%s)
329ANALYSIS_DURATION=$((ANALYSIS_END - ANALYSIS_START))
330
331echo "Completed in ${ANALYSIS_DURATION}s"
332echo ""
333echo "--- Analysis Result ---"
334cat "$OUTPUT_DIR/analysis.json"
335echo ""
336echo "-----------------------"
337echo ""
338
339# Check if updates are required (parse JSON output)
340UPDATES_REQUIRED=$(grep -o '"updates_required":\s*true' "$OUTPUT_DIR/analysis.json" || echo "")
341
342if [[ -z "$UPDATES_REQUIRED" ]]; then
343 echo "=== No documentation updates required ==="
344 echo "Analysis determined no documentation changes are needed."
345 cat "$OUTPUT_DIR/analysis.json"
346 exit 0
347fi
348
349echo "Documentation updates ARE required."
350echo ""
351
352# Extract planned changes for the next phase
353ANALYSIS_OUTPUT=$(cat "$OUTPUT_DIR/analysis.json")
354
355if [[ "$DRY_RUN" == "true" ]]; then
356 # Combined Preview Phase (dry-run): Show what would change
357 echo "=== Preview: Generating Proposed Changes ==="
358 echo "Model: $WRITING_MODEL"
359 echo "Started at: $(date '+%H:%M:%S')"
360 echo ""
361
362 PREVIEW_START=$(date +%s)
363
364 # Write preview prompt to temp file
365 cat > "$OUTPUT_DIR/preview-prompt.txt" << PREVIEW_EOF
366Generate a PREVIEW of the documentation changes. Do NOT modify any files.
367
368Based on this analysis:
369$ANALYSIS_OUTPUT
370
371For each planned change:
3721. Read the current file
3732. Show the CURRENT section that would be modified
3743. Show the PROPOSED new content
3754. Generate a unified diff
376
377Output format:
378---
379## File: [path]
380
381### Current:
382(paste exact current content)
383
384### Proposed:
385(paste proposed new content)
386
387### Diff:
388(unified diff with - and + lines)
389---
390
391Show the ACTUAL content, not summaries.
392PREVIEW_EOF
393
394 droid exec \
395 -m "$WRITING_MODEL" \
396 --auto low \
397 -f "$OUTPUT_DIR/preview-prompt.txt" \
398 > "$OUTPUT_DIR/preview.md" 2>&1 || true
399
400 PREVIEW_END=$(date +%s)
401 PREVIEW_DURATION=$((PREVIEW_END - PREVIEW_START))
402
403 echo "Completed in ${PREVIEW_DURATION}s"
404 echo ""
405 echo "--- Proposed Changes ---"
406 cat "$OUTPUT_DIR/preview.md"
407 echo "------------------------"
408
409 echo ""
410 echo "=== Dry run complete ==="
411 echo "Total time: Analysis ${ANALYSIS_DURATION}s + Preview ${PREVIEW_DURATION}s = $((ANALYSIS_DURATION + PREVIEW_DURATION))s"
412 echo "To apply changes, run without --dry-run flag."
413 echo "Output saved to: $OUTPUT_DIR/"
414 exit 0
415fi
416
417# Combined Phase: Apply Changes + Generate Summary (using writing model)
418echo "=== Applying Documentation Changes ==="
419echo "Model: $WRITING_MODEL"
420echo "Started at: $(date '+%H:%M:%S')"
421echo ""
422
423APPLY_START=$(date +%s)
424
425# Write apply prompt to temp file
426cat > "$OUTPUT_DIR/apply-prompt.txt" << APPLY_EOF
427Apply the documentation changes specified in this analysis:
428
429$ANALYSIS_OUTPUT
430
431Instructions:
4321. For each planned change, edit the specified file
4332. Follow the mdBook format and style from docs/AGENTS.md
4343. Use {#kb action::Name} syntax for keybindings
4354. After making changes, output a brief summary
436
437Output format:
438## Changes Applied
439- [file]: [what was changed]
440
441## Summary for PR
442[2-3 sentence summary suitable for a PR description]
443APPLY_EOF
444
445droid exec \
446 -m "$WRITING_MODEL" \
447 --auto medium \
448 -f "$OUTPUT_DIR/apply-prompt.txt" \
449 > "$OUTPUT_DIR/apply-report.md" 2>&1 || true
450
451APPLY_END=$(date +%s)
452APPLY_DURATION=$((APPLY_END - APPLY_START))
453
454echo "Completed in ${APPLY_DURATION}s"
455echo ""
456echo "--- Apply Report ---"
457cat "$OUTPUT_DIR/apply-report.md"
458echo "--------------------"
459echo ""
460
461# Format with Prettier (only changed files)
462echo "=== Formatting with Prettier ==="
463cd "$REPO_ROOT"
464
465CHANGED_DOCS=$(git diff --name-only docs/src/ 2>/dev/null | sed 's|^docs/||' | tr '\n' ' ')
466
467if [[ -n "$CHANGED_DOCS" ]]; then
468 echo "Formatting: $CHANGED_DOCS"
469 if command -v pnpm &> /dev/null; then
470 (cd docs && pnpm dlx prettier@3.5.0 $CHANGED_DOCS --write) 2>/dev/null || true
471 elif command -v prettier &> /dev/null; then
472 (cd docs && prettier --write $CHANGED_DOCS) 2>/dev/null || true
473 fi
474 echo "Done"
475else
476 echo "No changed docs files to format"
477fi
478echo ""
479
480# Generate summary from the apply report
481cp "$OUTPUT_DIR/apply-report.md" "$OUTPUT_DIR/phase6-summary.md"
482
483# Phase 7: Create Branch and PR
484echo "=== Phase 7: Create Branch and PR ==="
485
486# Check if there are actual changes
487if git -C "$REPO_ROOT" diff --quiet docs/src/; then
488 echo "No documentation changes detected after Phase 5"
489 echo ""
490 echo "=== Test Complete (no changes to commit) ==="
491 exit 0
492fi
493
494# Check if gh CLI is available
495if ! command -v gh &> /dev/null; then
496 echo "Warning: gh CLI not found. Skipping PR creation."
497 echo "Install from https://cli.github.com/ to enable automatic PR creation."
498 echo ""
499 echo "Documentation changes (git status):"
500 git -C "$REPO_ROOT" status --short docs/src/
501 echo ""
502 echo "To review the diff:"
503 echo " git diff docs/src/"
504 echo ""
505 echo "To discard changes:"
506 echo " git checkout docs/src/"
507 exit 0
508fi
509
510cd "$REPO_ROOT"
511
512# Daily batch branch - one branch per day, multiple commits accumulate
513BRANCH_NAME="docs/auto-update-$(date +%Y-%m-%d)"
514
515# Stash local changes from phase 5
516echo "Stashing documentation changes..."
517git stash push -m "docs-automation-changes" -- docs/src/
518
519# Check if branch already exists on remote
520if git ls-remote --exit-code --heads origin "$BRANCH_NAME" > /dev/null 2>&1; then
521 echo "Branch $BRANCH_NAME exists, checking out and updating..."
522 git fetch origin "$BRANCH_NAME"
523 git checkout -B "$BRANCH_NAME" "origin/$BRANCH_NAME"
524else
525 echo "Creating new branch $BRANCH_NAME from main..."
526 git fetch origin main
527 git checkout -B "$BRANCH_NAME" origin/main
528fi
529
530# Apply stashed changes
531echo "Applying documentation changes..."
532git stash pop || true
533
534# Stage and commit
535git add docs/src/
536
537# Get source PR info for attribution
538SOURCE_PR_INFO=""
539TRIGGER_INFO=""
540if [[ -n "$PR_NUMBER" ]]; then
541 # Fetch PR details: title, author, url
542 PR_DETAILS=$(gh pr view "$PR_NUMBER" --json title,author,url 2>/dev/null || echo "{}")
543 SOURCE_TITLE=$(echo "$PR_DETAILS" | jq -r '.title // "Unknown"')
544 SOURCE_AUTHOR=$(echo "$PR_DETAILS" | jq -r '.author.login // "Unknown"')
545 SOURCE_URL=$(echo "$PR_DETAILS" | jq -r '.url // ""')
546
547 TRIGGER_INFO="Triggered by: PR #$PR_NUMBER"
548 SOURCE_PR_INFO="
549---
550**Source**: [#$PR_NUMBER]($SOURCE_URL) - $SOURCE_TITLE
551**Author**: @$SOURCE_AUTHOR
552"
553elif [[ -n "$SOURCE_BRANCH" ]]; then
554 TRIGGER_INFO="Triggered by: branch $SOURCE_BRANCH"
555 SOURCE_PR_INFO="
556---
557**Source**: Branch \`$SOURCE_BRANCH\`
558"
559fi
560
561# Build commit message
562SUMMARY=$(head -50 < "$OUTPUT_DIR/phase6-summary.md" 2>/dev/null || echo "Automated documentation update")
563
564git commit -m "docs: auto-update documentation
565
566${SUMMARY}
567
568${TRIGGER_INFO}
569
570Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>" || {
571 echo "Nothing to commit"
572 exit 0
573}
574
575# Push
576echo "Pushing to origin/$BRANCH_NAME..."
577git push -u origin "$BRANCH_NAME"
578
579# Build the PR body section for this update
580PR_BODY_SECTION="## Update from $(date '+%Y-%m-%d %H:%M')
581$SOURCE_PR_INFO
582$(cat "$OUTPUT_DIR/phase6-summary.md")
583"
584
585# Check if PR already exists for this branch
586EXISTING_PR=$(gh pr list --head "$BRANCH_NAME" --json number,url,body --jq '.[0]' 2>/dev/null || echo "")
587
588if [[ -n "$EXISTING_PR" && "$EXISTING_PR" != "null" ]]; then
589 PR_NUM=$(echo "$EXISTING_PR" | jq -r '.number')
590 PR_URL=$(echo "$EXISTING_PR" | jq -r '.url')
591 EXISTING_BODY=$(echo "$EXISTING_PR" | jq -r '.body // ""')
592
593 # Append new summary to existing PR body
594 echo "Updating PR body with new summary..."
595 NEW_BODY="${EXISTING_BODY}
596
597---
598
599${PR_BODY_SECTION}"
600
601 echo "$NEW_BODY" > "$OUTPUT_DIR/updated-pr-body.md"
602 gh pr edit "$PR_NUM" --body-file "$OUTPUT_DIR/updated-pr-body.md"
603
604 echo ""
605 echo "=== Updated existing PR ==="
606 echo "PR #$PR_NUM: $PR_URL"
607 echo "New commit added and PR description updated."
608else
609 # Create new PR with full body
610 echo "Creating new PR..."
611 echo "$PR_BODY_SECTION" > "$OUTPUT_DIR/new-pr-body.md"
612
613 PR_URL=$(gh pr create \
614 --title "docs: automated documentation update ($(date +%Y-%m-%d))" \
615 --body-file "$OUTPUT_DIR/new-pr-body.md" \
616 --base main 2>&1) || {
617 echo "Failed to create PR: $PR_URL"
618 exit 1
619 }
620 echo ""
621 echo "=== PR Created ==="
622 echo "$PR_URL"
623fi
624
625echo ""
626echo "=== Test Complete ==="
627echo "Total time: Analysis ${ANALYSIS_DURATION}s + Apply ${APPLY_DURATION}s = $((ANALYSIS_DURATION + APPLY_DURATION))s"
628echo "All outputs saved to: $OUTPUT_DIR/"