From 2ec2b4c600a43a3127d3924a0dc66f29e44ac60f Mon Sep 17 00:00:00 2001 From: morgankrey Date: Wed, 25 Feb 2026 19:02:37 -0600 Subject: [PATCH] Fix and improve docs automation scripts (#50120) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes issues discovered while running the docs automation workflow for the first time, plus improvements based on the v0.225 run where 44 suggestions overwhelmed a single Droid invocation. ### docs-suggest-publish - Ignore untracked files when checking for clean working directory - Add `--auto high` flag to droid exec for non-interactive use - Add error handling to show droid output on failure - Remove non-existent `documentation` label from PR creation - Use `--write` flag for prettier to fix formatting - **Batch suggestions** into groups of 10 (configurable with `--batch-size`) to prevent Droid from dropping suggestions when context is too large - **Pre-PR docs build validation** — runs `generate-action-metadata` + `mdbook build` before creating the PR to catch invalid `{#action}` and `{#kb}` references locally instead of waiting for CI (skippable with `--skip-validation`) - **Prompt guardrail** — instructs Droid not to invent `{#kb}` or `{#action}` references, only reusing action names already present in docs files - **Stable release detection** — at publish time, checks each queued PR's merge commit against the latest stable release tag. PRs already in stable get annotated "ALREADY IN STABLE" so Droid applies content changes without adding incorrect Preview callouts - **Feature flag detection** — parses `crates/feature_flags/src/flags.rs` for all feature flag struct names, then checks each PR's diff for references. PRs behind feature flags are skipped entirely since those features aren't generally available yet ### docs-strip-preview-callouts - Remove non-existent `documentation` label from PR creation - Add `Release Notes: - N/A` to generated PR body (fixes Danger bot check) ## Context These scripts were run for the first time as part of the v0.225 release. Issues found: 1. The `documentation` label doesn't exist in this repo 2. Droid exec needs `--auto high` for non-interactive execution 3. Prettier needs `--write` to actually fix files (was running in check mode) 4. Untracked files should not block the workflow 5. Sending all 44 suggestions in one Droid invocation only applied 2 — batching in groups of 10 fixed this 6. Droid hallucinated action names (`settings::OpenSettings`, `gpui::Modifiers::secondary_key`) that broke the docs preprocessor build 7. PRs that shipped in stable v0.225 incorrectly got Preview callouts because the queue doesn't distinguish preview-only from already-in-stable 8. PRs behind feature flags (subagents, git graph) got documented despite not being generally available Release Notes: - N/A --- script/docs-strip-preview-callouts | 11 +- script/docs-suggest-publish | 274 +++++++++++++++++++++++++---- 2 files changed, 243 insertions(+), 42 deletions(-) diff --git a/script/docs-strip-preview-callouts b/script/docs-strip-preview-callouts index dc4d1eb1764a88a89b7406b8b02577bb7471ceb7..ca88fa22b5e12f2a0af09dd005334e662b128c54 100755 --- a/script/docs-strip-preview-callouts +++ b/script/docs-strip-preview-callouts @@ -117,8 +117,8 @@ if [[ "$DRY_RUN" == "true" ]]; then exit 0 fi -# Check for clean working state -if [[ -n "$(git status --porcelain docs/)" ]]; then +# Check for clean working state (ignore untracked files) +if [[ -n "$(git status --porcelain docs/ | grep -v '^??' || true)" ]]; then error "docs/ directory has uncommitted changes. Please commit or stash first." fi @@ -213,8 +213,11 @@ And: > **Changed in Preview (v0.XXX).** See [release notes](/releases#0.XXX). \`\`\` -These features are now in Stable, so the callouts are no longer needed." \ - --label "documentation" +These features are now in Stable, so the callouts are no longer needed. + +Release Notes: + +- N/A" PR_URL=$(gh pr view --json url --jq '.url') diff --git a/script/docs-suggest-publish b/script/docs-suggest-publish index 4f3e3ac4d552960bb362170081e81283bb76619c..23578785159b5fd720e84d3658f7f76dddf3ada9 100755 --- a/script/docs-suggest-publish +++ b/script/docs-suggest-publish @@ -7,16 +7,19 @@ # # This script: # 1. Reads pending suggestions from the docs/suggestions-pending branch -# 2. Uses Droid to apply all suggestions directly to docs files +# 2. Uses Droid to apply suggestions in batches (default 10 per batch) # 3. Runs docs formatting -# 4. Creates a draft PR for human review/merge -# 5. Optionally resets the suggestions branch after successful PR creation +# 4. Validates docs build (action references, JSON schemas, links) +# 5. Creates a draft PR for human review/merge +# 6. Optionally resets the suggestions branch after successful PR creation # # Options: -# --dry-run Show what would be done without creating PR -# --keep-queue Don't reset the suggestions branch after PR creation -# --model MODEL Override Droid model used for auto-apply -# --verbose Show detailed progress +# --dry-run Show what would be done without creating PR +# --keep-queue Don't reset the suggestions branch after PR creation +# --model MODEL Override Droid model used for auto-apply +# --batch-size N Suggestions per Droid invocation (default: 10) +# --skip-validation Skip the docs build validation step +# --verbose Show detailed progress # # Run this as part of the preview release workflow. @@ -25,7 +28,9 @@ set -euo pipefail DRY_RUN=false KEEP_QUEUE=false VERBOSE=false -MODEL="${DROID_MODEL:-claude-sonnet-4-5-20250929}" +SKIP_VALIDATION=false +BATCH_SIZE=10 +MODEL="${DROID_MODEL:-claude-sonnet-4-5-latest}" SUGGESTIONS_BRANCH="docs/suggestions-pending" @@ -66,8 +71,16 @@ while [[ $# -gt 0 ]]; do MODEL="$2" shift 2 ;; + --batch-size) + BATCH_SIZE="$2" + shift 2 + ;; + --skip-validation) + SKIP_VALIDATION=true + shift + ;; -h|--help) - head -26 "$0" | tail -24 + head -30 "$0" | tail -28 exit 0 ;; *) @@ -137,7 +150,11 @@ if [[ -n "$(git status --porcelain | grep -v '^??' || true)" ]]; then error "Working directory has uncommitted changes. Please commit or stash first." fi -for command in git gh jq droid; do +REQUIRED_COMMANDS=(git gh jq droid) +if [[ "$SKIP_VALIDATION" != "true" ]]; then + REQUIRED_COMMANDS+=(mdbook) +fi +for command in "${REQUIRED_COMMANDS[@]}"; do if ! command -v "$command" > /dev/null 2>&1; then error "Required command not found: $command" fi @@ -157,24 +174,165 @@ git checkout -b "$DOCS_BRANCH" origin/main TMPDIR=$(mktemp -d) trap 'rm -rf "$TMPDIR"' EXIT -SUGGESTIONS_FILE="$TMPDIR/suggestions.md" -APPLY_PROMPT_FILE="$TMPDIR/apply-prompt.md" APPLY_SUMMARY_FILE="$TMPDIR/apply-summary.md" - -# Combine queued suggestion files into one input -for file in $(echo "$MANIFEST" | jq -r '.suggestions[].file'); do - { - echo "## Source: $file" - echo "" - git show "origin/$SUGGESTIONS_BRANCH:$file" 2>/dev/null || error "Suggestion file missing: $file" - echo "" - echo "---" +touch "$APPLY_SUMMARY_FILE" + +# Collect suggestion files into an array +SUGGESTION_FILES=() +while IFS= read -r file; do + SUGGESTION_FILES+=("$file") +done < <(echo "$MANIFEST" | jq -r '.suggestions[].file') + +# Determine which PRs are already in the latest stable release. +# Suggestions queued with --preview may reference features that shipped in stable +# by the time this script runs, so their Preview callouts should be stripped. +STABLE_PRS=() +STABLE_TAG=$(git tag -l 'v*' --sort=-v:refname | grep -v 'pre' | head -1 || true) +if [[ -n "$STABLE_TAG" ]]; then + log "Latest stable release tag: $STABLE_TAG" + for file in "${SUGGESTION_FILES[@]}"; do + pr_num=$(echo "$MANIFEST" | jq -r --arg f "$file" '.suggestions[] | select(.file == $f) | .pr') + # Find the merge commit for this PR + merge_sha=$(gh pr view "$pr_num" --json mergeCommit --jq '.mergeCommit.oid' 2>/dev/null || true) + if [[ -n "$merge_sha" ]] && git merge-base --is-ancestor "$merge_sha" "$STABLE_TAG" 2>/dev/null; then + STABLE_PRS+=("$pr_num") + log "PR #$pr_num is in stable ($STABLE_TAG)" + fi + done + if [[ ${#STABLE_PRS[@]} -gt 0 ]]; then + echo -e "${YELLOW}Note:${NC} ${#STABLE_PRS[@]} suggestion(s) are for PRs already in stable ($STABLE_TAG)." + echo " Preview callouts will be stripped for: ${STABLE_PRS[*]}" echo "" - } >> "$SUGGESTIONS_FILE" -done + fi +else + log "No stable release tag found, treating all suggestions as preview-only" +fi + +# Determine which PRs touch code gated behind feature flags. +# Features behind flags aren't generally available and shouldn't be documented yet. +FLAGGED_PRS=() +FLAGS_FILE="$REPO_ROOT/crates/feature_flags/src/flags.rs" +if [[ -f "$FLAGS_FILE" ]]; then + # Extract feature flag struct names (e.g. SubagentsFeatureFlag, GitGraphFeatureFlag) + FLAG_NAMES=$(grep -oE 'pub struct \w+FeatureFlag' "$FLAGS_FILE" | awk '{print $3}') + if [[ -n "$FLAG_NAMES" ]]; then + FLAG_PATTERN=$(echo "$FLAG_NAMES" | tr '\n' '|' | sed 's/|$//') + log "Feature flags found: $(echo "$FLAG_NAMES" | tr '\n' ' ')" + for file in "${SUGGESTION_FILES[@]}"; do + pr_num=$(echo "$MANIFEST" | jq -r --arg f "$file" '.suggestions[] | select(.file == $f) | .pr') + # Skip PRs already marked as stable (no need to double-check) + is_already_stable=false + for stable_pr in "${STABLE_PRS[@]+"${STABLE_PRS[@]}"}"; do + if [[ "$stable_pr" == "$pr_num" ]]; then + is_already_stable=true + break + fi + done + if [[ "$is_already_stable" == "true" ]]; then + continue + fi + # Check if the PR diff references any feature flag + pr_diff=$(gh pr diff "$pr_num" 2>/dev/null || true) + if [[ -n "$pr_diff" ]] && echo "$pr_diff" | grep -qE "$FLAG_PATTERN"; then + matched_flags=$(echo "$pr_diff" | grep -oE "$FLAG_PATTERN" | sort -u | tr '\n' ', ' | sed 's/,$//') + FLAGGED_PRS+=("$pr_num") + log "PR #$pr_num is behind feature flag(s): $matched_flags" + fi + done + if [[ ${#FLAGGED_PRS[@]} -gt 0 ]]; then + echo -e "${YELLOW}Note:${NC} ${#FLAGGED_PRS[@]} suggestion(s) are for features behind feature flags." + echo " These will be skipped: ${FLAGGED_PRS[*]}" + echo "" + fi + fi +else + log "Feature flags file not found, skipping flag detection" +fi + +# Split into batches +TOTAL=${#SUGGESTION_FILES[@]} +BATCH_COUNT=$(( (TOTAL + BATCH_SIZE - 1) / BATCH_SIZE )) + +if [[ "$BATCH_COUNT" -gt 1 ]]; then + echo "Processing $TOTAL suggestions in $BATCH_COUNT batches of up to $BATCH_SIZE..." +else + echo "Processing $TOTAL suggestions..." +fi +echo "" + +for (( batch=0; batch> "$APPLY_SUMMARY_FILE" + continue + fi + + BATCH_HAS_SUGGESTIONS=true + + # Check if PR is already in stable + is_stable=false + for stable_pr in "${STABLE_PRS[@]+"${STABLE_PRS[@]}"}"; do + if [[ "$stable_pr" == "$pr_num" ]]; then + is_stable=true + break + fi + done + { + echo "## Source: $file" + if [[ "$is_stable" == "true" ]]; then + echo "" + echo "> **ALREADY IN STABLE**: PR #$pr_num shipped in $STABLE_TAG." + echo "> Do NOT add Preview or Changed-in-Preview callouts for this suggestion." + echo "> Apply the documentation content only, without any preview-related callouts." + fi + echo "" + git show "origin/$SUGGESTIONS_BRANCH:$file" 2>/dev/null || error "Suggestion file missing: $file" + echo "" + echo "---" + echo "" + } >> "$BATCH_SUGGESTIONS_FILE" + done + + # Skip this batch if all its suggestions were flagged + if [[ "$BATCH_HAS_SUGGESTIONS" == "false" ]]; then + echo -e " ${YELLOW}Batch $BATCH_NUM skipped (all suggestions behind feature flags)${NC}" + continue + fi -# Build auto-apply prompt -cat > "$APPLY_PROMPT_FILE" << 'EOF' + # Build auto-apply prompt for this batch + cat > "$BATCH_PROMPT_FILE" << 'EOF' # Documentation Auto-Apply Request (Preview Release) Apply all queued documentation suggestions below directly to docs files in this repository. @@ -201,7 +359,15 @@ Before making edits, read and follow these rule files: 6. Keep preview callout semantics correct: - Additive features: `> **Preview:** ...` - Behavior modifications: `> **Changed in Preview (vX.XXX).** ...` + - **Exception**: Suggestions marked "ALREADY IN STABLE" must NOT get any preview callouts. + These features already shipped in a stable release. Apply the content changes only. + - Suggestions for features behind feature flags have been pre-filtered and excluded. + If you encounter references to feature-flagged functionality, do not document it. 7. If a suggestion is too ambiguous to apply safely, skip it and explain why in the summary. +8. **Do not invent `{#kb}` or `{#action}` references.** Only use action names that already + appear in the existing docs files you are editing. If unsure whether an action name is + valid, use plain text instead. The docs build validates all action references against + the compiled binary and will reject unknown names. ## Output format (after making edits) @@ -218,15 +384,28 @@ Do not include a patch in the response; apply edits directly to files. EOF -cat "$SUGGESTIONS_FILE" >> "$APPLY_PROMPT_FILE" + cat "$BATCH_SUGGESTIONS_FILE" >> "$BATCH_PROMPT_FILE" -log "Running Droid auto-apply with model: $MODEL" -if ! droid exec -m "$MODEL" -f "$APPLY_PROMPT_FILE" --auto high > "$APPLY_SUMMARY_FILE" 2>&1; then - echo "Droid exec output:" - cat "$APPLY_SUMMARY_FILE" - error "Droid exec failed. See output above." -fi -log "Droid completed, checking results..." + log "Running Droid auto-apply (batch $BATCH_NUM) with model: $MODEL" + if ! droid exec -m "$MODEL" -f "$BATCH_PROMPT_FILE" --auto high > "$BATCH_SUMMARY_FILE" 2>&1; then + echo "Droid exec output (batch $BATCH_NUM):" + cat "$BATCH_SUMMARY_FILE" + error "Droid exec failed on batch $BATCH_NUM. See output above." + fi + + # Append batch summary + { + echo "### Batch $BATCH_NUM" + echo "" + cat "$BATCH_SUMMARY_FILE" + echo "" + } >> "$APPLY_SUMMARY_FILE" + + echo -e " ${GREEN}Batch $BATCH_NUM complete${NC}" +done +echo "" + +log "All batches completed, checking results..." if [[ -n "$(git status --porcelain | grep -v '^??' | grep -vE '^.. docs/' || true)" ]]; then error "Auto-apply modified non-doc files. Revert and re-run." @@ -243,6 +422,27 @@ if [[ -z "$(git status --porcelain docs/ | grep '^.. docs/src/' || true)" ]]; th error "No docs/src changes remain after formatting; aborting PR creation." fi +# Validate docs build before creating PR +if [[ "$SKIP_VALIDATION" != "true" ]]; then + echo "Validating docs build..." + log "Generating action metadata..." + if ! ./script/generate-action-metadata > /dev/null 2>&1; then + echo -e "${YELLOW}Warning:${NC} Could not generate action metadata (cargo build may have failed)." + echo "Skipping docs build validation. CI will still catch errors." + else + VALIDATION_DIR="$TMPDIR/docs-validation" + if ! mdbook build ./docs --dest-dir="$VALIDATION_DIR" 2>"$TMPDIR/validation-errors.txt"; then + echo "" + echo -e "${RED}Docs build validation failed:${NC}" + cat "$TMPDIR/validation-errors.txt" + echo "" + error "Fix the errors above and re-run, or use --skip-validation to bypass." + fi + echo -e "${GREEN}Docs build validation passed.${NC}" + fi + echo "" +fi + # Build PR body from suggestions PR_BODY_FILE="$TMPDIR/pr-body.md" cat > "$PR_BODY_FILE" << 'EOF' @@ -314,7 +514,7 @@ Release Notes: EOF git add docs/ -git commit -m "docs: auto-apply preview release suggestions +git commit -m "docs: Auto-apply preview release suggestions Auto-applied queued documentation suggestions from: $(echo "$MANIFEST" | jq -r '.suggestions[] | "- PR #\(.pr)"') @@ -328,7 +528,7 @@ git push -u origin "$DOCS_BRANCH" log "Creating PR..." PR_URL=$(gh pr create \ --draft \ - --title "docs: auto-apply preview release suggestions" \ + --title "docs: Apply preview release suggestions" \ --body-file "$PR_BODY_FILE") echo "" @@ -370,6 +570,7 @@ EOF Previous suggestions published in: $PR_URL" + # Force push required: replacing the orphan suggestions branch with a clean slate git push -f origin "${SUGGESTIONS_BRANCH}-reset:$SUGGESTIONS_BRANCH" git checkout "$ORIGINAL_BRANCH" git branch -D "${SUGGESTIONS_BRANCH}-reset" @@ -381,9 +582,6 @@ else echo "Suggestions queue kept (--keep-queue). Remember to reset manually after PR is merged." fi -# Cleanup - - echo "" echo -e "${GREEN}Done!${NC}" echo ""