From 2df11f7bae344bda5e0ad6c0a003d01e4c9e9624 Mon Sep 17 00:00:00 2001 From: morgankrey Date: Mon, 16 Feb 2026 20:32:51 -0600 Subject: [PATCH] background-agent: Scaffold week-one crash MVP pipeline (#49299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - add a new crash issue-linking subagent prompt (`.factory/prompts/crash/link-issues.md`) - add a scheduled/manual GitHub workflow for week-one background-agent runs (`.github/workflows/background_agent_mvp.yml`) - add Sentry candidate selection script to rank top crashes by solvability × population (`script/select-sentry-crash-candidates`) - add a local dry-run runner for end-to-end MVP execution without push/PR actions (`script/run-background-agent-mvp-local`) ## Guardrails in this MVP - draft PRs only (no auto-merge) - reviewer routing defaults to: `eholk,morgankrey,osiewicz,bennetbo` - pipeline order is: investigate -> link-issues -> fix ## Validation - `python3 -m py_compile script/select-sentry-crash-candidates script/run-background-agent-mvp-local` - `python3 script/select-sentry-crash-candidates --help` - `python3 script/run-background-agent-mvp-local --help` --------- Co-authored-by: John D. Swanson --- .factory/prompts/crash/link-issues.md | 89 +++++++ .github/workflows/background_agent_mvp.yml | 284 +++++++++++++++++++++ script/run-background-agent-mvp-local | 262 +++++++++++++++++++ script/select-sentry-crash-candidates | 242 ++++++++++++++++++ 4 files changed, 877 insertions(+) create mode 100644 .factory/prompts/crash/link-issues.md create mode 100644 .github/workflows/background_agent_mvp.yml create mode 100755 script/run-background-agent-mvp-local create mode 100755 script/select-sentry-crash-candidates diff --git a/.factory/prompts/crash/link-issues.md b/.factory/prompts/crash/link-issues.md new file mode 100644 index 0000000000000000000000000000000000000000..0413c791ed683bcb2b4248ffa329a164d1e786fd --- /dev/null +++ b/.factory/prompts/crash/link-issues.md @@ -0,0 +1,89 @@ +# Crash Issue Linking + +You are linking a crash to potentially related GitHub issues so human reviewers can quickly validate whether a fix may resolve multiple reports. + +## Inputs + +Before starting, you should have: + +1. **Crash report** (from `script/sentry-fetch ` or Sentry MCP) +2. **ANALYSIS.md** from investigation phase, including root cause and crash site + +If either is missing, stop and report what is missing. + +## Goal + +Search GitHub issues and produce a reviewer-ready shortlist grouped by confidence: + +- **High confidence** +- **Medium confidence** +- **Low confidence** + +The output is advisory only. Humans must confirm before adding closing keywords or making release claims. + +## Workflow + +### Step 1: Build Search Signals + +Extract concrete signals from the crash + analysis: + +1. Crash site function, file, and crate +2. Error message / panic text +3. Key stack frames (especially in-app) +4. Reproduction trigger phrasing (user actions) +5. Affected platform/version tags if available + +### Step 2: Search GitHub Issues + +Search **only** issues in `zed-industries/zed` (prefer `gh issue list` / `gh issue view` / GraphQL if available) by: + +1. Panic/error text +2. Function/file names +3. Crate/module names + symptom keywords +4. Similar reproduction patterns + +Check both open and recently closed issues in `zed-industries/zed`. + +### Step 3: Score Confidence + +Assign confidence based on evidence quality: + +- **High:** direct technical overlap (same crash site or same invariant violation with matching repro language) +- **Medium:** partial overlap (same subsystem and symptom, but indirect stack/repro match) +- **Low:** thematic similarity only (same area/keywords without solid technical match) + +Avoid inflated confidence. If uncertain, downgrade. + +### Step 4: Produce Structured Output + +Write `LINKED_ISSUES.md` using this exact structure: + +```markdown +# Potentially Related GitHub Issues + +## High Confidence +- [#12345](https://github.com/zed-industries/zed/issues/12345) — + - Why: <1-2 sentence evidence-backed rationale> + - Evidence: <stack frame / error text / repro alignment> + +## Medium Confidence +- ... + +## Low Confidence +- ... + +## Reviewer Checklist +- [ ] Confirm High confidence issues should be referenced in PR body +- [ ] Confirm any issue should receive closing keywords (`Fixes #...`) +- [ ] Reject false positives before merge +``` + +If no credible matches are found, keep sections present and write `- None found` under each. + +## Rules + +- Do not fabricate issues or URLs. +- Do not include issues from any repository other than `zed-industries/zed`. +- Do not add closing keywords automatically. +- Keep rationale short and evidence-based. +- Favor precision over recall. diff --git a/.github/workflows/background_agent_mvp.yml b/.github/workflows/background_agent_mvp.yml new file mode 100644 index 0000000000000000000000000000000000000000..02f9bf04cded52b9b2ac952ec362594ae4332f54 --- /dev/null +++ b/.github/workflows/background_agent_mvp.yml @@ -0,0 +1,284 @@ +name: background_agent_mvp + +on: + schedule: + - cron: "0 16 * * 1-5" + workflow_dispatch: + inputs: + crash_ids: + description: "Optional comma-separated Sentry issue IDs (e.g. ZED-4VS,ZED-123)" + required: false + type: string + reviewers: + description: "Optional comma-separated GitHub reviewer handles" + required: false + type: string + top: + description: "Top N candidates when crash_ids is empty" + required: false + type: string + default: "3" + +permissions: + contents: write + pull-requests: write + +env: + FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }} + DROID_MODEL: claude-opus-4-5-20251101 + SENTRY_ORG: zed-dev + +jobs: + run-mvp: + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Droid CLI + run: | + curl -fsSL https://app.factory.ai/cli | sh + echo "${HOME}/.local/bin" >> "$GITHUB_PATH" + echo "DROID_BIN=${HOME}/.local/bin/droid" >> "$GITHUB_ENV" + "${HOME}/.local/bin/droid" --version + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Resolve reviewers + id: reviewers + env: + INPUT_REVIEWERS: ${{ inputs.reviewers }} + DEFAULT_REVIEWERS: ${{ vars.BACKGROUND_AGENT_REVIEWERS }} + run: | + set -euo pipefail + if [ -z "$DEFAULT_REVIEWERS" ]; then + DEFAULT_REVIEWERS="eholk,morgankrey,osiewicz,bennetbo" + fi + REVIEWERS="${INPUT_REVIEWERS:-$DEFAULT_REVIEWERS}" + REVIEWERS="$(echo "$REVIEWERS" | tr -d '[:space:]')" + echo "reviewers=$REVIEWERS" >> "$GITHUB_OUTPUT" + + - name: Select crash candidates + id: candidates + env: + INPUT_CRASH_IDS: ${{ inputs.crash_ids }} + INPUT_TOP: ${{ inputs.top }} + SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_BACKGROUND_AGENT_MVP_TOKEN }} + run: | + set -euo pipefail + + PREFETCH_DIR="/tmp/crash-data" + ARGS=(--select-only --prefetch-dir "$PREFETCH_DIR" --org "$SENTRY_ORG") + if [ -n "$INPUT_CRASH_IDS" ]; then + ARGS+=(--crash-ids "$INPUT_CRASH_IDS") + else + TARGET_DRAFT_PRS="${INPUT_TOP:-3}" + if ! [[ "$TARGET_DRAFT_PRS" =~ ^[0-9]+$ ]] || [ "$TARGET_DRAFT_PRS" -lt 1 ]; then + TARGET_DRAFT_PRS="3" + fi + CANDIDATE_TOP=$((TARGET_DRAFT_PRS * 5)) + if [ "$CANDIDATE_TOP" -gt 100 ]; then + CANDIDATE_TOP=100 + fi + ARGS+=(--top "$CANDIDATE_TOP" --sample-size 100) + fi + + IDS="$(python3 script/run-background-agent-mvp-local "${ARGS[@]}")" + + if [ -z "$IDS" ]; then + echo "No candidates selected" + exit 1 + fi + + echo "Using crash IDs: $IDS" + echo "ids=$IDS" >> "$GITHUB_OUTPUT" + + - name: Run background agent pipeline per crash + id: pipeline + env: + GH_TOKEN: ${{ github.token }} + REVIEWERS: ${{ steps.reviewers.outputs.reviewers }} + CRASH_IDS: ${{ steps.candidates.outputs.ids }} + TARGET_DRAFT_PRS_INPUT: ${{ inputs.top }} + run: | + set -euo pipefail + + git config user.name "factory-droid[bot]" + git config user.email "138933559+factory-droid[bot]@users.noreply.github.com" + + # Crash ID format validation regex + CRASH_ID_PATTERN='^[A-Za-z0-9]+-[A-Za-z0-9]+$' + TARGET_DRAFT_PRS="${TARGET_DRAFT_PRS_INPUT:-3}" + if ! [[ "$TARGET_DRAFT_PRS" =~ ^[0-9]+$ ]] || [ "$TARGET_DRAFT_PRS" -lt 1 ]; then + TARGET_DRAFT_PRS="3" + fi + CREATED_DRAFT_PRS=0 + + IFS=',' read -r -a CRASH_ID_ARRAY <<< "$CRASH_IDS" + + for CRASH_ID in "${CRASH_ID_ARRAY[@]}"; do + if [ "$CREATED_DRAFT_PRS" -ge "$TARGET_DRAFT_PRS" ]; then + echo "Reached target draft PR count ($TARGET_DRAFT_PRS), stopping candidate processing" + break + fi + + CRASH_ID="$(echo "$CRASH_ID" | xargs)" + [ -z "$CRASH_ID" ] && continue + + # Validate crash ID format to prevent injection via branch names or prompts + if ! [[ "$CRASH_ID" =~ $CRASH_ID_PATTERN ]]; then + echo "ERROR: Invalid crash ID format: '$CRASH_ID' — skipping" + continue + fi + + BRANCH="background-agent/mvp-${CRASH_ID,,}-$(date +%Y%m%d)" + echo "Running crash pipeline for $CRASH_ID on $BRANCH" + + # Deduplication: skip if a draft PR already exists for this crash + EXISTING_BRANCH_PR="$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number' || echo "")" + if [ -n "$EXISTING_BRANCH_PR" ]; then + echo "Draft PR #$EXISTING_BRANCH_PR already exists for $CRASH_ID — skipping" + continue + fi + + git fetch origin main + git checkout -B "$BRANCH" origin/main + + CRASH_DATA_FILE="/tmp/crash-data/crash-${CRASH_ID}.md" + if [ ! -f "$CRASH_DATA_FILE" ]; then + echo "WARNING: No pre-fetched crash data for $CRASH_ID at $CRASH_DATA_FILE — skipping" + continue + fi + + python3 -c " + import sys + crash_id, data_file = sys.argv[1], sys.argv[2] + prompt = f'''You are running the weekly background crash-fix MVP pipeline for crash {crash_id}. + + The crash report has been pre-fetched and is available at: {data_file} + Read this file to get the crash data. Do not call script/sentry-fetch. + + Required workflow: + 1. Read the crash report from {data_file} + 2. Follow .factory/prompts/crash/investigate.md and write ANALYSIS.md + 3. Follow .factory/prompts/crash/link-issues.md and write LINKED_ISSUES.md + 4. Follow .factory/prompts/crash/fix.md to implement a minimal fix with tests + 5. Run validators required by the fix prompt for the affected crate(s) + 6. Write PR_BODY.md with sections: + - Crash Summary + - Root Cause + - Fix + - Validation + - Potentially Related Issues (High/Medium/Low from LINKED_ISSUES.md) + - Reviewer Checklist + + Constraints: + - Do not merge or auto-approve. + - Keep changes narrowly scoped to this crash. + - Do not modify files in .github/, .factory/, or script/ directories. + - When investigating git history, limit your search to the last 2 weeks of commits. Do not traverse older history. + - If the crash is not solvable with available context, write a clear blocker summary to PR_BODY.md. + ''' + import textwrap + with open('/tmp/background-agent-prompt.md', 'w') as f: + f.write(textwrap.dedent(prompt)) + " "$CRASH_ID" "$CRASH_DATA_FILE" + + if ! "$DROID_BIN" exec --auto medium -m "$DROID_MODEL" -f /tmp/background-agent-prompt.md; then + echo "Droid execution failed for $CRASH_ID, continuing to next candidate" + continue + fi + + for REPORT_FILE in ANALYSIS.md LINKED_ISSUES.md PR_BODY.md; do + if [ -f "$REPORT_FILE" ]; then + echo "::group::${CRASH_ID} ${REPORT_FILE}" + cat "$REPORT_FILE" + echo "::endgroup::" + fi + done + + if git diff --quiet; then + echo "No code changes produced for $CRASH_ID" + continue + fi + + # Stage only expected file types — not git add -A + git add -- '*.rs' '*.toml' 'Cargo.lock' 'ANALYSIS.md' 'LINKED_ISSUES.md' 'PR_BODY.md' + + # Reject changes to protected paths + PROTECTED_CHANGES="$(git diff --cached --name-only | grep -E '^(\.github/|\.factory/|script/)' || true)" + if [ -n "$PROTECTED_CHANGES" ]; then + echo "ERROR: Agent modified protected paths — aborting commit for $CRASH_ID:" + echo "$PROTECTED_CHANGES" + git reset HEAD -- . + continue + fi + + if ! git diff --cached --quiet; then + git commit -m "fix: draft crash fix for ${CRASH_ID}" + fi + + git push -u origin "$BRANCH" + + TITLE="fix: draft crash fix for ${CRASH_ID}" + BODY_FILE="PR_BODY.md" + if [ ! -f "$BODY_FILE" ]; then + BODY_FILE="/tmp/pr-body-${CRASH_ID}.md" + printf "Automated draft crash-fix pipeline output for %s.\n\nNo PR_BODY.md was generated by the agent; please review commit and linked artifacts manually.\n" "$CRASH_ID" > "$BODY_FILE" + fi + + EXISTING_PR="$(gh pr list --head "$BRANCH" --json number --jq '.[0].number')" + if [ -n "$EXISTING_PR" ]; then + gh pr edit "$EXISTING_PR" --title "$TITLE" --body-file "$BODY_FILE" + PR_NUMBER="$EXISTING_PR" + else + PR_URL="$(gh pr create --draft --base main --head "$BRANCH" --title "$TITLE" --body-file "$BODY_FILE")" + PR_NUMBER="$(basename "$PR_URL")" + fi + + if [ -n "$REVIEWERS" ]; then + IFS=',' read -r -a REVIEWER_ARRAY <<< "$REVIEWERS" + for REVIEWER in "${REVIEWER_ARRAY[@]}"; do + [ -z "$REVIEWER" ] && continue + gh pr edit "$PR_NUMBER" --add-reviewer "$REVIEWER" || true + done + fi + + CREATED_DRAFT_PRS=$((CREATED_DRAFT_PRS + 1)) + echo "Created/updated draft PRs this run: $CREATED_DRAFT_PRS/$TARGET_DRAFT_PRS" + done + + echo "created_draft_prs=$CREATED_DRAFT_PRS" >> "$GITHUB_OUTPUT" + echo "target_draft_prs=$TARGET_DRAFT_PRS" >> "$GITHUB_OUTPUT" + + - name: Cleanup pre-fetched crash data + if: always() + run: rm -rf /tmp/crash-data + + - name: Workflow summary + if: always() + env: + SUMMARY_CRASH_IDS: ${{ steps.candidates.outputs.ids }} + SUMMARY_REVIEWERS: ${{ steps.reviewers.outputs.reviewers }} + SUMMARY_CREATED_DRAFT_PRS: ${{ steps.pipeline.outputs.created_draft_prs }} + SUMMARY_TARGET_DRAFT_PRS: ${{ steps.pipeline.outputs.target_draft_prs }} + run: | + { + echo "## Background Agent MVP" + echo "" + echo "- Crash IDs: ${SUMMARY_CRASH_IDS:-none}" + echo "- Reviewer routing: ${SUMMARY_REVIEWERS:-NOT CONFIGURED}" + echo "- Draft PRs created: ${SUMMARY_CREATED_DRAFT_PRS:-0}/${SUMMARY_TARGET_DRAFT_PRS:-3}" + echo "- Pipeline: investigate -> link-issues -> fix -> draft PR" + } >> "$GITHUB_STEP_SUMMARY" + +concurrency: + group: background-agent-mvp + cancel-in-progress: false diff --git a/script/run-background-agent-mvp-local b/script/run-background-agent-mvp-local new file mode 100755 index 0000000000000000000000000000000000000000..a229b6af9e54fa58f79e5818a4145c9da8eda210 --- /dev/null +++ b/script/run-background-agent-mvp-local @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +"""Run the background crash-agent MVP pipeline locally. + +Default mode is dry-run: generate branch + agent output without commit/push/PR. +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +from datetime import datetime +from pathlib import Path + + +CRASH_ID_PATTERN = re.compile(r"^[A-Za-z0-9]+-[A-Za-z0-9]+$") + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def run(command: list[str], check: bool = True, capture_output: bool = False) -> subprocess.CompletedProcess: + return subprocess.run( + command, + cwd=REPO_ROOT, + text=True, + check=check, + capture_output=capture_output, + ) + + +def validate_crash_ids(ids: list[str]) -> list[str]: + valid = [] + for crash_id in ids: + if not CRASH_ID_PATTERN.match(crash_id): + print(f"WARNING: Skipping invalid crash ID format: '{crash_id}'", file=sys.stderr) + continue + valid.append(crash_id) + return valid + + +MAX_TOP = 100 + + +def prefetch_crash_data(crashes: list[dict[str, str]], output_dir: str) -> None: + """Fetch crash reports and save to output_dir/crash-{ID}.md. + + Each crash item must contain: + - crash_id: short ID used by the pipeline (e.g. ZED-202) + - fetch_id: identifier passed to script/sentry-fetch (short or numeric) + """ + os.makedirs(output_dir, exist_ok=True) + for crash in crashes: + crash_id = crash["crash_id"] + fetch_id = crash["fetch_id"] + output_path = os.path.join(output_dir, f"crash-{crash_id}.md") + result = run( + ["python3", "script/sentry-fetch", fetch_id], + check=False, + capture_output=True, + ) + if result.returncode != 0: + print( + f"WARNING: Failed to fetch crash data for {crash_id} " + f"(fetch id: {fetch_id}): {result.stderr.strip()}", + file=sys.stderr, + ) + continue + with open(output_path, "w", encoding="utf-8") as f: + f.write(result.stdout) + print( + f"Fetched crash data for {crash_id} (fetch id: {fetch_id}) -> {output_path}", + file=sys.stderr, + ) + + +def resolve_crashes(args) -> list[dict[str, str]]: + if args.crash_ids: + raw = [item.strip() for item in args.crash_ids.split(",") if item.strip()] + crash_ids = validate_crash_ids(raw) + return [{"crash_id": crash_id, "fetch_id": crash_id} for crash_id in crash_ids] + + top = min(args.top, MAX_TOP) + if args.top > MAX_TOP: + print(f"Capping --top from {args.top} to {MAX_TOP}", file=sys.stderr) + + with tempfile.NamedTemporaryFile(mode="w+", suffix=".json", delete=False) as file: + output_path = file.name + + try: + run( + [ + "python3", + "script/select-sentry-crash-candidates", + "--org", + args.org, + "--top", + str(top), + "--sample-size", + str(args.sample_size), + "--output", + output_path, + ], + capture_output=True, + ) + with open(output_path, "r", encoding="utf-8") as file: + payload = json.load(file) + crashes = [] + for item in payload.get("selected", []): + crash_id = item.get("short_id") + issue_id = item.get("issue_id") + if not crash_id: + continue + crashes.append( + { + "crash_id": crash_id, + "fetch_id": str(issue_id) if issue_id else crash_id, + } + ) + + valid_crash_ids = set(validate_crash_ids([item["crash_id"] for item in crashes])) + return [item for item in crashes if item["crash_id"] in valid_crash_ids] + finally: + try: + os.remove(output_path) + except OSError: + pass + + +def write_prompt(crash_id: str, crash_data_file: str | None = None) -> Path: + if crash_data_file: + fetch_step = ( + f"The crash report has been pre-fetched and is available at: {crash_data_file}\n" + f"Read this file to get the crash data. Do not call script/sentry-fetch.\n" + f"\n" + f"1. Read the crash report from {crash_data_file}" + ) + else: + fetch_step = f"1. Fetch crash data with: script/sentry-fetch {crash_id}" + + prompt = f"""You are running the weekly background crash-fix MVP pipeline for crash {crash_id}. + +Required workflow: +{fetch_step} +2. Follow .factory/prompts/crash/investigate.md and write ANALYSIS.md +3. Follow .factory/prompts/crash/link-issues.md and write LINKED_ISSUES.md +4. Follow .factory/prompts/crash/fix.md to implement a minimal fix with tests +5. Run validators required by the fix prompt for the affected crate(s) +6. Write PR_BODY.md with sections: + - Crash Summary + - Root Cause + - Fix + - Validation + - Potentially Related Issues (High/Medium/Low from LINKED_ISSUES.md) + - Reviewer Checklist + +Constraints: +- Keep changes narrowly scoped to this crash. +- If the crash is not solvable with available context, write a clear blocker summary to PR_BODY.md. +""" + + file_path = Path(tempfile.gettempdir()) / f"background-agent-{crash_id}.md" + file_path.write_text(prompt, encoding="utf-8") + return file_path + + +def run_pipeline_for_crash(args, crash_id: str) -> dict: + branch = f"background-agent/mvp-{crash_id.lower()}-{datetime.utcnow().strftime('%Y%m%d')}" + + if args.reset_branch: + run(["git", "fetch", "origin", "main"], check=False) + run(["git", "checkout", "-B", branch, "origin/main"]) + + prompt_file = write_prompt(crash_id) + try: + droid_command = [ + args.droid_bin, + "exec", + "--auto", + "medium", + "-m", + args.model, + "-f", + str(prompt_file), + ] + completed = run(droid_command, check=False) + + has_changes = run(["git", "diff", "--quiet"], check=False).returncode != 0 + return { + "crash_id": crash_id, + "branch": branch, + "droid_exit_code": completed.returncode, + "has_changes": has_changes, + } + finally: + try: + prompt_file.unlink() + except OSError: + pass + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run local background-agent MVP dry-run workflow") + parser.add_argument("--crash-ids", help="Comma-separated crash IDs (e.g. ZED-4VS,ZED-123)") + parser.add_argument("--top", type=int, default=3, help="Top N crashes when --crash-ids is omitted (max 100)") + parser.add_argument( + "--sample-size", + type=int, + default=100, + help="Number of unresolved issues to consider for candidate selection", + ) + parser.add_argument("--org", default="zed-dev", help="Sentry org slug") + parser.add_argument( + "--select-only", + action="store_true", + help="Resolve crash IDs and print them comma-separated, then exit. No agent execution.", + ) + parser.add_argument( + "--prefetch-dir", + help="When used with --select-only, also fetch crash data via script/sentry-fetch " + "and save reports to DIR/crash-{ID}.md for each resolved ID.", + ) + parser.add_argument("--model", default=os.environ.get("DROID_MODEL", "claude-opus-4-5-20251101")) + parser.add_argument("--droid-bin", default=os.environ.get("DROID_BIN", "droid")) + parser.add_argument( + "--reset-branch", + action="store_true", + help="For each crash, checkout a fresh local branch from origin/main", + ) + args = parser.parse_args() + + crashes = resolve_crashes(args) + if not crashes: + print("No crash IDs were selected.", file=sys.stderr) + return 1 + + crash_ids = [item["crash_id"] for item in crashes] + + if args.select_only: + if args.prefetch_dir: + prefetch_crash_data(crashes, args.prefetch_dir) + print(",".join(crash_ids)) + return 0 + + print(f"Running local dry-run for crashes: {', '.join(crash_ids)}") + results = [run_pipeline_for_crash(args, crash_id) for crash_id in crash_ids] + + print("\nRun summary:") + for result in results: + print( + f"- {result['crash_id']}: droid_exit={result['droid_exit_code']} " + f"changes={result['has_changes']} branch={result['branch']}" + ) + + failures = [result for result in results if result["droid_exit_code"] != 0] + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/select-sentry-crash-candidates b/script/select-sentry-crash-candidates new file mode 100755 index 0000000000000000000000000000000000000000..63aeb8e9279e716fdd9a1a17c4002e45608e1929 --- /dev/null +++ b/script/select-sentry-crash-candidates @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""Select top Sentry crash candidates ranked by solvability x impact. + +Usage: + script/select-sentry-crash-candidates --top 3 --output /tmp/candidates.json +""" + +import argparse +import configparser +import json +import math +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + +SENTRY_BASE_URL = "https://sentry.io/api/0" +DEFAULT_SENTRY_ORG = "zed-dev" +DEFAULT_QUERY = "is:unresolved issue.category:error" + + +class FetchError(Exception): + pass + + +def find_auth_token() -> str | None: + token = os.environ.get("SENTRY_AUTH_TOKEN") + if token: + return token + + sentryclirc_path = os.path.expanduser("~/.sentryclirc") + if os.path.isfile(sentryclirc_path): + config = configparser.ConfigParser() + try: + config.read(sentryclirc_path) + token = config.get("auth", "token", fallback=None) + if token: + return token + except configparser.Error: + return None + + return None + + +def api_get(path: str, token: str): + url = f"{SENTRY_BASE_URL}{path}" + request = urllib.request.Request(url) + request.add_header("Authorization", f"Bearer {token}") + request.add_header("Accept", "application/json") + + try: + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + body = error.read().decode("utf-8", errors="replace") + try: + detail = json.loads(body).get("detail", body) + except (json.JSONDecodeError, AttributeError): + detail = body + raise FetchError(f"Sentry API returned HTTP {error.code} for {path}: {detail}") + except urllib.error.URLError as error: + raise FetchError(f"Failed to connect to Sentry API: {error.reason}") + + +def fetch_issues(token: str, organization: str, limit: int, query: str): + encoded_query = urllib.parse.quote(query) + path = ( + f"/organizations/{organization}/issues/" + f"?limit={limit}&sort=freq&query={encoded_query}" + ) + return api_get(path, token) + + +def fetch_latest_event(token: str, issue_id: str): + return api_get(f"/issues/{issue_id}/events/latest/", token) + + +def parse_int(value, fallback=0) -> int: + try: + return int(value) + except (TypeError, ValueError): + return fallback + + +def in_app_frame_count(event) -> int: + entries = event.get("entries", []) + count = 0 + for entry in entries: + if entry.get("type") != "exception": + continue + exceptions = entry.get("data", {}).get("values", []) + for exception in exceptions: + frames = (exception.get("stacktrace") or {}).get("frames") or [] + count += sum(1 for frame in frames if frame.get("inApp") or frame.get("in_app")) + return count + + +def crash_signal_text(issue, event) -> str: + title = (issue.get("title") or "").lower() + culprit = (issue.get("culprit") or "").lower() + message = "" + + entries = event.get("entries", []) + for entry in entries: + if entry.get("type") != "exception": + continue + exceptions = entry.get("data", {}).get("values", []) + for exception in exceptions: + value = exception.get("value") + if value: + message = value.lower() + break + if message: + break + + return f"{title} {culprit} {message}".strip() + + +def solvable_factor(issue, event) -> tuple[float, list[str]]: + factor = 0.6 + reasons: list[str] = [] + + in_app_frames = in_app_frame_count(event) + if in_app_frames >= 6: + factor += 0.5 + reasons.append("strong in-app stack coverage") + elif in_app_frames >= 3: + factor += 0.3 + reasons.append("moderate in-app stack coverage") + else: + factor -= 0.1 + reasons.append("limited in-app stack coverage") + + signal_text = crash_signal_text(issue, event) + if "panic" in signal_text or "assert" in signal_text: + factor += 0.2 + reasons.append("panic/assert style failure") + + if "out of memory" in signal_text or "oom" in signal_text: + factor -= 0.35 + reasons.append("likely resource/system failure") + + if "segmentation fault" in signal_text or "sigsegv" in signal_text: + factor -= 0.2 + reasons.append("low-level crash signal") + + level = (issue.get("level") or "").lower() + if level == "error": + factor += 0.1 + + return max(0.2, min(1.5, factor)), reasons + + +def candidate_payload(issue, event): + issue_id = str(issue.get("id")) + short_id = issue.get("shortId") or issue_id + issue_count = parse_int(issue.get("count"), 0) + user_count = parse_int(issue.get("userCount"), 0) + population_score = issue_count + (user_count * 10) + solvability, reasons = solvable_factor(issue, event) + + score = int(math.floor(population_score * solvability)) + issue_url = f"https://sentry.io/organizations/{DEFAULT_SENTRY_ORG}/issues/{issue_id}/" + + return { + "issue_id": issue_id, + "short_id": short_id, + "title": issue.get("title") or "Unknown", + "count": issue_count, + "user_count": user_count, + "population_score": population_score, + "solvability_factor": round(solvability, 2), + "score": score, + "sentry_url": issue_url, + "reasons": reasons, + } + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Select top Sentry crash candidates ranked by solvability x impact." + ) + parser.add_argument("--org", default=DEFAULT_SENTRY_ORG, help="Sentry organization slug") + parser.add_argument("--query", default=DEFAULT_QUERY, help="Sentry issue query") + parser.add_argument("--top", type=int, default=3, help="Number of candidates to select") + parser.add_argument( + "--sample-size", + type=int, + default=25, + help="Number of unresolved issues to consider before ranking", + ) + parser.add_argument("--output", required=True, help="Output JSON file path") + args = parser.parse_args() + + token = find_auth_token() + if not token: + print( + "Error: No Sentry auth token found. Set SENTRY_AUTH_TOKEN or run sentry-cli login.", + file=sys.stderr, + ) + return 1 + + try: + issues = fetch_issues(token, args.org, args.sample_size, args.query) + except FetchError as error: + print(f"Error fetching issues: {error}", file=sys.stderr) + return 1 + + candidates = [] + for issue in issues: + issue_id = issue.get("id") + if not issue_id: + continue + + try: + event = fetch_latest_event(token, str(issue_id)) + except FetchError: + continue + + candidates.append(candidate_payload(issue, event)) + + candidates.sort(key=lambda candidate: candidate["score"], reverse=True) + selected = candidates[: max(1, args.top)] + + output = { + "organization": args.org, + "query": args.query, + "sample_size": args.sample_size, + "top": args.top, + "selected": selected, + } + + with open(args.output, "w", encoding="utf-8") as file: + json.dump(output, file, indent=2) + + print(json.dumps(output, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main())