diff --git a/script/github-track-duplicate-bot-effectiveness.py b/script/github-track-duplicate-bot-effectiveness.py index ca1ec5a9165bb9264dac1ad3fba7345a12d90f55..1ae62f36463d6059bed628b81c04aedcec792eac 100644 --- a/script/github-track-duplicate-bot-effectiveness.py +++ b/script/github-track-duplicate-bot-effectiveness.py @@ -24,6 +24,7 @@ import functools import os import re import sys +from datetime import datetime, timezone import requests @@ -39,10 +40,22 @@ BOT_START_DATE = "2026-02-18" NEEDS_TRIAGE_LABEL = "state:needs triage" DEFAULT_PROJECT_NUMBER = 76 VALID_CLOSED_AS_VALUES = {"duplicate", "not_planned", "completed"} -# Bump this when the duplicate-detection bot's behavior changes in a way that -# could affect outcome rates (e.g. prompt rewrites, model swaps, candidate -# filtering changes). Don't bump for unrelated changes like comment formatting. -BOT_VERSION = "v2" +# Add a new tuple when you deploy a new version of the bot that you want to +# keep track of (e.g. the prompt gets a rewrite or the model gets swapped). +# Newest first, please. The datetime is for the deployment time (merge to maain). +BOT_VERSION_TIMELINE = [ + ("v2", datetime(2026, 2, 26, 14, 9, tzinfo=timezone.utc)), + ("v1", datetime(2026, 2, 18, tzinfo=timezone.utc)), +] + + +def bot_version_for_time(date_string): + """Return the bot version that was active at the given ISO 8601 timestamp.""" + timestamp = datetime.fromisoformat(date_string.replace("Z", "+00:00")) + for version, deployed in BOT_VERSION_TIMELINE: + if timestamp >= deployed: + return version + return BOT_VERSION_TIMELINE[-1][0] def github_api_get(path, params=None): @@ -82,10 +95,10 @@ def fetch_issue(issue_number): } -def get_bot_duplicate_comment(issue_number): - """Get the bot's duplicate-detection comment body from an issue. +def get_bot_comment_with_time(issue_number): + """Get the bot's duplicate-detection comment and its timestamp from an issue. - Returns the comment body if found, else None. + Returns {"body": str, "created_at": str} if found, else None. """ comments_path = f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/comments" page = 1 @@ -94,7 +107,7 @@ def get_bot_duplicate_comment(issue_number): author = (comment.get("user") or {}).get("login", "") body = comment.get("body", "") if author == BOT_LOGIN and body.startswith(BOT_COMMENT_PREFIX): - return body + return {"body": body, "created_at": comment.get("created_at", "")} page += 1 return None @@ -265,7 +278,7 @@ def set_field_value(item_id, field_name, value): ) -def add_or_update_project_item(issue_node_id, outcome, closed_as=None, status="Auto-classified", notes=None): +def add_or_update_project_item(issue_node_id, outcome, closed_as=None, status="Auto-classified", notes=None, bot_comment_time=None): """Add an issue to the project board (or update it if already there), setting field values.""" item_id = find_project_item(issue_node_id) if item_id: @@ -283,7 +296,8 @@ def add_or_update_project_item(issue_node_id, outcome, closed_as=None, status="A if notes: set_field_value(item_id, "Notes", notes) - set_field_value(item_id, "Bot version", BOT_VERSION) + if bot_comment_time: + set_field_value(item_id, "Bot version", bot_version_for_time(bot_comment_time)) return item_id @@ -302,14 +316,14 @@ def classify_closed(issue_number, closer_login, state_reason): print(f" Skipping: author '{author}' is a staff member") return - bot_comment = get_bot_duplicate_comment(issue_number) + bot_comment = get_bot_comment_with_time(issue_number) bot_commented = bot_comment is not None print(f" Bot commented: {bot_commented}") closer_is_author = closer_login == author if bot_commented and closer_is_author: - classify_as_success(issue, state_reason) + classify_as_success(issue, bot_comment, state_reason) elif bot_commented and not closer_is_author: # Only authors, staff, and triagers can close issues, so # a non-author closer is always someone with elevated permissions. @@ -320,7 +334,7 @@ def classify_closed(issue_number, closer_login, state_reason): print(" Skipping: no bot comment and not closed as duplicate") -def classify_as_success(issue, state_reason): +def classify_as_success(issue, bot_comment, state_reason): """Author closed their own issue after the bot commented.""" if state_reason == "duplicate": status = "Auto-classified" @@ -340,6 +354,7 @@ def classify_as_success(issue, state_reason): closed_as=state_reason, status=status, notes=notes, + bot_comment_time=bot_comment["created_at"], ) @@ -356,12 +371,13 @@ def classify_non_author_closed(issue, bot_comment, state_reason): closed_as=state_reason, status="Needs review", notes=notes, + bot_comment_time=bot_comment["created_at"], ) def classify_as_assist(issue, bot_comment): """Staff member closed as duplicate after the bot commented. Check if the dup matches.""" - suggested = parse_suggested_issues(bot_comment) + suggested = parse_suggested_issues(bot_comment["body"]) original = None try: original = get_closed_as_duplicate_of(issue["number"]) @@ -388,7 +404,8 @@ def classify_as_assist(issue, bot_comment): print(f" -> Possible Assist, needs review ({notes})") add_or_update_project_item( - issue["node_id"], outcome="Assist", closed_as="duplicate", status=status, notes=notes) + issue["node_id"], outcome="Assist", closed_as="duplicate", status=status, notes=notes, + bot_comment_time=bot_comment["created_at"]) def classify_as_missed_opportunity(issue): @@ -425,16 +442,18 @@ def classify_open(): f"type is {type_name}" if type_name not in ("Bug", "Crash") else f"author {author} is staff" if is_staff_member(author) else "already on the board" if find_project_item(node_id) - else "no bot duplicate comment found" if not get_bot_duplicate_comment(number) + else "no bot duplicate comment found" if not (bot_comment := get_bot_comment_with_time(number)) else None ) + if skip_reason: print(f" #{number}: skipping, {skip_reason}") skipped += 1 continue print(f" #{number}: adding as Noise") - add_or_update_project_item(node_id, outcome="Noise", status="Auto-classified") + add_or_update_project_item(node_id, outcome="Noise", status="Auto-classified", + bot_comment_time=bot_comment["created_at"]) added += 1 except Exception as error: # broad catch: one issue failing shouldn't stop the sweep print(f" #{number}: error processing issue, skipping: {error}")