From 511be9a3ffa032da6bab82ddfdd2e492c68298e3 Mon Sep 17 00:00:00 2001 From: Lena <241371603+zelenenka@users.noreply.github.com> Date: Fri, 27 Feb 2026 12:56:39 +0100 Subject: [PATCH] Fix version reporting for duplicates bot (#50286) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Obviously™, we add the bot-commented issues onto the tracking board at the time of the issues getting closed, which is very much not the same time as when the bot commented. We could very well be adding the issue with a v1 bot comment to the board when v2 is already deployed. This commit stops messing up those stats. Release Notes: - N/A --- ...ithub-track-duplicate-bot-effectiveness.py | 53 +++++++++++++------ 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/script/github-track-duplicate-bot-effectiveness.py b/script/github-track-duplicate-bot-effectiveness.py index ca1ec5a9165bb9264dac1ad3fba7345a12d90f55..1ae62f36463d6059bed628b81c04aedcec792eac 100644 --- a/script/github-track-duplicate-bot-effectiveness.py +++ b/script/github-track-duplicate-bot-effectiveness.py @@ -24,6 +24,7 @@ import functools import os import re import sys +from datetime import datetime, timezone import requests @@ -39,10 +40,22 @@ BOT_START_DATE = "2026-02-18" NEEDS_TRIAGE_LABEL = "state:needs triage" DEFAULT_PROJECT_NUMBER = 76 VALID_CLOSED_AS_VALUES = {"duplicate", "not_planned", "completed"} -# Bump this when the duplicate-detection bot's behavior changes in a way that -# could affect outcome rates (e.g. prompt rewrites, model swaps, candidate -# filtering changes). Don't bump for unrelated changes like comment formatting. -BOT_VERSION = "v2" +# Add a new tuple when you deploy a new version of the bot that you want to +# keep track of (e.g. the prompt gets a rewrite or the model gets swapped). +# Newest first, please. The datetime is for the deployment time (merge to maain). +BOT_VERSION_TIMELINE = [ + ("v2", datetime(2026, 2, 26, 14, 9, tzinfo=timezone.utc)), + ("v1", datetime(2026, 2, 18, tzinfo=timezone.utc)), +] + + +def bot_version_for_time(date_string): + """Return the bot version that was active at the given ISO 8601 timestamp.""" + timestamp = datetime.fromisoformat(date_string.replace("Z", "+00:00")) + for version, deployed in BOT_VERSION_TIMELINE: + if timestamp >= deployed: + return version + return BOT_VERSION_TIMELINE[-1][0] def github_api_get(path, params=None): @@ -82,10 +95,10 @@ def fetch_issue(issue_number): } -def get_bot_duplicate_comment(issue_number): - """Get the bot's duplicate-detection comment body from an issue. +def get_bot_comment_with_time(issue_number): + """Get the bot's duplicate-detection comment and its timestamp from an issue. - Returns the comment body if found, else None. + Returns {"body": str, "created_at": str} if found, else None. """ comments_path = f"/repos/{REPO_OWNER}/{REPO_NAME}/issues/{issue_number}/comments" page = 1 @@ -94,7 +107,7 @@ def get_bot_duplicate_comment(issue_number): author = (comment.get("user") or {}).get("login", "") body = comment.get("body", "") if author == BOT_LOGIN and body.startswith(BOT_COMMENT_PREFIX): - return body + return {"body": body, "created_at": comment.get("created_at", "")} page += 1 return None @@ -265,7 +278,7 @@ def set_field_value(item_id, field_name, value): ) -def add_or_update_project_item(issue_node_id, outcome, closed_as=None, status="Auto-classified", notes=None): +def add_or_update_project_item(issue_node_id, outcome, closed_as=None, status="Auto-classified", notes=None, bot_comment_time=None): """Add an issue to the project board (or update it if already there), setting field values.""" item_id = find_project_item(issue_node_id) if item_id: @@ -283,7 +296,8 @@ def add_or_update_project_item(issue_node_id, outcome, closed_as=None, status="A if notes: set_field_value(item_id, "Notes", notes) - set_field_value(item_id, "Bot version", BOT_VERSION) + if bot_comment_time: + set_field_value(item_id, "Bot version", bot_version_for_time(bot_comment_time)) return item_id @@ -302,14 +316,14 @@ def classify_closed(issue_number, closer_login, state_reason): print(f" Skipping: author '{author}' is a staff member") return - bot_comment = get_bot_duplicate_comment(issue_number) + bot_comment = get_bot_comment_with_time(issue_number) bot_commented = bot_comment is not None print(f" Bot commented: {bot_commented}") closer_is_author = closer_login == author if bot_commented and closer_is_author: - classify_as_success(issue, state_reason) + classify_as_success(issue, bot_comment, state_reason) elif bot_commented and not closer_is_author: # Only authors, staff, and triagers can close issues, so # a non-author closer is always someone with elevated permissions. @@ -320,7 +334,7 @@ def classify_closed(issue_number, closer_login, state_reason): print(" Skipping: no bot comment and not closed as duplicate") -def classify_as_success(issue, state_reason): +def classify_as_success(issue, bot_comment, state_reason): """Author closed their own issue after the bot commented.""" if state_reason == "duplicate": status = "Auto-classified" @@ -340,6 +354,7 @@ def classify_as_success(issue, state_reason): closed_as=state_reason, status=status, notes=notes, + bot_comment_time=bot_comment["created_at"], ) @@ -356,12 +371,13 @@ def classify_non_author_closed(issue, bot_comment, state_reason): closed_as=state_reason, status="Needs review", notes=notes, + bot_comment_time=bot_comment["created_at"], ) def classify_as_assist(issue, bot_comment): """Staff member closed as duplicate after the bot commented. Check if the dup matches.""" - suggested = parse_suggested_issues(bot_comment) + suggested = parse_suggested_issues(bot_comment["body"]) original = None try: original = get_closed_as_duplicate_of(issue["number"]) @@ -388,7 +404,8 @@ def classify_as_assist(issue, bot_comment): print(f" -> Possible Assist, needs review ({notes})") add_or_update_project_item( - issue["node_id"], outcome="Assist", closed_as="duplicate", status=status, notes=notes) + issue["node_id"], outcome="Assist", closed_as="duplicate", status=status, notes=notes, + bot_comment_time=bot_comment["created_at"]) def classify_as_missed_opportunity(issue): @@ -425,16 +442,18 @@ def classify_open(): f"type is {type_name}" if type_name not in ("Bug", "Crash") else f"author {author} is staff" if is_staff_member(author) else "already on the board" if find_project_item(node_id) - else "no bot duplicate comment found" if not get_bot_duplicate_comment(number) + else "no bot duplicate comment found" if not (bot_comment := get_bot_comment_with_time(number)) else None ) + if skip_reason: print(f" #{number}: skipping, {skip_reason}") skipped += 1 continue print(f" #{number}: adding as Noise") - add_or_update_project_item(node_id, outcome="Noise", status="Auto-classified") + add_or_update_project_item(node_id, outcome="Noise", status="Auto-classified", + bot_comment_time=bot_comment["created_at"]) added += 1 except Exception as error: # broad catch: one issue failing shouldn't stop the sweep print(f" #{number}: error processing issue, skipping: {error}")