From 8f99de6d40465bc0477b207300e5c92d2368becd Mon Sep 17 00:00:00 2001 From: James Graham Date: Fri, 11 Oct 2024 16:11:15 +0100 Subject: [PATCH] Add a rule to sync [webcompat:sightline] whiteboard entry This should be set on all bugs that are part of the webcompat metrics set, and by no bugs that aren't. This initial approach just gets all the bugs that are either in that set or have the whiteboard entry, and implements the logic to update them in the client. Given that we expect initially to have ~700 bugs in that set, this means we need to set the maximum number of bugs to update to something rather high. --- bugbot/gcp.py | 37 ++++++++++ .../webcompat_platform_without_keyword.py | 17 +---- bugbot/rules/webcompat_sightline.py | 74 +++++++++++++++++++ scripts/cron_run_hourly.sh | 3 + 4 files changed, 116 insertions(+), 15 deletions(-) create mode 100644 bugbot/gcp.py create mode 100644 bugbot/rules/webcompat_sightline.py diff --git a/bugbot/gcp.py b/bugbot/gcp.py new file mode 100644 index 000000000..6b10d01f4 --- /dev/null +++ b/bugbot/gcp.py @@ -0,0 +1,37 @@ +from typing import Iterable, Optional + +from google.cloud import bigquery +from google.oauth2 import service_account + +from bugbot import utils + +SCOPES = { + "cloud-platform": "https://www.googleapis.com/auth/cloud-platform", + "drive": "https://www.googleapis.com/auth/drive", +} + + +def get_bigquery_client( + project: str, scopes: Optional[Iterable[str]] = None +) -> bigquery.Client: + """Get a bigquery.Client for a given project + + Args: + project: Name of the project. + scopes: Optional iterable containing the scopes the client should have. + By default this will be the cloud-platform scopes required to + run queries. + Returns: + bigquery.Client + """ + scope_urls = ( + [SCOPES["cloud-platform"]] + if scopes is None + else [SCOPES.get(item, item) for item in scopes] + ) + + credentials = service_account.Credentials.from_service_account_info( + utils.get_gcp_service_account_info() + ).with_scopes(scope_urls) + + return bigquery.Client(project=project, credentials=credentials) diff --git a/bugbot/rules/webcompat_platform_without_keyword.py b/bugbot/rules/webcompat_platform_without_keyword.py index 7d2b2e37f..85b0a371f 100644 --- a/bugbot/rules/webcompat_platform_without_keyword.py +++ b/bugbot/rules/webcompat_platform_without_keyword.py @@ -2,10 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -from google.cloud import bigquery -from google.oauth2 import service_account - -from bugbot import utils +from bugbot import gcp from bugbot.bzcleaner import BzCleaner @@ -34,17 +31,7 @@ def get_core_bug_ids(self): project = "moz-fx-dev-dschubert-wckb" dataset = "webcompat_knowledge_base" - credentials = service_account.Credentials.from_service_account_info( - utils.get_gcp_service_account_info() - ).with_scopes( - [ - "https://www.googleapis.com/auth/cloud-platform", - "https://www.googleapis.com/auth/drive", - ] - ) - - client = bigquery.Client(project=project, credentials=credentials) - + client = gcp.get_bigquery_client(project, ["cloud-platform", "drive"]) query = f""" SELECT core_bug FROM `{project}.{dataset}.prioritized_kb_entries` as kb_entries diff --git a/bugbot/rules/webcompat_sightline.py b/bugbot/rules/webcompat_sightline.py new file mode 100644 index 000000000..9bd7c78e6 --- /dev/null +++ b/bugbot/rules/webcompat_sightline.py @@ -0,0 +1,74 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from typing import Any, Optional + +from bugbot import gcp +from bugbot.bzcleaner import BzCleaner + + +class WebcompatSightline(BzCleaner): + WHITEBOARD_ENTRY = "[webcompat:sightline]" + + def __init__(self): + super().__init__() + self.sightline_ids = set() + + def description(self) -> str: + return "Web Compat site report in the sightline metric set" + + def filter_no_nag_keyword(self) -> bool: + return False + + def has_default_products(self) -> bool: + return False + + def handle_bug( + self, bug: dict[str, Any], data: dict[str, Any] + ) -> Optional[dict[str, Any]]: + bug_id = str(bug["id"]) + whiteboard = bug["whiteboard"] + + if bug["id"] in self.sightline_ids: + if self.WHITEBOARD_ENTRY not in whiteboard: + self.autofix_changes[bug_id] = { + "whiteboard": whiteboard + self.WHITEBOARD_ENTRY + } + elif self.WHITEBOARD_ENTRY in whiteboard: + self.autofix_changes[bug_id] = { + "whiteboard": whiteboard.replace(self.WHITEBOARD_ENTRY, "") + } + + return None + + def get_bz_params(self, date) -> dict[str, Any]: + fields = ["id", "summary", "whiteboard"] + self.sightline_ids = self.get_sightline_bug_ids() + # Get all bugs that either have, or should have, the [webcompat:sightline] + # whiteboard entry + return { + "include_fields": fields, + "j_top": "OR", + "f1": "bug_id", + "o1": "anyexact", + "v1": ",".join(str(item) for item in self.sightline_ids), + "f2": "status_whiteboard", + "o2": "substring", + "v2": self.WHITEBOARD_ENTRY, + } + + def get_sightline_bug_ids(self) -> set[int]: + project = "moz-fx-dev-dschubert-wckb" + dataset = "webcompat_knowledge_base" + + client = gcp.get_bigquery_client(project, ["cloud-platform", "drive"]) + query = f""" + SELECT number FROM `{project}.{dataset}.webcompat_topline_metric_site_reports` as bugs + """ + + return {row["number"] for row in client.query(query).result()} + + +if __name__ == "__main__": + WebcompatSightline().run() diff --git a/scripts/cron_run_hourly.sh b/scripts/cron_run_hourly.sh index 1353efefa..146e27b14 100755 --- a/scripts/cron_run_hourly.sh +++ b/scripts/cron_run_hourly.sh @@ -75,4 +75,7 @@ python -m bugbot.rules.duplicate_copy_metadata --production # Add `webcompat:platform-bug` keyword to bugs without a platform keyword python -m bugbot.rules.webcompat_platform_without_keyword --production +# Add `[webcompat:sightline]` whiteboard entry to bugs in sightline metric set +python -m bugbot.rules.webcompat_sightline --production + source ./scripts/cron_common_end.sh