Monitor Launchpad binary builds (Infra)

tools/release/lp_build_monitor_recipe.py

@@ -7,7 +7,14 @@
 
 from lazr.restfulclient.errors import BadRequest
 
-from utils import get_build_recipe, get_date_utc_now
+from utils import (
+    get_source_build_recipe,
+    get_date_utc_now,
+    LPBuild,
+    LPSourceBuild,
+    LPBinaryBuild,
+    LPSourcePackageRecipe,
+)
 
 """
 This script triggers a build on a recipe (for the platforms configured on LP),
@@ -21,7 +28,12 @@
 LP_POLLING_DELAY = 60
 
 
-def start_all_build(build_recipe):
+def start_all_source_builds(build_recipe: LPSourcePackageRecipe):
+    """
+    Starts building a source recipe on all distro configured for it on
+    launchpad targeting the default archive connected to the recipe
+    (the daily build archive)
+    """
     for series in build_recipe.distroseries:
         try:
             build_recipe.requestBuild(
@@ -33,7 +45,20 @@ def start_all_build(build_recipe):
             print("An identical build of this recipe is already pending")
 
 
-def wait_every_build_started(build_recipe):
+def are_binary_builds_ongoing(build_recipe: LPSourcePackageRecipe) -> bool:
+    """
+    Returns whether there is any pending or ongoing build in the default
+    archive connected to the build_recipe.
+    """
+    # Note: this is a different definition of pending than the one used in
+    #       in the buildstatus enum, this means pending (as in: the build
+    #       is ready to start but no worker was assigned to it yet) and
+    #       building (as in: the build is ongoing).
+    build_counters = build_recipe.daily_build_archive.getBuildCounters()
+    return build_counters["pending"] > 0
+
+
+def wait_every_source_build_started(build_recipe: LPSourcePackageRecipe):
     pending_builds = build_recipe.getPendingBuildInfo()
     while pending_builds:
         print("Waiting some builds that are pending:")
@@ -47,7 +72,36 @@ def wait_every_build_started(build_recipe):
         pending_builds = build_recipe.getPendingBuildInfo()
 
 
-def get_new_builds(build_recipe, started_datetime):
+def recipe_name_to_source_name(name: str) -> str:
+    """
+    In Checkbox the recipe name is in the form of
+    source-name-with-dashes-risk, this function removes risk
+    """
+    return name.rsplit("-", 1)[0]
+
+
+def get_all_binary_builds(
+    build_recipe: LPSourcePackageRecipe, started_datetime
+) -> list[LPBinaryBuild]:
+    """
+    Returns all builds of the current calculated recipe target
+    started after started_date (UTC+0)
+    """
+    recipe_target = recipe_name_to_source_name(build_recipe.name)
+    builds = build_recipe.daily_build_archive.getBuildRecords(
+        source_name=recipe_target
+    )
+    return list(
+        itertools.takewhile(
+            lambda build: build.date_first_dispatched > started_datetime,
+            builds,
+        )
+    )
+
+
+def get_all_source_builds(
+    build_recipe: LPSourcePackageRecipe, started_datetime
+) -> list[LPSourceBuild]:
     """
     Returns all builds of a recipe since started_date (UTC+0)
     """
@@ -63,7 +117,51 @@ def get_new_builds(build_recipe, started_datetime):
     )
 
 
-def monitor_retry_builds(build_recipe, start_date):
+def monitor_retry_binary_builds(
+    source_recipe: LPSourcePackageRecipe, start_time
+) -> list[LPBinaryBuild]:
+    """
+    This function will monitor all binary builds in the default archive for
+    a given recipe and react to them exactly as monitor_retry_builds does.
+
+    This is different from monitor_retry_builds because it
+    will keep monitoring for new builds till there is nothing pending or
+    building in the archive. The reason this is done is that there does not
+    seem to be a way to wait for all pending binary builds in an archive to
+    start so we have no way of knowing when all pending builds actually
+    started.
+    """
+    builds_unrecoverable = []
+    start_checking_binary = start_time
+    # source builds will trigger new binary build
+    while are_binary_builds_ongoing(source_recipe):
+        # to avoid a race condition, lets get the time at the beginning of the
+        # iteration that we will use to filters binary builds the next iteration
+        iteration_start_time = get_date_utc_now()
+
+        # this filters the binary builds from start_checking_binary (the
+        # beginning of the previous iteration). This is done because if a build
+        # was included in the previous get_all_binary_builds it has been
+        # taken to completion by monitor_retry_builds either failing it
+        # completely or making it pass. What matters is that we don't
+        # under-monitor, as re-getting the same build will deterministically
+        # yield the same result resulting in the wcs. in a duplicated
+        # builds_unrecoverable record
+        binary_builds_to_check = get_all_binary_builds(
+            source_recipe, start_checking_binary
+        )
+        builds_unrecoverable += monitor_retry_builds(binary_builds_to_check)
+        start_checking_binary = iteration_start_time
+
+        if not binary_builds_to_check:
+            # this may be because all builds for this recipe are done, either
+            # way lets not spam LP with requests
+            time.sleep(LP_POLLING_DELAY)
+
+    return builds_unrecoverable
+
+
+def monitor_retry_builds(builds_to_check: list[LPBuild]) -> list[LPBuild]:
     """
     This function monitors the builds that were started after start_date
 
@@ -81,11 +179,9 @@ def monitor_retry_builds(build_recipe, start_date):
     - Cancelling build - retry or note failure
     - Cancelled build - retry or note failure
 
-    If a build can not be retried, once the monitoring of the others
-    is finished, this function will raise SystemExit with the list of
-    failed builds
+    This function returns the list of builds that didn't succeed and
+    it was unable to retry
     """
-    builds_to_check = get_new_builds(build_recipe, start_date)
     builds_unrecoverable = []
     builds_ok = []
     while builds_to_check:
@@ -116,6 +212,33 @@ def monitor_retry_builds(build_recipe, start_date):
             print(f"Build failed with status '{buildstate}' ")
             print(f"  unrecoverable: {build.web_link}")
 
+    return builds_unrecoverable
+
+
+def build_monitor_recipe(project_name: str, recipe_name: str):
+    """
+    This will start the recipe with the given name under the given project
+    and trigger->(monitor and retry) any source build till they are either
+    unrecoverably failed or successful, then (wait->monitor->retry-?>wait)
+    any binary build that the source landing in the ppa will trigger till
+    they all unrecoverably fail or succeed.
+
+    This will raise SystemExit with a human readable message if any build
+    unrecoverably fails.
+    """
+    start_time = get_date_utc_now()
+    build_recipe = get_source_build_recipe(project_name, recipe_name)
+
+    start_all_source_builds(build_recipe)
+
+    wait_every_source_build_started(build_recipe)
+    source_builds_to_check = get_all_source_builds(build_recipe, start_time)
+    builds_unrecoverable = monitor_retry_builds(source_builds_to_check)
+
+    builds_unrecoverable += monitor_retry_binary_builds(
+        build_recipe, start_time
+    )
+
     if builds_unrecoverable:
         weblinks_of_failed = "\n".join(
             build.web_link for build in builds_unrecoverable
@@ -126,17 +249,6 @@ def monitor_retry_builds(build_recipe, start_date):
         )
 
 
-def build_monitor_recipe(project_name: str, recipe_name: str):
-    start_time = get_date_utc_now()
-    build_recipe = get_build_recipe(project_name, recipe_name)
-
-    start_all_build(build_recipe)
-
-    wait_every_build_started(build_recipe)
-
-    monitor_retry_builds(build_recipe, start_time)
-
-
 def parse_args(argv: list[str]) -> argparse.Namespace:
     parser = argparse.ArgumentParser("Update a recipe of a specific project")
     parser.add_argument("project", help="Unique name of the project")

tools/release/lp_update_recipe.py

@@ -29,7 +29,7 @@
 import textwrap
 import argparse
 
-from utils import get_build_recipe
+from utils import get_source_build_recipe
 
 
 def get_build_path(recipe_name: str) -> str:
@@ -64,7 +64,7 @@ def update_build_recipe(
     version: str,
     revision: str,
 ):
-    lp_recipe = get_build_recipe(project_name, recipe_name)
+    lp_recipe = get_source_build_recipe(project_name, recipe_name)
     new_recipe = get_updated_build_recipe(recipe_name, version, revision)
     lp_recipe.recipe_text = new_recipe
     lp_recipe.lp_save()