diff --git a/.github/workflows/deb-daily-builds.yml b/.github/workflows/deb-daily-builds.yml index 9dc06f436d..1136524708 100644 --- a/.github/workflows/deb-daily-builds.yml +++ b/.github/workflows/deb-daily-builds.yml @@ -5,26 +5,70 @@ on: workflow_call: jobs: - deb_daily_builds: - name: Debian packages daily build + ppa_update: + name: Sync PPA history with monorepo runs-on: [self-hosted, linux, large] steps: - name: Install dependencies run: | sudo apt update -qq - sudo apt install -qq -y python3-launchpadlib python3-setuptools-scm - git clone -b main https://git.launchpad.net/~hook25/ppa-dev-tools /tmp/ppa-dev-tools + sudo apt install -qq -y python3-launchpadlib - name: Checkout checkbox monorepo uses: actions/checkout@v3 with: fetch-depth: 0 - uses: Wandalen/wretry.action@a163f62ae554a8f3cbe27b23db15b60c0ae2e93c # v1.3.0 - name: Import to LP, ask for a build and wait result + name: Make LP pull the monorepo env: LP_CREDENTIALS: ${{ secrets.LP_CREDS }} PYTHONUNBUFFERED: 1 with: - attempt_delay: 600000 # 10min + attempt_delay: 6000 # 1 min + attempt_limit: 60 # max 1 hour of retries + command: | + tools/release/lp-request-import.py "~checkbox-dev/checkbox/+git/checkbox" + ppa_build: + name: Trigger and monitor PPA builds + runs-on: [self-hosted, linux, large] + needs: ppa_update + strategy: + matrix: + recipe: + - checkbox-ng-edge + - checkbox-support-edge + - checkbox-provider-base-edge + - checkbox-provider-resource-edge + - checkbox-provider-certification-server-edge + - checkbox-provider-certification-client-edge + - checkbox-provider-gpgpu-edge + - checkbox-provider-sru-edge + - checkbox-provider-tpm2-edge + steps: + - name: Install dependencies + run: | + sudo apt update -qq + sudo apt install -qq -y python3-launchpadlib + - name: Checkout checkbox monorepo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: Wandalen/wretry.action@a163f62ae554a8f3cbe27b23db15b60c0ae2e93c # v1.3.0 + name: Update the recipe in the checkbox PPA + env: + LP_CREDENTIALS: ${{ secrets.LP_CREDS }} + PYTHONUNBUFFERED: 1 + with: + attempt_delay: 6000 # 1 min + attempt_limit: 60 # max 1 hour of retries + command: | + tools/release/lp_update_recipe.py checkbox --recipe ${{ matrix.recipe }} --new-version $(tools/release/get_version.py --dev-suffix --output-format deb) --revision $GITHUB_SHA + - uses: Wandalen/wretry.action@a163f62ae554a8f3cbe27b23db15b60c0ae2e93c # v1.3.0 + name: Build and wait result + env: + LP_CREDENTIALS: ${{ secrets.LP_CREDS }} + PYTHONUNBUFFERED: 1 + with: + attempt_delay: 60000 # 10min attempt_limit: 3 command: | - tools/release/deb_daily_builds.py + tools/release/lp_build_monitor_recipe.py checkbox ${{ matrix.recipe }} diff --git a/tools/release/deb_daily_builds.py b/tools/release/deb_daily_builds.py deleted file mode 100755 index f3a6660e58..0000000000 --- a/tools/release/deb_daily_builds.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 -# This file is part of Checkbox. -# -# Copyright 2022-2023 Canonical Ltd. -# Written by: -# Sylvain Pineau -# -# Checkbox is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, -# as published by the Free Software Foundation. -# -# Checkbox is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Checkbox. If not, see . - -import os -import atexit -import tempfile -import subprocess - -from functools import partial -from contextlib import suppress -from get_version import get_version, OutputFormats - -CONFIG_PPA_DEV_TOOLS = """{{ - 'wait_max_age_hours' : 10, - 'exit_on_only_build_failure' : True, - 'wait_seconds' : 60, - 'name' : '{}' -}} -""" - - -def run(*args, **kwargs): - """wrapper for subprocess.run.""" - try: - return subprocess.run( - *args, **kwargs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - except subprocess.CalledProcessError as e: - print("{}\n{}".format(e, e.output.decode())) - raise SystemExit(1) - - -def _del_file(path): - with suppress(FileNotFoundError): - os.remove(path) - - -def check_build(name) -> bool: - """ - Checks if a build was succesful, returns true if it was - """ - handle, path = tempfile.mkstemp(text=True) - # try to remove the file before exit - atexit.register(partial(_del_file, path)) - with os.fdopen(handle, "w") as f: - f.write(CONFIG_PPA_DEV_TOOLS.format(name)) - with suppress(subprocess.CalledProcessError): - subprocess.check_call( - [ - "/tmp/ppa-dev-tools/scripts/ppa", - "wait", - "ppa:checkbox-dev/edge", - "-C", - path, - ] - ) - return True - return False - - -def main(): - """Parse the checkbox monorepo to trigger deb daily builds in Launchpad. - The daily builds will be stored under the checkbox-dev/edge PPA.""" - # First request code import (GitHub -> Launchpad) - run( - "./tools/release/lp-request-import.py " - "~checkbox-dev/checkbox/+git/checkbox", - shell=True, - check=True, - ) - projects = {} - for path, dirs, files in os.walk("."): - if "debian" in dirs: - project_path = os.path.relpath(path) - # Tweak the provider paths to get names in the following form: - # providers/base -> checkbox-provider-base - project_name = project_path.replace("s/", "-") - if project_name.startswith("provider"): - project_name = "checkbox-" + project_name - projects[project_name] = project_path - to_check = [] - # Find projects new commits from the last 24 hours - for name, path in sorted(projects.items(), key=lambda i: i[1]): - version = get_version( - dev_suffix=True, output_format=OutputFormats.DEB, verbose=True - ) - output = ( - run( - "./tools/release/lp-recipe-update-build.py checkbox " - "--recipe {} -n {}".format(name + "-edge", version), - shell=True, - check=True, - ) - .stdout.decode() - .rstrip() - ) - print(output) - # We have started the build, store it here so it can - # be checked after. - to_check.append(name) - - checked = [(name, check_build(name)) for name in to_check] - any_failed = False - for name, ok in checked: - if not ok: - any_failed = True - print("Failed to build:", name) - if any_failed: - raise SystemExit("Some build failed") - - -if __name__ == "__main__": - main() diff --git a/tools/release/lp-recipe-update-build.py b/tools/release/lp-recipe-update-build.py deleted file mode 100755 index c2c2970975..0000000000 --- a/tools/release/lp-recipe-update-build.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/python3 -# This file is part of Checkbox. -# -# Copyright 2016 Canonical Ltd. -# Written by: -# Sylvain Pineau -# -# Checkbox is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, -# as published by the Free Software Foundation. -# -# Checkbox is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Checkbox. If not, see . - -""" -Kicks off a recipe build for a branch in Launchpad which has an associated -recipe. -Meant to be used as part of a checkbox release to the Hardware Certification -public PPA and the ~checkbox-dev testing PPA. - -NOTE: The Hardware Certification public PPA has no further use over the -~checkbox-dev testing PPA. It is kept for historical reasons and should be -removed at some point. -""" - -import os -import re -import sys - -from launchpadlib.credentials import Credentials -from launchpadlib.launchpad import Launchpad -from lazr.restfulclient.errors import BadRequest -from argparse import ArgumentParser - - -def main(): - parser = ArgumentParser("Invoke a recipe build on specified branch") - parser.add_argument("project", help="Unique name of the project") - parser.add_argument( - "--recipe", - "-r", - help="Recipe name to build with. If there is only one " - "then that will be used by default, if not then " - "this must be specified.", - ) - parser.add_argument( - "--new-version", - "-n", - help="New version to use in the recipe " - "(for debian changelog) and bzr tags.", - ) - args = parser.parse_args() - - credentials = Credentials.from_string(os.getenv("LP_CREDENTIALS")) - lp = Launchpad( - credentials, None, None, service_root="production", version="devel" - ) - try: - project = lp.projects[args.project] - except KeyError: - parser.error("{} was not found in Launchpad.".format(args.project)) - - if project.recipes.total_size == 0: - parser.error("{} does not have any recipes.".format(args.project)) - else: - build_recipe = None - - if project.recipes.total_size == 1: - build_recipe = project.recipes[0] - elif args.recipe: - for recipe in project.recipes: - if recipe.name == args.recipe: - build_recipe = recipe - else: - all_recipe_names = [recipe.name for recipe in project.recipes] - parser.error( - "I don't know which recipe from " - "{project} you want to use, specify " - "one of '{recipes}' using --recipe".format( - project=args.project, recipes=", ".join(all_recipe_names) - ) - ) - - new_version = args.new_version - - text = build_recipe.recipe_text - # 0.28.0rc1 → 0.28.0~rc1 - deb_version = re.sub(r"(rc\d+)$", "~\g<0>", new_version) - # 0.28.0rc1 → 0.28.0_rc1 - deb_tag = re.sub(r"(rc\d+)$", "_\g<0>", new_version) - text = re.sub( - r"deb-version .*?~ppa", - "deb-version {}~ppa".format(deb_version), - text, - ) - # v0.27.0 → v0.28.0rc1 - text = re.sub(r"\bv\d\S+", "v{}".format(new_version), text) - # debian-0.27.0-1 → debian-0.28.0_rc1-1 - text = re.sub(r"debian-(.*)$", "debian-{}-1".format(deb_tag), text) - # {debupstream}+{revtime:monorepo}+git{git-commit:monorepo} → version - text = re.sub(r"{debupstream}\S+", new_version, text) - # 2.9.dev38+g896ae8978 → 2.9.dev57+g703bc6517 - text = re.sub( - r"deb-version \d\S+", - "deb-version {}".format(new_version), - text, - ) - build_recipe.recipe_text = text - build_recipe.lp_save() - if build_recipe: - for series in build_recipe.distroseries: - try: - build_recipe.requestBuild( - pocket="Release", - distroseries=series, - archive=build_recipe.daily_build_archive_link, - ) - except BadRequest: - print( - "An identical build of this recipe is " - "already pending" - ) - print("Check builds status: " + build_recipe.web_link) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/release/lp_build_monitor_recipe.py b/tools/release/lp_build_monitor_recipe.py old mode 100644 new mode 100755 index 44d6e125c0..abd789586d --- a/tools/release/lp_build_monitor_recipe.py +++ b/tools/release/lp_build_monitor_recipe.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import sys import time import argparse diff --git a/tools/release/utils.py b/tools/release/utils.py index 70bcc3653b..b1cb43254e 100644 --- a/tools/release/utils.py +++ b/tools/release/utils.py @@ -47,4 +47,4 @@ def get_build_recipe(project_name: str, recipe_name: str): ) def get_date_utc_now(): - return datetime.datetime.now(tz=datetime.UTC) + return datetime.datetime.now(tz=datetime.timezone.utc)