diff --git a/buildbot.tac b/buildbot.tac index 79cbe6b1..d98fb788 100644 --- a/buildbot.tac +++ b/buildbot.tac @@ -28,10 +28,11 @@ log_basedir = "/var/log/buildbot" rotateLength = 20000000 maxRotatedFiles = 30 + +last_two_dirs = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:] +master_name = last_two_dirs[-1] # Last two directories. autogen and -cfg_from_basedir = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:] + [ - "master.cfg" -] +cfg_from_basedir = last_two_dirs + ["master.cfg"] configfile = os.path.join(*cfg_from_basedir) # Default umask for server @@ -41,8 +42,9 @@ umask = None # directory; do not edit it. application = service.Application("buildmaster") # fmt: skip +# This logfile is monitored. It must end in .log. logfile = LogFile.fromFullPath( - os.path.join(log_basedir, "%s"), + os.path.join(log_basedir, f'{master_name}.log'), rotateLength=rotateLength, maxRotatedFiles=maxRotatedFiles, ) diff --git a/define_masters.py b/define_masters.py index 4b62abed..264db50f 100755 --- a/define_masters.py +++ b/define_masters.py @@ -3,25 +3,31 @@ import os import shutil +from collections import defaultdict + import yaml BASE_PATH = "autogen/" config = {"private": {}} -exec(open("master-private.cfg").read(), config, {}) +with open("master-private.cfg", "r") as file: + exec(file.read(), config, {}) + +master_variables = config["private"]["master-variables"] with open("os_info.yaml", encoding="utf-8") as file: OS_INFO = yaml.safe_load(file) -platforms = {} +platforms = defaultdict(dict) for os_name in OS_INFO: if "install_only" in OS_INFO[os_name] and OS_INFO[os_name]["install_only"]: continue for arch in OS_INFO[os_name]["arch"]: - builder_name = arch + "-" + os_name - if arch not in platforms: - platforms[arch] = [] - platforms[arch].append(builder_name) + builder_name = f"{arch}-{os_name}" + platforms[arch][os_name] = { + 'image_tag': OS_INFO[os_name]['image_tag'], + 'tags': OS_INFO[os_name]['tags'] if 'tags' in OS_INFO[os_name] else [] + } # Clear old configurations if os.path.exists(BASE_PATH): @@ -33,32 +39,25 @@ # create multiple masters # "max_builds" is defined is master-private.py num_masters = ( - int(len(platforms[arch]) / config["private"]["master-variables"]["max_builds"]) - + 1 + int(len(platforms[arch]) / master_variables["max_builds"]) + 1 ) for master_id in range(num_masters): - dir_path = BASE_PATH + arch + "-master-" + str(master_id) + dir_path = f'{BASE_PATH}{arch}-master-{master_id}' os.makedirs(dir_path) - master_config = {} - master_config["builders"] = platforms[arch] - master_config["workers"] = config["private"]["master-variables"]["workers"][ - arch - ] - master_config["log_name"] = ( - "master-docker-" + arch + "-" + str(master_id) + ".log" - ) + master_config = { + 'builders': {arch: platforms[arch]}, + 'workers': master_variables["workers"][arch], + 'log_name': f'master-docker-{arch}-{master_id}.log' + } - with open(dir_path + "/master-config.yaml", mode="w", encoding="utf-8") as file: + with open(f"{dir_path}/master-config.yaml", mode="w", + encoding="utf-8") as file: yaml.dump(master_config, file) shutil.copyfile("master.cfg", dir_path + "/master.cfg") shutil.copyfile("master-private.cfg", dir_path + "/master-private.cfg") + shutil.copyfile("buildbot.tac", dir_path + "/buildbot.tac") - buildbot_tac = ( - open("buildbot.tac", encoding="utf-8").read() % master_config["log_name"] - ) - with open(dir_path + "/buildbot.tac", mode="w", encoding="utf-8") as f: - f.write(buildbot_tac) print(arch, len(master_config["builders"])) diff --git a/master-config.yaml-sample b/master-config.yaml-sample index 7f9bcc32..9423d05e 100644 --- a/master-config.yaml-sample +++ b/master-config.yaml-sample @@ -1,14 +1,76 @@ builders: -- aarch64-centos-stream9 -- aarch64-debian-11 -- aarch64-debian-12 -- aarch64-debian-sid -- aarch64-rhel-8 -- aarch64-rhel-9 -- aarch64-ubuntu-2004 -- aarch64-ubuntu-2204 + aarch64: + centos-stream9: + image_tag: centos-stream9 + tags: + - release_packages + - autobake + debian-11: + image_tag: debian-11 + tags: + - release_packages + - autobake + debian-12: + image_tag: debian-12 + tags: + - release_packages + - autobake + debian-sid: + image_tag: debian-sid + tags: + - autobake + - bleeding_edge + fedora-39: + image_tag: fedora-39 + tags: + - autobake + - bleeding_edge + fedora-40: + image_tag: fedora-40 + tags: + - autobake + - bleeding_edge + fedora-41: + image_tag: fedora-41 + tags: + - autobake + - bleeding_edge + openeuler-2403: + image_tag: openeuler-2403 + tags: + - autobake + - bleeding_edge + rhel-8: + image_tag: rhel-8 + tags: + - autobake + - bleeding_edge + rhel-9: + image_tag: rhel-9 + tags: + - autobake + - bleeding_edge + ubuntu-2004: + image_tag: ubuntu-2004 + tags: + - autobake + - bleeding_edge + ubuntu-2204: + image_tag: ubuntu-2204 + tags: + - autobake + - bleeding_edge + ubuntu-2404: + image_tag: ubuntu-2404 + tags: + - autobake + - bleeding_edge + ubuntu-2410: + image_tag: ubuntu-2410 + tags: + - autobake + - bleeding_edge log_name: master-docker-aarch64-0.log -port: 9998 workers: - aarch64-bbw1 - aarch64-bbw2 diff --git a/master.cfg b/master.cfg index 8b0e29be..0aea869a 100644 --- a/master.cfg +++ b/master.cfg @@ -14,7 +14,6 @@ from common_factories import ( getSourceTarball, ) from constants import ( - BUILDERS_GALERA_MTR, GITHUB_STATUS_BUILDERS, OS_INFO, SAVED_PACKAGE_BRANCHES, @@ -79,40 +78,45 @@ c["workers"] = [] workers = defaultdict(list) +# For each worker in master_config ['aarch64-bbw1', 2, 3, 4] for w_name in master_config["workers"]: jobs = 7 + worker_name = w_name[:-1] # aarch64-bbw + worker_id = w_name[-1] # 1, 2, 3, 4 + + for arch in master_config["builders"]: + builders = master_config["builders"][arch] + for os_name in builders: + os_definition = builders[os_name] + image_tag = os_definition['image_tag'] + + # Skip s390x non-SLES builders on SLES host (bbw2) + if ("s390x" in arch + and (worker_id == "2") + and ("sles" not in os_name)): + continue + + if image_tag.startswith("ubuntu"): + image_tag = image_tag[:-2] + "." + image_tag[-2:] + + quay_name = f'{os.environ["CONTAINER_REGISTRY_URL"]}{image_tag}' + if arch.startswith("x86"): + os_name += "-i386" + quay_name += "-386" + + base_name, name, worker_instance = createWorker( + worker_name, + worker_id, + os_name, + quay_name, + jobs=jobs, + save_packages=True, + shm_size="15G", + ) - for builder in master_config["builders"]: - worker_name = w_name[:-1] - worker_id = w_name[-1] - - os_name = "-".join(builder.split("-")[1:]) - image_tag = "".join(os_name.split("-")) - - # Skip s390x non-SLES builders on SLES host (bbw2) - if ("s390x" in builder) and (worker_id == "2") and ("sles" not in os_name): - continue - - if image_tag.startswith("ubuntu"): - image_tag = image_tag[:-2] + "." + image_tag[-2:] - - quay_name = os.environ["CONTAINER_REGISTRY_URL"] + image_tag - if builder.startswith("x86"): - os_name += "-i386" - quay_name += "-386" - - base_name, name, worker_instance = createWorker( - worker_name, - worker_id, - os_name, - quay_name, - jobs=jobs, - save_packages=True, - shm_size="15G", - ) + workers[base_name].append(name) + c["workers"].append(worker_instance) - workers[base_name].append(name) - c["workers"].append(worker_instance) ####### FACTORY CODE @@ -231,80 +235,58 @@ f_deb_autobake.addStep( c["builders"] = [] -for builder in master_config["builders"]: - splits = builder.split("-") - arch = splits[0] - os_name = "-".join(splits[1:]) +for arch in master_config["builders"]: + builders_group = master_config["builders"][arch] + for os_name in builders_group: + worker_prefix = arch + worker_suffix = '' + + if arch == "amd64": + worker_prefix = "x64" - if arch == "amd64": - arch = "x64" - worker_name = arch + "-bbw-docker-" + os_name + if arch == "x86": + worker_prefix = 'x64' + worker_suffix = '-i386' + worker_name = f'{worker_prefix}-bbw-docker-{os_name}{worker_suffix}' - if arch == "x86": - worker_name = "x64-bbw-docker-" + os_name + "-i386" + build_type = OS_INFO[os_name]["type"] - build_type = OS_INFO[os_name]["type"] + builder = f'{arch}-{os_name}' - # Add builder only if it's not a protected branches one - if builder not in GITHUB_STATUS_BUILDERS: tags = [os_name] - if arch == "s390x" and builder in BUILDERS_GALERA_MTR: - tags += ["experimental"] - if "sid" in builder or "stream-9" in builder: - tags += ["bleeding-edge"] + # Add builder only if it's not a protected branches one + if builder not in GITHUB_STATUS_BUILDERS: + c["builders"].append( + util.BuilderConfig( + name=builder, + workernames=workers[worker_name], + tags=tags, + collapseRequests=True, + nextBuild=nextBuild, + canStartBuild=canStartBuild, + locks=getLocks, + factory=f_quick_build, + ) + ) + + factory_instance = f_deb_autobake if build_type != "rpm" else f_rpm_autobake + properties = { + "verbose_build": "VERBOSE=1" if arch == "ppc4le" else None, + "rpm_type": "".join(os_name.split("-")) if build_type == "rpm" else None + } + + tags += [build_type, "autobake"] + c["builders"].append( util.BuilderConfig( - name=builder, + name=builder + "-" + build_type + "-autobake", workernames=workers[worker_name], tags=tags, collapseRequests=True, nextBuild=nextBuild, canStartBuild=canStartBuild, locks=getLocks, - factory=f_quick_build, + properties=properties, + factory=factory_instance, ) ) - - factory_instance = f_deb_autobake - properties = {} - - if arch == "ppc64le": - properties["verbose_build"] = "VERBOSE=1" - if build_type == "rpm": - properties["rpm_type"] = "".join(os_name.split("-")) - factory_instance = f_rpm_autobake - tags = [os_name, build_type, "autobake"] - # From mariadb.org-tools/release/prep - under - # Dirs for buildbot.mariadb.org - if builder in [ - "aarch64-openeuler-2403", - "amd64-openeuler-2403", - "s390x-ubuntu-2004", - "s390x-rhel-8", - "s390x-sles-15", - "ppc64le-rhel-9", - "s390x-rhel-9", - "ppc64le-ubuntu-2204", - "s390x-ubuntu-2204", - "amd64-debian-sid", - "aarch64-debian-sid", - "ppc64le-debian-sid", - "amd64-opensuse-1505", - "amd64-opensuse-1506", - "amd64-sles-1505", - "s390x-sles-1505", - ]: - tags += ["release_packages"] - c["builders"].append( - util.BuilderConfig( - name=builder + "-" + build_type + "-autobake", - workernames=workers[worker_name], - tags=tags, - collapseRequests=True, - nextBuild=nextBuild, - canStartBuild=canStartBuild, - locks=getLocks, - properties=properties, - factory=factory_instance, - ) - ) diff --git a/os_info.yaml b/os_info.yaml index 48d12a83..d993c3f2 100644 --- a/os_info.yaml +++ b/os_info.yaml @@ -1,33 +1,53 @@ --- almalinux-8: + image_tag: almalinux-8 version_name: 8 + tags: + - release_packages + - autobake arch: - amd64 - aarch64 type: rpm install_only: True almalinux-9: + image_tag: almalinux-9 version_name: 9 + tags: + - release_packages + - autobake arch: - amd64 - aarch64 type: rpm install_only: True centos-stream9: + image_tag: centos-stream9 version_name: 9 + tags: + - release_packages + - autobake arch: - amd64 - aarch64 - ppc64le type: rpm debian-11: + image_tag: debian-11 version_name: bullseye + tags: + - release_packages + - autobake arch: - amd64 - aarch64 type: deb debian-12: + image_tag: debian-12 version_name: bookworm + tags: + - release_packages + - autobake arch: - amd64 - aarch64 @@ -35,7 +55,11 @@ debian-12: - x86 type: deb debian-sid: + image_tag: debian-sid version_name: sid + tags: + - autobake + - bleeding_edge arch: - amd64 - aarch64 @@ -43,40 +67,68 @@ debian-sid: - x86 type: deb fedora-39: + image_tag: fedora-39 version_name: 39 arch: - amd64 - aarch64 type: rpm + tags: + - autobake + - bleeding_edge fedora-40: + image_tag: fedora-40 + tags: + - autobake + - bleeding_edge version_name: 40 arch: - amd64 - aarch64 type: rpm fedora-41: + image_tag: fedora-41 + tags: + - autobake + - bleeding_edge version_name: 41 arch: - amd64 - aarch64 type: rpm openeuler-2403: + image_tag: openeuler-2403 + tags: + - autobake + - bleeding_edge version_name: 24.03 arch: - amd64 - aarch64 type: rpm opensuse-1505: + image_tag: opensuse-1505 + tags: + - autobake + - bleeding_edge version_name: 155 arch: - amd64 type: rpm opensuse-1506: + image_tag: opensuse-1506 + tags: + - autobake + - bleeding_edge version_name: 156 arch: - amd64 type: rpm rhel-8: + image_tag: rhel-8 + tags: + - autobake + - bleeding_edge version_name: 8 arch: - amd64 @@ -85,6 +137,10 @@ rhel-8: - s390x type: rpm rhel-9: + image_tag: rhel-9 + tags: + - autobake + - bleeding_edge version_name: 9 arch: - amd64 @@ -93,30 +149,50 @@ rhel-9: - s390x type: rpm rockylinux-8: + image_tag: rockylinux-8 + tags: + - autobake + - bleeding_edge version_name: 8 arch: - amd64 type: rpm install_only: True rockylinux-9: + image_tag: rockylinux-9 + tags: + - autobake + - bleeding_edge version_name: 9 arch: - amd64 type: rpm install_only: True sles-1505: + image_tag: sles-1505 + tags: + - autobake + - bleeding_edge version_name: 15.5 arch: - amd64 # TEMP - currently short on hardware - s390x type: rpm sles-1506: + image_tag: sles-1506 + tags: + - autobake + - bleeding_edge version_name: 15.6 arch: - amd64 - s390x type: rpm ubuntu-2004: + image_tag: ubuntu-2004 + tags: + - autobake + - bleeding_edge version_name: focal arch: - amd64 @@ -125,6 +201,10 @@ ubuntu-2004: - s390x type: deb ubuntu-2204: + image_tag: ubuntu-2204 + tags: + - autobake + - bleeding_edge version_name: jammy arch: - amd64 @@ -133,6 +213,10 @@ ubuntu-2204: - s390x type: deb ubuntu-2404: + image_tag: ubuntu-2404 + tags: + - autobake + - bleeding_edge version_name: noble arch: - amd64 @@ -141,6 +225,10 @@ ubuntu-2404: - s390x type: deb ubuntu-2410: + image_tag: ubuntu-2410 + tags: + - autobake + - bleeding_edge version_name: oracular arch: - amd64 diff --git a/utils.py b/utils.py index 76731a9c..e779c20a 100644 --- a/utils.py +++ b/utils.py @@ -7,6 +7,9 @@ import docker from pyzabbix import ZabbixAPI +# TODO(cvicentiu) remove +# from builder import DockerBuilder, WorkerMachine + from buildbot.buildrequest import BuildRequest from buildbot.interfaces import IProperties from buildbot.master import BuildMaster @@ -83,7 +86,7 @@ def createWorker( ], ) -> Tuple[str, str, worker.DockerLatentWorker]: worker_name = f"{worker_name_prefix}{worker_id}-docker" - name = f"{worker_name}{worker_type}{worker_name_suffix}" + name = f"{worker_name}-{worker_type}{worker_name_suffix}" # TODO(cvicentiu) Remove this list when refactoring YAML. b_name = worker_name_prefix @@ -668,3 +671,17 @@ def mtrEnv(props: IProperties) -> dict: mtr_add_env[key] = value return mtr_add_env return MTR_ENV + + +# TODO(cvicentiu) remove +# def create_latent_workers(machine: WorkerMachine, +# builders: list[DockerBuilder] +# ) -> list[worker.DockerLatentWorker]: +# result = [] +# for builder in builders: +# worker = createWorker(machine.name, +# '', +# builder.distro_name, +# builder.image) +# result.append(worker) +# return result diff --git a/validate_master_cfg.sh b/validate_master_cfg.sh index 5418b7d9..255ab365 100755 --- a/validate_master_cfg.sh +++ b/validate_master_cfg.sh @@ -68,6 +68,7 @@ fi command -v python3 >/dev/null || err "python3 command not found" +. ./docker-compose/.env python3 define_masters.py # not checking libvirt config file (//TEMP we need to find a solution # to not check ssh connection)