Cleanup define_masters

* os_info.yaml now provides tags / os
* os_info.yaml specifies directly the image tag. This removes the need
  to do transformations within buildbot configuration code.
* define_masters no longer generates a custom buildbot.tac by replacing
  the log file for the master with a custom name. buildbot.tac now
  uses a log file based on the master directory.
* master-config.yaml now groups builders by architecture.

buildbot.tac

@@ -28,10 +28,11 @@ log_basedir = "/var/log/buildbot"
 
 rotateLength = 20000000
 maxRotatedFiles = 30
+
+last_two_dirs = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:]
+master_name = last_two_dirs[-1]
 # Last two directories. autogen and <master-name>
-cfg_from_basedir = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:] + [
-    "master.cfg"
-]
+cfg_from_basedir = last_two_dirs + ["master.cfg"]
 
 configfile = os.path.join(*cfg_from_basedir)
 # Default umask for server
@@ -41,8 +42,9 @@ umask = None
 # directory; do not edit it.
 application = service.Application("buildmaster")  # fmt: skip
 
+# This logfile is monitored. It must end in .log.
 logfile = LogFile.fromFullPath(
-    os.path.join(log_basedir, "%s"),
+    os.path.join(log_basedir, f'{master_name}.log'),
     rotateLength=rotateLength,
     maxRotatedFiles=maxRotatedFiles,
 )

define_masters.py

@@ -3,25 +3,31 @@
 import os
 import shutil
 
+from collections import defaultdict
+
 import yaml
 
 BASE_PATH = "autogen/"
 config = {"private": {}}
-exec(open("master-private.cfg").read(), config, {})
+with open("master-private.cfg", "r") as file:
+    exec(file.read(), config, {})
+
+master_variables = config["private"]["master-variables"]
 
 with open("os_info.yaml", encoding="utf-8") as file:
     OS_INFO = yaml.safe_load(file)
 
-platforms = {}
+platforms = defaultdict(dict)
 
 for os_name in OS_INFO:
     if "install_only" in OS_INFO[os_name] and OS_INFO[os_name]["install_only"]:
         continue
     for arch in OS_INFO[os_name]["arch"]:
-        builder_name = arch + "-" + os_name
-        if arch not in platforms:
-            platforms[arch] = []
-        platforms[arch].append(builder_name)
+        builder_name = f"{arch}-{os_name}"
+        platforms[arch][os_name] = {
+            'image_tag': OS_INFO[os_name]['image_tag'],
+            'tags': OS_INFO[os_name]['tags'] if 'tags' in OS_INFO[os_name] else []
+        }
 
 # Clear old configurations
 if os.path.exists(BASE_PATH):
@@ -33,32 +39,25 @@
     # create multiple masters
     # "max_builds" is defined is master-private.py
     num_masters = (
-        int(len(platforms[arch]) / config["private"]["master-variables"]["max_builds"])
-        + 1
+        int(len(platforms[arch]) / master_variables["max_builds"]) + 1
     )
 
     for master_id in range(num_masters):
-        dir_path = BASE_PATH + arch + "-master-" + str(master_id)
+        dir_path = f'{BASE_PATH}{arch}-master-{master_id}'
         os.makedirs(dir_path)
 
-        master_config = {}
-        master_config["builders"] = platforms[arch]
-        master_config["workers"] = config["private"]["master-variables"]["workers"][
-            arch
-        ]
-        master_config["log_name"] = (
-            "master-docker-" + arch + "-" + str(master_id) + ".log"
-        )
+        master_config = {
+            'builders': {arch: platforms[arch]},
+            'workers': master_variables["workers"][arch],
+            'log_name': f'master-docker-{arch}-{master_id}.log'
+        }
 
-        with open(dir_path + "/master-config.yaml", mode="w", encoding="utf-8") as file:
+        with open(f"{dir_path}/master-config.yaml", mode="w",
+                  encoding="utf-8") as file:
             yaml.dump(master_config, file)
 
         shutil.copyfile("master.cfg", dir_path + "/master.cfg")
         shutil.copyfile("master-private.cfg", dir_path + "/master-private.cfg")
+        shutil.copyfile("buildbot.tac", dir_path + "/buildbot.tac")
 
-        buildbot_tac = (
-            open("buildbot.tac", encoding="utf-8").read() % master_config["log_name"]
-        )
-        with open(dir_path + "/buildbot.tac", mode="w", encoding="utf-8") as f:
-            f.write(buildbot_tac)
     print(arch, len(master_config["builders"]))

master-config.yaml-sample

@@ -1,14 +1,76 @@
 builders:
-- aarch64-centos-stream9
-- aarch64-debian-11
-- aarch64-debian-12
-- aarch64-debian-sid
-- aarch64-rhel-8
-- aarch64-rhel-9
-- aarch64-ubuntu-2004
-- aarch64-ubuntu-2204
+  aarch64:
+    centos-stream9:
+      image_tag: centos-stream9
+      tags:
+      - release_packages
+      - autobake
+    debian-11:
+      image_tag: debian-11
+      tags:
+      - release_packages
+      - autobake
+    debian-12:
+      image_tag: debian-12
+      tags:
+      - release_packages
+      - autobake
+    debian-sid:
+      image_tag: debian-sid
+      tags:
+      - autobake
+      - bleeding_edge
+    fedora-39:
+      image_tag: fedora-39
+      tags:
+      - autobake
+      - bleeding_edge
+    fedora-40:
+      image_tag: fedora-40
+      tags:
+      - autobake
+      - bleeding_edge
+    fedora-41:
+      image_tag: fedora-41
+      tags:
+      - autobake
+      - bleeding_edge
+    openeuler-2403:
+      image_tag: openeuler-2403
+      tags:
+      - autobake
+      - bleeding_edge
+    rhel-8:
+      image_tag: rhel-8
+      tags:
+      - autobake
+      - bleeding_edge
+    rhel-9:
+      image_tag: rhel-9
+      tags:
+      - autobake
+      - bleeding_edge
+    ubuntu-2004:
+      image_tag: ubuntu-2004
+      tags:
+      - autobake
+      - bleeding_edge
+    ubuntu-2204:
+      image_tag: ubuntu-2204
+      tags:
+      - autobake
+      - bleeding_edge
+    ubuntu-2404:
+      image_tag: ubuntu-2404
+      tags:
+      - autobake
+      - bleeding_edge
+    ubuntu-2410:
+      image_tag: ubuntu-2410
+      tags:
+      - autobake
+      - bleeding_edge
 log_name: master-docker-aarch64-0.log
-port: 9998
 workers:
 - aarch64-bbw1
 - aarch64-bbw2

master.cfg

@@ -14,7 +14,6 @@ from common_factories import (
     getSourceTarball,
 )
 from constants import (
-    BUILDERS_GALERA_MTR,
     GITHUB_STATUS_BUILDERS,
     OS_INFO,
     SAVED_PACKAGE_BRANCHES,
@@ -79,40 +78,45 @@ c["workers"] = []
 
 workers = defaultdict(list)
 
+# For each worker in master_config ['aarch64-bbw1', 2, 3, 4]
 for w_name in master_config["workers"]:
     jobs = 7
+    worker_name = w_name[:-1]  # aarch64-bbw
+    worker_id = w_name[-1]     # 1, 2, 3, 4
+
+    for arch in master_config["builders"]:
+        builders = master_config["builders"][arch]
+        for os_name in builders:
+            os_definition = builders[os_name]
+            image_tag = os_definition['image_tag']
+
+            # Skip s390x non-SLES builders on SLES host (bbw2)
+            if ("s390x" in arch
+                    and (worker_id == "2")
+                    and ("sles" not in os_name)):
+                continue
+
+            if image_tag.startswith("ubuntu"):
+                image_tag = image_tag[:-2] + "." + image_tag[-2:]
+
+            quay_name = f'{os.environ["CONTAINER_REGISTRY_URL"]}{image_tag}'
+            if arch.startswith("x86"):
+                os_name += "-i386"
+                quay_name += "-386"
+
+            base_name, name, worker_instance = createWorker(
+                worker_name,
+                worker_id,
+                os_name,
+                quay_name,
+                jobs=jobs,
+                save_packages=True,
+                shm_size="15G",
+            )
 
-    for builder in master_config["builders"]:
-        worker_name = w_name[:-1]
-        worker_id = w_name[-1]
-
-        os_name = "-".join(builder.split("-")[1:])
-        image_tag = "".join(os_name.split("-"))
-
-        # Skip s390x non-SLES builders on SLES host (bbw2)
-        if ("s390x" in builder) and (worker_id == "2") and ("sles" not in os_name):
-            continue
-
-        if image_tag.startswith("ubuntu"):
-            image_tag = image_tag[:-2] + "." + image_tag[-2:]
-
-        quay_name = os.environ["CONTAINER_REGISTRY_URL"] + image_tag
-        if builder.startswith("x86"):
-            os_name += "-i386"
-            quay_name += "-386"
-
-        base_name, name, worker_instance = createWorker(
-            worker_name,
-            worker_id,
-            os_name,
-            quay_name,
-            jobs=jobs,
-            save_packages=True,
-            shm_size="15G",
-        )
+            workers[base_name].append(name)
+            c["workers"].append(worker_instance)
 
-        workers[base_name].append(name)
-        c["workers"].append(worker_instance)
 
 ####### FACTORY CODE
 
@@ -231,80 +235,58 @@ f_deb_autobake.addStep(
 
 c["builders"] = []
 
-for builder in master_config["builders"]:
-    splits = builder.split("-")
-    arch = splits[0]
-    os_name = "-".join(splits[1:])
+for arch in master_config["builders"]:
+    builders_group = master_config["builders"][arch]
+    for os_name in builders_group:
+        worker_prefix = arch
+        worker_suffix = ''
+
+        if arch == "amd64":
+            worker_prefix = "x64"
 
-    if arch == "amd64":
-        arch = "x64"
-    worker_name = arch + "-bbw-docker-" + os_name
+        if arch == "x86":
+            worker_prefix = 'x64'
+            worker_suffix = '-i386'
+        worker_name = f'{worker_prefix}-bbw-docker-{os_name}{worker_suffix}'
 
-    if arch == "x86":
-        worker_name = "x64-bbw-docker-" + os_name + "-i386"
+        build_type = OS_INFO[os_name]["type"]
 
-    build_type = OS_INFO[os_name]["type"]
+        builder = f'{arch}-{os_name}'
 
-    # Add builder only if it's not a protected branches one
-    if builder not in GITHUB_STATUS_BUILDERS:
         tags = [os_name]
-        if arch == "s390x" and builder in BUILDERS_GALERA_MTR:
-            tags += ["experimental"]
-        if "sid" in builder or "stream-9" in builder:
-            tags += ["bleeding-edge"]
+        # Add builder only if it's not a protected branches one
+        if builder not in GITHUB_STATUS_BUILDERS:
+            c["builders"].append(
+                util.BuilderConfig(
+                    name=builder,
+                    workernames=workers[worker_name],
+                    tags=tags,
+                    collapseRequests=True,
+                    nextBuild=nextBuild,
+                    canStartBuild=canStartBuild,
+                    locks=getLocks,
+                    factory=f_quick_build,
+                )
+            )
+
+        factory_instance = f_deb_autobake if build_type != "rpm" else f_rpm_autobake
+        properties = {
+            "verbose_build": "VERBOSE=1" if arch == "ppc4le" else None,
+            "rpm_type": "".join(os_name.split("-")) if build_type == "rpm" else None
+        }
+
+        tags += [build_type, "autobake"]
+
         c["builders"].append(
             util.BuilderConfig(
-                name=builder,
+                name=builder + "-" + build_type + "-autobake",
                 workernames=workers[worker_name],
                 tags=tags,
                 collapseRequests=True,
                 nextBuild=nextBuild,
                 canStartBuild=canStartBuild,
                 locks=getLocks,
-                factory=f_quick_build,
+                properties=properties,
+                factory=factory_instance,
             )
         )
-
-    factory_instance = f_deb_autobake
-    properties = {}
-
-    if arch == "ppc64le":
-        properties["verbose_build"] = "VERBOSE=1"
-    if build_type == "rpm":
-        properties["rpm_type"] = "".join(os_name.split("-"))
-        factory_instance = f_rpm_autobake
-    tags = [os_name, build_type, "autobake"]
-    # From mariadb.org-tools/release/prep - under
-    # Dirs for buildbot.mariadb.org
-    if builder in [
-        "aarch64-openeuler-2403",
-        "amd64-openeuler-2403",
-        "s390x-ubuntu-2004",
-        "s390x-rhel-8",
-        "s390x-sles-15",
-        "ppc64le-rhel-9",
-        "s390x-rhel-9",
-        "ppc64le-ubuntu-2204",
-        "s390x-ubuntu-2204",
-        "amd64-debian-sid",
-        "aarch64-debian-sid",
-        "ppc64le-debian-sid",
-        "amd64-opensuse-1505",
-        "amd64-opensuse-1506",
-        "amd64-sles-1505",
-        "s390x-sles-1505",
-    ]:
-        tags += ["release_packages"]
-    c["builders"].append(
-        util.BuilderConfig(
-            name=builder + "-" + build_type + "-autobake",
-            workernames=workers[worker_name],
-            tags=tags,
-            collapseRequests=True,
-            nextBuild=nextBuild,
-            canStartBuild=canStartBuild,
-            locks=getLocks,
-            properties=properties,
-            factory=factory_instance,
-        )
-    )