WIP2

MariaDB · Jan 1, 2025 · 58b5d75 · 58b5d75
1 parent 7b697d1
commit 58b5d75
Show file tree

Hide file tree

Showing 8 changed files with 116 additions and 18 deletions.
diff --git a/builder.py b/builder.py
@@ -0,0 +1,44 @@
+from dataclasses import dataclass
+
+from buildbot import interfaces, steps, util
+
+
+@dataclass
+class WorkerMachine(object):
+    arch: str
+    name: str
+
+
+
+@dataclass
+class DistroDockerBuild:
+    name: str           # Distro name, such as ubuntu, rhel
+    # Version number (such as 12 (Debian), 22.04 (Ubuntu), 8 (Rhel)
+    version: str
+    version_alias: str          # Distro alias, such as Bookworm, Noble
+    docker_environment: DockerConfig    # Runtime environment config
+
+    tags: list[str]              # List of tags, presented in the UI
+    # All the build steps the docker build must run.
+    buildsteps: list[BuildStep]
+    properties: dict[str, str]
+
+    def __str__(self):
+        return f'DistroDockerBuild: {self.name}'
+
+    def create_builder_factory(self) -> util.BuildFactory:
+        f = util.BuildFactory()
+        f.addStep(StartContainerStep(self.docker_environment).generate())
+        f.addStep(FetchTarballStep().generate())
+        f.addStep(ConfigureMariaDBStep(cmake).generate())
+        f.addStep(CompileStep().generate())
+        f.addStep(MTRStep().generate())
+        f.addStep(StopContainerStep.generate())
+        return f
+        ...
+
+    def start_container_step(self) -> Build:
+        ...
+
+    def stop_container(self):
+        ...
diff --git a/define_masters.py b/define_masters.py
@@ -5,27 +5,27 @@
 
 import yaml
 
+from collections import defaultdict
+
 BASE_PATH = "autogen/"
 config = {"private": {}}
 exec(open("master-private.cfg").read(), config, {})
 
 with open("os_info.yaml", encoding="utf-8") as file:
     os_info = yaml.safe_load(file)
 
-platforms = {}
+platforms = defaultdict(list)
 
 for os_name in os_info:
     if "install_only" in os_info[os_name] and os_info[os_name]["install_only"]:
         continue
     for arch in os_info[os_name]["arch"]:
         builder_name = arch + "-" + os_name
-        if arch not in platforms:
-            platforms[arch] = []
         platforms[arch].append({
             os_name: {
                 'image_tag': os_info[os_name]['image_tag'],
                 # TODO(cvicentiu) load tags from os_info.
-                'tags': ['autobake', 'release_packages', 'bleeding_edge']
+                'tags': os_info[os_name]['tags']
             }
         })
 
@@ -47,6 +47,7 @@
     for master_id in range(num_masters):
         dir_path = f'{BASE_PATH}{arch}-master-{master_id}'
         os.makedirs(dir_path)
+        print(f'DEFINING MASTERS: {arch}, {master_id}')
 
         master_config = {
             'builders': {arch: platforms[arch]},
@@ -55,10 +56,13 @@
             'log_name': f'master-docker-{arch}-{master_id}.log'
 
         }
+        print('Before', master_config['workers'])
         with open(f"{dir_path}/master-config.yaml", mode="w",
                   encoding="utf-8") as file:
             yaml.dump(master_config, file)
 
+        print('After', master_config['workers'])
+
         shutil.copyfile("master.cfg", dir_path + "/master.cfg")
         shutil.copyfile("master-private.cfg", dir_path + "/master-private.cfg")
 

diff --git a/docker-compose/.env.orig b/docker-compose/.env.orig
@@ -0,0 +1,23 @@
+TITLE="MariaDB CI"
+TITLE_URL=https://github.com/MariaDB/server
+BUILDMASTER_URL=https://buildbot.mariadb.org/
+BUILDMASTER_WG_IP=100.64.100.1
+MQ_ROUTER_URL=ws://127.0.0.1:8080/ws
+MASTER_PACKAGES_DIR="/mnt/autofs/master_packages"
+MASTER_CREDENTIALS_DIR="/srv/buildbot/master/master-credential-provider"
+GALERA_PACKAGES_DIR="/mnt/autofs/galera_packages"
+ARTIFACTS_URL="https://ci.mariadb.org"
+NGINX_ARTIFACTS_VHOST="ci.mariadb.org"
+NGINX_BUILDBOT_VHOST="buildbot.mariadb.org"
+NGINX_CR_HOST_WG_ADDR="100.64.100.20:8080"
+ENVIRON="PROD"
+BRANCH="main"
+MASTER_NONLATENT_DOCKERLIBRARY_WORKER="bb-rhel8-docker"
+MASTER_NONLATENT_BINTARS_WORKERS='{ "ro-apexis-bbw03-x64": { "max_builds": 2, "jobs": 12 }, "bg-bbw1-x64": { "max_builds": 1, "jobs": 12 } }'
+MASTER_NONLATENT_BINTARS_VM_PORT="10000"
+<<<<<<< HEAD
+MASTER_NONLATENT_BINTARS_WORKER_PORT="10007"
+=======
+MASTER_NONLATENT_BINTARS_WORKER_PORT="10002"
+>>>>>>> 8c3a905 (WIP)
+CONTAINER_REGISTRY_URL="quay.io/mariadb-foundation/bb-worker:"
diff --git a/master-config.yaml-sample b/master-config.yaml-sample
@@ -45,7 +45,9 @@ builders:
 log_name: master-docker-aarch64-0.log
 port: 9998
 workers:
-- aarch64-bbw1
+- aarch64-bbw1:
+    skip_builds:
+      - "*sles*"  # This is a sles host
 - aarch64-bbw2
 - aarch64-bbw3
 - aarch64-bbw4
diff --git a/master.cfg b/master.cfg
@@ -106,9 +106,9 @@ for w_name in master_config["workers"]:
                     and ("sles" not in os_name)):
                 continue
 
-            if image_tag.startswith("ubuntu"):
-                image_tag = image_tag[:-2] + "." + image_tag[-2:]
+            print('Image tag:', image_tag)
 
+            # TODO: check image tags, if they match quay
             quay_name = f'{os.environ["CONTAINER_REGISTRY_URL"]}{image_tag}'
             if arch.startswith("x86"):
                 os_name += "-i386"

diff --git a/os_info.yaml b/os_info.yaml
@@ -7,6 +7,7 @@ almalinux-8:
     - aarch64
   type: rpm
   install_only: True
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 almalinux-9:
   image_tag: almalinux-9
   version_name: 9
@@ -15,6 +16,7 @@ almalinux-9:
     - aarch64
   type: rpm
   install_only: True
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 centos-stream9:
   image_tag: centos-stream9
   version_name: 9
@@ -23,13 +25,15 @@ centos-stream9:
     - aarch64
     - ppc64le
   type: rpm
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 debian-11:
   image_tag: debian-11
   version_name: bullseye
   arch:
     - amd64
     - aarch64
   type: deb
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 debian-12:
   image_tag: debian-12
   version_name: bookworm
@@ -39,6 +43,7 @@ debian-12:
     - ppc64le
     - x86
   type: deb
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 debian-sid:
   image_tag: debian-sid
   version_name: sid
@@ -48,46 +53,53 @@ debian-sid:
     - ppc64le
     - x86
   type: deb
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 fedora-39:
   image_tag: fedora-39
   version_name: 39
   arch:
     - amd64
     - aarch64
   type: rpm
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 fedora-40:
   image_tag: fedora-40
   version_name: 40
   arch:
     - amd64
     - aarch64
   type: rpm
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 fedora-41:
   image_tag: fedora-41
   version_name: 41
   arch:
     - amd64
     - aarch64
   type: rpm
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 openeuler-2403:
   image_tag: openeuler-2403
   version_name: 24.03
   arch:
     - amd64
     - aarch64
   type: rpm
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 opensuse-1505:
   image_tag: opensuse-1505
   version_name: 155
   arch:
     - amd64
   type: rpm
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 opensuse-1506:
   image_tag: opensuse-1506
   version_name: 156
   arch:
     - amd64
   type: rpm
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 rhel-8:
   image_tag: rhel-8
   version_name: 8
@@ -97,6 +109,7 @@ rhel-8:
     - ppc64le
     - s390x
   type: rpm
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 rhel-9:
   image_tag: rhel-9
   version_name: 9
@@ -113,18 +126,21 @@ rockylinux-8:
     - amd64
   type: rpm
   install_only: True
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 rockylinux-9:
   image_tag: rockylinux-9
   version_name: 9
   arch:
     - amd64
   type: rpm
   install_only: True
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 sles-1505:
   image_tag: sles-1505
   version_name: 15.5
   arch:
     - amd64
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 # TEMP - currently short on hardware - s390x
   type: rpm
 sles-1506:
@@ -134,6 +150,7 @@ sles-1506:
     - amd64
     - s390x
   type: rpm
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 ubuntu-2004:
   image_tag: ubuntu-2004
   version_name: focal
@@ -143,6 +160,7 @@ ubuntu-2004:
     - ppc64le
     - s390x
   type: deb
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 ubuntu-2204:
   image_tag: ubuntu-2204
   version_name: jammy
@@ -152,6 +170,7 @@ ubuntu-2204:
     - ppc64le
     - s390x
   type: deb
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 ubuntu-2404:
   image_tag: ubuntu-2404
   version_name: noble
@@ -161,10 +180,12 @@ ubuntu-2404:
     - ppc64le
     - s390x
   type: deb
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
 ubuntu-2410:
   image_tag: ubuntu-2410
   version_name: oracular
   arch:
     - amd64
     - aarch64
   type: deb
+  tags: ['autobake', 'release_packages', 'bleeding_edge']
diff --git a/steps/cmake/cmake_options.py b/steps/cmake/cmake_options.py
@@ -7,6 +7,7 @@
 # typos, as it's very easy to have a typo go unnoticed.
 class CMAKE(Enum):
     AR = 'AR'
+    BUILD_TYPE = 'BUILD_TYPE'
     CXX_COMPILER = 'CXX_COMPILER',
     CXX_FLAGS = 'CXX_FLAGS'
     C_COMPILER = 'C_COMPILER',

diff --git a/steps/configure.py b/steps/configure.py
@@ -1,10 +1,10 @@
-from .cmake.compilers import GCCCompilerCommand
-from .base import BuildStep
+from buildbot import interfaces, steps
 
+from .base import BuildStep
+from .cmake.cmake_options import CMakeOption, CMAKE
+from .cmake.compilers import CompilerCommand
 from .cmake.generator import CMakeGenerator
-from .cmake.compilers import GCCCompiler
-
-from buildbot import interfaces, steps
+from .cmake.mariadb_options import BuildType
 
 
 class ConfigureMariaDBCMakeStep(BuildStep):
@@ -21,11 +21,14 @@ def generate(self) -> interfaces.IBuildStep:
         )
 
 
-class ConfigureDebugBuild(ConfigureMariaDBCMakeStep):
-    def __init__(self):
-        cmake = CMakeGenerator()
-        cmake.set_compiler(GCCCompiler())
+def simple_debug_build(compiler: CompilerCommand,
+                       use_ccache: bool) -> BuildStep:
+    cmake = CMakeGenerator()
+    cmake.set_compiler(compiler)
+    if use_ccache:
         cmake.use_ccache()
+    cmake.append_flags([
+        CMakeOption(CMAKE.BUILD_TYPE, BuildType.DEBUG),
+    ])
 
-        super().__init__(build_type='Debug Build',
-                         cmake_generator=cmake)
+    return ConfigureMariaDBCMakeStep('Debug Build', cmake)