packaging: provide centos 9 stream builds (fluent#6039)

* packaging: add centos/9 build target based on centos 9 stream Signed-off-by: Patrick Stephens <[email protected]> * packaging: switch to JSON file for target config Signed-off-by: Patrick Stephens <[email protected]> * packaging: add centos/9 test Signed-off-by: Patrick Stephens <[email protected]> * packaging: update install script and version Signed-off-by: Patrick Stephens <[email protected]> * packaging: fix linting issues Signed-off-by: Patrick Stephens <[email protected]> * packaging: fix linting issues Signed-off-by: Patrick Stephens <[email protected]> * packaging: remove td-agent-bit in testing Signed-off-by: Patrick Stephens <[email protected]> * packaging: disable FLB_TD in build Signed-off-by: Patrick Stephens <[email protected]> * packaging: disable FLB_TD in build Signed-off-by: Patrick Stephens <[email protected]> * packaging: provide default container name Signed-off-by: Patrick Stephens <[email protected]> Signed-off-by: Patrick Stephens <[email protected]>
nikomauno · Sep 12, 2022 · afe2cbd · afe2cbd
1 parent 5928fa1
commit afe2cbd
Show file tree

Hide file tree

Showing 11 changed files with 130 additions and 17 deletions.
diff --git a/.github/actions/generate-package-build-matrix/action.yaml b/.github/actions/generate-package-build-matrix/action.yaml
@@ -35,11 +35,21 @@ runs:
         if [[ -d "packaging" ]]; then
           BUILD_TYPE="1.9"
         fi
+        if [[ -f "packaging/build-config.json" ]]; then
+          BUILD_TYPE="2.0"
+        fi
         echo "Detected type: $BUILD_TYPE"
         echo ::set-output name=BUILD_TYPE::$BUILD_TYPE
       shell: bash
       working-directory: version-check
 
+    - name: 2.0 targets
+      if: steps.determine-build-type.outputs.BUILD_TYPE == '2.0'
+      run: |
+        matrix=$(echo '{ "distro" : '$(jq -cr '.linux_targets' packaging/build-config.json)'}'|jq -c .)
+        echo "MATRIX=$matrix" >> $GITHUB_ENV
+      shell: bash
+
     - name: 1.9 targets
       if: steps.determine-build-type.outputs.BUILD_TYPE == '1.9'
       run: |

diff --git a/codebase-structure.svg b/codebase-structure.svg
diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile
@@ -11,7 +11,7 @@
 # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/
 
 # Set this to the current release version: it gets done so as part of the release.
-ARG RELEASE_VERSION=1.9.6
+ARG RELEASE_VERSION=2.0.0
 
 # For multi-arch builds - assumption is running on an AMD64 host
 FROM multiarch/qemu-user-static:x86_64-arm as qemu-arm32

diff --git a/install.sh b/install.sh
@@ -49,7 +49,7 @@ enabled=1
 EOF
 sed -i 's|VERSION_ARCH_SUBSTR|\$releasever/\$basearch/|g' /etc/yum.repos.d/fluent-bit.repo
 cat /etc/yum.repos.d/fluent-bit.repo
-yum -y install fluent-bit || yum -y install td-agent-bit
+yum -y install fluent-bit
 SCRIPT
     ;;
     centos|centoslinux|rhel|redhatenterpriselinuxserver|fedora|rocky|almalinux)
@@ -66,7 +66,7 @@ enabled=1
 EOF
 sed -i 's|VERSION_ARCH_SUBSTR|\$releasever/\$basearch/|g' /etc/yum.repos.d/fluent-bit.repo
 cat /etc/yum.repos.d/fluent-bit.repo
-yum -y install fluent-bit || yum -y install td-agent-bit
+yum -y install fluent-bit
 SCRIPT
     ;;
     ubuntu|debian)
@@ -81,7 +81,7 @@ deb [signed-by=/usr/share/keyrings/fluentbit-keyring.gpg] $RELEASE_URL/${OS}/${C
 EOF
 cat /etc/apt/sources.list.d/fluent-bit.list
 apt-get -y update
-apt-get -y install fluent-bit || apt-get -y install td-agent-bit
+apt-get -y install fluent-bit
 SCRIPT
     ;;
     *)

diff --git a/packaging/build-config.json b/packaging/build-config.json
@@ -0,0 +1,25 @@
+{
+    "linux_targets": [
+        "amazonlinux/2",
+        "amazonlinux/2.arm64v8",
+        "centos/7",
+        "centos/7.arm64v8",
+        "centos/8",
+        "centos/8.arm64v8",
+        "centos/9",
+        "centos/9.arm64v8",
+        "debian/buster",
+        "debian/buster.arm64v8",
+        "debian/bullseye",
+        "debian/bullseye.arm64v8",
+        "ubuntu/16.04",
+        "ubuntu/18.04",
+        "ubuntu/18.04.arm64v8",
+        "ubuntu/20.04",
+        "ubuntu/20.04.arm64v8",
+        "ubuntu/22.04",
+        "ubuntu/22.04.arm64v8",
+        "raspbian/buster",
+        "raspbian/bullseye"
+      ]
+}
diff --git a/packaging/distros/centos/Dockerfile b/packaging/distros/centos/Dockerfile
@@ -98,6 +98,41 @@ ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL
 ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
 ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
 
+FROM quay.io/centos/centos:stream9 as centos-9-base
+
+# Add for the YAML development libraries
+# hadolint ignore=DL3033,DL3041
+RUN dnf -y install 'dnf-command(config-manager)' && dnf -y config-manager --set-enabled crb && \
+    dnf -y install rpm-build ca-certificates gcc gcc-c++ cmake make bash \
+                   wget unzip systemd-devel wget flex bison \
+                   postgresql-libs postgresql-devel postgresql-server postgresql \
+                   cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel && \
+    dnf clean all
+
+ARG FLB_OUT_PGSQL=On
+ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL
+
+# hadolint ignore=DL3029
+FROM --platform=arm64 quay.io/centos/centos:stream9 as centos-9.arm64v8-base
+
+COPY --from=multiarch-aarch64 /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64-static
+
+# Add for the YAML development libraries
+# hadolint ignore=DL3033,DL3041
+RUN dnf -y install 'dnf-command(config-manager)' && dnf -y config-manager --set-enabled crb && \
+    dnf -y install rpm-build ca-certificates gcc gcc-c++ cmake make bash \
+                   wget unzip systemd-devel wget flex bison \
+                   postgresql-libs postgresql-devel postgresql-server postgresql \
+                   cyrus-sasl-lib openssl openssl-libs openssl-devel libyaml-devel && \
+    dnf clean all
+
+ARG FLB_OUT_PGSQL=On
+ENV FLB_OUT_PGSQL=$FLB_OUT_PGSQL
+
+# Need larger page size
+ARG FLB_JEMALLOC_OPTIONS="--with-lg-page=16 --with-lg-quantum=3"
+ENV FLB_JEMALLOC_OPTIONS=$FLB_JEMALLOC_OPTIONS
+
 # Common build for all distributions now
 # hadolint ignore=DL3006
 FROM $BASE_BUILDER as builder
@@ -113,9 +148,9 @@ WORKDIR /tmp/fluent-bit/build/
 # CMake configuration variables
 # Unused
 ARG CFLAGS
-ARG CMAKE_INSTALL_PREFIX=/opt/td-agent-bit/
+ARG CMAKE_INSTALL_PREFIX=/opt/fluent-bit/
 ARG CMAKE_INSTALL_SYSCONFDIR=/etc/
-ARG FLB_TD=On
+ARG FLB_TD=Off
 ARG FLB_RELEASE=On
 ARG FLB_TRACE=On
 ARG FLB_SQLDB=On

diff --git a/packaging/local-build-all.sh b/packaging/local-build-all.sh
@@ -14,12 +14,8 @@ SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 # aarch64
 #
 
-# Ensure this is updated for new targets
-# We do the arm64 targets at the end as ideally the amd64 ones trigger any issues with dependencies.
-declare -a TARGETS=("amazonlinux/2"
-"centos/7" "centos/8" "debian/buster" "debian/bullseye" "raspbian/buster" "raspbian/bullseye" "ubuntu/16.04" "ubuntu/18.04" "ubuntu/20.04"
-"amazonlinux/2.arm64v8" "centos/7.arm64v8" "centos/8.arm64v8" "debian/buster.arm64v8" "debian/bullseye.arm64v8" "ubuntu/18.04.arm64v8" "ubuntu/20.04.arm64v8"
-)
+# The local file with all the supported build configs in
+JSON_FILE_NAME=${JSON_FILE_NAME:-$SCRIPT_DIR/build-config.json}
 
 # Output checks are easier plus do not want to fill up git
 PACKAGING_OUTPUT_DIR=${PACKAGING_OUTPUT_DIR:-test}
@@ -28,7 +24,7 @@ rm -rf "${PACKAGING_OUTPUT_DIR:?}/*"
 
 # Iterate over each target and attempt to build it.
 # Verify that an RPM or DEB is created.
-for DISTRO in "${TARGETS[@]}"
+jq -cr '.linux_targets[]' "$JSON_FILE_NAME" | while read -r DISTRO
 do
     echo "$DISTRO"
     FLB_OUT_DIR="$PACKAGING_OUTPUT_DIR" /bin/bash "$SCRIPT_DIR"/build.sh -d "$DISTRO" "$@"

diff --git a/packaging/test-release-packages.sh b/packaging/test-release-packages.sh
@@ -19,6 +19,7 @@ APT_TARGETS=("ubuntu:18.04"
 
 YUM_TARGETS=("centos:7"
     "rockylinux:8"
+    "quay.io/centos/centos:stream9"
     "amazonlinux:2")
 
 for IMAGE in "${APT_TARGETS[@]}"

diff --git a/packaging/testing/smoke/container/container-smoke-test.sh b/packaging/testing/smoke/container/container-smoke-test.sh
@@ -18,6 +18,8 @@ SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 # Simple smoke test script of each local container architecture that runs up and then
 # ensure we can access the web server.
 
+CONTAINER_NAME=${CONTAINER_NAME:-smoke-test}
+
 # Ensure the container name is valid as we need this to retrieve the exposed ports.
 CONTAINER_NAME=${CONTAINER_NAME//\//-}
 CONTAINER_ARCH=${CONTAINER_ARCH:-linux/amd64}

diff --git a/packaging/testing/smoke/packages/Dockerfile.centos9 b/packaging/testing/smoke/packages/Dockerfile.centos9
@@ -0,0 +1,45 @@
+# For staging upgrade we use the 'staging-upgrade-prep' as the base
+ARG STAGING_BASE=dokken/centos-9
+
+ARG RELEASE_URL=https://packages.fluentbit.io
+ARG RELEASE_KEY=https://packages.fluentbit.io/fluentbit.key
+
+# hadolint ignore=DL3006
+FROM dokken/centos-9 as official-install
+ARG RELEASE_URL
+ENV FLUENT_BIT_PACKAGES_URL=${RELEASE_URL}
+
+ARG RELEASE_KEY
+ENV FLUENT_BIT_PACKAGES_KEY=${RELEASE_KEY}
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh
+RUN systemctl enable fluent-bit
+
+COPY ./test.sh /test.sh
+RUN chmod a+x /test.sh
+
+FROM official-install as staging-upgrade-prep
+RUN rm -f /etc/yum.repos.d/*-bit.repo
+
+# hadolint ignore=DL3006
+FROM ${STAGING_BASE} as staging-install
+ARG STAGING_VERSION
+ENV STAGING_VERSION=${STAGING_VERSION}
+
+ARG STAGING_URL
+ENV FLUENT_BIT_PACKAGES_URL=${STAGING_URL}
+
+ARG STAGING_KEY=${STAGING_URL}/fluentbit.key
+ENV FLUENT_BIT_PACKAGES_KEY=${STAGING_KEY}
+
+RUN rpm --import "$FLUENT_BIT_PACKAGES_KEY" && \
+    wget -nv "$FLUENT_BIT_PACKAGES_URL/centos-9.repo" -O /etc/yum.repos.d/staging.repo
+# hadolint ignore=DL3032
+RUN yum update -y && yum install -y fluent-bit && \
+    systemctl enable fluent-bit
+
+COPY ./test.sh /test.sh
+RUN chmod a+x /test.sh
+
+FROM staging-install as staging-upgrade
diff --git a/packaging/testing/smoke/packages/test.sh b/packaging/testing/smoke/packages/test.sh
@@ -7,14 +7,14 @@ if [ "$SKIP_TEST" = "yes" ]; then
 fi
 
 echo "Check package installed"
-rpm -q fluent-bit || rpm -q td-agent-bit || dpkg -l fluent-bit || dpkg -l td-agent-bit
+rpm -q fluent-bit || dpkg -l fluent-bit
 
 echo "Check service enabled"
-systemctl is-enabled fluent-bit || systemctl is-enabled td-agent-bit
+systemctl is-enabled fluent-bit
 
 until systemctl is-system-running; do
     sleep 10
 done
 
 echo "Check service running"
-systemctl status -q --no-pager fluent-bit || systemctl status -q --no-pager td-agent-bit
+systemctl status -q --no-pager fluent-bit