From d84650e7397e6fc2a31fb575a28a5332bb37f8c1 Mon Sep 17 00:00:00 2001 From: Carlos Henderson Date: Tue, 6 Feb 2024 11:55:30 +0100 Subject: [PATCH] Update packaging build process --- CHANGELOG.md | 1 + contrib/packaging/build | 25 +++--- contrib/packaging/build.cfg | 14 ++-- contrib/packaging/build_archive | 135 ++++++++++++++++++-------------- 4 files changed, 97 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c448a4..f85857b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Restructured source layout for pypi packaging. - Restructured and updated documentation. - Switched from setup.py to pyproject.toml + - Updated package build process to support Rocky linux and base on major version of OS. ### Removed - Removed `check_latest_version` function. diff --git a/contrib/packaging/build b/contrib/packaging/build index 2e21ecc..a1962ee 100755 --- a/contrib/packaging/build +++ b/contrib/packaging/build @@ -101,7 +101,7 @@ class BuildTarget: self.retrieve_assets() self.teardown_environment() -class CentOS8(BuildTarget): +class Rocky8(BuildTarget): def setup_environment(self): super().setup_environment() # rpmdev-setuptree creates the RPM build directory in the home env directory, so a symlink @@ -124,7 +124,7 @@ class CentOS8(BuildTarget): run(["rpmdev-setuptree"], cwd=self.tmp_build_dir, check=True) def copy_archives(self): - # CentOS needs the archive in a different location for Ubuntu and Debian so it's overridden here. + # Rocky needs the archive in a different location compared to Ubuntu and Debian so it's overridden here. src = os.path.join(self.build_cfg["_common"]["archive"]["path"], self.build_cfg["_common"]["archive"]["filename"]) dst = os.path.join(self.tmp_build_dir, "rpmbuild", "SOURCES", os.path.basename(src)) print(f"Copy archive file {src} {dst}") @@ -140,9 +140,9 @@ class CentOS8(BuildTarget): home_rpm_path.unlink() -class Rocky9(CentOS8): +class Rocky9(Rocky8): """ - Rocky 9.0 steps _might_ be the same as CentOS8. + Rocky 9.0 steps _might_ be the same as Rocky8. """ pass @@ -253,17 +253,18 @@ def release_info(release_files): return rel_to_dict(rel) rel = release_info(release_files) -distro = "{}_{}".format(rel["id"], rel["version_id"]) +# Strip the minor version since the script is designed to work on major versions. +distro = "{}{}".format(rel["id"], rel["version_id"].split(".", 1)[0]) build_cfg = load_config() targets = { - "ubuntu_18.04": Ubuntu1804, - "ubuntu_20.04": Ubuntu2004, - "ubuntu_22.04": Ubuntu2204, - "debian_10": Debian10, - "debian_11": Debian11, - "centos_8": CentOS8, - "rocky_9.0": Rocky9, + "ubuntu18": Ubuntu1804, + "ubuntu20": Ubuntu2004, + "ubuntu22": Ubuntu2204, + "debian10": Debian10, + "debian11": Debian11, + "rocky8": Rocky8, + "rocky9": Rocky9, } if distro not in build_cfg: diff --git a/contrib/packaging/build.cfg b/contrib/packaging/build.cfg index fa9dad1..2fa5a98 100644 --- a/contrib/packaging/build.cfg +++ b/contrib/packaging/build.cfg @@ -12,7 +12,7 @@ "email": "nzlosh@yhaoo.com" } }, - "centos_8": { + "rocky8": { "build_deps": [ "gcc", "rpm-build", @@ -41,7 +41,7 @@ ], "build_assets": [] }, - "rocky_9.0": { + "rocky9": { "build_deps": [ "gcc", "rpm-build", @@ -70,7 +70,7 @@ ], "build_assets": [] }, - "debian_10": { + "debian10": { "build_deps": [ "build-essential", "devscripts", @@ -96,7 +96,7 @@ ], "build_assets": [] }, - "debian_11": { + "debian11": { "build_deps": [ "build-essential", "devscripts", @@ -122,7 +122,7 @@ ], "build_assets": [] }, - "ubuntu_18.04": { + "ubuntu18": { "build_deps": [ "build-essential", "devscripts", @@ -148,7 +148,7 @@ ], "build_assets": [] }, - "ubuntu_20.04": { + "ubuntu20": { "build_deps": [ "build-essential", "devscripts", @@ -174,7 +174,7 @@ ], "build_assets": [] }, - "ubuntu_22.04": { + "ubuntu22": { "build_deps": [ "build-essential", "devscripts", diff --git a/contrib/packaging/build_archive b/contrib/packaging/build_archive index 4576d57..e4921f6 100755 --- a/contrib/packaging/build_archive +++ b/contrib/packaging/build_archive @@ -26,81 +26,87 @@ ERR_STACKSTORM_VERSION=2.2.0 # # https://gist.github.com/fernandoaleman/1377211/d78d13bd8f134e7d9b9bc3da5895c859d7cbf294 # https://gist.githubusercontent.com/fernandoaleman/1377169/raw/3e841ca1a887dd21f3fcb35a3e74b0cc2fc4977b/create-repo-metadata.sh - -function centos_8_install +function title +{ + echo -e "\033[38;5;206;48;5;57m${1}\033[0m" +} +function rocky8_install { dnf install -y ${PKGS[@]} } -function rocky_9.0_install +function rocky9_install { dnf install -y ${PKGS[@]} } -function ubuntu_18.04_install +function ubuntu18_install { apt install ${PKGS[@]} } -function ubuntu_20.04_install +function ubuntu20_install { apt install ${PKGS[@]} } -function ubuntu_22.04_install +function ubuntu22_install { apt install ${PKGS[@]} } -function debian_10_install +function debian10_install { apt install ${PKGS[@]} } -function debian_11_install +function debian11_install { apt install ${PKGS[@]} } function install_system_requirements { - echo INSTALL SYSTEM REQUIREMENTS + title "INSTALL SYSTEM REQUIREMENTS" ${DISTRO_COMBO}_install } function create_virtual_environment { - echo CREATE VIRTUAL ENVIRONMENT - $PYBIN -m venv $ROOT - cd $ROOT - mkdir $ROOT/{plugins,backends,data} + title "CREATE VIRTUAL ENVIRONMENT" + $PYBIN -m venv "$ROOT/venv" + cd "$ROOT" + mkdir "$ROOT"/{plugins,backends,data} } function fetch_archives { - echo DOWNLOAD ARCHIVES + title "DOWNLOAD ARCHIVES" export BACKENDS=("err-backend-discord" "err-backend-slackv3" "err-backend-gitter" "err-backend-mattermost" "err-backend-botframework") - cd $ROOT/backends - wget https://github.com/errbotio/err-backend-discord/archive/refs/tags/v2.1.0.tar.gz -O err-backend-discord-v2.1.0.tar.gz - wget https://github.com/errbotio/err-backend-slackv3/archive/refs/tags/v0.2.0.tar.gz -O err-backend-slackv3-v0.2.0.tar.gz - wget https://github.com/errbotio/err-backend-mattermost/archive/refs/tags/2.1.0.tar.gz -O err-backend-mattermost-v2.1.0.tar.gz - wget https://github.com/nzlosh/err-backend-gitter/archive/refs/tags/v0.1.0.tar.gz -O err-backend-gitter-v0.1.0.tar.gz - wget https://github.com/nzlosh/err-backend-botframework/archive/refs/tags/v0.1.0.tar.gz -O err-backend-botframework-v0.1.0.tar.gz - - cd $ROOT/plugins + mkdir "${BUILD_DIR}/backends" + cd "${BUILD_DIR}" + wget "https://github.com/errbotio/err-backend-discord/archive/refs/tags/v4.0.0.tar.gz" -O "backends/err-backend-discord-v4.0.0.tar.gz" + wget "https://github.com/errbotio/err-backend-slackv3/archive/refs/tags/v0.2.1.tar.gz" -O "backends/err-backend-slackv3-v0.2.1.tar.gz" + wget "https://github.com/errbotio/err-backend-mattermost/archive/refs/tags/3.0.0.tar.gz" -O "backends/err-backend-mattermost-v3.0.0.tar.gz" + wget "https://github.com/nzlosh/err-backend-gitter/archive/refs/tags/v0.1.0.tar.gz" -O "backends/err-backend-gitter-v0.1.0.tar.gz" + wget "https://github.com/nzlosh/err-backend-botframework/archive/refs/tags/v0.1.0.tar.gz" -O "backends/err-backend-botframework-v0.1.0.tar.gz" + + mkdir "${BUILD_DIR}/plugins" export PLUGINS=("err-stackstorm") - wget https://github.com/nzlosh/err-stackstorm/archive/refs/tags/v${ERR_STACKSTORM_VERSION}.tar.gz -O err-stackstorm-v${ERR_STACKSTORM_VERSION}.tar.gz + wget "https://github.com/nzlosh/err-stackstorm/archive/refs/tags/v${ERR_STACKSTORM_VERSION}.tar.gz" -O "plugins/err-stackstorm-v${ERR_STACKSTORM_VERSION}.tar.gz" } function install_extensions { - echo INSTALL ERRBOT EXTENSIONS - for location in $ROOT/backends $ROOT/plugins + title "INSTALL ERRBOT EXTENSIONS" + for location in backends plugins do # extract - for targz in ${location}/*.tar.gz + for targz in "${BUILD_DIR}/${location}"/*.tar.gz do - cd "$location" - tar xvf "$targz" && rm "$targz" + tar xf "$targz" -C "${ROOT}/${location}" done # install dependencies - for proj in ${location}/* + for proj in "${ROOT}/${location}"/* do - pip install -r "${proj}/requirements.txt" + # Install from pyproject + test -f "${proj}/pyproject.toml" && pip install -e "${proj}" + # Install from requirements + test -f "${proj}/requirements.txt" && pip install -r "${proj}/requirements.txt" done done @@ -109,42 +115,60 @@ function install_extensions cd $location for backend in ${BACKENDS[@]} do - mv -f *${backend}* "$backend" + # Move archive directory names to correct errbot names. + mv -f *"${backend}"* "$backend" done done - for location in $ROOT/plugins + for location in "$ROOT/plugins" do - cd $location + cd "$location" for plugin in ${PLUGINS[@]} do - mv -f *${plugin}* "$plugin" + mv -f *"${plugin}"* "$plugin" done done } function prune_installation { - echo PRUNE INSTALLATION - find $ROOT -iname '*.pyc' -delete - for pycache in $(find $ROOT -iname '__pycache__' -type d) + title "PRUNE INSTALLATION" + # Remove python cache files and hidden files. + find "$ROOT/venv" -iname '*.pyc' -delete + # Remove unrequired directories + for dir in '__pycache__' ".github" "docs" "tests" do - test -d "${pycache}" && rm -rf "${pycache}" + find "${ROOT}/venv" -iname "$dir" -type d -exec rm -rf "{}" \; done + for f in '.*' 'manifest.in' + do + # Remove hidden project files + find "${ROOT}/venv" -iname '.*' -delete + done + # Remove non-core code from err-stackstorm for excess in contrib docs tests do - rm -rf "$ROOT/plugins/err-stackstorm/${excess}" + rm -rf "$ROOT/plugins/err-stackstorm/${excess}" done + # Remove temp build dir + rm -rf "$BUILD_DIR" } function install_errbot { - echo "INSTALL ERRBOT ($($ROOT/bin/pip --version))" - source $ROOT/bin/activate + title "INSTALL ERRBOT ($($ROOT/venv/bin/pip --version))" + source "${ROOT}/venv/bin/activate" pip install --upgrade pip - pip install errbot + # Use tmp dir to download and build errbot/plugins. + export BUILD_DIR=$(mktemp -d "$ROOT"/build.XXXXXX) + cd "$BUILD_DIR" + ERRBOT_VERSION="6.2.0" + wget "https://github.com/errbotio/errbot/archive/refs/tags/${ERRBOT_VERSION}.tar.gz" -O errbot-v${ERRBOT_VERSION}.tar.gz + tar xf errbot-v${ERRBOT_VERSION}.tar.gz + cd errbot-${ERRBOT_VERSION} + pip install -e .[IRC,XMPP,telegram] fetch_archives install_extensions prune_installation @@ -156,40 +180,33 @@ function build_archive install_system_requirements create_virtual_environment install_errbot - tar czvf /opt/err-stackstorm_${ERR_STACKSTORM_VERSION}_${DISTRO_COMBO}_x86_64.tar.gz $ROOT + tar czf "/opt/err-stackstorm_${ERR_STACKSTORM_VERSION}_${DISTRO_COMBO}_x86_64.tar.gz" "$ROOT" } export DISTRO=$(source /etc/os-release; echo $ID) export DISTRO_VERSION=$(source /etc/os-release; echo $VERSION_ID) -export DISTRO_COMBO="${DISTRO}_${DISTRO_VERSION}" -export ROOT=/opt/errbot +# Strip the minor version since the script is designed to work on major versions. +export DISTRO_COMBO="${DISTRO}${DISTRO_VERSION%.*}" +export ROOT="/opt/errbot" -case "${DISTRO}_${DISTRO_VERSION}" in - centos_8) +case "${DISTRO_COMBO}" in + rocky8) export PKGS=(python38 virtualenv python38-devel git wget) export PYBIN=/usr/bin/python3.8 ;; - rocky_9.0) + rocky9) export PKGS=(python39 python3-devel tar gcc git wget) export PYBIN=/usr/bin/python3.9 ;; - ubuntu_18.04) - export PKGS=(python3.7-minimal git) - export PYBIN=/usr/bin/python3.7 - ;; - ubuntu_20.04) + ubuntu20) export PKGS=(python3.8-minimal git) export PYBIN=/usr/bin/python3.8 ;; - ubuntu_22.04) + ubuntu22) export PKGS=(python3.10-minimal git) export PYBIN=/usr/bin/python3.10 ;; - debian_10) - export PKGS=(python3.7-minimal git wget) - export PYBIN=/usr/bin/python3.7 - ;; - debian_11) + debian11) export PKGS=(python3.9-minimal git wget) export PYBIN=/usr/bin/python3.9 ;;