From 053cada8be7930000c3ba6f567d9c69a669aab08 Mon Sep 17 00:00:00 2001 From: Craig Thacker Date: Wed, 21 Feb 2024 21:55:24 +0000 Subject: [PATCH] Update --- containers/alpine/Dockerfile | 2 +- containers/rhel-podman-in-podman/start.sh | 115 ------------ containers/rhel/Dockerfile | 173 ++++++++++++++++++ .../Dockerfile => rhel/Test.Dockerfile} | 2 +- .../containers.conf | 0 containers/rhel/env.sh | 45 +++++ .../podman-containers.conf | 0 containers/rhel/start.sh | 105 +++++++++++ containers/ubuntu/Dockerfile | 2 +- 9 files changed, 326 insertions(+), 118 deletions(-) delete mode 100644 containers/rhel-podman-in-podman/start.sh create mode 100644 containers/rhel/Dockerfile rename containers/{rhel-podman-in-podman/Dockerfile => rhel/Test.Dockerfile} (99%) rename containers/{rhel-podman-in-podman => rhel}/containers.conf (100%) create mode 100644 containers/rhel/env.sh rename containers/{rhel-podman-in-podman => rhel}/podman-containers.conf (100%) create mode 100644 containers/rhel/start.sh diff --git a/containers/alpine/Dockerfile b/containers/alpine/Dockerfile index 448ff9f..11216c3 100644 --- a/containers/alpine/Dockerfile +++ b/containers/alpine/Dockerfile @@ -1,6 +1,6 @@ FROM alpine:latest -LABEL org.opencontainers.image.title=alpine-cicd-base +LABEL org.opencontainers.image.title=alpine LABEL org.opencontainers.image.source=https://github.com/libre-devops/azdo-agent-containers #Set args with blank values - these will be over-written with the CLI diff --git a/containers/rhel-podman-in-podman/start.sh b/containers/rhel-podman-in-podman/start.sh deleted file mode 100644 index 8b90ff9..0000000 --- a/containers/rhel-podman-in-podman/start.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash - -set -e - -print_header() { - lightcyan='\033[1;36m' - nocolor='\033[0m' - echo -e "${lightcyan}$1${nocolor}" -} - -AZP_AGENT_NAME="azdo-agent-lnx-$(date +'%d%m%Y')-$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c10)" - -USER="microsoft" -REPO="azure-pipelines-agent" -OS="linux" -ARCH="x64" -PACKAGE="tar.gz" - -if [ "$(command -v jq)" ] && [ "$(command -v curl)" ] && [ "$(command -v sed)" ]; then - print_header "0. Checking jq, curl and sed are installed..." - - else - echo "You do not have the needed packages to run the script, please install them" && exit 1 - -fi - -azdoLatestAgentVersion="$(curl --silent "https://api.github.com/repos/${USER}/${REPO}/releases/latest" | jq -r .tag_name)" && \ - -strippedTagAzDoAgentVersion="$(echo "${azdoLatestAgentVersion}" | sed 's/v//')" && \ - -AZP_AGENTPACKAGE_URL="https://vstsagentpackage.azureedge.net/agent/${strippedTagAzDoAgentVersion}/vsts-agent-${OS}-${ARCH}-${strippedTagAzDoAgentVersion}.${PACKAGE}" - -if [ -z "${AZP_URL}" ]; then - echo 1>&2 "error: missing AZP_URL environment variable" - exit 1 -fi - -if [ -z "${AZP_TOKEN_FILE}" ]; then - if [ -z "${AZP_TOKEN}" ]; then - echo 1>&2 "error: missing AZP_TOKEN environment variable" - exit 1 - fi - - AZP_TOKEN_FILE=/azp/.token - echo -n "${AZP_TOKEN}" > "${AZP_TOKEN_FILE}" -fi - -unset AZP_TOKEN - -if [ -n "${AZP_WORK}" ]; then - mkdir -p "${AZP_WORK}" -fi - -rm -rf /azp/agent -mkdir /azp/agent -cd /azp/agent - -export AGENT_ALLOW_RUNASROOT="1" - -cleanup() { - if [ -e config.sh ]; then - print_header "Cleanup. Removing Azure Pipelines agent..." - - ./config.sh remove --unattended \ - --auth PAT \ - --token "$(cat "${AZP_TOKEN_FILE}")" - fi -} - -# Let the agent ignore the token env variables -export VSO_AGENT_IGNORE=AZP_TOKEN,AZP_TOKEN_FILE - -print_header "1. Determining matching Azure Pipelines agent..." - -AZP_AGENT_RESPONSE=$(curl -LsS \ - -u user:"$(cat "${AZP_TOKEN_FILE}")" \ - -H 'Accept:application/json;api-version=3.0-preview' \ - "${AZP_URL}/_apis/distributedtask/packages/agent?platform=linux-x64") - -if echo "${AZP_AGENT_RESPONSE}" | jq . >/dev/null 2>&1; then - AZP_AGENTPACKAGE_URL="$(echo "${AZP_AGENT_RESPONSE}" \ - | jq -r '.value | map([.version.major,.version.minor,.version.patch,.downloadUrl]) | sort | .[length-1] | .[3]')" -fi - -if [ -z "${AZP_AGENTPACKAGE_URL}" ] || [ "${AZP_AGENTPACKAGE_URL}" == "null" ]; then - echo 1>&2 "error: could not determine a matching Azure Pipelines agent - check that account '$AZP_URL' is correct and the token is valid for that account" - exit 1 -fi - -print_header "2. Downloading and installing Azure Pipelines agent..." - -curl -LsS "${AZP_AGENTPACKAGE_URL}" | tar -xz & wait $! - -source ./env.sh - -print_header "3. Configuring Azure Pipelines agent..." - -./config.sh --unattended \ - --agent "${AZP_AGENT_NAME:-$(hostname)}" \ - --url "${AZP_URL}" \ - --auth PAT \ - --token "$(cat "${AZP_TOKEN_FILE}")" \ - --pool "${AZP_POOL}" \ - --work "${AZP_WORK:-_work}" \ - --replace \ - --acceptTeeEula & wait $! - -print_header "4. Running Azure Pipelines agent..." - -trap 'cleanup; exit 130' INT -trap 'cleanup; exit 143' TERM - -# To be aware of TERM and INT signals call run.sh -# Running it with the --once flag at the end will shut down the agent after the build is executed -./run.sh & wait $! diff --git a/containers/rhel/Dockerfile b/containers/rhel/Dockerfile new file mode 100644 index 0000000..465a57c --- /dev/null +++ b/containers/rhel/Dockerfile @@ -0,0 +1,173 @@ +#Use supplier image +FROM registry.access.redhat.com/ubi9/ubi:latest + +LABEL org.opencontainers.image.title=rhel +LABEL org.opencontainers.image.source=https://github.com/libre-devops/azdo-agent-containers + +RUN rm -rf /bin/sh && ln -sf /bin/bash /bin/sh + +#Set args with blank values - these will be over-written with the CLI +ARG NORMAL_USER=builder +ARG DEBIAN_FRONTEND=noninteractive +#Set args with blank values - these will be over-written with the CLI +ARG AZP_URL=https://dev.azure.com/Example +ARG AZP_TOKEN=ExamplePatToken +ARG AZP_AGENT_NAME=Example +ARG AZP_POOL=Default +ARG AZP_WORK=_work +ARG AZP_DIRECTORY=/home/$NORMAL_USER +ARG ACCEPT_EULA=y +ARG TARGETARCH="linux-x64" + +#Set the environment with the CLI-passed arguements +ENV AZP_URL ${AZP_URL} +ENV AZP_TOKEN ${AZP_TOKEN} +ENV AZP_AGENT_NAME ${AZP_AGENT_NAME} +ENV AZP_POOL ${AZP_POOL} +ENV AZP_WORK ${AZP_WORK} +ENV AZP_DIRECTORY ${AZP_DIRECTORY} +ENV ACCEPT_EULA ${ACCEPT_EULA} +ENV TARGETARCH ${TARGETARCH} + +ENV NORMAL_USER ${NORMAL_USER} +ENV DEBIAN_FRONTEND=noninteractive + +# Environment variables for pyenv +ENV HOME /home/${NORMAL_USER} +ENV PYENV_ROOT /home/${NORMAL_USER}/.pyenv +ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH + +#Set path vars +ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt:/opt/bin:/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.local/bin:/home/${NORMAL_USER}/.local:/home/${NORMAL_USER}:/home/${NORMAL_USER}/.tenv:/home/${NORMAL_USER}/.tenv/bin:/home/${NORMAL_USER}/.pkenv:/home/${NORMAL_USER}/.pkenv/bin:/home/${NORMAL_USER}/.pyenv:/home/${NORMAL_USER}/.pyenv/bin:/home/${NORMAL_USER}/.pyenv/shims:/home/${NORMAL_USER}/.local/bin" +ENV PATHVAR="PATH=${PATH}" + +# RHEL9 doesn't have readline-devel in the standard repos +ENV READLINE_DEVEL_VERSION="8.1-4" + +#Declare user expectation, I am performing root actions, so use root. +USER root + +#Install needed packages as well as setup python with args and pip +RUN useradd -ms /bin/bash ${NORMAL_USER} && \ + mkdir -p /home/linuxbrew && \ + chown -R ${NORMAL_USER}:${NORMAL_USER} /home/linuxbrew && \ + dnf update -y && dnf upgrade -y && dnf install -y yum-utils dnf sudo && \ + MAJOR_OS_VERSION=$(grep VERSION_ID /etc/os-release | cut -d '"' -f 2 | cut -d '.' -f 1) && \ + rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-${MAJOR_OS_VERSION}.noarch.rpm && \ + /usr/bin/crb enable && \ + sudo dnf install -y \ + bash \ + bzip2-devel \ + ca-certificates \ + gcc \ + git \ + gnupg \ + gnupg2 \ + jq \ + libffi-devel \ + libicu-devel \ + make \ + ncurses-devel \ + openssl-devel \ + sqlite-devel \ + unzip \ + wget \ + xz-devel \ + zip \ + zlib-devel && \ + echo $PATHVAR > /etc/environment && \ + LATEST_VERSION=$(curl https://api.github.com/repos/sigstore/cosign/releases/latest | jq -r .tag_name | tr -d "v\", ") && \ + curl -O -L "https://github.com/sigstore/cosign/releases/latest/download/cosign-${LATEST_VERSION}-1.x86_64.rpm" && \ + rpm -ivh cosign-${LATEST_VERSION}-1.x86_64.rpm && \ + curl -O https://mirror.stream.centos.org/9-stream/AppStream/x86_64/os/Packages/readline-devel-${READLINE_DEVEL_VERSION}.el9.x86_64.rpm && \ + dnf install -y readline-devel-${READLINE_DEVEL_VERSION}.el9.x86_64.rpm + +RUN POWERSHELL_RELEASE_URL=$(curl -s -L https://api.github.com/repos/PowerShell/PowerShell/releases/latest | jq -r '.assets[] | select(.name | endswith("linux-x64.tar.gz")) | .browser_download_url') && \ + curl -L $POWERSHELL_RELEASE_URL -o /tmp/powershell.tar.gz && \ + mkdir -p /opt/microsoft/powershell/7 && \ + tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 && \ + chmod +x /opt/microsoft/powershell/7/pwsh && \ + ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh && \ + ln -s /usr/bin/pwsh /usr/bin/powershell + +RUN git clone https://github.com/pyenv/pyenv.git /home/${NORMAL_USER}/.pyenv && \ + eval "$(pyenv init --path)" && \ + pyenvLatestStable=$(pyenv install --list | grep -v - | grep -E "^\s*[0-9]+\.[0-9]+\.[0-9]+$" | tail -1) && \ + pyenv install $pyenvLatestStable && \ + pyenv global $pyenvLatestStable && \ + pip install --upgrade pip + +#Install Azure Modules for Powershell - This can take a while, so setting as final step to shorten potential rebuilds +RUN pwsh -Command Set-PSRepository -Name "PSGallery" -InstallationPolicy Trusted && \ + pwsh -Command Install-Module -Name Az -Force -AllowClobber -Scope AllUsers -Repository PSGallery && \ + pwsh -Command Install-Module -Name Microsoft.Graph -Force -AllowClobber -Scope AllUsers -Repository PSGallery && \ + pwsh -Command Install-Module -Name Pester -Force -AllowClobber -Scope AllUsers -Repository PSGallery + +RUN LATEST_VERSION=$(curl --silent https://api.github.com/repos/tofuutils/tenv/releases/latest|jq -r .tag_name) && \ + curl -O -L "https://github.com/tofuutils/tenv/releases/latest/download/tenv_${LATEST_VERSION}_amd64.rpm" && \ + rpm -ivh "tenv_${LATEST_VERSION}_amd64.rpm" + +# Install Packer Env +RUN git clone https://github.com/iamhsa/pkenv.git /home/${NORMAL_USER}/.pkenv && \ + PACKER_LATEST_URL=$(curl -sL https://releases.hashicorp.com/packer/index.json | jq -r '.versions[].builds[].url' | egrep -v 'rc|beta|alpha' | egrep 'linux.*amd64' | tail -1) && \ + PACKER_LATEST_VERSION=$(echo "$PACKER_LATEST_URL" | awk -F '/' '{print $6}' | sed 's/packer_//' | sed 's/_linux_amd64.zip//') && \ + pkenv install ${PACKER_LATEST_VERSION} && \ + pkenv use ${PACKER_LATEST_VERSION} + +RUN chown -R ${NORMAL_USER}:${NORMAL_USER} /opt && \ + chown -R ${NORMAL_USER}:${NORMAL_USER} /home/${NORMAL_USER} + +RUN dnf -y module enable container-tools:rhel9; dnf -y update; rpm --restore --quiet shadow-utils; \ +dnf -y install crun podman podman-docker fuse-overlayfs /etc/containers/storage.conf --exclude container-selinux; \ +rm -rf /var/cache /var/log/dnf* /var/log/yum.* + +RUN useradd podman; \ +echo podman:10000:5000 > /etc/subuid; \ +echo podman:10000:5000 > /etc/subgid; + +VOLUME /var/lib/containers +RUN mkdir -p /home/podman/.local/share/containers +RUN chown podman:podman -R /home/podman && usermod -aG podman ${NORMAL_USER} +VOLUME /home/podman/.local/share/containers + +#https://raw.githubusercontent.com/containers/libpod/master/contrib/podmanimage/stable/containers.conf +ADD containers.conf /etc/containers/containers.conf +#https://raw.githubusercontent.com/containers/libpod/master/contrib/podmanimage/stable/podman-containers.conf +ADD podman-containers.conf /home/podman/.config/containers/containers.conf + +#chmod containers.conf and adjust storage.conf to enable Fuse storage. +RUN chmod 644 /etc/containers/containers.conf; sed -i -e 's|^#mount_program|mount_program|g' -e '/additionalimage.*/a "/var/lib/shared",' -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' /etc/containers/storage.conf +RUN mkdir -p /var/lib/shared/overlay-images /var/lib/shared/overlay-layers /var/lib/shared/vfs-images /var/lib/shared/vfs-layers; \ + touch /var/lib/shared/overlay-images/images.lock; \ + touch /var/lib/shared/overlay-layers/layers.lock; \ + touch /var/lib/shared/vfs-images/images.lock; \ + touch /var/lib/shared/vfs-layers/layers.lock + +ENV _CONTAINERS_USERNS_CONFIGURED="" + +COPY start.sh /home/${NORMAL_USER}/start.sh +COPY env.sh /home/${NORMAL_USER}/env.sh +RUN chmod +x /home/${NORMAL_USER}/start.sh && \ + chmod +x /home/${NORMAL_USER}/env.sh && \ + chown -R "${NORMAL_USER}:${NORMAL_USER}" /home/${NORMAL_USER} +USER ${NORMAL_USER} +WORKDIR /home/${NORMAL_USER} +CMD [ "./start.sh" ] + +RUN tenv tf install latest --verbose && \ + tenv tf use latest --verbose && \ + tenv tofu install latest --verbose && \ + tenv tofu use latest --verbose && \ + tenv tg install latest --verbose && \ + tenv tg use latest --verbose + +RUN pip3 install --user \ + pipenv \ + virtualenv \ + terraform-compliance \ + checkov \ + pywinrm + + +# Install Azure-CLI as seperate step for cacheing speed +RUN pip3 install --user azure-cli diff --git a/containers/rhel-podman-in-podman/Dockerfile b/containers/rhel/Test.Dockerfile similarity index 99% rename from containers/rhel-podman-in-podman/Dockerfile rename to containers/rhel/Test.Dockerfile index 27ee691..d8717f7 100644 --- a/containers/rhel-podman-in-podman/Dockerfile +++ b/containers/rhel/Test.Dockerfile @@ -107,7 +107,7 @@ RUN curl -s "https://get.sdkman.io" | bash && \ #Don't include container-selinux and remove #directories used by yum that are just taking #up space. -RUN dnf -y module enable container-tools:rhel8; dnf -y update; rpm --restore --quiet shadow-utils; \ +RUN dnf -y module enable container-tools:rhel9; dnf -y update; rpm --restore --quiet shadow-utils; \ dnf -y install crun podman podman-docker fuse-overlayfs /etc/containers/storage.conf --exclude container-selinux; \ rm -rf /var/cache /var/log/dnf* /var/log/yum.* diff --git a/containers/rhel-podman-in-podman/containers.conf b/containers/rhel/containers.conf similarity index 100% rename from containers/rhel-podman-in-podman/containers.conf rename to containers/rhel/containers.conf diff --git a/containers/rhel/env.sh b/containers/rhel/env.sh new file mode 100644 index 0000000..b991431 --- /dev/null +++ b/containers/rhel/env.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +varCheckList=( + 'LANG' + 'JAVA_HOME' + 'ANT_HOME' + 'M2_HOME' + 'ANDROID_HOME' + 'GRADLE_HOME' + 'NVM_BIN' + 'NVM_PATH' + 'VSTS_HTTP_PROXY' + 'VSTS_HTTP_PROXY_USERNAME' + 'VSTS_HTTP_PROXY_PASSWORD' + 'LD_LIBRARY_PATH' + 'PERL5LIB' + 'AGENT_TOOLSDIRECTORY' + ) + +envContents="" + +if [ -f ".env" ]; then + envContents=$(cat .env) +else + touch .env +fi + +function writeVar() +{ + local checkVar="$1" + local checkDelim="${1}=" + if test "${envContents#*$checkDelim}" = "$envContents" + then + if [ -z "${!checkVar}" ]; then + echo "${checkVar}=${!checkVar}">>.env + fi + fi +} + +echo "$PATH">.path + +for var_name in "${varCheckList[@]}" +do + writeVar "${var_name}" +done diff --git a/containers/rhel-podman-in-podman/podman-containers.conf b/containers/rhel/podman-containers.conf similarity index 100% rename from containers/rhel-podman-in-podman/podman-containers.conf rename to containers/rhel/podman-containers.conf diff --git a/containers/rhel/start.sh b/containers/rhel/start.sh new file mode 100644 index 0000000..e2e7dea --- /dev/null +++ b/containers/rhel/start.sh @@ -0,0 +1,105 @@ +#!/bin/bash +set -e + +if [ -z "${AZP_URL}" ]; then + echo 1>&2 "error: missing AZP_URL environment variable" + exit 1 +fi + +if [ -z "${AZP_TOKEN_FILE}" ]; then + if [ -z "${AZP_TOKEN}" ]; then + echo 1>&2 "error: missing AZP_TOKEN environment variable" + exit 1 + fi + + AZP_TOKEN_FILE="${AZP_DIRECTORY}/.token" + echo -n "${AZP_TOKEN}" >"${AZP_TOKEN_FILE}" +fi + +unset AZP_TOKEN + +if [ -n "${AZP_WORK}" ]; then + mkdir -p "${AZP_WORK}" +fi + +cleanup() { + trap "" EXIT + + if [ -e ./config.sh ]; then + print_header "Cleanup. Removing Azure Pipelines agent..." + + # If the agent has some running jobs, the configuration removal process will fail. + # So, give it some time to finish the job. + while true; do + ./config.sh remove --unattended --auth "PAT" --token "$(cat "${AZP_TOKEN_FILE}")" && break + + echo "Retrying in 30 seconds..." + sleep 30 + done + fi +} + +print_header() { + local lightcyan="\033[1;36m" + local nocolor="\033[0m" + echo -e "\n${lightcyan}$1${nocolor}\n" +} + +if [ "$(command -v jq)" ] && [ "$(command -v curl)" ] && [ "$(command -v sed)" ]; then + print_header "0. Checking jq, curl and sed are installed..." + +else + echo "You do not have the needed packages to run the script, please install them" && exit 1 + +fi + +# Let the agent ignore the token env variables +export VSO_AGENT_IGNORE="AZP_TOKEN,AZP_TOKEN_FILE" + +print_header "1. Determining matching Azure Pipelines agent..." + +AZP_AGENT_PACKAGES=$(curl -LsS \ + -u user:"$(cat "${AZP_TOKEN_FILE}")" \ + -H "Accept:application/json;" \ + "${AZP_URL}/_apis/distributedtask/packages/agent?platform=${TARGETARCH}&top=1") + +AZP_AGENT_PACKAGE_LATEST_URL=$(echo "${AZP_AGENT_PACKAGES}" | jq -r ".value[0].downloadUrl") + +if [ -z "${AZP_AGENT_PACKAGE_LATEST_URL}" ] || [ "${AZP_AGENT_PACKAGE_LATEST_URL}" == "null" ]; then + echo 1>&2 "error: could not determine a matching Azure Pipelines agent" + echo 1>&2 "check that account ${AZP_URL} is correct and the token is valid for that account" + exit 1 +fi + +print_header "2. Downloading and extracting Azure Pipelines agent..." + +curl -LsS "${AZP_AGENT_PACKAGE_LATEST_URL}" | tar -xz & +wait $! + +source ./env.sh + +trap "cleanup; exit 0" EXIT +trap "cleanup; exit 130" INT +trap "cleanup; exit 143" TERM + +print_header "3. Configuring Azure Pipelines agent..." + +./config.sh --unattended \ + --agent "${AZP_AGENT_NAME:-$(hostname)}" \ + --url "${AZP_URL}" \ + --auth "PAT" \ + --token "$(cat "${AZP_TOKEN_FILE}")" \ + --pool "${AZP_POOL:-Default}" \ + --work "${AZP_WORK:-_work}" \ + --replace \ + --acceptTeeEula & +wait $! + +print_header "4. Running Azure Pipelines agent..." + +chmod +x ./run.sh + +# To be aware of TERM and INT signals call ./run.sh +# Running it with the --once flag at the end will shut down the agent after the build is executed +./run.sh "$@" & +wait $! diff --git a/containers/ubuntu/Dockerfile b/containers/ubuntu/Dockerfile index b810fa5..b1fa76b 100644 --- a/containers/ubuntu/Dockerfile +++ b/containers/ubuntu/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:latest -LABEL org.opencontainers.image.title=azdo-agent-containers +LABEL org.opencontainers.image.title=ubuntu LABEL org.opencontainers.image.source=https://github.com/libre-devops/azdo-agent-containers RUN rm -rf /bin/sh && ln -sf /bin/bash /bin/sh