diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index effaad6..7756fd3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: - id: mixed-line-ending - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.9 + rev: v0.8.3 hooks: - id: ruff-format - id: ruff diff --git a/README.md b/README.md index def988e..a0c3920 100644 --- a/README.md +++ b/README.md @@ -72,12 +72,13 @@ repo file then you can find it in the `Repo Download` on the Copr page. Currently, we don't build automatically because there are no tests and an unnoticed push to `main` could silently break our RHEL/Fedora pipelines. -To create a new build: +To create a new SRPM, either: -- run `rpmdev-bumpspec mini-tps.spec` to bump the release in the spec file -- run `packit srpm --no-update-release` to create a SRPM -- Go to [@osci/mini-tps New Build](https://copr.fedorainfracloud.org/coprs/g/osci/mini-tps/add_build_upload), - select the SRPM file and hit `Build` +- run `packit srpm` or +- run `rpmdev-bumpspec mini-tps.spec` and `packit srpm --no-update-release` + +Then, go to [@osci/mini-tps New Build](https://copr.fedorainfracloud.org/coprs/g/osci/mini-tps/add_build_upload), +select the SRPM file and hit `Build` To enable the automatic builds, all you need to do is uncomment the `copr_build` job in the [.packit.yaml](.packit.yaml). diff --git a/mini-tps.spec b/mini-tps.spec index 210b07b..117d7de 100644 --- a/mini-tps.spec +++ b/mini-tps.spec @@ -6,7 +6,7 @@ Name: mini-tps Version: 0.1 -Release: 183%{?dist} +Release: 184%{?dist} Summary: Mini TPS - Test Package Sanity License: GPLv2 @@ -73,6 +73,11 @@ install -pD -m 0755 profiles/fedora/prepare-system %{buildroot}%{_libexecdir}/mi %changelog +* Tue Dec 17 2024 Jiri Popelka - 0.1-184 +- Konflux imported: Get RPMs from build intead of from a (sub)task(s) +- Don't install redhat-internal-cert-install rpm +- viewer: ignore UTF-8 errors + * Fri Nov 01 2024 Michal Srb - 0.1-183 - Remove Resilient Storage repos for 10 diff --git a/mtps-get-builds b/mtps-get-builds index 46a7b63..9c40920 100755 --- a/mtps-get-builds +++ b/mtps-get-builds @@ -22,9 +22,9 @@ if [ -z "${YUMDNFCMD:-}" ]; then source "$(dirname "${BASH_SOURCE[0]}")/mtps-set RET_NO_RPMS_IN_BREW=7 RET_NO_RPMS_IN_REPOS=8 -rpms_from_answer() { - local arch="$1" && shift - local xml="$1" && shift +build_rpms_info2rpms() { + local arch="$1" && shift # architecture + local xml="$1" && shift # listBuildRPMs response local tmpfile="$(mktemp)" echo "$xml" > "$tmpfile" local cmd="cat //member[value/string='$arch']/../member[name='nvr']/value/string/text()" @@ -33,8 +33,8 @@ rpms_from_answer() { echo "$rpm" } -build_from_answer() { - local xml="$1" && shift +build_rpms_info2buid_id() { + local xml="$1" && shift # listBuildRPMs response local build_id= build_id="$(echo "$xml" | xmllint --xpath 'string(//member[name="build_id"]/value/int/text())' -)" debug "Brew build id: $build_id" @@ -55,18 +55,16 @@ brew_list_build_rpms() { # Build can be: NVR or build number. local build="$1" && shift local re='^[0-9]+$' - if [[ $build =~ $re ]] ; then - build="$build" - fi + [[ $build =~ $re ]] && build="$build" local query="$(cat << EOF -listBuildRPMs - -$build - - - + listBuildRPMs + + + $build + + EOF )" @@ -83,7 +81,7 @@ mk_repo() { repo_dir="${repo_dir%%/}" local repo="$(cat << EOF [brew-${brew_id}] -name=Repo for $brew_id Brew build +name=Brew build $brew_id repo baseurl=file:///${repo_dir}/ enabled=$repo_enabled gpgcheck=0 @@ -302,12 +300,12 @@ debug "Create repo: $GB_CREATEREPO" build_rpms="$(brew_list_build_rpms $GB_BUILD)" # rpms_from_build is a list of packages in form NVR: dhcp-relay-4.3.6-27.el8+7 dhcp-server-4.3.6-27.el8+7 -rpms_from_build="$(rpms_from_answer "$GB_ARCH" "$build_rpms")" +rpms_from_build="$(build_rpms_info2rpms "$GB_ARCH" "$build_rpms")" rpms_from_build_all="$rpms_from_build" -rpms_from_build_noarch="$(rpms_from_answer "noarch" "$build_rpms")" +rpms_from_build_noarch="$(build_rpms_info2rpms "noarch" "$build_rpms")" rpms_from_build_noarch_all="$rpms_from_build_noarch" -srpm="$(rpms_from_answer "src" "$build_rpms").src.rpm" -brew_build_id="$(build_from_answer "$build_rpms")" +srpm="$(build_rpms_info2rpms "src" "$build_rpms").src.rpm" +brew_build_id="$(build_rpms_info2buid_id "$build_rpms")" srpm_url="$(mk_url "$srpm" "${srpm%%.src.rpm}" "src")" echo "Brew build ID: $brew_build_id" @@ -323,7 +321,7 @@ if [ -n "$GB_ONLYINREPO" ]; then # Purpose: # # * Brew builds always have 1 input SRPM. - # * Brew builds can have many PRM. + # * Brew builds can have many RPMs. # # RCM team can choose only some RPM. And make AppStream / BaseOS / Compose # only from some RPMs, ignoring other RPMs from Brew build. This options diff --git a/mtps-get-module b/mtps-get-module index df70804..2e33807 100755 --- a/mtps-get-module +++ b/mtps-get-module @@ -98,17 +98,19 @@ send_query() { brew_get_build() { # getBuild query - # Build can be: NVR or build number. + # Build can be: int ID or a string NVR local build="$1" && shift + local re='^[0-9]+$' + [[ $build =~ $re ]] && build="$build" local query="$(cat << EOF -getBuild - -$build - - - + getBuild + + + $build + + EOF )" @@ -118,17 +120,17 @@ EOF brew_get_latest_builds() { # getLatestBuilds query - # Build can be: NVR or build number. - local tag="$1" && shift + # Tag is int (ID) + local tag_id="$1" && shift local query="$(cat << EOF -getLatestBuilds - -$tag - - - + getLatestBuilds + + + $tag_id + + EOF )" diff --git a/mtps-get-task b/mtps-get-task index c1d8c65..665f6f4 100755 --- a/mtps-get-task +++ b/mtps-get-task @@ -133,21 +133,23 @@ send_query() { brew_get_build() { # getBuild query - # Build can be: NVR or build number. + # Build can be: int ID or a string NVR local build="$1" && shift + local re='^[0-9]+$' + [[ $build =~ $re ]] && build="$build" local query="$(cat << EOF -getBuild - -$build - - - + getBuild + + + $build + + EOF )" - debug "Brew query for listBuildRPMs: ${query:0:200}...(truncated)" +# debug "Brew query for getBuild: ${query:0:200}...(truncated)" send_query "$query" } @@ -158,19 +160,40 @@ brew_list_build_tags() { local query="$(cat << EOF -listTags - - -$build - - + listTags + + + $build + + EOF )" - debug "Brew query for listTags: ${query:0:200}...(truncated)" +# debug "Brew query for listTags: ${query:0:200}...(truncated)" send_query "$query" } +brew_list_build_rpms() { + # listBuildRPMs query + # Build can be: NVR or build number. + local build="$1" && shift + local re='^[0-9]+$' + [[ $build =~ $re ]] && build="$build" + local query="$(cat << EOF + + + listBuildRPMs + + + $build + + + +EOF + )" +# debug "Brew query for listBuildRPMs: ${query:0:200}...(truncated)" + send_query "$query" +} brew_list_builds() { # listBuilds query @@ -178,45 +201,45 @@ brew_list_builds() { local query="$(cat << EOF -listBuilds - - - - -__starstar -1 - - -taskID -$task - - - - + listBuilds + + + + + __starstar + 1 + + + taskID + $task + + + + EOF )" - debug "Brew query for listBuilds: ${query:0:200}...(truncated)" +# debug "Brew query for listBuilds: ${query:0:200}...(truncated)" send_query "$query" } brew_get_task_result() { # getTaskResult query - # Build can be: NVR or build number. - local task="$1" && shift + # Task: ID of task queried + local task_id="$1" && shift local query="$(cat << EOF getTaskResult - $task + $task_id EOF )" - debug "Brew query for getTaskResult: ${query:0:200}...(truncated)" +# debug "Brew query for getTaskResult: ${query:0:200}...(truncated)" send_query "$query" } @@ -226,54 +249,54 @@ brew_get_user() { local query="$(cat << EOF -getUser - - -$user_id - - + getUser + + + $user_id + + EOF )" - debug "Brew query for getUser: ${query:0:200}...(truncated)" +# debug "Brew query for getUser: ${query:0:200}...(truncated)" send_query "$query" } brew_get_task_info() { # getTaskInfo query - local task="$1" && shift + local task_id="$1" && shift local query="$(cat << EOF -getTaskInfo - - -$task - - + getTaskInfo + + + $task_id + + EOF )" - debug "Brew query for getTaskInfo: ${query:0:200}...(truncated)" +# debug "Brew query for getTaskInfo: ${query:0:200}...(truncated)" send_query "$query" } brew_get_task_request() { # getTaskRequest query - local task="$1" && shift + local task_id="$1" && shift local query="$(cat << EOF -getTaskRequest - - -$task - - + getTaskRequest + + + $task_id + + EOF )" - debug "Brew query for getTaskRequest: ${query:0:200}...(truncated)" +# debug "Brew query for getTaskRequest: ${query:0:200}...(truncated)" send_query "$query" } @@ -289,6 +312,18 @@ taskinfo2owner() { echo "$owner" } +taskinfo2method() { + local xml="$1" && shift + local tmpfile + tmpfile="$(mktemp)" + echo "$xml" > "$tmpfile" + local cmd="cat //member[name='method']/value/string/text()" + local method + method="$(echo "$cmd" | xmllint --shell "$tmpfile" | sed -n -e '/^[[:alnum:]]/p')" + debug "Method:\n${method}\n---" + echo "$method" +} + task_request2target() { local xml="$1" && shift local tmpfile @@ -301,6 +336,25 @@ task_request2target() { echo "$target" } +build_rpms_info2rpms() { + local arch="$1" && shift # architecture + local xml="$1" && shift # listBuildRPMs response + local tmpfile="$(mktemp)" + echo "$xml" > "$tmpfile" + local cmd="cat //member[value/string='$arch']/../member[name='nvr']/value/string/text()" + rpms="$(echo "$cmd" | xmllint --shell "$tmpfile" | sed -n -e '/^[[:alnum:]]/p')" + if [[ -n "${rpms}" ]]; then + # Add .{arch}.rpm to each line + declare -a lines + while IFS= read -r line; do + lines+=("${line}.${arch}.rpm") + done <<< "$rpms" + rpms=$(printf "%s\n" "${lines[@]}") + fi + debug "${arch} rpms:\n${rpms}\n---" + echo "$rpms" +} + ownerinfo2user() { local xml="$1" && shift local tmpfile @@ -321,27 +375,26 @@ ownerinfo2krb() { local cmd="cat //member[name='krb_principal']/value/string/text()" local email email="$(echo "$cmd" | xmllint --shell "$tmpfile" | sed -n -e '/^[[:alnum:]]/p')" - debug "Task owner email:\n${email}\n---" + debug "Task owner kerberos:\n${email}\n---" echo "$email" } brew_get_task_children() { # getTaskChildren query - # Build can be: NVR or build number. - local task="$1" && shift + local task_id="$1" && shift local query="$(cat << EOF getTaskChildren - $task + $task_id EOF )" - debug "Brew query for getTaskChildren: ${query:0:200}...(truncated)" +# debug "Brew query for getTaskChildren: ${query:0:200}...(truncated)" send_query "$query" } @@ -352,7 +405,8 @@ mk_repo() { local repo_enabled="${1:0}" && shift repo_dir="${repo_dir##/}" repo_dir="${repo_dir%%/}" - local repo="$(cat << EOF + local repo + repo="$(cat << EOF [brew-${task_id}] name=Repo for $task_id Brew build baseurl=file:///${repo_dir}/ @@ -370,7 +424,8 @@ mk_named_repo() { local repo_enabled="${1:0}" && shift repo_dir="${repo_dir##/}" repo_dir="${repo_dir%%/}" - local repo="$(cat << EOF + local repo + repo="$(cat << EOF [${repo_name}] name=Repo for $repo_name baseurl=file:///${repo_dir}/ @@ -477,15 +532,15 @@ mk_url() { local srpm_filename="$1" && shift local is_scratch="$1" && shift if [ "$is_scratch" == "no" ]; then - # https://download.devel.redhat.com/brewroot/packages/curl/7.61.1/34.el8/x86_64/curl-7.61.1-34.el8.x86_64.rpm - # https://kojipkgs.fedoraproject.org//packages/packit/0.87.1/1.fc40/src/packit-0.87.1-1.fc40.src.rpm filename=$(basename "$filename") srpm_filename=$(basename "$srpm_filename") name="$(from_rpm_name "${srpm_filename}" "name")" - rel="$(from_rpm_name "${srpm_filename}" "rel")" ver="$(from_rpm_name "${srpm_filename}" "ver")" + rel="$(from_rpm_name "${srpm_filename}" "rel")" arch="$(from_rpm_name "${filename}" "arch")" - url="${BREWROOT}/packages/${srpm_pkg_name}/${ver}/${rel}/${arch}/$filename" + # https://download.devel.redhat.com/brewroot/packages/curl/7.61.1/34.el8/x86_64/curl-7.61.1-34.el8.x86_64.rpm + # https://kojipkgs.fedoraproject.org//packages/packit/0.87.1/1.fc40/src/packit-0.87.1-1.fc40.src.rpm + url="${BREWROOT}/packages/${name}/${ver}/${rel}/${arch}/$filename" else # https://download.devel.redhat.com/brewroot/work/tasks/237/57520237/curl-7.61.1-34.el8.x86_64.rpm # https://kojipkgs.fedoraproject.org//work/tasks/9873/109769873/packit-0.87.1-1.fc40.src.rpm @@ -657,60 +712,75 @@ buildtags="$(buildtags_from_answer "$listbuildtags")" buildtags="$(echo "$buildtags" | tr '\n' ' ')" buildtags="${buildtags%% }" -TASKS="$TASK" -if [ -n "$RECURSIVE" ]; then - task_children="$(brew_get_task_children "$TASK")" - task_children="$(children_from_answer "$task_children")" - TASKS="$TASKS $task_children" - TASKS="${TASKS%% }" -fi +echo "BUILD ID: $buildid" +echo "BUILD TAGS: $buildtags" -srpm_url= -volume_name= -build_source= -srpm_pkg_file= -srpm_pkg_name= -rpms_from_task_all="" -rpms_from_task_noarch_all="" -for task in $TASKS; do - debug "Process task: $task" - task_result="$(brew_get_task_result "$task")" # Holds plain XML - rpms_from_task="$(task_result2rpms "$ARCH" "$task_result")" - rpms_from_task_noarch="$(task_result2rpms "noarch" "$task_result")" - srpm="$(task_result2srpm "$task_result")" - if [ -n "$srpm" ]; then - srpm_pkg_file="$(basename "$srpm")" - srpm_pkg_name="$(from_rpm_name "${srpm_pkg_file}" "name")" - if [ -z "$nvr" ]; then - nvr="${srpm_pkg_file%%.src.rpm}" - fi - build="$(brew_get_build "$nvr")" - volume_name="$(build2volume_name "$build")" - build_source="$(build2build_source "$build")" - srpm_url="$(mk_url "$srpm" "$srpm_pkg_file" "$is_scratch")" +task_method="$(taskinfo2method "$task_info")" +debug "task_method: $task_method" +if [[ "$task_method" == "build" ]]; then + TASKS="$TASK" + if [ -n "$RECURSIVE" ]; then + task_children="$(brew_get_task_children "$TASK")" + task_children="$(children_from_answer "$task_children")" + TASKS="$TASKS $task_children" + TASKS="${TASKS%% }" fi - rpms_from_task_all+=" $rpms_from_task" - rpms_from_task_noarch_all+=" $rpms_from_task_noarch" -done -# remove trailing whitespaces -rpms_from_task_all="${rpms_from_task_all%%+( )}" -rpms_from_task_noarch_all="${rpms_from_task_noarch_all%%+( )}" + srpm_url= + volume_name= + build_source= + srpm_pkg_file= + srpm_pkg_name= + rpms_arch_all="" + rpms_noarch_all="" + for task in $TASKS; do + task_result="$(brew_get_task_result "$task")" # Holds plain XML + rpms_from_task="$(task_result2rpms "$ARCH" "$task_result")" + rpms_from_task_noarch="$(task_result2rpms "noarch" "$task_result")" + srpm="$(task_result2srpm "$task_result")" + if [ -n "$srpm" ]; then + srpm_pkg_file="$(basename "$srpm")" + srpm_pkg_name="$(from_rpm_name "${srpm_pkg_file}" "name")" + if [ -z "$nvr" ]; then + nvr="${srpm_pkg_file%%.src.rpm}" + fi + build="$(brew_get_build "$nvr")" + volume_name="$(build2volume_name "$build")" + build_source="$(build2build_source "$build")" + srpm_url="$(mk_url "$srpm" "$srpm_pkg_file" "$is_scratch")" + fi + rpms_arch_all+=" $rpms_from_task" + rpms_noarch_all+=" $rpms_from_task_noarch" + done -echo "NVR: $nvr" -echo "BUILD ID: $buildid" -echo "BUILD TAGS: $buildtags" -echo "SRPM PKG NAME: $srpm_pkg_name" -echo "SRPM PKG FILE: $srpm_pkg_file" -echo "SRPM URL: $srpm_url" -echo "BUILD SOURCE: $build_source" + # remove trailing whitespaces + rpms_arch_all="${rpms_arch_all%%+( )}" + rpms_noarch_all="${rpms_noarch_all%%+( )}" -if [ -n "$SRPM" ]; then - # Print only SRPM information - exit 0 + echo "NVR: $nvr" + echo "SRPM PKG NAME: $srpm_pkg_name" + echo "SRPM PKG FILE: $srpm_pkg_file" + echo "SRPM URL: $srpm_url" + echo "BUILD SOURCE: $build_source" + + if [ -n "$SRPM" ]; then + # Print only SRPM information + exit 0 + fi +# Builds imported from Konflux don't have descendant/child tasks to take the (S)RPMs from. +# This is much easier than taking the (S)RPMs from (sub)task(s) +# but can't be used for the 'build' method because of scratch builds (they don't have build id) +elif [[ "$task_method" == "cg_import" ]]; then + build_rpms="$(brew_list_build_rpms "$buildid")" + rpms_arch_all="$(build_rpms_info2rpms "$ARCH" "$build_rpms")" + rpms_noarch_all="$(build_rpms_info2rpms "noarch" "$build_rpms")" + srpm_pkg_file="$(build_rpms_info2rpms "src" "$build_rpms")" +else + echo "Unknown Brew method: $task_method" + exit 1 fi -if [ -z "${rpms_from_task_all}${rpms_from_task_noarch_all}" ]; then +if [ -z "${rpms_arch_all}${rpms_noarch_all}" ]; then echo "There are no ${ARCH}/noarch RPMs for Brew/Koji task ${TASK}" exit "$RET_NO_RPMS_IN_BREW" fi @@ -724,18 +794,18 @@ if [ -n "$ONLYINREPO" ]; then # RCM team can choose only some RPM. And make AppStream / BaseOS / Compose # only from some RPMs, ignoring other RPMs from Brew build. This options # says to ignore RPMs that are absent. - rpms_from_task_all="$(filter_available "$ARCH" "$rpms_from_task_all")" - rpms_from_task_noarch_all="$(filter_available "noarch" "$rpms_from_task_noarch_all")" + rpms_arch_all="$(filter_available "$ARCH" "$rpms_arch_all")" + rpms_noarch_all="$(filter_available "noarch" "$rpms_noarch_all")" fi -if [ -z "${rpms_from_task_all}${rpms_from_task_noarch_all}" ]; then +if [ -z "${rpms_arch_all}${rpms_noarch_all}" ]; then # all packages have been filtered out above echo "None of the packages is available in repos. Is it a new package?" exit "$RET_NO_RPMS_IN_REPOS" fi declare -A urls # associative array (dictionary, map) -for pkg in $rpms_from_task_all $rpms_from_task_noarch_all; do +for pkg in $rpms_arch_all $rpms_noarch_all; do url="$(mk_url "$pkg" "$srpm_pkg_file" "$is_scratch")" urls["$url"]=1 # in order to deduplicate, store url as key, value is unimportant done diff --git a/mtps-tag b/mtps-tag index fd0eb24..d6ce4a0 100755 --- a/mtps-tag +++ b/mtps-tag @@ -41,12 +41,12 @@ tagged2nvrs() { echo "$nvrs" } -tag2tag_id() { +tag_info2tag_id() { local xml="$1" && shift local task= - volume_name="$(echo "$xml" | xmllint --xpath 'string(//member[name="id"]/value/int/text())' -)" - debug "Volume name: $volume_name" - echo "$volume_name" + tag_id="$(echo "$xml" | xmllint --xpath 'string(//member[name="id"]/value/int/text())' -)" + debug "Tag name: $tag_id" + echo "$tag_id" } send_query() { @@ -64,18 +64,22 @@ send_query() { # brew_get_tag() { + # tag may be either + # a string (the tag name) or an int (the tag ID) local tag="$1" && shift + local re='^[0-9]+$' + [[ $tag =~ $re ]] && tag="$tag" local query="$(cat << EOF -getTag - - - -$tag - - - + getTag + + + + $tag + + + EOF )" @@ -85,18 +89,21 @@ EOF brew_list_tagged() { + # tag: tag name or ID number local tag="$1" && shift + local re='^[0-9]+$' + [[ $tag =~ $re ]] && tag="$tag" local query="$(cat << EOF -listTagged - - - -$tag - - - + listTagged + + + + $tag + + + EOF )" @@ -126,7 +133,7 @@ EOF DEBUG="${DEBUG:-}" PROG="${PROG:-${0##*/}}" -TAG_NAME="${TAG_NAME:-}" +tag_id="${tag_id:-}" GET_TASKS="${GET_TASKS:-}" GET_NVRS="${GET_NVRS:-}" GET_TAG_ID="${GET_TAG_ID:-}" @@ -139,7 +146,7 @@ eval set -- "$opt" while [[ $# -gt 0 ]]; do case "$1" in --tname) - TAG_NAME="$2" + tag_id="$2" shift 2 ;; -t|--tasks) @@ -172,15 +179,15 @@ while [[ $# -gt 0 ]]; do done # Test correct invocation -if [ -z "$TAG_NAME" ]; then +if [ -z "$tag_id" ]; then echo "Use: $PROG -h for help." exit fi -debug "Tag: $TAG_NAME" +debug "Tag: $tag_id" if [[ -n "$GET_NVRS" || -n "$GET_TASKS" ]]; then - list_tagged="$(brew_list_tagged "$TAG_NAME")" + list_tagged="$(brew_list_tagged "$tag_id")" if [ -n "$GET_NVRS" ]; then nvrs="$(tagged2nvrs "$list_tagged")" echo "$nvrs" @@ -192,8 +199,8 @@ if [[ -n "$GET_NVRS" || -n "$GET_TASKS" ]]; then fi if [ -n "$GET_TAG_ID" ]; then - tag="$(brew_get_tag "$TAG_NAME")" - tag_id="$(tag2tag_id "$tag")" + tag="$(brew_get_tag "$tag_id")" + tag_id="$(tag_info2tag_id "$tag")" echo "$tag_id" exit 0 fi