Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

toolkit: added RPMs snapshots. #3734

Merged
merged 19 commits into from
Sep 16, 2022
Merged

toolkit: added RPMs snapshots. #3734

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
2baaa99
toolkit: added RPMs snapshots.
PawelWMS Sep 10, 2022
1810ad8
Fixing typo.
PawelWMS Sep 10, 2022
a1f71e6
Removing invalid '-q' argument.
PawelWMS Sep 12, 2022
e6c0e47
Merging 'main' ARM64 fix in 'livepatching.spec'.
PawelWMS Sep 12, 2022
1e9b26f
Revert "Merging 'main' ARM64 fix in 'livepatching.spec'."
PawelWMS Sep 12, 2022
8f6e3ac
Addressing comments.
PawelWMS Sep 12, 2022
adf940c
Adding separate target for RPMs snapshots.
PawelWMS Sep 12, 2022
7fdc2b7
Merging 'main' ARM64 fix in 'livepatching.spec'.
PawelWMS Sep 12, 2022
0cb91ee
Revert "Merging 'main' ARM64 fix in 'livepatching.spec'."
PawelWMS Sep 12, 2022
82c2a5b
Adding missed defines.
PawelWMS Sep 13, 2022
6632f7c
Initial work to switch to a Golang snapshot generator.
PawelWMS Sep 15, 2022
5c68671
Replacing Bash script with Go tool.
PawelWMS Sep 15, 2022
982b97b
Fixing the tool.
PawelWMS Sep 15, 2022
9bcb41e
Simplifying 'make' files.
PawelWMS Sep 15, 2022
a3ae0af
Updating clean-up steps.
PawelWMS Sep 15, 2022
6285bd4
Removing Bash scripts.
PawelWMS Sep 15, 2022
fcdd52a
Clean-up.
PawelWMS Sep 15, 2022
4a396fa
Sorting methods, adding comments.
PawelWMS Sep 15, 2022
e893487
Updating clean-up steps.
PawelWMS Sep 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions toolkit/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,14 @@ ifeq ($(USE_PREVIEW_REPO),y)
endif
endif

ifneq ($(strip $(SRPM_PACK_LIST)),)
LOCAL_SPECS = $(wildcard $(addprefix $(SPECS_DIR)/*/,$(addsuffix .spec,$(strip SRPM_PACK_LIST))))
else # Empty pack list, build all under $(SPECS_DIR)
LOCAL_SPECS = $(shell find $(SPECS_DIR)/ -type f -name '*.spec')
endif

LOCAL_SPEC_DIRS = $(foreach spec,$(LOCAL_SPECS),$(dir $(spec)))

CA_CERT ?=
TLS_CERT ?=
TLS_KEY ?=
Expand Down
10 changes: 3 additions & 7 deletions toolkit/scripts/srpm_pack.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,13 @@ toolchain_spec_list = $(toolchain_build_dir)/toolchain_specs.txt
srpm_pack_list_file = $(BUILD_SRPMS_DIR)/pack_list.txt

ifneq ($(strip $(SRPM_PACK_LIST)),)
local_specs = $(wildcard $(addprefix $(SPECS_DIR)/*/,$(addsuffix .spec,$(strip SRPM_PACK_LIST))))
$(srpm_pack_list_file): $(depend_SRPM_PACK_LIST)
@echo $(strip $(SRPM_PACK_LIST)) | tr " " "\n" > $(srpm_pack_list_file)
else # Empty pack list, build all under $(SPECS_DIR)
local_specs = $(shell find $(SPECS_DIR)/ -type f -name '*.spec')
$(srpm_pack_list_file): $(depend_SRPM_PACK_LIST)
@touch $@
endif

local_spec_dirs = $(foreach spec,$(local_specs),$(dir $(spec)))

$(call create_folder,$(BUILD_DIR))
$(call create_folder,$(BUILD_SRPMS_DIR))
$(call create_folder,$(SRPM_BUILD_CHROOT_DIR))
Expand All @@ -55,8 +51,8 @@ $(BUILD_SRPMS_DIR): $(STATUS_FLAGS_DIR)/build_srpms.flag
@echo Finished updating $@

ifeq ($(DOWNLOAD_SRPMS),y)
$(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(SPECS_DIR)
for spec in $(local_specs); do \
$(STATUS_FLAGS_DIR)/build_srpms.flag: $(LOCAL_SPECS) $(LOCAL_SPEC_DIRS) $(SPECS_DIR)
for spec in $(LOCAL_SPECS); do \
spec_file=$${spec} && \
srpm_file=$$(rpmspec -q $${spec_file} --srpm --define='with_check 1' --define='dist $(DIST_TAG)' --queryformat %{NAME}-%{VERSION}-%{RELEASE}.src.rpm) && \
for url in $(SRPM_URL_LIST); do \
Expand All @@ -78,7 +74,7 @@ $(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(SPECS_
$(STATUS_FLAGS_DIR)/build_toolchain_srpms.flag: $(STATUS_FLAGS_DIR)/build_srpms.flag
@touch $@
else
$(STATUS_FLAGS_DIR)/build_srpms.flag: $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(go-srpmpacker) $(srpm_pack_list_file)
$(STATUS_FLAGS_DIR)/build_srpms.flag: $(chroot_worker) $(LOCAL_SPECS) $(LOCAL_SPEC_DIRS) $(SPECS_DIR) $(go-srpmpacker) $(srpm_pack_list_file)
GODEBUG=netdns=go $(go-srpmpacker) \
--dir=$(SPECS_DIR) \
--output-dir=$(BUILD_SRPMS_DIR) \
Expand Down
92 changes: 73 additions & 19 deletions toolkit/scripts/toolkit.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,36 +12,90 @@ toolkit_component_extra_files = \
$(PROJECT_ROOT)/LICENSES-AND-NOTICES/LICENSE.md \
$(toolkit_root)/.gitignore

mariner_repos_dir = $(SPECS_DIR)/mariner-repos
mariner_repos_dir = $(PROJECT_ROOT)/SPECS/mariner-repos
PawelWMS marked this conversation as resolved.
Show resolved Hide resolved
mariner_repos_files = $(wildcard $(mariner_repos_dir)/*.repo)
rpms_snapshot_name = rpms_snapshot.json
specs_dir_name = $(notdir $(SPECS_DIR))
toolkit_remove_archive = $(OUT_DIR)/toolkit-*.tar*
toolkit_root_files = $(wildcard $(toolkit_root)/*)
toolkit_version = $(RELEASE_VERSION)-$(build_arch)

# Build files
rpms_snapshot_dir_name = rpms_snapshots
rpms_snapshot_build_dir = $(BUILD_DIR)/$(rpms_snapshot_dir_name)
rpms_snapshot_logs_path = $(LOGS_DIR)/$(rpms_snapshot_dir_name)/rpms_snapshot.log
rpms_snapshot_per_specs = $(rpms_snapshot_build_dir)/$(specs_dir_name)_$(rpms_snapshot_name)

toolkit_build_dir = $(BUILD_DIR)/toolkit_prep
toolkit_archive = $(toolkit_build_dir)/toolkit.tar
toolkit_archive_versioned = $(toolkit_build_dir)/toolkit_versioned.tar

toolkit_prep_dir = $(toolkit_build_dir)/toolkit
toolkit_release_file = $(toolkit_prep_dir)/version.txt
toolkit_release_file_relative_path = $(toolkit_release_file:$(toolkit_build_dir)/%=%)
toolkit_rpms_snapshot_file = $(toolkit_prep_dir)/$(rpms_snapshot_name)
toolkit_rpms_snapshot_file_relative_path = $(toolkit_rpms_snapshot_file:$(toolkit_build_dir)/%=%)
toolkit_repos_dir = $(toolkit_prep_dir)/repos
toolkit_tools_dir = $(toolkit_prep_dir)/tools/toolkit_bins

$(call create_folder,"$(rpms_snapshot_build_dir)")
$(call create_folder,"$(toolkit_prep_dir)")

# Outputs
toolkit_version = $(RELEASE_VERSION)-$(build_arch)
toolkit_archive = $(OUT_DIR)/toolkit-$(toolkit_version).tar.gz
toolkit_remove_archive = $(OUT_DIR)/toolkit-*.tar.gz
toolkit_build_dir = $(BUILD_DIR)/toolkit
toolkit_repos_dir = $(toolkit_build_dir)/repos
toolkit_tools_dir = $(toolkit_build_dir)/tools/toolkit_bins
toolkit_release_file = $(toolkit_build_dir)/version.txt
toolkit_archive_versioned_compressed = $(OUT_DIR)/toolkit-$(toolkit_version).tar.gz
rpms_snapshot = $(OUT_DIR)/$(rpms_snapshot_name)

.PHONY: package-toolkit clean-package-toolkit
.PHONY: package-toolkit rpms-snapshot clean-package-toolkit clean-rpms-snapshot

clean: clean-package-toolkit clean-rpms-snapshot

clean: clean-package-toolkit
clean-package-toolkit:
rm -f $(toolkit_remove_archive)
rm -rf $(toolkit_build_dir)

package-toolkit: go-tools
rm -rf $(toolkit_build_dir) && \
mkdir -p $(toolkit_build_dir) && \
clean-rpms-snapshot:
rm -f $(rpms_snapshot)
rm -rf $(rpms_snapshot_build_dir)
rm -r $(rpms_snapshot_logs_path)

package-toolkit: $(toolkit_archive_versioned_compressed)
@echo "Toolkit packed under '$(toolkit_archive_versioned_compressed)'."

$(toolkit_archive_versioned_compressed): $(toolkit_archive) $(rpms_snapshot) $(depend_SPECS_DIR)
cp $(toolkit_archive) $(toolkit_archive_versioned) && \
echo "$(toolkit_version)" > $(toolkit_release_file) && \
cp $(rpms_snapshot) $(toolkit_rpms_snapshot_file) && \
tar --update -f $(toolkit_archive_versioned) -C $(toolkit_build_dir) $(toolkit_release_file_relative_path) $(toolkit_rpms_snapshot_file_relative_path) && \
$(ARCHIVE_TOOL) --best -c $(toolkit_archive_versioned) > $(toolkit_archive_versioned_compressed)

$(toolkit_archive): $(go_tool_targets) $(mariner_repos_files) $(toolkit_component_extra_files) $(toolkit_root_files)
rm -rf $(toolkit_prep_dir) && \
mkdir -p $(toolkit_prep_dir) && \
mkdir -p $(toolkit_repos_dir) && \
mkdir -p $(toolkit_tools_dir) && \
cp -r $(toolkit_root)/* $(toolkit_build_dir) && \
cp $(mariner_repos_dir)/*.repo $(toolkit_repos_dir) && \
cp $(toolkit_component_extra_files) $(toolkit_build_dir) && \
cp -r $(toolkit_root_files) $(toolkit_prep_dir) && \
cp $(mariner_repos_files) $(toolkit_repos_dir) && \
cp $(toolkit_component_extra_files) $(toolkit_prep_dir) && \
cp $(go_tool_targets) $(toolkit_tools_dir) && \
echo "$(toolkit_version)" > $(toolkit_release_file) && \
rm -rf $(toolkit_build_dir)/out && \
tar -I $(ARCHIVE_TOOL) -cvp -f $(toolkit_archive) -C $(toolkit_build_dir)/.. $(notdir $(toolkit_build_dir))
rm -rf $(toolkit_prep_dir)/out && \
tar -cvp -f $(toolkit_archive) -C $(dir $(toolkit_prep_dir)) $(notdir $(toolkit_prep_dir))

rpms-snapshot: $(rpms_snapshot)
@echo "RPMs snapshot generated under '$(rpms_snapshot)'."

$(rpms_snapshot): $(rpms_snapshot_per_specs) $(depend_SPECS_DIR)
PawelWMS marked this conversation as resolved.
Show resolved Hide resolved
cp $(rpms_snapshot_per_specs) $(rpms_snapshot)

$(rpms_snapshot_per_specs): $(go-rpmssnapshot) $(chroot_worker) $(LOCAL_SPECS) $(LOCAL_SPEC_DIRS) $(SPECS_DIR)
PawelWMS marked this conversation as resolved.
Show resolved Hide resolved
@mkdir -p "$(rpms_snapshot_build_dir)"
$(go-rpmssnapshot) \
--input="$(SPECS_DIR)" \
--output="$(rpms_snapshot_per_specs)" \
--build-dir="$(rpms_snapshot_build_dir)" \
--dist-tag=$(DIST_TAG) \
--worker-tar="$(chroot_worker)" \
--log-level=$(LOG_LEVEL) \
--log-file="$(rpms_snapshot_logs_path)"

print-build-summary:
sed -E -n 's:^.+level=info msg="Built \(([^\)]+)\) -> \[(.+)\].+$:\1\t\2:gp' $(LOGS_DIR)/pkggen/rpmbuilding/* | tee $(LOGS_DIR)/pkggen/build-summary.csv
1 change: 1 addition & 0 deletions toolkit/scripts/tools.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ go_tool_list = \
liveinstaller \
pkgworker \
roast \
rpmssnapshot \
scheduler \
specreader \
srpmpacker \
Expand Down
4 changes: 2 additions & 2 deletions toolkit/scripts/utils.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ endef
######## VARIABLE DEPENDENCY TRACKING ########

# List of variables to watch for changes.
watch_vars=PACKAGE_BUILD_LIST PACKAGE_REBUILD_LIST PACKAGE_IGNORE_LIST REPO_LIST CONFIG_FILE STOP_ON_PKG_FAIL TOOLCHAIN_ARCHIVE REBUILD_TOOLCHAIN SRPM_PACK_LIST
watch_vars=PACKAGE_BUILD_LIST PACKAGE_REBUILD_LIST PACKAGE_IGNORE_LIST REPO_LIST CONFIG_FILE STOP_ON_PKG_FAIL TOOLCHAIN_ARCHIVE REBUILD_TOOLCHAIN SRPM_PACK_LIST SPECS_DIR
# Current list: $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_REPO_LIST) $(depend_CONFIG_FILE) $(depend_STOP_ON_PKG_FAIL)
# $(depend_TOOLCHAIN_ARCHIVE) $(depend_REBUILD_TOOLCHAIN) $(depend_SRPM_PACK_LIST)
# $(depend_TOOLCHAIN_ARCHIVE) $(depend_REBUILD_TOOLCHAIN) $(depend_SRPM_PACK_LIST) $(depend_SPECS_DIR)

.PHONY: variable_depends_on_phony clean-variable_depends_on_phony
clean: clean-variable_depends_on_phony
Expand Down
7 changes: 5 additions & 2 deletions toolkit/tools/internal/rpm/rpm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,11 @@ func QuerySPEC(specFile, sourceDir, queryFormat string, defines map[string]strin
}

// QuerySPECForBuiltRPMs queries a SPEC file with queryFormat. Returns only the subpackages, which generate a .rpm file.
func QuerySPECForBuiltRPMs(specFile, sourceDir, queryFormat string, defines map[string]string) (result []string, err error) {
const builtRPMsSwitch = "--builtrpms"
func QuerySPECForBuiltRPMs(specFile, sourceDir string, defines map[string]string) (result []string, err error) {
const (
builtRPMsSwitch = "--builtrpms"
queryFormat = ""
)

return QuerySPEC(specFile, sourceDir, queryFormat, defines, builtRPMsSwitch)
}
Expand Down
Loading