toolkit: added RPMs snapshots. (#3734)

microsoft · Sep 16, 2022 · c4a4c00 · c4a4c00
1 parent de62fd4
commit c4a4c00
Show file tree

Hide file tree

Showing 8 changed files with 390 additions and 30 deletions.
diff --git a/toolkit/Makefile b/toolkit/Makefile
@@ -119,6 +119,14 @@ ifeq ($(USE_PREVIEW_REPO),y)
    endif
 endif
 
+ifneq ($(strip $(SRPM_PACK_LIST)),)
+LOCAL_SPECS = $(wildcard $(addprefix $(SPECS_DIR)/*/,$(addsuffix .spec,$(strip SRPM_PACK_LIST))))
+else # Empty pack list, build all under $(SPECS_DIR)
+LOCAL_SPECS = $(shell find $(SPECS_DIR)/ -type f -name '*.spec')
+endif
+
+LOCAL_SPEC_DIRS = $(foreach spec,$(LOCAL_SPECS),$(dir $(spec)))
+
 CA_CERT     ?=
 TLS_CERT    ?=
 TLS_KEY     ?=

diff --git a/toolkit/scripts/srpm_pack.mk b/toolkit/scripts/srpm_pack.mk
@@ -20,17 +20,13 @@ toolchain_spec_list = $(toolchain_build_dir)/toolchain_specs.txt
 srpm_pack_list_file = $(BUILD_SRPMS_DIR)/pack_list.txt
 
 ifneq ($(strip $(SRPM_PACK_LIST)),)
-local_specs = $(wildcard $(addprefix $(SPECS_DIR)/*/,$(addsuffix .spec,$(strip SRPM_PACK_LIST))))
 $(srpm_pack_list_file): $(depend_SRPM_PACK_LIST)
 	@echo $(strip $(SRPM_PACK_LIST)) | tr " " "\n" > $(srpm_pack_list_file)
 else # Empty pack list, build all under $(SPECS_DIR)
-local_specs = $(shell find $(SPECS_DIR)/ -type f -name '*.spec')
 $(srpm_pack_list_file): $(depend_SRPM_PACK_LIST)
 	@touch $@
 endif
 
-local_spec_dirs = $(foreach spec,$(local_specs),$(dir $(spec)))
-
 $(call create_folder,$(BUILD_DIR))
 $(call create_folder,$(BUILD_SRPMS_DIR))
 $(call create_folder,$(SRPM_BUILD_CHROOT_DIR))
@@ -55,8 +51,8 @@ $(BUILD_SRPMS_DIR): $(STATUS_FLAGS_DIR)/build_srpms.flag
 	@echo Finished updating $@
 
 ifeq ($(DOWNLOAD_SRPMS),y)
-$(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(SPECS_DIR)
-	for spec in $(local_specs); do \
+$(STATUS_FLAGS_DIR)/build_srpms.flag: $(LOCAL_SPECS) $(LOCAL_SPEC_DIRS) $(SPECS_DIR)
+	for spec in $(LOCAL_SPECS); do \
 		spec_file=$${spec} && \
 		srpm_file=$$(rpmspec -q $${spec_file} --srpm --define='with_check 1' --define='dist $(DIST_TAG)' --queryformat %{NAME}-%{VERSION}-%{RELEASE}.src.rpm) && \
 		for url in $(SRPM_URL_LIST); do \
@@ -78,7 +74,7 @@ $(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(SPECS_
 $(STATUS_FLAGS_DIR)/build_toolchain_srpms.flag: $(STATUS_FLAGS_DIR)/build_srpms.flag
 	@touch $@
 else
-$(STATUS_FLAGS_DIR)/build_srpms.flag: $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(go-srpmpacker) $(srpm_pack_list_file)
+$(STATUS_FLAGS_DIR)/build_srpms.flag: $(chroot_worker) $(LOCAL_SPECS) $(LOCAL_SPEC_DIRS) $(SPECS_DIR) $(go-srpmpacker) $(srpm_pack_list_file)
 	GODEBUG=netdns=go $(go-srpmpacker) \
 		--dir=$(SPECS_DIR) \
 		--output-dir=$(BUILD_SRPMS_DIR) \

diff --git a/toolkit/scripts/toolkit.mk b/toolkit/scripts/toolkit.mk
@@ -12,36 +12,90 @@ toolkit_component_extra_files = \
 	$(PROJECT_ROOT)/LICENSES-AND-NOTICES/LICENSE.md \
 	$(toolkit_root)/.gitignore
 
-mariner_repos_dir = $(SPECS_DIR)/mariner-repos
+mariner_repos_dir = $(PROJECT_ROOT)/SPECS/mariner-repos
+mariner_repos_files = $(wildcard $(mariner_repos_dir)/*.repo)
+rpms_snapshot_name = rpms_snapshot.json
+specs_dir_name = $(notdir $(SPECS_DIR))
+toolkit_remove_archive = $(OUT_DIR)/toolkit-*.tar*
+toolkit_root_files = $(wildcard $(toolkit_root)/*)
+toolkit_version   = $(RELEASE_VERSION)-$(build_arch)
+
+# Build files
+rpms_snapshot_dir_name = rpms_snapshots
+rpms_snapshot_build_dir = $(BUILD_DIR)/$(rpms_snapshot_dir_name)
+rpms_snapshot_logs_path = $(LOGS_DIR)/$(rpms_snapshot_dir_name)/rpms_snapshot.log
+rpms_snapshot_per_specs = $(rpms_snapshot_build_dir)/$(specs_dir_name)_$(rpms_snapshot_name)
+
+toolkit_build_dir   = $(BUILD_DIR)/toolkit_prep
+toolkit_archive   = $(toolkit_build_dir)/toolkit.tar
+toolkit_archive_versioned   = $(toolkit_build_dir)/toolkit_versioned.tar
+
+toolkit_prep_dir = $(toolkit_build_dir)/toolkit
+toolkit_release_file = $(toolkit_prep_dir)/version.txt
+toolkit_release_file_relative_path = $(toolkit_release_file:$(toolkit_build_dir)/%=%)
+toolkit_rpms_snapshot_file = $(toolkit_prep_dir)/$(rpms_snapshot_name)
+toolkit_rpms_snapshot_file_relative_path = $(toolkit_rpms_snapshot_file:$(toolkit_build_dir)/%=%)
+toolkit_repos_dir = $(toolkit_prep_dir)/repos
+toolkit_tools_dir = $(toolkit_prep_dir)/tools/toolkit_bins
+
+$(call create_folder,"$(rpms_snapshot_build_dir)")
+$(call create_folder,"$(toolkit_prep_dir)")
 
 # Outputs
-toolkit_version   = $(RELEASE_VERSION)-$(build_arch)
-toolkit_archive   = $(OUT_DIR)/toolkit-$(toolkit_version).tar.gz
-toolkit_remove_archive = $(OUT_DIR)/toolkit-*.tar.gz
-toolkit_build_dir = $(BUILD_DIR)/toolkit
-toolkit_repos_dir = $(toolkit_build_dir)/repos
-toolkit_tools_dir = $(toolkit_build_dir)/tools/toolkit_bins
-toolkit_release_file = $(toolkit_build_dir)/version.txt
+toolkit_archive_versioned_compressed   = $(OUT_DIR)/toolkit-$(toolkit_version).tar.gz
+rpms_snapshot = $(OUT_DIR)/$(rpms_snapshot_name)
 
-.PHONY: package-toolkit clean-package-toolkit
+.PHONY: package-toolkit rpms-snapshot clean-package-toolkit clean-rpms-snapshot
+
+clean: clean-package-toolkit clean-rpms-snapshot
 
-clean: clean-package-toolkit
 clean-package-toolkit:
 	rm -f $(toolkit_remove_archive)
 	rm -rf $(toolkit_build_dir)
 
-package-toolkit: go-tools
-	rm -rf $(toolkit_build_dir) && \
-	mkdir -p $(toolkit_build_dir) && \
+clean-rpms-snapshot:
+	rm -f $(rpms_snapshot)
+	rm -rf $(rpms_snapshot_build_dir)
+	rm -r $(rpms_snapshot_logs_path)
+
+package-toolkit: $(toolkit_archive_versioned_compressed)
+	@echo "Toolkit packed under '$(toolkit_archive_versioned_compressed)'."
+
+$(toolkit_archive_versioned_compressed): $(toolkit_archive) $(rpms_snapshot) $(depend_SPECS_DIR)
+	cp $(toolkit_archive) $(toolkit_archive_versioned) && \
+	echo "$(toolkit_version)" > $(toolkit_release_file) && \
+	cp $(rpms_snapshot) $(toolkit_rpms_snapshot_file) && \
+	tar --update -f $(toolkit_archive_versioned) -C $(toolkit_build_dir) $(toolkit_release_file_relative_path) $(toolkit_rpms_snapshot_file_relative_path) && \
+	$(ARCHIVE_TOOL) --best -c $(toolkit_archive_versioned) > $(toolkit_archive_versioned_compressed)
+
+$(toolkit_archive): $(go_tool_targets) $(mariner_repos_files) $(toolkit_component_extra_files) $(toolkit_root_files)
+	rm -rf $(toolkit_prep_dir) && \
+	mkdir -p $(toolkit_prep_dir) && \
 	mkdir -p $(toolkit_repos_dir) && \
 	mkdir -p $(toolkit_tools_dir) && \
-	cp -r $(toolkit_root)/* $(toolkit_build_dir) && \
-	cp $(mariner_repos_dir)/*.repo $(toolkit_repos_dir) && \
-	cp $(toolkit_component_extra_files) $(toolkit_build_dir) && \
+	cp -r $(toolkit_root_files) $(toolkit_prep_dir) && \
+	cp $(mariner_repos_files) $(toolkit_repos_dir) && \
+	cp $(toolkit_component_extra_files) $(toolkit_prep_dir) && \
 	cp $(go_tool_targets) $(toolkit_tools_dir) && \
-	echo "$(toolkit_version)" > $(toolkit_release_file) && \
-	rm -rf $(toolkit_build_dir)/out && \
-	tar -I $(ARCHIVE_TOOL) -cvp -f $(toolkit_archive) -C $(toolkit_build_dir)/.. $(notdir $(toolkit_build_dir))
+	rm -rf $(toolkit_prep_dir)/out && \
+	tar -cvp -f $(toolkit_archive) -C $(dir $(toolkit_prep_dir)) $(notdir $(toolkit_prep_dir))
+
+rpms-snapshot: $(rpms_snapshot)
+	@echo "RPMs snapshot generated under '$(rpms_snapshot)'."
+
+$(rpms_snapshot): $(rpms_snapshot_per_specs) $(depend_SPECS_DIR)
+	cp $(rpms_snapshot_per_specs) $(rpms_snapshot)
+
+$(rpms_snapshot_per_specs): $(go-rpmssnapshot) $(chroot_worker) $(LOCAL_SPECS) $(LOCAL_SPEC_DIRS) $(SPECS_DIR)
+	@mkdir -p "$(rpms_snapshot_build_dir)"
+	$(go-rpmssnapshot) \
+		--input="$(SPECS_DIR)" \
+		--output="$(rpms_snapshot_per_specs)" \
+		--build-dir="$(rpms_snapshot_build_dir)" \
+		--dist-tag=$(DIST_TAG) \
+		--worker-tar="$(chroot_worker)" \
+		--log-level=$(LOG_LEVEL) \
+		--log-file="$(rpms_snapshot_logs_path)"
 
 print-build-summary:
 	sed -E -n 's:^.+level=info msg="Built \(([^\)]+)\) -> \[(.+)\].+$:\1\t\2:gp' $(LOGS_DIR)/pkggen/rpmbuilding/* | tee $(LOGS_DIR)/pkggen/build-summary.csv
diff --git a/toolkit/scripts/tools.mk b/toolkit/scripts/tools.mk
@@ -27,6 +27,7 @@ go_tool_list = \
 	liveinstaller \
 	pkgworker \
 	roast \
+	rpmssnapshot \
 	scheduler \
 	specreader \
 	srpmpacker \

diff --git a/toolkit/scripts/utils.mk b/toolkit/scripts/utils.mk
@@ -42,9 +42,9 @@ endef
 ######## VARIABLE DEPENDENCY TRACKING ########
 
 # List of variables to watch for changes.
-watch_vars=PACKAGE_BUILD_LIST PACKAGE_REBUILD_LIST PACKAGE_IGNORE_LIST REPO_LIST CONFIG_FILE STOP_ON_PKG_FAIL TOOLCHAIN_ARCHIVE REBUILD_TOOLCHAIN SRPM_PACK_LIST
+watch_vars=PACKAGE_BUILD_LIST PACKAGE_REBUILD_LIST PACKAGE_IGNORE_LIST REPO_LIST CONFIG_FILE STOP_ON_PKG_FAIL TOOLCHAIN_ARCHIVE REBUILD_TOOLCHAIN SRPM_PACK_LIST SPECS_DIR
 # Current list: $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_REPO_LIST) $(depend_CONFIG_FILE) $(depend_STOP_ON_PKG_FAIL)
-#					$(depend_TOOLCHAIN_ARCHIVE) $(depend_REBUILD_TOOLCHAIN) $(depend_SRPM_PACK_LIST)
+#					$(depend_TOOLCHAIN_ARCHIVE) $(depend_REBUILD_TOOLCHAIN) $(depend_SRPM_PACK_LIST) $(depend_SPECS_DIR)
 
 .PHONY: variable_depends_on_phony clean-variable_depends_on_phony
 clean: clean-variable_depends_on_phony

diff --git a/toolkit/tools/internal/rpm/rpm.go b/toolkit/tools/internal/rpm/rpm.go
@@ -189,8 +189,11 @@ func QuerySPEC(specFile, sourceDir, queryFormat string, defines map[string]strin
 }
 
 // QuerySPECForBuiltRPMs queries a SPEC file with queryFormat. Returns only the subpackages, which generate a .rpm file.
-func QuerySPECForBuiltRPMs(specFile, sourceDir, queryFormat string, defines map[string]string) (result []string, err error) {
-	const builtRPMsSwitch = "--builtrpms"
+func QuerySPECForBuiltRPMs(specFile, sourceDir string, defines map[string]string) (result []string, err error) {
+	const (
+		builtRPMsSwitch = "--builtrpms"
+		queryFormat     = ""
+	)
 
 	return QuerySPEC(specFile, sourceDir, queryFormat, defines, builtRPMsSwitch)
 }