Skip to content

Commit

Permalink
spksrc.requirement.mk: Unified method to process requirements files
Browse files Browse the repository at this point in the history
  • Loading branch information
th0ma7 committed Feb 2, 2025
1 parent 25945f0 commit 5fee872
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 69 deletions.
104 changes: 104 additions & 0 deletions mk/spksrc.requirement.mk
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
### Wheel rules
# Process wheels for modules listed in WHEELS.
# 1. wheel_download
# 2. wheel_compile
# 3. wheel_install
#
# Targets are executed in the following order:
# requirement_msg_target
# pre_requirement_target (override with PRE_REQUIREMENT_TARGET)
# requirement_target (override with REQUIREMENT_TARGET)
# post_requirement_target (override with POST_REQUIREMENT_TARGET)
# Variables:
# WHEELS List of wheels to go through

# When wheel is called from:
# spksrc.spk.mk: MAKECMDGOALS is empty (needs to be set to wheel)
# make wheel-<arch>-<tcversion> : MAKECMDGOALS is wheel
# make download-wheels : MAKECMDGOALS is download-wheels
REQUIREMENT_GOAL := $(if $(MAKECMDGOALS),$(MAKECMDGOALS),requirement)

ifeq ($(strip $(PRE_REQUIREMENT_TARGET)),)
PRE_REQUIREMENT_TARGET = pre_requirement_target
else
$(PRE_REQUIREMENT_TARGET): requirement_msg_target
endif
ifeq ($(strip $(REQUIREMENT_TARGET)),)
REQUIREMENT_TARGET = requirement_target
else
$(REQUIREMENT_TARGET): $(BUILD_REQUIREMENT_TARGET)
endif
ifeq ($(strip $(POST_REQUIREMENT_TARGET)),)
POST_REQUIREMENT_TARGET = post_requirement_target
else
$(POST_REQUIREMENT_TARGET): $(REQUIREMENT_TARGET)
endif

requirement_msg_target:
@$(MSG) "Processing wheel for $(NAME)"

pre_requirement_target: requirement_msg_target

requirement-%:
ifneq ($(strip $(REQUIREMENT)),)
@$(MSG) $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) REQUIREMENT=\"$(REQUIREMENT)\" requirement
@MAKEFLAGS= $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) REQUIREMENT="$(REQUIREMENT)" requirement --no-print-directory
else
$(error No python wheel requirement to process)
endif

requirement_target: SHELL:=/bin/bash
requirement_target: pre_requirement_target
ifneq ($(wildcard $(REQUIREMENT)),)
@set -e ; \
while IFS= read -r requirement ; do \
$(MSG) Processing requirement file [$${requirement}] ; \
wheel=$${requirement#*requirements-*.txt:} ; \
prefix=$$(basename $${requirement%%:*}) ; \
case $${prefix} in \
requirements-abi3.txt) type=abi3 ;; \
requirements-crossenv*.txt) type=crossenv ;; \
requirements-pure.txt) type=pure ;; \
esac ; \
$(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_TYPE=\"$${type}\" REQUIREMENT_GOAL=\"$(REQUIREMENT_GOAL)\" requirement ; \
MAKEFLAGS= $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_TYPE="$${type}" REQUIREMENT_GOAL="$(REQUIREMENT_GOAL)" requirement --no-print-directory || exit 1 ; \
done < <(grep -svH -e "^\#" -e "^\$$" $(wildcard $(REQUIREMENT)) | sed 's/\s* #.*//')
else
@for requirement in $(REQUIREMENT) ; do \
$(MSG) Processing requirement unit [$${requirement}] ; \
wheel=$$(echo $${requirement} | sed -E "s/^(abi3|build|cross|crossenv|pure)://") ; \
case $${requirement} in \
abi3:*) type=abi3 ;; \
build:*) type=build ;; \
cross:*) type=cross ;; \
crossenv:*) type=crossenv ;; \
pure:*) type=pure ;; \
*) type=$(WHEEL_DEFAULT_PREFIX) ;; \
esac ; \
version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \
if [ "$$(grep -s egg <<< $${requirement})" ]; then \
name=$$(echo $${wheel#*egg=} | cut -f1 -d=) ; \
wheel_url=$$(echo $${wheel%%#egg=*}) ; \
else \
name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|build|cross|crossenv|pure)://") ; \
fi ; \
if [ ! "$${version}" ]; then \
$(MSG) Fetching latest version available ; \
query="curl -s https://pypi.org/pypi/$${name}/json" ; \
query+=" | jq -r '.releases[][]" ; \
query+=" | select(.packagetype==\"sdist\")" ; \
query+=" | .filename'" ; \
query+=" | sort -V" ; \
query+=" | tail -1" ; \
query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \
query+=" | awk -F'-' '{print \$$2}'" ; \
version=$$(eval $${query} 2>/dev/null) ; \
fi ; \
$(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$(or $(WHEEL_TYPE),$${type})\" WHEEL_URL=\"$${wheel_url}\" $(REQUIREMENT_GOAL) ; \
MAKEFLAGS= $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$(or $(WHEEL_TYPE),$${type})" WHEEL_URL="$${wheel_url}" $(REQUIREMENT_GOAL) --no-print-directory ; \
done
endif

post_requirement_target: $(REQUIREMENT_TARGET)

requirement: $(POST_REQUIREMENT_TARGET)
86 changes: 17 additions & 69 deletions mk/spksrc.wheel.mk
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# Targets are executed in the following order:
# wheel_msg_target
# pre_wheel_target (override with PRE_WHEEL_TARGET)
# build_wheel_target (override with WHEEL_TARGET)
# wheel_target (override with WHEEL_TARGET)
# post_wheel_target (override with POST_WHEEL_TARGET)
# Variables:
# WHEELS List of wheels to go through
Expand All @@ -24,6 +24,9 @@ WHEEL_COOKIE = $(WORK_DIR)/.wheel_done
## python wheel specific configurations
include ../../mk/spksrc.wheel-env.mk

## requirement processing routines
include ../../mk/spksrc.requirement.mk

## python wheel specific configurations
include ../../mk/spksrc.crossenv.mk

Expand Down Expand Up @@ -54,7 +57,7 @@ else
$(PRE_WHEEL_TARGET): wheel_msg_target
endif
ifeq ($(strip $(WHEEL_TARGET)),)
WHEEL_TARGET = build_wheel_target
WHEEL_TARGET = wheel_target
else
$(WHEEL_TARGET): $(BUILD_WHEEL_TARGET)
endif
Expand All @@ -74,77 +77,22 @@ ifneq ($(strip $(WHEELS)),)
@$(MSG) $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) WHEELS=\"$(WHEELS)\" wheel
@MAKEFLAGS= $(MAKE) ARCH=$(firstword $(subst -, ,$*)) TCVERSION=$(lastword $(subst -, ,$*)) WHEELS="$(WHEELS)" wheel --no-print-directory
else
$(error No wheel to process)
$(error No python wheel to process)
endif

build_wheel_target: SHELL:=/bin/bash
build_wheel_target: pre_wheel_target
wheel_target: SHELL:=/bin/bash
wheel_target: pre_wheel_target
ifneq ($(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))),)
@set -e ; \
while IFS= read -r requirement ; do \
$(MSG) Processing requirement [$${requirement}] ; \
wheel=$${requirement#*requirements-*.txt:} ; \
file=$$(basename $${requirement%%:*}) ; \
case $${file} in \
requirements-pure.txt) type=pure ;; \
requirements-crossenv*.txt) type=crossenv ;; \
requirements-abi3.txt) type=abi3 ;; \
*) type=$(WHEEL_DEFAULT_PREFIX) ;; \
esac ; \
version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \
if [ "$$(grep -s egg <<< $${wheel})" ]; then \
name=$$(echo $${wheel#*egg=} | cut -f1 -d=) ; \
wheel=$$(echo $${wheel%%#egg=*}) ; \
else \
name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|crossenv|pure)://") ; \
fi ; \
if [ ! "$${version}" ]; then \
$(MSG) Fetching latest version available ; \
query="curl -s https://pypi.org/pypi/$${name}/json" ; \
query+=" | jq -r '.releases[][]" ; \
query+=" | select(.packagetype==\"sdist\")" ; \
query+=" | .filename'" ; \
query+=" | sort -V" ; \
query+=" | tail -1" ; \
query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \
query+=" | awk -F'-' '{print \$$2}'" ; \
version=$$(eval $${query} 2>/dev/null) ; \
fi ; \
$(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(WHEEL_GOAL) ; \
MAKEFLAGS= $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(WHEEL_GOAL) --no-print-directory || exit 1 ; \
done < <(grep -svH -e "^\#" -e "^\$$" $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))) | sed 's/\s* #.*//')
endif
ifneq ($(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)),)
@for requirement in $(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)) ; do \
$(MSG) Processing requirement [$${requirement}] ; \
wheel=$$(echo $${requirement} | sed -E "s/^(abi3|crossenv|pure)://") ; \
case $${requirement} in \
abi3:*) type=abi3 ;; \
crossenv:*) type=crossenv ;; \
pure:*) type=pure ;; \
*) type=$(WHEEL_DEFAULT_PREFIX) ;; \
esac ; \
version=$$(echo $${wheel} | grep -oP '(?<=([<>=]=))[^ ]*' || echo "") ; \
if [ "$$(grep -s egg <<< $${requirement})" ]; then \
name=$$(echo $${wheel#*egg=} | cut -f1 -d=) ; \
wheel=$$(echo $${wheel%%#egg=*}) ; \
else \
name=$$(echo $${wheel%%[<>=]=*} | sed -E "s/^(abi3|crossenv|pure)://") ; \
fi ; \
if [ ! "$${version}" ]; then \
$(MSG) Fetching latest version available ; \
query="curl -s https://pypi.org/pypi/$${name}/json" ; \
query+=" | jq -r '.releases[][]" ; \
query+=" | select(.packagetype==\"sdist\")" ; \
query+=" | .filename'" ; \
query+=" | sort -V" ; \
query+=" | tail -1" ; \
query+=" | sed -e 's/.tar.gz//g' -e 's/.zip//g'" ; \
query+=" | awk -F'-' '{print \$$2}'" ; \
version=$$(eval $${query} 2>/dev/null) ; \
fi ; \
$(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${wheel}\" WHEEL_NAME=\"$${name}\" WHEEL_VERSION=\"$${version}\" WHEEL_TYPE=\"$${type}\" $(WHEEL_GOAL) ; \
MAKEFLAGS= $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${wheel}" WHEEL_NAME="$${name}" WHEEL_VERSION="$${version}" WHEEL_TYPE="$${type}" $(WHEEL_GOAL) --no-print-directory ; \
for requirement in $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))) ; do \
$(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${requirement}\" REQUIREMENT_GOAL=\"$(WHEEL_GOAL)\" requirement ; \
MAKEFLAGS= $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${requirement}" REQUIREMENT_GOAL="$(WHEEL_GOAL)" requirement --no-print-directory || exit 1 ; \
done
else
@set -e ; \
for requirement in $(filter-out $(addprefix src/,$(notdir $(wildcard $(abspath $(addprefix $(WORK_DIR)/../,$(WHEELS)))))),$(WHEELS)) ; do \
$(MSG) $(MAKE) ARCH=$(ARCH) TCVERSION=$(TCVERSION) REQUIREMENT=\"$${requirement}\" REQUIREMENT_GOAL=\"$(WHEEL_GOAL)\" requirement ; \
MAKEFLAGS= $(MAKE) ARCH="$(ARCH)" TCVERSION="$(TCVERSION)" REQUIREMENT="$${requirement}" REQUIREMENT_GOAL="$(WHEEL_GOAL)" requirement --no-print-directory || exit 1 ; \
done
endif

Expand Down

0 comments on commit 5fee872

Please sign in to comment.