From c20578f6fd9a6829c3a2f800bf51f68b48367db5 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Mon, 9 Sep 2024 13:13:30 +0530 Subject: [PATCH] Update makefile, auto-help and better serve (#4109) - The current make -j parallelization doesn't kill the remaining process when one of them errors out, the causes confusion if let's css/js is not building. - This PR also refractors the help target to use comment based generator which is more maintainable. - Update the node scripts to not require `collectstatic` during development, the runserver -> static server can find all the all files from packages and serve them on demand. - `make serve` now also installs python & node packages and builds the static_src if they are outdated. --------- Co-authored-by: Fredrik Jonsson --- .gitignore | 1 + Makefile | 126 +++++++++++++++++++++++---------------------------- package.json | 3 +- 3 files changed, 58 insertions(+), 72 deletions(-) diff --git a/.gitignore b/.gitignore index 00fa002c23..ed260cee0b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ __coverage__/ node_modules/ /npm-debug.log /.idea/ +.cache/ # Distribution / packaging .Python diff --git a/Makefile b/Makefile index 1f6342c06e..9607299f86 100644 --- a/Makefile +++ b/Makefile @@ -1,55 +1,35 @@ +DJANGO_ADDRESS = $(or $(DJANGO_SERVE_ADDRESS), 127.0.0.1) DJANGO_PORT = 9001 +DJANGO_SETTINGS_MODULE = hypha.settings.dev JS_VENDOR_DIR = ./hypha/static_src/javascript/vendor JS_ESM_DIR = ./hypha/static_src/javascript/esm + .PHONY: help -help: - @echo "Usage:" - @echo " make help prints this help." - @echo " make build build js and css resources for development" - @echo " make cov-html generate html coverage report" - @echo " make lint run css, js and python linting." - @echo " make fmt run code formatters on all code." - @echo " make lint-fix try fixing plausible python linting issues." - @echo " make py-test run all python tests and display coverage" - @echo " make test run linting and test and generate html coverage report" - @echo " make serve-docs run documentation development server" - @echo " make serve-django run Django development server on port 9001." - @echo " make serve run Django and docs preview server, also watch and compile frontend changes" - @echo " make watch watch js and css resources for development" - @echo " make download-esm-modules download esm modules from npm and copy to static_src" - @echo " make clean-test-files remove test files during test" +help: ## Show this help + @echo "\nSpecify a command. The choices are:\n" + @grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[0;36m%-12s\033[m %s\n", $$1, $$2}' + @echo "" + .PHONY: serve -serve: - $(MAKE) -j3 watch serve-django serve-docs +serve: .cache/tandem .cache/py-packages .cache/dev-build-fe ## Run Django and docs preview server, also watch and compile frontend changes + @.cache/tandem \ + 'python manage.py runserver_plus $(DJANGO_ADDRESS):$(DJANGO_PORT) --settings=$(DJANGO_SETTINGS_MODULE)' \ + 'npm:watch:*' \ + 'mkdocs serve' + +.PHONY: test +test: lint py-test cov-html ## Run all tests and generate coverage report -.PHONY: build -build: - @echo "Build js and css resources for development." - npm run dev:build .PHONY: fmt -fmt: - @echo "run code formatters on all code." - python -m ruff check --fix . - python -m ruff format . - npx prettier . --write - djhtml hypha/ +fmt: ## Run code formatters on all code + @pre-commit run --all-files -.PHONY: cov-html -cov-html: -ifneq ("$(wildcard .coverage)","") - @rm -rf htmlcov - @echo "Generate html coverage report…" - coverage html - @echo "Open 'htmlcov/index.html' in your browser to see the report." -else - $(error Unable to generate html coverage report, please run 'make test' or 'make py-test') -endif .PHONY: lint -lint: +lint: ## Run all linters @echo "Checking python code style with ruff" ruff check . ruff format --check . @@ -59,55 +39,61 @@ lint: npm run lint -.PHONY: lint-fix -lint-fix: - @echo "Try fixing plausible python linting issues." - ruff check --fix . - .PHONY: py-test -py-test: +py-test: .cache/py-packages ## Run python tests @echo "Running python tests" pytest --reuse-db --cov --cov-report term:skip-covered -.PHONY: serve-django -serve-django: - python manage.py runserver 0.0.0.0:$(DJANGO_PORT) --settings=hypha.settings.dev - -.PHONY: clean-test-files -clean-test-files: @echo "Removing test files generated during test" - find media/ -iname 'test_*.pdf' -delete - find media/ -iname 'test_image*' -delete - find media/ -iname '*.dat' -delete - find media/ -type d -empty -delete - rm -rf media/temp_uploads/* + @find media/ -iname 'test_*.pdf' -o -iname 'test_image*' -o -iname '*.dat' -delete + @find media/ -type d -empty -delete + @rm -rf media/temp_uploads/* -.PHONY: test -test: lint py-test cov-html clean-test-files -.PHONY: serve-docs -serve-docs: - @echo "Serve and watch documentation locally:" - mkdocs serve +.PHONY: cov-html +cov-html: ## Generate html coverage report +ifneq ("$(wildcard .coverage)","") + @rm -rf htmlcov + @echo "Generate html coverage report…" + coverage html + @echo "Open 'htmlcov/index.html' in your browser to see the report." +else + $(error Unable to generate html coverage report, please run 'make test' or 'make py-test') +endif -.PHONY: watch -watch: - @echo "Watch js and css resources for development." - npm run watch .PHONY: download-esm-modules -download-esm-modules: +download-esm-modules: ## Download ESM modules pip install download-esm download-esm @github/relative-time-element $(JS_ESM_DIR) download-esm @github/filter-input-element $(JS_ESM_DIR) download-esm choices.js $(JS_ESM_DIR) -.PHONY: copy-npm-scripts -copy-npm-scripts: - # Used by "npm install" + +.cache/tandem: ## Install tandem, a tool to run multiple commands in parallel + @mkdir -p $$(dirname $@) + @curl -fsSL https://raw.githubusercontent.com/rosszurowski/tandem/main/install.sh | bash -s -- --dest="$$(dirname $@)" + + +.cache/dev-build-fe: .cache/npm-packages $(shell find hypha/static_src) ## Build frontend resources for development + @mkdir -p $$(dirname $@) + @.cache/tandem 'npm:dev:build:*' + @touch $@ + + +.cache/py-packages: requirements-dev.txt requirements-docs.txt ## Install python packages + @mkdir -p $$(dirname $@) + pip install -r requirements-dev.txt -r requirements-docs.txt + @touch $@ + + +.cache/npm-packages: package.json ## Install node packages and copy javascript files to vendor directory + @mkdir -p $$(dirname $@) + NODE_ENV=development npm install cp node_modules/htmx.org/dist/htmx.min.js $(JS_VENDOR_DIR)/htmx.min.js cp node_modules/htmx.org/dist/ext/multi-swap.js $(JS_VENDOR_DIR)/htmx-ext-multi-swap.min.js cp node_modules/alpinejs/dist/cdn.min.js $(JS_VENDOR_DIR)/alpine.min.js cp node_modules/@alpinejs/focus/dist/cdn.min.js $(JS_VENDOR_DIR)/alpine-focus.min.js cp node_modules/daterangepicker/moment.min.js $(JS_VENDOR_DIR)/moment.min.js cp node_modules/daterangepicker/daterangepicker.js $(JS_VENDOR_DIR)/daterangepicker.min.js + @touch $@ diff --git a/package.json b/package.json index 608ca115f1..68cd67df74 100644 --- a/package.json +++ b/package.json @@ -63,9 +63,8 @@ "watch:js": "babel --watch ./hypha/static_src/javascript --out-dir ./hypha/static_compiled/js", "watch:sass": "sass --watch ./hypha/static_src/sass:./hypha/static_compiled/css --load-path ./hypha/static_src/sass", "watch:tailwind": "tailwindcss -i ./hypha/static_src/tailwind/main.css -o ./hypha/static_compiled/css/tailwind-output.css --watch", - "watch:static": "nodemon --delay 3 --exec \"npm run collectstatic\" --watch ./hypha/static_compiled --ext css,js,json,png,svg", "watch:lint": "nodemon --exec \"npm run lint\" --watch ./hypha/static_src/sass --watch ./hypha/static_src/javscript --ext scss,js", - "dev:build": "npm-run-all --print-label --serial clean --parallel dev:build:* --serial collectstatic", + "dev:build": "npm-run-all --print-label --serial clean --parallel dev:build:*", "dev:build:js": "npm run build:js", "dev:build:sass": "sass ./hypha/static_src/sass:./hypha/static_compiled/css --load-path ./hypha/static_src/sass", "dev:build:lint": "npm run lint",