Update makefile, auto-help and better serve (#4109)

- The current make -j parallelization doesn't kill the remaining process
when one of them errors out, the causes confusion if let's css/js is not
building.

- This PR also refractors the help target to use comment based generator
which is more maintainable.

- Update the node scripts to not require `collectstatic` during
development, the runserver -> static server can find all the all files
from packages and serve them on demand.
- `make serve` now also installs python & node packages and builds the
static_src if they are outdated.

---------

Co-authored-by: Fredrik Jonsson <[email protected]>

.gitignore

@@ -13,6 +13,7 @@ __coverage__/
 node_modules/
 /npm-debug.log
 /.idea/
+.cache/
 
 # Distribution / packaging
 .Python

Makefile

@@ -1,55 +1,35 @@
+DJANGO_ADDRESS = $(or $(DJANGO_SERVE_ADDRESS), 127.0.0.1)
 DJANGO_PORT = 9001
+DJANGO_SETTINGS_MODULE = hypha.settings.dev
 JS_VENDOR_DIR = ./hypha/static_src/javascript/vendor
 JS_ESM_DIR = ./hypha/static_src/javascript/esm
 
+
 .PHONY: help
-help:
-	@echo "Usage:"
-	@echo "  make help               prints this help."
-	@echo "  make build              build js and css resources for development"
-	@echo "  make cov-html           generate html coverage report"
-	@echo "  make lint               run css, js and python linting."
-	@echo "  make fmt                run code formatters on all code."
-	@echo "  make lint-fix           try fixing plausible python linting issues."
-	@echo "  make py-test            run all python tests and display coverage"
-	@echo "  make test               run linting and test and generate html coverage report"
-	@echo "  make serve-docs         run documentation development server"
-	@echo "  make serve-django       run Django development server on port 9001."
-	@echo "  make serve              run Django and docs preview server, also watch and compile frontend changes"
-	@echo "  make watch              watch js and css resources for development"
-	@echo "  make download-esm-modules  download esm modules from npm and copy to static_src"
-	@echo "  make clean-test-files  remove test files during test"
+help: ## Show this help
+	@echo "\nSpecify a command. The choices are:\n"
+	@grep -E '^[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[0;36m%-12s\033[m %s\n", $$1, $$2}'
+	@echo ""
+
 
 .PHONY: serve
-serve:
-	$(MAKE) -j3 watch serve-django serve-docs
+serve: .cache/tandem .cache/py-packages .cache/dev-build-fe  ## Run Django and docs preview server, also watch and compile frontend changes
+	@.cache/tandem \
+		'python manage.py runserver_plus $(DJANGO_ADDRESS):$(DJANGO_PORT) --settings=$(DJANGO_SETTINGS_MODULE)' \
+		'npm:watch:*' \
+		'mkdocs serve'
+
+.PHONY: test
+test: lint py-test cov-html  ## Run all tests and generate coverage report
 
-.PHONY: build
-build:
-	@echo "Build js and css resources for development."
-	npm run dev:build
 
 .PHONY: fmt
-fmt:
-	@echo "run code formatters on all code."
-	python -m ruff check --fix .
-	python -m ruff format .
-	npx prettier . --write
-	djhtml hypha/
+fmt:  ## Run code formatters on all code
+	@pre-commit run --all-files
 
-.PHONY: cov-html
-cov-html:
-ifneq ("$(wildcard .coverage)","")
-	@rm -rf htmlcov
-	@echo "Generate html coverage report…"
-	coverage html
-	@echo "Open 'htmlcov/index.html' in your browser to see the report."
-else
-	$(error Unable to generate html coverage report, please run 'make test' or 'make py-test')
-endif
 
 .PHONY: lint
-lint:
+lint:  ## Run all linters
 	@echo "Checking python code style with ruff"
 	ruff check .
 	ruff format --check .
@@ -59,55 +39,61 @@ lint:
 	npm run lint
 
 
-.PHONY: lint-fix
-lint-fix:
-	@echo "Try fixing plausible python linting issues."
-	ruff check --fix .
-
 .PHONY: py-test
-py-test:
+py-test: .cache/py-packages  ## Run python tests
 	@echo "Running python tests"
 	pytest --reuse-db --cov --cov-report term:skip-covered
 
-.PHONY: serve-django
-serve-django:
-	python manage.py runserver 0.0.0.0:$(DJANGO_PORT) --settings=hypha.settings.dev
-
-.PHONY: clean-test-files
-clean-test-files:
 	@echo "Removing test files generated during test"
-	find media/ -iname 'test_*.pdf' -delete
-	find media/ -iname 'test_image*' -delete
-	find media/ -iname '*.dat' -delete
-	find media/ -type d -empty -delete
-	rm -rf media/temp_uploads/*
+	@find media/ -iname 'test_*.pdf' -o -iname 'test_image*' -o -iname '*.dat' -delete
+	@find media/ -type d -empty -delete
+	@rm -rf media/temp_uploads/*
 
-.PHONY: test
-test: lint py-test cov-html clean-test-files
 
-.PHONY: serve-docs
-serve-docs:
-	@echo "Serve and watch documentation locally:"
-	mkdocs serve
+.PHONY: cov-html
+cov-html:  ## Generate html coverage report
+ifneq ("$(wildcard .coverage)","")
+	@rm -rf htmlcov
+	@echo "Generate html coverage report…"
+	coverage html
+	@echo "Open 'htmlcov/index.html' in your browser to see the report."
+else
+	$(error Unable to generate html coverage report, please run 'make test' or 'make py-test')
+endif
 
-.PHONY: watch
-watch:
-	@echo "Watch js and css resources for development."
-	npm run watch
 
 .PHONY: download-esm-modules
-download-esm-modules:
+download-esm-modules:  ## Download ESM modules
 	pip install download-esm
 	download-esm @github/relative-time-element $(JS_ESM_DIR)
 	download-esm @github/filter-input-element $(JS_ESM_DIR)
 	download-esm choices.js $(JS_ESM_DIR)
 
-.PHONY: copy-npm-scripts
-copy-npm-scripts:
-	# Used by "npm install"
+
+.cache/tandem:  ## Install tandem, a tool to run multiple commands in parallel
+	@mkdir -p $$(dirname $@)
+	@curl -fsSL https://raw.githubusercontent.com/rosszurowski/tandem/main/install.sh | bash -s -- --dest="$$(dirname $@)"
+
+
+.cache/dev-build-fe: .cache/npm-packages $(shell find hypha/static_src)  ## Build frontend resources for development
+	@mkdir -p $$(dirname $@)
+	@.cache/tandem 'npm:dev:build:*'
+	@touch $@
+
+
+.cache/py-packages: requirements-dev.txt requirements-docs.txt  ## Install python packages
+	@mkdir -p $$(dirname $@)
+	pip install -r requirements-dev.txt -r requirements-docs.txt
+	@touch $@
+
+
+.cache/npm-packages: package.json  	## Install node packages and copy javascript files to vendor directory
+	@mkdir -p $$(dirname $@)
+	NODE_ENV=development npm install
 	cp node_modules/htmx.org/dist/htmx.min.js $(JS_VENDOR_DIR)/htmx.min.js
 	cp node_modules/htmx.org/dist/ext/multi-swap.js $(JS_VENDOR_DIR)/htmx-ext-multi-swap.min.js
 	cp node_modules/alpinejs/dist/cdn.min.js $(JS_VENDOR_DIR)/alpine.min.js
 	cp node_modules/@alpinejs/focus/dist/cdn.min.js $(JS_VENDOR_DIR)/alpine-focus.min.js
 	cp node_modules/daterangepicker/moment.min.js $(JS_VENDOR_DIR)/moment.min.js
 	cp node_modules/daterangepicker/daterangepicker.js $(JS_VENDOR_DIR)/daterangepicker.min.js
+	@touch $@

package.json

@@ -63,9 +63,8 @@
     "watch:js": "babel --watch ./hypha/static_src/javascript --out-dir ./hypha/static_compiled/js",
     "watch:sass": "sass --watch ./hypha/static_src/sass:./hypha/static_compiled/css --load-path ./hypha/static_src/sass",
     "watch:tailwind": "tailwindcss -i ./hypha/static_src/tailwind/main.css -o ./hypha/static_compiled/css/tailwind-output.css --watch",
-    "watch:static": "nodemon --delay 3 --exec \"npm run collectstatic\" --watch ./hypha/static_compiled --ext css,js,json,png,svg",
     "watch:lint": "nodemon --exec \"npm run lint\" --watch ./hypha/static_src/sass --watch ./hypha/static_src/javscript --ext scss,js",
-    "dev:build": "npm-run-all --print-label --serial clean --parallel dev:build:* --serial collectstatic",
+    "dev:build": "npm-run-all --print-label --serial clean --parallel dev:build:*",
     "dev:build:js": "npm run build:js",
     "dev:build:sass": "sass ./hypha/static_src/sass:./hypha/static_compiled/css --load-path ./hypha/static_src/sass",
     "dev:build:lint": "npm run lint",