tooling: Make cloe a super-build of all packages

New: - Add `cloe-meta` package to replace previous role of `cloe`. - Add `cloe-plugins-core` package recipe (but do not build). - Add editable builds to GitHub workflow `build-cloe` matrix. Changed: - Package `cloe` provides all Cloe packages compiled in one go. This is a boon to development, as we make `cloe` editable and only have to work with a single package. It also massively speeds up compilation: - Conan and CMake configuration is only performed once. - All cores can now be utilized much more effectively during the build process. - Do not aggressively lint everything The developer can do this themselves by setting a cmake define: https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_CLANG_TIDY.html - Move all individual tests into single project tests Still support multiple profiles, but this makes it easier to support the cloe super-build and should also make it easier to develop and maintain tests. - Renamed top-level Make targets: - `status` is now `status-all` - `export` is now `export-all` - `deploy` is now `deploy-all` - `clean` is now `clean-all` - `purge` is now `purge-all` - The following top-level Make targets just refer to `cloe` super-build package: - `package` - `smoketest` - `smoketest-deps` - `status` - `export` Fixed: - Plugin Conan configurations do not export correct library path. Removed: - Work-In-Progress Lua files.
eclipse · Dec 4, 2023 · 9f6e8d6 · 9f6e8d6
1 parent a9c7c65
commit 9f6e8d6
Show file tree

Hide file tree

Showing 133 changed files with 990 additions and 1,191 deletions.
diff --git a/.github/workflows/build-cloe.yaml b/.github/workflows/build-cloe.yaml
@@ -28,9 +28,20 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04, ubuntu-22.04]
-        build_type: [RelWithDebInfo]
-        package_target: [package]
+        os:
+          - "ubuntu-20.04"
+          - "ubuntu-22.04"
+        conan_profile:
+          - "cloe-normal"
+        package_target:
+          # 1. Build each test configuration in Conan cache and run all tests
+          - "export-all smoketest-deps smoketest"
+
+          # 2. Build cloe super-package in editable mode and run tests
+          - "editable all smoketest TEST_CONANFILES=tests/conanfile_all.py"
+
+          # 3. Build individual packages in editable mode and run tests
+          - "editable-select build-all smoketest TEST_CONANFILES=tests/conanfile_split.py"
     env:
       CONAN_NON_INTERACTIVE: "yes"
       DEBIAN_FRONTEND: noninteractive
@@ -59,12 +70,9 @@ jobs:
       - name: Configure Conan
         run: |
           make setup-conan
-      - name: Build cloe w/ package
+          conan config set general.default_profile=${{ matrix.conan_profile }}
+          conan config set general.default_build_profile=${{ matrix.conan_profile }}
+          make export-cli
+      - name: Build cloe
         run: |
           make ${{ matrix.package_target }}
-      - name: Build smoketest dependencies
-        run: |
-          make smoketest-deps
-      - name: Run smoketests
-        run: |
-          make smoketest
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -0,0 +1,30 @@
+# This CMakeLists.txt configures a super-build containing everything
+# from this repo.
+#
+# It is currently experimental.
+#
+
+cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
+
+project(cloe LANGUAGES CXX)
+
+set(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/runtime/cmake")
+
+# Since a super-build does not export packages individually via Conan,
+# we cannot depend on Conan-generated CMake config files, instead we
+# use the CMake targets directly as if they were already found.
+set(CLOE_FIND_PACKAGES OFF CACHE BOOL "Call find_package() for cloe packages" FORCE)
+
+# Ensure output goes to one place so cloe-launch can find the plugins
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin)
+
+# Ensure we can test from this level.
+include(CTest)
+
+add_subdirectory(fable)
+add_subdirectory(runtime)
+add_subdirectory(models)
+add_subdirectory(oak)
+add_subdirectory(engine)
+add_subdirectory(plugins)
diff --git a/Makefile b/Makefile
@@ -3,6 +3,11 @@
 # This file contains Makefile targets for the cloe project.
 #
 
+# Make configuration:
+SHELL := /bin/bash
+GNUMAKEFLAGS := --no-print-directory
+SUBMAKEFLAGS :=
+
 CLOE_ROOT   := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 CLOE_LAUNCH := PYTHONPATH="${CLOE_ROOT}/cli" python3 -m cloe_launch
 
@@ -20,78 +25,96 @@ AG := $(or \
 
 # Build configuration:
 BUILD_DIR       := build
-LOCKFILE_SOURCE := conanfile.py
-BUILD_LOCKFILE  := ${BUILD_DIR}/conan.lock
-LOCKFILE_OPTION := --lockfile="${CLOE_ROOT}/${BUILD_LOCKFILE}"
 INSTALL_DIR     := /usr/local
 CONAN_OPTIONS   :=
 
+# Lockfile for cloe-meta:
+META_LOCKFILE_SOURCE := meta/conanfile.py
+META_BUILD_LOCKFILE  := meta/build/conan.lock
+META_LOCKFILE_OPTION := --lockfile="${CLOE_ROOT}/${META_BUILD_LOCKFILE}"
+
 .DEFAULT_GOAL := help
 .PHONY: help
 .SILENT: help
 help::
 	$(call print_help_usage)
 	echo
+	$(call print_help_section, "Default target")
+	$(call print_help_target, help, "show this help on available targets")
+	echo
 
 # Setup targets ---------------------------------------------------------------
 include Makefile.setup
 
+${META_BUILD_LOCKFILE}:
+	${MAKE} -C meta LOCKFILE_SOURCE=conanfile.py lockfile
+
+.PHONY: lockfile
+lockfile: ${META_BUILD_LOCKFILE}
+
 # Workspace targets -----------------------------------------------------------
 help::
 	$(call print_help_section, "Available workspace targets")
-	$(call print_help_target, status, "show status of each of the Conan packages")
-	$(call print_help_target, smoketest-deps, "build system test pre-requisites")
-	$(call print_help_target, smoketest, "run system tests")
-	$(call print_help_target, docs, "generate documentation")
-	$(call print_help_target, deploy, "deploy Cloe to INSTALL_DIR [=${INSTALL_DIR}]")
-	$(call print_help_target, deploy-cli, "install ${_yel}cloe-launch${_rst} with ${_dim}${PIPX}${_rst}")
-	$(call print_help_target, export-cli, "export ${_yel}cloe-launch-profile${_rst} Conan recipe")
+	$(call print_help_target, docs, "generate Doxygen and Sphinx documentation")
 	echo
 
-${BUILD_LOCKFILE}:
-	${MAKE} -f Makefile.package SOURCE_CONANFILE=/dev/null LOCKFILE_SOURCE=${LOCKFILE_SOURCE} ${BUILD_LOCKFILE}
-
-.PHONY: lockfile
-lockfile: ${BUILD_LOCKFILE}
+.PHONY: docs
+docs:
+	$(call print_header, "Generating Doxygen documentation...")
+	${MAKE} -C docs doxygen
+	$(call print_header, "Generating Sphinx documentation...")
+	${MAKE} -C docs html
 
-.PHONY: status
-status: ${BUILD_LOCKFILE}
-	@for pkg in ${ALL_PKGS}; do \
-		[ -d $${pkg} ] || continue; \
-		${MAKE} LOCKFILE_SOURCE="" LOCKFILE_OPTION=${LOCKFILE_OPTION} -C $${pkg} status || true; \
-	done
+help::
+	$(call print_help_target, export-cli, "export ${_yel}cloe-launch-profile${_rst} Conan recipe")
+	$(call print_help_target, deploy-cli, "install ${_yel}cloe-launch${_rst} with ${_dim}${PIPX}${_rst}")
+	echo
 
-.PHONY: deploy
-deploy:
-	$(call print_header, "Deploying binaries to ${INSTALL_DIR}...")
-	conan install ${CONAN_OPTIONS} --install-folder ${BUILD_DIR}/deploy -g deploy .
-	mkdir -p ${INSTALL_DIR}
-	cp -r ${BUILD_DIR}/deploy/cloe-*/* ${INSTALL_DIR}/
+.PHONY: export-cli
+export-cli:
+	${MAKE} -C cli export
 
 .PHONY: deploy-cli
 deploy-cli:
 	$(call print_header, "Deploying cloe-launch binary with pip...")
 	${MAKE} -C cli install
 
-.PHONY: export-cli
-export-cli:
-	${MAKE} -C cli export
+help::
+	$(call print_help_target, lockfile, "create a lockfile for cloe-meta package")
+	$(call print_help_target, status-all, "show status of each of the Conan packages")
+	$(call print_help_target, export-all, "export all package sources to Conan cache")
+	$(call print_help_target, build-all, "build individual packages locally in-source")
+	$(call print_help_target, deploy-all, "deploy Cloe to INSTALL_DIR [=${INSTALL_DIR}]")
+	$(call print_help_target, clean-all, "clean entire repository of temporary files")
+	$(call print_help_target, purge-all, "remove all cloe packages (in any version) from Conan cache")
+	echo
 
-export: export-cli
-package: export-cli
+.PHONY: build-all
+build-all: lockfile
+	${MAKE} all-select UNSELECT_PKGS="meta" CONAN_OPTIONS="${CONAN_OPTIONS} ${META_LOCKFILE_OPTION}"
 
-.PHONY: docs
-docs:
-	$(call print_header, "Generating Doxygen documentation...")
-	${MAKE} -C docs doxygen
-	$(call print_header, "Generating Sphinx documentation...")
-	${MAKE} -C docs html
+.PHONY: status-all
+status-all: ${META_BUILD_LOCKFILE}
+	@for pkg in ${ALL_PKGS}; do \
+		[ -d $${pkg} ] || continue; \
+		${MAKE} LOCKFILE_SOURCE="" LOCKFILE_OPTION=${META_LOCKFILE_OPTION} -C $${pkg} status || true; \
+	done
 
-.PHONY: smoketest-deps
-smoketest-deps: export-cli smoketest-deps-select
+.PHONY: export-all
+export-all:
+	$(call print_header, "Exporting all cloe Conan packages...")
+	${MAKE} export-select export-cli export
 
-.PHONY: smoketest
-smoketest: smoketest-select
+.PHONY: deploy-all
+deploy-all:
+	$(call print_header, "Deploying binaries to ${INSTALL_DIR}...")
+	conan install ${CONAN_OPTIONS} --install-folder ${BUILD_DIR}/deploy -g deploy .
+	mkdir -p ${INSTALL_DIR}
+	cp -r ${BUILD_DIR}/deploy/cloe-*/* ${INSTALL_DIR}/
+
+.PHONY: clean-all
+clean-all:
+	${MAKE} clean clean-select
 
 .PHONY: purge-all
 purge-all:
@@ -122,6 +145,8 @@ todos:
 	${AG} FIXME
 	${AG} XXX
 
+# Hidden development targets --------------------------------------------------
+
 .PHONY: grep-uuids
 grep-uuids:
 	${AG} "\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b"
@@ -134,5 +159,12 @@ find-missing-eol:
 sanitize-files:
 	git grep --cached -Ilz '' | while IFS= read -rd '' f; do tail -c1 < "$$f" | read -r _ || echo >> "$$f"; done
 
-# Build targets ---------------------------------------------------------------
+# Micro-packages build targets ------------------------------------------------
 include Makefile.all
+
+# Mono-package build targets --------------------------------------------------
+DISABLE_HELP_PREAMBLE := true
+help::
+	@printf "Available $(_yel)cloe$(_rst) package targets:\n"
+
+include Makefile.package
diff --git a/Makefile.all b/Makefile.all
@@ -35,8 +35,8 @@ SHELL := /bin/bash
 GNUMAKEFLAGS := --no-print-directory
 SUBMAKEFLAGS :=
 
-META_PKG := cloe
-PLUGIN_PKGS := $(wildcard plugins/*)
+META_PKG := meta
+PLUGIN_PKGS := plugins/basic plugins/gndtruth_extractor plugins/minimator plugins/mocks plugins/noisy_sensor plugins/speedometer plugins/virtue
 ALL_PKGS := fable runtime models oak engine ${PLUGIN_PKGS} ${META_PKG}
 WITHOUT_PKGS :=
 UNSELECT_PKGS := ${WITHOUT_PKGS}
@@ -62,6 +62,7 @@ models: runtime
 oak: runtime
 engine: models oak
 ${PLUGIN_PKGS}: runtime models
+${META_PKG}: fable runtime models oak engine ${PLUGIN_PKGS}
 
 ## BUILD_POLICY
 ##   Usage: make BUILD_POLICY="missing"
@@ -106,22 +107,13 @@ ${1}-each: ${4}
 endef
 
 REGEX_TARGET := 's/(-vendor|-select)?-each//'
-$(filter-out ${META_PKG}, ${ALL_PKGS} ${ALL_VENDOR}):
+${ALL_PKGS} ${ALL_VENDOR}:
 	${MAKE} -C $@ $(shell echo ${MAKECMDGOALS} | sed -re ${REGEX_TARGET})
 
-# Re-define ${META_PKG} target to use Makefile.package, and only run for targets
-# where it makes sense, since "${META_PKG}" is a Conan meta-package.
-${META_PKG}:
-	for case in export package package-outdated list purge clean smoketest smoketest-deps; do \
-		if [ "$$(echo '${MAKECMDGOALS}' | sed -re ${REGEX_TARGET})" == "$${case}" ]; then \
-			${MAKE} -f Makefile.package CONAN_OPTIONS="${CONAN_OPTIONS}" $${case} || exit 1; \
-		fi \
-	done
-
 # Usage: $(call make_vendor_target, TARGET-NAME, HELP-DESCRIPTION, HELP-CATEGORY)
-# define make_vendor_target
-# $(eval $(call _make_target_rules,${1},${2},${3},${SELECT_VENDOR}))
-# endef
+define make_vendor_target
+$(eval $(call _make_target_rules,${1},${2},${3},${SELECT_VENDOR}))
+endef
 
 # Usage: $(call make_every_target, TARGET-NAME, HELP-DESCRIPTION, HELP-CATEGORY)
 define make_every_target
@@ -139,19 +131,16 @@ endef
 .PHONY: help
 .SILENT: help
 help::
-	$(call print_help_section, "Available build targets")
-
-# $(call make_vendor_target, export-vendor, "export all vendor packages", "[conan-cache]")
-# $(call make_vendor_target, package-vendor, "create all vendor packages", "[conan-cache]")
-# $(call make_vendor_target, download-vendor, "download or build vendor packages", "[conan-cache]")
+	$(call print_help_section, "Available multi-package targets")
 
+ifneq "${ALL_VENDOR}" ""
 help::
-	echo
-
-$(call make_every_target, export, "export all package recipes", "[conan-cache]")
+$(call make_vendor_target, export-vendor, "export all vendor packages", "[conan-cache]")
+$(call make_vendor_target, package-vendor, "create all vendor packages", "[conan-cache]")
+$(call make_vendor_target, download-vendor, "download or build vendor packages", "[conan-cache]")
 help::
-	$(call print_help_target, package, "create ${META_PKG} package and plugins", "[conan-cache]")
 	echo
+endif
 
 $(call make_select_target, export-select, "export selected packages", "[conan-cache]")
 $(call make_select_target, package-select, "create selected packages with policy", "[conan-cache]")
@@ -179,34 +168,19 @@ $(call make_select_target, clean-select, "remove build artifacts", "[in-source]"
 help::
 	echo
 	$(call print_help_subsection, "Options")
-#	$(call print_help_option, WITH_VENDOR, "", "include optional vendor packages from ${_grn}UNSELECT_VENDOR${_rst}")
+ifneq "${ALL_VENDOR}" ""
+	$(call print_help_option, WITH_VENDOR, "", "include optional vendor packages from ${_grn}UNSELECT_VENDOR${_rst}")
+endif
 	$(call print_help_option, WITH_PKGS, "", "include optional packages from ${_grn}UNSELECT_PKGS${_rst}")
 	$(call print_help_option, LOCKFILE_SOURCE, "", "use specified conanfile as lockfile source for build")
 	echo
 	$(call print_help_subsection, "Defines")
 	$(call print_help_option, BUILD_POLICY, ${BUILD_POLICY})
 	$(call print_help_define, CONAN_OPTIONS, ${CONAN_OPTIONS})
-#	$(call print_help_define_lines, UNSELECT_VENDOR, ${UNSELECT_VENDOR})
-#	$(call print_help_define_lines, SELECT_VENDOR, ${SELECT_VENDOR})
+ifneq "${ALL_VENDOR}" ""
+	$(call print_help_define_lines, UNSELECT_VENDOR, ${UNSELECT_VENDOR})
+	$(call print_help_define_lines, SELECT_VENDOR, ${SELECT_VENDOR})
+endif
 	$(call print_help_define_lines, UNSELECT_PKGS, ${UNSELECT_PKGS})
 	$(call print_help_define_lines, SELECT_PKGS, ${SELECT_PKGS})
 	echo
-
-.PHONY: package
-package: export-select
-	# Build cloe with all targets and options together.
-	#
-	#   This is different from the package target in that it always builds the
-	#   packages from this workspace, the ones in SELECT_PKGS.
-	#   This is different from the package-select target in that it builds them
-	#   all together and thereby uses the correct dependency resolution with
-	#   overrides and options.
-	TARGETS=$$( \
-		for pkg in ${SELECT_PKGS}; do \
-			if [ ! -d $${pkg} ]; then \
-				continue; \
-			fi; \
-			echo -n "--build=$$(make --no-print-directory -C $${pkg} info-name) "; \
-		done; \
-	) && \
-	${MAKE} -f Makefile.package CONAN_OPTIONS="${CONAN_OPTIONS} $$TARGETS" package
diff --git a/README.md b/README.md
@@ -84,14 +84,15 @@ See the Conan [documentation][6] for more information on how to do this.
 
 To build all packages, you should run the following:
 
-    make package
+    make export-all
+    make -C meta package
 
 This will export all Conan recipes from this repository and create the cloe
 package. Conan will download and build all necessary dependencies. Should
 any errors occur during the build, you may have to force Conan to build
 all packages instead of re-using packages it finds:
 ```
-    make package CONAN_OPTIONS="--build"
+    make -C meta package CONAN_OPTIONS="--build"
 ```
 Run `make help` to get an overview of the available targets we expect you to
 use. For more details on how this is done, have a look at the Makefiles in the
@@ -166,7 +167,7 @@ Note that the above examples show the verbose output of the `cloe-launch` tool.
 Integration and system tests can be run to ensure that all the packages built
 are working together as expected:
 
-    make smoketest-deps
+    make export-all smoketest-deps
     make smoketest
 
 This will build packages in the required configurations as defined by the