Config restructure: distribute module configs via MQTT (#201)

Changes to reduce configuration parsing effort by parsing it once by the manager and then distribute it to the modules via MQTT.

This is achieved by first parsing the MQTT settings from the config, passing these to the modules whilst spawning them. The modules themselves then ask for their module config via MQTT, which is in turn provided to them from the manager.

schemas, manifests, types, interfaces, error definitions are published as retained topics that modules can access when needed.

* Enable code coverage
* Update clang-format lint job to use v1.3.1 of run-clang-format
* Add additional Config tests
* Add user-config test
* Add test for module that has an interface using types that implements a cmd and a var
* Add simple config serialization test
* Add test for config file in legacy json format
* Add test for config file that does not exist
* Exit early if parse_command_line returned false
This is for example the case in the --help and --version parameters
* Log an error if a wrong module is started for a given module id
This can happen if the wrong binary is manually selected for a standalone module
* Update clang-tidy config
* Add RuntimeSettings as member to ManagerSettings to avoid code duplication
* Move filesystem helper functions into their own header
* Move BootException to the types header
* Use a shared MQTTAbstraction for config getting and EVerest afterwards
* Add misc-const-correctness check to clang-tidy config
Reorder checks so enabled checks come first, followed by checks that are deactivated
* Use std::size_t to avoid narrowing conversions
* Use int64_t instead of int in return value to avoid narrowing conversion
* Use constexpr instead of define for MQTT_BUF_SIZE
* Fix config not being checked in check_config command
* Move version_information.txt filename to constant
* Move MQTT get timeout to constant
* Use std::size_t everywhere
* re-order member ConfigBase, ManagerConfig and Config implementations
this now better reflects the order from the config header file that got shuffled around during breaking up into different classes
* Add ImplementationInfo struct replacing json
* Add build* to .gitignore
* Make extract_implementation_info into a free function
Add create_printable_identifier free function and use it in the member function
* Move MQTTSettings into its own files
* Use factory method create_mqtt_client to initialize MQTTAbstractionImpl based on MQTTSettings
* Remove lcov coverage, add gcovr xml coverage report
Move this configuration to tests/CMakeLists.txt
* Reduce usage of using for nlohmann::json in headers
* Re-introduce functionality to RuntimeSession ctor with config file param
This prevents everest-testing from breaking until it is refactored to use the new ctor + environment variables
* Updated catch2 to latest stable release 3.7.1
* Move testing logic to tests subdir
Set coverage flags directly on the appropriate targets
To make this work from the tests directory set CMake policy CMP0079
* Re-use the config that's already loaded in the ManagerSettings
Don't load the same yaml file twice
* Bump version to 0.19.0
* Add documentation with sequence and class diagrams about MQTT config feature

Signed-off-by: Kai-Uwe Hermann <[email protected]>

* refactor(tests): dry

Signed-off-by: aw <[email protected]>

---------

Signed-off-by: Kai-Uwe Hermann <[email protected]>
Signed-off-by: aw <[email protected]>
Co-authored-by: aw <[email protected]>

.clang-tidy

@@ -1,11 +1,20 @@
 Checks: >
-    *,
+    bugprone*,
+    misc-const-correctness,
     -llvmlibc*,
-    -fuchsia-default-arguments-calls,
     -fuchsia-overloaded-operator,
     -fuchsia-statically-constructed-objects,
     -readability-function-cognitive-complexity,
     -modernize-use-trailing-return-type,
     -abseil-string-find-startswith,
-    -abseil-string-find-str-contains
-HeaderFilterRegex:  ".*"
+    -abseil-string-find-str-contains,
+    -readability-identifier-length,
+    -fuchsia-default-arguments-calls,
+    -fuchsia-default-arguments-declarations,
+    -altera-struct-pack-align,
+    -performance-enum-size,
+    -altera*,
+    -misc-non-private-member-variables-in-classes,
+    -cppcoreguidelines-non-private-member-variables-in-classes,
+    -bugprone-easily-swappable-parameters
+HeaderFilterRegex:  ".*"

.github/workflows/lint.yaml

@@ -14,7 +14,7 @@ jobs:
         with:
           path: source
       - name: Run clang-format
-        uses: everest/everest-ci/github-actions/run-clang-format@v1.1.0
+        uses: everest/everest-ci/github-actions/run-clang-format@v1.3.1
         with:
           source-dir: source
           extensions: hpp,cpp

.gitignore

@@ -1,5 +1,6 @@
 *build
 *build-cross
+build*
 !everestrs-build
 target
 bazel-bin

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.14)
 
 project(everest-framework
-    VERSION 0.18.1
+    VERSION 0.19.0
     DESCRIPTION "The open operating system for e-mobility charging stations"
     LANGUAGES CXX C
 )
@@ -28,6 +28,14 @@ ev_setup_python_executable(
     PYTHON_VENV_PATH ${${PROJECT_NAME}_PYTHON_VENV_PATH}
 )
 
+if((${CMAKE_PROJECT_NAME} STREQUAL ${PROJECT_NAME} OR ${PROJECT_NAME}_BUILD_TESTING) AND BUILD_TESTING)
+    set(EVEREST_FRAMEWORK_BUILD_TESTING ON)
+    # this policy allows us to link gcov to targets defined in other directories
+    if(POLICY CMP0079)
+        set(CMAKE_POLICY_DEFAULT_CMP0079 NEW)
+    endif()
+endif()
+
 # make own cmake modules available
 list(INSERT CMAKE_MODULE_PATH 0 "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
@@ -165,16 +173,9 @@ if (FRAMEWORK_INSTALL)
 endif ()
 
 # testing
-# FIXME (aw): move testing logic into tests/CMakeLists.txt
-if((${CMAKE_PROJECT_NAME} STREQUAL ${PROJECT_NAME} OR ${PROJECT_NAME}_BUILD_TESTING) AND BUILD_TESTING)
-    if(NOT DISABLE_EDM)
-        list(APPEND CMAKE_MODULE_PATH ${CPM_PACKAGE_catch2_SOURCE_DIR}/extras)
-    endif()
+if(EVEREST_FRAMEWORK_BUILD_TESTING)
     include(CTest)
-    set(CMAKE_BUILD_TYPE Debug CACHE STRING "Build type" FORCE)
-
     add_subdirectory(tests)
-
 else()
     message(STATUS "Not running unit tests")
 endif()

README.md

@@ -2,4 +2,4 @@
 
 This subproject of EVerest is providing a mechanism to manage dependencies between different modules communicating with an wrapped MQTT protocol. On startup it parses a set of configuration file, checks them agains the manifests of different modules and launches each module needed.
 
-All documentation and the issue tracking can be found in our main repository here: https://github.com/EVerest/everest
+Additional documentation can be found in [docs](docs).

dependencies.yaml

@@ -68,8 +68,8 @@ date:
     ]
 catch2:
   git: https://github.com/catchorg/Catch2.git
-  git_tag: v3.4.0
-  cmake_condition: "BUILD_TESTING"
+  git_tag: v3.7.1
+  cmake_condition: "EVEREST_FRAMEWORK_BUILD_TESTING"
 pybind11:
   git: https://github.com/pybind/pybind11.git
   git_tag: v2.11.1

docs/MQTTConfigDistribution.md

@@ -0,0 +1,120 @@
+# Module configuration distributed via MQTT
+
+Since everest-framework 0.19.0 the module configuration is parsed once
+by the manager and then distributed to the modules via MQTT.
+This is achieved by parsing the MQTT settings from the config,
+spawning the modules and passing these MQTT settings to them.
+The modules themselves then ask for their module config via MQTT,
+which is in turn provided to them from the manager.
+After the modules have received their config, their init() function is called.
+Afterwards they signal ready to the manager.
+The manager sends out the global ready signal
+once it has received all Module ready signals.
+
+The following sequence diagram illustrates this startup process
+
+```mermaid
+sequenceDiagram
+    create participant manager
+    create participant ManagerSettings
+    manager-)ManagerSettings: ManagerSettings(prefix, config_path)
+    ManagerSettings-->>manager: return ms
+    create participant ManagerConfig
+    manager-)ManagerConfig: ManagerConfig(ms)
+    create participant MQTTAbstraction
+    manager-)MQTTAbstraction: MQTTAbstraction(ms.mqtt_settings)
+    MQTTAbstraction-->>manager: return mqtt_abstraction
+    activate manager
+    manager->>manager: start_modules()
+    manager->>ManagerConfig: serialize()
+    ManagerConfig-->>manager: serialized_config
+    manager->>MQTTAbstraction: publish(interfaces, types, schemas, manifests, settings, retain=true)
+    loop For every module
+        manager->>manager: spawn_modules(Module)
+        create participant Module
+        manager->>Module: spawn Module
+        Module->>MQTTAbstraction: get(Config)
+        MQTTAbstraction->>manager: get(Config of Module)
+        manager-->>MQTTAbstraction: publish(module configs, mappings)
+        MQTTAbstraction-->>Module: publish(module configs, mappings)
+        Module->>Module: init
+        Module->>MQTTAbstraction: publish(ready)
+        MQTTAbstraction->>manager: publish(ready of Module)
+    end
+    manager->>MQTTAbstraction: publish global ready
+```
+
+Class diagram
+
+```mermaid
+classDiagram
+ConfigBase <|-- ManagerConfig
+ConfigBase <|-- Config
+MQTTSettings *-- ConfigBase
+ManagerSettings *-- ManagerConfig
+
+note for ConfigBase "
+Baseclass containing json config, manifests, interfaces,
+types and functions to access this information which
+needs to be available in all derived classes
+"
+
+class ManagerSettings{
+    +fs::path configs_dir
+    +fs::path schemas_dir
+    +fs::path interfaces_dir
+    +fs::path types_dir
+    +fs::path errors_dir
+    +fs::path config_file
+    +fs::path www_dir
+    +int controller_port
+    +int controller_rpc_timeout_ms
+    +std::string run_as_user
+    +std::string version_information
+    +nlohmann::json config
+    +MQTTSettings mqtt_settings
+    +std::unique_ptr<RuntimeSettings> runtime_settings
+    +ManagerSettings(const std::string& prefix, const std::string& config)
+    +const RuntimeSettings& get_runtime_settings()
+}
+
+class MQTTSettings{
+    +std::string broker_socket_path
+    +std::string broker_host
+    +int broker_port
+    +std::string everest_prefix
+    +std::string external_prefix
+    +bool uses_socket()
+}
+
+class ConfigBase{
+    #const MQTTSettings mqtt_settings
+    +ConfigBase(const MQTTSettings& mqtt_settings)
+}
+
+class ManagerConfig{
+    -const ManagerSettings& ms
+    +ManagerConfig(const ManagerSettings& ms)
+    +nlohmann::json serialize()
+    -load_and_validate_manifest(const std::string& module_id, const nlohmann::json& module_config)
+    -std::tuple~nlohmann::json, int64_t~ load_and_validate_with_schema(const fs::path& file_path, const nlohmann::json& schema)
+    -nlohmann::json resolve_interface(const std::string& intf_name)
+    -nlohmann::json load_interface_file(const std::string& intf_name)
+    -resolve_all_requirements()
+    -parse(nlohmann::json config)
+}
+
+class Config{
+    +Config(const MQTTSettings& mqtt_settings, nlohmann::json config)
+    +bool module_provides(const std::string& module_name, const std::string& impl_id);
+    +nlohmann::json get_module_cmds(const std::string& module_name, const std::string& impl_id)
+    +nlohmann::json resolve_requirement(const std::string& module_id, const std::string& requirement_id)
+    +std::list~Requirement~ get_requirements(const std::string& module_id)
+    +RequirementInitialization get_requirement_initialization(const std::string& module_id)
+    +ModuleConfigs get_module_configs(const std::string& module_id)
+    +nlohmann::json get_module_json_config(const std::string& module_id)
+    +ModuleInfo get_module_info(const std::string& module_id)
+    +std::optional~<~TelemetryConfig~ get_telemetry_config()
+    +nlohmann::json get_interface_definition(const std::string& interface_name) const;
+}
+```

docs/README.md

@@ -0,0 +1,3 @@
+# Additional documentation about the inner workings of EVerest Framework
+
+[MQTT Config distribution](MQTTConfigDistribution.md)

everestjs/everestjs.cpp

@@ -11,6 +11,7 @@
 #include <sys/prctl.h>
 
 #include <chrono>
+#include <cstddef>
 #include <future>
 #include <iostream>
 #include <map>
@@ -26,6 +27,9 @@
 #include <utils/error/error_manager_impl.hpp>
 #include <utils/error/error_manager_req.hpp>
 #include <utils/error/error_state_monitor.hpp>
+#include <utils/filesystem.hpp>
+#include <utils/module_config.hpp>
+#include <utils/mqtt_settings.hpp>
 
 namespace EverestJs {
 
@@ -564,15 +568,45 @@ static Napi::Value boot_module(const Napi::CallbackInfo& info) {
 
         const auto& module_id = settings.Get("module").ToString().Utf8Value();
         const auto& prefix = settings.Get("prefix").ToString().Utf8Value();
-        const auto& config_file = settings.Get("config_file").ToString().Utf8Value();
+        const auto& mqtt_everest_prefix = settings.Get("mqtt_everest_prefix").ToString().Utf8Value();
+        const auto& mqtt_external_prefix = settings.Get("mqtt_external_prefix").ToString().Utf8Value();
+        const auto& mqtt_broker_socket_path = settings.Get("mqtt_broker_socket_path").ToString().Utf8Value();
+        const auto& mqtt_server_address = settings.Get("mqtt_server_address").ToString().Utf8Value();
+        const auto& mqtt_server_port = settings.Get("mqtt_server_port").ToString().Utf8Value();
         const bool validate_schema = settings.Get("validate_schema").ToBoolean().Value();
 
-        auto rs = std::make_shared<Everest::RuntimeSettings>(prefix, config_file);
-
+        namespace fs = std::filesystem;
+        fs::path logging_config_file =
+            Everest::assert_file(settings.Get("logging_config_file").ToString().Utf8Value(), "Default logging config");
         // initialize logging as early as possible
-        Everest::Logging::init(rs->logging_config_file, module_id);
+        Everest::Logging::init(logging_config_file.string(), module_id);
+        std::shared_ptr<Everest::MQTTAbstraction> mqtt;
+        Everest::MQTTSettings mqtt_settings{};
+        if (mqtt_broker_socket_path.empty()) {
+            auto mqtt_broker_port = Everest::defaults::MQTT_BROKER_PORT;
+            try {
+                mqtt_broker_port = std::stoi(mqtt_server_port);
+            } catch (...) {
+                EVLOG_warning << "Could not parse MQTT broker port, using default: " << mqtt_broker_port;
+            }
+
+            Everest::populate_mqtt_settings(mqtt_settings, mqtt_server_address, mqtt_broker_port, mqtt_everest_prefix,
+                                            mqtt_external_prefix);
+        } else {
+            Everest::populate_mqtt_settings(mqtt_settings, mqtt_broker_socket_path, mqtt_everest_prefix,
+                                            mqtt_external_prefix);
+        }
+
+        mqtt = std::make_shared<Everest::MQTTAbstraction>(mqtt_settings);
+        mqtt->connect();
+        mqtt->spawn_main_loop_thread();
+
+        const auto result = Everest::get_module_config(mqtt, module_id);
+
+        auto rs = std::make_unique<Everest::RuntimeSettings>(result.at("settings"));
+
+        auto config = std::make_unique<Everest::Config>(mqtt_settings, result);
 
-        auto config = std::make_unique<Everest::Config>(rs);
         if (!config->contains(module_id)) {
             EVTHROW(EVEXCEPTION(Everest::EverestConfigError,
                                 "Module with identifier '" << module_id << "' not found in config!"));
@@ -600,9 +634,11 @@ static Napi::Value boot_module(const Napi::CallbackInfo& info) {
         // provides property: iterate over every implementation that this modules provides
         auto provided_impls_prop = Napi::Object::New(env);
         auto provided_cmds_prop = Napi::Object::New(env);
+        const auto& interface_definitions = config->get_interface_definitions();
         for (const auto& impl_definition : module_impls.items()) {
             const auto& impl_id = impl_definition.key();
-            const auto& impl_intf = module_impls[impl_id];
+            const auto& interface_name = module_impls.at(impl_id).get<std::string>();
+            const auto& impl_intf = interface_definitions.at(interface_name);
 
             auto impl_prop = Napi::Object::New(env);
 
@@ -710,7 +746,7 @@ static Napi::Value boot_module(const Napi::CallbackInfo& info) {
             }
             auto req_array_prop = Napi::Array::New(env);
             auto req_mod_cmds_array = Napi::Array::New(env);
-            for (size_t i = 0; i < req_route_list.size(); i++) {
+            for (std::size_t i = 0; i < req_route_list.size(); i++) {
                 auto req_route = req_route_list[i];
                 const std::string& requirement_module_id = req_route["module_id"];
                 const std::string& requirement_impl_id = req_route["implementation_id"];
@@ -844,7 +880,7 @@ static Napi::Value boot_module(const Napi::CallbackInfo& info) {
         json module_config = config->get_module_json_config(module_id);
 
         auto module_info = config->get_module_info(module_id);
-        populate_module_info_path_from_runtime_settings(module_info, rs);
+        populate_module_info_path_from_runtime_settings(module_info, *rs);
 
         auto module_config_prop = Napi::Object::New(env);
         auto module_config_impl_prop = Napi::Object::New(env);
@@ -897,10 +933,8 @@ static Napi::Value boot_module(const Napi::CallbackInfo& info) {
         module_this.DefineProperty(Napi::PropertyDescriptor::Value("info", module_info_prop, napi_enumerable));
 
         // connect to mqtt server and start mqtt mainloop thread
-        auto everest_handle =
-            std::make_unique<Everest::Everest>(module_id, *config, validate_schema, rs->mqtt_broker_socket_path,
-                                               rs->mqtt_broker_host, rs->mqtt_broker_port, rs->mqtt_everest_prefix,
-                                               rs->mqtt_external_prefix, rs->telemetry_prefix, rs->telemetry_enabled);
+        auto everest_handle = std::make_unique<Everest::Everest>(module_id, *config, validate_schema, mqtt,
+                                                                 rs->telemetry_prefix, rs->telemetry_enabled);
 
         ctx = new EvModCtx(std::move(everest_handle), module_manifest, env);
 

everestjs/index.js

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0
-// Copyright 2020 - 2023 Pionix GmbH and Contributors to EVerest
+// Copyright Pionix GmbH and Contributors to EVerest
 const util = require('util');
 const addon = require('./everestjs.node');
 
@@ -28,9 +28,12 @@ const EverestModule = function EverestModule(handler_setup, user_settings) {
   const env_settings = {
     module: process.env.EV_MODULE,
     prefix: process.env.EV_PREFIX,
-    config_file: process.env.EV_CONF_FILE,
-    mqtt_server_address: process.env.MQTT_SERVER_ADDRESS,
-    mqtt_server_port: process.env.MQTT_SERVER_PORT,
+    logging_config_file: process.env.EV_LOG_CONF_FILE,
+    mqtt_everest_prefix: process.env.EV_MQTT_EVEREST_PREFIX,
+    mqtt_external_prefix: process.env.EV_MQTT_EXTERNAL_PREFIX,
+    mqtt_broker_socket_path: process.env.EV_MQTT_BROKER_SOCKET_PATH,
+    mqtt_server_address: process.env.EV_MQTT_BROKER_HOST,
+    mqtt_server_port: process.env.EV_MQTT_BROKER_PORT,
     validate_schema: process.env.EV_VALIDATE_SCHEMA,
   };
 
@@ -43,7 +46,12 @@ const EverestModule = function EverestModule(handler_setup, user_settings) {
   const config = {
     module: settings.module,
     prefix: settings.prefix,
-    config_file: settings.config_file,
+    logging_config_file: settings.logging_config_file,
+    mqtt_everest_prefix: settings.mqtt_everest_prefix,
+    mqtt_external_prefix: helpers.get_default(settings, 'mqtt_external_prefix', ''),
+    mqtt_broker_socket_path: helpers.get_default(settings, 'mqtt_broker_socket_path', ''),
+    mqtt_server_address: helpers.get_default(settings, 'mqtt_server_address', ''),
+    mqtt_server_port: helpers.get_default(settings, 'mqtt_server_port', 0),
     validate_schema: helpers.get_default(settings, 'validate_schema', false),
   };