Add first draft of a telemetry mechanism (#73)

This allows a module to use telemetry.publish() to publish a Everest::TelemetryMap This TelemetryMap takes std::string as keys and boost::variant with std::string, bool, int and double values telemetry.publish() takes 3 or 4 parameters: category: a string to categorize the telemetry, for example livedata, session, etc. subcategory: optional, a subcategory of the telemetry, for example "events". If this parameter is missing "type" will be assumed instead type: the type of telemetry, for example power_meter if it is powermeter related telemetry: a Everest::TelemetryMap All C++ modules that want to use telemetry must set the "enable_telemetry" option in their manifest to true and then be updated using ev-cli >= 0.0.18 Add enable_telemetry option to manifest Move telemetry publishing logic from ModuleAdapter to Everest Add telemetry to everestjs Add information if telemtry is enabled to ModuleInfo Extend TelemetryEntry variant to cover more integer types Signed-off-by: Kai-Uwe Hermann <[email protected]>
EVerest · Feb 23, 2023 · 7afb87f · 7afb87f
1 parent cd1a72d
commit 7afb87f
Show file tree

Hide file tree

Showing 18 changed files with 270 additions and 22 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.14)
 
 project(everest-framework
-    VERSION 0.3
+    VERSION 0.4
     DESCRIPTION "The open operating system for e-mobility charging stations"
 	LANGUAGES CXX C
 )

diff --git a/everestjs/conversions.cpp b/everestjs/conversions.cpp
@@ -52,6 +52,33 @@ Everest::json convertToJson(const Napi::Value& value) {
                         napi_valuetype_strings[value.Type()]));
 }
 
+Everest::TelemetryMap convertToTelemetryMap(const Napi::Object& obj) {
+    BOOST_LOG_FUNCTION();
+    Everest::TelemetryMap telemetry;
+    Napi::Array keys = obj.GetPropertyNames();
+    for (uint64_t i = 0; i < keys.Length(); i++) {
+        Napi::Value key = keys[i];
+        if (key.IsString()) {
+            std::string k = key.As<Napi::String>();
+            Napi::Value value = Napi::Value(obj[k]);
+            if (value.IsString()) {
+                telemetry[k] = std::string(value.As<Napi::String>());
+            } else if (value.IsNumber()) {
+                int intNumber = value.As<Napi::Number>();
+                double floatNumber = value.As<Napi::Number>();
+                if (floatNumber == intNumber) {
+                    telemetry[k] = intNumber;
+                } else {
+                    telemetry[k] = floatNumber;
+                }
+            } else if (value.IsBoolean()) {
+                telemetry[k] = bool(value.As<Napi::Boolean>());
+            }
+        }
+    }
+    return telemetry;
+}
+
 Napi::Value convertToNapiValue(const Napi::Env& env, const json& value) {
     BOOST_LOG_FUNCTION();
 

diff --git a/everestjs/conversions.hpp b/everestjs/conversions.hpp
@@ -26,6 +26,7 @@ static const char* const napi_valuetype_strings[] = {
 };
 
 Everest::json convertToJson(const Napi::Value& value);
+Everest::TelemetryMap convertToTelemetryMap(const Napi::Object& obj);
 Napi::Value convertToNapiValue(const Napi::Env& env, const Everest::json& value);
 
 } // namespace EverestJs

diff --git a/everestjs/everestjs.cpp b/everestjs/everestjs.cpp
@@ -234,6 +234,33 @@ static Napi::Value mqtt_subscribe(const Napi::CallbackInfo& info) {
     return env.Undefined();
 }
 
+static Napi::Value telemetry_publish(const Napi::CallbackInfo& info) {
+    BOOST_LOG_FUNCTION();
+
+    const auto& env = info.Env();
+    try {
+        auto length = info.Length();
+        if (length == 3 || length == 4) {
+            const auto& category = info[0].ToString().Utf8Value();
+            const auto& subcategory = info[1].ToString().Utf8Value();
+            std::string type = subcategory;
+            if (length == 3) {
+                // assume it's category, subcategory, telemetry
+                auto telemetry = convertToTelemetryMap(info[2].As<Napi::Object>());
+                ctx->everest.telemetry_publish(category, subcategory, subcategory, telemetry);
+            } else if (length == 4) {
+                // assume it's category, subcategory, type, telemetry
+                auto telemetry = convertToTelemetryMap(info[3].As<Napi::Object>());
+                ctx->everest.telemetry_publish(category, subcategory, type, telemetry);
+            }
+        }
+    } catch (std::exception& e) {
+        EVLOG_AND_RETHROW(env);
+    }
+
+    return env.Undefined();
+}
+
 static Napi::Value call_cmd(const Requirement& req, const std::string& cmd_name, const Napi::CallbackInfo& info) {
     BOOST_LOG_FUNCTION();
 
@@ -299,10 +326,11 @@ static Napi::Value boot_module(const Napi::CallbackInfo& info) {
         Everest::Logging::update_process_name(module_identifier);
 
         // connect to mqtt server and start mqtt mainloop thread
-        ctx =
-            new EvModCtx(Everest::Everest::get_instance(module_id, *config, validate_schema, rs.mqtt_broker_host,
-                                                        rs.mqtt_broker_port, rs.mqtt_everest_prefix, rs.mqtt_external_prefix),
-                         module_manifest, env);
+        ctx = new EvModCtx(Everest::Everest::get_instance(module_id, *config, validate_schema, rs.mqtt_broker_host,
+                                                          rs.mqtt_broker_port, rs.mqtt_everest_prefix,
+                                                          rs.mqtt_external_prefix, rs.telemetry_prefix,
+                                                          rs.telemetry_enabled),
+                           module_manifest, env);
         ctx->everest.connect();
 
         ctx->js_module_ref = Napi::Persistent(module_this);
@@ -483,6 +511,15 @@ static Napi::Value boot_module(const Napi::CallbackInfo& info) {
             module_this.DefineProperty(Napi::PropertyDescriptor::Value("mqtt", mqtt_prop, napi_enumerable));
         }
 
+        // telemetry property
+        if (module_manifest.contains("enable_telemetry") && module_manifest["enable_telemetry"] == true) {
+            auto telemetry_prop = Napi::Object::New(env);
+            telemetry_prop.DefineProperty(Napi::PropertyDescriptor::Value(
+                "publish", Napi::Function::New(env, telemetry_publish), napi_enumerable));
+
+            module_this.DefineProperty(Napi::PropertyDescriptor::Value("telemetry", telemetry_prop, napi_enumerable));
+        }
+
         // config property
         json module_config = config->get_module_json_config(module_id);
         auto module_config_prop = Napi::Object::New(env);
@@ -519,6 +556,10 @@ static Napi::Value boot_module(const Napi::CallbackInfo& info) {
         module_info_prop.DefineProperty(Napi::PropertyDescriptor::Value(
             "everest_prefix", Napi::String::New(env, prefix), napi_enumerable));
 
+        // set telemetry_enabled
+        module_info_prop.DefineProperty(Napi::PropertyDescriptor::Value(
+            "telemetry_enabled", Napi::Boolean::New(env, ctx->everest.is_telemetry_enabled()), napi_enumerable));
+
         module_this.DefineProperty(Napi::PropertyDescriptor::Value("info", module_info_prop, napi_enumerable));
 
         ctx->everest.register_on_ready_handler(framework_ready_handler);

diff --git a/everestjs/index.js b/everestjs/index.js
@@ -58,6 +58,7 @@ const EverestModule = function EverestModule(handler_setup, user_settings) {
     info: this.info,
     config: this.config,
     mqtt: this.mqtt,
+    telemetry: this.telemetry,
   };
 
   if (this.mqtt === undefined) {
@@ -68,6 +69,14 @@ const EverestModule = function EverestModule(handler_setup, user_settings) {
     Object.defineProperty(this, 'mqtt', missing_mqtt_getter);
   }
 
+  if (this.telemetry === undefined) {
+    const missing_telemetry_getter = {
+      get() { throw new Error('Telemetry not available - missing enable_telemetry in manifest?'); },
+    };
+    Object.defineProperty(module_setup, 'telemetry', missing_telemetry_getter);
+    Object.defineProperty(this, 'telemetry', missing_telemetry_getter);
+  }
+
   // check, if we need to register cmds
   if (typeof handler_setup === 'undefined') {
     if (Object.keys(available_handlers.provides).length !== 0) {

diff --git a/everestjs/package.json b/everestjs/package.json
@@ -1,7 +1,7 @@
 {
   "name": "everestjs",
   "main": "index.js",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "EVerest API for node.js",
   "dependencies": {
     "node-addon-api": "^3.2.1"

diff --git a/everestpy/everestpy.cpp b/everestpy/everestpy.cpp
@@ -135,7 +135,8 @@ int initialize(const std::string& prefix, const std::string& config_file, const
 
         Everest::Everest& everest =
             Everest::Everest::get_instance(module_id, config, rs.validate_schema, rs.mqtt_broker_host,
-                                           rs.mqtt_broker_port, rs.mqtt_everest_prefix, rs.mqtt_external_prefix);
+                                           rs.mqtt_broker_port, rs.mqtt_everest_prefix, rs.mqtt_external_prefix,
+                                           rs.telemetry_prefix, rs.telemetry_enabled);
 
         EVLOG_info << fmt::format("Initializing module {}...", module_identifier);
 
@@ -265,6 +266,7 @@ int initialize(const std::string& prefix, const std::string& config_file, const
         auto module_configs = config.get_module_configs(module_id);
         auto module_info = config.get_module_info(module_id);
         module_info.everest_prefix = rs.prefix.string();
+        module_info.telemetry_enabled = everest.is_telemetry_enabled();
 
         everest_py.module_callbacks.init(module_configs, module_info);
 
@@ -326,7 +328,8 @@ PYBIND11_MODULE(everestpy, m) {
         .def_readonly("authors", &ModuleInfo::authors)
         .def_readonly("license", &ModuleInfo::license)
         .def_readonly("id", &ModuleInfo::id)
-        .def_readonly("everest_prefix", &ModuleInfo::everest_prefix);
+        .def_readonly("everest_prefix", &ModuleInfo::everest_prefix)
+        .def_readonly("telemetry_enabled", &ModuleInfo::telemetry_enabled);
     py::class_<CmdWithArguments>(m, "CmdWithArguments")
         .def(py::init<>())
         .def_readwrite("cmd", &CmdWithArguments::cmd)
@@ -354,5 +357,5 @@ PYBIND11_MODULE(everestpy, m) {
     m.def("register_pre_init_callback", &register_pre_init_callback);
     m.def("register_ready_callback", &register_ready_callback);
 
-    m.attr("__version__") = "0.3";
+    m.attr("__version__") = "0.4";
 }
diff --git a/include/framework/ModuleAdapter.hpp b/include/framework/ModuleAdapter.hpp
@@ -4,9 +4,11 @@
 #define MODULE_ADAPTER_HPP
 
 #include "everest.hpp"
-#include <iostream>
+#include <everest/logging.hpp>
+#include <utils/date.hpp>
 
 #include <iomanip>
+#include <iostream>
 
 namespace Everest {
 
@@ -95,13 +97,16 @@ struct ModuleAdapter {
     using SubscribeFunc = std::function<void(const Requirement&, const std::string&, ValueCallback)>;
     using ExtMqttPublishFunc = std::function<void(const std::string&, const std::string&)>;
     using ExtMqttSubscribeFunc = std::function<void(const std::string&, StringHandler)>;
+    using TelemetryPublishFunc =
+        std::function<void(const std::string&, const std::string&, const std::string&, const TelemetryMap&)>;
 
     CallFunc call;
     PublishFunc publish;
     SubscribeFunc subscribe;
     ExtMqttPublishFunc ext_mqtt_publish;
     ExtMqttSubscribeFunc ext_mqtt_subscribe;
     std::vector<cmd> registered_commands;
+    TelemetryPublishFunc telemetry_publish;
 
     void check_complete() {
         // FIXME (aw): I should throw if some of my handlers are not set
@@ -172,6 +177,23 @@ class MqttProvider {
     ModuleAdapter& ev;
 };
 
+class TelemetryProvider {
+public:
+    TelemetryProvider(ModuleAdapter& ev) : ev(ev){};
+
+    void publish(const std::string& category, const std::string& subcategory, const std::string& type,
+                 const TelemetryMap& telemetry) {
+        ev.telemetry_publish(category, subcategory, type, telemetry);
+    }
+
+    void publish(const std::string& category, const std::string& subcategory, const TelemetryMap& telemetry) {
+        publish(category, subcategory, subcategory, telemetry);
+    }
+
+private:
+    ModuleAdapter& ev;
+};
+
 } // namespace Everest
 
 #endif // MODULE_ADAPTER_HPP
diff --git a/include/framework/everest.hpp b/include/framework/everest.hpp
@@ -27,6 +27,10 @@ struct cmd {
     ReturnType return_type; ///< The return type
 };
 
+using TelemetryEntry =
+    boost::variant<std::string, const char*, bool, int, int32_t, uint32_t, int64_t, uint64_t, double>;
+using TelemetryMap = std::map<std::string, TelemetryEntry>;
+
 ///
 /// \brief Contains the EVerest framework that provides convenience functionality for implementing EVerest modules
 ///
@@ -48,10 +52,13 @@ class Everest {
     json module_classes;
     std::string mqtt_everest_prefix;
     std::string mqtt_external_prefix;
+    std::string telemetry_prefix;
+    boost::optional<TelemetryConfig> telemetry_config;
+    bool telemetry_enabled;
 
     Everest(std::string module_id, Config config, bool validate_data_with_schema,
             const std::string& mqtt_server_address, int mqtt_server_port, const std::string& mqtt_everest_prefix,
-            const std::string& mqtt_external_prefix);
+            const std::string& mqtt_external_prefix, const std::string& telemetry_prefix, bool telemetry_enabled);
 
     void handle_ready(json data);
 
@@ -103,6 +110,21 @@ class Everest {
     ///
     void provide_external_mqtt_handler(const std::string& topic, const StringHandler& handler);
 
+    ///
+    /// \brief publishes the given telemetry \p data on the given \p topic
+    ///
+    void telemetry_publish(const std::string& topic, const std::string& data);
+
+    ///
+    /// \brief publishes the given telemetry \p telemetry on a topic constructed from \p category \p subcategory and \p
+    /// type
+    ///
+    void telemetry_publish(const std::string& category, const std::string& subcategory, const std::string& type,
+                           const TelemetryMap& telemetry);
+
+    /// \returns true if telemetry is enabled
+    bool is_telemetry_enabled();
+
     ///
     /// \brief Chccks if all commands of a module that are listed in its manifest are available
     ///
@@ -144,9 +166,10 @@ class Everest {
     /// with the known json schemas is needed this can be activated by setting \p validate_data_with_schema to true
     static Everest& get_instance(std::string module_id, Config config, bool validate_data_with_schema,
                                  const std::string& mqtt_server_address, int mqtt_server_port,
-                                 const std::string& mqtt_everest_prefix, const std::string& mqtt_external_prefix) {
+                                 const std::string& mqtt_everest_prefix, const std::string& mqtt_external_prefix,
+                                 const std::string& telemetry_prefix, bool telemetry_enabled) {
         static Everest instance(module_id, config, validate_data_with_schema, mqtt_server_address, mqtt_server_port,
-                                mqtt_everest_prefix, mqtt_external_prefix);
+                                mqtt_everest_prefix, mqtt_external_prefix, telemetry_prefix, telemetry_enabled);
 
         return instance;
     }

diff --git a/include/framework/runtime.hpp b/include/framework/runtime.hpp
@@ -76,6 +76,8 @@ inline constexpr auto MQTT_BROKER_HOST = "localhost";
 inline constexpr auto MQTT_BROKER_PORT = 1883;
 inline constexpr auto MQTT_EVEREST_PREFIX = "everest";
 inline constexpr auto MQTT_EXTERNAL_PREFIX = "";
+inline constexpr auto TELEMETRY_PREFIX = "everest-telemetry";
+inline constexpr auto TELEMETRY_ENABLED = false;
 
 } // namespace defaults
 
@@ -103,6 +105,8 @@ struct RuntimeSettings {
     int mqtt_broker_port;
     std::string mqtt_everest_prefix;
     std::string mqtt_external_prefix;
+    std::string telemetry_prefix;
+    bool telemetry_enabled;
 
     nlohmann::json config;
 

diff --git a/include/utils/config.hpp b/include/utils/config.hpp
@@ -7,6 +7,7 @@
 #include <regex>
 #include <set>
 #include <string>
+#include <unordered_map>
 
 #include <nlohmann/json-schema.hpp>
 
@@ -53,6 +54,8 @@ class Config {
     json types;
     schemas _schemas;
 
+    std::unordered_map<std::string, boost::optional<TelemetryConfig>> telemetry_configs;
+
     ///
     /// \brief loads the contents of the interface file referenced by the give \p intf_name from disk and validates its
     /// contents
@@ -120,6 +123,10 @@ class Config {
     /// \returns a ModuleInfo object
     ModuleInfo get_module_info(const std::string& module_id);
 
+    ///
+    /// \returns a TelemetryConfig if this has been configured
+    boost::optional<TelemetryConfig> get_telemetry_config(const std::string& module_id);
+
     ///
     /// \returns a json object that contains the manifests
     const json& get_manifests();

diff --git a/include/utils/types.hpp b/include/utils/types.hpp
@@ -32,8 +32,7 @@ using Object = json::object_t;
 using Handler = std::function<void(json)>;
 using StringHandler = std::function<void(std::string)>;
 
-enum class HandlerType
-{
+enum class HandlerType {
     Call,
     Result,
     SubscribeVar,
@@ -62,8 +61,7 @@ struct TypedHandler {
 using Token = std::shared_ptr<TypedHandler>;
 
 /// \brief MQTT Quality of service
-enum class QOS
-{
+enum class QOS {
     QOS0, ///< At most once delivery
     QOS1, ///< At least once delivery
     QOS2  ///< Exactly once delivery
@@ -75,6 +73,11 @@ struct ModuleInfo {
     std::string license;
     std::string id;
     std::string everest_prefix;
+    bool telemetry_enabled;
+};
+
+struct TelemetryConfig {
+    int id;
 };
 
 struct Requirement {