ENG-1596: Facilitate OTEL traces and metrics, including sending to mo…

…nitoring backend

Makefile

@@ -101,5 +101,20 @@ test-flaky-assertions: .EXPORT_ALL_VARIABLES
 test-failing-assertions: .EXPORT_ALL_VARIABLES
 	poetry run rasa test e2e e2e_tests_with_assertions/failing --e2e-results
 
-make test-passing-stub-custom-actions: .EXPORT_ALL_VARIABLES
+test-passing-stub-custom-actions: .EXPORT_ALL_VARIABLES
 	poetry run rasa test e2e e2e_tests_with_stub_custom_actions/passing --e2e-results
+
+set-otel-resource-attributes: ## Set OTEL_RESOURCE_ATTRIBUTES with rasa version and git info
+	. scripts/set-otel-resource-attributes.sh
+
+run-otel-collector: ## Run OTEL collector, which would recieve traces and metrics, and export them to OTEL monitoring backend
+	docker compose -f otel-docker-compose.yml run --remove-orphans --build --name otel-collector -d -P otel-collector
+
+print-otel-collector-logs: ## Print OTEL collector logs on console
+	docker logs otel-collector
+
+otel-collector-health-check: ## Conduct health check on OTEL collector (requires curl and jq tools)
+	curl -sf http://localhost:13133/health/status | jq '.status' | grep 'Server available'
+
+stop-otel-collector: ## Stop OTEL collector
+	docker compose -f otel-docker-compose.yml down otel-collector -v --remove-orphans --rmi all

README.md

@@ -480,3 +480,20 @@ rasa test e2e e2e_tests_with_assertions/tests/path/to/a/target/test.yml
 ```
 
 ------
+
+### Enable `Open Telemetry` tracing and metrics
+1. Uncomment (and update if required) the `tracing` and `metrics` blocks in the
+[`endpoints.yml`](./endpoints.yml) file.
+2. _If_ intending to export traces and metrics to monitoring backend `Honeycomb` cloud for visualisation, then set API key `HONEYCOMB_API_KEY`.
+    - If intending to send to a different monitoring backend, then update [`otel-collector-config.yml`](./otel-collector-config.yml) and [`otel-docker-compose.yml`](./otel-docker-compose.yml) files accordingly, and set required API key(s) if required.
+3. To enable mapping traces or metrics to corresponding `rasa-pro` version, `rasa-calm-demo` git repository branch and commit hash, set `OTEL_RESOURCE_ATTRIBUTES`
+via `` `make set-otel-resource-attributes` `` (note the backticks in the command. Those are to be included, and _not_ to be omitted).
+    - If intending to set other attributes also, such as git tag etc, then set `OTEL_RESOURCE_ATTRIBUTES` 
+    manually: `export OTEL_RESOURCE_ATTRIBUTES=key1=value,key2=value2,key-n=value-n`
+4. Start OTEL collector as a `docker` container (named `otel-collector`): `make run-otel-collector`
+    - **Optional**: A health check on the collector can be performed via `make otel-collector-health-check` (Note: `curl` and `jq` utilities required for this)
+5. Train and use Rasa bot assistant.
+6. View traces and metrics: Now generated traces and metrics would be visible in:
+    - `Honeycomb`'s web UI (or other chosen monitoring backend, if using a different one).
+    - Also in `otel-collector`'s console logs. To troubleshoot in case traces or metrics do not appear on the monitoring backend, start by checking these logs via:
+        - `make print-otel-collector-logs`

endpoints.yml

@@ -58,12 +58,19 @@ action_endpoint:
 nlg:
   type: rephrase
 
-#tracing:
-#  type: jaeger
+# Run OTEL collector (via `otel-docker-compose.yml`), and uncomment this section, to send traces to it.
+# tracing:
+#  type: otlp
+#  endpoint: localhost:4317
+#  host: localhost
+#  service_name: rasa
+
+# Run OTEL collector (via `otel-docker-compose.yml`), and uncomment this section, to send metrics to it.
+# metrics:
+#  type: otlp
+#  endpoint: localhost:4317
 #  host: localhost
-#  port: 6831
 #  service_name: rasa
-#  sync_export: ~
 
 vector_store:
   type: qdrant

otel-collector-config.yml

@@ -0,0 +1,34 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+        cors:
+          allowed_origins:
+            - "*"
+
+exporters:
+  debug:
+    verbosity: detailed
+  otlp/honeycomb:
+    endpoint: "api.honeycomb.io:443"
+    headers:
+      "x-honeycomb-team": "${env:HONEYCOMB_API_KEY}"
+      "x-honeycomb-dataset": "metrics"
+
+extensions:
+  health_check:
+    endpoint: "0.0.0.0:13133"
+    path: "/health/status" 
+
+service:
+  extensions: [health_check]
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      exporters: [debug]
+    traces:
+      receivers: [otlp]
+      exporters: [debug, otlp/honeycomb]

otel-docker-compose.yml

@@ -0,0 +1,14 @@
+services:
+  otel-collector:
+    container_name: otel-collector
+    image: otel/opentelemetry-collector-contrib:0.114.0
+    restart: always
+    command: ["--config=/etc/otel-collector-config.yaml", ""]
+    volumes:
+      - ./otel-collector-config.yml:/etc/otel-collector-config.yaml
+    ports:
+      - 4317:4317
+      - 13133:13133
+    environment:
+      - HONEYCOMB_API_KEY=${HONEYCOMB_API_KEY}
+      - OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES}

scripts/set-otel-resource-attributes.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -eu pipefail
+
+RASA_CALM_DEMO_BRANCH="$(git branch --show-current)"
+RASA_PRO_VERSION="$(rasa --version | grep 'Rasa Pro Version' | tr -d ' ' | awk -F ':' '{print $2}')"
+RASA_CALM_DEMO_SHA="$(git rev-parse --short HEAD)"
+
+echo "RASA_CALM_DEMO_BRANCH=${RASA_CALM_DEMO_BRANCH}"
+echo "RASA_PRO_VERSION=${RASA_PRO_VERSION}"
+echo "RASA_CALM_DEMO_SHA=${RASA_CALM_DEMO_SHA}"
+
+OTEL_RESOURCE_ATTRIBUTES=rasa-pro-version="${RASA_PRO_VERSION}",rasa-calm-demo-sha="${RASA_CALM_DEMO_SHA}",rasa-calm-demo-branch="${RASA_CALM_DEMO_BRANCH}"
+echo "OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES}"
+
+export RASA_CALM_DEMO_BRANCH
+export RASA_PRO_VERSION
+export OTEL_RESOURCE_ATTRIBUTES