From c5778b11f39e6784a2d3dd9a077c92784a4a2471 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 3 Jun 2024 17:04:52 -0400 Subject: [PATCH 1/2] refactor: Drop legacy interfaces Drop the following legacy interfaces: - interface_grafana_support - interface_influxdb - interface_elasticsearch These interfaces are no longer in use and will be replaced by COS. Signed-off-by: Jason C. Nucciarone --- src/charm.py | 72 +-------------- src/interface_elasticsearch.py | 68 -------------- src/interface_grafana_source.py | 72 --------------- src/interface_influxdb.py | 156 -------------------------------- 4 files changed, 1 insertion(+), 367 deletions(-) delete mode 100644 src/interface_elasticsearch.py delete mode 100644 src/interface_grafana_source.py delete mode 100644 src/interface_influxdb.py diff --git a/src/charm.py b/src/charm.py index 6e2cf01..70b180a 100755 --- a/src/charm.py +++ b/src/charm.py @@ -12,9 +12,6 @@ from typing import List from charms.fluentbit.v0.fluentbit import FluentbitClient -from interface_elasticsearch import Elasticsearch -from interface_grafana_source import GrafanaSource -from interface_influxdb import InfluxDB from interface_prolog_epilog import PrologEpilog from interface_slurmctld_peer import SlurmctldPeer from interface_slurmd import Slurmd @@ -55,10 +52,6 @@ def __init__(self, *args): self._slurmrestd = Slurmrestd(self, "slurmrestd") self._slurmctld_peer = SlurmctldPeer(self, "slurmctld-peer") self._prolog_epilog = PrologEpilog(self, "prolog-epilog") - - self._grafana = GrafanaSource(self, "grafana-source") - self._influxdb = InfluxDB(self, "influxdb-api") - self._elasticsearch = Elasticsearch(self, "elasticsearch") self._fluentbit = FluentbitClient(self, "fluentbit") event_handler_bindings = { @@ -82,16 +75,10 @@ def __init__(self, *args): # Addons lifecycle events self._prolog_epilog.on.prolog_epilog_available: self._on_write_slurm_config, self._prolog_epilog.on.prolog_epilog_unavailable: self._on_write_slurm_config, - self._grafana.on.grafana_available: self._on_grafana_available, - self._influxdb.on.influxdb_available: self._on_influxdb_available, - self._influxdb.on.influxdb_unavailable: self._on_write_slurm_config, - self._elasticsearch.on.elasticsearch_available: self._on_elasticsearch_available, - self._elasticsearch.on.elasticsearch_unavailable: self._on_write_slurm_config, # actions self.on.show_current_config_action: self._on_show_current_config, self.on.drain_action: self._drain_nodes_action, self.on.resume_action: self._resume_nodes_action, - self.on.influxdb_info_action: self._infludb_info_action, } for event, handler in event_handler_bindings.items(): self.framework.observe(event, handler) @@ -146,7 +133,6 @@ def _addons_info(self): return { **self._assemble_prolog_epilog(), **self._assemble_acct_gather_addon(), - **self._assemble_elastic_search_addon(), } def _assemble_prolog_epilog(self) -> dict: @@ -163,16 +149,8 @@ def _assemble_prolog_epilog(self) -> dict: def _assemble_acct_gather_addon(self): """Generate the acct gather section of the addons.""" logger.debug("## Generating acct gather configuration") - addons = {} - - influxdb_info = self._get_influxdb_info() - if influxdb_info: - addons["acct_gather"] = influxdb_info - addons["acct_gather"]["default"] = "all" - addons["acct_gather_profile"] = "acct_gather_profile/influxdb" - - # it is possible to setup influxdb or hdf5 profiles without the + # it is possible to setup hdf5 profiles without the # relation, using the custom-config section of slurm.conf. We need to # support setting up the acct_gather configuration for this scenario acct_gather_custom = self.config.get("acct-gather-custom") @@ -186,18 +164,6 @@ def _assemble_acct_gather_addon(self): return addons - def _assemble_elastic_search_addon(self): - """Generate the acct gather section of the addons.""" - logger.debug("## Generating elastic search addon configuration") - addon = {} - - elasticsearch_ingress = self._elasticsearch.elasticsearch_ingress - if elasticsearch_ingress: - suffix = f"/{self.cluster_name}/jobcomp" - addon = {"elasticsearch_address": f"{elasticsearch_ingress}{suffix}"} - - return addon - def set_slurmd_available(self, flag: bool): """Set stored value of slurmd available.""" self._stored.slurmd_available = flag @@ -508,30 +474,6 @@ def _resume_nodes(self, nodelist): update_cmd = f"update nodename={nodes} state=resume" self._slurm_manager.slurm_cmd("scontrol", update_cmd) - def _on_grafana_available(self, event): - """Create the grafana-source if we are the leader and have influxdb.""" - if not self._is_leader(): - return - - influxdb_info = self._get_influxdb_info() - - if influxdb_info: - self._grafana.set_grafana_source_info(influxdb_info) - else: - logger.error("## Can not set Grafana source: missing influxdb relation") - - def _on_influxdb_available(self, event): - """Assemble addons to forward slurm data to influxdb.""" - self._on_write_slurm_config(event) - - def _on_elasticsearch_available(self, event): - """Assemble addons to forward Slurm data to elasticsearch.""" - self._on_write_slurm_config(event) - - def _get_influxdb_info(self) -> dict: - """Return influxdb info.""" - return self._influxdb.get_influxdb_info() - def _drain_nodes_action(self, event): """Drain specified nodes.""" nodes = event.params["nodename"] @@ -561,18 +503,6 @@ def _resume_nodes_action(self, event): except subprocess.CalledProcessError as e: event.fail(message=f"Error resuming {nodes}: {e.output}") - def _infludb_info_action(self, event): - influxdb_info = self._get_influxdb_info() - - if not influxdb_info: - info = "not related" - else: - # Juju does not like underscores in dictionaries - info = {k.replace("_", "-"): v for k, v in influxdb_info.items()} - - logger.debug(f"## InfluxDB-info action: {influxdb_info}") - event.set_results({"influxdb": info}) - if __name__ == "__main__": main(SlurmctldCharm) diff --git a/src/interface_elasticsearch.py b/src/interface_elasticsearch.py deleted file mode 100644 index d61179d..0000000 --- a/src/interface_elasticsearch.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python3 -"""Elasticserch interface.""" -import logging - -from ops.framework import EventBase, EventSource, Object, ObjectEvents, StoredState - -logger = logging.getLogger() - - -class ElasticsearchAvailableEvent(EventBase): - """ElasticsearchAvailable event.""" - - -class ElasticsearchUnAvailableEvent(EventBase): - """ElasticsearchUnAvailable event.""" - - -class ElasticsearchEvents(ObjectEvents): - """ElasticsearchEvents.""" - - elasticsearch_available = EventSource(ElasticsearchAvailableEvent) - elasticsearch_unavailable = EventSource(ElasticsearchUnAvailableEvent) - - -class Elasticsearch(Object): - """Elasticsearch interface.""" - - _stored = StoredState() - on = ElasticsearchEvents() - - def __init__(self, charm, relation_name): - """Observe relation events.""" - super().__init__(charm, relation_name) - self._charm = charm - self._relation_name = relation_name - - self._stored.set_default(elasticsearch_host=str(), elasticsearch_port=str()) - - self.framework.observe( - self._charm.on[self._relation_name].relation_changed, self._on_relation_changed - ) - - self.framework.observe( - self._charm.on[self._relation_name].relation_broken, self._on_relation_broken - ) - - def _on_relation_changed(self, event): - """Get server address and store it.""" - ingress = event.relation.data[event.unit]["ingress-address"] - self._stored.elasticsearch_host = ingress - # elastic charm can't change the port, so hardcode it here - self._stored.elasticsearch_port = "9200" - self.on.elasticsearch_available.emit() - - def _on_relation_broken(self, event): - """Clear elasticsearch_ingress and emit elasticsearch_unavailable.""" - self._stored.elasticsearch_ingress = "" - self.on.elasticsearch_unavailable.emit() - - @property - def elasticsearch_ingress(self) -> str: - """Return elasticsearch_ingress.""" - host = self._stored.elasticsearch_host - port = self._stored.elasticsearch_port - if host: - return f"http://{host}:{port}" - else: - return "" diff --git a/src/interface_grafana_source.py b/src/interface_grafana_source.py deleted file mode 100644 index 8c7ba05..0000000 --- a/src/interface_grafana_source.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -"""Grafana Source Interface.""" -import logging - -from ops.framework import EventBase, EventSource, Object, ObjectEvents - -logger = logging.getLogger() - - -class GrafanaSourceAvailableEvent(EventBase): - """GrafanaSourceAvailable event.""" - - -class GrafanaSourceEvents(ObjectEvents): - """GrafanaSourceEvents.""" - - grafana_available = EventSource(GrafanaSourceAvailableEvent) - - -class GrafanaSource(Object): - """Grafana Source Interface.""" - - on = GrafanaSourceEvents() - - def __init__(self, charm, relation_name): - """Observe relation events.""" - super().__init__(charm, relation_name) - self._charm = charm - self._relation_name = relation_name - - self.framework.observe( - self._charm.on[self._relation_name].relation_joined, - self._on_relation_joined, - ) - - self.framework.observe( - self._charm.on[self._relation_name].relation_broken, - self._on_relation_broken, - ) - - def _on_relation_joined(self, event): - if self.framework.model.unit.is_leader(): - self.on.grafana_available.emit() - - def _on_relation_broken(self, event): - if self.framework.model.unit.is_leader(): - app_relation_data = self._relation.data[self.model.app] - app_relation_data["url"] = "" - app_relation_data["username"] = "" - app_relation_data["password"] = "" - app_relation_data["database"] = "" - - @property - def _relation(self): - return self.framework.model.get_relation(self._relation_name) - - @property - def is_joined(self) -> bool: - """Return True if self._relation is not None.""" - return self._relation is not None - - def set_grafana_source_info(self, influxdb: dict): - """Set grafana source info on relation.""" - logger.debug("## sending data to grafana") - - if self.framework.model.unit.is_leader(): - app_relation_data = self._relation.data[self.model.app] - app_relation_data["type"] = "influxdb" - app_relation_data["url"] = f"{influxdb['ingress']}:{influxdb['port']}" - app_relation_data["username"] = influxdb["user"] - app_relation_data["password"] = influxdb["password"] - app_relation_data["database"] = influxdb["database"] diff --git a/src/interface_influxdb.py b/src/interface_influxdb.py deleted file mode 100644 index f41747a..0000000 --- a/src/interface_influxdb.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python3 -"""AcctGather (Influxdb) interface.""" -import json -import logging -import secrets -import string - -import influxdb -from ops.framework import EventBase, EventSource, Object, ObjectEvents, StoredState - -logger = logging.getLogger() - - -class InfluxDBAvailableEvent(EventBase): - """InfluxDBAvailable event.""" - - -class InfluxDBUnAvailableEvent(EventBase): - """InfluxDBUnAvailable event.""" - - -class InfluxDBEvents(ObjectEvents): - """InfluxDBEvents.""" - - influxdb_available = EventSource(InfluxDBAvailableEvent) - influxdb_unavailable = EventSource(InfluxDBUnAvailableEvent) - - -def generate_password(length=20): - """Generate a random password.""" - alphabet = string.ascii_letters + string.digits - return "".join(secrets.choice(alphabet) for _ in range(length)) - - -class InfluxDB(Object): - """InfluxDB interface.""" - - _stored = StoredState() - on = InfluxDBEvents() - - def __init__(self, charm, relation_name): - """Observe relation events.""" - super().__init__(charm, relation_name) - self._charm = charm - self._relation_name = relation_name - - self._INFLUX_USER = "slurm" - self._INFLUX_PRIVILEGE = "all" - self._INFLUX_DATABASE = self._charm.cluster_name - - self._stored.set_default( - influxdb_info=str(), - influxdb_admin_info=str(), - ) - - self.framework.observe( - self._charm.on[self._relation_name].relation_changed, - self._on_relation_changed, - ) - - self.framework.observe( - self._charm.on[self._relation_name].relation_broken, - self._on_relation_broken, - ) - - def _on_relation_changed(self, event): - """Store influxdb_ingress in the charm.""" - if self.framework.model.unit.is_leader(): - if not self._stored.influxdb_admin_info: - ingress = event.relation.data[event.unit]["ingress-address"] - port = event.relation.data[event.unit].get("port") - user = event.relation.data[event.unit].get("user") - password = event.relation.data[event.unit].get("password") - - if all([ingress, port, user, password]): - admin_info = { - "ingress": ingress, - "port": port, - "user": user, - "password": password, - } - self._stored.influxdb_admin_info = json.dumps(admin_info) - - # Influxdb client - client = influxdb.InfluxDBClient(ingress, port, user, password) - - # Influxdb slurm user password - influx_slurm_password = generate_password() - - # Only create the user and db if they don't already exist - users = [db["user"] for db in client.get_list_users()] - logger.debug(f"## users in influxdb: {users}") - if self._INFLUX_USER not in users: - logger.debug(f"## Creating influxdb user: {self._INFLUX_USER}") - client.create_user(self._INFLUX_USER, influx_slurm_password) - - databases = [db["name"] for db in client.get_list_database()] - if self._INFLUX_DATABASE not in databases: - logger.debug(f"## Creating influxdb db: {self._INFLUX_DATABASE}") - client.create_database(self._INFLUX_DATABASE) - - client.grant_privilege( - self._INFLUX_PRIVILEGE, self._INFLUX_DATABASE, self._INFLUX_USER - ) - - # select default retention policy - policies = client.get_list_retention_policies(self._INFLUX_DATABASE) - policy = "slurm" - for p in policies: - if p["default"]: - policy = p["name"] - - # Dump influxdb_info to json and set it to state - influxdb_info = { - "ingress": ingress, - "port": port, - "user": self._INFLUX_USER, - "password": influx_slurm_password, - "database": self._INFLUX_DATABASE, - "retention_policy": policy, - } - self._stored.influxdb_info = json.dumps(influxdb_info) - self.on.influxdb_available.emit() - - def _on_relation_broken(self, event): - """Remove the database and user from influxdb.""" - if self.framework.model.unit.is_leader(): - if self._stored.influxdb_admin_info: - influxdb_admin_info = json.loads(self._stored.influxdb_admin_info) - - client = influxdb.InfluxDBClient( - influxdb_admin_info["ingress"], - influxdb_admin_info["port"], - influxdb_admin_info["user"], - influxdb_admin_info["password"], - ) - - databases = [db["name"] for db in client.get_list_database()] - if self._INFLUX_DATABASE in databases: - client.drop_database(self._INFLUX_DATABASE) - - users = [db["user"] for db in client.get_list_users()] - if self._INFLUX_USER in users: - client.drop_user(self._INFLUX_USER) - - self._stored.influxdb_info = "" - self._stored.influxdb_admin_info = "" - self.on.influxdb_unavailable.emit() - - def get_influxdb_info(self) -> dict: - """Return the influxdb info.""" - influxdb_info = self._stored.influxdb_info - if influxdb_info: - return json.loads(influxdb_info) - else: - return {} From 992f0104b3b9e729222d94ce5b7d9882b06df323 Mon Sep 17 00:00:00 2001 From: "Jason C. Nucciarone" Date: Mon, 3 Jun 2024 17:16:40 -0400 Subject: [PATCH 2/2] refactor: Drop legacy integrations and actions Signed-off-by: Jason C. Nucciarone --- charmcraft.yaml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/charmcraft.yaml b/charmcraft.yaml index 9b57f0f..28ebc3d 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -32,18 +32,11 @@ requires: interface: slurmdbd slurmrestd: interface: slurmrestd - influxdb-api: - interface: influxdb-api - elasticsearch: - interface: elasticsearch fluentbit: interface: fluentbit provides: prolog-epilog: interface: prolog-epilog - grafana-source: - interface: grafana-source - scope: global assumes: - juju @@ -211,10 +204,3 @@ actions: The nodes to resume, using the Slurm format, e.g. `node-[1,2]`. required: - nodename - - influxdb-info: - description: > - Get InfluxDB info. - - This action returns the host, port, username, password, database, and - retention policy regarding to InfluxDB.