CSS-12576 add support for mysql catalogs

README.md

@@ -61,7 +61,7 @@ Adding a catalog to Trino requires user or service account credentiials. For thi
 ### Juju secrets
 Juju secrets are used to manage connector credentials. The format of these differ by connector type. Note: the same secret can be shared by multiple trino catalogs.
 
-For PostgreSQL (`postgresql-user-creds.yaml`):
+For PostgreSQL or MySQL (`postgresql-user-creds.yaml` or `mysql-user-creds.yaml`):
 ```
 rw: 
   user: trino
@@ -117,6 +117,7 @@ For Google sheets (`gsheets-service-accounts.yaml`):
 These secrets can be created by running the following:
 ```
 juju add-secret postgresql-credentials replicas#file=postgresql-user-creds.yaml cert#file=certificates.yaml
+juju add-secret mysql-credentials replicas#file=mysql-user-creds.yaml
 juju add-secret bigquery-service-accounts service-accounts#file=bigquery-service-accounts.yaml
 juju add-secret gsheets-service-accounts service-accounts#file=gsheets-service-accounts.yaml
 ```
@@ -135,6 +136,9 @@ catalogs:
     backend: dwh
     database: example  
     secret-id: crt7gpnmp25c760ji150
+  mysql_example: 
+    backend: mysql
+    secret-id: crt7gpnmp25c760ji150
   ge_bigquery:
     backend: bigquery
     project: <project-id>
@@ -152,6 +156,14 @@ backends:
       case-insensitive-name-matching=true
       decimal-mapping=allow_overflow
       decimal-rounding-mode=HALF_UP
+  mysql:
+    connector: mysql
+    url: jdbc:mysql://<database-host>:3306
+    params: sslMode=REQUIRED
+    config: |
+      case-insensitive-name-matching=true
+      decimal-mapping=allow_overflow
+      decimal-rounding-mode=HALF_UP
   bigquery:
     connector: bigquery
     config: |

requirements.txt

@@ -5,5 +5,7 @@ jsonschema==4.17.3
 bcrypt==4.0.1
 cosl==0.0.10
 cerberus==1.3.5
+# 0.28.x introduced some issues: https://github.com/gtsystem/lightkube/issues/78
+httpx==0.27.2 
 lightkube==0.15.4
 lightkube-models==1.28.1.4

src/catalog_manager.py

@@ -13,8 +13,8 @@
 from literals import (
     BIGQUERY_BACKEND_SCHEMA,
     GSHEETS_BACKEND_SCHEMA,
-    POSTGRESQL_BACKEND_SCHEMA,
     REPLICA_SCHEMA,
+    SQL_BACKEND_SCHEMA,
 )
 from utils import add_cert_to_truststore, validate_keys
 
@@ -151,24 +151,24 @@ def configure_catalogs(self):
             raise
 
 
-class PostgresqlCatalog(CatalogBase):
-    """Class for handling the PostgreSQL connector."""
+class SqlCatalog(CatalogBase):
+    """Class for handling the PostgreSQL and MySQL connectors."""
 
     def _get_credentials(self):
-        """Handle PostgreSQL catalog configuration.
+        """Handle PostgreSQL/MySQL catalog configuration.
 
         Returns:
             replicas: the database replica configuration.
         """
-        validate_keys(self.backend, POSTGRESQL_BACKEND_SCHEMA)
+        validate_keys(self.backend, SQL_BACKEND_SCHEMA)
         secret = self._get_secret_content(self.info["secret-id"])
         replicas = yaml.safe_load(secret["replicas"])
         certs = yaml.safe_load(secret.get("cert", ""))
         self._add_certs(certs)
         return replicas
 
     def _create_properties(self, replicas):
-        """Create the PostgreSQL connector catalog files.
+        """Create the PostgreSQL/MySQL connector catalog files.
 
         Args:
             replicas: the database replica configuration.
@@ -184,7 +184,7 @@ def _create_properties(self, replicas):
             suffix = replica_info.get("suffix", "")
 
             catalog_name = f"{self.name}{suffix}"
-            url = f"{self.backend['url']}/{self.info['database']}"
+            url = f"{self.backend['url']}/{self.info.get('database','')}"
             if self.backend.get("params"):
                 url = f"{url}?{self.backend['params']}"
 

src/charm.py

@@ -34,7 +34,7 @@
 )
 from ops.pebble import CheckStatus, ExecError
 
-from catalog_manager import BigqueryCatalog, GsheetCatalog, PostgresqlCatalog
+from catalog_manager import BigqueryCatalog, GsheetCatalog, SqlCatalog
 from literals import (
     CATALOG_DIR,
     CATALOG_SCHEMA,
@@ -467,7 +467,8 @@ def _create_catalog_instance(self, truststore_pwd, name, info, backend):
             ValueError: in case the backend type is not supported.
         """
         catalog_map = {
-            "postgresql": PostgresqlCatalog,
+            "postgresql": SqlCatalog,
+            "mysql": SqlCatalog,
             "bigquery": BigqueryCatalog,
             "gsheets": GsheetCatalog,
         }

src/literals.py

@@ -74,7 +74,7 @@
     "secret-id": {"type": "string"},
 }
 
-POSTGRESQL_BACKEND_SCHEMA = {
+SQL_BACKEND_SCHEMA = {
     "connector": {"type": "string"},
     "url": {"type": "string"},
     "params": {"type": "string"},

tests/integration/helpers.py

@@ -83,6 +83,12 @@
   password: pwd2
 """  # nosec
 
+MYSQL_REPLICA_SECRET = """\
+ro:
+  user: trino_ro
+  password: pwd3
+"""  # nosec
+
 CATALOG_QUERY = "SHOW CATALOGS"
 TRINO_USER = "trino"
 
@@ -338,6 +344,8 @@ async def add_juju_secret(ops_test: OpsTest, connector_type: str):
     """
     if connector_type == "postgresql":
         data_args = [f"replicas={POSTGRESQL_REPLICA_SECRET}"]  # nosec
+    elif connector_type == "mysql":
+        data_args = [f"replicas=={MYSQL_REPLICA_SECRET}"]  # nosec
     elif connector_type == "bigquery":
         data_args = [f"service-accounts={BIGQUERY_SECRET}"]  # nosec
     elif connector_type == "gsheets":
@@ -355,6 +363,7 @@ async def add_juju_secret(ops_test: OpsTest, connector_type: str):
 
 async def create_catalog_config(
     postgresql_secret_id,
+    mysql_secret_id,
     bigquery_secret_id,
     gsheets_secret_id,
     include_bigquery=True,
@@ -363,6 +372,7 @@ async def create_catalog_config(
 
     Args:
         postgresql_secret_id: the juju secret id for postgresql
+        mysql_secret_id: the juju secret id for postgresql
         bigquery_secret_id: the juju secret id for bigquery
         gsheets_secret_id: the juju secret id for gsheets
         include_bigquery: flag to indicate if bigquery configuration should be included.
@@ -377,6 +387,9 @@ async def create_catalog_config(
                 backend: dwh
                 database: example
                 secret-id: {postgresql_secret_id}
+            mysql:
+                backend: mysql
+                secret-id: {mysql_secret_id}
             bigquery:
                 backend: bigquery
                 project: project-12345
@@ -394,6 +407,14 @@ async def create_catalog_config(
                     case-insensitive-name-matching=true
                     decimal-mapping=allow_overflow
                     decimal-rounding-mode=HALF_UP
+            mysql:
+                connector: mysql
+                url: jdbc:mysql://mysql.com:3306
+                params: sslMode=REQUIRED
+                config: |
+                    case-insensitive-name-matching=true
+                    decimal-mapping=allow_overflow
+                    decimal-rounding-mode=HALF_UP
             bigquery:
                 connector: bigquery
                 config: |
@@ -408,6 +429,9 @@ async def create_catalog_config(
                 backend: dwh
                 database: example
                 secret-id: {postgresql_secret_id}
+            mysql:
+                backend: mysql
+                secret-id: {mysql_secret_id}
             gsheets-1:
                 backend: gsheets
                 metasheet-id: 1Es4HhWALUQjoa-bQh4a8B5HROz7dpGMfq_HbfoaW5LM
@@ -421,6 +445,14 @@ async def create_catalog_config(
                     case-insensitive-name-matching=true
                     decimal-mapping=allow_overflow
                     decimal-rounding-mode=HALF_UP
+            mysql:
+                connector: mysql
+                url: jdbc:mysql://mysql.com:3306
+                params: sslMode=REQUIRED
+                config: |
+                    case-insensitive-name-matching=true
+                    decimal-mapping=allow_overflow
+                    decimal-rounding-mode=HALF_UP
             gsheets:
                 connector: gsheets
         """

tests/integration/test_charm.py

@@ -42,28 +42,39 @@ async def test_basic_client(self, ops_test: OpsTest):
     async def test_catalog_config(self, ops_test: OpsTest):
         """Adds a PostgreSQL and BigQuery connector and asserts catalogs added."""
         postgresql_secret_id = await add_juju_secret(ops_test, "postgresql")
+        mysql_secret_id = await add_juju_secret(ops_test, "mysql")
         bigquery_secret_id = await add_juju_secret(ops_test, "bigquery")
         gsheet_secret_id = await add_juju_secret(ops_test, "gsheets")
 
         for app in ["trino-k8s", "trino-k8s-worker"]:
             await ops_test.model.grant_secret("postgresql-secret", app)
+            await ops_test.model.grant_secret("mysql-secret", app)
             await ops_test.model.grant_secret("bigquery-secret", app)
             await ops_test.model.grant_secret("gsheets-secret", app)
 
         catalog_config = await create_catalog_config(
-            postgresql_secret_id, bigquery_secret_id, gsheet_secret_id, True
+            postgresql_secret_id,
+            mysql_secret_id,
+            bigquery_secret_id,
+            gsheet_secret_id,
+            True,
         )
         catalogs = await update_catalog_config(
             ops_test, catalog_config, TRINO_USER
         )
 
         # Verify that both catalogs have been added.
         assert "postgresql-1" in str(catalogs)
+        assert "mysql" in str(catalogs)
         assert "bigquery" in str(catalogs)
         assert "gsheets-1" in str(catalogs)
 
         updated_catalog_config = await create_catalog_config(
-            postgresql_secret_id, bigquery_secret_id, gsheet_secret_id, False
+            postgresql_secret_id,
+            mysql_secret_id,
+            bigquery_secret_id,
+            gsheet_secret_id,
+            False,
         )
 
         catalogs = await update_catalog_config(
@@ -72,6 +83,7 @@ async def test_catalog_config(self, ops_test: OpsTest):
 
         # Verify that only the bigquery catalog has been removed.
         assert "postgresql-1" in str(catalogs)
+        assert "mysql" in str(catalogs)
         assert "bigquery" not in str(catalogs)
         assert "gsheets-1" in str(catalogs)
 

tests/unit/helpers.py

@@ -48,6 +48,11 @@
   user: trino_ro
   password: pwd2
 """  # nosec
+MYSQL_REPLICA_SECRET = """\
+ro:
+  user: trino_ro
+  password: pwd3
+"""  # nosec
 POSTGRESQL_REPLICA_CERT = """\
 cert: |
   -----BEGIN CERTIFICATE-----
@@ -61,6 +66,7 @@
 POSTGRESQL_2_CATALOG_PATH = (
     "/usr/lib/trino/etc/catalog/postgresql-2.properties"
 )
+MYSQL_CATALOG_PATH = "/usr/lib/trino/etc/catalog/mysql.properties"
 BIGQUERY_CATALOG_PATH = "/usr/lib/trino/etc/catalog/bigquery.properties"
 RANGER_PROPERTIES_PATH = "/usr/lib/ranger/install.properties"
 POLICY_MGR_URL = "http://ranger-k8s:6080"