diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96a9db7..b01ab27 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,7 @@ juju debug-log charmcraft pack # the --destructive-mode flag can be used to pack the charm using the current host. # Deploy the charm -juju deploy ./airbyte-k8s_ubuntu-22.04-amd64.charm --resource airbyte-api-server=airbyte/airbyte-api-server:0.60.0 --resource airbyte-bootloader=airbyte/bootloader:0.60.0 --resource airbyte-connector-builder-server=airbyte/connector-builder-server:0.60.0 --resource airbyte-cron=airbyte/cron:0.60.0 --resource airbyte-pod-sweeper=bitnami/kubectl:1.29.4 --resource airbyte-server=airbyte/server:0.60.0 --resource airbyte-worker=airbyte/worker:0.60.0 +juju deploy ./airbyte-k8s_ubuntu-22.04-amd64.charm --resource airbyte-api-server=airbyte/airbyte-api-server:0.60.0 --resource airbyte-bootloader=airbyte/bootloader:0.60.0 --resource airbyte-connector-builder-server=airbyte/connector-builder-server:0.60.0 --resource airbyte-cron=airbyte/cron:0.60.0 --resource airbyte-pod-sweeper=bitnami/kubectl:1.29.4 --resource airbyte-server=airbyte/server:0.60.0 --resource airbyte-workers=airbyte/worker:0.60.0 # Deploy ui charm (Only if modifying UI charm, otherwise deploy using `juju deploy airbyte-ui-k8s --channel edge`) juju deploy ./airbyte-ui-k8s_ubuntu-22.04-amd64.charm --resource airbyte-webapp=airbyte/webapp:0.60.0 diff --git a/charmcraft.yaml b/charmcraft.yaml index d718a0a..e24909e 100644 --- a/charmcraft.yaml +++ b/charmcraft.yaml @@ -7,7 +7,7 @@ name: airbyte-k8s type: charm title: Airbyte Server -summary: Airbyte server operator +summary: Airbyte Server operator description: | Airbyte is an open-source data integration platform designed to centralize and streamline the process of extracting and loading data from various sources into @@ -81,7 +81,6 @@ provides: # General configuration documentation: https://juju.is/docs/sdk/config config: options: - # An example config option to customise the log level of the workload log-level: description: | Configures the log level. @@ -90,11 +89,279 @@ config: default: "INFO" type: string + ##### Airbyte services config ##### temporal-host: description: Temporal server host. default: "temporal-k8s:7233" type: string + webapp-url: + description: URL for the Airbyte webapp. + default: "http://airbyte-ui-k8s:8080" + type: string + + ##### Secrets config ##### + secret-persistence: + description: | + Defines the Secret Persistence type. + + If set, acceptable values are: "GOOGLE_SECRET_MANAGER", "AWS_SECRET_MANAGER", "TESTING_CONFIG_DB_TABLE", "VAULT" + type: string + + secret-store-gcp-project-id: + description: Defines the GCP Project to store secrets in. Alpha support. + type: string + + secret-store-gcp-credentials: + description: | + Defines the JSON credentials used to read/write Airbyte Configuration to Google Secret Manager. + + These credentials must have Secret Manager Read/Write access. Alpha support. + type: string + + vault-address: + description: Defines the vault address to read/write Airbyte Configuration to Hashicorp Vault. Alpha Support. + type: string + + vault-prefix: + description: Defines the vault path prefix. Empty by default. Alpha Support. + type: string + + vault-auth-token: + description: The token used for vault authentication. Alpha Support. + type: string + + vault-auth-method: + description: How vault will perform authentication. Currently, only supports Token auth. Defaults to "token". Alpha Support. + default: "token" + type: string + + aws-access-key: + description: Defines the aws_access_key_id from the AWS credentials to use for AWS Secret Manager. + type: string + + aws-secret-access-key: + description: Defines aws_secret_access_key to use for the AWS Secret Manager. + type: string + + aws-kms-key-arn: + description: Optional param that defines the KMS Encryption key used for the AWS Secret Manager. + type: string + + aws-secret-manager-secret-tags: + description: | + Defines the tags that will be included to all writes to the AWS Secret Manager. + + The format should be "key1=value1,key2=value2". + type: string + + ##### Jobs config ##### + sync-job-retries-complete-failures-max-successive: + description: Max number of successive attempts in which no data was synchronized before failing the job. + default: 5 + type: int + + sync-job-retries-complete-failures-max-total: + description: Max number of attempts in which no data was synchronized before failing the job. + default: 10 + type: int + + sync-job-retries-complete-failures-backoff-min-interval-s: + description: Minimum backoff interval in seconds between failed attempts in which no data was synchronized. + default: 10 + type: int + + sync-job-retries-complete-failures-backoff-max-interval-s: + description: Maximum backoff interval in seconds between failed attempts in which no data was synchronized. + default: 1800 + type: int + + sync-job-retries-complete-failures-backoff-base: + description: Exponential base of the backoff interval between failed attempts in which no data was synchronized. + default: 3 + type: int + + sync-job-retries-partial-failures-max-successive: + description: Max number of successive attempts in which some data was synchronized before failing the job. + default: 1000 + type: int + + sync-job-retries-partial-failures-max-total: + description: Max number of attempts in which some data was synchronized before failing the job. + default: 20 + type: int + + sync-job-max-timeout-days: + description: Number of days a sync job will execute for before timing out. + default: 3 + type: int + + job-main-container-cpu-request: + description: Job container's minimum CPU usage. Defaults to none. + type: string + + job-main-container-cpu-limit: + description: Job container's maximum CPU usage. Defaults to none. + type: string + + job-main-container-memory-request: + description: Job container's minimum RAM usage. Defaults to none. + type: string + + job-main-container-memory-limit: + description: Job container's maximum RAM usage. Defaults to none. + type: string + + ##### Connections config ##### + max-fields-per-connections: + description: Maximum number of fields able to be selected for a single connection. + default: 20000 + type: int + + max-days-of-only-failed-jobs-before-connection-disable: + description: Number of consecuative days of only failed jobs before the connection is disabled. + default: 14 + type: int + + max-failed-jobs-in-a-row-before-connection-disable: + description: Number of consecuative failed jobs before the connection is disabled. + default: 20 + type: int + + ##### Worker config ##### + max-spec-workers: + description: Maximum number of Spec workers each Airbyte Worker container can support. Defaults to 5. + default: 5 + type: int + + max-check-workers: + description: Maximum number of Check workers each Airbyte Worker container can support. Defaults to 5. + default: 5 + type: int + + max-sync-workers: + description: Maximum number of Sync workers each Airbyte Worker container can support. Defaults to 5. + default: 5 + type: int + + max-discover-workers: + description: Maximum number of Discover workers each Airbyte Worker container can support. Defaults to 5. + default: 5 + type: int + + ##### Data retention config ##### + temporal-history-retention-in-days: + description: Retention period of the job history in Temporal, defaults to 30 days. + default: 30 + type: int + + ##### Kubernetes config ##### + job-kube-tolerations: + description: | + Defines one or more Job pod tolerations. + + Tolerations are separated by ';'. Each toleration contains k=v pairs mentioning some/all + of key, effect, operator and value and separated by ','. + type: string + + job-kube-node-selectors: + description: | + Defines one or more Job pod node selectors. + + Each k=v pair is separated by a ','. For example: key1=value1,key2=value2. + It is the pod node selectors of the "sync" job. It also serves as the + default pod node selectors fallback for other jobs. + type: string + + job-kube-annotations: + description: | + Defines one or more Job pod annotations. + + Each k=v pair is separated by a ','. For example: key1=value1,key2=value2. + It is the pod annotations of the "sync" job. It also serves as the + default pod annotations fallback for other jobs. + type: string + + job-kube-main-container-image-pull-policy: + description: Defines the Job pod connector image pull policy. + default: "IfNotPresent" + type: string + + job-kube-main-container-image-pull-secret: + description: Defines the Job pod connector image pull secret. Useful when hosting private images. + type: string + + job-kube-sidecar-container-image-pull-policy: + description: | + Defines the image pull policy on the sidecar containers in the Job pod. + + Useful when there are cluster policies enforcing to always pull. + default: "IfNotPresent" + type: string + + job-kube-socat-image: + description: Defines the Job pod socat image. e.g. alpine/socat:1.7.4.3-r0 + type: string + + job-kube-busybox-image: + description: Defines the Job pod busybox image. e.g. busybox:1.28 + type: string + + job-kube-curl-image: + description: Defines the Job pod curl image. e.g. curlimages/curl:7.83.1 + type: string + + job-kube-namespace: + description: | + Defines the Kubernetes namespace Job pods are created in. + + Defaults to the current namespace. + type: string + + ##### Jobs config ##### + spec-job-kube-node-selectors: + description: | + Defines one or more pod node selectors for the spec job. + + Each k=v pair is separated by a ','. For example: key1=value1,key2=value2. + type: string + + check-job-kube-node-selectors: + description: | + Defines one or more pod node selectors for the check job. + + Each k=v pair is separated by a ','. For example: key1=value1,key2=value2. + type: string + + discover-job-kube-node-selectors: + description: | + Defines one or more pod node selectors for the discover job. + + Each k=v pair is separated by a ','. For example: key1=value1,key2=value2. + type: string + + spec-job-kube-annotations: + description: | + Defines one or more pod annotations for the spec job. + + Each k=v pair is separated by a ','. For example: key1=value1,key2=value2 + type: string + + check-job-kube-annotations: + description: | + Defines one or more pod annotations for the check job. + + Each k=v pair is separated by a ','. For example: key1=value1,key2=value2 + type: string + + discover-job-kube-annotations: + description: | + Defines one or more pod annotations for the discover job. + + Each k=v pair is separated by a ','. For example: key1=value1,key2=value2 + type: string + + ##### Logging config ##### storage-type: description: | Storage type for logs. @@ -129,6 +396,7 @@ config: default: "airbyte-state-storage" type: string + ##### Miscellaneous config ##### pod-running-ttl-minutes: description: Number of minutes until a running job pod is removed. default: 240 @@ -144,11 +412,6 @@ config: default: 1440 type: int - webapp-url: - description: URL for the Airbyte webapp. - default: "http://airbyte-ui-k8s:8080" - type: string - # The containers and resources metadata apply to Kubernetes charms only. # See https://juju.is/docs/sdk/metadata-reference for a checklist and guidance. diff --git a/src/charm.py b/src/charm.py index 1f19868..075155a 100755 --- a/src/charm.py +++ b/src/charm.py @@ -18,6 +18,7 @@ from charm_helpers import create_env from literals import ( AIRBYTE_API_PORT, + AIRBYTE_VERSION, BUCKET_CONFIGS, CONNECTOR_BUILDER_SERVER_API_PORT, CONTAINERS, @@ -179,6 +180,7 @@ def _on_update_status(self, event): if not all_valid_plans: return + self.unit.set_workload_version(f"v{AIRBYTE_VERSION}") self.unit.status = ActiveStatus() if self.unit.is_leader(): self.airbyte_ui._provide_server_status() @@ -304,6 +306,7 @@ def _update(self, event): ) env = create_env(self.model.name, self.app.name, container_name, self.config, self._state) + env = {k: v for k, v in env.items() if v is not None} pebble_layer = get_pebble_layer(container_name, env) container.add_layer(container_name, pebble_layer, combine=True) container.replan() diff --git a/src/charm_helpers.py b/src/charm_helpers.py index f18a3da..ec66b5f 100644 --- a/src/charm_helpers.py +++ b/src/charm_helpers.py @@ -34,18 +34,83 @@ def create_env(model_name, app_name, container_name, config, state): port = db_conn["port"] db_name = db_conn["dbname"] db_url = f"jdbc:postgresql://{host}:{port}/{db_name}" + secret_persistence = config["secret-persistence"] + if secret_persistence: + secret_persistence = config["secret-persistence"].value # Some defaults are extracted from Helm chart: # https://github.com/airbytehq/airbyte-platform/tree/v0.60.0/charts/airbyte env = { **BASE_ENV, - "DATABASE_URL": db_url, - "DATABASE_USER": db_conn["user"], - "DATABASE_PASSWORD": db_conn["password"], - "DATABASE_DB": db_name, - "DATABASE_HOST": host, - "DATABASE_PORT": port, + # Airbye services config + "LOG_LEVEL": config["log-level"].value, "TEMPORAL_HOST": config["temporal-host"], + "WEBAPP_URL": config["webapp-url"], + # Secrets config + "SECRET_PERSISTENCE": secret_persistence, + "SECRET_STORE_GCP_PROJECT_ID": config["secret-store-gcp-project-id"], + "SECRET_STORE_GCP_CREDENTIALS": config["secret-store-gcp-credentials"], + "VAULT_ADDRESS": config["vault-address"], + "VAULT_PREFIX": config["vault-prefix"], + "VAULT_AUTH_TOKEN": config["vault-auth-token"], + "VAULT_AUTH_METHOD": config["vault-auth-method"].value, + "AWS_ACCESS_KEY": config["aws-access-key"], + "AWS_SECRET_ACCESS_KEY": config["aws-secret-access-key"], + "AWS_KMS_KEY_ARN": config["aws-kms-key-arn"], + "AWS_SECRET_MANAGER_SECRET_TAGS": config["aws-secret-manager-secret-tags"], + # Jobs config + "SYNC_JOB_RETRIES_COMPLETE_FAILURES_MAX_SUCCESSIVE": config[ + "sync-job-retries-complete-failures-max-successive" + ], + "SYNC_JOB_RETRIES_COMPLETE_FAILURES_MAX_TOTAL": config["sync-job-retries-complete-failures-max-total"], + "SYNC_JOB_RETRIES_COMPLETE_FAILURES_BACKOFF_MIN_INTERVAL_S": config[ + "sync-job-retries-complete-failures-backoff-min-interval-s" + ], + "SYNC_JOB_RETRIES_COMPLETE_FAILURES_BACKOFF_MAX_INTERVAL_S": config[ + "sync-job-retries-complete-failures-backoff-max-interval-s" + ], + "SYNC_JOB_RETRIES_COMPLETE_FAILURES_BACKOFF_BASE": config["sync-job-retries-complete-failures-backoff-base"], + "SYNC_JOB_RETRIES_PARTIAL_FAILURES_MAX_SUCCESSIVE": config["sync-job-retries-partial-failures-max-successive"], + "SYNC_JOB_RETRIES_PARTIAL_FAILURES_MAX_TOTAL": config["sync-job-retries-partial-failures-max-total"], + "SYNC_JOB_MAX_TIMEOUT_DAYS": config["sync-job-max-timeout-days"], + "JOB_MAIN_CONTAINER_CPU_REQUEST": config["job-main-container-cpu-request"], + "JOB_MAIN_CONTAINER_CPU_LIMIT": config["job-main-container-cpu-limit"], + "JOB_MAIN_CONTAINER_MEMORY_REQUEST": config["job-main-container-memory-request"], + "JOB_MAIN_CONTAINER_MEMORY_LIMIT": config["job-main-container-memory-limit"], + # Connections config + "MAX_FIELDS_PER_CONNECTION": config["max-fields-per-connections"], + "MAX_DAYS_OF_ONLY_FAILED_JOBS_BEFORE_CONNECTION_DISABLE": config[ + "max-days-of-only-failed-jobs-before-connection-disable" + ], + "MAX_FAILED_JOBS_IN_A_ROW_BEFORE_CONNECTION_DISABLE": config[ + "max-failed-jobs-in-a-row-before-connection-disable" + ], + # Worker config + "MAX_SPEC_WORKERS": config["max-spec-workers"], + "MAX_CHECK_WORKERS": config["max-check-workers"], + "MAX_SYNC_WORKERS": config["max-sync-workers"], + "MAX_DISCOVER_WORKERS": config["max-discover-workers"], + # Data retention config + "TEMPORAL_HISTORY_RETENTION_IN_DAYS": config["temporal-history-retention-in-days"], + # Kubernetes config + "JOB_KUBE_TOLERATIONS": config["job-kube-tolerations"], + "JOB_KUBE_NODE_SELECTORS": config["job-kube-node-selectors"], + "JOB_KUBE_ANNOTATIONS": config["job-kube-annotations"], + "JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY": config["job-kube-main-container-image-pull-policy"].value, + "JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_SECRET": config["job-kube-main-container-image-pull-secret"], + "JOB_KUBE_SIDECAR_CONTAINER_IMAGE_PULL_POLICY": config["job-kube-sidecar-container-image-pull-policy"].value, + "JOB_KUBE_SOCAT_IMAGE": config["job-kube-socat-image"], + "JOB_KUBE_BUSYBOX_IMAGE": config["job-kube-busybox-image"], + "JOB_KUBE_CURL_IMAGE": config["job-kube-curl-image"], + "JOB_KUBE_NAMESPACE": config["job-kube-namespace"] or model_name, + # Jobs config + "SPEC_JOB_KUBE_NODE_SELECTORS": config["spec-job-kube-node-selectors"], + "CHECK_JOB_KUBE_NODE_SELECTORS": config["check-job-kube-node-selectors"], + "DISCOVER_JOB_KUBE_NODE_SELECTORS": config["discover-job-kube-node-selectors"], + "SPEC_JOB_KUBE_ANNOTATIONS": config["spec-job-kube-annotations"], + "CHECK_JOB_KUBE_ANNOTATIONS": config["check-job-kube-annotations"], + "DISCOVER_JOB_KUBE_ANNOTATIONS": config["discover-job-kube-annotations"], + # Logging config "WORKER_LOGS_STORAGE_TYPE": config["storage-type"].value, "WORKER_STATE_STORAGE_TYPE": config["storage-type"].value, "STORAGE_TYPE": config["storage-type"].value, @@ -54,10 +119,15 @@ def create_env(model_name, app_name, container_name, config, state): "STORAGE_BUCKET_STATE": config["storage-bucket-state"], "STORAGE_BUCKET_WORKLOAD_OUTPUT": config["storage-bucket-workload-output"], "STORAGE_BUCKET_ACTIVITY_PAYLOAD": config["storage-bucket-activity-payload"], - "LOG_LEVEL": config["log-level"].value, + # Database config + "DATABASE_URL": db_url, + "DATABASE_USER": db_conn["user"], + "DATABASE_PASSWORD": db_conn["password"], + "DATABASE_DB": db_name, + "DATABASE_HOST": host, + "DATABASE_PORT": port, "KEYCLOAK_DATABASE_URL": db_url + "?currentSchema=keycloak", "JOB_KUBE_SERVICEACCOUNT": app_name, - "JOB_KUBE_NAMESPACE": model_name, "RUNNING_TTL_MINUTES": config["pod-running-ttl-minutes"], "SUCCEEDED_TTL_MINUTES": config["pod-successful-ttl-minutes"], "UNSUCCESSFUL_TTL_MINUTES": config["pod-unsuccessful-ttl-minutes"], @@ -67,7 +137,6 @@ def create_env(model_name, app_name, container_name, config, state): "CONNECTOR_BUILDER_SERVER_API_HOST": f"{app_name}:{CONNECTOR_BUILDER_SERVER_API_PORT}", "CONNECTOR_BUILDER_API_HOST": f"{app_name}:{CONNECTOR_BUILDER_SERVER_API_PORT}", "AIRBYTE_API_HOST": f"{app_name}:{AIRBYTE_API_PORT}/api/public", - "WEBAPP_URL": config["webapp-url"], "AIRBYTE_URL": config["webapp-url"], } diff --git a/src/s3_helpers.py b/src/s3_helpers.py index 612f560..fb3a66b 100644 --- a/src/s3_helpers.py +++ b/src/s3_helpers.py @@ -51,7 +51,7 @@ def create_bucket_if_not_exists(self, bucket_name): s3_bucket = self.s3_resource.Bucket(bucket_name) try: s3_bucket.meta.client.head_bucket(Bucket=bucket_name) - logger.info("Bucket %s exists.", bucket_name) + logger.info("Bucket %s exists. Skipping creation.", bucket_name) exists = True except ClientError as e: error_code = int(e.response["Error"]["Code"]) diff --git a/src/structured_config.py b/src/structured_config.py index d5978d8..9d3f0c5 100644 --- a/src/structured_config.py +++ b/src/structured_config.py @@ -7,6 +7,7 @@ """Structured configuration for the charm.""" import logging +import re from enum import Enum from typing import Optional @@ -33,11 +34,84 @@ class StorageType(str, Enum): s3 = "S3" +class SecretPersistenceType(str, Enum): + """Enum for the `secret-persistence` field.""" + + GOOGLE_SECRET_MANAGER = "GOOGLE_SECRET_MANAGER" # nosec + AWS_SECRET_MANAGER = "AWS_SECRET_MANAGER" # nosec + TESTING_CONFIG_DB_TABLE = "TESTING_CONFIG_DB_TABLE" + VAULT = "VAULT" + + secret_persistence: Optional["SecretPersistenceType"] = None # Optional field + + +class VaultAuthType(str, Enum): + """Enum for the `vault-auth-method` field.""" + + token = "token" # nosec + + +class ImagePullPolicyType(str, Enum): + """Enum for the `*-image-pull-policy` field.""" + + Always = "Always" + IfNotPresent = "IfNotPresent" + Never = "Never" + + class CharmConfig(BaseConfigModel): """Manager for the structured configuration.""" log_level: LogLevelType temporal_host: str + webapp_url: Optional[str] + secret_persistence: Optional[SecretPersistenceType] + secret_store_gcp_project_id: Optional[str] + secret_store_gcp_credentials: Optional[str] + vault_address: Optional[str] + vault_prefix: Optional[str] + vault_auth_token: Optional[str] + vault_auth_method: VaultAuthType + aws_access_key: Optional[str] + aws_secret_access_key: Optional[str] + aws_kms_key_arn: Optional[str] + aws_secret_manager_secret_tags: Optional[str] + sync_job_retries_complete_failures_max_successive: Optional[int] + sync_job_retries_complete_failures_max_total: Optional[int] + sync_job_retries_complete_failures_backoff_min_interval_s: Optional[int] + sync_job_retries_complete_failures_backoff_max_interval_s: Optional[int] + sync_job_retries_complete_failures_backoff_base: Optional[int] + sync_job_retries_partial_failures_max_successive: Optional[int] + sync_job_retries_partial_failures_max_total: Optional[int] + sync_job_max_timeout_days: Optional[int] + job_main_container_cpu_request: Optional[str] + job_main_container_cpu_limit: Optional[str] + job_main_container_memory_request: Optional[str] + job_main_container_memory_limit: Optional[str] + max_fields_per_connections: Optional[int] + max_days_of_only_failed_jobs_before_connection_disable: Optional[int] + max_failed_jobs_in_a_row_before_connection_disable: Optional[int] + max_spec_workers: Optional[int] + max_check_workers: Optional[int] + max_sync_workers: Optional[int] + max_discover_workers: Optional[int] + temporal_history_retention_in_days: Optional[int] + job_kube_tolerations: Optional[str] + job_kube_node_selectors: Optional[str] + job_kube_annotations: Optional[str] + job_kube_main_container_image_pull_policy: Optional[ImagePullPolicyType] + job_kube_main_container_image_pull_secret: Optional[str] + job_kube_sidecar_container_image_pull_policy: Optional[ImagePullPolicyType] + job_kube_socat_image: Optional[str] + job_kube_busybox_image: Optional[str] + job_kube_curl_image: Optional[str] + job_kube_namespace: Optional[str] + spec_job_kube_node_selectors: Optional[str] + check_job_kube_node_selectors: Optional[str] + discover_job_kube_node_selectors: Optional[str] + spec_job_kube_annotations: Optional[str] + check_job_kube_annotations: Optional[str] + discover_job_kube_annotations: Optional[str] storage_type: StorageType storage_bucket_logs: str logs_ttl: int @@ -47,7 +121,6 @@ class CharmConfig(BaseConfigModel): pod_running_ttl_minutes: int pod_successful_ttl_minutes: int pod_unsuccessful_ttl_minutes: int - webapp_url: Optional[str] @validator("*", pre=True) @classmethod @@ -66,7 +139,7 @@ def blank_string(cls, value): @validator("pod_running_ttl_minutes", "pod_successful_ttl_minutes", "pod_unsuccessful_ttl_minutes") @classmethod - def pod_ttl_minutes_validator(cls, value: str) -> Optional[int]: + def greater_than_zero(cls, value: str) -> Optional[int]: """Check validity of `*-ttl-minutes` fields. Args: @@ -85,7 +158,7 @@ def pod_ttl_minutes_validator(cls, value: str) -> Optional[int]: @validator("logs_ttl") @classmethod - def logs_ttl_validator(cls, value: str) -> Optional[int]: + def zero_or_greater(cls, value: str) -> Optional[int]: """Check validity of `logs-ttl` fields. Args: @@ -101,3 +174,52 @@ def logs_ttl_validator(cls, value: str) -> Optional[int]: if int_value >= 0: return int_value raise ValueError("Value out of range.") + + @validator("job_main_container_cpu_request", "job_main_container_cpu_limit") + @classmethod + def cpu_validator(cls, value: str) -> Optional[str]: + """Check validity of `*-cpu-request/limit` fields. + + Args: + value: CPU request/limit value + + Returns: + value: CPU request/limit value + + Raises: + ValueError: in the case when the value is invalid + """ + millicores_pattern = re.compile(r"^\d+m$") + + if millicores_pattern.match(value): + return value + + int_value = int(value) + if int_value > 0: + return value + raise ValueError("Invalid CPU request/limit value.") + + @validator("job_main_container_memory_request", "job_main_container_memory_limit") + @classmethod + def memory_validator(cls, value: str) -> Optional[str]: + """Check validity of `*-memory-request/limit` fields. + + Args: + value: Memory request/limit value + + Returns: + value: Memory request/limit value + + Raises: + ValueError: in the case when the value is invalid + """ + memory_pattern = re.compile(r"^[1-9]\d*(Ei|Pi|Ti|Gi|Mi|Ki)?$") + + if memory_pattern.match(value): + return value + + # Check if the input is a valid integer (bytes) + int_value = int(value) + if int_value > 0: + return value + raise ValueError("Invalid CPU request/limit value.") diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index ca13a03..ccd1837 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -336,40 +336,63 @@ def create_plan(container_name, storage_type): "override": "replace", "environment": { **BASE_ENV, + "AIRBYTE_API_HOST": "airbyte-k8s:8006/api/public", + "AIRBYTE_SERVER_HOST": "airbyte-k8s:8001", + "AIRBYTE_URL": "http://airbyte-ui-k8s:8080", "AWS_ACCESS_KEY_ID": "access", "AWS_SECRET_ACCESS_KEY": "secret", + "CONFIG_API_HOST": "airbyte-k8s:8001", + "CONNECTOR_BUILDER_API_HOST": "airbyte-k8s:80", + "CONNECTOR_BUILDER_API_URL": "/connector-builder-api", + "CONNECTOR_BUILDER_SERVER_API_HOST": "airbyte-k8s:80", "DATABASE_DB": "airbyte-k8s_db", "DATABASE_HOST": "myhost", "DATABASE_PASSWORD": "inner-light", "DATABASE_PORT": "5432", "DATABASE_URL": "jdbc:postgresql://myhost:5432/airbyte-k8s_db", "DATABASE_USER": "jean-luc@db", - "JOB_KUBE_NAMESPACE": MODEL_NAME, - "JOB_KUBE_SERVICEACCOUNT": APP_NAME, + "INTERNAL_API_HOST": "airbyte-k8s:8001", + "JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION": "0.29.15.001", + "JOB_KUBE_MAIN_CONTAINER_IMAGE_PULL_POLICY": "IfNotPresent", + "JOB_KUBE_NAMESPACE": "airbyte-model", + "JOB_KUBE_SERVICEACCOUNT": "airbyte-k8s", + "JOB_KUBE_SIDECAR_CONTAINER_IMAGE_PULL_POLICY": "IfNotPresent", "KEYCLOAK_DATABASE_URL": "jdbc:postgresql://myhost:5432/airbyte-k8s_db?currentSchema=keycloak", + "KEYCLOAK_INTERNAL_HOST": "localhost", "LOG_LEVEL": "INFO", + "MAX_CHECK_WORKERS": 5, + "MAX_DAYS_OF_ONLY_FAILED_JOBS_BEFORE_CONNECTION_DISABLE": 14, + "MAX_DISCOVER_WORKERS": 5, + "MAX_FAILED_JOBS_IN_A_ROW_BEFORE_CONNECTION_DISABLE": 20, + "MAX_FIELDS_PER_CONNECTION": 20000, + "MAX_SPEC_WORKERS": 5, + "MAX_SYNC_WORKERS": 5, "RUNNING_TTL_MINUTES": 240, "S3_LOG_BUCKET": "airbyte-dev-logs", + "SHOULD_RUN_NOTIFY_WORKFLOWS": "true", "STORAGE_BUCKET_ACTIVITY_PAYLOAD": "airbyte-payload-storage", "STORAGE_BUCKET_LOG": "airbyte-dev-logs", "STORAGE_BUCKET_STATE": "airbyte-state-storage", "STORAGE_BUCKET_WORKLOAD_OUTPUT": "airbyte-state-storage", "STORAGE_TYPE": storage_type, "SUCCEEDED_TTL_MINUTES": 30, + "SYNC_JOB_MAX_TIMEOUT_DAYS": 3, + "SYNC_JOB_RETRIES_COMPLETE_FAILURES_BACKOFF_BASE": 3, + "SYNC_JOB_RETRIES_COMPLETE_FAILURES_BACKOFF_MAX_INTERVAL_S": 1800, + "SYNC_JOB_RETRIES_COMPLETE_FAILURES_BACKOFF_MIN_INTERVAL_S": 10, + "SYNC_JOB_RETRIES_COMPLETE_FAILURES_MAX_SUCCESSIVE": 5, + "SYNC_JOB_RETRIES_COMPLETE_FAILURES_MAX_TOTAL": 10, + "SYNC_JOB_RETRIES_PARTIAL_FAILURES_MAX_SUCCESSIVE": 1000, + "SYNC_JOB_RETRIES_PARTIAL_FAILURES_MAX_TOTAL": 20, + "TEMPORAL_HISTORY_RETENTION_IN_DAYS": 30, "TEMPORAL_HOST": "temporal-k8s:7233", + "TEMPORAL_WORKER_PORTS": "9001,9002,9003,9004,9005,9006,9007,9008,9009,9010,9011,9012,9013,9014,9015,9016,9017,9018,9019,9020,9021,9022,9023,9024,9025,9026,9027,9028,9029,9030", "UNSUCCESSFUL_TTL_MINUTES": 1440, + "VAULT_AUTH_METHOD": "token", "WEBAPP_URL": "http://airbyte-ui-k8s:8080", - "AIRBYTE_URL": "http://airbyte-ui-k8s:8080", - "WORKERS_MICRONAUT_ENVIRONMENTS": "control-plane", - "WORKER_ENVIRONMENT": "kubernetes", "WORKER_LOGS_STORAGE_TYPE": storage_type, "WORKER_STATE_STORAGE_TYPE": storage_type, - "AIRBYTE_API_HOST": "airbyte-k8s:8006/api/public", - "AIRBYTE_SERVER_HOST": "airbyte-k8s:8001", - "CONFIG_API_HOST": "airbyte-k8s:8001", - "CONNECTOR_BUILDER_API_HOST": "airbyte-k8s:80", - "CONNECTOR_BUILDER_SERVER_API_HOST": "airbyte-k8s:80", - "INTERNAL_API_HOST": "airbyte-k8s:8001", + "WORKLOAD_API_HOST": "localhost", }, }, }, diff --git a/tests/unit/test_structured_config.py b/tests/unit/test_structured_config.py index cf7bd97..c0bda66 100644 --- a/tests/unit/test_structured_config.py +++ b/tests/unit/test_structured_config.py @@ -47,6 +47,22 @@ def test_product_related_values(_harness) -> None: check_valid_values(_harness, "storage-type", accepted_values) +def test_cpu_related_values(_harness) -> None: + """Test specific parameters for each field.""" + erroneus_values = ["-123", "0", "100f"] + check_invalid_values(_harness, "job-main-container-cpu-limit", erroneus_values) + accepted_values = ["200m", "4"] + check_valid_values(_harness, "job-main-container-cpu-limit", accepted_values) + + +def test_memory_related_values(_harness) -> None: + """Test specific parameters for each field.""" + erroneus_values = ["-123", "0", "100f"] + check_invalid_values(_harness, "job-main-container-memory-limit", erroneus_values) + accepted_values = ["4Gi", "256Mi"] + check_valid_values(_harness, "job-main-container-memory-limit", accepted_values) + + def check_valid_values(_harness, field: str, accepted_values: list) -> None: """Check the correctness of the passed values for a field.