IT-4241: create documentDB instance and allow connection from ECS API…

… container

app.py

@@ -1,12 +1,15 @@
 from os import environ
 
 import aws_cdk as cdk
+from aws_cdk import aws_ec2 as ec2
 
 from src.ecs_stack import EcsStack
 from src.load_balancer_stack import LoadBalancerStack
 from src.network_stack import NetworkStack
 from src.service_props import ServiceProps, ServiceSecret
 from src.service_stack import LoadBalancedServiceStack, ServiceStack
+from src.docdb_props import DocdbProps
+from src.docdb_stack import DocdbStack
 
 # get the environment and set environment specific variables
 VALID_ENVIRONMENTS = ["dev", "stage", "prod"]
@@ -43,6 +46,7 @@
 fully_qualified_domain_name = environment_variables["FQDN"]
 environment_tags = environment_variables["TAGS"]
 agora_version = "4.0.0-rc1"
+docdb_master_username = "master"
 
 # Define stacks
 cdk_app = cdk.App()
@@ -58,6 +62,19 @@
     vpc_cidr=environment_variables["VPC_CIDR"],
 )
 
+docdb_props = DocdbProps(
+    instance_type=ec2.InstanceType.of(
+        ec2.InstanceClass.MEMORY5, ec2.InstanceSize.LARGE
+    ),
+    master_username=docdb_master_username,
+)
+docdb_stack = DocdbStack(
+    scope=cdk_app,
+    construct_id=f"{stack_name_prefix}-docdb",
+    vpc=network_stack.vpc,
+    props=docdb_props,
+)
+
 ecs_stack = EcsStack(
     scope=cdk_app,
     construct_id=f"{stack_name_prefix}-ecs",
@@ -84,21 +101,16 @@
         "NODE_ENV": "development",
         "MONGODB_PORT": "27017",
         "MONGODB_NAME": "agora",
+        "MONDODB_USER": docdb_master_username,
+        "MONGODB_HOST": docdb_stack.cluster.cluster_endpoint.hostname,
     },
     container_secrets=[
         ServiceSecret(
-            secret_name=f"{stack_name_prefix}/MongodbUsername",
-            environment_key="MONGODB_USER",
-        ),
-        ServiceSecret(
-            secret_name=f"{stack_name_prefix}/MongodbPassword",
+            secret_name=docdb_stack.master_password_secret.secret_name,
             environment_key="MONGODB_PASS",
-        ),
-        ServiceSecret(
-            secret_name=f"{stack_name_prefix}/MongodbHost",
-            environment_key="MONGODB_HOST",
-        ),
+        )
     ],
+    container_security_groups=[docdb_stack.access_docdb_security_group],
 )
 api_stack = ServiceStack(
     scope=cdk_app,
@@ -107,6 +119,7 @@
     cluster=ecs_stack.cluster,
     props=api_props,
 )
+api_stack.add_dependency(docdb_stack)
 
 app_props = ServiceProps(
     container_name="agora-app",
@@ -118,7 +131,6 @@
         "APP_VERSION": f"{agora_version}",
         "CSR_API_URL": f"http://{fully_qualified_domain_name}/api/v1",
         "SSR_API_URL": "http://agora-api:3333/v1",
-        "ROLLBAR_TOKEN": "e788198867474855a996485580b08d03",
         "TAG_NAME": f"agora/v${agora_version}",
     },
 )

requirements.txt

@@ -1,3 +1,3 @@
-aws-cdk-lib~=2.139
+aws-cdk-lib~=2.176
 constructs~=10.0
 boto3~=1.34

src/docdb_props.py

@@ -0,0 +1,14 @@
+from aws_cdk import aws_ec2 as ec2
+
+
+class DocdbProps:
+    """
+    DocumentDB properties
+
+    instance_type: What type of instance to start for the replicas
+    master_username: The database admin account username
+    """
+
+    def __init__(self, instance_type: ec2.InstanceType, master_username: str) -> None:
+        self.instance_type = instance_type
+        self.master_username = master_username

src/docdb_stack.py

@@ -0,0 +1,94 @@
+import aws_cdk as cdk
+from aws_cdk import (
+    aws_docdb as docdb,
+    aws_ec2 as ec2,
+    aws_secretsmanager as sm,
+)
+from src.docdb_props import DocdbProps
+
+from constructs import Construct
+
+
+class DocdbStack(cdk.Stack):
+    """
+    DocumentDB cluster
+    """
+
+    def __init__(
+        self,
+        scope: Construct,
+        construct_id: str,
+        vpc: ec2.Vpc,
+        props: DocdbProps,
+        **kwargs,
+    ) -> None:
+        super().__init__(scope, construct_id, **kwargs)
+
+        self.master_password_secret = sm.Secret(
+            self,
+            f"{construct_id}-master-password",
+            generate_secret_string=sm.SecretStringGenerator(
+                password_length=32, exclude_punctuation=True
+            ),
+        )
+
+        self.access_docdb_security_group = ec2.SecurityGroup(
+            self,
+            f"{construct_id}-access-docdb-sg",
+            vpc=vpc,
+            description="Instances with access to document DB servers",
+        )
+        self.docdb_security_group = ec2.SecurityGroup(
+            self,
+            f"{construct_id}-docdb-sg",
+            vpc=vpc,
+            description="Document DB server management and access ports",
+        )
+        self.docdb_security_group.add_ingress_rule(
+            peer=self.access_docdb_security_group,
+            connection=ec2.Port.tcp_range(27017, 27030),
+        )
+        self.docdb_security_group.add_ingress_rule(
+            peer=self.access_docdb_security_group, connection=ec2.Port.tcp(28017)
+        )
+
+        cluster_parameter_group = docdb.ClusterParameterGroup(
+            self,
+            "ClusterParameterGroup",
+            family="docdb5.0",
+            parameters={
+                "audit_logs": "disabled",
+                "audit_logs": "disabled",
+                "profiler": "enabled",
+                "profiler_sampling_rate": "1.0",
+                "profiler_threshold_ms": "50",
+                "change_stream_log_retention_duration": "10800",
+                "tls": "disabled",
+                "ttl_monitor": "disabled",
+            },
+            db_cluster_parameter_group_name=f"{construct_id}-cluster-parameter-group",
+        )
+
+        # https://docs.aws.amazon.com/cdk/api/v2/python/aws_cdk.aws_docdb/DatabaseCluster.html
+        self.cluster = docdb.DatabaseCluster(
+            self,
+            "Database",
+            master_user=docdb.Login(
+                username=props.master_username,
+                password=self.master_password_secret.secret_value,
+            ),
+            instance_type=props.instance_type,
+            vpc=vpc,
+            vpc_subnets=ec2.SubnetSelection(
+                subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS
+            ),
+            parameter_group=cluster_parameter_group,
+            removal_policy=cdk.RemovalPolicy.DESTROY,
+            storage_encrypted=True,
+            preferred_maintenance_window="sat:06:54-sat:07:24",
+            port=27017,
+            export_profiler_logs_to_cloud_watch=True,
+            security_group=self.docdb_security_group,
+        )
+
+        self.cluster.add_security_groups(self.access_docdb_security_group)

src/service_props.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from typing import List, Optional, Sequence
 
-from aws_cdk import aws_ecs as ecs
+from aws_cdk import aws_ecs as ecs, aws_ec2 as ec2
 
 CONTAINER_LOCATION_PATH_ID = "path://"
 
@@ -62,6 +62,7 @@ class ServiceProps:
     auto_scale_max_capacity: the fargate auto scaling maximum capacity
     container_command: Optional commands to run during the container startup
     container_healthcheck: Optional health check configuration for the container
+    container_security_groups: Optional List of security groups for the container
     """
 
     def __init__(
@@ -77,6 +78,7 @@ def __init__(
         auto_scale_max_capacity: int = 1,
         container_command: Optional[Sequence[str]] = None,
         container_healthcheck: Optional[ecs.HealthCheck] = None,
+        container_security_groups: Optional[List[ec2.SecurityGroup]] = None,
     ) -> None:
         self.container_name = container_name
         self.container_port = container_port
@@ -106,3 +108,8 @@ def __init__(
         self.auto_scale_max_capacity = auto_scale_max_capacity
         self.container_command = container_command
         self.container_healthcheck = container_healthcheck
+
+        if container_security_groups is None:
+            self.container_security_groups = []
+        else:
+            self.container_security_groups = container_security_groups

src/service_stack.py

@@ -132,6 +132,8 @@ def _get_secret(scope: Construct, id: str, name: str) -> sm.Secret:
             peer=ec2.Peer.ipv4("0.0.0.0/0"),
             connection=ec2.Port.tcp(props.container_port),
         )
+        self.security_groups = props.container_security_groups.copy()
+        self.security_groups.append(self.security_group)
 
         # attach ECS task to ECS cluster
         self.service = ecs.FargateService(
@@ -141,7 +143,7 @@ def _get_secret(scope: Construct, id: str, name: str) -> sm.Secret:
             task_definition=self.task_definition,
             enable_execute_command=True,
             circuit_breaker=ecs.DeploymentCircuitBreaker(enable=True, rollback=True),
-            security_groups=([self.security_group]),
+            security_groups=self.security_groups,
             service_connect_configuration=ecs.ServiceConnectProps(
                 log_driver=ecs.LogDrivers.aws_logs(stream_prefix=f"{construct_id}"),
                 services=[

tests/unit/test_docdb_stack.py

@@ -0,0 +1,62 @@
+import aws_cdk as cdk
+from aws_cdk import aws_ec2 as ec2, assertions as assertions
+
+from src.network_stack import NetworkStack
+from src.docdb_stack import DocdbStack
+from src.docdb_props import DocdbProps
+
+
+def test_docdb_created():
+    cdk_app = cdk.App()
+    master_username = "myuser"
+    vpc_cidr = "10.254.192.0/24"
+    network_stack = NetworkStack(cdk_app, "NetworkStack", vpc_cidr=vpc_cidr)
+
+    docdb_props = DocdbProps(
+        instance_type=ec2.InstanceType.of(
+            ec2.InstanceClass.MEMORY5, ec2.InstanceSize.LARGE
+        ),
+        master_username=master_username,
+    )
+    docdb_stack = DocdbStack(
+        scope=cdk_app,
+        construct_id="docdb",
+        vpc=network_stack.vpc,
+        props=docdb_props,
+    )
+
+    template = assertions.Template.from_stack(docdb_stack)
+    template.has_resource_properties(
+        "AWS::DocDB::DBClusterParameterGroup",
+        {
+            "Parameters": {
+                "audit_logs": "disabled",
+                "profiler": "enabled",
+                "profiler_sampling_rate": "1.0",
+                "profiler_threshold_ms": "50",
+                "change_stream_log_retention_duration": "10800",
+                "tls": "disabled",
+                "ttl_monitor": "disabled",
+            }
+        },
+    )
+    template.has_resource_properties(
+        "AWS::DocDB::DBCluster",
+        {
+            "MasterUsername": master_username,
+            "MasterUserPassword": assertions.Match.any_value(),
+            "DBSubnetGroupName": assertions.Match.any_value(),
+            "DBClusterParameterGroupName": assertions.Match.any_value(),
+            "StorageEncrypted": True,
+            "PreferredMaintenanceWindow": "sat:06:54-sat:07:24",
+            "Port": 27017,
+            "EnableCloudwatchLogsExports": ["profiler"],
+            "VpcSecurityGroupIds": [
+                assertions.Match.any_value(),
+                assertions.Match.any_value(),
+            ],
+        },
+    )
+    template.resource_properties_count_is(
+        "AWS::EC2::SecurityGroup", assertions.Match.any_value(), 2
+    )

tools/setup.sh

@@ -4,7 +4,7 @@
 set -euxo pipefail
 
 # Install Node.js dependencies
-npm install -g aws-cdk@2.151.0
+npm install -g aws-cdk@2.176.0
 
 # Install Python dependencies
 python -m venv .venv