DBTP-457 Add DBT PaaS CodeBuild configuration (#464)

Co-authored-by: Anthony Roy <[email protected]>
Co-authored-by: Lawrence Goldstien <[email protected]>
Co-authored-by: Yusuf <[email protected]>
Co-authored-by: Cameron Lamb <[email protected]>
Co-authored-by: Anthony Roy <[email protected]>

README.md

@@ -3,3 +3,4 @@
 A [Wagtail](https://www.wagtail.io)-based intranet for the Department for Business & Trade.
 
 Project documentation is available [here](https://uktrade.github.io/digital-workspace-v2/).
+

buildpack.json

@@ -0,0 +1,13 @@
+{
+  "buildpacks": [
+    {
+      "paketo-buildpacks": "python"
+    },
+    {
+      "paketo-buildpacks": "nodejs"
+    },
+    {
+      "fagiani" : "run"
+    }
+  ]
+}

codebuild/process.yml

@@ -0,0 +1,12 @@
+application:
+  name: intranet
+  process:
+    # The way DBT PaaS works currently we need one entry
+    # here for each of the entries in the Procfile.
+    # The only actual different in the three images created
+    # is the process they start up with.
+    # There is a plan ensure it only needs to build one
+    # image in the near future.
+    - web
+    - beat # celery beat
+    - worker # celery worker

codebuild/web/buildspec.yml

@@ -0,0 +1,20 @@
+version: 0.2
+env:
+  parameter-store:
+    SLACK_WORKSPACE_ID: "/codebuild/slack_workspace_id"
+    SLACK_CHANNEL_ID: "/codebuild/slack_channel_id"
+    SLACK_TOKEN: "/codebuild/slack_api_token"
+  variables:
+    PAKETO_BUILDER_VERSION: 0.2.443-full
+    LIFECYCLE_VERSION: 0.16.5
+
+phases:
+  # install:
+
+  pre_build:
+    commands:
+      - codebuild-breakpoint
+
+  build:
+    commands:
+      - /work/build.sh

docker-compose.yml

@@ -73,7 +73,7 @@ services:
       - opensearch
       - celery
     environment:
-      - DJANGO_SETTINGS_MODULE=config.settings.local
+      - DJANGO_SETTINGS_MODULE=config.settings.developer
     stdin_open: true
     tty: true
 
@@ -86,12 +86,12 @@ services:
       - db
       - redis
     environment:
-      - DJANGO_SETTINGS_MODULE=config.settings.local
+      - DJANGO_SETTINGS_MODULE=config.settings.developer
 
   celery-beat:
     image: digital_workspace/wagtail:latest
     environment:
-      - DJANGO_SETTINGS_MODULE=config.settings.local
+      - DJANGO_SETTINGS_MODULE=config.settings.developer
     volumes:
       - .:/app
     profiles:

src/config/settings/base.py

@@ -8,6 +8,7 @@
 from dbt_copilot_python.network import setup_allowed_hosts
 from dbt_copilot_python.utility import is_copilot
 from django.urls import reverse_lazy
+from django_log_formatter_ecs import ECSFormatter
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
 
@@ -25,7 +26,7 @@
 VCAP_SERVICES = env.json("VCAP_SERVICES", {})
 
 # Set required configuration from environment
-# Should be one of the following: "local", "test", "dev", "staging", "training", "prod"
+# Should be one of the following: "local", "test", "dev", "staging", "training", "prod", "build"
 APP_ENV = env.str("APP_ENV", "local")
 GIT_COMMIT = env.str("GIT_COMMIT", None)
 
@@ -47,16 +48,14 @@
     AWS_REGION = app_bucket_creds["aws_region"]
     AWS_S3_REGION_NAME = app_bucket_creds["aws_region"]
     AWS_STORAGE_BUCKET_NAME = app_bucket_creds["bucket_name"]
+    AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID")
+    AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
+    AWS_S3_HOST = "s3-eu-west-2.amazonaws.com"
 else:
     AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
     AWS_REGION = env("AWS_REGION")
     AWS_S3_REGION_NAME = env("AWS_REGION", default="eu-west-2")
 
-# You don't seem to be able to sign S3 URLs with VCAP S3 creds
-AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID")
-AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
-AWS_S3_HOST = "s3-eu-west-2.amazonaws.com"
-
 # Asset path used in parser
 NEW_ASSET_PATH = env("NEW_ASSET_PATH")
 
@@ -432,6 +431,10 @@
         credentials["host"],
         credentials["port"],
     )
+elif is_copilot():
+    CELERY_BROKER_URL = (
+        env("CELERY_BROKER_URL", default=None) + "?ssl_cert_reqs=required"
+    )
 else:
     CELERY_BROKER_URL = env("CELERY_BROKER_URL", default=None)
 
@@ -482,12 +485,23 @@
     "version": 1,
     "disable_existing_loggers": False,
     "formatters": {
+        "ecs_formatter": {
+            "()": ECSFormatter,
+        },
         "simple": {
             "format": "{asctime} {levelname} {message}",
             "style": "{",
         },
     },
     "handlers": {
+        "ecs": {
+            "class": "logging.StreamHandler",
+            "formatter": "ecs_formatter",
+        },
+        "simple": {
+            "class": "logging.StreamHandler",
+            "formatter": "simple",
+        },
         "stdout": {
             "class": "logging.StreamHandler",
             "stream": sys.stdout,
@@ -501,6 +515,8 @@
     "loggers": {
         "django": {
             "handlers": [
+                "ecs",
+                "simple",
                 "stdout",
             ],
             "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"),
@@ -515,13 +531,17 @@
         },
         "django.server": {
             "handlers": [
+                "ecs",
+                "simple",
                 "stdout",
             ],
             "level": os.getenv("DJANGO_SERVER_LOG_LEVEL", "INFO"),
             "propagate": False,
         },
         "django.db.backends": {
             "handlers": [
+                "ecs",
+                "simple",
                 "stdout",
             ],
             "level": os.getenv("DJANGO_DB_LOG_LEVEL", "INFO"),

src/config/settings/build.py

@@ -0,0 +1,3 @@
+from .base import *  # noqa
+
+APP_ENV = "build"

src/config/settings/local.py → src/config/settings/developer.py

@@ -1,6 +1,5 @@
 from .base import *  # noqa
 
-
 CAN_ELEVATE_SSO_USER_PERMISSIONS = True
 
 # Set to True if you need to upload documents and you are not running

src/config/settings/gov_paas.py

@@ -0,0 +1,13 @@
+from .prod import *  # noqa F403
+
+INSTALLED_APPS += [  # noqa F405
+    "elasticapm.contrib.django",
+]
+
+ELASTIC_APM = {
+    "SERVICE_NAME": "Digital Workspace",
+    "SECRET_TOKEN": env("ELASTIC_APM_SECRET_TOKEN"),  # noqa F405
+    "SERVER_URL": env("ELASTIC_APM_SERVER_URL"),  # noqa F405
+    "ENVIRONMENT": env("APP_ENV"),  # noqa F405
+    "SERVER_TIMEOUT": env("ELASTIC_APM_SERVER_TIMEOUT", default="20s"),  # noqa F405
+}

src/config/settings/prod.py

@@ -1 +1,27 @@
-from .env import *  # noqa
+from .base import *  # noqa
+
+DEBUG = False
+
+AWS_S3_URL_PROTOCOL = "https:"
+AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN")  # noqa F405
+AWS_QUERYSTRING_AUTH = False
+
+SESSION_COOKIE_AGE = 60 * 60
+
+SECURE_BROWSER_XSS_FILTER = True
+X_FRAME_OPTIONS = "DENY"
+SECURE_CONTENT_TYPE_NOSNIFF = True
+SECURE_HSTS_SECONDS = 15768000
+SECURE_HSTS_INCLUDE_SUBDOMAINS = True
+SECURE_HSTS_PRELOAD = True
+SECURE_SSL_REDIRECT = True
+CSRF_COOKIE_SECURE = True
+SESSION_COOKIE_SECURE = True
+SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+LOGGING["root"]["handlers"] = [  # noqa F405
+    "ecs",
+    "simple",
+]
+LOGGING["loggers"]["django"]["propagate"] = False  # noqa F405
+LOGGING["loggers"]["django.db.backends"]["propagate"] = False  # noqa F405

src/config/settings/test.py

@@ -2,7 +2,6 @@
 
 from .base import *  # noqa
 
-
 APP_ENV = "test"
 DEBUG = True
 TEMPLATE_DEBUG = True
@@ -18,10 +17,6 @@
     "AUTO_UPDATE": False,
 }
 
-INSTALLED_APPS += [  # noqa F405
-    "django_extensions",
-]
-
 LOGGING["handlers"] |= {  # noqa F405
     "file": {
         "level": "DEBUG",

src/extended_search/apps.py

@@ -11,5 +11,5 @@ def ready(self):
 
         search_settings.initialise_field_dict()
         search_settings.initialise_env_dict()
-        if settings.APP_ENV != "test":
+        if settings.APP_ENV not in ["test", "build"]:
             search_settings.initialise_db_dict()

user-post.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+# Exit early if something goes wrong
+set -e
+
+echo "Running post build script"
+
+echo "Running pip install"
+pip install -r requirements.txt
+
+echo "Running npm ci"
+npm ci
+
+echo "Renaming .env.ci to .env"
+mv ".env.ci" ".env"
+
+cd src
+
+echo "Running collectstatic"
+python manage.py collectstatic --settings=config.settings.build --noinput
+
+echo "Renaming .env to .env.ci"
+cd ../
+mv ".env" ".env.ci"