From 0152fb18b9269e4acd1e9aef292b678db91838fb Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 27 Sep 2024 09:40:54 -0700 Subject: [PATCH 01/10] Dummy change to allow the PR to be opened. --- database/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/README.md b/database/README.md index cb1c2984c9..e16720a47a 100644 --- a/database/README.md +++ b/database/README.md @@ -6,7 +6,7 @@ | ---------- | ------- | ------------------------------------ | -------------------- | | node | 18.x.x | https://nodejs.org/en/ | JavaScript Runtime | | npm | 10.x.x | https://www.npmjs.com/ | Node Package Manager | -| PostgreSQL | 12.5 | https://www.postgresql.org/download/ | PSQL database | +| PostgreSQL | 14.2 | https://www.postgresql.org/download/ | PSQL database | | PostGIS | 3 | https://postgis.net/ | GIS (spatial) tools |
From 31fa8d9bce72582d809c80252c85ad4fada168e9 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Tue, 22 Oct 2024 16:32:21 -0700 Subject: [PATCH 02/10] WIP - Experiment --- database/.docker/db/Dockerfile | 2 +- database/.pipeline/templates/db.bc.yaml | 4 +- database/.pipeline/templates/db.dc.yaml | 35 +++---- .../prereqs/postgres-postgis.is.yaml | 91 +++++++++++++++++++ env_config/env.docker | 2 +- 5 files changed, 106 insertions(+), 28 deletions(-) create mode 100644 database/.pipeline/templates/prereqs/postgres-postgis.is.yaml diff --git a/database/.docker/db/Dockerfile b/database/.docker/db/Dockerfile index effd413ed7..a4bd1dee86 100644 --- a/database/.docker/db/Dockerfile +++ b/database/.docker/db/Dockerfile @@ -2,7 +2,7 @@ # This DockerFile is used for local development (via compose.yml) only. # ######################################################################################################## -ARG POSTGRES_VERSION=12.5 +ARG POSTGRES_VERSION=15-bullseye FROM postgres:$POSTGRES_VERSION diff --git a/database/.pipeline/templates/db.bc.yaml b/database/.pipeline/templates/db.bc.yaml index 15bf4cbf7d..8205cd6377 100644 --- a/database/.pipeline/templates/db.bc.yaml +++ b/database/.pipeline/templates/db.bc.yaml @@ -1,7 +1,7 @@ kind: Template apiVersion: template.openshift.io/v1 metadata: - name: postgresql null + name: postgresql parameters: - name: NAME displayName: Name @@ -29,7 +29,7 @@ objects: annotations: from: kind: ImageStreamTag - name: postgis-postgres:12-31 + name: postgres-postgis:15-bullseye namespace: af2668-tools generation: 1 importPolicy: {} diff --git a/database/.pipeline/templates/db.dc.yaml b/database/.pipeline/templates/db.dc.yaml index e4b567fed5..55f4ca7e23 100644 --- a/database/.pipeline/templates/db.dc.yaml +++ b/database/.pipeline/templates/db.dc.yaml @@ -12,14 +12,18 @@ parameters: name: NAME required: true value: 'postgresql' - - description: The OpenShift ImageStream name. - displayName: IMAGE_STREAM_NAME - name: IMAGE_STREAM_NAME - value: 'postgis-postgres' - - description: The OpenShift Namespace where the ImageStream resides. - displayName: Namespace - name: IMAGE_STREAM_NAMESPACE + - name: IMAGE_STREAM_NAMESPACE + description: The OpenShift Namespace where the ImageStream resides. + required: true value: 'af2668-tools' + - name: IMAGE_STREAM_NAME + description: The OpenShift ImageStream name. + required: true + value: 'postgis-postgres' + - name: IMAGE_STREAM_VERSION + description: Version of PostgreSQL image to be used (9.2, 9.4, 9.5 or latest). + required: true + value: '15-bullseye' - description: The name of the OpenShift Service exposed for the database. displayName: Database Service Name name: DATABASE_SERVICE_NAME @@ -61,19 +65,6 @@ parameters: name: VOLUME_CAPACITY required: true value: '500Mi' - - description: Version of PostgreSQL image to be used (9.2, 9.4, 9.5 or latest). - displayName: Version of PostgreSQL Image - name: IMAGE_STREAM_VERSION - required: true - value: '12-31' - - description: Indicator to enable pgcrypto extension (provided out-of-the-box with PostgreSQL). - displayName: Flag to enable pgcrypto PostgreSQL extension. - name: PGCRYPTO_EXTENSION - value: 'Y' - - description: Indicator to enable postgis extension (from official PostgeSQL YUM repo). - displayName: Flag to enable postgis PostgreSQL extension. - name: POSTGIS_EXTENSION - value: 'Y' - name: CPU_REQUEST value: '50m' - name: CPU_LIMIT @@ -185,10 +176,6 @@ objects: secretKeyRef: key: database-name name: '${DATABASE_SERVICE_NAME}' - - name: PGCRYPTO_EXTENSION - value: 'N' - - name: POSTGIS_EXTENSION - value: 'N' - name: PGOPTIONS value: '-c maintenance_work_mem=128MB' - name: PGTZ diff --git a/database/.pipeline/templates/prereqs/postgres-postgis.is.yaml b/database/.pipeline/templates/prereqs/postgres-postgis.is.yaml new file mode 100644 index 0000000000..e310a33759 --- /dev/null +++ b/database/.pipeline/templates/prereqs/postgres-postgis.is.yaml @@ -0,0 +1,91 @@ +# ######################################################################################################## +# Creates a build config that builds the database dockerfile +# Creates an imagestream that references the built database image +# ######################################################################################################## + +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: postgresql +parameters: + # Imagestream + - name: IMAGE_STREAM_NAMESPACE + value: af2668-tools + - name: IMAGE_STREAM_NAME + value: postgres-postgis + - name: IMAGE_STREAM_VERSION + value: 15-bullseye + # Dockerfile + - name: SOURCE_CONTEXT_DIR + value: 'database' + - name: SOURCE_REPOSITORY_REF + value: dev + - name: SOURCE_REPOSITORY_URL + value: https://github.com/bcgov/biohubbc.git + - name: DOCKER_FILE_PATH + value: 'docker/db/Dockerfile' + # Resources + - name: CPU_REQUEST + value: '50m' + - name: CPU_LIMIT + value: '200m' + - name: MEMORY_REQUEST + value: '100Mi' + - name: MEMORY_LIMIT + value: '2Gi' +objects: + - kind: ImageStream + apiVersion: image.openshift.io/v1 + metadata: + name: '${NAME}' + labels: + shared: 'true' + spec: + lookupPolicy: + local: false + tags: + - name: '${TAG_NAME}' + annotations: + from: + kind: ImageStreamTag + name: '${IMAGE_STREAM_NAME}:${IMAGE_STREAM_VERSION}' + namespace: ${IMAGE_STREAM_NAMESPACE} + generation: 1 + importPolicy: {} + referencePolicy: + type: Source + + - kind: BuildConfig + apiVersion: v1 + metadata: + name: '${NAME}' + spec: + failedBuildsHistoryLimit: 5 + nodeSelector: + output: + to: + kind: ImageStreamTag + name: '${IMAGE_STREAM_NAME}:${IMAGE_STREAM_VERSION}' + postCommit: {} + resources: + requests: + cpu: ${CPU_REQUEST} + memory: ${MEMORY_REQUEST} + limits: + cpu: ${CPU_LIMIT} + memory: ${MEMORY_LIMIT} + runPolicy: SerialLatestOnly + source: + type: Git + git: + uri: '${SOURCE_REPOSITORY_URL}' + ref: '${SOURCE_REPOSITORY_REF}' + contextDir: '${SOURCE_CONTEXT_DIR}' + strategy: + type: Docker + dockerStrategy: + dockerfilePath: ${DOCKER_FILE_PATH} + successfulBuildsHistoryLimit: 5 + triggers: + - type: ConfigChange + - type: ImageChange diff --git a/env_config/env.docker b/env_config/env.docker index b78a07268b..6601156967 100644 --- a/env_config/env.docker +++ b/env_config/env.docker @@ -126,7 +126,7 @@ CB_API_HOST=https://moe-critterbase-api-dev.apps.silver.devops.gov.bc.ca/api # # See `biohubbc-creds` secret in openshift # ------------------------------------------------------------------------------ -POSTGRES_VERSION=12.5 +POSTGRES_VERSION=15-bullseye POSTGIS_VERSION=3 DB_HOST=db DB_ADMIN=postgres From bad87d35f3679784b6ce7b0c8acddd6e09d5a968 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Tue, 22 Oct 2024 16:33:40 -0700 Subject: [PATCH 03/10] ignore-skip From 581716afa9995825f66c2a2baa2c5f627c034d11 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 24 Oct 2024 11:03:05 -0700 Subject: [PATCH 04/10] Updates --- database/.pipeline/templates/db.dc.yaml | 6 ++- .../prereqs/postgres-postgis.is.yaml | 17 ++------- database/Dockerfile | 37 +++++++++++++++++++ 3 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 database/Dockerfile diff --git a/database/.pipeline/templates/db.dc.yaml b/database/.pipeline/templates/db.dc.yaml index 55f4ca7e23..3051a7b0be 100644 --- a/database/.pipeline/templates/db.dc.yaml +++ b/database/.pipeline/templates/db.dc.yaml @@ -19,7 +19,7 @@ parameters: - name: IMAGE_STREAM_NAME description: The OpenShift ImageStream name. required: true - value: 'postgis-postgres' + value: 'postgres-postgis' - name: IMAGE_STREAM_VERSION description: Version of PostgreSQL image to be used (9.2, 9.4, 9.5 or latest). required: true @@ -56,6 +56,10 @@ parameters: name: POSTGRESQL_DATABASE required: true value: 'biohubbc' + - name: PGDATA + description: Path to PostgreSQL data directory + value: '/var/lib/pgsql/data' + required: false - name: TZ description: Database timezone required: false diff --git a/database/.pipeline/templates/prereqs/postgres-postgis.is.yaml b/database/.pipeline/templates/prereqs/postgres-postgis.is.yaml index e310a33759..124b8fece2 100644 --- a/database/.pipeline/templates/prereqs/postgres-postgis.is.yaml +++ b/database/.pipeline/templates/prereqs/postgres-postgis.is.yaml @@ -23,7 +23,7 @@ parameters: - name: SOURCE_REPOSITORY_URL value: https://github.com/bcgov/biohubbc.git - name: DOCKER_FILE_PATH - value: 'docker/db/Dockerfile' + value: Dockerfile # Resources - name: CPU_REQUEST value: '50m' @@ -37,28 +37,17 @@ objects: - kind: ImageStream apiVersion: image.openshift.io/v1 metadata: - name: '${NAME}' + name: '${IMAGE_STREAM_NAME}' labels: shared: 'true' spec: lookupPolicy: local: false - tags: - - name: '${TAG_NAME}' - annotations: - from: - kind: ImageStreamTag - name: '${IMAGE_STREAM_NAME}:${IMAGE_STREAM_VERSION}' - namespace: ${IMAGE_STREAM_NAMESPACE} - generation: 1 - importPolicy: {} - referencePolicy: - type: Source - kind: BuildConfig apiVersion: v1 metadata: - name: '${NAME}' + name: '${IMAGE_STREAM_NAME}' spec: failedBuildsHistoryLimit: 5 nodeSelector: diff --git a/database/Dockerfile b/database/Dockerfile new file mode 100644 index 0000000000..162abcc19b --- /dev/null +++ b/database/Dockerfile @@ -0,0 +1,37 @@ +# ######################################################################################################## +# This DockerFile is used for Openshift deployments only. +# ######################################################################################################## + +ARG POSTGRES_VERSION=15-bullseye + +FROM postgres:$POSTGRES_VERSION + +# read env variables +ARG TZ=America/Vancouver +ARG POSTGIS_VERSION=3 + +ENV PORT=5432 + +# install postgis packages +RUN mkdir -p /opt/apps +RUN apt-get -qq update +RUN apt-get -qq install -y --no-install-recommends postgresql-$PG_MAJOR-postgis-$POSTGIS_VERSION +RUN apt-get -qq install -y --no-install-recommends postgresql-$PG_MAJOR-postgis-$POSTGIS_VERSION-scripts +RUN apt-get -qq install -y --no-install-recommends postgresql-$PG_MAJOR-pgrouting +RUN apt-get -qq install -y --no-install-recommends postgresql-$PG_MAJOR-pgrouting-scripts +RUN apt-get -qq install -y --no-install-recommends postgresql-server-dev-$PG_MAJOR +RUN apt-get -qq install -y --no-install-recommends pgbadger pg-activity wget unzip nano +RUN apt-get -qq purge -y --auto-remove postgresql-server-dev-$PG_MAJOR +RUN apt-get -qq autoremove -y +RUN apt-get -qq clean + +# set time zone +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# copy postgis init script to docker init directory +RUN mkdir -p /docker-entrypoint-initdb.d +COPY ./docker/db/create_postgis.sql /docker-entrypoint-initdb.d/postgis.sql + +EXPOSE $PORT + +CMD ["postgres"] From acfcaaaa95816709c0e76372b9e796facf0c4ba2 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 24 Oct 2024 11:42:05 -0700 Subject: [PATCH 05/10] Fix docker path --- database/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/Dockerfile b/database/Dockerfile index 162abcc19b..040319103a 100644 --- a/database/Dockerfile +++ b/database/Dockerfile @@ -30,7 +30,7 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # copy postgis init script to docker init directory RUN mkdir -p /docker-entrypoint-initdb.d -COPY ./docker/db/create_postgis.sql /docker-entrypoint-initdb.d/postgis.sql +COPY ./.docker/db/create_postgis.sql /docker-entrypoint-initdb.d/postgis.sql EXPOSE $PORT From 638fa0df8a67c3ad02208108de94c5e76970546a Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 24 Oct 2024 13:31:34 -0700 Subject: [PATCH 06/10] Updates --- database/.pipeline/lib/db.deploy.js | 4 +- database/.pipeline/templates/db.dc.yaml | 50 +++++++++---------------- 2 files changed, 19 insertions(+), 35 deletions(-) diff --git a/database/.pipeline/lib/db.deploy.js b/database/.pipeline/lib/db.deploy.js index cbc1485827..ed07608b81 100644 --- a/database/.pipeline/lib/db.deploy.js +++ b/database/.pipeline/lib/db.deploy.js @@ -31,7 +31,7 @@ const dbDeploy = async (settings) => { DATABASE_SERVICE_NAME: `${name}-postgresql${phases[phase].suffix}`, IMAGE_STREAM_NAME: name, IMAGE_STREAM_VERSION: phases.build.tag, - POSTGRESQL_DATABASE: 'biohubbc', + POSTGRES_DB: 'biohubbc', TZ: phases[phase].tz, IMAGE_STREAM_NAMESPACE: phases.build.namespace, VOLUME_CAPACITY: phases[phase].volumeCapacity, @@ -46,7 +46,7 @@ const dbDeploy = async (settings) => { oc.applyRecommendedLabels(objects, name, phase, changeId, instance); oc.importImageStreams(objects, phases[phase].tag, phases.build.namespace, phases.build.tag); - + await oc.applyAndDeploy(objects, instance); }; diff --git a/database/.pipeline/templates/db.dc.yaml b/database/.pipeline/templates/db.dc.yaml index 3051a7b0be..1a4674ec64 100644 --- a/database/.pipeline/templates/db.dc.yaml +++ b/database/.pipeline/templates/db.dc.yaml @@ -31,29 +31,18 @@ parameters: value: 'postgresql' - description: Username for PostgreSQL user that will be used for accessing the database. displayName: PostgreSQL Connection Username - name: POSTGRESQL_USER - required: true - value: 'postgres' - - description: Admin Username for PostgreSQL user that will be used for accessing the database. - displayName: PostgreSQL Admin Connection Username - name: POSTGRESQL_ADMIN_USER + name: POSTGRES_USER required: true value: 'postgres' - description: Password for the PostgreSQL connection user. displayName: PostgreSQL Connection Password from: '[a-zA-Z0-9]{16}' generate: expression - name: POSTGRESQL_PASSWORD - required: true - - description: Password for the PostgreSQL admin user. - displayName: PostgreSQL Admin Connection Password - from: '[a-zA-Z0-9]{16}' - generate: expression - name: POSTGRESQL_ADMIN_PASSWORD + name: POSTGRES_PASSWORD required: true - description: Name of the PostgreSQL database accessed. displayName: PostgreSQL Database Name - name: POSTGRESQL_DATABASE + name: POSTGRES_DB required: true value: 'biohubbc' - name: PGDATA @@ -85,16 +74,14 @@ objects: metadata: annotations: template.openshift.io/expose-database_name: "{.data['database-name']}" - template.openshift.io/expose-password: "{.data['database-user-password']}" - template.openshift.io/expose-admin-password: "{.data['database-admin-password']}" - template.openshift.io/expose-admin-username: "{.data['database-user']}" + template.openshift.io/expose-database-user: "{.data['database-admin']}" + template.openshift.io/expose-database-user-password: "{.data['database-admin-password']}" as-copy-of: biohubbc-creds name: '${DATABASE_SERVICE_NAME}' stringData: - database-name: '${POSTGRESQL_DATABASE}' - database-user-password: '${POSTGRESQL_PASSWORD}' - database-admin-password: '${POSTGRESQL_ADMIN_PASSWORD}' - database-user: '${POSTGRESQL_USER}' + database-name: '${POSTGRES_DB}' + database-user: '${POSTGRES_USER}' + database-user-password: '${POSTGRES_PASSWORD}' - kind: Service apiVersion: v1 @@ -160,26 +147,23 @@ objects: containers: - name: postgresql env: - - name: POSTGRESQL_USER - valueFrom: - secretKeyRef: - key: database-user - name: '${DATABASE_SERVICE_NAME}' - - name: POSTGRESQL_PASSWORD + - name: POSTGRES_DB valueFrom: secretKeyRef: - key: database-user-password + key: database-name name: '${DATABASE_SERVICE_NAME}' - - name: POSTGRESQL_ADMIN_PASSWORD + - name: POSTGRES_USER valueFrom: secretKeyRef: - key: database-admin-password + key: database-user name: '${DATABASE_SERVICE_NAME}' - - name: POSTGRESQL_DATABASE + - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - key: database-name + key: database-user-password name: '${DATABASE_SERVICE_NAME}' + - name: PGDATA + value: '${PGDATA}' - name: PGOPTIONS value: '-c maintenance_work_mem=128MB' - name: PGTZ @@ -200,7 +184,7 @@ objects: - '/bin/sh' - '-i' - '-c' - - psql -h 127.0.0.1 -U $POSTGRESQL_USER -q -d $POSTGRESQL_DATABASE -c 'SELECT 1' + - psql -h 127.0.0.1 -U $POSTGRES_USER -q -d $POSTGRES_DB -c 'SELECT 1' initialDelaySeconds: 5 timeoutSeconds: 1 resources: From 233ee9fbd554db9dc28c1710f284c6fe0c9bf495 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 24 Oct 2024 13:47:56 -0700 Subject: [PATCH 07/10] Update pgdata variable --- database/.pipeline/templates/db.dc.yaml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/database/.pipeline/templates/db.dc.yaml b/database/.pipeline/templates/db.dc.yaml index 1a4674ec64..febb2d081a 100644 --- a/database/.pipeline/templates/db.dc.yaml +++ b/database/.pipeline/templates/db.dc.yaml @@ -47,7 +47,7 @@ parameters: value: 'biohubbc' - name: PGDATA description: Path to PostgreSQL data directory - value: '/var/lib/pgsql/data' + value: '/var/lib/pgsql/data/pgdata' required: false - name: TZ description: Database timezone @@ -194,23 +194,21 @@ objects: limits: cpu: ${CPU_LIMIT} memory: ${MEMORY_LIMIT} - securityContext: - capabilities: {} - privileged: false + securityContext: {} terminationMessagePath: '/dev/termination-log' volumeMounts: - - mountPath: '/var/lib/pgsql/data' - name: '${DATABASE_SERVICE_NAME}-data' - - mountPath: '/var/run/postgresql' - name: '${DATABASE_SERVICE_NAME}-run' + - name: '${DATABASE_SERVICE_NAME}-data' + mountPath: '/var/lib/pgsql/data' + # - name: '${DATABASE_SERVICE_NAME}-run' + # mountPath: '/var/run/postgresql' dnsPolicy: ClusterFirst restartPolicy: Always volumes: - name: '${DATABASE_SERVICE_NAME}-data' persistentVolumeClaim: claimName: '${DATABASE_SERVICE_NAME}' - - name: '${DATABASE_SERVICE_NAME}-run' - emptyDir: {} + # - name: '${DATABASE_SERVICE_NAME}-run' + # emptyDir: {} triggers: - imageChangeParams: automatic: true @@ -220,6 +218,5 @@ objects: kind: ImageStreamTag name: '${IMAGE_STREAM_NAME}:${IMAGE_STREAM_VERSION}' namespace: '${IMAGE_STREAM_NAMESPACE}' - lastTriggeredImage: '' type: ImageChange - type: ConfigChange From aff5c35ef963b179d8297789057650d03cd82b34 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 24 Oct 2024 13:59:39 -0700 Subject: [PATCH 08/10] Fix secret --- database/.pipeline/templates/db.dc.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/.pipeline/templates/db.dc.yaml b/database/.pipeline/templates/db.dc.yaml index febb2d081a..762979b889 100644 --- a/database/.pipeline/templates/db.dc.yaml +++ b/database/.pipeline/templates/db.dc.yaml @@ -155,12 +155,12 @@ objects: - name: POSTGRES_USER valueFrom: secretKeyRef: - key: database-user + key: database-admin name: '${DATABASE_SERVICE_NAME}' - name: POSTGRES_PASSWORD valueFrom: secretKeyRef: - key: database-user-password + key: database-admin-password name: '${DATABASE_SERVICE_NAME}' - name: PGDATA value: '${PGDATA}' From 9b5e6e7090347309b5a1facca136c62d260c1a28 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 24 Oct 2024 14:04:03 -0700 Subject: [PATCH 09/10] ignore-skip From 2d865932ce74b7471c35f6a935aaed86966c1131 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 24 Oct 2024 15:01:12 -0700 Subject: [PATCH 10/10] Update pgdata path --- database/.pipeline/templates/db.dc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/.pipeline/templates/db.dc.yaml b/database/.pipeline/templates/db.dc.yaml index 762979b889..a76a801b89 100644 --- a/database/.pipeline/templates/db.dc.yaml +++ b/database/.pipeline/templates/db.dc.yaml @@ -47,7 +47,7 @@ parameters: value: 'biohubbc' - name: PGDATA description: Path to PostgreSQL data directory - value: '/var/lib/pgsql/data/pgdata' + value: '/var/lib/pgsql/data/userdata' required: false - name: TZ description: Database timezone