From e42260bec67badb9dccb42853b230c7b9adb9cf5 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 26 Nov 2024 17:03:24 -0600 Subject: [PATCH 1/7] Add up/down migration for department soft deletes and add new department --- .../migrations/1732661952305_department_reorg/down.sql | 3 +++ .../migrations/1732661952305_department_reorg/up.sql | 5 +++++ 2 files changed, 8 insertions(+) create mode 100644 moped-database/migrations/1732661952305_department_reorg/down.sql create mode 100644 moped-database/migrations/1732661952305_department_reorg/up.sql diff --git a/moped-database/migrations/1732661952305_department_reorg/down.sql b/moped-database/migrations/1732661952305_department_reorg/down.sql new file mode 100644 index 0000000000..e583fe7837 --- /dev/null +++ b/moped-database/migrations/1732661952305_department_reorg/down.sql @@ -0,0 +1,3 @@ +-- Updating moped_department table will be up only. If we need to revert, we will need do it manually or +-- update with a future migration. +SELECT 0; diff --git a/moped-database/migrations/1732661952305_department_reorg/up.sql b/moped-database/migrations/1732661952305_department_reorg/up.sql new file mode 100644 index 0000000000..8ae88c02fa --- /dev/null +++ b/moped-database/migrations/1732661952305_department_reorg/up.sql @@ -0,0 +1,5 @@ +-- Add soft deletes to department table +ALTER TABLE moped_department ADD is_deleted boolean DEFAULT FALSE; + +INSERT INTO "public"."moped_department" ("department_name", "department_abbreviation", "organization_id") VALUES +('Austin Transportation and Public Works', 'TPW', 1); From 1d0f5e086a4988a62589a052976ffef186cf15c3 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 26 Nov 2024 17:12:34 -0600 Subject: [PATCH 2/7] Soft delete ATD & PWD; update workgroup department ids to TPW id --- .../1732661952305_department_reorg/up.sql | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/moped-database/migrations/1732661952305_department_reorg/up.sql b/moped-database/migrations/1732661952305_department_reorg/up.sql index 8ae88c02fa..77c74f66df 100644 --- a/moped-database/migrations/1732661952305_department_reorg/up.sql +++ b/moped-database/migrations/1732661952305_department_reorg/up.sql @@ -3,3 +3,34 @@ ALTER TABLE moped_department ADD is_deleted boolean DEFAULT FALSE; INSERT INTO "public"."moped_department" ("department_name", "department_abbreviation", "organization_id") VALUES ('Austin Transportation and Public Works', 'TPW', 1); + +-- Soft delete existing department records that are merging into the new one +UPDATE moped_department SET is_deleted = TRUE WHERE department_name IN ('Austin Transportation', 'Public Works'); + +-- Find existing workgroup records that are associated with the departments that are merging +-- and update them to the new department row called Austin Transportation and Public Works +WITH department_todos AS ( + SELECT department_id AS ids + FROM + moped_department + WHERE + department_name IN ( + 'Austin Transportation', + 'Public Works' + ) +), + +new_department_row AS ( + SELECT department_id AS id + FROM + moped_department + WHERE + department_name = 'Austin Transportation and Public Works' +) + +UPDATE +moped_workgroup +SET + department_id = (SELECT id FROM new_department_row) +WHERE + department_id IN (SELECT ids FROM department_todos); From adca6c5a0e2b6449158711df69d1cd4472f76543 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 26 Nov 2024 18:00:15 -0600 Subject: [PATCH 3/7] Update entity departments too and add missing fk constraint --- .../1732661952305_department_reorg/up.sql | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/moped-database/migrations/1732661952305_department_reorg/up.sql b/moped-database/migrations/1732661952305_department_reorg/up.sql index 77c74f66df..a77a94158b 100644 --- a/moped-database/migrations/1732661952305_department_reorg/up.sql +++ b/moped-database/migrations/1732661952305_department_reorg/up.sql @@ -1,5 +1,6 @@ -- Add soft deletes to department table ALTER TABLE moped_department ADD is_deleted boolean DEFAULT FALSE; +COMMENT ON COLUMN moped_department.is_deleted IS 'Indicates soft deletion'; INSERT INTO "public"."moped_department" ("department_name", "department_abbreviation", "organization_id") VALUES ('Austin Transportation and Public Works', 'TPW', 1); @@ -34,3 +35,36 @@ SET department_id = (SELECT id FROM new_department_row) WHERE department_id IN (SELECT ids FROM department_todos); + +-- Find existing entity records that are associated with the departments that are merging +-- and update them to the new department row called Austin Transportation and Public Works +WITH department_todos AS ( + SELECT department_id AS ids + FROM + moped_department + WHERE + department_name IN ( + 'Austin Transportation', + 'Public Works' + ) +), + +new_department_row AS ( + SELECT department_id AS id + FROM + moped_department + WHERE + department_name = 'Austin Transportation and Public Works' +) + +UPDATE +moped_entity +SET + department_id = (SELECT id FROM new_department_row) +WHERE + department_id IN (SELECT ids FROM department_todos); + +-- Add foreign key constraints to entity table department_id column; moped_workgroup already has this constraint +ALTER TABLE moped_entity +ADD CONSTRAINT moped_entity_department_id_fkey FOREIGN KEY ("department_id") +REFERENCES moped_department ("department_id"); From f316579a5d1ee670853a7c846a08609ab0e321dc Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 26 Nov 2024 18:16:05 -0600 Subject: [PATCH 4/7] Add other fk constraints --- .../1732661952305_department_reorg/up.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/moped-database/migrations/1732661952305_department_reorg/up.sql b/moped-database/migrations/1732661952305_department_reorg/up.sql index a77a94158b..ef1d46a8b1 100644 --- a/moped-database/migrations/1732661952305_department_reorg/up.sql +++ b/moped-database/migrations/1732661952305_department_reorg/up.sql @@ -68,3 +68,17 @@ WHERE ALTER TABLE moped_entity ADD CONSTRAINT moped_entity_department_id_fkey FOREIGN KEY ("department_id") REFERENCES moped_department ("department_id"); + +-- Add foreign key constraints to entity table workgroup_id column +ALTER TABLE moped_entity +ADD CONSTRAINT moped_entity_workgroup_id_fkey FOREIGN KEY ("workgroup_id") +REFERENCES moped_workgroup ("workgroup_id"); + +-- Add foreign key constraints to department and entity tables for organization_id columns +ALTER TABLE moped_entity +ADD CONSTRAINT moped_entity_organization_id_fkey FOREIGN KEY ("organization_id") +REFERENCES moped_organization ("organization_id"); + +ALTER TABLE moped_department +ADD CONSTRAINT moped_entity_organization_id_fkey FOREIGN KEY ("organization_id") +REFERENCES moped_organization ("organization_id"); From 9107a73f96b64e589f48b3e6cf32b9a1ed8a4887 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 2 Dec 2024 15:56:29 -0600 Subject: [PATCH 5/7] Delete outdated row intended for null department --- .../migrations/1732661952305_department_reorg/up.sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moped-database/migrations/1732661952305_department_reorg/up.sql b/moped-database/migrations/1732661952305_department_reorg/up.sql index ef1d46a8b1..55edb0c7b5 100644 --- a/moped-database/migrations/1732661952305_department_reorg/up.sql +++ b/moped-database/migrations/1732661952305_department_reorg/up.sql @@ -82,3 +82,6 @@ REFERENCES moped_organization ("organization_id"); ALTER TABLE moped_department ADD CONSTRAINT moped_entity_organization_id_fkey FOREIGN KEY ("organization_id") REFERENCES moped_organization ("organization_id"); + +-- Remove row with 0 for id and 'None' for name; we're not using it and null is more appropriate for entities and workgroups with no department +DELETE FROM moped_department WHERE department_id = 0; From 4937eb152324334859d26eec91e99f8a9c627186 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 2 Dec 2024 16:36:29 -0600 Subject: [PATCH 6/7] Adding in the missing ETL readme here --- moped-etl/README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 moped-etl/README.md diff --git a/moped-etl/README.md b/moped-etl/README.md new file mode 100644 index 0000000000..fdfdb4c908 --- /dev/null +++ b/moped-etl/README.md @@ -0,0 +1,3 @@ +# Moped ETLs + +These ETLs are scheduled through our [Airflow](https://github.com/cityofaustin/atd-airflow) deployment. Each folder contains a containerized Python script and the Docker images are updated through Github Action workflows. From c2fc86c6709026a6013e72c1d31e272f3c97c348 Mon Sep 17 00:00:00 2001 From: Mike Date: Thu, 5 Dec 2024 11:21:23 -0600 Subject: [PATCH 7/7] Fix typo in department org fk constraint --- moped-database/migrations/1732661952305_department_reorg/up.sql | 2 +- moped-etl/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/moped-database/migrations/1732661952305_department_reorg/up.sql b/moped-database/migrations/1732661952305_department_reorg/up.sql index 55edb0c7b5..4ae3b9d1d9 100644 --- a/moped-database/migrations/1732661952305_department_reorg/up.sql +++ b/moped-database/migrations/1732661952305_department_reorg/up.sql @@ -80,7 +80,7 @@ ADD CONSTRAINT moped_entity_organization_id_fkey FOREIGN KEY ("organization_id") REFERENCES moped_organization ("organization_id"); ALTER TABLE moped_department -ADD CONSTRAINT moped_entity_organization_id_fkey FOREIGN KEY ("organization_id") +ADD CONSTRAINT moped_department_organization_id_fkey FOREIGN KEY ("organization_id") REFERENCES moped_organization ("organization_id"); -- Remove row with 0 for id and 'None' for name; we're not using it and null is more appropriate for entities and workgroups with no department diff --git a/moped-etl/README.md b/moped-etl/README.md index fdfdb4c908..d71ea0c683 100644 --- a/moped-etl/README.md +++ b/moped-etl/README.md @@ -1,3 +1,3 @@ # Moped ETLs -These ETLs are scheduled through our [Airflow](https://github.com/cityofaustin/atd-airflow) deployment. Each folder contains a containerized Python script and the Docker images are updated through Github Action workflows. +These ETLs are scheduled through our [Airflow](https://github.com/cityofaustin/atd-airflow) deployment. Each folder contains a containerized Python script and the Docker images are updated through GitHub Action workflows.