Implement operational database purging

cloud

@@ -112,6 +112,7 @@ function execute_command() {
     declare -r load_queue_trigger_topic="${prefix}load_queue_trigger"
     declare -r cache_bucket_name="${project}_${prefix}cache"
     declare -r pick_notifications_trigger_topic="${prefix}pick_notifications_trigger"
+    declare -r purge_op_db_trigger_topic="${prefix}purge_op_db_trigger"
     declare -r cache_redirect_function_name="cache_redirect"
     declare cache_redirector_url="https://${CLOUD_FUNCTION_REGION}"
     declare cache_redirector_url+="-${project}.cloudfunctions.net/"
@@ -135,15 +136,15 @@ function execute_command() {
     declare -r psql_database="${prefix}${version}"
     declare -r psql_clean_test_database="${prefix}${version}_clean_test"
     declare -r psql_empty_test_database="${prefix}${version}_empty_test"
-    declare -r psql_submitter="${psql_database}_submitter"
+    declare -r psql_editor="${psql_database}_editor"
     declare -r psql_pgpass_secret="${prefix}psql_pgpass"
     declare -r psql_kcidb_db=$(
         echo -n "postgresql: "
         # Use the local proxy for the shell
         if [ "$command" != "shell" ]; then
             echo -n "host=${psql_socket_dir}" | escape_whitespace
             echo -n " "
-            echo -n "user=${psql_submitter}"
+            echo -n "user=${psql_editor}"
             echo -n " "
         fi
         echo -n "dbname=${psql_database}" | escape_whitespace
@@ -156,22 +157,22 @@ function execute_command() {
         echo -n "postgresql:"
         echo -n "dbname=${psql_empty_test_database}" | escape_whitespace
     )
-    declare -a psql_args=("$project" "$psql_database" "$psql_submitter")
+    declare -a psql_args=("$project" "$psql_database" "$psql_editor")
     if "$test"; then
         psql_args+=(
             # Do not initialize the clean database
             "#$psql_clean_test_database"
-            "${psql_clean_test_database}_submitter"
+            "${psql_clean_test_database}_editor"
             "$psql_empty_test_database"
-            "${psql_empty_test_database}_submitter"
+            "${psql_empty_test_database}_editor"
         )
     fi
     declare -a -r psql_args
 
     # Enable fetching PostgreSQL passwords from their secrets
     password_set_secret "psql_superuser" "$project" "kcidb_psql_superuser"
     password_set_secret "psql_viewer" "$project" "kcidb_psql_viewer"
-    password_set_secret "psql_submitter" "$project" "${prefix}psql_submitter"
+    password_set_secret "psql_editor" "$project" "${prefix}psql_editor"
 
     declare -r bigquery_dataset="${prefix}${version}"
     declare -r bigquery_clean_test_dataset="${prefix}${version}_clean_test"
@@ -255,6 +256,7 @@ function execute_command() {
         --new-load-subscription="$new_load_subscription"
         --updated-publish="$updated_publish"
         --updated-topic="$updated_topic"
+        --purge-op-db-trigger-topic="$purge_op_db_trigger_topic"
         --updated-urls-topic="$updated_urls_topic"
         --spool-collection-path="$spool_collection_path"
         --extra-cc="$extra_cc"
@@ -263,6 +265,8 @@ function execute_command() {
         --smtp-topic="$smtp_topic"
         --smtp-subscription="$smtp_subscription"
         --pgpass-secret="$psql_pgpass_secret"
+        --op-database="$psql_kcidb_db"
+        --ar-database="$bigquery_kcidb_db"
         --database="$database"
         --clean-test-databases="$clean_test_databases"
         --empty-test-databases="$empty_test_databases"
@@ -294,7 +298,7 @@ function execute_command() {
 
     if [ "$command" == "deploy" ]; then
         sections_run "$sections" secrets_deploy "$project" \
-            "$psql_pgpass_secret" "$psql_submitter"
+            "$psql_pgpass_secret" "$psql_editor"
         sections_run "$sections" bigquery_deploy "${bigquery_args[@]}"
         sections_run "$sections" psql_deploy "${psql_args[@]}"
         sections_run "$sections" pubsub_deploy \
@@ -308,6 +312,7 @@ function execute_command() {
             --updated-debug-subscription="$updated_debug_subscription" \
             --pick-notifications-trigger-topic \
               "$pick_notifications_trigger_topic" \
+            --purge-op-db-trigger-topic "$purge_op_db_trigger_topic" \
             --smtp-topic="$smtp_topic" \
             --smtp-subscription="$smtp_subscription"
         sections_run "$sections" firestore_deploy "$project"
@@ -320,6 +325,7 @@ function execute_command() {
             --load-queue-trigger-topic="$load_queue_trigger_topic" \
             --pick-notifications-trigger-topic \
               "$pick_notifications_trigger_topic" \
+            --purge-op-db-trigger-topic "$purge_op_db_trigger_topic" \
             --updated-urls-topic="$updated_urls_topic" \
             --updated-topic="$updated_topic" \
             --cache-redirect-function-name="$cache_redirect_function_name" \
@@ -329,7 +335,8 @@ function execute_command() {
             --project="$project" \
             --prefix="$prefix" \
             --load-queue-trigger-topic="$load_queue_trigger_topic" \
-            --pick-notifications-trigger-topic="$pick_notifications_trigger_topic"
+            --pick-notifications-trigger-topic="$pick_notifications_trigger_topic" \
+            --purge-op-db-trigger-topic="$purge_op_db_trigger_topic"
         sections_run "$sections" submitters_deploy \
             "$project" "$new_topic" "${submitters[@]}"
     elif [ "$command" == "shell" ]; then
@@ -359,6 +366,7 @@ function execute_command() {
             --load-queue-trigger-topic="$load_queue_trigger_topic" \
             --pick-notifications-trigger-topic \
               "$pick_notifications_trigger_topic" \
+            --purge-op-db-trigger-topic "$purge_op_db_trigger_topic" \
             --new-topic="$new_topic" \
             --new-load-subscription="$new_load_subscription" \
             --new-debug-subscription="$new_debug_subscription" \

kcidb/cloud/cloud_functions.sh

@@ -16,14 +16,17 @@ declare _CLOUD_FUNCTIONS_SH=
 #       --new-topic=NAME --new-load-subscription=NAME
 #       --updated-publish=true|false
 #       --updated-topic=NAME
-#       --updated_urls_topic=NAME
+#       --purge-op-db-trigger-topic=NAME
+#       --updated-urls-topic=NAME
 #       --cache-bucket-name=NAME
 #       --cache-redirector-url=URL
 #       --spool-collection-path=PATH
 #       --extra-cc=ADDRS
 #       --smtp-to-addrs=ADDRS --smtp-password-secret=NAME
 #       --smtp-topic=NAME --smtp-subscription=NAME
 #       --pgpass-secret=NAME
+#       --op-database=SPEC
+#       --ar-database=SPEC
 #       --database=SPEC
 #       --clean-test-databases=SPEC_LIST
 #       --empty-test-databases=SPEC_LIST
@@ -36,6 +39,7 @@ function cloud_functions_env() {
                           heavy_asserts \
                           new_topic new_load_subscription \
                           updated_publish updated_topic \
+                          purge_op_db_trigger_topic \
                           updated_urls_topic \
                           spool_collection_path \
                           extra_cc \
@@ -44,6 +48,8 @@ function cloud_functions_env() {
                           pgpass_secret \
                           cache_bucket_name \
                           cache_redirector_url \
+                          op_database \
+                          ar_database \
                           database \
                           clean_test_databases \
                           empty_test_databases \
@@ -59,11 +65,14 @@ function cloud_functions_env() {
         [KCIDB_LOAD_QUEUE_OBJ_MAX]="8192"
         [KCIDB_LOAD_QUEUE_TIMEOUT_SEC]="30"
         [KCIDB_PGPASS_SECRET]="$pgpass_secret"
+        [KCIDB_OPERATIONAL_DATABASE]="$op_database"
+        [KCIDB_ARCHIVE_DATABASE]="$ar_database"
         [KCIDB_DATABASE]="$database"
         [KCIDB_DATABASE_LOAD_PERIOD_SEC]="180"
         [KCIDB_CLEAN_TEST_DATABASES]="$clean_test_databases"
         [KCIDB_EMPTY_TEST_DATABASES]="$empty_test_databases"
         [KCIDB_UPDATED_QUEUE_TOPIC]="$updated_topic"
+        [KCIDB_PURGE_OP_DB_TRIGGER_TOPIC]="$purge_op_db_trigger_topic"
         [KCIDB_UPDATED_URLS_TOPIC]="$updated_urls_topic"
         [KCIDB_SELECTED_SUBSCRIPTIONS]=""
         [KCIDB_SPOOL_COLLECTION_PATH]="$spool_collection_path"
@@ -117,6 +126,7 @@ function cloud_functions_env() {
 #       --project=NAME --prefix=PREFIX --source=PATH
 #       --load-queue-trigger-topic=NAME
 #       --pick-notifications-trigger-topic=NAME
+#       --purge-op-db-trigger-topic=NAME
 #       --updated-urls-topic=NAME
 #       --updated-topic=NAME
 #       --spool-collection-path=PATH
@@ -127,6 +137,7 @@ function cloud_functions_deploy() {
     params="$(getopt_vars sections project prefix source \
                           load_queue_trigger_topic \
                           pick_notifications_trigger_topic \
+                          purge_op_db_trigger_topic \
                           updated_urls_topic \
                           updated_topic \
                           spool_collection_path \
@@ -146,6 +157,14 @@ function cloud_functions_deploy() {
     trigger_event+="document.create"
     declare trigger_resource="projects/$project/databases/(default)/documents/"
     trigger_resource+="${spool_collection_path}/{notification_id}"
+    cloud_function_deploy "$sections" "$source" "$project" "$prefix" \
+                          purge_op_db \
+                          --env-vars-file "$env_yaml_file" \
+                          --trigger-topic "${purge_op_db_trigger_topic}" \
+                          --memory 256MB \
+                          --max-instances=1 \
+                          --timeout 540
+
     cloud_function_deploy "$sections" "$source" "$project" "$prefix" \
                           pick_notifications \
                           --env-vars-file "$env_yaml_file" \
@@ -209,6 +228,8 @@ function cloud_functions_withdraw() {
                           cache_redirect_function_name \
                           -- "$@")"
     eval "$params"
+    cloud_function_withdraw "$sections" "$project" "$prefix" \
+                            purge_op_db
     cloud_function_withdraw "$sections" "$project" "$prefix" \
                             pick_notifications
     cloud_function_withdraw "$sections" "$project" "$prefix" \

kcidb/cloud/password.sh

@@ -9,15 +9,15 @@ declare _PASSWORD_SH=
 declare -r -A PASSWORD_DESCS=(
     [smtp]="SMTP"
     [psql_superuser]="PostgreSQL superuser"
-    [psql_submitter]="PostgreSQL submitter user"
+    [psql_editor]="PostgreSQL editor user"
     [psql_viewer]="PostgreSQL viewer user"
 )
 
 # A map of password names and their "can be auto-generated" flags.
 # The corresponding password will be auto-generated if the flag is "true", and
 # no source file nor secret was specified for it.
 declare -A PASSWORD_GENERATE=(
-    [psql_submitter]="true"
+    [psql_editor]="true"
     [psql_viewer]="true"
 )
 

kcidb/cloud/psql.sh

@@ -225,22 +225,22 @@ function psql_database_exists() {
 # Setup a PostgreSQL database and its paraphernalia.
 # Expect the environment to be set up with variables permitting a libpq user
 # to connect to the database as a superuser.
-# Args: database init submitter viewer
+# Args: database init editor viewer
 function _psql_database_setup() {
     declare -r database="$1"; shift
     declare -r init="$1"; shift
-    declare -r submitter="$1"; shift
+    declare -r editor="$1"; shift
     declare -r viewer="$1"; shift
-    # Deploy viewer and submitter permissions
+    # Deploy viewer and editor permissions
     mute psql --dbname="$database" -e <<<"
         \\set ON_ERROR_STOP on
 
-        GRANT USAGE ON SCHEMA public TO $submitter, $viewer;
+        GRANT USAGE ON SCHEMA public TO $editor, $viewer;
 
         ALTER DEFAULT PRIVILEGES IN SCHEMA public
-        GRANT SELECT, INSERT, UPDATE ON TABLES TO $submitter;
-        GRANT SELECT, INSERT, UPDATE ON ALL TABLES
-        IN SCHEMA public TO $submitter;
+        GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO $editor;
+        GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES
+        IN SCHEMA public TO $editor;
 
         ALTER DEFAULT PRIVILEGES IN SCHEMA public
         GRANT SELECT ON TABLES TO $viewer;
@@ -257,16 +257,16 @@ function _psql_database_setup() {
 # Cleanup a PostgreSQL database and its paraphernalia.
 # Expect the environment to be set up with variables permitting a libpq user
 # to connect to the database as a superuser.
-# Args: database submitter viewer
+# Args: database editor viewer
 function _psql_database_cleanup() {
     declare -r database="$1"; shift
-    declare -r submitter="$1"; shift
+    declare -r editor="$1"; shift
     declare -r viewer="$1"; shift
-    # Withdraw viewer and submitter permissions
+    # Withdraw viewer and editor permissions
     mute psql --dbname="$database" -e <<<"
         /* Do not stop on errors in case users are already removed */
-        REVOKE SELECT, INSERT, UPDATE ON ALL TABLES IN SCHEMA public
-        FROM $submitter;
+        REVOKE SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public
+        FROM $editor;
         REVOKE SELECT ON ALL TABLES IN SCHEMA public
         FROM $viewer;
         \\set ON_ERROR_STOP on
@@ -282,18 +282,18 @@ function _psql_database_cleanup() {
 
 # Deploy PostgreSQL databases, if they don't exist
 # Do not initialize databases that are prepended with the hash sign ('#').
-# Args: project instance viewer [database submitter]...
+# Args: project instance viewer [database editor]...
 function psql_databases_deploy() {
     declare -r project="$1"; shift
     declare -r instance="$1"; shift
     declare -r viewer="$1"; shift
     declare database
-    declare submitter
+    declare editor
     declare init
 
     while (($#)); do
         database="$1"; shift
-        submitter="$1"; shift
+        editor="$1"; shift
 
         # Handle possible leading hash sign
         if [[ $database == \#* ]]; then
@@ -313,46 +313,46 @@ function psql_databases_deploy() {
                 --instance="$instance"
         fi
 
-        # Deploy the per-database submitter user
-        exists=$(psql_user_exists "$project" "$instance" "$submitter")
-        if ! "$exists" || password_is_specified psql_submitter; then
+        # Deploy the per-database editor user
+        exists=$(psql_user_exists "$project" "$instance" "$editor")
+        if ! "$exists" || password_is_specified psql_editor; then
             # Get and cache the password in the current shell first
-            password_get psql_submitter >/dev/null
+            password_get psql_editor >/dev/null
             # Create the user with the cached password
-            password_get psql_submitter |
-                psql_user_deploy "$project" "$instance" "$submitter"
+            password_get psql_editor |
+                psql_user_deploy "$project" "$instance" "$editor"
         fi
 
         # NOTE: The viewer user is created per-instance
 
         # Setup the database
         psql_proxy_session "$project" "$instance" \
-            _psql_database_setup "$database" "$init" "$submitter" "$viewer"
+            _psql_database_setup "$database" "$init" "$editor" "$viewer"
     done
 }
 
 # Withdraw PostgreSQL databases, if they exist
 # Cleanup all databases, even those prepended with the hash sign ('#').
-# Args: project instance viewer [database submitter]...
+# Args: project instance viewer [database editor]...
 function psql_databases_withdraw() {
     declare -r project="$1"; shift
     declare -r instance="$1"; shift
     declare -r viewer="$1"; shift
-    declare -a -r databases_and_submitters=("$@")
+    declare -a -r databases_and_editors=("$@")
     declare database
-    declare submitter
+    declare editor
 
     # Cleanup and remove the databases
-    set -- "${databases_and_submitters[@]}"
+    set -- "${databases_and_editors[@]}"
     while (($#)); do
         # Ignore possible leading hash sign
         database="${1###}"; shift
-        submitter="$1"; shift
+        editor="$1"; shift
         exists=$(psql_database_exists "$project" "$instance" "$database")
         if "$exists"; then
             # Cleanup the database
             psql_proxy_session "$project" "$instance" \
-                _psql_database_cleanup "$database" "$submitter" "$viewer"
+                _psql_database_cleanup "$database" "$editor" "$viewer"
             # Delete the database
             mute gcloud sql databases delete \
                 "$database" \
@@ -363,13 +363,13 @@ function psql_databases_withdraw() {
     done
 
     # Remove the users afterwards as they could be shared by databases
-    set -- "${databases_and_submitters[@]}"
+    set -- "${databases_and_editors[@]}"
     while (($#)); do
         # Discard the database name
         shift
-        submitter="$1"; shift
-        # Withdraw the submitter user
-        psql_user_withdraw "$project" "$instance" "$submitter"
+        editor="$1"; shift
+        # Withdraw the editor user
+        psql_user_withdraw "$project" "$instance" "$editor"
         # NOTE: The viewer user is per-instance
     done
 }
@@ -444,7 +444,7 @@ function psql_user_withdraw() {
 
 # Deploy (to) PostgreSQL.
 # Do not initialize databases that are prepended with the hash sign ('#').
-# Args: project [database submitter]...
+# Args: project [database editor]...
 function psql_deploy() {
     declare -r project="$1"; shift
     # Deploy the instance
@@ -455,7 +455,7 @@ function psql_deploy() {
 
 # Withdraw (from) PostgreSQL
 # Cleanup all databases, even those prepended with the hash sign ('#').
-# Args: project [database submitter]...
+# Args: project [database editor]...
 function psql_withdraw() {
     declare -r project="$1"; shift
     psql_databases_withdraw "$project" "$PSQL_INSTANCE" "$PSQL_VIEWER" "$@"