Skip to content

Commit

Permalink
cloud: Expose operational and archive databases
Browse files Browse the repository at this point in the history
Introduce the concepts of "operational database" and "archive database".

The "operational database" is the one where most operations are done,
and which has recent data only, as opposed to "archive database", which
keeps all the data, but is either slow or expensive to query. In our
case the former is PostgreSQL and the latter is BigQuery.

Export the operational database specification to Cloud Functions as
"KCIDB_OPERATIONAL_DATABASE", and the archive database as
"KCIDB_ARCHIVE_DATABASE".
  • Loading branch information
spbnick committed Jan 16, 2024
1 parent 6b515e4 commit 1e731f8
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 0 deletions.
2 changes: 2 additions & 0 deletions cloud
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,8 @@ function execute_command() {
--smtp-topic="$smtp_topic"
--smtp-subscription="$smtp_subscription"
--pgpass-secret="$psql_pgpass_secret"
--op-database="$psql_kcidb_db"
--ar-database="$bigquery_kcidb_db"
--database="$database"
--clean-test-databases="$clean_test_databases"
--empty-test-databases="$empty_test_databases"
Expand Down
6 changes: 6 additions & 0 deletions kcidb/cloud/cloud_functions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ declare _CLOUD_FUNCTIONS_SH=
# --smtp-to-addrs=ADDRS --smtp-password-secret=NAME
# --smtp-topic=NAME --smtp-subscription=NAME
# --pgpass-secret=NAME
# --op-database=SPEC
# --ar-database=SPEC
# --database=SPEC
# --clean-test-databases=SPEC_LIST
# --empty-test-databases=SPEC_LIST
Expand All @@ -44,6 +46,8 @@ function cloud_functions_env() {
pgpass_secret \
cache_bucket_name \
cache_redirector_url \
op_database \
ar_database \
database \
clean_test_databases \
empty_test_databases \
Expand All @@ -59,6 +63,8 @@ function cloud_functions_env() {
[KCIDB_LOAD_QUEUE_OBJ_MAX]="8192"
[KCIDB_LOAD_QUEUE_TIMEOUT_SEC]="30"
[KCIDB_PGPASS_SECRET]="$pgpass_secret"
[KCIDB_OPERATIONAL_DATABASE]="$op_database"
[KCIDB_ARCHIVE_DATABASE]="$ar_database"
[KCIDB_DATABASE]="$database"
[KCIDB_DATABASE_LOAD_PERIOD_SEC]="180"
[KCIDB_CLEAN_TEST_DATABASES]="$clean_test_databases"
Expand Down
6 changes: 6 additions & 0 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@
# Maximum time for pulling maximum amount of submissions from the queue
LOAD_QUEUE_TIMEOUT_SEC = float(os.environ["KCIDB_LOAD_QUEUE_TIMEOUT_SEC"])

# The specification for the operational database (a part of DATABASE spec)
OPERATIONAL_DATABASE = os.environ["KCIDB_OPERATIONAL_DATABASE"]

# The specification for the archive database (a part of DATABASE spec)
ARCHIVE_DATABASE = os.environ["KCIDB_ARCHIVE_DATABASE"]

# The specification for the database submissions should be loaded into
DATABASE = os.environ["KCIDB_DATABASE"]

Expand Down

0 comments on commit 1e731f8

Please sign in to comment.