From e7974fb930e0975dad4417cb3225b939b4add657 Mon Sep 17 00:00:00 2001 From: jbeemster Date: Thu, 1 Feb 2024 13:41:33 +0100 Subject: [PATCH] Update GCP quickstart to SLULA (closes #90) --- terraform/gcp/iglu_server/default/README.md | 3 ++- terraform/gcp/iglu_server/default/main.tf | 4 +++- terraform/gcp/iglu_server/default/terraform.tfvars | 3 +++ terraform/gcp/iglu_server/default/variables.tf | 11 +++++++++++ terraform/gcp/iglu_server/secure/README.md | 3 ++- terraform/gcp/iglu_server/secure/main.tf | 4 +++- terraform/gcp/iglu_server/secure/terraform.tfvars | 3 +++ terraform/gcp/iglu_server/secure/variables.tf | 11 +++++++++++ terraform/gcp/pipeline/default/README.md | 11 ++++++----- terraform/gcp/pipeline/default/main.tf | 8 ++++++-- terraform/gcp/pipeline/default/target_bigquery.tf | 4 +++- terraform/gcp/pipeline/default/target_postgres.tf | 8 ++++++-- terraform/gcp/pipeline/default/terraform.tfvars | 3 +++ terraform/gcp/pipeline/default/variables.tf | 11 +++++++++++ terraform/gcp/pipeline/secure/README.md | 11 ++++++----- terraform/gcp/pipeline/secure/main.tf | 8 ++++++-- terraform/gcp/pipeline/secure/target_bigquery.tf | 4 +++- terraform/gcp/pipeline/secure/target_postgres.tf | 8 ++++++-- terraform/gcp/pipeline/secure/terraform.tfvars | 3 +++ terraform/gcp/pipeline/secure/variables.tf | 11 +++++++++++ 20 files changed, 108 insertions(+), 24 deletions(-) diff --git a/terraform/gcp/iglu_server/default/README.md b/terraform/gcp/iglu_server/default/README.md index b21f7de..ff6631f 100644 --- a/terraform/gcp/iglu_server/default/README.md +++ b/terraform/gcp/iglu_server/default/README.md @@ -16,7 +16,7 @@ No providers. |------|--------|---------| | [iglu\_db](#module\_iglu\_db) | snowplow-devops/cloud-sql/google | 0.3.0 | | [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/lb/google | 0.3.0 | -| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.4.0 | +| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.5.0 | ## Resources @@ -36,6 +36,7 @@ No resources. | [region](#input\_region) | The name of the region to deploy within | `string` | n/a | yes | | [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | n/a | yes | | [subnetwork](#input\_subnetwork) | The name of the sub-network to deploy within | `string` | n/a | yes | +| [accept\_limited\_use\_license](#input\_accept\_limited\_use\_license) | Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/) | `bool` | `false` | no | | [labels](#input\_labels) | The labels to append to the resources in this module | `map(string)` | `{}` | no | | [ssh\_key\_pairs](#input\_ssh\_key\_pairs) | The list of SSH key-pairs to add to the servers |
list(object({
user_name = string
public_key = string
}))
| `[]` | no | | [ssl\_information](#input\_ssl\_information) | The ID of an Google Managed certificate to bind to the load balancer |
object({
enabled = bool
certificate_id = string
})
|
{
"certificate_id": "",
"enabled": false
}
| no | diff --git a/terraform/gcp/iglu_server/default/main.tf b/terraform/gcp/iglu_server/default/main.tf index f50d9a6..4811f48 100644 --- a/terraform/gcp/iglu_server/default/main.tf +++ b/terraform/gcp/iglu_server/default/main.tf @@ -19,7 +19,9 @@ module "iglu_db" { module "iglu_server" { source = "snowplow-devops/iglu-server-ce/google" - version = "0.4.0" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-iglu-server" diff --git a/terraform/gcp/iglu_server/default/terraform.tfvars b/terraform/gcp/iglu_server/default/terraform.tfvars index 69ec6f3..88a3b72 100644 --- a/terraform/gcp/iglu_server/default/terraform.tfvars +++ b/terraform/gcp/iglu_server/default/terraform.tfvars @@ -1,3 +1,6 @@ +# Please accept the terms of the Snowplow Limited Use License Agreement to proceed. (https://docs.snowplow.io/limited-use-license-1.0/) +accept_limited_use_license = false + # Will be prefixed to all resource names # Use this to easily identify the resources created and provide entropy for subsequent environments prefix = "sp" diff --git a/terraform/gcp/iglu_server/default/variables.tf b/terraform/gcp/iglu_server/default/variables.tf index a1e3404..be76165 100644 --- a/terraform/gcp/iglu_server/default/variables.tf +++ b/terraform/gcp/iglu_server/default/variables.tf @@ -1,3 +1,14 @@ +variable "accept_limited_use_license" { + description = "Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/)" + type = bool + default = false + + validation { + condition = var.accept_limited_use_license + error_message = "Please accept the terms of the Snowplow Limited Use License Agreement to proceed." + } +} + variable "prefix" { description = "Will be prefixed to all resource names. Use to easily identify the resources created" type = string diff --git a/terraform/gcp/iglu_server/secure/README.md b/terraform/gcp/iglu_server/secure/README.md index b21f7de..ff6631f 100644 --- a/terraform/gcp/iglu_server/secure/README.md +++ b/terraform/gcp/iglu_server/secure/README.md @@ -16,7 +16,7 @@ No providers. |------|--------|---------| | [iglu\_db](#module\_iglu\_db) | snowplow-devops/cloud-sql/google | 0.3.0 | | [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/lb/google | 0.3.0 | -| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.4.0 | +| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.5.0 | ## Resources @@ -36,6 +36,7 @@ No resources. | [region](#input\_region) | The name of the region to deploy within | `string` | n/a | yes | | [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | n/a | yes | | [subnetwork](#input\_subnetwork) | The name of the sub-network to deploy within | `string` | n/a | yes | +| [accept\_limited\_use\_license](#input\_accept\_limited\_use\_license) | Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/) | `bool` | `false` | no | | [labels](#input\_labels) | The labels to append to the resources in this module | `map(string)` | `{}` | no | | [ssh\_key\_pairs](#input\_ssh\_key\_pairs) | The list of SSH key-pairs to add to the servers |
list(object({
user_name = string
public_key = string
}))
| `[]` | no | | [ssl\_information](#input\_ssl\_information) | The ID of an Google Managed certificate to bind to the load balancer |
object({
enabled = bool
certificate_id = string
})
|
{
"certificate_id": "",
"enabled": false
}
| no | diff --git a/terraform/gcp/iglu_server/secure/main.tf b/terraform/gcp/iglu_server/secure/main.tf index 3255a14..c9bedb7 100644 --- a/terraform/gcp/iglu_server/secure/main.tf +++ b/terraform/gcp/iglu_server/secure/main.tf @@ -19,7 +19,9 @@ module "iglu_db" { module "iglu_server" { source = "snowplow-devops/iglu-server-ce/google" - version = "0.4.0" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-iglu-server" diff --git a/terraform/gcp/iglu_server/secure/terraform.tfvars b/terraform/gcp/iglu_server/secure/terraform.tfvars index 04f31d6..1d489ff 100644 --- a/terraform/gcp/iglu_server/secure/terraform.tfvars +++ b/terraform/gcp/iglu_server/secure/terraform.tfvars @@ -1,3 +1,6 @@ +# Please accept the terms of the Snowplow Limited Use License Agreement to proceed. (https://docs.snowplow.io/limited-use-license-1.0/) +accept_limited_use_license = false + # Will be prefixed to all resource names # Use this to easily identify the resources created and provide entropy for subsequent environments prefix = "sp" diff --git a/terraform/gcp/iglu_server/secure/variables.tf b/terraform/gcp/iglu_server/secure/variables.tf index a1e3404..be76165 100644 --- a/terraform/gcp/iglu_server/secure/variables.tf +++ b/terraform/gcp/iglu_server/secure/variables.tf @@ -1,3 +1,14 @@ +variable "accept_limited_use_license" { + description = "Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/)" + type = bool + default = false + + validation { + condition = var.accept_limited_use_license + error_message = "Please accept the terms of the Snowplow Limited Use License Agreement to proceed." + } +} + variable "prefix" { description = "Will be prefixed to all resource names. Use to easily identify the resources created" type = string diff --git a/terraform/gcp/pipeline/default/README.md b/terraform/gcp/pipeline/default/README.md index 6e858c1..5e792da 100644 --- a/terraform/gcp/pipeline/default/README.md +++ b/terraform/gcp/pipeline/default/README.md @@ -17,15 +17,15 @@ | Name | Source | Version | |------|--------|---------| | [bad\_1\_topic](#module\_bad\_1\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | -| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.2.0 | +| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.3.0 | | [bq\_bad\_rows\_topic](#module\_bq\_bad\_rows\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | | [collector\_lb](#module\_collector\_lb) | snowplow-devops/lb/google | 0.3.0 | -| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.4.0 | -| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.2.0 | +| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.5.0 | +| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.3.0 | | [enriched\_topic](#module\_enriched\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | | [postgres\_db](#module\_postgres\_db) | snowplow-devops/cloud-sql/google | 0.3.0 | -| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 | -| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 | +| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.4.0 | +| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.4.0 | | [raw\_topic](#module\_raw\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | ## Resources @@ -50,6 +50,7 @@ | [region](#input\_region) | The name of the region to deploy within | `string` | n/a | yes | | [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | n/a | yes | | [subnetwork](#input\_subnetwork) | The name of the sub-network to deploy within | `string` | n/a | yes | +| [accept\_limited\_use\_license](#input\_accept\_limited\_use\_license) | Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/) | `bool` | `false` | no | | [bigquery\_db\_enabled](#input\_bigquery\_db\_enabled) | Whether to enable loading into a BigQuery Dataset | `bool` | `false` | no | | [bigquery\_loader\_dead\_letter\_bucket\_deploy](#input\_bigquery\_loader\_dead\_letter\_bucket\_deploy) | Whether this module should create a new bucket with the specified name - if the bucket already exists set this to false | `bool` | `true` | no | | [bigquery\_loader\_dead\_letter\_bucket\_name](#input\_bigquery\_loader\_dead\_letter\_bucket\_name) | The name of the GCS bucket to use for dead-letter output of loader | `string` | `""` | no | diff --git a/terraform/gcp/pipeline/default/main.tf b/terraform/gcp/pipeline/default/main.tf index 777c132..f98fdd5 100644 --- a/terraform/gcp/pipeline/default/main.tf +++ b/terraform/gcp/pipeline/default/main.tf @@ -46,7 +46,9 @@ module "enriched_topic" { # 2. Deploy Collector stack module "collector_pubsub" { source = "snowplow-devops/collector-pubsub-ce/google" - version = "0.4.0" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-collector" @@ -86,7 +88,9 @@ module "collector_lb" { # 3. Deploy Enrichment module "enrich_pubsub" { source = "snowplow-devops/enrich-pubsub-ce/google" - version = "0.2.0" + version = "0.3.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-enrich" diff --git a/terraform/gcp/pipeline/default/target_bigquery.tf b/terraform/gcp/pipeline/default/target_bigquery.tf index 3f2e6e2..742d18f 100644 --- a/terraform/gcp/pipeline/default/target_bigquery.tf +++ b/terraform/gcp/pipeline/default/target_bigquery.tf @@ -37,7 +37,9 @@ locals { module "bigquery_loader" { source = "snowplow-devops/bigquery-loader-pubsub-ce/google" - version = "0.2.0" + version = "0.3.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.bigquery_db_enabled ? 1 : 0 diff --git a/terraform/gcp/pipeline/default/target_postgres.tf b/terraform/gcp/pipeline/default/target_postgres.tf index 5df8a62..15e5093 100644 --- a/terraform/gcp/pipeline/default/target_postgres.tf +++ b/terraform/gcp/pipeline/default/target_postgres.tf @@ -20,7 +20,9 @@ module "postgres_db" { module "postgres_loader_enriched" { source = "snowplow-devops/postgres-loader-pubsub-ce/google" - version = "0.3.0" + version = "0.4.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.postgres_db_enabled ? 1 : 0 @@ -55,7 +57,9 @@ module "postgres_loader_enriched" { module "postgres_loader_bad" { source = "snowplow-devops/postgres-loader-pubsub-ce/google" - version = "0.3.0" + version = "0.4.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.postgres_db_enabled ? 1 : 0 diff --git a/terraform/gcp/pipeline/default/terraform.tfvars b/terraform/gcp/pipeline/default/terraform.tfvars index fe58814..cc38f66 100644 --- a/terraform/gcp/pipeline/default/terraform.tfvars +++ b/terraform/gcp/pipeline/default/terraform.tfvars @@ -1,3 +1,6 @@ +# Please accept the terms of the Snowplow Limited Use License Agreement to proceed. (https://docs.snowplow.io/limited-use-license-1.0/) +accept_limited_use_license = false + # Will be prefixed to all resource names # Use this to easily identify the resources created and provide entropy for subsequent environments prefix = "sp" diff --git a/terraform/gcp/pipeline/default/variables.tf b/terraform/gcp/pipeline/default/variables.tf index 269727a..498cf19 100644 --- a/terraform/gcp/pipeline/default/variables.tf +++ b/terraform/gcp/pipeline/default/variables.tf @@ -1,3 +1,14 @@ +variable "accept_limited_use_license" { + description = "Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/)" + type = bool + default = false + + validation { + condition = var.accept_limited_use_license + error_message = "Please accept the terms of the Snowplow Limited Use License Agreement to proceed." + } +} + variable "prefix" { description = "Will be prefixed to all resource names. Use to easily identify the resources created" type = string diff --git a/terraform/gcp/pipeline/secure/README.md b/terraform/gcp/pipeline/secure/README.md index 6e858c1..5e792da 100644 --- a/terraform/gcp/pipeline/secure/README.md +++ b/terraform/gcp/pipeline/secure/README.md @@ -17,15 +17,15 @@ | Name | Source | Version | |------|--------|---------| | [bad\_1\_topic](#module\_bad\_1\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | -| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.2.0 | +| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.3.0 | | [bq\_bad\_rows\_topic](#module\_bq\_bad\_rows\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | | [collector\_lb](#module\_collector\_lb) | snowplow-devops/lb/google | 0.3.0 | -| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.4.0 | -| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.2.0 | +| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.5.0 | +| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.3.0 | | [enriched\_topic](#module\_enriched\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | | [postgres\_db](#module\_postgres\_db) | snowplow-devops/cloud-sql/google | 0.3.0 | -| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 | -| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 | +| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.4.0 | +| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.4.0 | | [raw\_topic](#module\_raw\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | ## Resources @@ -50,6 +50,7 @@ | [region](#input\_region) | The name of the region to deploy within | `string` | n/a | yes | | [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | n/a | yes | | [subnetwork](#input\_subnetwork) | The name of the sub-network to deploy within | `string` | n/a | yes | +| [accept\_limited\_use\_license](#input\_accept\_limited\_use\_license) | Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/) | `bool` | `false` | no | | [bigquery\_db\_enabled](#input\_bigquery\_db\_enabled) | Whether to enable loading into a BigQuery Dataset | `bool` | `false` | no | | [bigquery\_loader\_dead\_letter\_bucket\_deploy](#input\_bigquery\_loader\_dead\_letter\_bucket\_deploy) | Whether this module should create a new bucket with the specified name - if the bucket already exists set this to false | `bool` | `true` | no | | [bigquery\_loader\_dead\_letter\_bucket\_name](#input\_bigquery\_loader\_dead\_letter\_bucket\_name) | The name of the GCS bucket to use for dead-letter output of loader | `string` | `""` | no | diff --git a/terraform/gcp/pipeline/secure/main.tf b/terraform/gcp/pipeline/secure/main.tf index 789a356..655db3c 100644 --- a/terraform/gcp/pipeline/secure/main.tf +++ b/terraform/gcp/pipeline/secure/main.tf @@ -46,7 +46,9 @@ module "enriched_topic" { # 2. Deploy Collector stack module "collector_pubsub" { source = "snowplow-devops/collector-pubsub-ce/google" - version = "0.4.0" + version = "0.5.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-collector" @@ -88,7 +90,9 @@ module "collector_lb" { # 3. Deploy Enrichment module "enrich_pubsub" { source = "snowplow-devops/enrich-pubsub-ce/google" - version = "0.2.0" + version = "0.3.0" + + accept_limited_use_license = var.accept_limited_use_license name = "${var.prefix}-enrich" diff --git a/terraform/gcp/pipeline/secure/target_bigquery.tf b/terraform/gcp/pipeline/secure/target_bigquery.tf index dd930e9..dc768f4 100644 --- a/terraform/gcp/pipeline/secure/target_bigquery.tf +++ b/terraform/gcp/pipeline/secure/target_bigquery.tf @@ -37,7 +37,9 @@ locals { module "bigquery_loader" { source = "snowplow-devops/bigquery-loader-pubsub-ce/google" - version = "0.2.0" + version = "0.3.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.bigquery_db_enabled ? 1 : 0 diff --git a/terraform/gcp/pipeline/secure/target_postgres.tf b/terraform/gcp/pipeline/secure/target_postgres.tf index 052133f..b632ccd 100644 --- a/terraform/gcp/pipeline/secure/target_postgres.tf +++ b/terraform/gcp/pipeline/secure/target_postgres.tf @@ -20,7 +20,9 @@ module "postgres_db" { module "postgres_loader_enriched" { source = "snowplow-devops/postgres-loader-pubsub-ce/google" - version = "0.3.0" + version = "0.4.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.postgres_db_enabled ? 1 : 0 @@ -57,7 +59,9 @@ module "postgres_loader_enriched" { module "postgres_loader_bad" { source = "snowplow-devops/postgres-loader-pubsub-ce/google" - version = "0.3.0" + version = "0.4.0" + + accept_limited_use_license = var.accept_limited_use_license count = var.postgres_db_enabled ? 1 : 0 diff --git a/terraform/gcp/pipeline/secure/terraform.tfvars b/terraform/gcp/pipeline/secure/terraform.tfvars index 8267274..222038e 100644 --- a/terraform/gcp/pipeline/secure/terraform.tfvars +++ b/terraform/gcp/pipeline/secure/terraform.tfvars @@ -1,3 +1,6 @@ +# Please accept the terms of the Snowplow Limited Use License Agreement to proceed. (https://docs.snowplow.io/limited-use-license-1.0/) +accept_limited_use_license = false + # Will be prefixed to all resource names # Use this to easily identify the resources created and provide entropy for subsequent environments prefix = "sp" diff --git a/terraform/gcp/pipeline/secure/variables.tf b/terraform/gcp/pipeline/secure/variables.tf index 269727a..498cf19 100644 --- a/terraform/gcp/pipeline/secure/variables.tf +++ b/terraform/gcp/pipeline/secure/variables.tf @@ -1,3 +1,14 @@ +variable "accept_limited_use_license" { + description = "Acceptance of the SLULA terms (https://docs.snowplow.io/limited-use-license-1.0/)" + type = bool + default = false + + validation { + condition = var.accept_limited_use_license + error_message = "Please accept the terms of the Snowplow Limited Use License Agreement to proceed." + } +} + variable "prefix" { description = "Will be prefixed to all resource names. Use to easily identify the resources created" type = string