chore: Add BigQuery tests to AWS and GCP (#421)

cloudquery · Dec 12, 2023 · e32c5c1 · e32c5c1
1 parent b7ff919
commit e32c5c1
Show file tree

Hide file tree

Showing 10 changed files with 84 additions and 12 deletions.
diff --git a/.github/workflows/transformations_aws_compliance_free.yml b/.github/workflows/transformations_aws_compliance_free.yml
@@ -107,7 +107,7 @@ jobs:
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml
-          dbt run --target dev-pg --profiles-dir ./tests --select aws_compliance__foundational_security_free
+          dbt run --target dev-pg --profiles-dir ./tests
         if: needs.prepare.outputs.postgres == 'true'
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
@@ -122,4 +122,4 @@ jobs:
         if: needs.prepare.outputs.bigquery == 'true'
         run: |
           cloudquery migrate tests/bigquery.yml
-          dbt run --target dev-bigquery --profiles-dir ./tests --select aws_compliance__foundational_security_free
+          dbt run --target dev-bigquery --profiles-dir ./tests
diff --git a/.github/workflows/transformations_aws_compliance_premium.yml b/.github/workflows/transformations_aws_compliance_premium.yml
@@ -107,7 +107,7 @@ jobs:
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml
-          dbt run --target dev-pg --profiles-dir ./tests --select aws_compliance__foundational_security
+          dbt run --target dev-pg --profiles-dir ./tests
         if: needs.prepare.outputs.postgres == 'true'
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
@@ -122,4 +122,4 @@ jobs:
         if: needs.prepare.outputs.bigquery == 'true'
         run: |
           cloudquery migrate tests/bigquery.yml
-          dbt run --target dev-bigquery --profiles-dir ./tests --select aws_compliance__foundational_security
+          dbt run --target dev-bigquery --profiles-dir ./tests
diff --git a/transformations/aws/compliance-free/tests/bigquery.yml b/transformations/aws/compliance-free/tests/bigquery.yml
@@ -0,0 +1,18 @@
+kind: source
+spec:
+  name: aws
+  path: cloudquery/aws
+  version: "v22.19.2" # latest version of source aws plugin
+  destinations: ["bigquery"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: bigquery
+  path: cloudquery/bigquery
+  registry: cloudquery
+  version: "v3.3.9"
+  write_mode: "append"
+  spec:
+    project_id: cq-integration-tests
+    dataset_id: policies_premium_ci_test
diff --git a/transformations/aws/compliance-free/tests/profiles.yml b/transformations/aws/compliance-free/tests/profiles.yml
@@ -24,6 +24,6 @@ aws_compliance: # This should match the name in your dbt_project.yml
     dev-bigquery:
       type: bigquery
       method: oauth
-      project: cq-playground
-      dataset: bigquery-test # You can also use "schema" here
+      project: cq-integration-tests
+      dataset: policies_premium_ci_test # You can also use "schema" here
       threads: 4 # Must be a value of 1 or greater 
diff --git a/transformations/aws/compliance-premium/tests/bigquery.yml b/transformations/aws/compliance-premium/tests/bigquery.yml
@@ -0,0 +1,18 @@
+kind: source
+spec:
+  name: aws
+  path: cloudquery/aws
+  version: "v22.19.2" # latest version of source aws plugin
+  destinations: ["bigquery"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: bigquery
+  path: cloudquery/bigquery
+  registry: cloudquery
+  version: "v3.3.9"
+  write_mode: "append"
+  spec:
+    project_id: cq-integration-tests
+    dataset_id: policies_premium_ci_test
diff --git a/transformations/aws/compliance-premium/tests/profiles.yml b/transformations/aws/compliance-premium/tests/profiles.yml
@@ -24,6 +24,6 @@ aws_compliance: # This should match the name in your dbt_project.yml
     dev-bigquery:
       type: bigquery
       method: oauth
-      project: cq-playground
-      dataset: bigquery-test # You can also use "schema" here
+      project: cq-integration-tests
+      dataset: policies_premium_ci_test # You can also use "schema" here
       threads: 4 # Must be a value of 1 or greater 
diff --git a/transformations/gcp/compliance-free/tests/bigquery.yml b/transformations/gcp/compliance-free/tests/bigquery.yml
@@ -0,0 +1,18 @@
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  version: "v10.0.0" # latest version of source gcp plugin
+  destinations: ["bigquery"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: bigquery
+  path: cloudquery/bigquery
+  registry: cloudquery
+  version: "v3.3.9"
+  write_mode: "append"
+  spec:
+    project_id: cq-integration-tests
+    dataset_id: policies_premium_ci_test
diff --git a/transformations/gcp/compliance-free/tests/profiles.yml b/transformations/gcp/compliance-free/tests/profiles.yml
@@ -24,6 +24,6 @@ gcp_compliance: # This should match the name in your dbt_project.yml
     dev-bigquery:
       type: bigquery
       method: oauth
-      project: cq-playground
-      dataset: bigquery-test # You can also use "schema" here
+      project: cq-integration-tests
+      dataset: policies_premium_ci_test # You can also use "schema" here
       threads: 4 # Must be a value of 1 or greater 
diff --git a/transformations/gcp/compliance-premium/tests/bigquery.yml b/transformations/gcp/compliance-premium/tests/bigquery.yml
@@ -0,0 +1,18 @@
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  version: "v10.0.0" # latest version of source gcp plugin
+  destinations: ["bigquery"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: bigquery
+  path: cloudquery/bigquery
+  registry: cloudquery
+  version: "v3.3.9"
+  write_mode: "append"
+  spec:
+    project_id: cq-integration-tests
+    dataset_id: policies_premium_ci_test
diff --git a/transformations/gcp/compliance-premium/tests/profiles.yml b/transformations/gcp/compliance-premium/tests/profiles.yml
@@ -24,6 +24,6 @@ gcp_compliance: # This should match the name in your dbt_project.yml
     dev-bigquery:
       type: bigquery
       method: oauth
-      project: cq-playground
-      dataset: bigquery-test # You can also use "schema" here
+      project: cq-integration-tests
+      dataset: policies_premium_ci_test # You can also use "schema" here
       threads: 4 # Must be a value of 1 or greater