Merge branch 'main' into private/ron/aws_snowflake_found_free

cloudquery · Dec 13, 2023 · 54a4183 · 54a4183
2 parents 6627a22 + 17f53d8
commit 54a4183
Show file tree

Hide file tree

Showing 23 changed files with 114 additions and 33 deletions.
diff --git a/.github/workflows/transformations_aws_compliance_free.yml b/.github/workflows/transformations_aws_compliance_free.yml
@@ -107,19 +107,19 @@ jobs:
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml
-          dbt run --target dev-pg --profiles-dir ./tests --select aws_compliance__foundational_security_free
+          dbt run --target dev-pg --profiles-dir ./tests
         if: needs.prepare.outputs.postgres == 'true'
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Test Snowflake
         run: |
           cloudquery migrate tests/snowflake.yml
-          dbt run --target dev-snowflake --profiles-dir ./tests --select aws_compliance__foundational_security_free
+          dbt run --target dev-snowflake --profiles-dir ./tests
         if: needs.prepare.outputs.snowflake == 'true'
         env:
           SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}"
       - name: Test BigQuery
         if: needs.prepare.outputs.bigquery == 'true'
         run: |
           cloudquery migrate tests/bigquery.yml
-          dbt run --target dev-bigquery --profiles-dir ./tests --select aws_compliance__foundational_security_free
+          dbt run --target dev-bigquery --profiles-dir ./tests
diff --git a/.github/workflows/transformations_aws_compliance_premium.yml b/.github/workflows/transformations_aws_compliance_premium.yml
@@ -107,19 +107,19 @@ jobs:
       - name: Test Postgres
         run: |
           cloudquery migrate tests/postgres.yml
-          dbt run --target dev-pg --profiles-dir ./tests --select aws_compliance__foundational_security
+          dbt run --target dev-pg --profiles-dir ./tests
         if: needs.prepare.outputs.postgres == 'true'
         env:
           CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: Test Snowflake
         run: |
           cloudquery migrate tests/snowflake.yml
-          dbt run --target dev-snowflake --profiles-dir ./tests --select aws_compliance__foundational_security
+          dbt run --target dev-snowflake --profiles-dir ./tests
         if: needs.prepare.outputs.snowflake == 'true'
         env:
           SNOWFLAKE_CONNECTION_STRING: "${{ secrets.SNOW_USER }}:${{ secrets.SNOW_PASSWORD }}@${{ secrets.SNOW_ACCOUNT }}.${{ secrets.SNOW_REGION }}/${{ secrets.SNOW_DATABASE }}/${{ secrets.SNOW_SCHEMA }}?warehouse=${{ secrets.SNOW_WAREHOUSE }}"
       - name: Test BigQuery
         if: needs.prepare.outputs.bigquery == 'true'
         run: |
           cloudquery migrate tests/bigquery.yml
-          dbt run --target dev-bigquery --profiles-dir ./tests --select aws_compliance__foundational_security
+          dbt run --target dev-bigquery --profiles-dir ./tests
diff --git a/.github/workflows/validate_transformation_release.yml b/.github/workflows/validate_transformation_release.yml
@@ -127,18 +127,22 @@ jobs:
         run: |
           mkdir -p temp/build
           unzip -o ${{ needs.prepare.outputs.zipPath }} -d temp/build
+          cp -r ${{ needs.prepare.outputs.transformation_dir }}/seeds/ temp/build
       - name: Run Unpacked DBT Postgres
         if: needs.prepare.outputs.postgres == 'true'
         working-directory: ./temp/build
         run: |
+          dbt seed --target dev-pg --profiles-dir ../../${{ needs.prepare.outputs.transformation_dir }}/tests
           dbt run --target dev-pg --profiles-dir ../../${{ needs.prepare.outputs.transformation_dir }}/tests
       - name: Run Unpacked DBT Snowflake
         if: needs.prepare.outputs.snowflake == 'true'
         working-directory: ./temp/build
         run: |
+          dbt seed --target dev-snowflake --profiles-dir ../../${{ needs.prepare.outputs.transformation_dir }}/tests
           dbt run --target dev-snowflake --profiles-dir ../../${{ needs.prepare.outputs.transformation_dir }}/tests
       - name: Run Unpacked DBT BigQuery
         if: needs.prepare.outputs.bigquery == 'true'
         working-directory: ./temp/build
         run: |
+          dbt seed --target dev-bigquery --profiles-dir ../../${{ needs.prepare.outputs.transformation_dir }}/tests
           dbt run --target dev-bigquery --profiles-dir ../../${{ needs.prepare.outputs.transformation_dir }}/tests
diff --git a/.release-please-manifest.json b/.release-please-manifest.json
@@ -24,5 +24,7 @@
   "transformations/aws/encryption": "1.1.0",
   "transformations/aws/encryption+FILLER": "0.0.0",
   "transformations/aws/asset-inventory-free": "1.1.1",
-  "visualizations/aws/asset_inventory": "1.0.0"
+  "transformations/aws/asset-inventory+FILLER": "0.0.0",
+  "visualizations/aws/asset_inventory": "1.0.0",
+  "visualizations/aws/asset_inventory+FILLER": "0.0.0"
 }
diff --git a/scripts/dbt-pack/src/index.js b/scripts/dbt-pack/src/index.js
@@ -70,6 +70,9 @@ const addDependencies = (node, allNodes, allMacros, filesToPack) => {
     // Skip dbt internal macros
     return;
   }
+  if (node.resource_type !== "model" && node.resource_type !== "macro") {
+    return;
+  }
   filesToPack.add(node.original_file_path);
   const dependsOnNodes = node.depends_on?.nodes ?? [];
   for (const dependency of dependsOnNodes) {

diff --git a/transformations/aws/asset-inventory-free/README.md b/transformations/aws/asset-inventory-free/README.md
@@ -12,7 +12,7 @@ This package contains dbt models (views) that aggregates AWS Resources for AWS A
 
 One of the below databases:
 
-- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql/v6.1.3/docs)
+- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql)
 
 #### dbt Installation
 

diff --git a/transformations/aws/compliance-free/README.md b/transformations/aws/compliance-free/README.md
@@ -9,8 +9,8 @@
 
 One of the below databases
 
-- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql/v6.1.3/docs)
-- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake/v3.3.3/docs)
+- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql)
+- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake)
 
 ### What's in the pack
 

diff --git a/transformations/aws/compliance-free/tests/bigquery.yml b/transformations/aws/compliance-free/tests/bigquery.yml
@@ -0,0 +1,18 @@
+kind: source
+spec:
+  name: aws
+  path: cloudquery/aws
+  version: "v22.19.2" # latest version of source aws plugin
+  destinations: ["bigquery"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: bigquery
+  path: cloudquery/bigquery
+  registry: cloudquery
+  version: "v3.3.9"
+  write_mode: "append"
+  spec:
+    project_id: cq-integration-tests
+    dataset_id: policies_premium_ci_test
diff --git a/transformations/aws/compliance-free/tests/profiles.yml b/transformations/aws/compliance-free/tests/profiles.yml
@@ -24,6 +24,6 @@ aws_compliance: # This should match the name in your dbt_project.yml
     dev-bigquery:
       type: bigquery
       method: oauth
-      project: cq-playground
-      dataset: bigquery-test # You can also use "schema" here
+      project: cq-integration-tests
+      dataset: policies_premium_ci_test # You can also use "schema" here
       threads: 4 # Must be a value of 1 or greater 
diff --git a/transformations/aws/compliance-premium/README.md b/transformations/aws/compliance-premium/README.md
@@ -9,8 +9,8 @@
 
 One of the below databases
 
-- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql/v6.1.3/docs)
-- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake/v3.3.3/docs)
+- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql)
+- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake)
 - [BigQuery](https://hub.cloudquery.io/plugins/destination/cloudquery/bigquery)
 
 ### What's in the pack

diff --git a/transformations/aws/compliance-premium/tests/bigquery.yml b/transformations/aws/compliance-premium/tests/bigquery.yml
@@ -0,0 +1,18 @@
+kind: source
+spec:
+  name: aws
+  path: cloudquery/aws
+  version: "v22.19.2" # latest version of source aws plugin
+  destinations: ["bigquery"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: bigquery
+  path: cloudquery/bigquery
+  registry: cloudquery
+  version: "v3.3.9"
+  write_mode: "append"
+  spec:
+    project_id: cq-integration-tests
+    dataset_id: policies_premium_ci_test
diff --git a/transformations/aws/compliance-premium/tests/profiles.yml b/transformations/aws/compliance-premium/tests/profiles.yml
@@ -24,6 +24,6 @@ aws_compliance: # This should match the name in your dbt_project.yml
     dev-bigquery:
       type: bigquery
       method: oauth
-      project: cq-playground
-      dataset: bigquery-test # You can also use "schema" here
+      project: cq-integration-tests
+      dataset: policies_premium_ci_test # You can also use "schema" here
       threads: 4 # Must be a value of 1 or greater 
diff --git a/transformations/aws/cost/dbt_project.yml b/transformations/aws/cost/dbt_project.yml
@@ -12,7 +12,7 @@ profile: 'aws_cost'
 # These configurations specify where dbt should look for different types of files.
 # The `model-paths` config, for example, states that models in this project can be
 # found in the "models/" directory. You probably won't need to change these!
-model-paths: ["models", "../models"]
+model-paths: ["models"]
 analysis-paths: ["analyses"]
 test-paths: ["tests"]
 seed-paths: ["seeds"]

diff --git a/transformations/azure/compliance-free/README.md b/transformations/azure/compliance-free/README.md
@@ -79,8 +79,8 @@ dbt compile
 
 One of the below databases
 
-- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql/v6.1.3/docs)
-- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake/v3.3.3/docs)
+- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql)
+- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake)
 
 ### What's in the pack
 

diff --git a/transformations/azure/compliance-premium/README.md b/transformations/azure/compliance-premium/README.md
@@ -13,8 +13,8 @@ This package contains dbt models (views) that gives compliance insights from Clo
 
 One of the below databases:
 
-- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql/v6.1.3/docs)
-- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake/v3.3.3/docs)
+- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql)
+- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake)
 
 
 #### dbt Installation

diff --git a/transformations/gcp/compliance-free/README.md b/transformations/gcp/compliance-free/README.md
@@ -9,8 +9,8 @@
 
 One of the below databases
 
-- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql/v6.1.3/docs)
-- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake/v3.3.3/docs)
+- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql)
+- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake)
 - [BigQuery](https://hub.cloudquery.io/plugins/destination/cloudquery/bigquery)
 
 ### What's in the pack

diff --git a/transformations/gcp/compliance-free/tests/bigquery.yml b/transformations/gcp/compliance-free/tests/bigquery.yml
@@ -0,0 +1,18 @@
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  version: "v10.0.0" # latest version of source gcp plugin
+  destinations: ["bigquery"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: bigquery
+  path: cloudquery/bigquery
+  registry: cloudquery
+  version: "v3.3.9"
+  write_mode: "append"
+  spec:
+    project_id: cq-integration-tests
+    dataset_id: policies_premium_ci_test
diff --git a/transformations/gcp/compliance-free/tests/profiles.yml b/transformations/gcp/compliance-free/tests/profiles.yml
@@ -24,6 +24,6 @@ gcp_compliance: # This should match the name in your dbt_project.yml
     dev-bigquery:
       type: bigquery
       method: oauth
-      project: cq-playground
-      dataset: bigquery-test # You can also use "schema" here
+      project: cq-integration-tests
+      dataset: policies_premium_ci_test # You can also use "schema" here
       threads: 4 # Must be a value of 1 or greater 
diff --git a/transformations/gcp/compliance-premium/README.md b/transformations/gcp/compliance-premium/README.md
@@ -9,8 +9,8 @@
 
 One of the below databases
 
-- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql/v6.1.3/docs)
-- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake/v3.3.3/docs)
+- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql)
+- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake)
 - [BigQuery](https://hub.cloudquery.io/plugins/destination/cloudquery/bigquery)
 
 ### What's in the pack

diff --git a/transformations/gcp/compliance-premium/tests/bigquery.yml b/transformations/gcp/compliance-premium/tests/bigquery.yml
@@ -0,0 +1,18 @@
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  version: "v10.0.0" # latest version of source gcp plugin
+  destinations: ["bigquery"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: bigquery
+  path: cloudquery/bigquery
+  registry: cloudquery
+  version: "v3.3.9"
+  write_mode: "append"
+  spec:
+    project_id: cq-integration-tests
+    dataset_id: policies_premium_ci_test
diff --git a/transformations/gcp/compliance-premium/tests/profiles.yml b/transformations/gcp/compliance-premium/tests/profiles.yml
@@ -24,6 +24,6 @@ gcp_compliance: # This should match the name in your dbt_project.yml
     dev-bigquery:
       type: bigquery
       method: oauth
-      project: cq-playground
-      dataset: bigquery-test # You can also use "schema" here
+      project: cq-integration-tests
+      dataset: policies_premium_ci_test # You can also use "schema" here
       threads: 4 # Must be a value of 1 or greater 
diff --git a/transformations/k8s/compliance-free/README.md b/transformations/k8s/compliance-free/README.md
@@ -12,8 +12,8 @@ This package contains dbt models (views) that gives compliance insights from Clo
 
 One of the below databases
 
-- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql/v6.1.3/docs)
-- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake/v3.3.3/docs)
+- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql)
+- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake)
 
 ### What's in the pack
 

diff --git a/transformations/k8s/compliance-premium/README.md b/transformations/k8s/compliance-premium/README.md
@@ -12,8 +12,8 @@ This package contains dbt models (views) that gives compliance insights from Clo
 
 One of the below databases
 
-- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql/v6.1.3/docs)
-- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake/v3.3.3/docs)
+- [PostgreSQL](https://hub.cloudquery.io/plugins/destination/cloudquery/postgresql)
+- [Snowflake](https://hub.cloudquery.io/plugins/destination/cloudquery/snowflake)
 
 ### What's in the pack