From bac2e5d2abb5296bccbb2a168ab53d5e3dab2ed3 Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Tue, 12 Dec 2023 18:37:27 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20dbt=20run=20now=20functions=20for=20all?= =?UTF-8?q?=20compliances=E2=80=94AWS,=20GCP,=20AZURE=E2=80=94without=20sp?= =?UTF-8?q?ecifying=20the=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../aws_compliance__api_gateway_method_settings.sql | 2 ++ .../aws_compliance__foundational_security_free.sql | 2 +- .../models/aws_compliance__pci_dss_v3_2_1_free.sql | 2 ++ .../aws_compliance__api_gateway_method_settings.sql | 2 ++ .../models/aws_compliance__foundational_security.sql | 2 ++ .../models/aws_compliance__imds_v2.sql | 2 ++ .../models/aws_compliance__pci_dss_v3.2.1.sql | 2 ++ .../models/aws_compliance__public_egress.sql | 2 ++ .../models/aws_compliance__publicly_available.sql | 2 ++ .../efs/access_point_enforce_user_identity.sql | 2 -- .../aws_compliance__networks_acls_ingress_rules.sql | 2 ++ .../aws_compliance__security_group_egress_rules.sql | 2 ++ .../models/azure_compliance__cis_v1_3_0_free.sql | 2 ++ transformations/azure/compliance-free/profiles.yml | 12 ------------ .../models/azure_compliance__cis_v1_3_0.sql | 2 ++ .../azure/compliance-premium/profiles.yml | 12 ------------ .../azure/models/view_azure_nsg_dest_port_ranges.sql | 2 ++ .../azure/models/view_azure_nsg_rules.sql | 2 ++ .../models/view_azure_security_policy_parameters.sql | 2 ++ .../macros/{is_snowflake.sql => block_bigquery.sql} | 8 ++++---- transformations/macros/block_posgres.sql | 7 +++++++ transformations/macros/block_snowflake.sql | 7 +++++++ 22 files changed, 49 insertions(+), 31 deletions(-) delete mode 100644 transformations/azure/compliance-free/profiles.yml delete mode 100644 transformations/azure/compliance-premium/profiles.yml rename transformations/macros/{is_snowflake.sql => block_bigquery.sql} (57%) create mode 100644 transformations/macros/block_posgres.sql create mode 100644 transformations/macros/block_snowflake.sql diff --git a/transformations/aws/compliance-free/models/aws_compliance__api_gateway_method_settings.sql b/transformations/aws/compliance-free/models/aws_compliance__api_gateway_method_settings.sql index 579e5c7c6..bba559d35 100644 --- a/transformations/aws/compliance-free/models/aws_compliance__api_gateway_method_settings.sql +++ b/transformations/aws/compliance-free/models/aws_compliance__api_gateway_method_settings.sql @@ -1 +1,3 @@ +{{ config(enabled=block_bigquery()) }} + {{ api_gateway_method_settings() }} \ No newline at end of file diff --git a/transformations/aws/compliance-free/models/aws_compliance__foundational_security_free.sql b/transformations/aws/compliance-free/models/aws_compliance__foundational_security_free.sql index 48de7c74d..e63305dba 100644 --- a/transformations/aws/compliance-free/models/aws_compliance__foundational_security_free.sql +++ b/transformations/aws/compliance-free/models/aws_compliance__foundational_security_free.sql @@ -1,4 +1,4 @@ -{{ config(enabled=is_snowflake()) }} +{{ config(enabled=block_bigquery() and block_postgres()) }} with aggregated as ( diff --git a/transformations/aws/compliance-free/models/aws_compliance__pci_dss_v3_2_1_free.sql b/transformations/aws/compliance-free/models/aws_compliance__pci_dss_v3_2_1_free.sql index 2a1251e59..a11bffb3b 100644 --- a/transformations/aws/compliance-free/models/aws_compliance__pci_dss_v3_2_1_free.sql +++ b/transformations/aws/compliance-free/models/aws_compliance__pci_dss_v3_2_1_free.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery()) }} + with aggregated as ( ({{ autoscaling_groups_elb_check('pci_dss_v3.2.1','autoscaling.1') }}) diff --git a/transformations/aws/compliance-premium/models/aws_compliance__api_gateway_method_settings.sql b/transformations/aws/compliance-premium/models/aws_compliance__api_gateway_method_settings.sql index 579e5c7c6..bba559d35 100644 --- a/transformations/aws/compliance-premium/models/aws_compliance__api_gateway_method_settings.sql +++ b/transformations/aws/compliance-premium/models/aws_compliance__api_gateway_method_settings.sql @@ -1 +1,3 @@ +{{ config(enabled=block_bigquery()) }} + {{ api_gateway_method_settings() }} \ No newline at end of file diff --git a/transformations/aws/compliance-premium/models/aws_compliance__foundational_security.sql b/transformations/aws/compliance-premium/models/aws_compliance__foundational_security.sql index a2164234b..f830d4820 100644 --- a/transformations/aws/compliance-premium/models/aws_compliance__foundational_security.sql +++ b/transformations/aws/compliance-premium/models/aws_compliance__foundational_security.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery() and block_postgres()) }} + with aggregated as ( ({{ access_logs_enabled('foundational_security','cloudfront.5') }}) diff --git a/transformations/aws/compliance-premium/models/aws_compliance__imds_v2.sql b/transformations/aws/compliance-premium/models/aws_compliance__imds_v2.sql index d9cb9ef2c..5852df690 100644 --- a/transformations/aws/compliance-premium/models/aws_compliance__imds_v2.sql +++ b/transformations/aws/compliance-premium/models/aws_compliance__imds_v2.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery() and block_snowflake()) }} + with aggregated as ( ({{ ec2_not_imdsv2_instances('imds_v2','EC2-IMDSv2') }}) diff --git a/transformations/aws/compliance-premium/models/aws_compliance__pci_dss_v3.2.1.sql b/transformations/aws/compliance-premium/models/aws_compliance__pci_dss_v3.2.1.sql index 0126d8a05..c5553ba72 100644 --- a/transformations/aws/compliance-premium/models/aws_compliance__pci_dss_v3.2.1.sql +++ b/transformations/aws/compliance-premium/models/aws_compliance__pci_dss_v3.2.1.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery() and block_snowflake()) }} + with aggregated as ( ({{ autoscaling_groups_elb_check('pci_dss_v3.2.1','autoscaling.1') }}) diff --git a/transformations/aws/compliance-premium/models/aws_compliance__public_egress.sql b/transformations/aws/compliance-premium/models/aws_compliance__public_egress.sql index c394db40d..37b3175c8 100644 --- a/transformations/aws/compliance-premium/models/aws_compliance__public_egress.sql +++ b/transformations/aws/compliance-premium/models/aws_compliance__public_egress.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery() and block_snowflake()) }} + with aggregated as ( ({{ public_egress_sg_and_routing_instances('public_egress','ec2-all-instances-with-routes-and-security-groups') }}) diff --git a/transformations/aws/compliance-premium/models/aws_compliance__publicly_available.sql b/transformations/aws/compliance-premium/models/aws_compliance__publicly_available.sql index aad75748b..8de75f69b 100644 --- a/transformations/aws/compliance-premium/models/aws_compliance__publicly_available.sql +++ b/transformations/aws/compliance-premium/models/aws_compliance__publicly_available.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery() and block_snowflake()) }} + with aggregated as ( ({{ api_gw_publicly_accessible('publicly_available','API-Gateways') }}) diff --git a/transformations/aws/macros/efs/access_point_enforce_user_identity.sql b/transformations/aws/macros/efs/access_point_enforce_user_identity.sql index 75f490e26..ada9313ec 100644 --- a/transformations/aws/macros/efs/access_point_enforce_user_identity.sql +++ b/transformations/aws/macros/efs/access_point_enforce_user_identity.sql @@ -1,5 +1,3 @@ -{{ config(enabled=is_snowflake()) }} - {% macro access_point_enforce_user_identity(framework, check_id) %} {{ return(adapter.dispatch('access_point_enforce_user_identity')(framework, check_id)) }} {% endmacro %} diff --git a/transformations/aws/models/aws_compliance__networks_acls_ingress_rules.sql b/transformations/aws/models/aws_compliance__networks_acls_ingress_rules.sql index 0642eee7a..a42264819 100644 --- a/transformations/aws/models/aws_compliance__networks_acls_ingress_rules.sql +++ b/transformations/aws/models/aws_compliance__networks_acls_ingress_rules.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery()) }} + with aggregated as ( ({{ networks_acls_ingress_rules() }}) diff --git a/transformations/aws/models/aws_compliance__security_group_egress_rules.sql b/transformations/aws/models/aws_compliance__security_group_egress_rules.sql index ed4cf6a07..a36b6215e 100644 --- a/transformations/aws/models/aws_compliance__security_group_egress_rules.sql +++ b/transformations/aws/models/aws_compliance__security_group_egress_rules.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery()) }} + with aggregated as ( ({{ security_group_egress_rules() }}) diff --git a/transformations/azure/compliance-free/models/azure_compliance__cis_v1_3_0_free.sql b/transformations/azure/compliance-free/models/azure_compliance__cis_v1_3_0_free.sql index 41320f52f..bb25d30e6 100644 --- a/transformations/azure/compliance-free/models/azure_compliance__cis_v1_3_0_free.sql +++ b/transformations/azure/compliance-free/models/azure_compliance__cis_v1_3_0_free.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery()) }} + with aggregated as ( ({{iam_custom_subscription_owner_roles('cis_v1.3.0','1.21')}}) diff --git a/transformations/azure/compliance-free/profiles.yml b/transformations/azure/compliance-free/profiles.yml deleted file mode 100644 index a87ed4d91..000000000 --- a/transformations/azure/compliance-free/profiles.yml +++ /dev/null @@ -1,12 +0,0 @@ -azure_compliance: # This should match the name in your dbt_project.yml - target: dev - outputs: - dev: - type: postgres - host: 127.0.0.1 - user: postgres - pass: pass - port: 5432 - dbname: postgres - schema: public # default schema where dbt will build the models - threads: 1 # number of threads to use when running in parallel diff --git a/transformations/azure/compliance-premium/models/azure_compliance__cis_v1_3_0.sql b/transformations/azure/compliance-premium/models/azure_compliance__cis_v1_3_0.sql index 485c592c5..5ff93702d 100644 --- a/transformations/azure/compliance-premium/models/azure_compliance__cis_v1_3_0.sql +++ b/transformations/azure/compliance-premium/models/azure_compliance__cis_v1_3_0.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery()) }} + with aggregated as ( ({{iam_custom_subscription_owner_roles('cis_v1.3.0','1.21')}}) diff --git a/transformations/azure/compliance-premium/profiles.yml b/transformations/azure/compliance-premium/profiles.yml deleted file mode 100644 index a87ed4d91..000000000 --- a/transformations/azure/compliance-premium/profiles.yml +++ /dev/null @@ -1,12 +0,0 @@ -azure_compliance: # This should match the name in your dbt_project.yml - target: dev - outputs: - dev: - type: postgres - host: 127.0.0.1 - user: postgres - pass: pass - port: 5432 - dbname: postgres - schema: public # default schema where dbt will build the models - threads: 1 # number of threads to use when running in parallel diff --git a/transformations/azure/models/view_azure_nsg_dest_port_ranges.sql b/transformations/azure/models/view_azure_nsg_dest_port_ranges.sql index eb1a45937..57c6f592f 100644 --- a/transformations/azure/models/view_azure_nsg_dest_port_ranges.sql +++ b/transformations/azure/models/view_azure_nsg_dest_port_ranges.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery()) }} + with aggregated as ( ({{view_azure_nsg_dest_port_ranges()}}) diff --git a/transformations/azure/models/view_azure_nsg_rules.sql b/transformations/azure/models/view_azure_nsg_rules.sql index 82b034ba0..1b82b9158 100644 --- a/transformations/azure/models/view_azure_nsg_rules.sql +++ b/transformations/azure/models/view_azure_nsg_rules.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery()) }} + with aggregated as ( ({{view_azure_nsg_rules()}}) diff --git a/transformations/azure/models/view_azure_security_policy_parameters.sql b/transformations/azure/models/view_azure_security_policy_parameters.sql index e00be2b27..5fc9df57c 100644 --- a/transformations/azure/models/view_azure_security_policy_parameters.sql +++ b/transformations/azure/models/view_azure_security_policy_parameters.sql @@ -1,3 +1,5 @@ +{{ config(enabled=block_bigquery()) }} + with aggregated as ( ({{view_azure_security_policy_parameters()}}) diff --git a/transformations/macros/is_snowflake.sql b/transformations/macros/block_bigquery.sql similarity index 57% rename from transformations/macros/is_snowflake.sql rename to transformations/macros/block_bigquery.sql index c9f5f0ac8..2f1ed6046 100644 --- a/transformations/macros/is_snowflake.sql +++ b/transformations/macros/block_bigquery.sql @@ -1,7 +1,7 @@ -{% macro is_snowflake() %} - {% if target.name == 'snowflake' %} - {{ return(true) }} - {% else %} +{% macro block_bigquery() %} + {% if target.type == 'bigquery' %} {{ return(false) }} + {% else %} + {{ return(true) }} {% endif %} {% endmacro %} \ No newline at end of file diff --git a/transformations/macros/block_posgres.sql b/transformations/macros/block_posgres.sql new file mode 100644 index 000000000..63d472e1b --- /dev/null +++ b/transformations/macros/block_posgres.sql @@ -0,0 +1,7 @@ +{% macro block_postgres() %} + {% if target.type == 'postgres' %} + {{ return(false) }} + {% else %} + {{ return(true) }} + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/transformations/macros/block_snowflake.sql b/transformations/macros/block_snowflake.sql new file mode 100644 index 000000000..2faaf2100 --- /dev/null +++ b/transformations/macros/block_snowflake.sql @@ -0,0 +1,7 @@ +{% macro block_snowflake() %} + {% if target.type == 'snowflake' %} + {{ return(false) }} + {% else %} + {{ return(true) }} + {% endif %} +{% endmacro %} \ No newline at end of file