From 221f04d74a414c877707a4ddd08d68802f01a059 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Tue, 21 Nov 2023 17:23:12 +0200 Subject: [PATCH 01/49] feat: Update `aws_cloudtrail_trail_event_selectors` schema --- .gitignore | 2 +- .../macros/cloudtrail/enabled_in_all_regions.sql | 15 +++++++++++---- .../aws/macros/log_metric_filter_and_alarm.sql | 14 +++++++++++--- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index d48b33794..61e3fb1a7 100644 --- a/.gitignore +++ b/.gitignore @@ -157,7 +157,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +.idea/ .DS_Store transformations/aws_compliance/.user.yml diff --git a/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql b/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql index e43c21741..17b710eac 100644 --- a/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql +++ b/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql @@ -10,10 +10,17 @@ select aws_cloudtrail_trails.account_id, arn as resource_id, case - when is_multi_region_trail = FALSE or ( - is_multi_region_trail = TRUE and ( - read_write_type != 'All' or include_management_events = FALSE - )) then 'fail' + when aws_cloudtrail_trails.is_multi_region_trail = FALSE then 'fail' + when exists(select * + from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.event_selectors) as es + where es ->>'ReadWriteType' != 'All' or (es->>'IncludeManagementEvents')::boolean = FALSE) + then 'fail' + when exists(select * + from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.advanced_event_selectors) as aes + where exists(select * + from jsonb_array_elements(aes ->'FieldSelectors') as aes_fs + where aes_fs ->>'Field' = 'readOnly')) + then 'fail' else 'pass' end as status from aws_cloudtrail_trails diff --git a/transformations/aws/macros/log_metric_filter_and_alarm.sql b/transformations/aws/macros/log_metric_filter_and_alarm.sql index 1e8b4d729..970668b1c 100644 --- a/transformations/aws/macros/log_metric_filter_and_alarm.sql +++ b/transformations/aws/macros/log_metric_filter_and_alarm.sql @@ -8,6 +8,16 @@ with af as ( select distinct a.arn, a.actions_enabled, a.alarm_actions, m->'MetricStat'->'Metric'->>'MetricName' as metric_name -- TODO check from aws_cloudwatch_alarms a, jsonb_array_elements(a.metrics) as m +), +tes as ( + select trail_arn from aws_cloudtrail_trail_event_selectors + where exists( + select * from jsonb_array_elements(event_selectors) as es + where es ->>'ReadWriteType' = 'All' and (es->>'IncludeManagementEvents')::boolean = TRUE + ) or exists( + select * from jsonb_array_elements(advanced_event_selectors) as aes + where not exists(select * from jsonb_array_elements(aes ->'FieldSelectors') as aes_fs where aes_fs ->>'Field' = 'readOnly') + ) ) select t.account_id, @@ -15,14 +25,12 @@ select t.cloud_watch_logs_log_group_arn, mf.filter_pattern as pattern from aws_cloudtrail_trails t -inner join aws_cloudtrail_trail_event_selectors tes on t.arn = tes.trail_arn +inner join tes on t.arn = tes.trail_arn inner join aws_cloudwatchlogs_metric_filters mf on mf.log_group_name = t.cloudwatch_logs_log_group_name inner join af on mf.filter_name = af.metric_name inner join aws_sns_subscriptions ss on ss.topic_arn = ANY(af.alarm_actions) where t.is_multi_region_trail = TRUE and (t.status->>'IsLogging')::boolean = TRUE - and tes.include_management_events = TRUE - and tes.read_write_type = 'All' and ss.arn like 'aws:arn:%' {% endmacro %} From eac98bf5d3cf8e441b581fa3097b584f6dbd238c Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 09:50:15 +0200 Subject: [PATCH 02/49] bump aws in data-resilience --- transformations/aws/data-resilience/tests/postgres.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformations/aws/data-resilience/tests/postgres.yml b/transformations/aws/data-resilience/tests/postgres.yml index 8daf5d1e9..05bc5efac 100644 --- a/transformations/aws/data-resilience/tests/postgres.yml +++ b/transformations/aws/data-resilience/tests/postgres.yml @@ -2,7 +2,8 @@ kind: source spec: name: aws path: cloudquery/aws - version: "v22.19.2" # latest version of source aws plugin + registry: cloudquery + version: "v23.2.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- @@ -14,4 +15,4 @@ spec: version: "v7.1.1" # latest version of postgresql plugin spec: batch_size: 10000 - connection_string: ${CQ_DSN} \ No newline at end of file + connection_string: ${CQ_DSN} From 482310a2e9973c70c1575623efc8c437339b942f Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 09:53:44 +0200 Subject: [PATCH 03/49] bump aws in asset-free --- transformations/aws/asset-inventory-free/tests/postgres.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformations/aws/asset-inventory-free/tests/postgres.yml b/transformations/aws/asset-inventory-free/tests/postgres.yml index 053925358..0a1c3080a 100644 --- a/transformations/aws/asset-inventory-free/tests/postgres.yml +++ b/transformations/aws/asset-inventory-free/tests/postgres.yml @@ -2,7 +2,8 @@ kind: source spec: name: aws path: cloudquery/aws - version: "v22.19.2" # latest version of source aws plugin + registry: cloudquery + version: "v23.2.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- @@ -14,4 +15,4 @@ spec: version: "v7.1.2" # latest version of postgresql plugin spec: batch_size: 10000 - connection_string: ${CQ_DSN} \ No newline at end of file + connection_string: ${CQ_DSN} From b0aa0ce6114a41b0fd126f2d2fea0f62ce7eb6d9 Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 09:54:11 +0200 Subject: [PATCH 04/49] bump aws-pg in compliance free --- transformations/aws/compliance-free/tests/postgres.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformations/aws/compliance-free/tests/postgres.yml b/transformations/aws/compliance-free/tests/postgres.yml index 053925358..0a1c3080a 100644 --- a/transformations/aws/compliance-free/tests/postgres.yml +++ b/transformations/aws/compliance-free/tests/postgres.yml @@ -2,7 +2,8 @@ kind: source spec: name: aws path: cloudquery/aws - version: "v22.19.2" # latest version of source aws plugin + registry: cloudquery + version: "v23.2.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- @@ -14,4 +15,4 @@ spec: version: "v7.1.2" # latest version of postgresql plugin spec: batch_size: 10000 - connection_string: ${CQ_DSN} \ No newline at end of file + connection_string: ${CQ_DSN} From 303dd10c383c96b69f3b1963416373c0dd6bad0f Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 09:54:36 +0200 Subject: [PATCH 05/49] bumop aws in compliance-free-bq --- transformations/aws/compliance-free/tests/bigquery.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformations/aws/compliance-free/tests/bigquery.yml b/transformations/aws/compliance-free/tests/bigquery.yml index fcb4e3992..0960d000c 100644 --- a/transformations/aws/compliance-free/tests/bigquery.yml +++ b/transformations/aws/compliance-free/tests/bigquery.yml @@ -2,7 +2,8 @@ kind: source spec: name: aws path: cloudquery/aws - version: "v22.19.2" # latest version of source aws plugin + registry: cloudquery + version: "v23.2.0" # latest version of source aws plugin destinations: ["bigquery"] tables: ["*"] --- @@ -15,4 +16,4 @@ spec: write_mode: "append" spec: project_id: cq-integration-tests - dataset_id: policies_premium_ci_test \ No newline at end of file + dataset_id: policies_premium_ci_test From 46914ba9655c4cf0b4d3cc5e26f3158bcc55a131 Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 09:55:07 +0200 Subject: [PATCH 06/49] bump aws in compliance-free snowflake --- transformations/aws/compliance-free/tests/snowflake.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformations/aws/compliance-free/tests/snowflake.yml b/transformations/aws/compliance-free/tests/snowflake.yml index e791008dc..ad5782482 100644 --- a/transformations/aws/compliance-free/tests/snowflake.yml +++ b/transformations/aws/compliance-free/tests/snowflake.yml @@ -2,7 +2,8 @@ kind: source spec: name: aws path: cloudquery/aws - version: "v22.19.2" # latest version of source aws plugin + registry: cloudquery + version: "v23.2.0" # latest version of source aws plugin destinations: ["snowflake"] tables: ["*"] --- From 33b2ea2836fdc673582d9b2d101220cda41ef85a Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 09:55:28 +0200 Subject: [PATCH 07/49] Update bigquery.yml --- transformations/aws/compliance-premium/tests/bigquery.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformations/aws/compliance-premium/tests/bigquery.yml b/transformations/aws/compliance-premium/tests/bigquery.yml index fcb4e3992..0960d000c 100644 --- a/transformations/aws/compliance-premium/tests/bigquery.yml +++ b/transformations/aws/compliance-premium/tests/bigquery.yml @@ -2,7 +2,8 @@ kind: source spec: name: aws path: cloudquery/aws - version: "v22.19.2" # latest version of source aws plugin + registry: cloudquery + version: "v23.2.0" # latest version of source aws plugin destinations: ["bigquery"] tables: ["*"] --- @@ -15,4 +16,4 @@ spec: write_mode: "append" spec: project_id: cq-integration-tests - dataset_id: policies_premium_ci_test \ No newline at end of file + dataset_id: policies_premium_ci_test From c43492119b6a33e9706e5f9585b7fe8113017c8d Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 09:55:36 +0200 Subject: [PATCH 08/49] Update postgres.yml --- transformations/aws/compliance-premium/tests/postgres.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformations/aws/compliance-premium/tests/postgres.yml b/transformations/aws/compliance-premium/tests/postgres.yml index 053925358..0a1c3080a 100644 --- a/transformations/aws/compliance-premium/tests/postgres.yml +++ b/transformations/aws/compliance-premium/tests/postgres.yml @@ -2,7 +2,8 @@ kind: source spec: name: aws path: cloudquery/aws - version: "v22.19.2" # latest version of source aws plugin + registry: cloudquery + version: "v23.2.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- @@ -14,4 +15,4 @@ spec: version: "v7.1.2" # latest version of postgresql plugin spec: batch_size: 10000 - connection_string: ${CQ_DSN} \ No newline at end of file + connection_string: ${CQ_DSN} From f560430fa4fdd5ae5301d106a0aefe93197fd27c Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 09:55:47 +0200 Subject: [PATCH 09/49] Update snowflake.yml --- transformations/aws/compliance-premium/tests/snowflake.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index e791008dc..ad5782482 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -2,7 +2,8 @@ kind: source spec: name: aws path: cloudquery/aws - version: "v22.19.2" # latest version of source aws plugin + registry: cloudquery + version: "v23.2.0" # latest version of source aws plugin destinations: ["snowflake"] tables: ["*"] --- From fb81760e37428fcfd2062c7b97d074266bb32330 Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 09:56:32 +0200 Subject: [PATCH 10/49] Update postgres.yml --- transformations/aws/cost/tests/postgres.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformations/aws/cost/tests/postgres.yml b/transformations/aws/cost/tests/postgres.yml index 053925358..0a1c3080a 100644 --- a/transformations/aws/cost/tests/postgres.yml +++ b/transformations/aws/cost/tests/postgres.yml @@ -2,7 +2,8 @@ kind: source spec: name: aws path: cloudquery/aws - version: "v22.19.2" # latest version of source aws plugin + registry: cloudquery + version: "v23.2.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- @@ -14,4 +15,4 @@ spec: version: "v7.1.2" # latest version of postgresql plugin spec: batch_size: 10000 - connection_string: ${CQ_DSN} \ No newline at end of file + connection_string: ${CQ_DSN} From 88d0ad5d46e8a97cdc9057cf2e4bbd638caf523f Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 09:57:11 +0200 Subject: [PATCH 11/49] Update postgres.yml --- transformations/aws/encryption/tests/postgres.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformations/aws/encryption/tests/postgres.yml b/transformations/aws/encryption/tests/postgres.yml index 053925358..0a1c3080a 100644 --- a/transformations/aws/encryption/tests/postgres.yml +++ b/transformations/aws/encryption/tests/postgres.yml @@ -2,7 +2,8 @@ kind: source spec: name: aws path: cloudquery/aws - version: "v22.19.2" # latest version of source aws plugin + registry: cloudquery + version: "v23.2.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- @@ -14,4 +15,4 @@ spec: version: "v7.1.2" # latest version of postgresql plugin spec: batch_size: 10000 - connection_string: ${CQ_DSN} \ No newline at end of file + connection_string: ${CQ_DSN} From d5bfa300b8ec16487236a27509e1e90258813dc6 Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 10:12:19 +0200 Subject: [PATCH 12/49] Update bucket_access_logging.sql --- .../macros/cloudtrail/bucket_access_logging.sql | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/transformations/aws/macros/cloudtrail/bucket_access_logging.sql b/transformations/aws/macros/cloudtrail/bucket_access_logging.sql index 75ca04ddb..ecfb1fb12 100644 --- a/transformations/aws/macros/cloudtrail/bucket_access_logging.sql +++ b/transformations/aws/macros/cloudtrail/bucket_access_logging.sql @@ -12,11 +12,14 @@ select t.account_id, t.arn as resource_id, case - when b.logging_target_bucket is null or b.logging_target_prefix is null then 'fail' + when l.logging_enabled is null then 'fail' + when l.logging_enabled -> 'TargetBucket' is null then 'fail' + when l.logging_enabled -> 'TargetPrefix' is null then 'fail' else 'pass' end as status from aws_cloudtrail_trails t inner join aws_s3_buckets b on t.s3_bucket_name = b.name +inner join aws_s3_bucket_loggings l on b.arn = l.bucket_arn {% endmacro %} {% macro bigquery__bucket_access_logging(framework, check_id) %} @@ -27,11 +30,14 @@ select t.account_id, t.arn as resource_id, case - when b.logging_target_bucket is null or b.logging_target_prefix is null then 'fail' + when l.logging_enabled is null then 'fail' + when l.logging_enabled -> 'TargetBucket' is null then 'fail' + when l.logging_enabled -> 'TargetPrefix' is null then 'fail' else 'pass' end as status from {{ full_table_name("aws_cloudtrail_trails") }} t inner join {{ full_table_name("aws_s3_buckets") }} b on t.s3_bucket_name = b.name +inner join {{ full_table_name("aws_s3_bucket_loggings") }} l on b.arn = l.bucket_arn {% endmacro %} {% macro snowflake__bucket_access_logging(framework, check_id) %} @@ -42,9 +48,12 @@ select t.account_id, t.arn as resource_id, case - when b.logging_target_bucket is null or b.logging_target_prefix is null then 'fail' + when l.logging_enabled is null then 'fail' + when l.logging_enabled -> 'TargetBucket' is null then 'fail' + when l.logging_enabled -> 'TargetPrefix' is null then 'fail' else 'pass' end as status from aws_cloudtrail_trails t inner join aws_s3_buckets b on t.s3_bucket_name = b.name -{% endmacro %} \ No newline at end of file +inner join aws_s3_bucket_loggings l on b.arn = l.bucket_arn +{% endmacro %} From e6d3f6e05cb5c5f001cbe2270deb46846256309e Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 10:18:41 +0200 Subject: [PATCH 13/49] Update detector_enabled.sql --- transformations/aws/macros/guardduty/detector_enabled.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformations/aws/macros/guardduty/detector_enabled.sql b/transformations/aws/macros/guardduty/detector_enabled.sql index 2e0f0d121..964003baa 100644 --- a/transformations/aws/macros/guardduty/detector_enabled.sql +++ b/transformations/aws/macros/guardduty/detector_enabled.sql @@ -4,7 +4,7 @@ {% macro snowflake__detector_enabled(framework, check_id) %} with enabled_detector_regions as ( - select account_id, region + select request_account_id as account_id, request_region as region from aws_guardduty_detectors where status = 'ENABLED' ) @@ -19,7 +19,7 @@ select enabled = TRUE and e.region is null then 'fail' else 'pass' end AS status from aws_regions r -left join enabled_detector_regions e on e.region = r.region AND e.account_id = r.account_id +left join enabled_detector_regions e on e.region = r.region AND e.request_account_id = r.account_id union -- Add any detector that is enabled but all data sources are disabled select @@ -77,4 +77,4 @@ where {% endmacro %} {% macro default__detector_enabled(framework, check_id) %}{% endmacro %} - \ No newline at end of file + From 7e86bb971d03f7b2f8eb9291660fa1c62aa4b6a3 Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 10:19:06 +0200 Subject: [PATCH 14/49] Update detector_enabled.sql --- transformations/aws/macros/guardduty/detector_enabled.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformations/aws/macros/guardduty/detector_enabled.sql b/transformations/aws/macros/guardduty/detector_enabled.sql index 964003baa..5f948bad5 100644 --- a/transformations/aws/macros/guardduty/detector_enabled.sql +++ b/transformations/aws/macros/guardduty/detector_enabled.sql @@ -19,7 +19,7 @@ select enabled = TRUE and e.region is null then 'fail' else 'pass' end AS status from aws_regions r -left join enabled_detector_regions e on e.region = r.region AND e.request_account_id = r.account_id +left join enabled_detector_regions e on e.region = r.region AND e.account_id = r.account_id union -- Add any detector that is enabled but all data sources are disabled select From 521a791639e5dd4dd388cfd7393ed8a0d0d7426c Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 10:24:16 +0200 Subject: [PATCH 15/49] Update detector_enabled.sql --- transformations/aws/macros/guardduty/detector_enabled.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformations/aws/macros/guardduty/detector_enabled.sql b/transformations/aws/macros/guardduty/detector_enabled.sql index 5f948bad5..6fa6c80ef 100644 --- a/transformations/aws/macros/guardduty/detector_enabled.sql +++ b/transformations/aws/macros/guardduty/detector_enabled.sql @@ -41,7 +41,7 @@ where {% macro postgres__detector_enabled(framework, check_id) %} with enabled_detector_regions as ( - select account_id, region + select request_account_id as account_id, request_region as region from aws_guardduty_detectors where status = 'ENABLED' ) From 938b4277452af6f1bac44e043dcd886d9f5d89bb Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 12:01:44 +0200 Subject: [PATCH 16/49] Update unused_directconntect_connections.sql --- .../aws/macros/unused/unused_directconntect_connections.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformations/aws/macros/unused/unused_directconntect_connections.sql b/transformations/aws/macros/unused/unused_directconntect_connections.sql index 110c1b65a..8397f12a4 100644 --- a/transformations/aws/macros/unused/unused_directconntect_connections.sql +++ b/transformations/aws/macros/unused/unused_directconntect_connections.sql @@ -6,8 +6,8 @@ {% macro postgres__unused_directconntect_connections(framework, check_id) %} select - dc.account_id, - dc.arn as resource_id, + dc.request_account_id as account_id, + dc.arn as resource_id, rbc.cost from aws_directconnect_connections dc JOIN {{ ref('aws_cost__by_resources') }} rbc ON dc.arn = rbc.line_item_resource_id @@ -16,4 +16,4 @@ where dc.connection_state = 'down' {% macro snowflake__unused_directconntect_connections(framework, check_id) %} -{% endmacro %} \ No newline at end of file +{% endmacro %} From df1b25d1eeada8c9ac5d6c77f6ffc9caf92f7728 Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 13:28:40 +0200 Subject: [PATCH 17/49] Update snowflake.yml --- transformations/aws/compliance-free/tests/snowflake.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/transformations/aws/compliance-free/tests/snowflake.yml b/transformations/aws/compliance-free/tests/snowflake.yml index ad5782482..f2b8954c6 100644 --- a/transformations/aws/compliance-free/tests/snowflake.yml +++ b/transformations/aws/compliance-free/tests/snowflake.yml @@ -13,6 +13,7 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.4" # latest version of destination snowflake plugin + migrate_mode: forced # to be removed after tests pass spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 From 14a92a58a44588a83e087406d391439b8850f21a Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 13:28:54 +0200 Subject: [PATCH 18/49] Update snowflake.yml --- transformations/aws/compliance-premium/tests/snowflake.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index ad5782482..f2b8954c6 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -13,6 +13,7 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.4" # latest version of destination snowflake plugin + migrate_mode: forced # to be removed after tests pass spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 From a8b7dc57b0b207e00ff512de278253d20ae188eb Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 14:59:52 +0200 Subject: [PATCH 19/49] Update snowflake.yml --- transformations/aws/compliance-free/tests/snowflake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformations/aws/compliance-free/tests/snowflake.yml b/transformations/aws/compliance-free/tests/snowflake.yml index f2b8954c6..a3758e288 100644 --- a/transformations/aws/compliance-free/tests/snowflake.yml +++ b/transformations/aws/compliance-free/tests/snowflake.yml @@ -12,7 +12,7 @@ spec: name: snowflake path: cloudquery/snowflake registry: github - version: "v3.3.4" # latest version of destination snowflake plugin + version: "v3.3.5" # latest version of destination snowflake plugin migrate_mode: forced # to be removed after tests pass spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} From 441e01380f23da5078dc1e1cca485a6262d4d415 Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 21 Dec 2023 15:03:55 +0200 Subject: [PATCH 20/49] Update transformations/aws/compliance-free/tests/snowflake.yml --- transformations/aws/compliance-free/tests/snowflake.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/transformations/aws/compliance-free/tests/snowflake.yml b/transformations/aws/compliance-free/tests/snowflake.yml index a3758e288..dd08aa173 100644 --- a/transformations/aws/compliance-free/tests/snowflake.yml +++ b/transformations/aws/compliance-free/tests/snowflake.yml @@ -13,7 +13,6 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.5" # latest version of destination snowflake plugin - migrate_mode: forced # to be removed after tests pass spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 From add1149585b42f761c7d81d096360fc5a96484d4 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 21 Dec 2023 15:05:58 +0200 Subject: [PATCH 21/49] force migration inf compliance-free/bq --- transformations/aws/compliance-free/tests/bigquery.yml | 1 + transformations/aws/compliance-premium/tests/snowflake.yml | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/transformations/aws/compliance-free/tests/bigquery.yml b/transformations/aws/compliance-free/tests/bigquery.yml index 0960d000c..6f37903b2 100644 --- a/transformations/aws/compliance-free/tests/bigquery.yml +++ b/transformations/aws/compliance-free/tests/bigquery.yml @@ -14,6 +14,7 @@ spec: registry: cloudquery version: "v3.3.9" write_mode: "append" + migrate_mode: forced # to be removed after tests pass spec: project_id: cq-integration-tests dataset_id: policies_premium_ci_test diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index f2b8954c6..ad5782482 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -13,7 +13,6 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.4" # latest version of destination snowflake plugin - migrate_mode: forced # to be removed after tests pass spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 From c7cc14ba1652cf2461295b5a04c19abc22468310 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 21 Dec 2023 15:12:42 +0200 Subject: [PATCH 22/49] upd sf for compliance-premium --- transformations/aws/compliance-free/tests/bigquery.yml | 1 - transformations/aws/compliance-premium/tests/snowflake.yml | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transformations/aws/compliance-free/tests/bigquery.yml b/transformations/aws/compliance-free/tests/bigquery.yml index 6f37903b2..0960d000c 100644 --- a/transformations/aws/compliance-free/tests/bigquery.yml +++ b/transformations/aws/compliance-free/tests/bigquery.yml @@ -14,7 +14,6 @@ spec: registry: cloudquery version: "v3.3.9" write_mode: "append" - migrate_mode: forced # to be removed after tests pass spec: project_id: cq-integration-tests dataset_id: policies_premium_ci_test diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index ad5782482..a3758e288 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -12,7 +12,8 @@ spec: name: snowflake path: cloudquery/snowflake registry: github - version: "v3.3.4" # latest version of destination snowflake plugin + version: "v3.3.5" # latest version of destination snowflake plugin + migrate_mode: forced # to be removed after tests pass spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 From a9df666798d097df2981e5a9173cd5080214f551 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 21 Dec 2023 15:59:42 +0200 Subject: [PATCH 23/49] use request_ cols --- transformations/aws/macros/guardduty/detector_enabled.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/transformations/aws/macros/guardduty/detector_enabled.sql b/transformations/aws/macros/guardduty/detector_enabled.sql index 6fa6c80ef..c528dbb7a 100644 --- a/transformations/aws/macros/guardduty/detector_enabled.sql +++ b/transformations/aws/macros/guardduty/detector_enabled.sql @@ -26,8 +26,8 @@ select '{{framework}}' As framework, '{{check_id}}' As check_id, 'GuardDuty should be enabled (detectors)' AS title, - account_id, - region AS resource_id, + request_account_id as account_id, + request_region AS resource_id, case when data_sources:S3Logs:Status != 'ENABLED' AND data_sources:DNSLogs:Status != 'ENABLED' AND @@ -63,8 +63,8 @@ select '{{framework}}' as framework, '{{check_id}}' as check_id, 'GuardDuty should be enabled (detectors)' AS title, - account_id, - region AS resource_id, + request_account_id as account_id, + request_region AS resource_id, case when data_sources->'S3Logs'->>'Status' != 'ENABLED' AND data_sources->'DNSLogs'->>'Status' != 'ENABLED' AND From 6cd4b8188c376fd353b3d434431dda5ebb793c38 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 21 Dec 2023 17:11:38 +0200 Subject: [PATCH 24/49] use aws_iam_policy_versions --- transformations/aws/macros/iam/no_star.sql | 6 ++++- .../iam/policies_have_wildcard_actions.sql | 9 +++---- .../macros/iam/policies_with_admin_rights.sql | 27 +++++++++---------- .../macros/iam/wildcard_access_policies.sql | 15 +++++------ .../customer_policy_blocked_kms_actions.sql | 11 ++++---- .../iam_customer_policy_no_kms_decrypt.sql | 22 +++++++-------- 6 files changed, 46 insertions(+), 44 deletions(-) diff --git a/transformations/aws/macros/iam/no_star.sql b/transformations/aws/macros/iam/no_star.sql index 246dac2fd..61b2b1d74 100644 --- a/transformations/aws/macros/iam/no_star.sql +++ b/transformations/aws/macros/iam/no_star.sql @@ -7,7 +7,11 @@ {% macro postgres__no_star(framework, check_id) %} with pvs as ( - select id, (v->>'Document')::jsonb as document from aws_iam_policies, jsonb_array_elements(aws_iam_policies.policy_version_list) AS v + select + p.id, + pv.document_json as document + from aws_iam_policies p + inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn ), violations as ( select id, diff --git a/transformations/aws/macros/iam/policies_have_wildcard_actions.sql b/transformations/aws/macros/iam/policies_have_wildcard_actions.sql index 3d86c1668..00309fc47 100644 --- a/transformations/aws/macros/iam/policies_have_wildcard_actions.sql +++ b/transformations/aws/macros/iam/policies_have_wildcard_actions.sql @@ -11,11 +11,10 @@ SELECT FROM aws_iam_policies p - , lateral flatten(input => p.POLICY_VERSION_LIST) as f - , lateral flatten(input => parse_json(f.value:Document):Statement) as s -where f.value:IsDefaultVersion = 'true' AND s.value:Effect = 'Allow' - - ) + INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn + , lateral flatten(input => pv.document_json -> 'Statement') as s +where pv.is_default_version = true AND s ->> 'Effect' = 'Allow' +) select DISTINCT '{{framework}}' As framework, '{{check_id}}' As check_id, diff --git a/transformations/aws/macros/iam/policies_with_admin_rights.sql b/transformations/aws/macros/iam/policies_with_admin_rights.sql index 0a8d920b3..93eeb430b 100644 --- a/transformations/aws/macros/iam/policies_with_admin_rights.sql +++ b/transformations/aws/macros/iam/policies_with_admin_rights.sql @@ -8,12 +8,13 @@ SELECT p.id FROM aws_iam_policies p - , lateral flatten(input => p.POLICY_VERSION_LIST) as f - , lateral flatten(input => parse_json(f.value:Document):Statement) as s -where f.value:IsDefaultVersion = 'true' - and s.value:Effect = 'Allow' - and (s.value:Action = '*' or s.value:Action = '*:*') - and s.value:Resource = '*' + INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn + , lateral flatten(input => pv.document_json -> 'Statement') as s +where pv.is_default_version = 'true' AND s ->> 'Effect' = 'Allow' +where pv.is_default_version = true + and s ->> 'Effect' = 'Allow' + and (s ->> 'Action' = '*' or s ->> 'Action' = '*:*') + and s ->> 'Resource' = '*' ) select '{{framework}}' As framework, @@ -36,20 +37,18 @@ WHERE p.arn REGEXP '.*\d{12}.*' with iam_policies as ( select - id, - (v->>'Document')::jsonb AS document - from aws_iam_policies, jsonb_array_elements(aws_iam_policies.policy_version_list) AS v - where aws_iam_policies.default_version_id = v->>'VersionId' and arn not like 'arn:aws:iam::aws:policy%' + p.id, + v.document + from aws_iam_policies p + inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn + where pv.is_default_version = true and p.arn not like 'arn:aws:iam::aws:policy%' ), policy_statements as ( select id, JSONB_ARRAY_ELEMENTS( case JSONB_TYPEOF(document -> 'Statement') - when - 'string' then JSONB_BUILD_ARRAY( - document ->> 'Statement' - ) + when 'string' then JSONB_BUILD_ARRAY(document ->> 'Statement') when 'array' then document -> 'Statement' end ) as statement from diff --git a/transformations/aws/macros/iam/wildcard_access_policies.sql b/transformations/aws/macros/iam/wildcard_access_policies.sql index bbf861a56..e1ac468f7 100644 --- a/transformations/aws/macros/iam/wildcard_access_policies.sql +++ b/transformations/aws/macros/iam/wildcard_access_policies.sql @@ -8,20 +8,19 @@ with policy_statements as ( select - aws_iam_policies.id, + p.id, JSONB_ARRAY_ELEMENTS( - case JSONB_TYPEOF(((v->>'Document')::jsonb) -> 'Statement') + case JSONB_TYPEOF(v.document_json -> 'Statement') when - 'string' then JSONB_BUILD_ARRAY( - ((v->>'Document')::jsonb) ->> 'Statement' - ) + 'string' then JSONB_BUILD_ARRAY(v.document_json ->> 'Statement') when - 'array' then ((v->>'Document')::jsonb) -> 'Statement' + 'array' then v.document_json -> 'Statement' end ) as statement from - aws_iam_policies, jsonb_array_elements(aws_iam_policies.policy_version_list) AS v - where aws_iam_policies.arn not like 'arn:aws:iam::aws:policy%' + aws_iam_policies p + inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn + where p.arn not like 'arn:aws:iam::aws:policy%' ), allow_all_statements as ( diff --git a/transformations/aws/macros/kms/customer_policy_blocked_kms_actions.sql b/transformations/aws/macros/kms/customer_policy_blocked_kms_actions.sql index c5ec5520f..10f5c778b 100644 --- a/transformations/aws/macros/kms/customer_policy_blocked_kms_actions.sql +++ b/transformations/aws/macros/kms/customer_policy_blocked_kms_actions.sql @@ -8,11 +8,12 @@ with iam_policies as ( select - (v->>'Document')::jsonb AS document, - account_id, - arn, - id - from aws_iam_policies, jsonb_array_elements(aws_iam_policies.policy_version_list) AS v + pv.document_json as document, + p.account_id, + p.arn, + p.id + from aws_iam_policies p + inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn ), violations as ( diff --git a/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql b/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql index 4b0018e01..ab45fc8e2 100644 --- a/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql +++ b/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql @@ -2,19 +2,19 @@ wITH policy_with_decrypt AS ( SELECT DISTINCT arn FROM aws_iam_policies p - ,lateral flatten(input => p.POLICY_VERSION_LIST) as f - ,lateral flatten(input => parse_json(f.value:Document):Statement) as s - WHERE - s.value:Effect = 'Allow' + INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn + , lateral flatten(input => pv.document_json -> 'Statement') as s + WHERE + s ->> 'Effect' = 'Allow' AND - (s.value:Resource = '*' OR - s.value:Resource LIKE '%kms%') + (s ->> 'Resource' = '*' OR + s ->> 'Resource' LIKE '%kms%') AND - (s.value:Action = '*' - OR s.value:Action ILIKE '%kms:*%' - OR s.value:Action ILIKE '%kms:decrypt%' - OR s.value:Action ILIKE '%kms:reencryptfrom%' - OR s.value:Action ILIKE '%kms:reencrypt*%') + (s ->> 'Action' = '*' + OR s ->> 'Action' ILIKE '%kms:*%' + OR s ->> 'Action' ILIKE '%kms:decrypt%' + OR s ->> 'Action' ILIKE '%kms:reencryptfrom%' + OR s ->> 'Action' ILIKE '%kms:reencrypt*%') ) SELECT '{{framework}}' As framework, From 75a11a7ad1db98481e21a349032a565cae8b699a Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 21 Dec 2023 18:40:37 +0200 Subject: [PATCH 25/49] use aws_s3_bucket_policies --- .../macros/s3/publicly_readable_buckets.sql | 18 +++++------------- .../macros/s3/publicly_writable_buckets.sql | 18 +++++------------- 2 files changed, 10 insertions(+), 26 deletions(-) diff --git a/transformations/aws/macros/s3/publicly_readable_buckets.sql b/transformations/aws/macros/s3/publicly_readable_buckets.sql index d8b591b2e..cc11e1065 100644 --- a/transformations/aws/macros/s3/publicly_readable_buckets.sql +++ b/transformations/aws/macros/s3/publicly_readable_buckets.sql @@ -69,21 +69,13 @@ with policy_allow_public as ( from ( select - aws_s3_buckets.arn, - statements -> 'Principal' as principals + b.arn, + bp.policy_json -> 'Statement' -> 'Principal' as principals from - aws_s3_buckets, - jsonb_array_elements( - case jsonb_typeof(policy::jsonb -> 'Statement') - when - 'string' then jsonb_build_array( - policy::jsonb ->> 'Statement' - ) - when 'array' then policy::jsonb -> 'Statement' - end - ) as statements + aws_s3_buckets b + inner join aws_s3_bucket_policies bp on b.arn = bp.bucket_arn where - statements -> 'Effect' = '"Allow"' + bp.policy_json -> 'Statement' -> 'Effect' = '"Allow"' ) as foo where principals = '"*"' diff --git a/transformations/aws/macros/s3/publicly_writable_buckets.sql b/transformations/aws/macros/s3/publicly_writable_buckets.sql index ebc525cd0..ddaee49ed 100644 --- a/transformations/aws/macros/s3/publicly_writable_buckets.sql +++ b/transformations/aws/macros/s3/publicly_writable_buckets.sql @@ -69,21 +69,13 @@ with policy_allow_public as ( from ( select - aws_s3_buckets.arn, - statements -> 'Principal' as principals + b.arn, + bp.policy_json -> 'Statement' -> 'Principal' as principals from - aws_s3_buckets, - jsonb_array_elements( - case jsonb_typeof(policy::jsonb -> 'Statement') - when - 'string' then jsonb_build_array( - policy::jsonb ->> 'Statement' - ) - when 'array' then policy::jsonb -> 'Statement' - end - ) as statements + aws_s3_buckets b + inner join aws_s3_bucket_policies bp on b.arn = bp.bucket_arn where - statements -> 'Effect' = '"Allow"' + bp.policy_json -> 'Statement' -> 'Effect' = '"Allow"' ) as foo where principals = '"*"' From b7afc2312efe3ec7799897c608f5610ef79bb06e Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 22 Dec 2023 14:47:01 +0200 Subject: [PATCH 26/49] aws_s3_buckets relations --- .../aws/macros/s3/publicly_readable_buckets.sql | 12 ++++++++---- .../aws/macros/s3/publicly_writable_buckets.sql | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/transformations/aws/macros/s3/publicly_readable_buckets.sql b/transformations/aws/macros/s3/publicly_readable_buckets.sql index cc11e1065..72b7bdaac 100644 --- a/transformations/aws/macros/s3/publicly_readable_buckets.sql +++ b/transformations/aws/macros/s3/publicly_readable_buckets.sql @@ -47,16 +47,18 @@ LEFT JOIN aws_s3_buckets.arn = aws_s3_bucket_grants.bucket_arn LEFT JOIN policy_allow_public ON aws_s3_buckets.arn = policy_allow_public.arn +LEFT JOIN aws_s3_bucket_public_access_blocks ON + aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn WHERE ( - aws_s3_buckets.block_public_acls != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicAcls')::boolean != TRUE AND ( aws_s3_bucket_grants.grantee:URI::STRING = 'http://acs.amazonaws.com/groups/global/AllUsers' AND aws_s3_bucket_grants.permission IN ('READ_ACP', 'FULL_CONTROL') ) ) OR ( - aws_s3_buckets.block_public_policy != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicPolicy')::boolean != TRUE AND policy_allow_public.statement_count > 0 ) {% endmacro %} @@ -110,16 +112,18 @@ left join -- Principal = "*" left join policy_allow_public on aws_s3_buckets.arn = policy_allow_public.arn +left join aws_s3_bucket_public_access_blocks on + aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn where ( - aws_s3_buckets.block_public_acls != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicAcls')::boolean != TRUE and ( grantee->>'URI' = 'http://acs.amazonaws.com/groups/global/AllUsers' and permission in ('READ_ACP', 'FULL_CONTROL') ) ) or ( - aws_s3_buckets.block_public_policy != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicPolicy')::boolean != TRUE and policy_allow_public.statement_count > 0 ) {% endmacro %} diff --git a/transformations/aws/macros/s3/publicly_writable_buckets.sql b/transformations/aws/macros/s3/publicly_writable_buckets.sql index ddaee49ed..dbb24bd16 100644 --- a/transformations/aws/macros/s3/publicly_writable_buckets.sql +++ b/transformations/aws/macros/s3/publicly_writable_buckets.sql @@ -47,16 +47,18 @@ LEFT JOIN aws_s3_buckets.arn = aws_s3_bucket_grants.bucket_arn LEFT JOIN policy_allow_public ON aws_s3_buckets.arn = policy_allow_public.arn +LEFT JOIN aws_s3_bucket_public_access_blocks ON + aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn WHERE ( - aws_s3_buckets.block_public_acls != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicAcls')::boolean != TRUE AND ( aws_s3_bucket_grants.grantee:URI::STRING = 'http://acs.amazonaws.com/groups/global/AllUsers' AND aws_s3_bucket_grants.permission IN ('WRITE_ACP', 'FULL_CONTROL') ) ) OR ( - aws_s3_buckets.block_public_policy != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicPolicy')::boolean != TRUE AND policy_allow_public.statement_count > 0 ) {% endmacro %} @@ -110,16 +112,18 @@ left join -- Principal = "*" left join policy_allow_public on aws_s3_buckets.arn = policy_allow_public.arn +left join aws_s3_bucket_public_access_blocks on + aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn where ( - aws_s3_buckets.block_public_acls != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicAcls')::boolean != TRUE and ( grantee->>'URI' = 'http://acs.amazonaws.com/groups/global/AllUsers' and permission in ('WRITE_ACP', 'FULL_CONTROL') ) ) or ( - aws_s3_buckets.block_public_policy != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicPolicy')::boolean != TRUE and policy_allow_public.statement_count > 0 ) {% endmacro %} From ab8b274c840c87131bb583520ecabd27dac27727 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 22 Dec 2023 15:04:01 +0200 Subject: [PATCH 27/49] aws_s3_bucket_replications --- .../aws/macros/s3/s3_cross_region_replication.sql | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/transformations/aws/macros/s3/s3_cross_region_replication.sql b/transformations/aws/macros/s3/s3_cross_region_replication.sql index 9759a784a..7d4a50458 100644 --- a/transformations/aws/macros/s3/s3_cross_region_replication.sql +++ b/transformations/aws/macros/s3/s3_cross_region_replication.sql @@ -12,13 +12,10 @@ select aws_s3_buckets.account_id, aws_s3_buckets.arn as resource_id, case when - r->>'Status' is distinct from 'Enabled' + aws_s3_bucket_replications.replication_configuration -> 'Rule' ->>'Status' is distinct from 'Enabled' then 'fail' else 'pass' end as status from - aws_s3_buckets, JSONB_ARRAY_ELEMENTS( - case jsonb_typeof(replication_rules) - when 'array' then replication_rules - else '[]' end - ) as r + aws_s3_buckets + inner join aws_s3_bucket_replications on aws_s3_buckets.arn = aws_s3_bucket_replications.bucket_arn -- Note: This query doesn't validate that the destination bucket is actually in a different region {% endmacro %} From 935a03218ae9110bcd34bc6bf2164cae0f3d49ce Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 22 Dec 2023 15:12:27 +0200 Subject: [PATCH 28/49] aws_s3_bucket_policies --- transformations/aws/macros/s3/deny_http_requests.sql | 9 +++++---- .../aws/macros/s3/restrict_cross_account_actions.sql | 7 ++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/transformations/aws/macros/s3/deny_http_requests.sql b/transformations/aws/macros/s3/deny_http_requests.sql index 171010fa0..638430a1b 100644 --- a/transformations/aws/macros/s3/deny_http_requests.sql +++ b/transformations/aws/macros/s3/deny_http_requests.sql @@ -49,14 +49,15 @@ where from (select aws_s3_buckets.arn, statements, statements -> 'Principal' as principals - from aws_s3_buckets, + from aws_s3_buckets + inner join aws_s3_bucket_policies on aws_s3_buckets.arn = aws_s3_bucket_policies.bucket_arn, jsonb_array_elements( - case jsonb_typeof(policy -> 'Statement') + case jsonb_typeof(aws_s3_bucket_policies.policy_json -> 'Statement') when 'string' then jsonb_build_array( - policy ->> 'Statement' + aws_s3_bucket_policies.policy_json ->> 'Statement' ) - when 'array' then policy -> 'Statement' + when 'array' then aws_s3_bucket_policies.policy_json -> 'Statement' end ) as statements where statements -> 'Effect' = '"Deny"') as foo, diff --git a/transformations/aws/macros/s3/restrict_cross_account_actions.sql b/transformations/aws/macros/s3/restrict_cross_account_actions.sql index f9db971a1..0f4c1cb8f 100644 --- a/transformations/aws/macros/s3/restrict_cross_account_actions.sql +++ b/transformations/aws/macros/s3/restrict_cross_account_actions.sql @@ -97,10 +97,11 @@ FROM ( WHEN JSONB_TYPEOF(statements -> 'Action') = 'array' THEN statements -> 'Action' END AS actions - FROM aws_s3_buckets, + FROM aws_s3_buckets + INNER JOIN aws_s3_bucket_policies ON aws_s3_buckets.arn = aws_s3_bucket_policies.bucket_arn, jsonb_array_elements( - CASE JSONB_TYPEOF(policy -> 'Statement') - WHEN 'string' THEN JSONB_BUILD_ARRAY(policy ->> 'Statement') + CASE JSONB_TYPEOF(aws_s3_bucket_policies.policy_json -> 'Statement') + WHEN 'string' THEN JSONB_BUILD_ARRAY(aws_s3_bucket_policies.policy_json ->> 'Statement') WHEN 'array' THEN policy -> 'Statement' END ) AS statements From b705ea1f5d53d87aab727d39ccc63488e148ee99 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Fri, 22 Dec 2023 16:39:31 +0200 Subject: [PATCH 29/49] rm extra forced mode --- transformations/aws/compliance-premium/tests/snowflake.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index a3758e288..dd08aa173 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -13,7 +13,6 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.5" # latest version of destination snowflake plugin - migrate_mode: forced # to be removed after tests pass spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 From 80855d1167db6440599c37ca15cf0d27132b5645 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Sat, 23 Dec 2023 11:18:26 +0200 Subject: [PATCH 30/49] cloudtrail/bucket_access_logging:snowflake --- .../aws/macros/cloudtrail/bucket_access_logging.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformations/aws/macros/cloudtrail/bucket_access_logging.sql b/transformations/aws/macros/cloudtrail/bucket_access_logging.sql index ecfb1fb12..275d4144e 100644 --- a/transformations/aws/macros/cloudtrail/bucket_access_logging.sql +++ b/transformations/aws/macros/cloudtrail/bucket_access_logging.sql @@ -49,8 +49,8 @@ select t.arn as resource_id, case when l.logging_enabled is null then 'fail' - when l.logging_enabled -> 'TargetBucket' is null then 'fail' - when l.logging_enabled -> 'TargetPrefix' is null then 'fail' + when l.logging_enabled:TargetBucket is null then 'fail' + when l.logging_enabled:TargetPrefix is null then 'fail' else 'pass' end as status from aws_cloudtrail_trails t From b387f58e73ad3d531068dcb66513b5116ca48f7a Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Sat, 23 Dec 2023 11:24:44 +0200 Subject: [PATCH 31/49] cloudtrail/enabled_in_all_regions:snowflake --- .../cloudtrail/enabled_in_all_regions.sql | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql b/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql index 8896ad68e..5e3725ac2 100644 --- a/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql +++ b/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql @@ -39,10 +39,17 @@ select aws_cloudtrail_trails.account_id, arn as resource_id, case - when is_multi_region_trail = FALSE or ( - is_multi_region_trail = TRUE and ( - read_write_type != 'All' or include_management_events = FALSE - )) then 'fail' + when aws_cloudtrail_trails.is_multi_region_trail = FALSE then 'fail' + when exists(select * + from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.event_selectors) as es + where es ->>'ReadWriteType' != 'All' or (es->>'IncludeManagementEvents')::boolean = FALSE) + then 'fail' + when exists(select * + from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.advanced_event_selectors) as aes + where exists(select * + from jsonb_array_elements(aes ->'FieldSelectors') as aes_fs + where aes_fs ->>'Field' = 'readOnly')) + then 'fail' else 'pass' end as status from {{ full_table_name("aws_cloudtrail_trails") }} @@ -61,10 +68,17 @@ select aws_cloudtrail_trails.account_id, arn as resource_id, case - when is_multi_region_trail = FALSE or ( - is_multi_region_trail = TRUE and ( - read_write_type != 'All' or include_management_events = FALSE - )) then 'fail' + when aws_cloudtrail_trails.is_multi_region_trail = FALSE then 'fail' + when exists(select * + from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.event_selectors) as es + where es:ReadWriteType != 'All' or (es:IncludeManagementEvents)::boolean = FALSE) + then 'fail' + when exists(select * + from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.advanced_event_selectors) as aes + where exists(select * + from jsonb_array_elements(aes:FieldSelectors) as aes_fs + where aes_fs:Field = 'readOnly')) + then 'fail' else 'pass' end as status from aws_cloudtrail_trails From cce6d8bd37c771270700ad96d5ad7279558b4afa Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Tue, 26 Dec 2023 14:23:35 +0200 Subject: [PATCH 32/49] Updated log_metric and enabled_in_all_regions --- .../cloudtrail/enabled_in_all_regions.sql | 23 ++++-- .../macros/log_metric_filter_and_alarm.sql | 71 ++++++++++--------- 2 files changed, 53 insertions(+), 41 deletions(-) diff --git a/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql b/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql index 5e3725ac2..6e536ac19 100644 --- a/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql +++ b/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql @@ -61,6 +61,12 @@ inner join {% endmacro %} {% macro snowflake__cloudtrail_enabled_all_regions(framework, check_id) %} +with aes as +( + select * + from aws_cloudtrail_trail_event_selectors, + LATERAL FLATTEN (advanced_event_selectors) as aes +) select '{{framework}}' as framework, '{{check_id}}' as check_id, @@ -70,14 +76,17 @@ select case when aws_cloudtrail_trails.is_multi_region_trail = FALSE then 'fail' when exists(select * - from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.event_selectors) as es - where es:ReadWriteType != 'All' or (es:IncludeManagementEvents)::boolean = FALSE) + from aws_cloudtrail_trail_event_selectors, + LATERAL FLATTEN(event_selectors) as es + where es.value:ReadWriteType != 'All' or (es.value:IncludeManagementEvents)::boolean = FALSE + ) then 'fail' - when exists(select * - from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.advanced_event_selectors) as aes - where exists(select * - from jsonb_array_elements(aes:FieldSelectors) as aes_fs - where aes_fs:Field = 'readOnly')) + when exists( + select * + from aes, + LATERAL FLATTEN (value:FieldSelectors) as aes_fs + where aes_fs.value:Field = 'readOnly' + ) then 'fail' else 'pass' end as status diff --git a/transformations/aws/macros/log_metric_filter_and_alarm.sql b/transformations/aws/macros/log_metric_filter_and_alarm.sql index 90a1ed8f0..6e5ddac75 100644 --- a/transformations/aws/macros/log_metric_filter_and_alarm.sql +++ b/transformations/aws/macros/log_metric_filter_and_alarm.sql @@ -57,39 +57,42 @@ where t.is_multi_region_trail = TRUE {% endmacro %} {% macro snowflake__log_metric_filter_and_alarm() %} -WITH af AS ( - SELECT DISTINCT - a.arn, - a.actions_enabled, - a.alarm_actions, - m.value:MetricStat:Metric:MetricName AS metric_name - FROM aws_cloudwatch_alarms a, - LATERAL FLATTEN(input => a.metrics) AS m +with af as ( + select distinct a.arn, a.actions_enabled, a.alarm_actions, m.value:MetricStat:Metric:MetricName as metric_name -- TODO check + from aws_cloudwatch_alarms a, + LATERAL FLATTEN (metrics) as m +), +aes as ( +select * from aws_cloudtrail_trail_event_selectors, + LATERAL FLATTEN (advanced_event_selectors) as aes +), +tes as ( + select trail_arn from aws_cloudtrail_trail_event_selectors + where exists( + select * from + aws_cloudtrail_trail_event_selectors, + LATERAL FLATTEN(event_selectors) as es + where es.value:ReadWriteType = 'All' and (es.value:IncludeManagementEvents)::boolean = TRUE + ) + or exists( + select * from aes + where not exists ( + select * from aes, LATERAL FLATTEN(value:FieldSelectors) as aes_fs + where aes_fs.value:Field = 'readOnly' + ) + ) ) - -SELECT - t.account_id, - t.region, - t.cloud_watch_logs_log_group_arn, - mf.filter_pattern AS pattern -FROM - aws_cloudtrail_trails t -INNER JOIN - aws_cloudtrail_trail_event_selectors tes ON t.arn = tes.trail_arn -INNER JOIN - aws_cloudwatchlogs_metric_filters mf ON mf.log_group_name = t.cloudwatch_logs_log_group_name -INNER JOIN - af ON mf.filter_name = af.metric_name -INNER JOIN LATERAL ( - SELECT arn, topic_arn - FROM aws_sns_subscriptions ss - WHERE ARRAY_CONTAINS(ss.topic_arn::variant, af.alarm_actions) - LIMIT 1 -) ss ON TRUE -WHERE - t.is_multi_region_trail = TRUE - AND (t.status:IsLogging)::BOOLEAN = TRUE - AND tes.include_management_events = TRUE - AND tes.read_write_type = 'All' - AND ss.arn LIKE 'aws:arn:%' +select + t.account_id, + t.region, + t.cloud_watch_logs_log_group_arn, + mf.filter_pattern as pattern +from aws_cloudtrail_trails t +inner join tes on t.arn = tes.trail_arn +inner join aws_cloudwatchlogs_metric_filters mf on mf.log_group_name = t.cloudwatch_logs_log_group_name +inner join af on mf.filter_name = af.metric_name +inner join aws_sns_subscriptions ss on ARRAY_CONTAINS((ss.topic_arn)::variant, af.alarm_actions) +where t.is_multi_region_trail = TRUE + and (t.status:IsLogging)::boolean = TRUE + and ss.arn like 'aws:arn:%' {% endmacro %} \ No newline at end of file From b6fb42150b13a9f7717ea8cbc7735285d71a9e0f Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Wed, 27 Dec 2023 12:15:16 +0200 Subject: [PATCH 33/49] change ref to aws@v23.2.0 --- transformations/aws/asset-inventory-free/manifest.json | 2 +- transformations/aws/data-resilience/manifest.json | 2 +- transformations/aws/encryption/manifest.json | 2 +- visualizations/aws/asset_inventory/README.md | 2 +- visualizations/aws/asset_inventory/manifest.json | 2 +- visualizations/aws/compliance/README.md | 2 +- visualizations/aws/compliance/manifest.json | 2 +- visualizations/aws/data_resilience/README.md | 2 +- visualizations/aws/data_resilience/manifest.json | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/transformations/aws/asset-inventory-free/manifest.json b/transformations/aws/asset-inventory-free/manifest.json index 9fb912ffd..0de3b0eee 100644 --- a/transformations/aws/asset-inventory-free/manifest.json +++ b/transformations/aws/asset-inventory-free/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_asset_inventory_free.zip", - "plugin_deps": ["cloudquery/source/aws@v22.19.0"], + "plugin_deps": ["cloudquery/source/aws@v23.2.0"], "addon_deps": [] } diff --git a/transformations/aws/data-resilience/manifest.json b/transformations/aws/data-resilience/manifest.json index 29872780d..0fcaab7da 100644 --- a/transformations/aws/data-resilience/manifest.json +++ b/transformations/aws/data-resilience/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_data_resilience.zip", - "plugin_deps": ["cloudquery/source/aws@v22.19.0"], + "plugin_deps": ["cloudquery/source/aws@v23.2.0"], "addon_deps": [] } diff --git a/transformations/aws/encryption/manifest.json b/transformations/aws/encryption/manifest.json index 06172bb30..6147885fb 100644 --- a/transformations/aws/encryption/manifest.json +++ b/transformations/aws/encryption/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_encryption.zip", - "plugin_deps": ["cloudquery/source/aws@v22.19.0"], + "plugin_deps": ["cloudquery/source/aws@v23.2.0"], "addon_deps": [] } diff --git a/visualizations/aws/asset_inventory/README.md b/visualizations/aws/asset_inventory/README.md index d95eaf814..5f7703e08 100644 --- a/visualizations/aws/asset_inventory/README.md +++ b/visualizations/aws/asset_inventory/README.md @@ -2,7 +2,7 @@ ## Overview -This contains an AWS Asset Inventory Dashboard for Grafana on top of CloudQuery [AWS plugin](https://hub.cloudquery.io/plugins/source/cloudquery/aws/v22.19.0/docs) and [AWS Asset Inventory pack](https://hub.cloudquery.io/addons/transformation/cloudquery/aws-asset-inventory/). +This contains an AWS Asset Inventory Dashboard for Grafana on top of CloudQuery [AWS plugin](https://hub.cloudquery.io/plugins/source/cloudquery/aws/v23.2.0/docs) and [AWS Asset Inventory pack](https://hub.cloudquery.io/addons/transformation/cloudquery/aws-asset-inventory/). ## Requirements diff --git a/visualizations/aws/asset_inventory/manifest.json b/visualizations/aws/asset_inventory/manifest.json index ce723b967..0f99bde66 100644 --- a/visualizations/aws/asset_inventory/manifest.json +++ b/visualizations/aws/asset_inventory/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_asset_inventory.zip", - "plugin_deps": ["cloudquery/source/aws@v22.19.0"], + "plugin_deps": ["cloudquery/source/aws@v23.2.0"], "addon_deps": ["cloudquery/transformation/aws-asset-inventory@v1.0.0"] } \ No newline at end of file diff --git a/visualizations/aws/compliance/README.md b/visualizations/aws/compliance/README.md index bdb0f7997..57deffd6f 100644 --- a/visualizations/aws/compliance/README.md +++ b/visualizations/aws/compliance/README.md @@ -2,7 +2,7 @@ ## Overview -This contains an AWS Compliance Dashboard for Grafana on top of CloudQuery [AWS plugin](https://hub.cloudquery.io/plugins/source/cloudquery/aws/v22.19.0/docs) and [AWS Compliance pack](https://hub.cloudquery.io/addons/transformation/cloudquery/aws-compliance-free/v0.0.1/docs). +This contains an AWS Compliance Dashboard for Grafana on top of CloudQuery [AWS plugin](https://hub.cloudquery.io/plugins/source/cloudquery/aws/v23.2.0/docs) and [AWS Compliance pack](https://hub.cloudquery.io/addons/transformation/cloudquery/aws-compliance-free/v0.0.1/docs). ## Requirements diff --git a/visualizations/aws/compliance/manifest.json b/visualizations/aws/compliance/manifest.json index 7d3b1e604..5daaa588c 100644 --- a/visualizations/aws/compliance/manifest.json +++ b/visualizations/aws/compliance/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_compliance.zip", - "plugin_deps": ["cloudquery/source/aws@v22.19.0"], + "plugin_deps": ["cloudquery/source/aws@v23.2.0"], "addon_deps": ["cloudquery/transformation/aws-compliance-free@v0.0.1"] } diff --git a/visualizations/aws/data_resilience/README.md b/visualizations/aws/data_resilience/README.md index 4e3b8ae56..3c85cf407 100644 --- a/visualizations/aws/data_resilience/README.md +++ b/visualizations/aws/data_resilience/README.md @@ -2,7 +2,7 @@ ## Overview -This contains an AWS Resilience and Backup Dashboard for Grafana on top of CloudQuery [AWS plugin](https://hub.cloudquery.io/plugins/source/cloudquery/aws/v22.19.0/docs) and [AWS Compliance pack](https://hub.cloudquery.io/addons/transformation/cloudquery/aws-compliance-free/v0.0.1/docs). +This contains an AWS Resilience and Backup Dashboard for Grafana on top of CloudQuery [AWS plugin](https://hub.cloudquery.io/plugins/source/cloudquery/aws/v23.2.0/docs) and [AWS Compliance pack](https://hub.cloudquery.io/addons/transformation/cloudquery/aws-compliance-free/v0.0.1/docs). ## Requirements diff --git a/visualizations/aws/data_resilience/manifest.json b/visualizations/aws/data_resilience/manifest.json index a4359c897..12532aad9 100644 --- a/visualizations/aws/data_resilience/manifest.json +++ b/visualizations/aws/data_resilience/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_data_resilience.zip", - "plugin_deps": ["cloudquery/source/aws@v22.19.0"], + "plugin_deps": ["cloudquery/source/aws@v23.2.0"], "addon_deps": ["cloudquery/transformation/aws-data-resilience@v1.0.0"] } From fb9320efb6ac1244ec290e7c1be60345dfeddce3 Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Wed, 27 Dec 2023 15:54:22 +0200 Subject: [PATCH 34/49] Updated queries - no_star, policies_have_wildcard_actions, policies_with_admin_rights, iam_customer_policy_no_kms_decrypt, deny_http_requests, publicly_readable_buckets, publicly_writable_buckets, restrict_cross_account_actions, s3_cross_region_replication --- transformations/aws/macros/iam/no_star.sql | 34 +++++++++++++ .../iam/policies_have_wildcard_actions.sql | 42 +++++++++++++++- .../macros/iam/policies_with_admin_rights.sql | 16 +++--- .../iam_customer_policy_no_kms_decrypt.sql | 50 +++++++++++++++++-- .../aws/macros/s3/deny_http_requests.sql | 5 +- .../macros/s3/publicly_readable_buckets.sql | 13 ++--- .../macros/s3/publicly_writable_buckets.sql | 13 ++--- .../s3/restrict_cross_account_actions.sql | 27 +++++----- .../macros/s3/s3_cross_region_replication.sql | 16 ++++++ 9 files changed, 173 insertions(+), 43 deletions(-) diff --git a/transformations/aws/macros/iam/no_star.sql b/transformations/aws/macros/iam/no_star.sql index 61b2b1d74..72a892cdc 100644 --- a/transformations/aws/macros/iam/no_star.sql +++ b/transformations/aws/macros/iam/no_star.sql @@ -50,3 +50,37 @@ select distinct from aws_iam_policies left join violations on violations.id = aws_iam_policies.id {% endmacro %} + +{% macro snowflake__no_star(framework, check_id) %} +with pvs as ( + select + p.id, + pv.document_json as document + from aws_iam_policies p + inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn +), violations as ( + select + id, + COUNT(*) as violations + from pvs, + LATERAL FLATTEN(document:Statement) as statement, + LATERAL FLATTEN(statement.value:Resource) as resource, + LATERAL FLATTEN(statement.value:Action) as action + where statement.value:Effect = 'Allow' + and resource.value = '*' + and ( action.value = '*' or action.value = '*:*' ) + group by id +) + +select distinct + '{{framework}}' as framework, + '{{check_id}}' as check_id, + 'IAM policies should not allow full ''*'' administrative privileges' as title, + account_id, + arn AS resource_id, + case when + violations.id is not null AND violations.violations > 0 + then 'fail' else 'pass' end as status +from aws_iam_policies +left join violations on violations.id = aws_iam_policies.id +{% endmacro %} \ No newline at end of file diff --git a/transformations/aws/macros/iam/policies_have_wildcard_actions.sql b/transformations/aws/macros/iam/policies_have_wildcard_actions.sql index 00309fc47..d488ea7b4 100644 --- a/transformations/aws/macros/iam/policies_have_wildcard_actions.sql +++ b/transformations/aws/macros/iam/policies_have_wildcard_actions.sql @@ -1,4 +1,42 @@ {% macro policies_have_wildcard_actions(framework, check_id) %} + {{ return(adapter.dispatch('policies_have_wildcard_actions')(framework, check_id)) }} +{% endmacro %} + +{% macro default__policies_have_wildcard_actions(framework, check_id) %}{% endmacro %} + +{% macro postgres__policies_have_wildcard_actions(framework, check_id) %} +with bad_statements as ( +SELECT + p.account_id, + p.arn as resource_id, + CASE + WHEN s ->> 'Action' ~ '^[a-zA-Z0-9]+:\*$' + OR s ->> 'Action' = '*:*' THEN 1 + ELSE 0 + END as status + +FROM + aws_iam_policies p +INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn + , JSONB_ARRAY_ELEMENTS(pv.document_json -> 'Statement') as s +where pv.is_default_version = true AND s ->> 'Effect' = 'Allow' + + ) +select DISTINCT + '{{framework}}' As framework, + '{{check_id}}' As check_id, + 'IAM customer managed policies that you create should not allow wildcard actions for services' AS title, + account_id, + resource_id, + CASE + WHEN max(status) over(partition by resource_id) = 1 THEN 'fail' + ELSE 'pass' + END as status +FROM + bad_statements +{% endmacro %} + +{% macro snowflake__policies_have_wildcard_actions(framework, check_id) %} with bad_statements as ( SELECT p.account_id, @@ -12,8 +50,8 @@ SELECT FROM aws_iam_policies p INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn - , lateral flatten(input => pv.document_json -> 'Statement') as s -where pv.is_default_version = true AND s ->> 'Effect' = 'Allow' + , lateral flatten(input => pv.document_json:Statement) as s +where pv.is_default_version = true AND s.value:Effect = 'Allow' ) select DISTINCT '{{framework}}' As framework, diff --git a/transformations/aws/macros/iam/policies_with_admin_rights.sql b/transformations/aws/macros/iam/policies_with_admin_rights.sql index 93eeb430b..fad0ee0fb 100644 --- a/transformations/aws/macros/iam/policies_with_admin_rights.sql +++ b/transformations/aws/macros/iam/policies_with_admin_rights.sql @@ -9,12 +9,11 @@ SELECT FROM aws_iam_policies p INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn - , lateral flatten(input => pv.document_json -> 'Statement') as s -where pv.is_default_version = 'true' AND s ->> 'Effect' = 'Allow' -where pv.is_default_version = true - and s ->> 'Effect' = 'Allow' - and (s ->> 'Action' = '*' or s ->> 'Action' = '*:*') - and s ->> 'Resource' = '*' + , lateral flatten(input => pv.document_json:Statement) as s +where pv.is_default_version = 'true' AND s.value:Effect = 'Allow' + and s.value:Effect = 'Allow' + and (s.value:Action = '*' or s.value:Action = '*:*') + and s.value:Resource = '*' ) select '{{framework}}' As framework, @@ -34,11 +33,10 @@ WHERE p.arn REGEXP '.*\d{12}.*' {% endmacro %} {% macro postgres__policies_with_admin_rights(framework, check_id) %} - with iam_policies as ( select - p.id, - v.document + p.id as id, + pv.document_json as document from aws_iam_policies p inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn where pv.is_default_version = true and p.arn not like 'arn:aws:iam::aws:policy%' diff --git a/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql b/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql index ab45fc8e2..f4ec0c67a 100644 --- a/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql +++ b/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql @@ -1,9 +1,16 @@ {% macro iam_customer_policy_no_kms_decrypt(framework, check_id) %} wITH policy_with_decrypt AS ( + {{ return(adapter.dispatch('iam_customer_policy_no_kms_decrypt')(framework, check_id)) }} +{% endmacro %} + +{% macro default__iam_customer_policy_no_kms_decrypt(framework, check_id) %}{% endmacro %} + +{% macro postgres__iam_customer_policy_no_kms_decrypt(framework, check_id) %} +WITH policy_with_decrypt AS ( SELECT DISTINCT arn FROM aws_iam_policies p INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn - , lateral flatten(input => pv.document_json -> 'Statement') as s + , JSONB_ARRAY_ELEMENTS(pv.document_json -> 'Statement') as s WHERE s ->> 'Effect' = 'Allow' AND @@ -11,10 +18,43 @@ wITH policy_with_decrypt AS ( s ->> 'Resource' LIKE '%kms%') AND (s ->> 'Action' = '*' - OR s ->> 'Action' ILIKE '%kms:*%' - OR s ->> 'Action' ILIKE '%kms:decrypt%' - OR s ->> 'Action' ILIKE '%kms:reencryptfrom%' - OR s ->> 'Action' ILIKE '%kms:reencrypt*%') + OR s ->> 'Action' LIKE '%kms:*%' + OR s ->> 'Action' LIKE '%kms:decrypt%' + OR s ->> 'Action' LIKE '%kms:reencryptfrom%' + OR s ->> 'Action' LIKE '%kms:reencrypt*%') +) +SELECT + '{{framework}}' As framework, + '{{check_id}}' As check_id, + 'IAM customer managed policies should not allow decryption actions on all KMS keys' AS title, + i.account_id, + i.arn AS resource_id, + CASE + WHEN d.arn IS NULL THEN 'pass' + ELSE 'fail' + END AS status +FROM + aws_iam_policies i +LEFT JOIN policy_with_decrypt d ON i.arn = d.arn +{% endmacro %} + +{% macro snowflake__iam_customer_policy_no_kms_decrypt(framework, check_id) %} +WITH policy_with_decrypt AS ( + SELECT DISTINCT arn + FROM aws_iam_policies p + INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn + , lateral flatten(input => pv.document_json:Statement) as s + WHERE + s.value:Effect = 'Allow' + AND + (s.value:Resource = '*' OR + s.value:Resource LIKE '%kms%') + AND + (s.value:Action = '*' + OR s.value:Action ILIKE '%kms:*%' + OR s.value:Action ILIKE '%kms:decrypt%' + OR s.value:Action ILIKE '%kms:reencryptfrom%' + OR s.value:Action ILIKE '%kms:reencrypt*%') ) SELECT '{{framework}}' As framework, diff --git a/transformations/aws/macros/s3/deny_http_requests.sql b/transformations/aws/macros/s3/deny_http_requests.sql index 638430a1b..be83867eb 100644 --- a/transformations/aws/macros/s3/deny_http_requests.sql +++ b/transformations/aws/macros/s3/deny_http_requests.sql @@ -20,8 +20,9 @@ WHERE b.arn, statements.value AS statement FROM - aws_s3_buckets AS b, - LATERAL FLATTEN(INPUT => IFF(TYPEOF(b.policy:Statement) = 'STRING', TO_ARRAY(b.policy:Statement), b.policy:Statement)) AS statements + aws_s3_buckets AS b + inner join aws_s3_bucket_policies on b.arn = aws_s3_bucket_policies.bucket_arn, + LATERAL FLATTEN(INPUT => IFF(TYPEOF(aws_s3_bucket_policies.policy_json:Statement) = 'STRING', TO_ARRAY(aws_s3_bucket_policies.policy_json:Statement), aws_s3_bucket_policies.policy_json:Statement)) AS statements WHERE GET_PATH(statement, 'Effect')::STRING = 'Deny' AND GET_PATH(statement, 'Condition.Bool.aws:SecureTransport')::STRING = 'false' diff --git a/transformations/aws/macros/s3/publicly_readable_buckets.sql b/transformations/aws/macros/s3/publicly_readable_buckets.sql index 72b7bdaac..a43754d1e 100644 --- a/transformations/aws/macros/s3/publicly_readable_buckets.sql +++ b/transformations/aws/macros/s3/publicly_readable_buckets.sql @@ -13,10 +13,11 @@ WITH policy_allow_public AS ( aws_s3_buckets.arn, statements.value:Principal AS principals FROM - aws_s3_buckets, - LATERAL FLATTEN(INPUT => IFF(TYPEOF(policy:Statement) = 'STRING', - TO_ARRAY(policy:Statement), - policy:Statement)) AS statements + aws_s3_buckets + inner join aws_s3_bucket_policies bp on aws_s3_buckets.arn = bp.bucket_arn, + LATERAL FLATTEN(INPUT => IFF(TYPEOF(bp.policy_json:Statement) = 'STRING', + TO_ARRAY(bp.policy_json:Statement), + bp.policy_json:Statement)) AS statements WHERE statements.value:Effect::STRING = 'Allow' ) AS foo @@ -51,14 +52,14 @@ LEFT JOIN aws_s3_bucket_public_access_blocks ON aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn WHERE ( - (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicAcls')::boolean != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration:BlockPublicAcls)::boolean != TRUE AND ( aws_s3_bucket_grants.grantee:URI::STRING = 'http://acs.amazonaws.com/groups/global/AllUsers' AND aws_s3_bucket_grants.permission IN ('READ_ACP', 'FULL_CONTROL') ) ) OR ( - (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicPolicy')::boolean != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration:BlockPublicPolicy)::boolean != TRUE AND policy_allow_public.statement_count > 0 ) {% endmacro %} diff --git a/transformations/aws/macros/s3/publicly_writable_buckets.sql b/transformations/aws/macros/s3/publicly_writable_buckets.sql index dbb24bd16..71d79b152 100644 --- a/transformations/aws/macros/s3/publicly_writable_buckets.sql +++ b/transformations/aws/macros/s3/publicly_writable_buckets.sql @@ -13,10 +13,11 @@ WITH policy_allow_public AS ( aws_s3_buckets.arn, statements.value:Principal AS principals FROM - aws_s3_buckets, - LATERAL FLATTEN(INPUT => IFF(TYPEOF(policy:Statement) = 'STRING', - TO_ARRAY(policy:Statement), - policy:Statement)) AS statements + aws_s3_buckets + inner join aws_s3_bucket_policies bp on aws_s3_buckets.arn = bp.bucket_arn, + LATERAL FLATTEN(INPUT => IFF(TYPEOF(bp.policy_json:Statement) = 'STRING', + TO_ARRAY(bp.policy_json:Statement), + bp.policy_json:Statement)) AS statements WHERE statements.value:Effect::STRING = 'Allow' ) AS foo @@ -51,14 +52,14 @@ LEFT JOIN aws_s3_bucket_public_access_blocks ON aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn WHERE ( - (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicAcls')::boolean != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration:BlockPublicAcls)::boolean != TRUE AND ( aws_s3_bucket_grants.grantee:URI::STRING = 'http://acs.amazonaws.com/groups/global/AllUsers' AND aws_s3_bucket_grants.permission IN ('WRITE_ACP', 'FULL_CONTROL') ) ) OR ( - (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicPolicy')::boolean != TRUE + (aws_s3_bucket_public_access_blocks.public_access_block_configuration:BlockPublicPolicy)::boolean != TRUE AND policy_allow_public.statement_count > 0 ) {% endmacro %} diff --git a/transformations/aws/macros/s3/restrict_cross_account_actions.sql b/transformations/aws/macros/s3/restrict_cross_account_actions.sql index 0f4c1cb8f..c942baaff 100644 --- a/transformations/aws/macros/s3/restrict_cross_account_actions.sql +++ b/transformations/aws/macros/s3/restrict_cross_account_actions.sql @@ -13,9 +13,9 @@ select FROM ( SELECT aws_s3_buckets.arn, - account_id, - name, - region, + aws_s3_buckets.account_id, + aws_s3_buckets.name, + aws_s3_buckets.region, -- For each Statement return an array containing the principals CASE WHEN @@ -38,11 +38,12 @@ FROM ( statements.VALUE:Action END AS actions FROM - aws_s3_buckets, + aws_s3_buckets + inner join aws_s3_bucket_policies bp on aws_s3_buckets.arn = bp.bucket_arn, LATERAL FLATTEN( INPUT => CASE - WHEN TYPEOF(policy:Statement) = 'STRING' THEN TO_ARRAY(policy:Statement) - WHEN TYPEOF(policy:Statement) = 'ARRAY' THEN policy:Statement + WHEN TYPEOF(bp.policy_json:Statement) = 'STRING' THEN TO_ARRAY(bp.policy_json:Statement) + WHEN TYPEOF(bp.policy_json:Statement) = 'ARRAY' THEN bp.policy_json:Statement END ) statements WHERE @@ -74,10 +75,10 @@ select arn as resource_id, 'fail' as status -- TODO FIXME FROM ( - SELECT aws_s3_buckets.arn, - account_id, - name, - region, + SELECT b.arn, + b.account_id, + b.name, + b.region, -- For each Statement return an array containing the principals CASE WHEN @@ -97,12 +98,12 @@ FROM ( WHEN JSONB_TYPEOF(statements -> 'Action') = 'array' THEN statements -> 'Action' END AS actions - FROM aws_s3_buckets - INNER JOIN aws_s3_bucket_policies ON aws_s3_buckets.arn = aws_s3_bucket_policies.bucket_arn, + FROM aws_s3_buckets b + INNER JOIN aws_s3_bucket_policies ON b.arn = aws_s3_bucket_policies.bucket_arn, jsonb_array_elements( CASE JSONB_TYPEOF(aws_s3_bucket_policies.policy_json -> 'Statement') WHEN 'string' THEN JSONB_BUILD_ARRAY(aws_s3_bucket_policies.policy_json ->> 'Statement') - WHEN 'array' THEN policy -> 'Statement' + WHEN 'array' THEN aws_s3_bucket_policies.policy_json -> 'Statement' END ) AS statements WHERE statements -> 'Effect' = '"Allow"') AS flatten_statements, diff --git a/transformations/aws/macros/s3/s3_cross_region_replication.sql b/transformations/aws/macros/s3/s3_cross_region_replication.sql index 7d4a50458..4eafd8eb1 100644 --- a/transformations/aws/macros/s3/s3_cross_region_replication.sql +++ b/transformations/aws/macros/s3/s3_cross_region_replication.sql @@ -19,3 +19,19 @@ from inner join aws_s3_bucket_replications on aws_s3_buckets.arn = aws_s3_bucket_replications.bucket_arn -- Note: This query doesn't validate that the destination bucket is actually in a different region {% endmacro %} + +{% macro snowflake__s3_cross_region_replication(framework, check_id) %} +select + '{{framework}}' as framework, + '{{check_id}}' as check_id, + 'S3 buckets with replication rules should be enabled' as title, + aws_s3_buckets.account_id, + aws_s3_buckets.arn as resource_id, + case when + aws_s3_bucket_replications.replication_configuration:Rule:Status is distinct from 'Enabled' + then 'fail' else 'pass' end as status +from + aws_s3_buckets + inner join aws_s3_bucket_replications on aws_s3_buckets.arn = aws_s3_bucket_replications.bucket_arn +-- Note: This query doesn't validate that the destination bucket is actually in a different region +{% endmacro %} \ No newline at end of file From 005aab96406600f8242095dc48ff7c42e8d6b09c Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Wed, 27 Dec 2023 16:07:12 +0200 Subject: [PATCH 35/49] force migrate for tests --- transformations/aws/compliance-free/tests/snowflake.yml | 1 + transformations/aws/compliance-premium/tests/snowflake.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/transformations/aws/compliance-free/tests/snowflake.yml b/transformations/aws/compliance-free/tests/snowflake.yml index dd08aa173..c3a3e9f30 100644 --- a/transformations/aws/compliance-free/tests/snowflake.yml +++ b/transformations/aws/compliance-free/tests/snowflake.yml @@ -13,6 +13,7 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.5" # latest version of destination snowflake plugin + migrate_mode: forced spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index dd08aa173..c3a3e9f30 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -13,6 +13,7 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.5" # latest version of destination snowflake plugin + migrate_mode: forced spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 From 794808fb2720f2315ea9f85a6fe472d8e460dafb Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Wed, 27 Dec 2023 16:38:11 +0200 Subject: [PATCH 36/49] use v23.3.0 --- transformations/aws/compliance-premium/tests/bigquery.yml | 2 +- transformations/aws/compliance-premium/tests/postgres.yml | 2 +- transformations/aws/compliance-premium/tests/snowflake.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/transformations/aws/compliance-premium/tests/bigquery.yml b/transformations/aws/compliance-premium/tests/bigquery.yml index 0960d000c..ac682b64b 100644 --- a/transformations/aws/compliance-premium/tests/bigquery.yml +++ b/transformations/aws/compliance-premium/tests/bigquery.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.2.0" # latest version of source aws plugin + version: "v23.3.0" # latest version of source aws plugin destinations: ["bigquery"] tables: ["*"] --- diff --git a/transformations/aws/compliance-premium/tests/postgres.yml b/transformations/aws/compliance-premium/tests/postgres.yml index 0a1c3080a..2c8527e68 100644 --- a/transformations/aws/compliance-premium/tests/postgres.yml +++ b/transformations/aws/compliance-premium/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.2.0" # latest version of source aws plugin + version: "v23.3.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index c3a3e9f30..3b839ed21 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.2.0" # latest version of source aws plugin + version: "v23.3.0" # latest version of source aws plugin destinations: ["snowflake"] tables: ["*"] --- From 355ed435ca16c97a1e1fc7f397239860a8496c9c Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Wed, 27 Dec 2023 16:38:40 +0200 Subject: [PATCH 37/49] use v23.3.0 --- transformations/aws/asset-inventory-free/manifest.json | 2 +- transformations/aws/asset-inventory-free/tests/postgres.yml | 2 +- transformations/aws/compliance-free/tests/bigquery.yml | 2 +- transformations/aws/compliance-free/tests/postgres.yml | 2 +- transformations/aws/compliance-free/tests/snowflake.yml | 2 +- transformations/aws/cost/tests/postgres.yml | 2 +- transformations/aws/data-resilience/manifest.json | 2 +- transformations/aws/data-resilience/tests/postgres.yml | 2 +- transformations/aws/encryption/manifest.json | 2 +- transformations/aws/encryption/tests/postgres.yml | 2 +- visualizations/aws/asset_inventory/manifest.json | 2 +- visualizations/aws/compliance/manifest.json | 2 +- visualizations/aws/data_resilience/manifest.json | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/transformations/aws/asset-inventory-free/manifest.json b/transformations/aws/asset-inventory-free/manifest.json index 0de3b0eee..225ecaa0c 100644 --- a/transformations/aws/asset-inventory-free/manifest.json +++ b/transformations/aws/asset-inventory-free/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_asset_inventory_free.zip", - "plugin_deps": ["cloudquery/source/aws@v23.2.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.0"], "addon_deps": [] } diff --git a/transformations/aws/asset-inventory-free/tests/postgres.yml b/transformations/aws/asset-inventory-free/tests/postgres.yml index 0a1c3080a..2c8527e68 100644 --- a/transformations/aws/asset-inventory-free/tests/postgres.yml +++ b/transformations/aws/asset-inventory-free/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.2.0" # latest version of source aws plugin + version: "v23.3.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/compliance-free/tests/bigquery.yml b/transformations/aws/compliance-free/tests/bigquery.yml index 0960d000c..ac682b64b 100644 --- a/transformations/aws/compliance-free/tests/bigquery.yml +++ b/transformations/aws/compliance-free/tests/bigquery.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.2.0" # latest version of source aws plugin + version: "v23.3.0" # latest version of source aws plugin destinations: ["bigquery"] tables: ["*"] --- diff --git a/transformations/aws/compliance-free/tests/postgres.yml b/transformations/aws/compliance-free/tests/postgres.yml index 0a1c3080a..2c8527e68 100644 --- a/transformations/aws/compliance-free/tests/postgres.yml +++ b/transformations/aws/compliance-free/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.2.0" # latest version of source aws plugin + version: "v23.3.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/compliance-free/tests/snowflake.yml b/transformations/aws/compliance-free/tests/snowflake.yml index c3a3e9f30..3b839ed21 100644 --- a/transformations/aws/compliance-free/tests/snowflake.yml +++ b/transformations/aws/compliance-free/tests/snowflake.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.2.0" # latest version of source aws plugin + version: "v23.3.0" # latest version of source aws plugin destinations: ["snowflake"] tables: ["*"] --- diff --git a/transformations/aws/cost/tests/postgres.yml b/transformations/aws/cost/tests/postgres.yml index 0a1c3080a..2c8527e68 100644 --- a/transformations/aws/cost/tests/postgres.yml +++ b/transformations/aws/cost/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.2.0" # latest version of source aws plugin + version: "v23.3.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/data-resilience/manifest.json b/transformations/aws/data-resilience/manifest.json index 0fcaab7da..51892b6bb 100644 --- a/transformations/aws/data-resilience/manifest.json +++ b/transformations/aws/data-resilience/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_data_resilience.zip", - "plugin_deps": ["cloudquery/source/aws@v23.2.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.0"], "addon_deps": [] } diff --git a/transformations/aws/data-resilience/tests/postgres.yml b/transformations/aws/data-resilience/tests/postgres.yml index 0a1c3080a..2c8527e68 100644 --- a/transformations/aws/data-resilience/tests/postgres.yml +++ b/transformations/aws/data-resilience/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.2.0" # latest version of source aws plugin + version: "v23.3.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/encryption/manifest.json b/transformations/aws/encryption/manifest.json index 6147885fb..42c7c3032 100644 --- a/transformations/aws/encryption/manifest.json +++ b/transformations/aws/encryption/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_encryption.zip", - "plugin_deps": ["cloudquery/source/aws@v23.2.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.0"], "addon_deps": [] } diff --git a/transformations/aws/encryption/tests/postgres.yml b/transformations/aws/encryption/tests/postgres.yml index 0a1c3080a..2c8527e68 100644 --- a/transformations/aws/encryption/tests/postgres.yml +++ b/transformations/aws/encryption/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.2.0" # latest version of source aws plugin + version: "v23.3.0" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/visualizations/aws/asset_inventory/manifest.json b/visualizations/aws/asset_inventory/manifest.json index 0f99bde66..2034c780c 100644 --- a/visualizations/aws/asset_inventory/manifest.json +++ b/visualizations/aws/asset_inventory/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_asset_inventory.zip", - "plugin_deps": ["cloudquery/source/aws@v23.2.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.0"], "addon_deps": ["cloudquery/transformation/aws-asset-inventory@v1.0.0"] } \ No newline at end of file diff --git a/visualizations/aws/compliance/manifest.json b/visualizations/aws/compliance/manifest.json index 5daaa588c..c3a27e39a 100644 --- a/visualizations/aws/compliance/manifest.json +++ b/visualizations/aws/compliance/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_compliance.zip", - "plugin_deps": ["cloudquery/source/aws@v23.2.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.0"], "addon_deps": ["cloudquery/transformation/aws-compliance-free@v0.0.1"] } diff --git a/visualizations/aws/data_resilience/manifest.json b/visualizations/aws/data_resilience/manifest.json index 12532aad9..8e9acfef1 100644 --- a/visualizations/aws/data_resilience/manifest.json +++ b/visualizations/aws/data_resilience/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_data_resilience.zip", - "plugin_deps": ["cloudquery/source/aws@v23.2.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.0"], "addon_deps": ["cloudquery/transformation/aws-data-resilience@v1.0.0"] } From 46441bc30a2036535d0381fa76ebaccf48b62d30 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Wed, 27 Dec 2023 17:36:54 +0200 Subject: [PATCH 38/49] Revert "force migrate for tests" This reverts commit 005aab96406600f8242095dc48ff7c42e8d6b09c. --- transformations/aws/compliance-free/tests/snowflake.yml | 1 - transformations/aws/compliance-premium/tests/snowflake.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/transformations/aws/compliance-free/tests/snowflake.yml b/transformations/aws/compliance-free/tests/snowflake.yml index 3b839ed21..e7740945e 100644 --- a/transformations/aws/compliance-free/tests/snowflake.yml +++ b/transformations/aws/compliance-free/tests/snowflake.yml @@ -13,7 +13,6 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.5" # latest version of destination snowflake plugin - migrate_mode: forced spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index 3b839ed21..e7740945e 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -13,7 +13,6 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.5" # latest version of destination snowflake plugin - migrate_mode: forced spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 From a0c64e29bbbebe98a83bf833298df24c49608fc1 Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Wed, 27 Dec 2023 18:23:33 +0200 Subject: [PATCH 39/49] Updated queries cloudtrail_enabled_all_regions, bucket_access_logging --- .../aws/macros/cloudtrail/bucket_access_logging.sql | 4 ++-- .../aws/macros/cloudtrail/enabled_in_all_regions.sql | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/transformations/aws/macros/cloudtrail/bucket_access_logging.sql b/transformations/aws/macros/cloudtrail/bucket_access_logging.sql index 275d4144e..98b6619f6 100644 --- a/transformations/aws/macros/cloudtrail/bucket_access_logging.sql +++ b/transformations/aws/macros/cloudtrail/bucket_access_logging.sql @@ -31,8 +31,8 @@ select t.arn as resource_id, case when l.logging_enabled is null then 'fail' - when l.logging_enabled -> 'TargetBucket' is null then 'fail' - when l.logging_enabled -> 'TargetPrefix' is null then 'fail' + when l.logging_enabled.TargetBucket is null then 'fail' + when l.logging_enabled.TargetPrefix is null then 'fail' else 'pass' end as status from {{ full_table_name("aws_cloudtrail_trails") }} t diff --git a/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql b/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql index 6e536ac19..2e2b916fd 100644 --- a/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql +++ b/transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql @@ -41,14 +41,15 @@ select case when aws_cloudtrail_trails.is_multi_region_trail = FALSE then 'fail' when exists(select * - from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.event_selectors) as es - where es ->>'ReadWriteType' != 'All' or (es->>'IncludeManagementEvents')::boolean = FALSE) + from UNNEST(JSON_QUERY_ARRAY(aws_cloudtrail_trail_event_selectors.event_selectors)) AS es + where JSON_VALUE(es.ReadWriteType) != 'All' or (CAST( JSON_VALUE(es.IncludeManagementEvents) AS BOOL)= FALSE ) + ) then 'fail' when exists(select * - from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.advanced_event_selectors) as aes + from UNNEST(JSON_QUERY_ARRAY(aws_cloudtrail_trail_event_selectors.advanced_event_selectors)) AS aes where exists(select * - from jsonb_array_elements(aes ->'FieldSelectors') as aes_fs - where aes_fs ->>'Field' = 'readOnly')) + from UNNEST(JSON_QUERY_ARRAY(aes.FieldSelectors)) as aes_fs + where JSON_VALUE(aes_fs.Field) = 'readOnly')) then 'fail' else 'pass' end as status From 9c52378ddcadf0aa03a5e0fa5da471716bc1a55e Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Wed, 27 Dec 2023 21:22:45 +0200 Subject: [PATCH 40/49] Updated manifest compliances --- transformations/aws/compliance-free/manifest.json | 2 +- transformations/aws/compliance-premium/manifest.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transformations/aws/compliance-free/manifest.json b/transformations/aws/compliance-free/manifest.json index 3239855fe..58c4348cf 100644 --- a/transformations/aws/compliance-free/manifest.json +++ b/transformations/aws/compliance-free/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_compliance_free.zip", - "plugin_deps": ["cloudquery/source/aws@v22.18.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.0"], "addon_deps": [] } diff --git a/transformations/aws/compliance-premium/manifest.json b/transformations/aws/compliance-premium/manifest.json index df2e14ab6..d673667a3 100644 --- a/transformations/aws/compliance-premium/manifest.json +++ b/transformations/aws/compliance-premium/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_compliance_premium.zip", - "plugin_deps": ["cloudquery/source/aws@v22.18.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.0"], "addon_deps": [] } From 191fdd2d962ff122ae24f53f787193ca9d880656 Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Thu, 28 Dec 2023 12:25:18 +0200 Subject: [PATCH 41/49] Updated queries elastic_beanstalk_stream_logs_to_cloudwatch, s3_bucket_level_public_access_prohibited, s3_bucket_logging_enabled, s3_version_lifecycle_policy_check --- ...astic_beanstalk_stream_logs_to_cloudwatch.sql | 4 ++-- .../s3_bucket_level_public_access_prohibited.sql | 16 +++++++++------- .../aws/macros/s3/s3_bucket_logging_enabled.sql | 10 ++++++---- .../s3/s3_version_lifecycle_policy_check.sql | 8 ++++++-- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/transformations/aws/macros/elasticbeanstalk/elastic_beanstalk_stream_logs_to_cloudwatch.sql b/transformations/aws/macros/elasticbeanstalk/elastic_beanstalk_stream_logs_to_cloudwatch.sql index 2242d659c..de7b91f36 100644 --- a/transformations/aws/macros/elasticbeanstalk/elastic_beanstalk_stream_logs_to_cloudwatch.sql +++ b/transformations/aws/macros/elasticbeanstalk/elastic_beanstalk_stream_logs_to_cloudwatch.sql @@ -1,7 +1,7 @@ {% macro elastic_beanstalk_stream_logs_to_cloudwatch(framework, check_id) %} with flat_configs as ( select - c.environment_id, + c.environment_arn, f.value:Namespace:Value::string as is_log_streaming from @@ -26,5 +26,5 @@ SELECT END as status FROM aws_elasticbeanstalk_environments e JOIN flat_configs as fc - ON e.environment_id = fc.environment_id + ON e.environment_id = fc.environment_arn {% endmacro %} \ No newline at end of file diff --git a/transformations/aws/macros/s3/s3_bucket_level_public_access_prohibited.sql b/transformations/aws/macros/s3/s3_bucket_level_public_access_prohibited.sql index c80e33e45..bf8380127 100644 --- a/transformations/aws/macros/s3/s3_bucket_level_public_access_prohibited.sql +++ b/transformations/aws/macros/s3/s3_bucket_level_public_access_prohibited.sql @@ -3,15 +3,17 @@ select '{{framework}}' As framework, '{{check_id}}' As check_id, 'S3 Block Public Access setting should be enabled at the bucket-level' AS title, - account_id, - arn AS resource_id, + b.account_id, + b.arn AS resource_id, CASE - when block_public_acls - and block_public_policy - and ignore_public_acls - and restrict_public_buckets THEN 'pass' + when pab.public_access_block_configuration:block_public_acls + and pab.public_access_block_configuration:block_public_policy + and pab.public_access_block_configuration:ignore_public_acls + and pab.public_access_block_configuration:restrict_public_buckets THEN 'pass' ELSE 'fail' END AS status FROM - aws_s3_buckets + aws_s3_buckets as b +LEFT JOIN + aws_s3_bucket_public_access_blocks as pab on pab.bucket_arn = b.arn {% endmacro %} \ No newline at end of file diff --git a/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql b/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql index 836058828..d570b839b 100644 --- a/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql +++ b/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql @@ -3,12 +3,14 @@ select '{{framework}}' As framework, '{{check_id}}' As check_id, 'S3 bucket server access logging should be enabled' AS title, - account_id, - arn AS resource_id, + b.account_id, + b.arn AS resource_id, CASE - when logging_target_bucket IS NOT NULL + when bl.logging_enabled:TargetBucket IS NOT NULL THEN 'pass' ELSE 'fail' END AS status FROM - aws_s3_buckets + aws_s3_buckets as b +LEFT JOIN + aws_s3_bucket_loggings as bl on bl.bucket_arn = b.arn {% endmacro %} \ No newline at end of file diff --git a/transformations/aws/macros/s3/s3_version_lifecycle_policy_check.sql b/transformations/aws/macros/s3/s3_version_lifecycle_policy_check.sql index 230deb88a..f3aa24bb3 100644 --- a/transformations/aws/macros/s3/s3_version_lifecycle_policy_check.sql +++ b/transformations/aws/macros/s3/s3_version_lifecycle_policy_check.sql @@ -11,9 +11,13 @@ select END AS status FROM aws_s3_buckets AS b +LEFT JOIN + aws_s3_bucket_versionings AS bv + ON + b.arn = bv.bucket_arn LEFT JOIN aws_s3_bucket_lifecycles AS l -ON + ON b.arn = l.bucket_arn -where b.versioning_status = 'Enabled' +where bv.status = 'Enabled' {% endmacro %} \ No newline at end of file From 4e5cc6988c009af5f9cfca85412b1a01f428b7fc Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Thu, 28 Dec 2023 13:12:18 +0200 Subject: [PATCH 42/49] check query --- .../aws/macros/s3/s3_bucket_logging_enabled.sql | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql b/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql index d570b839b..836058828 100644 --- a/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql +++ b/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql @@ -3,14 +3,12 @@ select '{{framework}}' As framework, '{{check_id}}' As check_id, 'S3 bucket server access logging should be enabled' AS title, - b.account_id, - b.arn AS resource_id, + account_id, + arn AS resource_id, CASE - when bl.logging_enabled:TargetBucket IS NOT NULL + when logging_target_bucket IS NOT NULL THEN 'pass' ELSE 'fail' END AS status FROM - aws_s3_buckets as b -LEFT JOIN - aws_s3_bucket_loggings as bl on bl.bucket_arn = b.arn + aws_s3_buckets {% endmacro %} \ No newline at end of file From 35c0fd0a1ae02f7a141a560050aafcbeb2fc66a0 Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 28 Dec 2023 13:25:26 +0200 Subject: [PATCH 43/49] Update aws to `v23.3.1` --- transformations/aws/asset-inventory-free/manifest.json | 2 +- transformations/aws/asset-inventory-free/tests/postgres.yml | 2 +- transformations/aws/compliance-free/manifest.json | 2 +- transformations/aws/compliance-free/tests/bigquery.yml | 2 +- transformations/aws/compliance-free/tests/postgres.yml | 2 +- transformations/aws/compliance-free/tests/snowflake.yml | 2 +- transformations/aws/compliance-premium/manifest.json | 2 +- transformations/aws/compliance-premium/tests/bigquery.yml | 2 +- transformations/aws/compliance-premium/tests/postgres.yml | 2 +- transformations/aws/compliance-premium/tests/snowflake.yml | 2 +- transformations/aws/cost/tests/postgres.yml | 2 +- transformations/aws/data-resilience/manifest.json | 2 +- transformations/aws/data-resilience/tests/postgres.yml | 2 +- transformations/aws/encryption/manifest.json | 2 +- transformations/aws/encryption/tests/postgres.yml | 2 +- visualizations/aws/asset_inventory/manifest.json | 2 +- visualizations/aws/compliance/manifest.json | 2 +- visualizations/aws/data_resilience/manifest.json | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/transformations/aws/asset-inventory-free/manifest.json b/transformations/aws/asset-inventory-free/manifest.json index 225ecaa0c..089fec786 100644 --- a/transformations/aws/asset-inventory-free/manifest.json +++ b/transformations/aws/asset-inventory-free/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_asset_inventory_free.zip", - "plugin_deps": ["cloudquery/source/aws@v23.3.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.1"], "addon_deps": [] } diff --git a/transformations/aws/asset-inventory-free/tests/postgres.yml b/transformations/aws/asset-inventory-free/tests/postgres.yml index 2c8527e68..4877a6f20 100644 --- a/transformations/aws/asset-inventory-free/tests/postgres.yml +++ b/transformations/aws/asset-inventory-free/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.3.0" # latest version of source aws plugin + version: "v23.3.1" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/compliance-free/manifest.json b/transformations/aws/compliance-free/manifest.json index 58c4348cf..ecfe1a367 100644 --- a/transformations/aws/compliance-free/manifest.json +++ b/transformations/aws/compliance-free/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_compliance_free.zip", - "plugin_deps": ["cloudquery/source/aws@v23.3.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.1"], "addon_deps": [] } diff --git a/transformations/aws/compliance-free/tests/bigquery.yml b/transformations/aws/compliance-free/tests/bigquery.yml index ac682b64b..4e5febc5d 100644 --- a/transformations/aws/compliance-free/tests/bigquery.yml +++ b/transformations/aws/compliance-free/tests/bigquery.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.3.0" # latest version of source aws plugin + version: "v23.3.1" # latest version of source aws plugin destinations: ["bigquery"] tables: ["*"] --- diff --git a/transformations/aws/compliance-free/tests/postgres.yml b/transformations/aws/compliance-free/tests/postgres.yml index 2c8527e68..4877a6f20 100644 --- a/transformations/aws/compliance-free/tests/postgres.yml +++ b/transformations/aws/compliance-free/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.3.0" # latest version of source aws plugin + version: "v23.3.1" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/compliance-free/tests/snowflake.yml b/transformations/aws/compliance-free/tests/snowflake.yml index e7740945e..936222b99 100644 --- a/transformations/aws/compliance-free/tests/snowflake.yml +++ b/transformations/aws/compliance-free/tests/snowflake.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.3.0" # latest version of source aws plugin + version: "v23.3.1" # latest version of source aws plugin destinations: ["snowflake"] tables: ["*"] --- diff --git a/transformations/aws/compliance-premium/manifest.json b/transformations/aws/compliance-premium/manifest.json index d673667a3..173311720 100644 --- a/transformations/aws/compliance-premium/manifest.json +++ b/transformations/aws/compliance-premium/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_compliance_premium.zip", - "plugin_deps": ["cloudquery/source/aws@v23.3.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.1"], "addon_deps": [] } diff --git a/transformations/aws/compliance-premium/tests/bigquery.yml b/transformations/aws/compliance-premium/tests/bigquery.yml index ac682b64b..4e5febc5d 100644 --- a/transformations/aws/compliance-premium/tests/bigquery.yml +++ b/transformations/aws/compliance-premium/tests/bigquery.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.3.0" # latest version of source aws plugin + version: "v23.3.1" # latest version of source aws plugin destinations: ["bigquery"] tables: ["*"] --- diff --git a/transformations/aws/compliance-premium/tests/postgres.yml b/transformations/aws/compliance-premium/tests/postgres.yml index 2c8527e68..4877a6f20 100644 --- a/transformations/aws/compliance-premium/tests/postgres.yml +++ b/transformations/aws/compliance-premium/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.3.0" # latest version of source aws plugin + version: "v23.3.1" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index e7740945e..936222b99 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.3.0" # latest version of source aws plugin + version: "v23.3.1" # latest version of source aws plugin destinations: ["snowflake"] tables: ["*"] --- diff --git a/transformations/aws/cost/tests/postgres.yml b/transformations/aws/cost/tests/postgres.yml index 2c8527e68..4877a6f20 100644 --- a/transformations/aws/cost/tests/postgres.yml +++ b/transformations/aws/cost/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.3.0" # latest version of source aws plugin + version: "v23.3.1" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/data-resilience/manifest.json b/transformations/aws/data-resilience/manifest.json index 51892b6bb..cba8baa24 100644 --- a/transformations/aws/data-resilience/manifest.json +++ b/transformations/aws/data-resilience/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_data_resilience.zip", - "plugin_deps": ["cloudquery/source/aws@v23.3.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.1"], "addon_deps": [] } diff --git a/transformations/aws/data-resilience/tests/postgres.yml b/transformations/aws/data-resilience/tests/postgres.yml index 2c8527e68..4877a6f20 100644 --- a/transformations/aws/data-resilience/tests/postgres.yml +++ b/transformations/aws/data-resilience/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.3.0" # latest version of source aws plugin + version: "v23.3.1" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/transformations/aws/encryption/manifest.json b/transformations/aws/encryption/manifest.json index 42c7c3032..8dbcf15ad 100644 --- a/transformations/aws/encryption/manifest.json +++ b/transformations/aws/encryption/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_encryption.zip", - "plugin_deps": ["cloudquery/source/aws@v23.3.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.1"], "addon_deps": [] } diff --git a/transformations/aws/encryption/tests/postgres.yml b/transformations/aws/encryption/tests/postgres.yml index 2c8527e68..4877a6f20 100644 --- a/transformations/aws/encryption/tests/postgres.yml +++ b/transformations/aws/encryption/tests/postgres.yml @@ -3,7 +3,7 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v23.3.0" # latest version of source aws plugin + version: "v23.3.1" # latest version of source aws plugin destinations: ["postgresql"] tables: ["*"] --- diff --git a/visualizations/aws/asset_inventory/manifest.json b/visualizations/aws/asset_inventory/manifest.json index 2034c780c..90ed0971c 100644 --- a/visualizations/aws/asset_inventory/manifest.json +++ b/visualizations/aws/asset_inventory/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_asset_inventory.zip", - "plugin_deps": ["cloudquery/source/aws@v23.3.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.1"], "addon_deps": ["cloudquery/transformation/aws-asset-inventory@v1.0.0"] } \ No newline at end of file diff --git a/visualizations/aws/compliance/manifest.json b/visualizations/aws/compliance/manifest.json index c3a27e39a..79b4907f3 100644 --- a/visualizations/aws/compliance/manifest.json +++ b/visualizations/aws/compliance/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_compliance.zip", - "plugin_deps": ["cloudquery/source/aws@v23.3.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.1"], "addon_deps": ["cloudquery/transformation/aws-compliance-free@v0.0.1"] } diff --git a/visualizations/aws/data_resilience/manifest.json b/visualizations/aws/data_resilience/manifest.json index 8e9acfef1..1123d19c0 100644 --- a/visualizations/aws/data_resilience/manifest.json +++ b/visualizations/aws/data_resilience/manifest.json @@ -8,6 +8,6 @@ "message": "@./changelog.md", "doc": "./README.md", "path": "./build/aws_data_resilience.zip", - "plugin_deps": ["cloudquery/source/aws@v23.3.0"], + "plugin_deps": ["cloudquery/source/aws@v23.3.1"], "addon_deps": ["cloudquery/transformation/aws-data-resilience@v1.0.0"] } From bdfb813ab906e2a5037c0cfbeb3c49cd1093e9ca Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Thu, 28 Dec 2023 15:20:57 +0200 Subject: [PATCH 44/49] Updated query s3_bucket_logging_enabled --- .../aws/macros/s3/s3_bucket_logging_enabled.sql | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql b/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql index 836058828..d570b839b 100644 --- a/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql +++ b/transformations/aws/macros/s3/s3_bucket_logging_enabled.sql @@ -3,12 +3,14 @@ select '{{framework}}' As framework, '{{check_id}}' As check_id, 'S3 bucket server access logging should be enabled' AS title, - account_id, - arn AS resource_id, + b.account_id, + b.arn AS resource_id, CASE - when logging_target_bucket IS NOT NULL + when bl.logging_enabled:TargetBucket IS NOT NULL THEN 'pass' ELSE 'fail' END AS status FROM - aws_s3_buckets + aws_s3_buckets as b +LEFT JOIN + aws_s3_bucket_loggings as bl on bl.bucket_arn = b.arn {% endmacro %} \ No newline at end of file From 4bdcce2e0576df60da1ca8e086dd4cd9a8e88f3d Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 28 Dec 2023 16:08:54 +0200 Subject: [PATCH 45/49] Update transformations/aws/compliance-premium/tests/snowflake.yml --- transformations/aws/compliance-premium/tests/snowflake.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index 936222b99..cda5f1bb1 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -4,6 +4,7 @@ spec: path: cloudquery/aws registry: cloudquery version: "v23.3.1" # latest version of source aws plugin + migrate-mode: forced # tmp destinations: ["snowflake"] tables: ["*"] --- From 6b2f7cc6f7aa07a2a43907e1ee64df55e967df35 Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Thu, 28 Dec 2023 16:22:53 +0200 Subject: [PATCH 46/49] Updated query elastic_beanstalk_stream_logs_to_cloudwatch --- .../elastic_beanstalk_stream_logs_to_cloudwatch.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformations/aws/macros/elasticbeanstalk/elastic_beanstalk_stream_logs_to_cloudwatch.sql b/transformations/aws/macros/elasticbeanstalk/elastic_beanstalk_stream_logs_to_cloudwatch.sql index de7b91f36..1ab8c8e40 100644 --- a/transformations/aws/macros/elasticbeanstalk/elastic_beanstalk_stream_logs_to_cloudwatch.sql +++ b/transformations/aws/macros/elasticbeanstalk/elastic_beanstalk_stream_logs_to_cloudwatch.sql @@ -26,5 +26,5 @@ SELECT END as status FROM aws_elasticbeanstalk_environments e JOIN flat_configs as fc - ON e.environment_id = fc.environment_arn + ON e.arn = fc.environment_arn {% endmacro %} \ No newline at end of file From 3c80494f1251706f27d2c70114c107973cab691c Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 28 Dec 2023 16:27:07 +0200 Subject: [PATCH 47/49] Update transformations/aws/compliance-premium/tests/snowflake.yml --- transformations/aws/compliance-premium/tests/snowflake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index cda5f1bb1..4b1078465 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -4,7 +4,7 @@ spec: path: cloudquery/aws registry: cloudquery version: "v23.3.1" # latest version of source aws plugin - migrate-mode: forced # tmp + migrate_mode: forced # tmp destinations: ["snowflake"] tables: ["*"] --- From 946f63f6a67158a11492c96d2d99d107ab0e337d Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 28 Dec 2023 17:02:33 +0200 Subject: [PATCH 48/49] tmp force migration --- transformations/aws/compliance-premium/tests/snowflake.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index b720849b6..4a7327421 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -4,7 +4,6 @@ spec: path: cloudquery/aws registry: cloudquery version: "v23.3.1" # latest version of source aws plugin - migrate_mode: forced # tmp destinations: ["snowflake"] tables: ["*"] --- @@ -14,6 +13,7 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.6" # latest version of destination snowflake plugin + migrate_mode: forced # tmp spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50 From 5be873d82b44502f394a20151c973f18cdbe7590 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 28 Dec 2023 17:37:54 +0200 Subject: [PATCH 49/49] no forced migration --- transformations/aws/compliance-premium/tests/snowflake.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/transformations/aws/compliance-premium/tests/snowflake.yml b/transformations/aws/compliance-premium/tests/snowflake.yml index 4a7327421..3a384a5a9 100644 --- a/transformations/aws/compliance-premium/tests/snowflake.yml +++ b/transformations/aws/compliance-premium/tests/snowflake.yml @@ -13,7 +13,6 @@ spec: path: cloudquery/snowflake registry: github version: "v3.3.6" # latest version of destination snowflake plugin - migrate_mode: forced # tmp spec: connection_string: ${SNOWFLAKE_CONNECTION_STRING} migrate_concurrency: 50