From 2878a2148f182738b9888457abd223a1c6b783da Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Fri, 29 Dec 2023 01:15:23 +0200 Subject: [PATCH 1/6] Updated query log_metric_filter_and_alarm --- .../macros/log_metric_filter_and_alarm.sql | 63 +++++++------------ 1 file changed, 21 insertions(+), 42 deletions(-) diff --git a/transformations/aws/macros/log_metric_filter_and_alarm.sql b/transformations/aws/macros/log_metric_filter_and_alarm.sql index 6e5ddac75..d684c167a 100644 --- a/transformations/aws/macros/log_metric_filter_and_alarm.sql +++ b/transformations/aws/macros/log_metric_filter_and_alarm.sql @@ -36,63 +36,42 @@ where t.is_multi_region_trail = TRUE {% macro bigquery__log_metric_filter_and_alarm() %} with af as ( - select distinct a.arn, a.actions_enabled, ARRAY_TO_STRING(a.alarm_actions, ',') as alarm_actions, JSON_VALUE(m.MetricStat.Metric.MetricName) as metric_name -- TODO check - from {{ full_table_name("aws_cloudwatch_alarms") }} a, UNNEST(JSON_QUERY_ARRAY(metrics)) as m -) -select - t.account_id, - t.region, - t.cloud_watch_logs_log_group_arn, - mf.filter_pattern as pattern -from {{ full_table_name("aws_cloudtrail_trails") }} t -inner join {{ full_table_name("aws_cloudtrail_trail_event_selectors") }} tes on t.arn = tes.trail_arn -inner join {{ full_table_name("aws_cloudwatchlogs_metric_filters") }} mf on mf.log_group_name = t.cloudwatch_logs_log_group_name -inner join af on mf.filter_name = af.metric_name -inner join {{ full_table_name("aws_sns_subscriptions") }} ss on ss.topic_arn in UNNEST(SPLIT(af.alarm_actions, ',')) -where t.is_multi_region_trail = TRUE - and CAST( JSON_VALUE(t.status.IsLogging) AS BOOL) = TRUE - and tes.include_management_events = TRUE - and tes.read_write_type = 'All' - and ss.arn like 'aws:arn:%' -{% endmacro %} - -{% macro snowflake__log_metric_filter_and_alarm() %} -with af as ( - select distinct a.arn, a.actions_enabled, a.alarm_actions, m.value:MetricStat:Metric:MetricName as metric_name -- TODO check - from aws_cloudwatch_alarms a, - LATERAL FLATTEN (metrics) as m + select distinct a.arn, a.actions_enabled, ARRAY_TO_STRING(a.alarm_actions, ',') as alarm_actions, JSON_VALUE(m.MetricStat.Metric.MetricName) as metric_name + from {{ full_table_name("aws_cloudwatch_alarms") }} a, + UNNEST(JSON_QUERY_ARRAY(metrics)) as m ), aes as ( -select * from aws_cloudtrail_trail_event_selectors, - LATERAL FLATTEN (advanced_event_selectors) as aes +select * from {{ full_table_name("aws_cloudtrail_trail_event_selectors") }} , + UNNEST(JSON_QUERY_ARRAY(advanced_event_selectors)) as aes ), tes as ( - select trail_arn from aws_cloudtrail_trail_event_selectors + select trail_arn from {{ full_table_name("aws_cloudtrail_trail_event_selectors") }} where exists( select * from - aws_cloudtrail_trail_event_selectors, - LATERAL FLATTEN(event_selectors) as es - where es.value:ReadWriteType = 'All' and (es.value:IncludeManagementEvents)::boolean = TRUE + {{ full_table_name("aws_cloudtrail_trail_event_selectors") }}, + UNNEST(JSON_QUERY_ARRAY(event_selectors)) as es + where JSON_VALUE(es.ReadWriteType) = 'All' and CAST( JSON_VALUE(es.IncludeManagementEvents) AS BOOL) = TRUE ) - or exists( - select * from aes - where not exists ( - select * from aes, LATERAL FLATTEN(value:FieldSelectors) as aes_fs - where aes_fs.value:Field = 'readOnly' - ) - ) + or exists( + select * from aes + where not exists ( + select * from aes, + UNNEST(JSON_QUERY_ARRAY(aes.aes.FieldSelectors)) as aes_fs + where JSON_VALUE(aes_fs.Field) = 'readOnly' + ) + ) ) select t.account_id, t.region, t.cloud_watch_logs_log_group_arn, mf.filter_pattern as pattern -from aws_cloudtrail_trails t +from {{ full_table_name("aws_cloudtrail_trails") }} t inner join tes on t.arn = tes.trail_arn -inner join aws_cloudwatchlogs_metric_filters mf on mf.log_group_name = t.cloudwatch_logs_log_group_name +inner join {{ full_table_name("aws_cloudwatchlogs_metric_filters") }} mf on mf.log_group_name = t.cloudwatch_logs_log_group_name inner join af on mf.filter_name = af.metric_name -inner join aws_sns_subscriptions ss on ARRAY_CONTAINS((ss.topic_arn)::variant, af.alarm_actions) +inner join {{ full_table_name("aws_sns_subscriptions") }} ss on ss.topic_arn in UNNEST(SPLIT(af.alarm_actions, ',')) where t.is_multi_region_trail = TRUE - and (t.status:IsLogging)::boolean = TRUE + and CAST( JSON_VALUE(t.status.IsLogging) AS BOOL) = TRUE and ss.arn like 'aws:arn:%' {% endmacro %} \ No newline at end of file From 52e68892f76a29a8f919cecf0ec764c360f5fb28 Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Fri, 29 Dec 2023 01:23:17 +0200 Subject: [PATCH 2/6] Updated query log_metric_filter_and_alarm - snowflake --- .../macros/log_metric_filter_and_alarm.sql | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/transformations/aws/macros/log_metric_filter_and_alarm.sql b/transformations/aws/macros/log_metric_filter_and_alarm.sql index d684c167a..bfd774681 100644 --- a/transformations/aws/macros/log_metric_filter_and_alarm.sql +++ b/transformations/aws/macros/log_metric_filter_and_alarm.sql @@ -74,4 +74,45 @@ inner join {{ full_table_name("aws_sns_subscriptions") }} ss on ss.topic_arn in where t.is_multi_region_trail = TRUE and CAST( JSON_VALUE(t.status.IsLogging) AS BOOL) = TRUE and ss.arn like 'aws:arn:%' +{% endmacro %} + +{% macro snowflake__log_metric_filter_and_alarm(framework, check_id) %} +with af as ( + select distinct a.arn, a.actions_enabled, a.alarm_actions, m.value:MetricStat:Metric:MetricName as metric_name -- TODO check + from aws_cloudwatch_alarms a, + LATERAL FLATTEN (metrics) as m +), +aes as ( +select * from aws_cloudtrail_trail_event_selectors, + LATERAL FLATTEN (advanced_event_selectors) as aes +), +tes as ( + select trail_arn from aws_cloudtrail_trail_event_selectors + where exists( + select * from + aws_cloudtrail_trail_event_selectors, + LATERAL FLATTEN(event_selectors) as es + where es.value:ReadWriteType = 'All' and (es.value:IncludeManagementEvents)::boolean = TRUE + ) + or exists( + select * from aes + where not exists ( + select * from aes, LATERAL FLATTEN(value:FieldSelectors) as aes_fs + where aes_fs.value:Field = 'readOnly' + ) + ) +) +select + t.account_id, + t.region, + t.cloud_watch_logs_log_group_arn, + mf.filter_pattern as pattern +from aws_cloudtrail_trails t +inner join tes on t.arn = tes.trail_arn +inner join aws_cloudwatchlogs_metric_filters mf on mf.log_group_name = t.cloudwatch_logs_log_group_name +inner join af on mf.filter_name = af.metric_name +inner join aws_sns_subscriptions ss on ARRAY_CONTAINS((ss.topic_arn)::variant, af.alarm_actions) +where t.is_multi_region_trail = TRUE + and (t.status:IsLogging)::boolean = TRUE + and ss.arn like 'aws:arn:%' {% endmacro %} \ No newline at end of file From a73a2255d81dffae35d24700459fcd24a050174f Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Fri, 29 Dec 2023 01:24:34 +0200 Subject: [PATCH 3/6] Updated 1 query log_metric_filter_and_alarm - snowflake --- transformations/aws/macros/log_metric_filter_and_alarm.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformations/aws/macros/log_metric_filter_and_alarm.sql b/transformations/aws/macros/log_metric_filter_and_alarm.sql index bfd774681..bd4b205bb 100644 --- a/transformations/aws/macros/log_metric_filter_and_alarm.sql +++ b/transformations/aws/macros/log_metric_filter_and_alarm.sql @@ -76,7 +76,7 @@ where t.is_multi_region_trail = TRUE and ss.arn like 'aws:arn:%' {% endmacro %} -{% macro snowflake__log_metric_filter_and_alarm(framework, check_id) %} +{% macro snowflake__log_metric_filter_and_alarm() %} with af as ( select distinct a.arn, a.actions_enabled, a.alarm_actions, m.value:MetricStat:Metric:MetricName as metric_name -- TODO check from aws_cloudwatch_alarms a, From 58b6d968cd8c83468604c9c2c446a4ae69558a7b Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Fri, 29 Dec 2023 11:40:00 +0200 Subject: [PATCH 4/6] Updated query 2 log_metric_filter_and_alarm - snowflake --- .../aws/macros/log_metric_filter_and_alarm.sql | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/transformations/aws/macros/log_metric_filter_and_alarm.sql b/transformations/aws/macros/log_metric_filter_and_alarm.sql index bd4b205bb..828b1788c 100644 --- a/transformations/aws/macros/log_metric_filter_and_alarm.sql +++ b/transformations/aws/macros/log_metric_filter_and_alarm.sql @@ -40,10 +40,6 @@ with af as ( from {{ full_table_name("aws_cloudwatch_alarms") }} a, UNNEST(JSON_QUERY_ARRAY(metrics)) as m ), -aes as ( -select * from {{ full_table_name("aws_cloudtrail_trail_event_selectors") }} , - UNNEST(JSON_QUERY_ARRAY(advanced_event_selectors)) as aes -), tes as ( select trail_arn from {{ full_table_name("aws_cloudtrail_trail_event_selectors") }} where exists( @@ -53,10 +49,9 @@ tes as ( where JSON_VALUE(es.ReadWriteType) = 'All' and CAST( JSON_VALUE(es.IncludeManagementEvents) AS BOOL) = TRUE ) or exists( - select * from aes + select * from UNNEST(JSON_QUERY_ARRAY(advanced_event_selectors)) as aes where not exists ( - select * from aes, - UNNEST(JSON_QUERY_ARRAY(aes.aes.FieldSelectors)) as aes_fs + select * from UNNEST(JSON_QUERY_ARRAY(aes.FieldSelectors)) as aes_fs where JSON_VALUE(aes_fs.Field) = 'readOnly' ) ) From 68364728175cdbe99da86cec9a20a1e501bb8a86 Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Fri, 29 Dec 2023 11:49:48 +0200 Subject: [PATCH 5/6] Update 3 query Co-authored-by: Alex Shcherbakov --- transformations/aws/macros/log_metric_filter_and_alarm.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformations/aws/macros/log_metric_filter_and_alarm.sql b/transformations/aws/macros/log_metric_filter_and_alarm.sql index 828b1788c..12185d23a 100644 --- a/transformations/aws/macros/log_metric_filter_and_alarm.sql +++ b/transformations/aws/macros/log_metric_filter_and_alarm.sql @@ -36,7 +36,7 @@ where t.is_multi_region_trail = TRUE {% macro bigquery__log_metric_filter_and_alarm() %} with af as ( - select distinct a.arn, a.actions_enabled, ARRAY_TO_STRING(a.alarm_actions, ',') as alarm_actions, JSON_VALUE(m.MetricStat.Metric.MetricName) as metric_name + select distinct a.arn, a.actions_enabled, ARRAY_TO_STRING(a.alarm_actions, ',') as alarm_actions, JSON_VALUE(m.MetricStat.Metric.MetricName) as metric_name from {{ full_table_name("aws_cloudwatch_alarms") }} a, UNNEST(JSON_QUERY_ARRAY(metrics)) as m ), From 41822db3b4c1e38db84730801df22cdc4abb13c0 Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Fri, 29 Dec 2023 11:50:03 +0200 Subject: [PATCH 6/6] Update 4 query Co-authored-by: Alex Shcherbakov --- transformations/aws/macros/log_metric_filter_and_alarm.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/transformations/aws/macros/log_metric_filter_and_alarm.sql b/transformations/aws/macros/log_metric_filter_and_alarm.sql index 12185d23a..c79c15bc1 100644 --- a/transformations/aws/macros/log_metric_filter_and_alarm.sql +++ b/transformations/aws/macros/log_metric_filter_and_alarm.sql @@ -43,9 +43,7 @@ with af as ( tes as ( select trail_arn from {{ full_table_name("aws_cloudtrail_trail_event_selectors") }} where exists( - select * from - {{ full_table_name("aws_cloudtrail_trail_event_selectors") }}, - UNNEST(JSON_QUERY_ARRAY(event_selectors)) as es + select * from UNNEST(JSON_QUERY_ARRAY(event_selectors)) as es where JSON_VALUE(es.ReadWriteType) = 'All' and CAST( JSON_VALUE(es.IncludeManagementEvents) AS BOOL) = TRUE ) or exists(