feat!: Update aws to v23.3.1

.gitignore

@@ -157,7 +157,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
 
 .DS_Store
 transformations/aws_compliance/.user.yml

transformations/aws/asset-inventory-free/manifest.json

@@ -8,6 +8,6 @@
   "message": "@./changelog.md",
   "doc": "./README.md",
   "path": "./build/aws_asset_inventory_free.zip",
-  "plugin_deps": ["cloudquery/source/aws@v22.19.0"],
+  "plugin_deps": ["cloudquery/source/aws@v23.3.1"],
   "addon_deps": []
 }

transformations/aws/asset-inventory-free/tests/postgres.yml

@@ -2,7 +2,8 @@ kind: source
 spec:
   name: aws
   path: cloudquery/aws
-  version: "v22.19.2" # latest version of source aws plugin
+  registry: cloudquery
+  version: "v23.3.1" # latest version of source aws plugin
   destinations: ["postgresql"]
   tables: ["*"]
 ---
@@ -14,4 +15,4 @@ spec:
   version: "v7.1.3" # latest version of postgresql plugin
   spec:
     batch_size: 10000
-    connection_string: ${CQ_DSN}
+    connection_string: ${CQ_DSN}

transformations/aws/compliance-free/manifest.json

@@ -8,6 +8,6 @@
   "message": "@./changelog.md",
   "doc": "./README.md",
   "path": "./build/aws_compliance_free.zip",
-  "plugin_deps": ["cloudquery/source/aws@v22.18.0"],
+  "plugin_deps": ["cloudquery/source/aws@v23.3.1"],
   "addon_deps": []
 }

transformations/aws/compliance-free/tests/bigquery.yml

@@ -2,7 +2,8 @@ kind: source
 spec:
   name: aws
   path: cloudquery/aws
-  version: "v22.19.2" # latest version of source aws plugin
+  registry: cloudquery
+  version: "v23.3.1" # latest version of source aws plugin
   destinations: ["bigquery"]
   tables: ["*"]
 ---
@@ -15,4 +16,4 @@ spec:
   write_mode: "append"
   spec:
     project_id: cq-integration-tests
-    dataset_id: policies_premium_ci_test
+    dataset_id: policies_premium_ci_test

transformations/aws/compliance-free/tests/postgres.yml

@@ -2,7 +2,8 @@ kind: source
 spec:
   name: aws
   path: cloudquery/aws
-  version: "v22.19.2" # latest version of source aws plugin
+  registry: cloudquery
+  version: "v23.3.1" # latest version of source aws plugin
   destinations: ["postgresql"]
   tables: ["*"]
 ---
@@ -14,4 +15,4 @@ spec:
   version: "v7.1.3" # latest version of postgresql plugin
   spec:
     batch_size: 10000
-    connection_string: ${CQ_DSN}
+    connection_string: ${CQ_DSN}

transformations/aws/compliance-free/tests/snowflake.yml

@@ -2,7 +2,8 @@ kind: source
 spec:
   name: aws
   path: cloudquery/aws
-  version: "v22.19.2" # latest version of source aws plugin
+  registry: cloudquery
+  version: "v23.3.1" # latest version of source aws plugin
   destinations: ["snowflake"]
   tables: ["*"]
 ---

transformations/aws/compliance-premium/manifest.json

@@ -8,6 +8,6 @@
   "message": "@./changelog.md",
   "doc": "./README.md",
   "path": "./build/aws_compliance_premium.zip",
-  "plugin_deps": ["cloudquery/source/aws@v22.18.0"],
+  "plugin_deps": ["cloudquery/source/aws@v23.3.1"],
   "addon_deps": []
 }

transformations/aws/compliance-premium/tests/bigquery.yml

@@ -2,7 +2,8 @@ kind: source
 spec:
   name: aws
   path: cloudquery/aws
-  version: "v22.19.2" # latest version of source aws plugin
+  registry: cloudquery
+  version: "v23.3.1" # latest version of source aws plugin
   destinations: ["bigquery"]
   tables: ["*"]
 ---
@@ -15,4 +16,4 @@ spec:
   write_mode: "append"
   spec:
     project_id: cq-integration-tests
-    dataset_id: policies_premium_ci_test
+    dataset_id: policies_premium_ci_test

transformations/aws/compliance-premium/tests/postgres.yml

@@ -2,7 +2,8 @@ kind: source
 spec:
   name: aws
   path: cloudquery/aws
-  version: "v22.19.2" # latest version of source aws plugin
+  registry: cloudquery
+  version: "v23.3.1" # latest version of source aws plugin
   destinations: ["postgresql"]
   tables: ["*"]
 ---
@@ -14,4 +15,4 @@ spec:
   version: "v7.1.3" # latest version of postgresql plugin
   spec:
     batch_size: 10000
-    connection_string: ${CQ_DSN}
+    connection_string: ${CQ_DSN}

transformations/aws/compliance-premium/tests/snowflake.yml

@@ -2,7 +2,8 @@ kind: source
 spec:
   name: aws
   path: cloudquery/aws
-  version: "v22.19.2" # latest version of source aws plugin
+  registry: cloudquery
+  version: "v23.3.1" # latest version of source aws plugin
   destinations: ["snowflake"]
   tables: ["*"]
 ---

transformations/aws/cost/tests/postgres.yml

@@ -2,7 +2,8 @@ kind: source
 spec:
   name: aws
   path: cloudquery/aws
-  version: "v22.19.2" # latest version of source aws plugin
+  registry: cloudquery
+  version: "v23.3.1" # latest version of source aws plugin
   destinations: ["postgresql"]
   tables: ["*"]
 ---
@@ -14,4 +15,4 @@ spec:
   version: "v7.1.3" # latest version of postgresql plugin
   spec:
     batch_size: 10000
-    connection_string: ${CQ_DSN}
+    connection_string: ${CQ_DSN}

transformations/aws/data-resilience/manifest.json

@@ -8,6 +8,6 @@
   "message": "@./changelog.md",
   "doc": "./README.md",
   "path": "./build/aws_data_resilience.zip",
-  "plugin_deps": ["cloudquery/source/aws@v22.19.0"],
+  "plugin_deps": ["cloudquery/source/aws@v23.3.1"],
   "addon_deps": []
 }

transformations/aws/data-resilience/tests/postgres.yml

@@ -2,7 +2,8 @@ kind: source
 spec:
   name: aws
   path: cloudquery/aws
-  version: "v22.19.2" # latest version of source aws plugin
+  registry: cloudquery
+  version: "v23.3.1" # latest version of source aws plugin
   destinations: ["postgresql"]
   tables: ["*"]
 ---
@@ -14,4 +15,4 @@ spec:
   version: "v7.1.3" # latest version of postgresql plugin
   spec:
     batch_size: 10000
-    connection_string: ${CQ_DSN}
+    connection_string: ${CQ_DSN}

transformations/aws/encryption/manifest.json

@@ -8,6 +8,6 @@
   "message": "@./changelog.md",
   "doc": "./README.md",
   "path": "./build/aws_encryption.zip",
-  "plugin_deps": ["cloudquery/source/aws@v22.19.0"],
+  "plugin_deps": ["cloudquery/source/aws@v23.3.1"],
   "addon_deps": []
 }

transformations/aws/encryption/tests/postgres.yml

@@ -2,7 +2,8 @@ kind: source
 spec:
   name: aws
   path: cloudquery/aws
-  version: "v22.19.2" # latest version of source aws plugin
+  registry: cloudquery
+  version: "v23.3.1" # latest version of source aws plugin
   destinations: ["postgresql"]
   tables: ["*"]
 ---
@@ -14,4 +15,4 @@ spec:
   version: "v7.1.3" # latest version of postgresql plugin
   spec:
     batch_size: 10000
-    connection_string: ${CQ_DSN}
+    connection_string: ${CQ_DSN}

transformations/aws/macros/cloudtrail/bucket_access_logging.sql

@@ -12,11 +12,14 @@ select
     t.account_id,
     t.arn as resource_id,
     case
-        when b.logging_target_bucket is null or b.logging_target_prefix is null then 'fail'
+        when l.logging_enabled is null then 'fail'
+        when l.logging_enabled -> 'TargetBucket' is null then 'fail'
+        when l.logging_enabled -> 'TargetPrefix' is null then 'fail'
         else 'pass'
     end as status
 from aws_cloudtrail_trails t
 inner join aws_s3_buckets b on t.s3_bucket_name = b.name
+inner join aws_s3_bucket_loggings l on b.arn = l.bucket_arn
 {% endmacro %}
 
 {% macro bigquery__bucket_access_logging(framework, check_id) %}
@@ -27,11 +30,14 @@ select
     t.account_id,
     t.arn as resource_id,
     case
-        when b.logging_target_bucket is null or b.logging_target_prefix is null then 'fail'
+        when l.logging_enabled is null then 'fail'
+        when l.logging_enabled.TargetBucket is null then 'fail'
+        when l.logging_enabled.TargetPrefix is null then 'fail'
         else 'pass'
     end as status
 from {{ full_table_name("aws_cloudtrail_trails") }} t
 inner join {{ full_table_name("aws_s3_buckets") }} b on t.s3_bucket_name = b.name
+inner join {{ full_table_name("aws_s3_bucket_loggings") }} l on b.arn = l.bucket_arn
 {% endmacro %}
 
 {% macro snowflake__bucket_access_logging(framework, check_id) %}
@@ -42,9 +48,12 @@ select
     t.account_id,
     t.arn as resource_id,
     case
-        when b.logging_target_bucket is null or b.logging_target_prefix is null then 'fail'
+        when l.logging_enabled is null then 'fail'
+        when l.logging_enabled:TargetBucket is null then 'fail'
+        when l.logging_enabled:TargetPrefix is null then 'fail'
         else 'pass'
     end as status
 from aws_cloudtrail_trails t
 inner join aws_s3_buckets b on t.s3_bucket_name = b.name
-{% endmacro %}
+inner join aws_s3_bucket_loggings l on b.arn = l.bucket_arn
+{% endmacro %}

transformations/aws/macros/cloudtrail/enabled_in_all_regions.sql

@@ -10,10 +10,17 @@ select
     aws_cloudtrail_trails.account_id,
     arn as resource_id,
     case
-        when is_multi_region_trail = FALSE or (
-                    is_multi_region_trail = TRUE and (
-                        read_write_type != 'All' or include_management_events = FALSE
-                )) then 'fail'
+        when aws_cloudtrail_trails.is_multi_region_trail = FALSE then 'fail'
+        when exists(select *
+                    from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.event_selectors) as es
+                    where es ->>'ReadWriteType' != 'All' or (es->>'IncludeManagementEvents')::boolean = FALSE)
+            then 'fail'
+        when exists(select *
+                    from jsonb_array_elements(aws_cloudtrail_trail_event_selectors.advanced_event_selectors) as aes
+                    where exists(select *
+                                 from jsonb_array_elements(aes ->'FieldSelectors') as aes_fs
+                                 where aes_fs ->>'Field' = 'readOnly'))
+            then 'fail'
         else 'pass'
     end as status
 from aws_cloudtrail_trails
@@ -32,10 +39,18 @@ select
     aws_cloudtrail_trails.account_id,
     arn as resource_id,
     case
-        when is_multi_region_trail = FALSE or (
-                    is_multi_region_trail = TRUE and (
-                        read_write_type != 'All' or include_management_events = FALSE
-                )) then 'fail'
+        when aws_cloudtrail_trails.is_multi_region_trail = FALSE then 'fail'
+        when exists(select *
+                    from UNNEST(JSON_QUERY_ARRAY(aws_cloudtrail_trail_event_selectors.event_selectors)) AS es
+                    where JSON_VALUE(es.ReadWriteType) != 'All' or (CAST( JSON_VALUE(es.IncludeManagementEvents) AS BOOL)= FALSE )
+        )
+            then 'fail'
+        when exists(select *
+                    from UNNEST(JSON_QUERY_ARRAY(aws_cloudtrail_trail_event_selectors.advanced_event_selectors)) AS aes
+                    where exists(select *
+                                 from UNNEST(JSON_QUERY_ARRAY(aes.FieldSelectors)) as aes_fs
+                                 where JSON_VALUE(aes_fs.Field) = 'readOnly'))
+            then 'fail'
         else 'pass'
     end as status
 from {{ full_table_name("aws_cloudtrail_trails") }}
@@ -47,17 +62,33 @@ inner join
 {% endmacro %}
 
 {% macro snowflake__cloudtrail_enabled_all_regions(framework, check_id) %}
+with aes as
+(
+  select *
+  from aws_cloudtrail_trail_event_selectors,
+  LATERAL FLATTEN (advanced_event_selectors) as aes
+)
 select
     '{{framework}}' as framework,
     '{{check_id}}' as check_id,
     'Ensure CloudTrail is enabled in all regions' as title,
     aws_cloudtrail_trails.account_id,
     arn as resource_id,
     case
-        when is_multi_region_trail = FALSE or (
-                    is_multi_region_trail = TRUE and (
-                        read_write_type != 'All' or include_management_events = FALSE
-                )) then 'fail'
+        when aws_cloudtrail_trails.is_multi_region_trail = FALSE then 'fail'
+        when exists(select *
+                    from aws_cloudtrail_trail_event_selectors,
+                    LATERAL FLATTEN(event_selectors) as es
+                    where es.value:ReadWriteType != 'All' or (es.value:IncludeManagementEvents)::boolean = FALSE
+                    )
+            then 'fail'
+        when exists(
+                    select *
+                   from aes,
+                  LATERAL FLATTEN (value:FieldSelectors) as aes_fs
+                    where aes_fs.value:Field = 'readOnly'
+                   )
+            then 'fail'
         else 'pass'
     end as status
 from aws_cloudtrail_trails

transformations/aws/macros/elasticbeanstalk/elastic_beanstalk_stream_logs_to_cloudwatch.sql

@@ -1,7 +1,7 @@
 {% macro elastic_beanstalk_stream_logs_to_cloudwatch(framework, check_id) %}
 with flat_configs as (
     select 
-        c.environment_id,
+        c.environment_arn,
         f.value:Namespace:Value::string as is_log_streaming
 
     from 
@@ -26,5 +26,5 @@ SELECT
     END as status
 FROM aws_elasticbeanstalk_environments e
 JOIN flat_configs as fc
-    ON e.environment_id = fc.environment_id
+    ON e.arn = fc.environment_arn
     {% endmacro %}

transformations/aws/macros/guardduty/detector_enabled.sql

@@ -4,7 +4,7 @@
 
 {% macro snowflake__detector_enabled(framework, check_id) %}
 with enabled_detector_regions as (
-    select account_id, region
+    select request_account_id as account_id, request_region as region
     from aws_guardduty_detectors
     where status = 'ENABLED'
 )
@@ -26,8 +26,8 @@ select
     '{{framework}}' As framework,
     '{{check_id}}' As check_id,
     'GuardDuty should be enabled (detectors)' AS title,
-    account_id,
-    region AS resource_id,
+    request_account_id as account_id,
+    request_region AS resource_id,
     case when
         data_sources:S3Logs:Status != 'ENABLED' AND
         data_sources:DNSLogs:Status != 'ENABLED' AND
@@ -41,7 +41,7 @@ where
 
 {% macro postgres__detector_enabled(framework, check_id) %}
 with enabled_detector_regions as (
-    select account_id, region
+    select request_account_id as account_id, request_region as region
     from aws_guardduty_detectors
     where status = 'ENABLED'
 )
@@ -63,8 +63,8 @@ select
     '{{framework}}' as framework,
     '{{check_id}}' as check_id,
     'GuardDuty should be enabled (detectors)' AS title,
-    account_id,
-    region AS resource_id,
+    request_account_id as account_id,
+    request_region AS resource_id,
     case when
         data_sources->'S3Logs'->>'Status' != 'ENABLED' AND
         data_sources->'DNSLogs'->>'Status' != 'ENABLED' AND
@@ -77,4 +77,4 @@ where
 {% endmacro %}
 
 {% macro default__detector_enabled(framework, check_id) %}{% endmacro %}
-
+