From 3186adff44c8ab4adf7f1800ce61bc8428097158 Mon Sep 17 00:00:00 2001 From: ronsh12 <101520407+ronsh12@users.noreply.github.com> Date: Wed, 17 Jan 2024 11:02:05 +0200 Subject: [PATCH] fix: Updated queries for joining with the column _cq_parent_id (#472) * fix: Updated queries for joining with the Column _cq_parent_id * minor fix to queries * Add bigquery changes --- transformations/aws/macros/iam/no_star.sql | 6 +- .../iam/policies_have_wildcard_actions.sql | 4 +- .../macros/iam/policies_with_admin_rights.sql | 4 +- .../iam_customer_policy_no_kms_decrypt.sql | 4 +- .../aws/macros/s3/deny_http_requests.sql | 20 +++--- .../macros/s3/publicly_readable_buckets.sql | 71 ++++++++++--------- 6 files changed, 55 insertions(+), 54 deletions(-) diff --git a/transformations/aws/macros/iam/no_star.sql b/transformations/aws/macros/iam/no_star.sql index 1817968e8..0dc7b3481 100644 --- a/transformations/aws/macros/iam/no_star.sql +++ b/transformations/aws/macros/iam/no_star.sql @@ -11,7 +11,7 @@ with pvs as ( p.id, pv.document_json as document from aws_iam_policies p - inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn + inner join aws_iam_policy_versions pv on pv._cq_parent_id = p._cq_id ), violations as ( select id, @@ -57,7 +57,7 @@ with pvs as ( p.id, pv.document_json as document from {{ full_table_name("aws_iam_policies") }} p - inner join {{ full_table_name("aws_iam_policy_versions") }} pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn + inner join {{ full_table_name("aws_iam_policy_versions") }} pv on pv._cq_parent_id = p._cq_id ), violations as ( select id, @@ -91,7 +91,7 @@ with pvs as ( p.id, pv.document_json as document from aws_iam_policies p - inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn + inner join aws_iam_policy_versions pv on pv._cq_parent_id = p._cq_id ), violations as ( select id, diff --git a/transformations/aws/macros/iam/policies_have_wildcard_actions.sql b/transformations/aws/macros/iam/policies_have_wildcard_actions.sql index d488ea7b4..4a586ffe6 100644 --- a/transformations/aws/macros/iam/policies_have_wildcard_actions.sql +++ b/transformations/aws/macros/iam/policies_have_wildcard_actions.sql @@ -17,7 +17,7 @@ SELECT FROM aws_iam_policies p -INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn +INNER JOIN aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id , JSONB_ARRAY_ELEMENTS(pv.document_json -> 'Statement') as s where pv.is_default_version = true AND s ->> 'Effect' = 'Allow' @@ -49,7 +49,7 @@ SELECT FROM aws_iam_policies p - INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn + INNER JOIN aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id , lateral flatten(input => pv.document_json:Statement) as s where pv.is_default_version = true AND s.value:Effect = 'Allow' ) diff --git a/transformations/aws/macros/iam/policies_with_admin_rights.sql b/transformations/aws/macros/iam/policies_with_admin_rights.sql index fad0ee0fb..1944976fd 100644 --- a/transformations/aws/macros/iam/policies_with_admin_rights.sql +++ b/transformations/aws/macros/iam/policies_with_admin_rights.sql @@ -8,7 +8,7 @@ SELECT p.id FROM aws_iam_policies p - INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn + INNER JOIN aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id , lateral flatten(input => pv.document_json:Statement) as s where pv.is_default_version = 'true' AND s.value:Effect = 'Allow' and s.value:Effect = 'Allow' @@ -38,7 +38,7 @@ with iam_policies as ( p.id as id, pv.document_json as document from aws_iam_policies p - inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn + inner join aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id where pv.is_default_version = true and p.arn not like 'arn:aws:iam::aws:policy%' ), policy_statements as ( diff --git a/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql b/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql index 859dd1f8a..c6d7a2957 100644 --- a/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql +++ b/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql @@ -8,7 +8,7 @@ WITH policy_with_decrypt AS ( SELECT DISTINCT arn FROM aws_iam_policies p - INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn + INNER JOIN aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id , JSONB_ARRAY_ELEMENTS(pv.document_json -> 'Statement') as s WHERE s ->> 'Effect' = 'Allow' @@ -41,7 +41,7 @@ LEFT JOIN policy_with_decrypt d ON i.arn = d.arn WITH policy_with_decrypt AS ( SELECT DISTINCT arn FROM aws_iam_policies p - INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn + INNER JOIN aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id , lateral flatten(input => pv.document_json:Statement) as s WHERE s.value:Effect = 'Allow' diff --git a/transformations/aws/macros/s3/deny_http_requests.sql b/transformations/aws/macros/s3/deny_http_requests.sql index e4008c0f4..8c45cf51d 100644 --- a/transformations/aws/macros/s3/deny_http_requests.sql +++ b/transformations/aws/macros/s3/deny_http_requests.sql @@ -21,8 +21,8 @@ WHERE statements.value AS statement FROM aws_s3_buckets AS b - inner join aws_s3_bucket_policies on b.arn = aws_s3_bucket_policies.bucket_arn, - LATERAL FLATTEN(INPUT => IFF(TYPEOF(aws_s3_bucket_policies.policy_json:Statement) = 'STRING', TO_ARRAY(aws_s3_bucket_policies.policy_json:Statement), aws_s3_bucket_policies.policy_json:Statement)) AS statements + inner join aws_s3_bucket_policies bp ON bp._cq_parent_id = b._cq_id, + LATERAL FLATTEN(INPUT => IFF(TYPEOF(bp.policy_json:Statement) = 'STRING', TO_ARRAY(bp.policy_json:Statement), bp.policy_json:Statement)) AS statements WHERE GET_PATH(statement, 'Effect')::STRING = 'Deny' AND GET_PATH(statement, 'Condition.Bool.aws:SecureTransport')::STRING = 'false' @@ -50,15 +50,15 @@ where from (select aws_s3_buckets.arn, statements, statements -> 'Principal' as principals - from aws_s3_buckets - inner join aws_s3_bucket_policies on aws_s3_buckets.arn = aws_s3_bucket_policies.bucket_arn, + from aws_s3_buckets b + inner join aws_s3_bucket_policies bp ON bp._cq_parent_id = b._cq_id, jsonb_array_elements( - case jsonb_typeof(aws_s3_bucket_policies.policy_json -> 'Statement') + case jsonb_typeof(bp.policy_json -> 'Statement') when 'string' then jsonb_build_array( - aws_s3_bucket_policies.policy_json ->> 'Statement' + bp.policy_json ->> 'Statement' ) - when 'array' then aws_s3_bucket_policies.policy_json -> 'Statement' + when 'array' then bp.policy_json -> 'Statement' end ) as statements where statements -> 'Effect' = '"Deny"') as foo, @@ -98,9 +98,9 @@ WHERE statements AS statement FROM {{ full_table_name("aws_s3_buckets") }} AS b - inner join {{ full_table_name("aws_s3_bucket_policies") }} - on b.arn = aws_s3_bucket_policies.bucket_arn, - UNNEST(JSON_QUERY_ARRAY(aws_s3_bucket_policies.policy_json.Statement)) AS statements + inner join {{ full_table_name("aws_s3_bucket_policies") }} bp + on bp._cq_parent_id = b._cq_id, + UNNEST(JSON_QUERY_ARRAY(bp.policy_json.Statement)) AS statements WHERE CAST(JSON_VALUE(statements.Effect) AS STRING) = 'Deny' AND CAST(JSON_VALUE(JSON_EXTRACT(statements, '$.Condition.Bool."aws:SecureTransport"')) AS STRING) = 'false' diff --git a/transformations/aws/macros/s3/publicly_readable_buckets.sql b/transformations/aws/macros/s3/publicly_readable_buckets.sql index 5dbc3ae83..53bdf4068 100644 --- a/transformations/aws/macros/s3/publicly_readable_buckets.sql +++ b/transformations/aws/macros/s3/publicly_readable_buckets.sql @@ -10,11 +10,11 @@ WITH policy_allow_public AS ( FROM ( SELECT - aws_s3_buckets.arn, + b.arn, statements.value:Principal AS principals FROM - aws_s3_buckets - inner join aws_s3_bucket_policies bp on aws_s3_buckets.arn = bp.bucket_arn, + aws_s3_buckets b + inner join aws_s3_bucket_policies bp ON bp._cq_parent_id = b._cq_id, LATERAL FLATTEN(INPUT => IFF(TYPEOF(bp.policy_json:Statement) = 'STRING', TO_ARRAY(bp.policy_json:Statement), bp.policy_json:Statement)) AS statements @@ -38,28 +38,28 @@ SELECT '{{framework}}' As framework, '{{check_id}}' As check_id, 'S3 buckets should prohibit public read access' AS title, - aws_s3_buckets.account_id, - aws_s3_buckets.arn AS resource_id, + b.account_id, + b.arn AS resource_id, 'fail' AS status -- TODO FIXME FROM - aws_s3_buckets + aws_s3_buckets b LEFT JOIN - aws_s3_bucket_grants ON - aws_s3_buckets.arn = aws_s3_bucket_grants.bucket_arn + aws_s3_bucket_grants bg + ON bg._cq_parent_id = b._cq_id LEFT JOIN policy_allow_public ON - aws_s3_buckets.arn = policy_allow_public.arn -LEFT JOIN aws_s3_bucket_public_access_blocks ON - aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn + b.arn = policy_allow_public.arn +LEFT JOIN aws_s3_bucket_public_access_blocks bpab + ON bpab._cq_parent_id = b._cq_id WHERE ( - (aws_s3_bucket_public_access_blocks.public_access_block_configuration:BlockPublicAcls)::boolean != TRUE + (bpab.public_access_block_configuration:BlockPublicAcls)::boolean != TRUE AND ( - aws_s3_bucket_grants.grantee:URI::STRING = 'http://acs.amazonaws.com/groups/global/AllUsers' - AND aws_s3_bucket_grants.permission IN ('READ_ACP', 'FULL_CONTROL') + bg.grantee:URI::STRING = 'http://acs.amazonaws.com/groups/global/AllUsers' + AND bg.permission IN ('READ_ACP', 'FULL_CONTROL') ) ) OR ( - (aws_s3_bucket_public_access_blocks.public_access_block_configuration:BlockPublicPolicy)::boolean != TRUE + (bpab.public_access_block_configuration:BlockPublicPolicy)::boolean != TRUE AND policy_allow_public.statement_count > 0 ) {% endmacro %} @@ -97,39 +97,40 @@ select '{{framework}}' as framework, '{{check_id}}' as check_id, 'S3 buckets should prohibit public read access' as title, - aws_s3_buckets.account_id, - aws_s3_buckets.arn as resource_id, + b.account_id, + b.arn as resource_id, 'fail' as status -- TODO FIXME from -- Find and join all bucket ACLS that givea public write access - aws_s3_buckets + aws_s3_buckets b left join - aws_s3_bucket_grants on - aws_s3_buckets.arn = aws_s3_bucket_grants.bucket_arn + aws_s3_bucket_grants bg + ON bg._cq_parent_id = b._cq_id -- Find all statements that could give public allow access -- Statements that give public access have 1) Effect == Allow 2) One of the following principal: -- Principal = {"AWS": "*"} -- Principal = {"AWS": ["arn:aws:iam::12345678910:root", "*"]} -- Principal = "*" left join policy_allow_public on - aws_s3_buckets.arn = policy_allow_public.arn -left join aws_s3_bucket_public_access_blocks on - aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn + b.arn = policy_allow_public.arn +left join aws_s3_bucket_public_access_blocks bpab + ON bpab._cq_parent_id = b._cq_id where ( - (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicAcls')::boolean != TRUE + (bpab.public_access_block_configuration -> 'BlockPublicAcls')::boolean != TRUE and ( grantee->>'URI' = 'http://acs.amazonaws.com/groups/global/AllUsers' and permission in ('READ_ACP', 'FULL_CONTROL') ) ) or ( - (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicPolicy')::boolean != TRUE + (bpab.public_access_block_configuration -> 'BlockPublicPolicy')::boolean != TRUE and policy_allow_public.statement_count > 0 ) {% endmacro %} {% macro default__publicly_readable_buckets(framework, check_id) %}{% endmacro %} + {% macro bigquery__publicly_readable_buckets(framework, check_id) %} with policy_allow_public as ( select @@ -162,34 +163,34 @@ select '{{framework}}' as framework, '{{check_id}}' as check_id, 'S3 buckets should prohibit public read access' as title, - aws_s3_buckets.account_id, - aws_s3_buckets.arn as resource_id, + b.account_id, + b.arn as resource_id, 'fail' as status -- TODO FIXME from -- Find and join all bucket ACLS that give a public write access - {{ full_table_name("aws_s3_buckets") }} + {{ full_table_name("aws_s3_buckets") }} b left join - {{ full_table_name("aws_s3_bucket_grants") }} on - aws_s3_buckets.arn = aws_s3_bucket_grants.bucket_arn + {{ full_table_name("aws_s3_bucket_grants") }} bg on + bg._cq_parent_id = b._cq_id -- Find all statements that could give public allow access -- Statements that give public access have 1) Effect == Allow 2) One of the following principal: -- Principal = {"AWS": "*"} -- Principal = {"AWS": ["arn:aws:iam::12345678910:root", "*"]} -- Principal = "*" left join policy_allow_public on - aws_s3_buckets.arn = policy_allow_public.arn -left join {{ full_table_name("aws_s3_bucket_public_access_blocks") }} on - aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn + b.arn = policy_allow_public.arn +left join {{ full_table_name("aws_s3_bucket_public_access_blocks") }} bpab + ON bpab._cq_parent_id = b._cq_id where ( - CAST( JSON_VALUE(aws_s3_bucket_public_access_blocks.public_access_block_configuration.BlockPublicAcls) AS BOOL) != TRUE + CAST( JSON_VALUE(bpab.public_access_block_configuration.BlockPublicAcls) AS BOOL) != TRUE and ( JSON_VALUE(grantee.URI) = 'http://acs.amazonaws.com/groups/global/AllUsers' and permission in ('READ_ACP', 'FULL_CONTROL') ) ) or ( - CAST( JSON_VALUE(aws_s3_bucket_public_access_blocks.public_access_block_configuration.BlockPublicPolicy) AS BOOL) != TRUE + CAST( JSON_VALUE(bpab.public_access_block_configuration.BlockPublicPolicy) AS BOOL) != TRUE and policy_allow_public.statement_count > 0 ) {% endmacro %} \ No newline at end of file