fix: Updated queries for joining with the column _cq_parent_id (#472)

* fix: Updated queries for joining with the Column _cq_parent_id * minor fix to queries * Add bigquery changes
cloudquery · Jan 17, 2024 · 3186adf · 3186adf
1 parent b97769e
commit 3186adf
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 54 deletions.
diff --git a/transformations/aws/macros/iam/no_star.sql b/transformations/aws/macros/iam/no_star.sql
@@ -11,7 +11,7 @@ with pvs as (
         p.id,
         pv.document_json as document
     from aws_iam_policies p
-    inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn
+    inner join aws_iam_policy_versions pv on pv._cq_parent_id = p._cq_id
 ), violations as (
     select
         id,
@@ -57,7 +57,7 @@ with pvs as (
         p.id,
         pv.document_json as document
     from {{ full_table_name("aws_iam_policies") }} p
-    inner join {{ full_table_name("aws_iam_policy_versions") }} pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn
+    inner join {{ full_table_name("aws_iam_policy_versions") }} pv on pv._cq_parent_id = p._cq_id
 ), violations as (
     select
         id,
@@ -91,7 +91,7 @@ with pvs as (
         p.id,
         pv.document_json as document
     from aws_iam_policies p
-    inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn
+    inner join aws_iam_policy_versions pv on pv._cq_parent_id = p._cq_id
 ), violations as (
     select
         id,

diff --git a/transformations/aws/macros/iam/policies_have_wildcard_actions.sql b/transformations/aws/macros/iam/policies_have_wildcard_actions.sql
@@ -17,7 +17,7 @@ SELECT
 
 FROM
     aws_iam_policies p
-INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn
+INNER JOIN aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id
 			, JSONB_ARRAY_ELEMENTS(pv.document_json -> 'Statement') as s
 where pv.is_default_version = true AND s ->> 'Effect' = 'Allow'
 
@@ -49,7 +49,7 @@ SELECT
 
 FROM
     aws_iam_policies p
-    INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn
+    INNER JOIN aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id
     , lateral flatten(input => pv.document_json:Statement) as s
 where pv.is_default_version = true AND s.value:Effect = 'Allow'
 )

diff --git a/transformations/aws/macros/iam/policies_with_admin_rights.sql b/transformations/aws/macros/iam/policies_with_admin_rights.sql
@@ -8,7 +8,7 @@ SELECT
     p.id
 FROM
     aws_iam_policies p
-    INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn
+    INNER JOIN aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id
     , lateral flatten(input => pv.document_json:Statement) as s
 where pv.is_default_version = 'true' AND s.value:Effect = 'Allow'
     and s.value:Effect = 'Allow'
@@ -38,7 +38,7 @@ with iam_policies as (
         p.id as id,
         pv.document_json as document
     from aws_iam_policies p
-    inner join aws_iam_policy_versions pv on p.account_id = pv.account_id AND p.arn = pv.policy_arn
+    inner join aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id
     where pv.is_default_version = true and p.arn not like 'arn:aws:iam::aws:policy%'
 ),
 policy_statements as (

diff --git a/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql b/transformations/aws/macros/kms/iam_customer_policy_no_kms_decrypt.sql
@@ -8,7 +8,7 @@
 WITH policy_with_decrypt AS (
     SELECT DISTINCT arn
     FROM aws_iam_policies p
-    INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn
+    INNER JOIN aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id
     , JSONB_ARRAY_ELEMENTS(pv.document_json -> 'Statement') as s
     WHERE
         s ->> 'Effect' = 'Allow'
@@ -41,7 +41,7 @@ LEFT JOIN policy_with_decrypt d ON i.arn = d.arn
 WITH policy_with_decrypt AS (
     SELECT DISTINCT arn
     FROM aws_iam_policies p
-    INNER JOIN aws_iam_policy_versions pv ON p.account_id = pv.account_id AND p.arn = pv.policy_arn
+    INNER JOIN aws_iam_policy_versions pv ON pv._cq_parent_id = p._cq_id
     , lateral flatten(input => pv.document_json:Statement) as s
     WHERE
         s.value:Effect = 'Allow'

diff --git a/transformations/aws/macros/s3/deny_http_requests.sql b/transformations/aws/macros/s3/deny_http_requests.sql
@@ -21,8 +21,8 @@ WHERE
                 statements.value AS statement
             FROM
                 aws_s3_buckets AS b
-            inner join aws_s3_bucket_policies on b.arn = aws_s3_bucket_policies.bucket_arn,
-                LATERAL FLATTEN(INPUT => IFF(TYPEOF(aws_s3_bucket_policies.policy_json:Statement) = 'STRING', TO_ARRAY(aws_s3_bucket_policies.policy_json:Statement), aws_s3_bucket_policies.policy_json:Statement)) AS statements
+            inner join aws_s3_bucket_policies bp ON bp._cq_parent_id = b._cq_id,
+                LATERAL FLATTEN(INPUT => IFF(TYPEOF(bp.policy_json:Statement) = 'STRING', TO_ARRAY(bp.policy_json:Statement), bp.policy_json:Statement)) AS statements
             WHERE
                 GET_PATH(statement, 'Effect')::STRING = 'Deny'
                 AND GET_PATH(statement, 'Condition.Bool.aws:SecureTransport')::STRING = 'false'
@@ -50,15 +50,15 @@ where
         from (select aws_s3_buckets.arn,
                      statements,
                      statements -> 'Principal' as principals
-              from aws_s3_buckets
-                   inner join aws_s3_bucket_policies on aws_s3_buckets.arn = aws_s3_bucket_policies.bucket_arn,
+              from aws_s3_buckets b
+                   inner join aws_s3_bucket_policies bp ON bp._cq_parent_id = b._cq_id,
                    jsonb_array_elements(
-                           case jsonb_typeof(aws_s3_bucket_policies.policy_json -> 'Statement')
+                           case jsonb_typeof(bp.policy_json -> 'Statement')
                                when
                                    'string' then jsonb_build_array(
-                                       aws_s3_bucket_policies.policy_json ->> 'Statement'
+                                       bp.policy_json ->> 'Statement'
                                    )
-                               when 'array' then aws_s3_bucket_policies.policy_json -> 'Statement'
+                               when 'array' then bp.policy_json -> 'Statement'
                                end
                        ) as statements
               where statements -> 'Effect' = '"Deny"') as foo,
@@ -98,9 +98,9 @@ WHERE
                 statements AS statement
             FROM
                 {{ full_table_name("aws_s3_buckets") }} AS b
-            inner join {{ full_table_name("aws_s3_bucket_policies") }}
-            on b.arn = aws_s3_bucket_policies.bucket_arn,
-            UNNEST(JSON_QUERY_ARRAY(aws_s3_bucket_policies.policy_json.Statement)) AS statements
+            inner join {{ full_table_name("aws_s3_bucket_policies") }} bp
+            on bp._cq_parent_id = b._cq_id,
+            UNNEST(JSON_QUERY_ARRAY(bp.policy_json.Statement)) AS statements
             WHERE
                 CAST(JSON_VALUE(statements.Effect) AS STRING) = 'Deny'
                 AND CAST(JSON_VALUE(JSON_EXTRACT(statements, '$.Condition.Bool."aws:SecureTransport"')) AS STRING) = 'false'

diff --git a/transformations/aws/macros/s3/publicly_readable_buckets.sql b/transformations/aws/macros/s3/publicly_readable_buckets.sql
@@ -10,11 +10,11 @@ WITH policy_allow_public AS (
     FROM
         (
             SELECT
-                aws_s3_buckets.arn,
+                b.arn,
                 statements.value:Principal AS principals
             FROM
-                aws_s3_buckets
-            inner join aws_s3_bucket_policies bp on aws_s3_buckets.arn = bp.bucket_arn,
+                aws_s3_buckets b
+            inner join aws_s3_bucket_policies bp ON bp._cq_parent_id = b._cq_id,
                 LATERAL FLATTEN(INPUT => IFF(TYPEOF(bp.policy_json:Statement) = 'STRING', 
                                               TO_ARRAY(bp.policy_json:Statement), 
                                               bp.policy_json:Statement)) AS statements
@@ -38,28 +38,28 @@ SELECT
     '{{framework}}' As framework,
     '{{check_id}}' As check_id,
     'S3 buckets should prohibit public read access' AS title,
-    aws_s3_buckets.account_id,
-    aws_s3_buckets.arn AS resource_id,
+    b.account_id,
+    b.arn AS resource_id,
     'fail' AS status -- TODO FIXME
 FROM
-    aws_s3_buckets
+    aws_s3_buckets b
 LEFT JOIN
-    aws_s3_bucket_grants ON
-        aws_s3_buckets.arn = aws_s3_bucket_grants.bucket_arn
+    aws_s3_bucket_grants bg
+        ON bg._cq_parent_id = b._cq_id
 LEFT JOIN policy_allow_public ON
-        aws_s3_buckets.arn = policy_allow_public.arn
-LEFT JOIN aws_s3_bucket_public_access_blocks ON
-        aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn
+        b.arn = policy_allow_public.arn
+LEFT JOIN aws_s3_bucket_public_access_blocks bpab
+        ON bpab._cq_parent_id = b._cq_id
 WHERE
     (
-        (aws_s3_bucket_public_access_blocks.public_access_block_configuration:BlockPublicAcls)::boolean != TRUE
+        (bpab.public_access_block_configuration:BlockPublicAcls)::boolean != TRUE
         AND (
-            aws_s3_bucket_grants.grantee:URI::STRING = 'http://acs.amazonaws.com/groups/global/AllUsers'
-            AND aws_s3_bucket_grants.permission IN ('READ_ACP', 'FULL_CONTROL')
+            bg.grantee:URI::STRING = 'http://acs.amazonaws.com/groups/global/AllUsers'
+            AND bg.permission IN ('READ_ACP', 'FULL_CONTROL')
         )
     )
     OR (
-        (aws_s3_bucket_public_access_blocks.public_access_block_configuration:BlockPublicPolicy)::boolean != TRUE
+        (bpab.public_access_block_configuration:BlockPublicPolicy)::boolean != TRUE
         AND policy_allow_public.statement_count > 0
     )
 {% endmacro %}
@@ -97,39 +97,40 @@ select
     '{{framework}}' as framework,
     '{{check_id}}' as check_id,
     'S3 buckets should prohibit public read access' as title,
-    aws_s3_buckets.account_id,
-    aws_s3_buckets.arn as resource_id,
+    b.account_id,
+    b.arn as resource_id,
     'fail' as status -- TODO FIXME
 from
     -- Find and join all bucket ACLS that givea public write access
-    aws_s3_buckets
+    aws_s3_buckets b
 left join
-    aws_s3_bucket_grants on
-            aws_s3_buckets.arn = aws_s3_bucket_grants.bucket_arn
+    aws_s3_bucket_grants bg
+        ON bg._cq_parent_id = b._cq_id
 -- Find all statements that could give public allow access 
 -- Statements that give public access have 1) Effect == Allow 2) One of the following principal:
 --       Principal = {"AWS": "*"}
 --       Principal = {"AWS": ["arn:aws:iam::12345678910:root", "*"]}
 --       Principal = "*"
 left join policy_allow_public on
-        aws_s3_buckets.arn = policy_allow_public.arn
-left join aws_s3_bucket_public_access_blocks on
-        aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn
+        b.arn = policy_allow_public.arn
+left join aws_s3_bucket_public_access_blocks bpab
+        ON bpab._cq_parent_id = b._cq_id
 where
     (
-        (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicAcls')::boolean != TRUE
+        (bpab.public_access_block_configuration -> 'BlockPublicAcls')::boolean != TRUE
         and (
             grantee->>'URI' = 'http://acs.amazonaws.com/groups/global/AllUsers'
             and permission in ('READ_ACP', 'FULL_CONTROL')
         )
     )
     or (
-        (aws_s3_bucket_public_access_blocks.public_access_block_configuration -> 'BlockPublicPolicy')::boolean != TRUE
+        (bpab.public_access_block_configuration -> 'BlockPublicPolicy')::boolean != TRUE
         and policy_allow_public.statement_count > 0
     )
 {% endmacro %}
 
 {% macro default__publicly_readable_buckets(framework, check_id) %}{% endmacro %}
+
 {% macro bigquery__publicly_readable_buckets(framework, check_id) %}
 with policy_allow_public as (
     select
@@ -162,34 +163,34 @@ select
     '{{framework}}' as framework,
     '{{check_id}}' as check_id,
     'S3 buckets should prohibit public read access' as title,
-    aws_s3_buckets.account_id,
-    aws_s3_buckets.arn as resource_id,
+    b.account_id,
+    b.arn as resource_id,
     'fail' as status -- TODO FIXME
 from
     -- Find and join all bucket ACLS that give a public write access
-    {{ full_table_name("aws_s3_buckets") }}
+    {{ full_table_name("aws_s3_buckets") }} b
 left join
-    {{ full_table_name("aws_s3_bucket_grants") }} on
-        aws_s3_buckets.arn = aws_s3_bucket_grants.bucket_arn
+    {{ full_table_name("aws_s3_bucket_grants") }} bg on
+        bg._cq_parent_id = b._cq_id
 -- Find all statements that could give public allow access 
 -- Statements that give public access have 1) Effect == Allow 2) One of the following principal:
 --       Principal = {"AWS": "*"}
 --       Principal = {"AWS": ["arn:aws:iam::12345678910:root", "*"]}
 --       Principal = "*"
 left join policy_allow_public on
-        aws_s3_buckets.arn = policy_allow_public.arn
-left join {{ full_table_name("aws_s3_bucket_public_access_blocks") }} on
-        aws_s3_buckets.arn = aws_s3_bucket_public_access_blocks.bucket_arn
+        b.arn = policy_allow_public.arn
+left join {{ full_table_name("aws_s3_bucket_public_access_blocks") }} bpab
+        ON bpab._cq_parent_id = b._cq_id
 where
     (
-        CAST( JSON_VALUE(aws_s3_bucket_public_access_blocks.public_access_block_configuration.BlockPublicAcls) AS BOOL) != TRUE
+        CAST( JSON_VALUE(bpab.public_access_block_configuration.BlockPublicAcls) AS BOOL) != TRUE
         and (
             JSON_VALUE(grantee.URI) = 'http://acs.amazonaws.com/groups/global/AllUsers'
             and permission in ('READ_ACP', 'FULL_CONTROL')
         )
     )
     or (
-       CAST( JSON_VALUE(aws_s3_bucket_public_access_blocks.public_access_block_configuration.BlockPublicPolicy) AS BOOL) != TRUE
+       CAST( JSON_VALUE(bpab.public_access_block_configuration.BlockPublicPolicy) AS BOOL) != TRUE
         and policy_allow_public.statement_count > 0
     )
 {% endmacro %}