Further improve testing and configure uploading SARIF results to GitH…

…ub Security tab
finos · Dec 6, 2023 · 1fc673e · 1fc673e
1 parent 03ee3b0
commit 1fc673e
Show file tree

Hide file tree

Showing 10 changed files with 111 additions and 47 deletions.
diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml
@@ -4,15 +4,20 @@
 
 name: Test-Bandit
 on: [push, pull_request]
+permissions:
+  contents: read
 
 jobs:
   bandit:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
     runs-on: ubuntu-latest
     env:
       NEW_BRANCH: new_branch
       MAIN_BRANCH: main_branch
-      NEW_BRANCH_REPORT_FILE: new_branch_bandit_report_file.txt
-      MAIN_BRANCH_REPORT_FILE: main_branch_bandit_report_file.txt
+      NEW_BRANCH_REPORT_FILE: new_branch_bandit_report_file.sarif
+      MAIN_BRANCH_REPORT_FILE: main_branch_bandit_report_file.sarif
       DIFF_REPORTS_FILE: diff_bandit_reports.txt
 
     steps:
@@ -29,20 +34,26 @@ jobs:
 
       - name: Install bandit
         run: |-
-          pip3 install bandit
+          pip3 install bandit bandit-sarif-formatter
 
       - name: Run bandit on the new branch
         run: |-
           bandit --configfile ${{ env.NEW_BRANCH }}/.github/workflows/bandit.conf \
-                 --exit-zero --recursive \
+                 --format sarif --exit-zero --recursive \
                  --output ${{ env.NEW_BRANCH_REPORT_FILE }} ${{ env.NEW_BRANCH }}/
 
       - name: Run bandit on the main branch
         run: |-
           bandit --configfile ${{ env.NEW_BRANCH }}/.github/workflows/bandit.conf \
-                 --exit-zero --recursive \
+                 --format sarif --exit-zero --recursive \
                  --output ${{ env.MAIN_BRANCH_REPORT_FILE }} ${{ env.MAIN_BRANCH }}/
 
+      - name: Upload bandit scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        if: success() || failure()
+        with:
+          sarif_file: "${{ env.NEW_BRANCH_REPORT_FILE }}"
+
       - name: Save report results for the new branch
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/cfn_lint.conf b/.github/workflows/cfn_lint.conf
@@ -1,4 +1,3 @@
 # Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 # Licensed under the Apache License, Version 2.0 https://aws.amazon.com/apache-2-0/
-
diff --git a/.github/workflows/cfn_lint.yaml → .github/workflows/cfn_lint.yml b/.github/workflows/cfn_lint.yaml → .github/workflows/cfn_lint.yml
@@ -4,15 +4,20 @@
 
 name: Test-CFN-Lint
 on: [push, pull_request]
+permissions:
+  contents: read
 
 jobs:
   cfn-lint:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
     runs-on: ubuntu-latest
     env:
       NEW_BRANCH: new_branch
       MAIN_BRANCH: main_branch
-      NEW_BRANCH_REPORT_FILE: new_branch_cfnlint_report_file.json
-      MAIN_BRANCH_REPORT_FILE: main_branch_cfnlint_report_file.json
+      NEW_BRANCH_REPORT_FILE: new_branch_cfnlint_report_file.sarif
+      MAIN_BRANCH_REPORT_FILE: main_branch_cfnlint_report_file.sarif
       DIFF_REPORTS_FILE: diff_cfnlint_reports.txt
 
     steps:
@@ -35,19 +40,25 @@ jobs:
       - name: Run cfn_lint on the new branch
         run: |-
           cfn-lint --config-file ${{ env.NEW_BRANCH }}/.github/workflows/cfn_lint.conf \
-                   --format json --output-file ${{ env.NEW_BRANCH_REPORT_FILE }} \
+                   --format sarif --output-file ${{ env.NEW_BRANCH_REPORT_FILE }} \
                    ${{ env.NEW_BRANCH }}/deployment/dev_environment_cloud9/**/*.yaml \
                    ${{ env.NEW_BRANCH }}/deployment/init_grid/**/*.yaml
         continue-on-error: true
 
       - name: Run cfn_lint on the main branch
         run: |-
           cfn-lint --config-file ${{ env.NEW_BRANCH }}/.github/workflows/cfn_lint.conf \
-                   --format json --output-file ${{ env.MAIN_BRANCH_REPORT_FILE }} \
+                   --format sarif --output-file ${{ env.MAIN_BRANCH_REPORT_FILE }} \
                    ${{ env.MAIN_BRANCH }}/deployment/dev_environment_cloud9/**/*.yaml \
                    ${{ env.MAIN_BRANCH }}/deployment/init_grid/**/*.yaml
         continue-on-error: true
 
+      - name: Upload cfn-lint scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        if: success() || failure()
+        with:
+          sarif_file: "${{ env.NEW_BRANCH_REPORT_FILE }}"
+
       - name: Save report results for the new branch
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/cfn_nag.yml b/.github/workflows/cfn_nag.yml
@@ -4,15 +4,20 @@
 
 name: Test-CFN-Nag
 on: [push, pull_request]
+permissions:
+  contents: read
 
 jobs:
   cfn_nag:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
     runs-on: ubuntu-latest
     env:
       NEW_BRANCH: new_branch
       MAIN_BRANCH: main_branch
-      NEW_BRANCH_REPORT_FILE: new_branch_cfn_nag_report_file.txt
-      MAIN_BRANCH_REPORT_FILE: main_branch_cfn_nag_report_file.txt
+      NEW_BRANCH_REPORT_FILE: new_branch_cfn_nag_report_file.sarif
+      MAIN_BRANCH_REPORT_FILE: main_branch_cfn_nag_report_file.sarif
       DIFF_REPORTS_FILE: diff_cfn_nag_reports.txt
 
     steps:
@@ -31,16 +36,22 @@ jobs:
         uses: stelligent/cfn_nag@master
         with:
           input_path: ${{ env.NEW_BRANCH }}/deployment/
-          extra_args: -g #-t '..*\/(cfn|cloudformation)\/..*\.yaml' # This flag doesnt seem to be working.
+          extra_args: '-g -o sarif' #-t '..*\/(cfn|cloudformation)\/..*\.yaml' # This flag doesnt seem to be working.
           output_path: ${{ env.NEW_BRANCH_REPORT_FILE }}
 
       - name: Run cfn_nag on main
         uses: stelligent/cfn_nag@master
         with:
           input_path: ${{ env.MAIN_BRANCH }}/deployment/
-          extra_args: -g #-t '..*\/(cfn|cloudformation)\/..*\.yaml' # This flag doesnt seem to be working.
+          extra_args: '-g -o sarif' #-t '..*\/(cfn|cloudformation)\/..*\.yaml' # This flag doesnt seem to be working.
           output_path: ${{ env.MAIN_BRANCH_REPORT_FILE }}
 
+      - name: Upload cnf_nag scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        if: success() || failure()
+        with:
+          sarif_file: "${{ env.NEW_BRANCH_REPORT_FILE }}"
+
       - name: Save report results for the new branch
         uses: actions/upload-artifact@v3
         with:

diff --git a/.github/workflows/checkov.yaml → .github/workflows/checkov.yml b/.github/workflows/checkov.yaml → .github/workflows/checkov.yml
@@ -4,15 +4,20 @@
 
 name: Test-Checkov
 on: [push, pull_request]
+permissions:
+  contents: read
 
 jobs:
   checkov:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
     runs-on: ubuntu-latest
     env:
       NEW_BRANCH: new_branch
       MAIN_BRANCH: main_branch
-      NEW_BRANCH_REPORT_FILE: new_branch_checkov_report_file.json
-      MAIN_BRANCH_REPORT_FILE: main_branch_checkov_report_file.json
+      NEW_BRANCH_REPORT_FILE: new_branch_checkov_report_file.sarif
+      MAIN_BRANCH_REPORT_FILE: main_branch_checkov_report_file.sarif
       DIFF_REPORTS_FILE: diff_checkov_reports.txt
 
     steps:
@@ -43,11 +48,10 @@ jobs:
           output_format: cli,sarif
           output_file_path: console,${{ env.MAIN_BRANCH_REPORT_FILE }}
 
-      # This action can be enabled when using GitHub Security Code Scanning feature
-      # - name: Upload checkov scan results to GitHub Security tab
-      #   uses: github/codeql-action/upload-sarif@v2
-      #   with:
-      #     sarif_file: "./../${{ env.NEW_BRANCH_REPORT_FILE }}"
+      - name: Upload checkov scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        with:
+          sarif_file: "${{ env.NEW_BRANCH_REPORT_FILE }}"
 
       - name: Save report results for the new branch
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/doc_builder.yaml b/.github/workflows/doc_builder.yaml
@@ -3,6 +3,9 @@ on:
   push:
     branches:
       - main
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Deploy docs

diff --git a/.github/workflows/flake8.conf b/.github/workflows/flake8.conf
@@ -6,5 +6,6 @@
 extend-ignore = E501,W503,E131
 exclude = .git,__pycache__
 max-line-length = 100
-statistics = True
-count = True
+statistics = False
+count = False
+format = sarif
diff --git a/.github/workflows/flake8.yml b/.github/workflows/flake8.yml
@@ -4,15 +4,20 @@
 
 name: Test-Flake8
 on: [push, pull_request]
+permissions:
+  contents: read
 
 jobs:
-  flake:
+  flake8:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
     runs-on: ubuntu-latest
     env:
       NEW_BRANCH: new_branch
       MAIN_BRANCH: main_branch
-      NEW_BRANCH_REPORT_FILE: new_branch_flake_report_file.txt
-      MAIN_BRANCH_REPORT_FILE: main_branch_flake_report_file.txt
+      NEW_BRANCH_REPORT_FILE: new_branch_flake_report_file.sarif
+      MAIN_BRANCH_REPORT_FILE: main_branch_flake_report_file.sarif
       DIFF_REPORTS_FILE: diff_reports.txt
 
     steps:
@@ -27,21 +32,27 @@ jobs:
           repository: awslabs/aws-htc-grid
           path: ${{ env.MAIN_BRANCH }}
 
-      - name: Install flake8
-        run: |-
-          pip3 install flake8
-          ls -l -a
-          pwd
-
       - name: Run flake8 on the new branch
-        run: |-
-          flake8 --config ${{ env.NEW_BRANCH }}/.github/workflows/flake8.conf \
-                 --exit-zero --output-file ${{ env.NEW_BRANCH_REPORT_FILE }} ${{ env.NEW_BRANCH }}
+        uses: py-actions/flake8@v2
+        with:
+          path: ${{ env.NEW_BRANCH }}
+          args: --config ${{ env.NEW_BRANCH }}/.github/workflows/flake8.conf --output-file ${{ env.NEW_BRANCH_REPORT_FILE }}
+          plugins: 'flake8-sarif'
+        continue-on-error: true
 
-      - name: Run flake8 on the main branch
-        run: |-
-          flake8 --config ${{ env.NEW_BRANCH }}/.github/workflows/flake8.conf \
-                 --exit-zero --output-file ${{ env.MAIN_BRANCH_REPORT_FILE }} ${{ env.MAIN_BRANCH }}
+      - name: Run flake8 on main
+        uses: py-actions/flake8@v2
+        with:
+          path: ${{ env.MAIN_BRANCH }}
+          args: --config ${{ env.NEW_BRANCH }}/.github/workflows/flake8.conf --output-file ${{ env.MAIN_BRANCH_REPORT_FILE }}
+          plugins: 'flake8-sarif'
+        continue-on-error: true
+
+      - name: Upload flake8 scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        if: success() || failure()
+        with:
+          sarif_file: "${{ env.NEW_BRANCH_REPORT_FILE }}"
 
       - name: Save report results for the new branch
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/trivy.conf b/.github/workflows/trivy.conf
@@ -17,6 +17,10 @@ scan:
     - '**/.terraform'
     - venv/
 
+misconfiguration:
+  terraform:
+    exclude-downloaded-modules: true
+
 # These flag dont seem to be working from the config file.
 
 # severity:

diff --git a/.github/workflows/trivy.yaml → .github/workflows/trivy.yml b/.github/workflows/trivy.yaml → .github/workflows/trivy.yml
@@ -4,15 +4,20 @@
 
 name: Test-Trivy
 on: [push, pull_request]
+permissions:
+  contents: read
 
 jobs:
   trivy:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
     runs-on: ubuntu-latest
     env:
       NEW_BRANCH: new_branch
       MAIN_BRANCH: main_branch
-      NEW_BRANCH_REPORT_FILE: new_branch_trivy_report_file.json
-      MAIN_BRANCH_REPORT_FILE: main_branch_trivy_report_file.json
+      NEW_BRANCH_REPORT_FILE: new_branch_trivy_report_file.sarif
+      MAIN_BRANCH_REPORT_FILE: main_branch_trivy_report_file.sarif
       DIFF_REPORTS_FILE: diff_trivy_reports.txt
 
     steps:
@@ -51,12 +56,16 @@ jobs:
         env:
           TRIVY_OUTPUT: ${{ env.MAIN_BRANCH_REPORT_FILE }}
 
-      # This action can be enabled when using GitHub Security Code Scanning feature
-      # - name: Upload Trivy scan results to GitHub Security tab
-      #   uses: github/codeql-action/upload-sarif@v2
-      #   if: success() || failure()
-      #   with:
-      #     sarif_file: "${{ env.NEW_BRANCH_REPORT_FILE }}"
+      # As per the following issue/solution: https://github.com/aquasecurity/trivy/issues/5003
+      - name: Fix trivy SARIF report for URI scheme
+        run: |-
+          sed -i 's#git::https:/##g' ${{ env.NEW_BRANCH_REPORT_FILE }}
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v2
+        if: success() || failure()
+        with:
+          sarif_file: "${{ env.NEW_BRANCH_REPORT_FILE }}"
 
       - name: Save report results for the new branch
         uses: actions/upload-artifact@v3