e2e:feature - update e2e/analysis tests for build locally (ZupIT#761)

* In older version i found some errors:
  * The analysis is not effective in check tools runned, because not apply assert in all tools runned by language, then i fixed this;
  * Occurs when horusec not found files required in project how requirements.txt and others, not exist validation if analysis was been executed with success, then i fixed this;
* In older version when run analysis always download all tools from dockerhub, and this not validate if current version is OK for create new release, then was implemented this new test. Note the struct AnalysisTestCase is shared into both tests, then was moved to centralized file.
* Another error found was that the error logs presented by the tool were scattered, so we centralized the SetAnalysisError function where, in addition to adding the error in the analysis, it also performs the unified log among all the tools.
* An error was found in the language detect package, if the user is using a project with git submodule it was not possible to copy the git folder as it was only a reference so we fixed these problems.
* In HCL language it was necessary to change your docker file seen at issue bridgecrewio/checkov#1947 where we found a problem in checkov for alpine version
* Was Updated all dockerfiles with image from python fix in version python:3.10.0-alpine3.14
* Was Update dockerfile of Ruby language for usage ruby:2.4.10-alpine base image, because brakeman tool is not suport for newest version of ruby
* Was Updated DockerAPI for pass GITHUB_TOKEN env because nancy and others tools use the github api's and get error GitHub rate-limiting on unauthenticated requests

Signed-off-by: wilian <[email protected]>

.github/workflows/e2e.yml

@@ -24,6 +24,9 @@ on:
 permissions: read-all
 jobs:
   test:
+    permissions:
+      contents: write
+      packages: write
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -38,6 +41,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v2
         with:
+          fetch-depth: 0
           submodules: true
 
       - name: Setup Golang
@@ -50,6 +54,14 @@ jobs:
         uses: docker-practice/actions-setup-docker@master
         with:
           docker_buildx: false
-
-      - name: Run e2e test for ${{ matrix.os }}
-        run: go test -v ./e2e/... -timeout 60m -failfast -race
+      - name: Build local images
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: sh ./deployments/scripts/build-all-tools.sh
+      - name: Run e2e analysis test for ${{ matrix.os }}
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: go test -v ./e2e/analysis/... -timeout 60m -failfast -race
+      - name: Run e2e commands test for ${{ matrix.os }}
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+        run: go test -v ./e2e/commands/... -timeout 60m -failfast -race

Makefile

@@ -34,23 +34,26 @@ test:
 	$(GO) test -v $(GO_LIST_TO_TEST) -race -timeout=5m -parallel=1 -failfast -short
 
 test-e2e:
+	sh ./deployments/scripts/build-all-tools.sh
 	$(GO) clean -testcache
-	$(GO) test -v ./e2e/... -timeout=10m -parallel=1 -failfast
+	$(GO) test -v ./e2e/analysis/... -timeout=30m -parallel=1 -failfast
+	$(GO) clean -testcache
+	$(GO) test -v ./e2e/commands/... -timeout=30m -parallel=1 -failfast
 
 fix-imports:
     ifeq (, $(shell which $(GO_IMPORTS)))
-		$(GO) get -u golang.org/x/tools/cmd/goimports
+		$(GO) install golang.org/x/tools/cmd/goimports
 		$(GO_IMPORTS) -local $(GO_IMPORTS_LOCAL) -w $(GO_FILES)
     else
 		$(GO_IMPORTS) -local $(GO_IMPORTS_LOCAL) -w $(GO_FILES)
     endif
 
 license:
-	$(GO) get -u github.com/google/addlicense
+	$(GO) install github.com/google/addlicense
 	@$(ADDLICENSE) -check -f ./copyright.txt $(shell find -regex '.*\.\(go\|js\|ts\|yml\|yaml\|sh\|dockerfile\)')
 
 license-fix:
-	$(GO) get -u github.com/google/addlicense
+	$(GO) install github.com/google/addlicense
 	@$(ADDLICENSE) -f ./copyright.txt $(shell find -regex '.*\.\(go\|js\|ts\|yml\|yaml\|sh\|dockerfile\)')
 
 security:

deployments/scripts/build-all-tools.sh

@@ -0,0 +1,40 @@
+# Copyright 2021 ZUP IT SERVICOS EM TECNOLOGIA E INOVACAO SA
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+IMAGE_BASE_NAME=$1
+IMAGE_TAG=$2
+
+setDefaultImageBaseName() {
+    if [ -z "$IMAGE_BASE_NAME" ]; then
+        IMAGE_BASE_NAME="local"
+    fi
+    if [ -z "$IMAGE_TAG" ]; then
+        IMAGE_TAG="local"
+    fi
+}
+
+getDirectoryAndImagesNameByLanguageName() {
+    # shellcheck disable=SC2044
+    for DOCKERFILE in $(find internal -type f -iname "Dockerfile"); do
+        LANGUAGE=$(echo $DOCKERFILE | cut -d "/" -f4)
+        if ! docker build -t "$IMAGE_BASE_NAME-$LANGUAGE:$IMAGE_TAG" -f "$DOCKERFILE" .; then
+            echo "Error on build docker file in path: $DOCKERFILE"
+            exit 1
+        fi
+    done
+}
+
+setDefaultImageBaseName
+
+getDirectoryAndImagesNameByLanguageName

e2e/analysis/analysis_suite_test.go

@@ -17,11 +17,14 @@ package analysis_test
 import (
 	"testing"
 
+	ginkgoconfig "github.com/onsi/ginkgo/config"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 func TestAnalysis(t *testing.T) {
+	ginkgoconfig.GinkgoConfig.FailFast = true
 	RegisterFailHandler(Fail)
 	defer GinkgoRecover()
 	RunSpecs(t, "Analysis Suite")

e2e/analysis/analysis_test.go

@@ -15,123 +15,132 @@
 package analysis_test
 
 import (
+	"encoding/json"
 	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
 
+	"github.com/ZupIT/horusec-devkit/pkg/enums/languages"
+
+	"github.com/google/uuid"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/onsi/gomega/gexec"
 
-	"github.com/ZupIT/horusec-devkit/pkg/enums/languages"
-	"github.com/ZupIT/horusec-devkit/pkg/enums/tools"
+	"github.com/ZupIT/horusec-devkit/pkg/utils/logger"
+	"github.com/ZupIT/horusec/config"
+	"github.com/ZupIT/horusec/e2e/analysis"
+	customimages "github.com/ZupIT/horusec/internal/entities/custom_images"
 	"github.com/ZupIT/horusec/internal/utils/testutil"
 )
 
-type AnalysisTestCase struct {
-	Language       languages.Language
-	AnalysisFolder string
-	ExpectedTools  []tools.Tool
-}
+const isWindows = runtime.GOOS == "windows"
+const isDarwin = runtime.GOOS == "darwin"
+const horusecConfigName = "horusec-config.json"
+
+var _ = Describe("Run a complete horusec analysis when build tools locally", func() {
+	tmpDir := CreateHorusecConfigAndReturnTMPDirectory()
+
+	Describe("Running e2e tests when build tools locally", func() {
+		allTestsCases := analysis.NewTestCase()
+
+		for idx, testCase := range allTestsCases {
+			if !isDarwin {
+				horusecConfigFilePathBuildLocally := filepath.Join(tmpDir, horusecConfigName)
+				RunTestCase(testCase, horusecConfigFilePathBuildLocally, "(build locally)", idx, len(allTestsCases))
 
-func (A AnalysisTestCase) RunAnalysisTestCase() (*gexec.Session, error) {
-	flags := map[string]string{
-		testutil.StartFlagProjectPath: A.AnalysisFolder,
+				time.Sleep(2 * time.Second)
+			}
+
+			horusecConfigFilePathDownloadFromDockerHub := filepath.Join(allTestsCases[idx].Command.Flags[testutil.StartFlagProjectPath], horusecConfigName)
+			RunTestCase(testCase, horusecConfigFilePathDownloadFromDockerHub, "(download from dockerhub)", idx, len(allTestsCases))
+
+			time.Sleep(2 * time.Second)
+		}
+	})
+
+	AfterEach(func() {
+		if err := os.RemoveAll(tmpDir); err != nil {
+			Fail(fmt.Sprintf("Error on remove tmp folder: %v", err))
+		}
+	})
+})
+
+func RunTestCase(testCase *analysis.TestCase, horusecConfigFilePath, runType string, idx, lenTestCase int) {
+	testCase.Command.Flags[testutil.GlobalFlagConfigFilePath] = horusecConfigFilePath
+	if isWindows && testCase.RequiredDocker {
+		logger.LogInfo("Tool ignored because is required docker: ", runType, testCase.Tool, fmt.Sprintf("[%v/%v]", idx+1, lenTestCase))
+		return
 	}
-	cmd := testutil.GinkgoGetHorusecCmdWithFlags(testutil.CmdStart, flags)
-	return gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
-}
+	logger.LogInfo("Preparing test for run e2e: ", runType, testCase.Tool, fmt.Sprintf("[%v/%v]", idx+1, lenTestCase))
+	session, err := testCase.RunAnalysisTestCase()
+	if err != nil {
+		Fail(fmt.Sprintf("Error on run analysis Test Case %s: %v", runType, err))
+	}
+	session.Wait(testutil.AverageTimeoutAnalyzeForExamplesFolder)
+	testCase.Command.Output = string(session.Out.Contents())
+	testCase.Command.ExitCode = session.ExitCode()
 
-var testcases = []AnalysisTestCase{
-	{
-		Language:       languages.Python,
-		AnalysisFolder: testutil.PythonExample1,
-		ExpectedTools:  []tools.Tool{tools.Bandit, tools.BundlerAudit, tools.Trivy},
-	},
-	{
-		Language:       languages.Ruby,
-		AnalysisFolder: testutil.RubyExample1,
-		ExpectedTools:  []tools.Tool{tools.Brakeman, tools.BundlerAudit, tools.Trivy},
-	},
-	{
-		Language:       languages.Javascript,
-		AnalysisFolder: testutil.JavaScriptExample1,
-		ExpectedTools:  []tools.Tool{tools.NpmAudit, tools.YarnAudit, tools.HorusecEngine, tools.Semgrep, tools.Trivy},
-	},
-	{
-		Language:       languages.Go,
-		AnalysisFolder: testutil.GoExample1,
-		ExpectedTools:  []tools.Tool{tools.GoSec, tools.Semgrep, tools.Nancy, tools.Trivy},
-	},
-	{
-		Language:       languages.CSharp,
-		AnalysisFolder: testutil.CsharpExample1,
-		ExpectedTools:  []tools.Tool{tools.SecurityCodeScan, tools.HorusecEngine, tools.DotnetCli, tools.Trivy},
-	},
-	{
-		Language:       languages.Java,
-		AnalysisFolder: testutil.JavaExample1,
-		ExpectedTools:  []tools.Tool{tools.HorusecEngine, tools.Semgrep, tools.Trivy},
-	},
-	{
-		Language:       languages.Kotlin,
-		AnalysisFolder: testutil.KotlinExample1,
-		ExpectedTools:  []tools.Tool{tools.HorusecEngine},
-	},
-	{
-		Language:       languages.Leaks,
-		AnalysisFolder: testutil.LeaksExample1,
-		ExpectedTools:  []tools.Tool{tools.GitLeaks, tools.HorusecEngine},
-	},
-	{
-		Language:       languages.PHP,
-		AnalysisFolder: testutil.PHPExample1,
-		ExpectedTools:  []tools.Tool{tools.Semgrep, tools.PhpCS, tools.Trivy},
-	},
-	{
-		Language:       languages.Dart,
-		AnalysisFolder: testutil.DartExample1,
-		ExpectedTools:  []tools.Tool{tools.HorusecEngine},
-	},
-	{
-		Language:       languages.Elixir,
-		AnalysisFolder: testutil.ElixirExample1,
-		ExpectedTools:  []tools.Tool{tools.MixAudit, tools.Sobelow},
-	},
-	{
-		Language:       languages.Nginx,
-		AnalysisFolder: testutil.NginxExample1,
-		ExpectedTools:  []tools.Tool{tools.HorusecEngine},
-	},
-	{
-		Language:       languages.Swift,
-		AnalysisFolder: testutil.SwiftExample1,
-		ExpectedTools:  []tools.Tool{tools.HorusecEngine},
-	},
+	It(fmt.Sprintf("Execute command without error for %s: %s", runType, testCase.Tool), func() {
+		Expect(testCase.Command.ExitCode).Should(Equal(0))
+	})
+
+	It(fmt.Sprintf("Validate is outputs expected exists on tool %s: %s", runType, testCase.Tool), func() {
+		for _, outputExpected := range testCase.Expected.OutputsContains {
+			Expect(testCase.Command.Output).Should(
+				ContainSubstring(outputExpected),
+				fmt.Sprintf("The output [%s] not exist in output", outputExpected),
+				testCase.Command.Output,
+			)
+		}
+	})
+
+	It(fmt.Sprintf("Validate is outputs not expected exists on tool %s: %s", runType, testCase.Tool), func() {
+		for _, outputNotExpected := range testCase.Expected.OutputsNotContains {
+			Expect(testCase.Command.Output).ShouldNot(
+				ContainSubstring(outputNotExpected),
+				fmt.Sprintf("The output [%s] exist in output", outputNotExpected),
+				testCase.Command.Output,
+			)
+		}
+	})
 }
 
-var _ = Describe("Run a complete horusec analysis", func() {
-	var (
-		session *gexec.Session
-	)
-
-	for _, tt := range testcases {
-		Describe(fmt.Sprintf("Running on %s codebase.", tt.Language.ToString()), func() {
-			session, _ = tt.RunAnalysisTestCase()
-			session.Wait(testutil.AverageTimeoutAnalyzeForExamplesFolder)
-
-			It("execute command without error", func() {
-				Expect(session.ExitCode()).To(Equal(0))
-			})
-
-			It("vulnerabilities were found", func() {
-				Expect(session.Out.Contents()).To(ContainSubstring("{HORUSEC_CLI} Vulnerability Hash expected to be FIXED"))
-				Expect(session.Out.Contents()).To(ContainSubstring("VULNERABILITIES WERE FOUND IN YOUR CODE"))
-			})
-
-			It("running all expected tools", func() {
-				for _, tool := range tt.ExpectedTools {
-					Expect(session.Out.Contents()).To(ContainSubstring(fmt.Sprintf("{HORUSEC_CLI} Running %s", tool.ToString())))
-				}
-			})
-		})
+func CreateHorusecConfigAndReturnTMPDirectory() string {
+	tmpDir := filepath.Join(os.TempDir(), "horusec-analysis-e2e-"+uuid.NewString())
+	horusecConfigPath := filepath.Join(tmpDir, horusecConfigName)
+
+	if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
+		Fail(fmt.Sprintf("Error on create tmp folder: %v", err))
 	}
-})
+
+	horusecConfigFile, err := os.Create(horusecConfigPath)
+	if err != nil {
+		Fail(fmt.Sprintf("Error on create config file for scan on e2e test %v", err))
+	}
+
+	horusecConfig := config.New()
+
+	newCustomImages := customimages.Default()
+	for k := range newCustomImages {
+		if k == languages.CSharp {
+			newCustomImages[k] = strings.ToLower("local-csharp:local")
+		} else {
+			newCustomImages[k] = strings.ToLower(fmt.Sprintf("local-%s:local", k))
+		}
+	}
+
+	horusecConfig.CustomImages = newCustomImages
+	horusecConfigBytes, err := json.MarshalIndent(horusecConfig.ToMapLowerCase(), "", "  ")
+	if err != nil {
+		Fail(fmt.Sprintf("Error on marshall horusec-config.json %v", err))
+	}
+
+	_, err = horusecConfigFile.Write(horusecConfigBytes)
+	if err != nil {
+		Fail(fmt.Sprintf("Error on write config file for scan on e2e test %v", err))
+	}
+	return tmpDir
+}