From 0df35e4b4fbcd78899f26ba7ce5bf81907280a2a Mon Sep 17 00:00:00 2001 From: matheusalcantarazup <84723211+matheusalcantarazup@users.noreply.github.com> Date: Mon, 3 Jan 2022 14:26:07 -0300 Subject: [PATCH] analyzer:chore - split analyzer implementation into runner (#909) Previously the Analyzer struct holds the entire pipeline of an analysis, which add a lot of complexity, since this pipeline has a lot of steps: Detect languages, orchestrate tools by languages, send analysis to Horusec manager and them print the results on stdout. This commit split the Analyzer struct into a new struct `runner`. This new struct will be responsible only to orchestrate all tools to their respective languages. The coded implemented on runner is basically a copy and paste from Analyzer implementation, so the behavior remains the same. Some documentation was also added to try do make more clear what are the steps that Analyzer do, and what a runner means. Signed-off-by: Matheus Alcantara --- cmd/app/start/start.go | 2 +- internal/controllers/analyzer/analyzer.go | 391 ++---------------- .../controllers/analyzer/analyzer_test.go | 61 +-- internal/controllers/analyzer/runner.go | 385 +++++++++++++++++ 4 files changed, 443 insertions(+), 396 deletions(-) create mode 100644 internal/controllers/analyzer/runner.go diff --git a/cmd/app/start/start.go b/cmd/app/start/start.go index 8cbe9bd25..2bcc71cec 100644 --- a/cmd/app/start/start.go +++ b/cmd/app/start/start.go @@ -356,7 +356,7 @@ func (s *Start) isRunPromptQuestion(cmd *cobra.Command) bool { func (s *Start) executeAnalysisDirectory() (totalVulns int, err error) { if s.analyzer == nil { - s.analyzer = analyzer.NewAnalyzer(s.configs) + s.analyzer = analyzer.New(s.configs) } return s.analyzer.Analyze() diff --git a/internal/controllers/analyzer/analyzer.go b/internal/controllers/analyzer/analyzer.go index 1166af07e..8de0ab958 100644 --- a/internal/controllers/analyzer/analyzer.go +++ b/internal/controllers/analyzer/analyzer.go @@ -16,14 +16,7 @@ package analyzer import ( "fmt" - "io" - "log" - "os" - "os/signal" - "path" - "path/filepath" "strings" - "sync" "time" "github.com/ZupIT/horusec-devkit/pkg/entities/analysis" @@ -34,61 +27,17 @@ import ( "github.com/ZupIT/horusec-devkit/pkg/enums/severities" enumsVulnerability "github.com/ZupIT/horusec-devkit/pkg/enums/vulnerability" "github.com/ZupIT/horusec-devkit/pkg/utils/logger" - "github.com/briandowns/spinner" "github.com/google/uuid" - "github.com/sirupsen/logrus" "github.com/ZupIT/horusec/config" languagedetect "github.com/ZupIT/horusec/internal/controllers/language_detect" "github.com/ZupIT/horusec/internal/controllers/printresults" - "github.com/ZupIT/horusec/internal/enums/images" "github.com/ZupIT/horusec/internal/helpers/messages" "github.com/ZupIT/horusec/internal/services/docker" - dockerClient "github.com/ZupIT/horusec/internal/services/docker/client" - "github.com/ZupIT/horusec/internal/services/formatters" - "github.com/ZupIT/horusec/internal/services/formatters/c/flawfinder" - dotnetcli "github.com/ZupIT/horusec/internal/services/formatters/csharp/dotnet_cli" - "github.com/ZupIT/horusec/internal/services/formatters/csharp/horuseccsharp" - "github.com/ZupIT/horusec/internal/services/formatters/csharp/scs" - "github.com/ZupIT/horusec/internal/services/formatters/dart/horusecdart" - "github.com/ZupIT/horusec/internal/services/formatters/elixir/mixaudit" - "github.com/ZupIT/horusec/internal/services/formatters/elixir/sobelow" - dependencycheck "github.com/ZupIT/horusec/internal/services/formatters/generic/dependency_check" - "github.com/ZupIT/horusec/internal/services/formatters/generic/semgrep" - "github.com/ZupIT/horusec/internal/services/formatters/generic/trivy" - "github.com/ZupIT/horusec/internal/services/formatters/go/gosec" - "github.com/ZupIT/horusec/internal/services/formatters/go/nancy" - "github.com/ZupIT/horusec/internal/services/formatters/hcl/checkov" - "github.com/ZupIT/horusec/internal/services/formatters/hcl/tfsec" - "github.com/ZupIT/horusec/internal/services/formatters/java/horusecjava" - "github.com/ZupIT/horusec/internal/services/formatters/javascript/horusecjavascript" - "github.com/ZupIT/horusec/internal/services/formatters/javascript/npmaudit" - "github.com/ZupIT/horusec/internal/services/formatters/javascript/yarnaudit" - "github.com/ZupIT/horusec/internal/services/formatters/kotlin/horuseckotlin" - "github.com/ZupIT/horusec/internal/services/formatters/leaks/gitleaks" - "github.com/ZupIT/horusec/internal/services/formatters/leaks/horusecleaks" - "github.com/ZupIT/horusec/internal/services/formatters/nginx/horusecnginx" - "github.com/ZupIT/horusec/internal/services/formatters/php/phpcs" - "github.com/ZupIT/horusec/internal/services/formatters/python/bandit" - "github.com/ZupIT/horusec/internal/services/formatters/python/safety" - "github.com/ZupIT/horusec/internal/services/formatters/ruby/brakeman" - "github.com/ZupIT/horusec/internal/services/formatters/ruby/bundler" - "github.com/ZupIT/horusec/internal/services/formatters/shell/shellcheck" - "github.com/ZupIT/horusec/internal/services/formatters/swift/horusecswift" - "github.com/ZupIT/horusec/internal/services/formatters/yaml/horuseckubernetes" - horusecAPI "github.com/ZupIT/horusec/internal/services/horusec_api" + "github.com/ZupIT/horusec/internal/services/docker/client" + horusec_api "github.com/ZupIT/horusec/internal/services/horusec_api" ) -const LoadingDelay = 200 * time.Millisecond - -// detectVulnerabilityFn is a func that detect vulnerabilities on path. -// detectVulnerabilityFn funcs run all in parallel, so a WaitGroup is required -// to synchronize states of running analysis. -// -// detectVulnerabilityFn funcs can also spawn other detectVulnerabilityFn funcs -// just passing the received WaitGroup to underlying funcs. -type detectVulnerabilityFn func(wg *sync.WaitGroup, path string) error - // LanguageDetect is the interface that detect all languages in some directory. type LanguageDetect interface { Detect(directory string) ([]languages.Language, error) @@ -108,64 +57,45 @@ type HorusecService interface { GetAnalysis(uuid.UUID) (*analysis.Analysis, error) } +// Analyzer is responsible to orchestrate the pipeline of an analysis. +// +// Basically, an analysis has the following steps: +// 1 - Detect all languages on project path. +// 2 - Execute all tools to all languages founded. +// 3 - Send analysis to Horusuec Manager if access token is set. +// 4 - Print analysis results. type Analyzer struct { - docker docker.Docker analysis *analysis.Analysis config *config.Config languageDetect LanguageDetect printController PrintResults horusec HorusecService - formatter formatters.IService - loading *spinner.Spinner + runner *runner } -//nolint:funlen -func NewAnalyzer(cfg *config.Config) *Analyzer { - entity := &analysis.Analysis{ +// New create a new analyzer to a given config. +func New(cfg *config.Config) *Analyzer { + analysiss := &analysis.Analysis{ ID: uuid.New(), CreatedAt: time.Now(), Status: enumsAnalysis.Running, } - dockerAPI := docker.New(dockerClient.NewDockerClient(), cfg, entity.ID) + dockerAPI := docker.New(client.NewDockerClient(), cfg, analysiss.ID) return &Analyzer{ - docker: dockerAPI, - analysis: entity, + analysis: analysiss, config: cfg, - languageDetect: languagedetect.NewLanguageDetect(cfg, entity.ID), - printController: printresults.NewPrintResults(entity, cfg), - horusec: horusecAPI.NewHorusecAPIService(cfg), - formatter: formatters.NewFormatterService(entity, dockerAPI, cfg), - loading: newScanLoading(cfg), - } -} - -func (a *Analyzer) Analyze() (totalVulns int, err error) { - a.removeTrashByInterruptProcess() - totalVulns, err = a.runAnalysis() - a.removeHorusecFolder() - return totalVulns, err -} - -func (a *Analyzer) removeTrashByInterruptProcess() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - go func() { - for range c { - a.removeHorusecFolder() - log.Fatal() - } - }() -} - -func (a *Analyzer) removeHorusecFolder() { - err := os.RemoveAll(filepath.Join(a.config.ProjectPath, ".horusec")) - logger.LogErrorWithLevel(messages.MsgErrorRemoveAnalysisFolder, err) - if !a.config.DisableDocker { - a.docker.DeleteContainersFromAPI() + languageDetect: languagedetect.NewLanguageDetect(cfg, analysiss.ID), + printController: printresults.NewPrintResults(analysiss, cfg), + horusec: horusec_api.NewHorusecAPIService(cfg), + runner: newRunner(cfg, analysiss, dockerAPI), } } -func (a *Analyzer) runAnalysis() (totalVulns int, err error) { +// Analyze start an analysis and return the total of vulnerabilities founded +// and an error if exists. +// +// nolint: funlen +func (a *Analyzer) Analyze() (int, error) { langs, err := a.languageDetect.Detect(a.config.ProjectPath) if err != nil { return 0, err @@ -176,10 +106,14 @@ func (a *Analyzer) runAnalysis() (totalVulns int, err error) { fmt.Println() } - a.startDetectVulnerabilities(langs) + for _, err := range a.runner.run(langs) { + a.setAnalysisError(err) + } + if err = a.sendAnalysis(); err != nil { logger.LogStringAsError(fmt.Sprintf("[HORUSEC] %s", err.Error())) } + return a.startPrintResults() } @@ -225,236 +159,6 @@ func (a *Analyzer) formatAnalysisToSendToAPI() { } } -// startDetectVulnerabilities handle execution of all analysis in parallel -// -// We ignore the funlen and gocyclo lint here because concurrency code is complicated -// nolint:funlen,gocyclo -func (a *Analyzer) startDetectVulnerabilities(langs []languages.Language) { - var wg sync.WaitGroup - done := make(chan struct{}) - - funcs := a.detectVulnerabilityFuncs() - - a.loading.Start() - - go func() { - defer close(done) - for _, language := range langs { - for _, subPath := range a.config.WorkDir.PathsOfLanguage(language) { - projectSubPath := subPath - a.logProjectSubPath(language, projectSubPath) - - if fn, exist := funcs[language]; exist { - wg.Add(1) - go func() { - defer wg.Done() - if err := fn(&wg, projectSubPath); err != nil { - a.setAnalysisError(err) - } - }() - } - } - } - wg.Wait() - }() - - timeout := time.After(time.Duration(a.config.TimeoutInSecondsAnalysis) * time.Second) - for { - select { - case <-done: - a.loading.Stop() - return - case <-timeout: - a.docker.DeleteContainersFromAPI() - a.config.IsTimeout = true - a.loading.Stop() - return - } - } -} - -// detectVulnerabilityFuncs returns a map of language and functions -// that detect vulnerabilities on some path. -// -// All Languages is greater than 15 -//nolint:funlen -func (a *Analyzer) detectVulnerabilityFuncs() map[languages.Language]detectVulnerabilityFn { - return map[languages.Language]detectVulnerabilityFn{ - languages.CSharp: a.detectVulnerabilityCsharp, - languages.Leaks: a.detectVulnerabilityLeaks, - languages.Go: a.detectVulnerabilityGo, - languages.Java: a.detectVulnerabilityJava, - languages.Kotlin: a.detectVulnerabilityKotlin, - languages.Javascript: a.detectVulnerabilityJavascript, - languages.Python: a.detectVulnerabilityPython, - languages.Ruby: a.detectVulnerabilityRuby, - languages.HCL: a.detectVulnerabilityHCL, - languages.Generic: a.detectVulnerabilityGeneric, - languages.Yaml: a.detectVulnerabilityYaml, - languages.C: a.detectVulnerabilityC, - languages.PHP: a.detectVulnerabilityPHP, - languages.Dart: a.detectVulnerabilityDart, - languages.Elixir: a.detectVulnerabilityElixir, - languages.Shell: a.detectVulnerabilityShell, - languages.Nginx: a.detectVulnerabilityNginx, - languages.Swift: a.detectVulneravilitySwift, - } -} - -func (a *Analyzer) detectVulneravilitySwift(_ *sync.WaitGroup, projectSubPath string) error { - horusecswift.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityCsharp(wg *sync.WaitGroup, projectSubPath string) error { - spawn(wg, horuseccsharp.NewFormatter(a.formatter), projectSubPath) - - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.CSharp)); err != nil { - return err - } - - spawn(wg, scs.NewFormatter(a.formatter), projectSubPath) - dotnetcli.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityLeaks(wg *sync.WaitGroup, projectSubPath string) error { - spawn(wg, horusecleaks.NewFormatter(a.formatter), projectSubPath) - - if a.config.EnableGitHistoryAnalysis { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Leaks)); err != nil { - return err - } - gitleaks.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - } - - return nil -} - -func (a *Analyzer) detectVulnerabilityGo(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Go)); err != nil { - return err - } - - spawn(wg, gosec.NewFormatter(a.formatter), projectSubPath) - nancy.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityJava(_ *sync.WaitGroup, projectSubPath string) error { - horusecjava.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityKotlin(_ *sync.WaitGroup, projectSubPath string) error { - horuseckotlin.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityNginx(_ *sync.WaitGroup, projectSubPath string) error { - horusecnginx.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityJavascript(wg *sync.WaitGroup, projectSubPath string) error { - spawn(wg, horusecjavascript.NewFormatter(a.formatter), projectSubPath) - - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Javascript)); err != nil { - return err - } - spawn(wg, yarnaudit.NewFormatter(a.formatter), projectSubPath) - npmaudit.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityPython(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Python)); err != nil { - return err - } - spawn(wg, bandit.NewFormatter(a.formatter), projectSubPath) - safety.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityRuby(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Ruby)); err != nil { - return err - } - spawn(wg, brakeman.NewFormatter(a.formatter), projectSubPath) - bundler.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityHCL(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.HCL)); err != nil { - return err - } - spawn(wg, tfsec.NewFormatter(a.formatter), projectSubPath) - checkov.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityYaml(_ *sync.WaitGroup, projectSubPath string) error { - horuseckubernetes.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityC(_ *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.C)); err != nil { - return err - } - flawfinder.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityPHP(_ *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.PHP)); err != nil { - return err - } - phpcs.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityGeneric(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Generic)); err != nil { - return err - } - - spawn(wg, trivy.NewFormatter(a.formatter), projectSubPath) - spawn(wg, semgrep.NewFormatter(a.formatter), projectSubPath) - dependencycheck.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityDart(_ *sync.WaitGroup, projectSubPath string) error { - horusecdart.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityElixir(wg *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Elixir)); err != nil { - return err - } - spawn(wg, mixaudit.NewFormatter(a.formatter), projectSubPath) - sobelow.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) detectVulnerabilityShell(_ *sync.WaitGroup, projectSubPath string) error { - if err := a.docker.PullImage(a.getCustomOrDefaultImage(languages.Shell)); err != nil { - return err - } - shellcheck.NewFormatter(a.formatter).StartAnalysis(projectSubPath) - return nil -} - -func (a *Analyzer) logProjectSubPath(language languages.Language, subPath string) { - if subPath != "" { - msg := fmt.Sprintf("Running %s in subpath: %s", language.ToString(), subPath) - logger.LogDebugWithLevel(msg) - } -} - // nolint:gocyclo func (a *Analyzer) checkIfNoExistHashAndLog(list []string) { for _, hash := range list { @@ -493,15 +197,6 @@ func (a *Analyzer) setAnalysisError(err error) { } } -func (a *Analyzer) getCustomOrDefaultImage(language languages.Language) string { - // Images can be set to empty on config file, so we need to use only if its not empty. - // If its empty we return the default value. - if customImage := a.config.CustomImages[language]; customImage != "" { - return customImage - } - return path.Join(images.DefaultRegistry, images.MapValues()[language]) -} - // SetFalsePositivesAndRiskAcceptInVulnerabilities set analysis vulnerabilities to false // positive or risk accept if the hash exists on falsePositive and riskAccept params. // @@ -565,11 +260,14 @@ func (a *Analyzer) sortVulnerabilitiesByCriticality() *analysis.Analysis { func (a *Analyzer) sortVulnerabilitiesByType() *analysis.Analysis { analysisVulnerabilities := a.getVulnerabilitiesByType(enumsVulnerability.Vulnerability) analysisVulnerabilities = append(analysisVulnerabilities, - a.getVulnerabilitiesByType(enumsVulnerability.RiskAccepted)...) + a.getVulnerabilitiesByType(enumsVulnerability.RiskAccepted)..., + ) analysisVulnerabilities = append(analysisVulnerabilities, - a.getVulnerabilitiesByType(enumsVulnerability.FalsePositive)...) + a.getVulnerabilitiesByType(enumsVulnerability.FalsePositive)..., + ) analysisVulnerabilities = append(analysisVulnerabilities, - a.getVulnerabilitiesByType(enumsVulnerability.Corrected)...) + a.getVulnerabilitiesByType(enumsVulnerability.Corrected)..., + ) a.analysis.AnalysisVulnerabilities = analysisVulnerabilities return a.analysis } @@ -672,22 +370,3 @@ func (a *Analyzer) removeVulnerabilitiesByTypes() *analysis.Analysis { return a.analysis } - -func spawn(wg *sync.WaitGroup, f formatters.IFormatter, src string) { - wg.Add(1) - go func() { - defer wg.Done() - f.StartAnalysis(src) - }() -} - -func newScanLoading(cfg *config.Config) *spinner.Spinner { - loading := spinner.New(spinner.CharSets[11], LoadingDelay) - loading.Suffix = messages.MsgInfoAnalysisLoading - - if cfg.LogLevel == logrus.DebugLevel.String() || cfg.LogLevel == logrus.TraceLevel.String() { - loading.Writer = io.Discard - } - - return loading -} diff --git a/internal/controllers/analyzer/analyzer_test.go b/internal/controllers/analyzer/analyzer_test.go index 2ceeb8e94..566a61bf8 100644 --- a/internal/controllers/analyzer/analyzer_test.go +++ b/internal/controllers/analyzer/analyzer_test.go @@ -25,7 +25,7 @@ import ( "os" "testing" - entitiesAnalysis "github.com/ZupIT/horusec-devkit/pkg/entities/analysis" + "github.com/ZupIT/horusec-devkit/pkg/entities/analysis" "github.com/ZupIT/horusec-devkit/pkg/entities/cli" "github.com/ZupIT/horusec-devkit/pkg/entities/vulnerability" "github.com/ZupIT/horusec-devkit/pkg/enums/languages" @@ -41,7 +41,6 @@ import ( "github.com/ZupIT/horusec/config" "github.com/ZupIT/horusec/internal/entities/workdir" "github.com/ZupIT/horusec/internal/services/docker" - "github.com/ZupIT/horusec/internal/services/formatters" "github.com/ZupIT/horusec/internal/utils/testutil" vulnhash "github.com/ZupIT/horusec/internal/utils/vuln_hash" ) @@ -56,7 +55,7 @@ func BenchmarkAnalyzerAnalyze(b *testing.B) { cfg := config.New() cfg.ProjectPath = testutil.GoExample - analyzer := NewAnalyzer(cfg) + analyzer := New(cfg) for i := 0; i < b.N; i++ { if _, err := analyzer.Analyze(); err != nil { @@ -110,10 +109,10 @@ func TestAnalyzerSetFalsePositivesAndRiskAcceptInVulnerabilities(t *testing.T) { for _, tt := range testcases { t.Run(tt.name, func(t *testing.T) { - analyzer := NewAnalyzer(config.New()) + analyzer := New(config.New()) analyzer.analysis.AnalysisVulnerabilities = append( - analyzer.analysis.AnalysisVulnerabilities, entitiesAnalysis.AnalysisVulnerabilities{ + analyzer.analysis.AnalysisVulnerabilities, analysis.AnalysisVulnerabilities{ AnalysisID: uuid.New(), Vulnerability: tt.vulnerability, }, @@ -144,7 +143,7 @@ func TestAnalyzerSetFalsePositivesAndRiskAcceptInVulnerabilities(t *testing.T) { func TestNewAnalyzer(t *testing.T) { t.Run("Should return type os struct correctly", func(t *testing.T) { - assert.IsType(t, &Analyzer{}, NewAnalyzer(&config.Config{})) + assert.IsType(t, &Analyzer{}, New(&config.Config{})) }) } @@ -153,7 +152,7 @@ func TestAnalyzerWithoutMock(t *testing.T) { cfg := config.New() cfg.ProjectPath = testutil.GoExample - controller := NewAnalyzer(cfg) + controller := New(cfg) _, err := controller.Analyze() assert.NoError(t, err) }) @@ -181,7 +180,7 @@ func TestAnalyzerWithoutMock(t *testing.T) { cfg.HorusecAPIUri = svr.URL defer svr.Close() - controller := NewAnalyzer(cfg) + controller := New(cfg) _, err := controller.Analyze() assert.NoError(t, err) }) @@ -218,7 +217,7 @@ func TestAnalyze(t *testing.T) { horusecAPIMock := testutil.NewHorusecAPIMock() horusecAPIMock.On("SendAnalysis").Return(nil) - horusecAPIMock.On("GetAnalysis").Return(&entitiesAnalysis.Analysis{}, nil) + horusecAPIMock.On("GetAnalysis").Return(&analysis.Analysis{}, nil) dockerMocker := testutil.NewDockerClientMock() dockerMocker.On("CreateLanguageAnalysisContainer").Return("", nil) @@ -231,19 +230,15 @@ func TestAnalyze(t *testing.T) { dockerMocker.On("ContainerRemove").Return(nil) dockerMocker.On("ContainerList").Return([]types.Container{{ID: "test"}}, nil) - dockerSDK := docker.New(dockerMocker, configs, uuid.New()) - controller := &Analyzer{ - docker: dockerSDK, config: configs, languageDetect: languageDetectMock, printController: printResultMock, horusec: horusecAPIMock, - formatter: formatters.NewFormatterService(&entitiesAnalysis.Analysis{}, dockerSDK, configs), - loading: newScanLoading(configs), + runner: newRunner(configs, new(analysis.Analysis), docker.New(dockerMocker, configs, uuid.New())), } - controller.analysis = &entitiesAnalysis.Analysis{ID: uuid.New()} + controller.analysis = &analysis.Analysis{ID: uuid.New()} totalVulns, err := controller.Analyze() assert.NoError(t, err) assert.Equal(t, 0, totalVulns) @@ -289,19 +284,15 @@ func TestAnalyze(t *testing.T) { dockerMocker.On("ContainerRemove").Return(nil) dockerMocker.On("ContainerList").Return([]types.Container{{ID: "test"}}, nil) - dockerSDK := docker.New(dockerMocker, configs, uuid.New()) - controller := &Analyzer{ - docker: dockerSDK, config: configs, languageDetect: languageDetectMock, printController: printResultMock, horusec: horusecAPIMock, - formatter: formatters.NewFormatterService(&entitiesAnalysis.Analysis{}, dockerSDK, configs), - loading: newScanLoading(configs), + runner: newRunner(configs, new(analysis.Analysis), docker.New(dockerMocker, configs, uuid.New())), } - controller.analysis = &entitiesAnalysis.Analysis{ID: uuid.New()} + controller.analysis = &analysis.Analysis{ID: uuid.New()} totalVulns, err := controller.Analyze() assert.NoError(t, err) assert.Equal(t, 0, totalVulns) @@ -319,7 +310,7 @@ func TestAnalyze(t *testing.T) { horusecAPIMock := testutil.NewHorusecAPIMock() horusecAPIMock.On("SendAnalysis").Return(nil) - horusecAPIMock.On("GetAnalysis").Return(&entitiesAnalysis.Analysis{}, nil) + horusecAPIMock.On("GetAnalysis").Return(&analysis.Analysis{}, nil) dockerMocker := testutil.NewDockerClientMock() dockerMocker.On("CreateLanguageAnalysisContainer").Return("", nil) @@ -332,19 +323,15 @@ func TestAnalyze(t *testing.T) { dockerMocker.On("ContainerRemove").Return(nil) dockerMocker.On("ContainerList").Return([]types.Container{{ID: "test"}}, nil) - dockerSDK := docker.New(dockerMocker, configs, uuid.New()) - controller := &Analyzer{ - docker: dockerSDK, config: configs, languageDetect: languageDetectMock, printController: printResultMock, horusec: horusecAPIMock, - formatter: formatters.NewFormatterService(&entitiesAnalysis.Analysis{}, dockerSDK, configs), - loading: newScanLoading(configs), + runner: newRunner(configs, new(analysis.Analysis), docker.New(dockerMocker, configs, uuid.New())), } - controller.analysis = &entitiesAnalysis.Analysis{ID: uuid.New()} + controller.analysis = &analysis.Analysis{ID: uuid.New()} totalVulns, err := controller.Analyze() assert.Error(t, err) assert.Equal(t, 0, totalVulns) @@ -359,13 +346,11 @@ func TestAnalyze(t *testing.T) { horusecAPI := testutil.NewHorusecAPIMock() horusecAPI.On("SendAnalysis").Return(nil) - horusecAPI.On("GetAnalysis").Return(new(entitiesAnalysis.Analysis), nil) - - docker := docker.New(testutil.NewDockerClientMock(), cfg, uuid.New()) + horusecAPI.On("GetAnalysis").Return(new(analysis.Analysis), nil) - analysis := new(entitiesAnalysis.Analysis) - analysis.AnalysisVulnerabilities = append( - analysis.AnalysisVulnerabilities, entitiesAnalysis.AnalysisVulnerabilities{ + analysiss := new(analysis.Analysis) + analysiss.AnalysisVulnerabilities = append( + analysiss.AnalysisVulnerabilities, analysis.AnalysisVulnerabilities{ Vulnerability: vulnerability.Vulnerability{ Severity: severities.Info, }, @@ -377,19 +362,17 @@ func TestAnalyze(t *testing.T) { pr.On("SetAnalysis") analyzer := &Analyzer{ - docker: docker, config: cfg, languageDetect: ld, printController: pr, horusec: horusecAPI, - formatter: formatters.NewFormatterService(analysis, docker, cfg), - loading: newScanLoading(cfg), - analysis: analysis, + analysis: analysiss, + runner: newRunner(cfg, analysiss, docker.New(testutil.NewDockerClientMock(), cfg, uuid.New())), } _, err := analyzer.Analyze() require.NoError(t, err, "Expected no error to execute analysis") - assert.Len(t, analysis.AnalysisVulnerabilities, 1, "Expected that analysis contains info vulnerabilities") + assert.Len(t, analysiss.AnalysisVulnerabilities, 1, "Expected that analysis contains info vulnerabilities") }) } diff --git a/internal/controllers/analyzer/runner.go b/internal/controllers/analyzer/runner.go new file mode 100644 index 000000000..10043eb95 --- /dev/null +++ b/internal/controllers/analyzer/runner.go @@ -0,0 +1,385 @@ +// Copyright 2020 ZUP IT SERVICOS EM TECNOLOGIA E INOVACAO SA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package analyzer + +import ( + "fmt" + "io" + "os" + "os/signal" + "path" + "path/filepath" + "sync" + "time" + + "github.com/ZupIT/horusec-devkit/pkg/entities/analysis" + "github.com/ZupIT/horusec-devkit/pkg/enums/languages" + "github.com/ZupIT/horusec-devkit/pkg/utils/logger" + "github.com/briandowns/spinner" + "github.com/sirupsen/logrus" + + "github.com/ZupIT/horusec/config" + "github.com/ZupIT/horusec/internal/enums/images" + "github.com/ZupIT/horusec/internal/helpers/messages" + "github.com/ZupIT/horusec/internal/services/docker" + "github.com/ZupIT/horusec/internal/services/formatters" + "github.com/ZupIT/horusec/internal/services/formatters/c/flawfinder" + dotnetcli "github.com/ZupIT/horusec/internal/services/formatters/csharp/dotnet_cli" + "github.com/ZupIT/horusec/internal/services/formatters/csharp/horuseccsharp" + "github.com/ZupIT/horusec/internal/services/formatters/csharp/scs" + "github.com/ZupIT/horusec/internal/services/formatters/dart/horusecdart" + "github.com/ZupIT/horusec/internal/services/formatters/elixir/mixaudit" + "github.com/ZupIT/horusec/internal/services/formatters/elixir/sobelow" + dependencycheck "github.com/ZupIT/horusec/internal/services/formatters/generic/dependency_check" + "github.com/ZupIT/horusec/internal/services/formatters/generic/semgrep" + "github.com/ZupIT/horusec/internal/services/formatters/generic/trivy" + "github.com/ZupIT/horusec/internal/services/formatters/go/gosec" + "github.com/ZupIT/horusec/internal/services/formatters/go/nancy" + "github.com/ZupIT/horusec/internal/services/formatters/hcl/checkov" + "github.com/ZupIT/horusec/internal/services/formatters/hcl/tfsec" + "github.com/ZupIT/horusec/internal/services/formatters/java/horusecjava" + "github.com/ZupIT/horusec/internal/services/formatters/javascript/horusecjavascript" + "github.com/ZupIT/horusec/internal/services/formatters/javascript/npmaudit" + "github.com/ZupIT/horusec/internal/services/formatters/javascript/yarnaudit" + "github.com/ZupIT/horusec/internal/services/formatters/kotlin/horuseckotlin" + "github.com/ZupIT/horusec/internal/services/formatters/leaks/gitleaks" + "github.com/ZupIT/horusec/internal/services/formatters/leaks/horusecleaks" + "github.com/ZupIT/horusec/internal/services/formatters/nginx/horusecnginx" + "github.com/ZupIT/horusec/internal/services/formatters/php/phpcs" + "github.com/ZupIT/horusec/internal/services/formatters/python/bandit" + "github.com/ZupIT/horusec/internal/services/formatters/python/safety" + "github.com/ZupIT/horusec/internal/services/formatters/ruby/brakeman" + "github.com/ZupIT/horusec/internal/services/formatters/ruby/bundler" + "github.com/ZupIT/horusec/internal/services/formatters/shell/shellcheck" + "github.com/ZupIT/horusec/internal/services/formatters/swift/horusecswift" + "github.com/ZupIT/horusec/internal/services/formatters/yaml/horuseckubernetes" +) + +const spinnerLoadingDelay = 200 * time.Millisecond + +// detectVulnerabilityFn is a func that detect vulnerabilities on path. +// detectVulnerabilityFn funcs run all in parallel, so a WaitGroup is required +// to synchronize states of running analysis. +// +// detectVulnerabilityFn funcs can also spawn other detectVulnerabilityFn funcs +// just passing the received WaitGroup to underlying funcs. +// +// Note that the argument path is a work dir path and not the project path, so this +// value can be empty. +type detectVulnerabilityFn func(wg *sync.WaitGroup, path string) error + +// runner is responsible to orchestrate all executions. +// +// For each language founded on project path, runner will run an analysis using +// the appropriate tool. +type runner struct { + loading *spinner.Spinner + config *config.Config + docker docker.Docker + formatter formatters.IService +} + +func newRunner(cfg *config.Config, analysiss *analysis.Analysis, dockerAPI *docker.API) *runner { + return &runner{ + loading: newScanLoading(cfg), + formatter: formatters.NewFormatterService(analysiss, dockerAPI, cfg), + config: cfg, + docker: dockerAPI, + } +} + +// run handle execution of all analysis in parallel +// +// nolint:funlen,gocyclo +func (r *runner) run(langs []languages.Language) []error { + r.removeTrashByInterruptProcess() + defer r.removeHorusecFolder() + + var ( + wg sync.WaitGroup + errors []error + mutex = new(sync.Mutex) + done = make(chan struct{}) + ) + + funcs := r.detectVulnerabilityFuncs() + + r.loading.Start() + + go func() { + defer close(done) + for _, language := range langs { + for _, subPath := range r.config.WorkDir.PathsOfLanguage(language) { + projectSubPath := subPath + r.logProjectSubPath(language, projectSubPath) + + if fn, exist := funcs[language]; exist { + wg.Add(1) + go func() { + defer wg.Done() + if err := fn(&wg, projectSubPath); err != nil { + mutex.Lock() + errors = append(errors, err) + mutex.Unlock() + } + }() + } + } + } + wg.Wait() + }() + + timeout := time.After(time.Duration(r.config.TimeoutInSecondsAnalysis) * time.Second) + for { + select { + case <-done: + r.loading.Stop() + return errors + case <-timeout: + r.docker.DeleteContainersFromAPI() + r.config.IsTimeout = true + r.loading.Stop() + return errors + } + } +} + +// detectVulnerabilityFuncs returns a map of language and a function +// that detect vulnerabilities on some path. +// +//nolint:funlen +func (r *runner) detectVulnerabilityFuncs() map[languages.Language]detectVulnerabilityFn { + return map[languages.Language]detectVulnerabilityFn{ + languages.CSharp: r.detectVulnerabilityCsharp, + languages.Leaks: r.detectVulnerabilityLeaks, + languages.Go: r.detectVulnerabilityGo, + languages.Java: r.detectVulnerabilityJava, + languages.Kotlin: r.detectVulnerabilityKotlin, + languages.Javascript: r.detectVulnerabilityJavascript, + languages.Python: r.detectVulnerabilityPython, + languages.Ruby: r.detectVulnerabilityRuby, + languages.HCL: r.detectVulnerabilityHCL, + languages.Generic: r.detectVulnerabilityGeneric, + languages.Yaml: r.detectVulnerabilityYaml, + languages.C: r.detectVulnerabilityC, + languages.PHP: r.detectVulnerabilityPHP, + languages.Dart: r.detectVulnerabilityDart, + languages.Elixir: r.detectVulnerabilityElixir, + languages.Shell: r.detectVulnerabilityShell, + languages.Nginx: r.detectVulnerabilityNginx, + languages.Swift: r.detectVulneravilitySwift, + } +} + +func (r *runner) detectVulneravilitySwift(_ *sync.WaitGroup, projectSubPath string) error { + horusecswift.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityCsharp(wg *sync.WaitGroup, projectSubPath string) error { + spawn(wg, horuseccsharp.NewFormatter(r.formatter), projectSubPath) + + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.CSharp)); err != nil { + return err + } + + spawn(wg, scs.NewFormatter(r.formatter), projectSubPath) + dotnetcli.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityLeaks(wg *sync.WaitGroup, projectSubPath string) error { + spawn(wg, horusecleaks.NewFormatter(r.formatter), projectSubPath) + + if r.config.EnableGitHistoryAnalysis { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Leaks)); err != nil { + return err + } + gitleaks.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + } + + return nil +} + +func (r *runner) detectVulnerabilityGo(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Go)); err != nil { + return err + } + + spawn(wg, gosec.NewFormatter(r.formatter), projectSubPath) + nancy.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityJava(_ *sync.WaitGroup, projectSubPath string) error { + horusecjava.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityKotlin(_ *sync.WaitGroup, projectSubPath string) error { + horuseckotlin.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityNginx(_ *sync.WaitGroup, projectSubPath string) error { + horusecnginx.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityJavascript(wg *sync.WaitGroup, projectSubPath string) error { + spawn(wg, horusecjavascript.NewFormatter(r.formatter), projectSubPath) + + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Javascript)); err != nil { + return err + } + spawn(wg, yarnaudit.NewFormatter(r.formatter), projectSubPath) + npmaudit.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityPython(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Python)); err != nil { + return err + } + spawn(wg, bandit.NewFormatter(r.formatter), projectSubPath) + safety.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityRuby(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Ruby)); err != nil { + return err + } + spawn(wg, brakeman.NewFormatter(r.formatter), projectSubPath) + bundler.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityHCL(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.HCL)); err != nil { + return err + } + spawn(wg, tfsec.NewFormatter(r.formatter), projectSubPath) + checkov.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityYaml(_ *sync.WaitGroup, projectSubPath string) error { + horuseckubernetes.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityC(_ *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.C)); err != nil { + return err + } + flawfinder.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityPHP(_ *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.PHP)); err != nil { + return err + } + phpcs.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityGeneric(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Generic)); err != nil { + return err + } + + spawn(wg, trivy.NewFormatter(r.formatter), projectSubPath) + spawn(wg, semgrep.NewFormatter(r.formatter), projectSubPath) + dependencycheck.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityDart(_ *sync.WaitGroup, projectSubPath string) error { + horusecdart.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityElixir(wg *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Elixir)); err != nil { + return err + } + spawn(wg, mixaudit.NewFormatter(r.formatter), projectSubPath) + sobelow.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) detectVulnerabilityShell(_ *sync.WaitGroup, projectSubPath string) error { + if err := r.docker.PullImage(r.getCustomOrDefaultImage(languages.Shell)); err != nil { + return err + } + shellcheck.NewFormatter(r.formatter).StartAnalysis(projectSubPath) + return nil +} + +func (r *runner) getCustomOrDefaultImage(language languages.Language) string { + // Images can be set to empty on config file, so we need to use only if its not empty. + // If its empty we return the default value. + if customImage := r.config.CustomImages[language]; customImage != "" { + return customImage + } + return path.Join(images.DefaultRegistry, images.MapValues()[language]) +} + +func (r *runner) logProjectSubPath(language languages.Language, subPath string) { + if subPath != "" { + msg := fmt.Sprintf("Running %s in subpath: %s", language.ToString(), subPath) + logger.LogDebugWithLevel(msg) + } +} + +func (r *runner) removeHorusecFolder() { + err := os.RemoveAll(filepath.Join(r.config.ProjectPath, ".horusec")) + logger.LogErrorWithLevel(messages.MsgErrorRemoveAnalysisFolder, err) + if !r.config.DisableDocker { + r.docker.DeleteContainersFromAPI() + } +} + +func (r *runner) removeTrashByInterruptProcess() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + for range c { + r.removeHorusecFolder() + os.Exit(1) + } + }() +} + +func spawn(wg *sync.WaitGroup, f formatters.IFormatter, src string) { + wg.Add(1) + go func() { + defer wg.Done() + f.StartAnalysis(src) + }() +} + +func newScanLoading(cfg *config.Config) *spinner.Spinner { + loading := spinner.New(spinner.CharSets[11], spinnerLoadingDelay) + loading.Suffix = messages.MsgInfoAnalysisLoading + + if cfg.LogLevel == logrus.DebugLevel.String() || cfg.LogLevel == logrus.TraceLevel.String() { + loading.Writer = io.Discard + } + + return loading +}