From a74fc7ab27db6a01004f3ddbab5d800196a1a91b Mon Sep 17 00:00:00 2001 From: Szabolcs Toth <54896607+tothszabi@users.noreply.github.com> Date: Wed, 10 Jan 2024 09:25:59 +0000 Subject: [PATCH] [CI-2284] Add configuration hash to the xml data (#190) --- .../xcresult3/action_test_summary.go | 23 ++ .../xcresult3/action_test_summary_group.go | 5 +- test/converters/xcresult3/converter.go | 29 ++- test/junit/xml.go | 17 +- test/test_test.go | 47 ++++- .../ios_device_config_xml_output.golden | 50 +++++ test/testdata/ios_xml_output.golden | 42 ++-- .../bitrise-io/go-utils/v2/command/command.go | 198 ++++++++++++++++++ .../go-utils/v2/command/errorcollector.go | 18 ++ vendor/modules.txt | 1 + 10 files changed, 388 insertions(+), 42 deletions(-) create mode 100644 test/testdata/ios_device_config_xml_output.golden create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/command/command.go create mode 100644 vendor/github.com/bitrise-io/go-utils/v2/command/errorcollector.go diff --git a/test/converters/xcresult3/action_test_summary.go b/test/converters/xcresult3/action_test_summary.go index 0a5d26a3..8745d56d 100644 --- a/test/converters/xcresult3/action_test_summary.go +++ b/test/converters/xcresult3/action_test_summary.go @@ -1,5 +1,10 @@ package xcresult3 +import ( + "crypto/md5" + "encoding/hex" +) + // Attachment ... type Attachment struct { Filename struct { @@ -48,8 +53,26 @@ type FailureSummaries struct { Values []ActionTestFailureSummary `json:"_values"` } +// Configuration ... +type Configuration struct { + Hash string +} + +// UnmarshalJSON ... +func (c *Configuration) UnmarshalJSON(data []byte) error { + if string(data) == "null" || string(data) == `""` { + return nil + } + + hash := md5.Sum(data) + c.Hash = hex.EncodeToString(hash[:]) + + return nil +} + // ActionTestSummary ... type ActionTestSummary struct { ActivitySummaries ActivitySummaries `json:"activitySummaries"` FailureSummaries FailureSummaries `json:"failureSummaries"` + Configuration Configuration `json:"configuration"` } diff --git a/test/converters/xcresult3/action_test_summary_group.go b/test/converters/xcresult3/action_test_summary_group.go index 671b5cc8..eb177481 100644 --- a/test/converters/xcresult3/action_test_summary_group.go +++ b/test/converters/xcresult3/action_test_summary_group.go @@ -7,6 +7,9 @@ import ( "strings" ) +// ErrSummaryNotFound ... +var ErrSummaryNotFound = errors.New("no summaryRef.ID.Value found for test case") + // ActionTestSummaryGroup ... type ActionTestSummaryGroup struct { Name Name `json:"name"` @@ -73,7 +76,7 @@ func (g ActionTestSummaryGroup) testsWithStatus() (tests []ActionTestSummaryGrou // loadActionTestSummary ... func (g ActionTestSummaryGroup) loadActionTestSummary(xcresultPath string) (ActionTestSummary, error) { if g.SummaryRef.ID.Value == "" { - return ActionTestSummary{}, errors.New("no summaryRef.ID.Value found for test case") + return ActionTestSummary{}, ErrSummaryNotFound } var summary ActionTestSummary diff --git a/test/converters/xcresult3/converter.go b/test/converters/xcresult3/converter.go index bc7f8a34..64c5822f 100644 --- a/test/converters/xcresult3/converter.go +++ b/test/converters/xcresult3/converter.go @@ -1,6 +1,7 @@ package xcresult3 import ( + "errors" "fmt" "path/filepath" "runtime" @@ -9,12 +10,13 @@ import ( "sync" "time" + "howett.net/plist" + "github.com/bitrise-io/go-utils/fileutil" "github.com/bitrise-io/go-utils/log" "github.com/bitrise-io/go-utils/pathutil" "github.com/bitrise-io/go-xcode/xcodeproject/serialized" "github.com/bitrise-steplib/steps-deploy-to-bitrise-io/test/junit" - "howett.net/plist" ) // Converter ... @@ -200,15 +202,19 @@ func genTestCase(test ActionTestSummaryGroup, xcresultPath, testResultDir string } } + testSummary, err := test.loadActionTestSummary(xcresultPath) + // Ignoring the SummaryNotFoundError error is on purpose because not having an action summary is a valid use case. + // For example, failed tests will always have a summary, but successful ones might have it or might not. + // If they do not have it, then that means that they did not log anything to the console, + // and they were not executed as device configuration tests. + if err != nil && !errors.Is(err, ErrSummaryNotFound) { + return junit.TestCase{}, err + } + var failure *junit.Failure var skipped *junit.Skipped switch test.TestStatus.Value { case "Failure": - testSummary, err := test.loadActionTestSummary(xcresultPath) - if err != nil { - return junit.TestCase{}, err - } - failureMessage := "" for _, aTestFailureSummary := range testSummary.FailureSummaries.Values { file := aTestFailureSummary.FileName.Value @@ -233,10 +239,11 @@ func genTestCase(test ActionTestSummaryGroup, xcresultPath, testResultDir string } return junit.TestCase{ - Name: test.Name.Value, - ClassName: strings.Split(test.Identifier.Value, "/")[0], - Failure: failure, - Skipped: skipped, - Time: duartion, + Name: test.Name.Value, + ConfigurationHash: testSummary.Configuration.Hash, + ClassName: strings.Split(test.Identifier.Value, "/")[0], + Failure: failure, + Skipped: skipped, + Time: duartion, }, nil } diff --git a/test/junit/xml.go b/test/junit/xml.go index b57253e1..7416c62a 100644 --- a/test/junit/xml.go +++ b/test/junit/xml.go @@ -24,14 +24,15 @@ type TestSuite struct { // TestCase ... type TestCase struct { - XMLName xml.Name `xml:"testcase"` - Name string `xml:"name,attr"` - ClassName string `xml:"classname,attr"` - Time float64 `xml:"time,attr"` - Failure *Failure `xml:"failure,omitempty"` - Skipped *Skipped `xml:"skipped,omitempty"` - Error *Error `xml:"error,omitempty"` - SystemErr string `xml:"system-err,omitempty"` + XMLName xml.Name `xml:"testcase"` + ConfigurationHash string `xml:"configuration-hash,attr"` + Name string `xml:"name,attr"` + ClassName string `xml:"classname,attr"` + Time float64 `xml:"time,attr"` + Failure *Failure `xml:"failure,omitempty"` + Skipped *Skipped `xml:"skipped,omitempty"` + Error *Error `xml:"error,omitempty"` + SystemErr string `xml:"system-err,omitempty"` } // Failure ... diff --git a/test/test_test.go b/test/test_test.go index 45dfd309..79039f2e 100644 --- a/test/test_test.go +++ b/test/test_test.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "net/http" "os" + "path" "path/filepath" "testing" "time" @@ -12,9 +13,12 @@ import ( "github.com/bitrise-io/bitrise/models" "github.com/bitrise-io/go-utils/fileutil" "github.com/bitrise-io/go-utils/pathutil" + "github.com/bitrise-io/go-utils/v2/command" + "github.com/bitrise-io/go-utils/v2/env" "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func createDummyFilesInDirWithContent(dir, content string, fileNames []string) error { @@ -158,7 +162,7 @@ func Test_Upload(t *testing.T) { } } -func Test_ParseTestResults(t *testing.T) { +func Test_ParseXctestResults(t *testing.T) { sampleTestSummariesPlist, err := fileutil.ReadStringFromFile(filepath.Join("testdata", "ios_testsummaries_plist.golden")) if err != nil { t.Fatal("unable to read golden file, error:", err) @@ -270,3 +274,44 @@ func Test_ParseTestResults(t *testing.T) { assert.Equal(t, sampleIOSXmlOutput, string(bundle[0].XMLContent)) } } + +func Test_ParseXctest3Results(t *testing.T) { + tmpDir := t.TempDir() + gitDir := path.Join(tmpDir, "git") + + // The xcresult3 format has many small encoded binary files, so it is better to use a real xcresult file. + // We are storing these in the sample-artifacts git repo. + cmd := command.NewFactory(env.NewRepository()).Create("git", []string{"clone", "--depth", "1", "https://github.com/bitrise-io/sample-artifacts.git", gitDir}, nil) + err := cmd.Run() + require.NoError(t, err) + + testDir := path.Join(tmpDir, "tests") + testResultDir := path.Join(testDir, "test-result") + err = os.MkdirAll(testDir, os.ModePerm) + require.NoError(t, err) + + phaseDir := path.Join(testResultDir, "phase") + err = os.MkdirAll(testDir, os.ModePerm) + require.NoError(t, err) + + if err := createDummyFilesInDirWithContent(testResultDir, `{"title": "test title"}`, []string{"step-info.json"}); err != nil { + t.Fatal("failed to create dummy files in dir, error:", err) + } + if err := createDummyFilesInDirWithContent(phaseDir, `{"name": "test name"}`, []string{"test-info.json"}); err != nil { + t.Fatal("failed to create dummy files in dir, error:", err) + } + + oldDir := path.Join(gitDir, "xcresults", "xcresult3-device-configuration-tests.xcresult") + newDir := path.Join(phaseDir, "xcresult3-device-configuration-tests.xcresult") + copyCmd := command.NewFactory(env.NewRepository()).Create("cp", []string{"-a", oldDir, newDir}, nil) + err = copyCmd.Run() + require.NoError(t, err) + + bundle, err := ParseTestResults(testDir) + require.NoError(t, err) + + want, err := fileutil.ReadStringFromFile(filepath.Join("testdata", "ios_device_config_xml_output.golden")) + + assert.Equal(t, 1, len(bundle)) + assert.Equal(t, want, string(bundle[0].XMLContent)) +} diff --git a/test/testdata/ios_device_config_xml_output.golden b/test/testdata/ios_device_config_xml_output.golden new file mode 100644 index 00000000..08dd3f3d --- /dev/null +++ b/test/testdata/ios_device_config_xml_output.golden @@ -0,0 +1,50 @@ + + + + + /Users/vagrant/git/DarkAndLightModeTests/DarkAndLightModeTests.swift:25 - failed - Reached 1 + + + + + + + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + /Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed + + + \ No newline at end of file diff --git a/test/testdata/ios_xml_output.golden b/test/testdata/ios_xml_output.golden index 4ec1638d..7b7c6e3c 100644 --- a/test/testdata/ios_xml_output.golden +++ b/test/testdata/ios_xml_output.golden @@ -1,40 +1,40 @@ - - - - + + + + - + - - + + - + /Users/vagrant/git/BitriseApp/BitriseAppUITests/BitriseOMUITests.swift:70 - XCTAssertTrue failed - No session found - - + + - - + + - - - - - - - - - + + + + + + + + + \ No newline at end of file diff --git a/vendor/github.com/bitrise-io/go-utils/v2/command/command.go b/vendor/github.com/bitrise-io/go-utils/v2/command/command.go new file mode 100644 index 00000000..4206c2b3 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/command/command.go @@ -0,0 +1,198 @@ +package command + +import ( + "errors" + "fmt" + "io" + "os/exec" + "strconv" + "strings" + + "github.com/bitrise-io/go-utils/v2/env" +) + +// ErrorFinder ... +type ErrorFinder func(out string) []string + +// Opts ... +type Opts struct { + Stdout io.Writer + Stderr io.Writer + Stdin io.Reader + Env []string + Dir string + ErrorFinder ErrorFinder +} + +// Factory ... +type Factory interface { + Create(name string, args []string, opts *Opts) Command +} + +type factory struct { + envRepository env.Repository +} + +// NewFactory ... +func NewFactory(envRepository env.Repository) Factory { + return factory{envRepository: envRepository} +} + +// Create ... +func (f factory) Create(name string, args []string, opts *Opts) Command { + cmd := exec.Command(name, args...) + var collector *errorCollector + + if opts != nil { + if opts.ErrorFinder != nil { + collector = &errorCollector{errorFinder: opts.ErrorFinder} + } + + cmd.Stdout = opts.Stdout + cmd.Stderr = opts.Stderr + cmd.Stdin = opts.Stdin + + // If Env is nil, the new process uses the current process's + // environment. + // If we pass env vars we want to append them to the + // current process's environment. + cmd.Env = append(f.envRepository.List(), opts.Env...) + cmd.Dir = opts.Dir + } + return &command{ + cmd: cmd, + errorCollector: collector, + } +} + +// Command ... +type Command interface { + PrintableCommandArgs() string + Run() error + RunAndReturnExitCode() (int, error) + RunAndReturnTrimmedOutput() (string, error) + RunAndReturnTrimmedCombinedOutput() (string, error) + Start() error + Wait() error +} + +type command struct { + cmd *exec.Cmd + errorCollector *errorCollector +} + +// PrintableCommandArgs ... +func (c command) PrintableCommandArgs() string { + return printableCommandArgs(false, c.cmd.Args) +} + +// Run ... +func (c *command) Run() error { + c.wrapOutputs() + + if err := c.cmd.Run(); err != nil { + return c.wrapError(err) + } + + return nil +} + +// RunAndReturnExitCode ... +func (c command) RunAndReturnExitCode() (int, error) { + c.wrapOutputs() + err := c.cmd.Run() + if err != nil { + err = c.wrapError(err) + } + + exitCode := c.cmd.ProcessState.ExitCode() + return exitCode, err +} + +// RunAndReturnTrimmedOutput ... +func (c command) RunAndReturnTrimmedOutput() (string, error) { + outBytes, err := c.cmd.Output() + outStr := string(outBytes) + if err != nil { + if c.errorCollector != nil { + c.errorCollector.collectErrors(outStr) + } + err = c.wrapError(err) + } + + return strings.TrimSpace(outStr), err +} + +// RunAndReturnTrimmedCombinedOutput ... +func (c command) RunAndReturnTrimmedCombinedOutput() (string, error) { + outBytes, err := c.cmd.CombinedOutput() + outStr := string(outBytes) + if err != nil { + if c.errorCollector != nil { + c.errorCollector.collectErrors(outStr) + } + err = c.wrapError(err) + } + + return strings.TrimSpace(outStr), err +} + +// Start ... +func (c command) Start() error { + c.wrapOutputs() + return c.cmd.Start() +} + +// Wait ... +func (c command) Wait() error { + err := c.cmd.Wait() + if err != nil { + err = c.wrapError(err) + } + + return err +} + +func printableCommandArgs(isQuoteFirst bool, fullCommandArgs []string) string { + var cmdArgsDecorated []string + for idx, anArg := range fullCommandArgs { + quotedArg := strconv.Quote(anArg) + if idx == 0 && !isQuoteFirst { + quotedArg = anArg + } + cmdArgsDecorated = append(cmdArgsDecorated, quotedArg) + } + + return strings.Join(cmdArgsDecorated, " ") +} + +func (c command) wrapError(err error) error { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + if c.errorCollector != nil && len(c.errorCollector.errorLines) > 0 { + return fmt.Errorf("command failed with exit status %d (%s): %w", exitErr.ExitCode(), c.PrintableCommandArgs(), errors.New(strings.Join(c.errorCollector.errorLines, "\n"))) + } + return fmt.Errorf("command failed with exit status %d (%s): %w", exitErr.ExitCode(), c.PrintableCommandArgs(), errors.New("check the command's output for details")) + } + return fmt.Errorf("executing command failed (%s): %w", c.PrintableCommandArgs(), err) +} + +func (c command) wrapOutputs() { + if c.errorCollector == nil { + return + } + + if c.cmd.Stdout != nil { + outWriter := io.MultiWriter(c.errorCollector, c.cmd.Stdout) + c.cmd.Stdout = outWriter + } else { + c.cmd.Stdout = c.errorCollector + } + + if c.cmd.Stderr != nil { + errWriter := io.MultiWriter(c.errorCollector, c.cmd.Stderr) + c.cmd.Stderr = errWriter + } else { + c.cmd.Stderr = c.errorCollector + } +} diff --git a/vendor/github.com/bitrise-io/go-utils/v2/command/errorcollector.go b/vendor/github.com/bitrise-io/go-utils/v2/command/errorcollector.go new file mode 100644 index 00000000..945e3ff2 --- /dev/null +++ b/vendor/github.com/bitrise-io/go-utils/v2/command/errorcollector.go @@ -0,0 +1,18 @@ +package command + +type errorCollector struct { + errorLines []string + errorFinder ErrorFinder +} + +func (e *errorCollector) Write(p []byte) (n int, err error) { + e.collectErrors(string(p)) + return len(p), nil +} + +func (e *errorCollector) collectErrors(output string) { + lines := e.errorFinder(output) + if len(lines) > 0 { + e.errorLines = append(e.errorLines, lines...) + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index f4754a6c..94cbf0cc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -41,6 +41,7 @@ github.com/bitrise-io/go-utils/urlutil github.com/bitrise-io/go-utils/ziputil # github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.19 ## explicit; go 1.17 +github.com/bitrise-io/go-utils/v2/command github.com/bitrise-io/go-utils/v2/env github.com/bitrise-io/go-utils/v2/errorutil github.com/bitrise-io/go-utils/v2/exitcode