[CI-2284] Add configuration hash to the xml data (#190)

bitrise-steplib · Jan 10, 2024 · a74fc7a · a74fc7a
1 parent e630e3e
commit a74fc7a
Show file tree

Hide file tree

Showing 10 changed files with 388 additions and 42 deletions.
diff --git a/test/converters/xcresult3/action_test_summary.go b/test/converters/xcresult3/action_test_summary.go
@@ -1,5 +1,10 @@
 package xcresult3
 
+import (
+	"crypto/md5"
+	"encoding/hex"
+)
+
 // Attachment ...
 type Attachment struct {
 	Filename struct {
@@ -48,8 +53,26 @@ type FailureSummaries struct {
 	Values []ActionTestFailureSummary `json:"_values"`
 }
 
+// Configuration ...
+type Configuration struct {
+	Hash string
+}
+
+// UnmarshalJSON ...
+func (c *Configuration) UnmarshalJSON(data []byte) error {
+	if string(data) == "null" || string(data) == `""` {
+		return nil
+	}
+
+	hash := md5.Sum(data)
+	c.Hash = hex.EncodeToString(hash[:])
+
+	return nil
+}
+
 // ActionTestSummary ...
 type ActionTestSummary struct {
 	ActivitySummaries ActivitySummaries `json:"activitySummaries"`
 	FailureSummaries  FailureSummaries  `json:"failureSummaries"`
+	Configuration     Configuration     `json:"configuration"`
 }
diff --git a/test/converters/xcresult3/action_test_summary_group.go b/test/converters/xcresult3/action_test_summary_group.go
@@ -7,6 +7,9 @@ import (
 	"strings"
 )
 
+// ErrSummaryNotFound ...
+var ErrSummaryNotFound = errors.New("no summaryRef.ID.Value found for test case")
+
 // ActionTestSummaryGroup ...
 type ActionTestSummaryGroup struct {
 	Name       Name       `json:"name"`
@@ -73,7 +76,7 @@ func (g ActionTestSummaryGroup) testsWithStatus() (tests []ActionTestSummaryGrou
 // loadActionTestSummary ...
 func (g ActionTestSummaryGroup) loadActionTestSummary(xcresultPath string) (ActionTestSummary, error) {
 	if g.SummaryRef.ID.Value == "" {
-		return ActionTestSummary{}, errors.New("no summaryRef.ID.Value found for test case")
+		return ActionTestSummary{}, ErrSummaryNotFound
 	}
 
 	var summary ActionTestSummary

diff --git a/test/converters/xcresult3/converter.go b/test/converters/xcresult3/converter.go
@@ -1,6 +1,7 @@
 package xcresult3
 
 import (
+	"errors"
 	"fmt"
 	"path/filepath"
 	"runtime"
@@ -9,12 +10,13 @@ import (
 	"sync"
 	"time"
 
+	"howett.net/plist"
+
 	"github.com/bitrise-io/go-utils/fileutil"
 	"github.com/bitrise-io/go-utils/log"
 	"github.com/bitrise-io/go-utils/pathutil"
 	"github.com/bitrise-io/go-xcode/xcodeproject/serialized"
 	"github.com/bitrise-steplib/steps-deploy-to-bitrise-io/test/junit"
-	"howett.net/plist"
 )
 
 // Converter ...
@@ -200,15 +202,19 @@ func genTestCase(test ActionTestSummaryGroup, xcresultPath, testResultDir string
 		}
 	}
 
+	testSummary, err := test.loadActionTestSummary(xcresultPath)
+	// Ignoring the SummaryNotFoundError error is on purpose because not having an action summary is a valid use case.
+	// For example, failed tests will always have a summary, but successful ones might have it or might not.
+	// If they do not have it, then that means that they did not log anything to the console,
+	// and they were not executed as device configuration tests.
+	if err != nil && !errors.Is(err, ErrSummaryNotFound) {
+		return junit.TestCase{}, err
+	}
+
 	var failure *junit.Failure
 	var skipped *junit.Skipped
 	switch test.TestStatus.Value {
 	case "Failure":
-		testSummary, err := test.loadActionTestSummary(xcresultPath)
-		if err != nil {
-			return junit.TestCase{}, err
-		}
-
 		failureMessage := ""
 		for _, aTestFailureSummary := range testSummary.FailureSummaries.Values {
 			file := aTestFailureSummary.FileName.Value
@@ -233,10 +239,11 @@ func genTestCase(test ActionTestSummaryGroup, xcresultPath, testResultDir string
 	}
 
 	return junit.TestCase{
-		Name:      test.Name.Value,
-		ClassName: strings.Split(test.Identifier.Value, "/")[0],
-		Failure:   failure,
-		Skipped:   skipped,
-		Time:      duartion,
+		Name:              test.Name.Value,
+		ConfigurationHash: testSummary.Configuration.Hash,
+		ClassName:         strings.Split(test.Identifier.Value, "/")[0],
+		Failure:           failure,
+		Skipped:           skipped,
+		Time:              duartion,
 	}, nil
 }
diff --git a/test/junit/xml.go b/test/junit/xml.go
@@ -24,14 +24,15 @@ type TestSuite struct {
 
 // TestCase ...
 type TestCase struct {
-	XMLName   xml.Name `xml:"testcase"`
-	Name      string   `xml:"name,attr"`
-	ClassName string   `xml:"classname,attr"`
-	Time      float64  `xml:"time,attr"`
-	Failure   *Failure `xml:"failure,omitempty"`
-	Skipped   *Skipped `xml:"skipped,omitempty"`
-	Error     *Error   `xml:"error,omitempty"`
-	SystemErr string   `xml:"system-err,omitempty"`
+	XMLName           xml.Name `xml:"testcase"`
+	ConfigurationHash string   `xml:"configuration-hash,attr"`
+	Name              string   `xml:"name,attr"`
+	ClassName         string   `xml:"classname,attr"`
+	Time              float64  `xml:"time,attr"`
+	Failure           *Failure `xml:"failure,omitempty"`
+	Skipped           *Skipped `xml:"skipped,omitempty"`
+	Error             *Error   `xml:"error,omitempty"`
+	SystemErr         string   `xml:"system-err,omitempty"`
 }
 
 // Failure ...

diff --git a/test/test_test.go b/test/test_test.go
@@ -5,16 +5,20 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"path"
 	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/bitrise-io/bitrise/models"
 	"github.com/bitrise-io/go-utils/fileutil"
 	"github.com/bitrise-io/go-utils/pathutil"
+	"github.com/bitrise-io/go-utils/v2/command"
+	"github.com/bitrise-io/go-utils/v2/env"
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func createDummyFilesInDirWithContent(dir, content string, fileNames []string) error {
@@ -158,7 +162,7 @@ func Test_Upload(t *testing.T) {
 	}
 }
 
-func Test_ParseTestResults(t *testing.T) {
+func Test_ParseXctestResults(t *testing.T) {
 	sampleTestSummariesPlist, err := fileutil.ReadStringFromFile(filepath.Join("testdata", "ios_testsummaries_plist.golden"))
 	if err != nil {
 		t.Fatal("unable to read golden file, error:", err)
@@ -270,3 +274,44 @@ func Test_ParseTestResults(t *testing.T) {
 		assert.Equal(t, sampleIOSXmlOutput, string(bundle[0].XMLContent))
 	}
 }
+
+func Test_ParseXctest3Results(t *testing.T) {
+	tmpDir := t.TempDir()
+	gitDir := path.Join(tmpDir, "git")
+
+	// The xcresult3 format has many small encoded binary files, so it is better to use a real xcresult file.
+	// We are storing these in the sample-artifacts git repo.
+	cmd := command.NewFactory(env.NewRepository()).Create("git", []string{"clone", "--depth", "1", "https://github.com/bitrise-io/sample-artifacts.git", gitDir}, nil)
+	err := cmd.Run()
+	require.NoError(t, err)
+
+	testDir := path.Join(tmpDir, "tests")
+	testResultDir := path.Join(testDir, "test-result")
+	err = os.MkdirAll(testDir, os.ModePerm)
+	require.NoError(t, err)
+
+	phaseDir := path.Join(testResultDir, "phase")
+	err = os.MkdirAll(testDir, os.ModePerm)
+	require.NoError(t, err)
+
+	if err := createDummyFilesInDirWithContent(testResultDir, `{"title": "test title"}`, []string{"step-info.json"}); err != nil {
+		t.Fatal("failed to create dummy files in dir, error:", err)
+	}
+	if err := createDummyFilesInDirWithContent(phaseDir, `{"name": "test name"}`, []string{"test-info.json"}); err != nil {
+		t.Fatal("failed to create dummy files in dir, error:", err)
+	}
+
+	oldDir := path.Join(gitDir, "xcresults", "xcresult3-device-configuration-tests.xcresult")
+	newDir := path.Join(phaseDir, "xcresult3-device-configuration-tests.xcresult")
+	copyCmd := command.NewFactory(env.NewRepository()).Create("cp", []string{"-a", oldDir, newDir}, nil)
+	err = copyCmd.Run()
+	require.NoError(t, err)
+
+	bundle, err := ParseTestResults(testDir)
+	require.NoError(t, err)
+
+	want, err := fileutil.ReadStringFromFile(filepath.Join("testdata", "ios_device_config_xml_output.golden"))
+
+	assert.Equal(t, 1, len(bundle))
+	assert.Equal(t, want, string(bundle[0].XMLContent))
+}
diff --git a/test/testdata/ios_device_config_xml_output.golden b/test/testdata/ios_device_config_xml_output.golden
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+ <testsuite name="DarkAndLightModeTests" tests="3" failures="1" skipped="0" errors="0" time="0.3750969171524048">
+  <testcase configuration-hash="" name="testExample()" classname="DarkAndLightModeTests" time="0.11242496967315674">
+   <failure>/Users/vagrant/git/DarkAndLightModeTests/DarkAndLightModeTests.swift:25 - failed - Reached 1</failure>
+  </testcase>
+  <testcase configuration-hash="" name="testExample()" classname="DarkAndLightModeTests" time="0.0011119842529296875"></testcase>
+  <testcase configuration-hash="" name="testPerformanceExample()" classname="DarkAndLightModeTests" time="0.26155996322631836"></testcase>
+ </testsuite>
+ <testsuite name="DarkAndLightModeUITests" tests="14" failures="12" skipped="0" errors="0" time="92.95631325244904">
+  <testcase configuration-hash="" name="testExample()" classname="DarkAndLightModeUITests" time="3.8267470598220825"></testcase>
+  <testcase configuration-hash="" name="testLaunchPerformance()" classname="DarkAndLightModeUITests" time="28.429319024086"></testcase>
+  <testcase configuration-hash="16789993f7012553276a039f8829bcbf" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="7.017634034156799">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="16789993f7012553276a039f8829bcbf" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="4.156491994857788">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="16789993f7012553276a039f8829bcbf" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="4.570378065109253">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="bb9ffdfb3f0e2ac59b09a83bd8c1ba55" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="7.531908988952637">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="bb9ffdfb3f0e2ac59b09a83bd8c1ba55" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="4.387096047401428">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="bb9ffdfb3f0e2ac59b09a83bd8c1ba55" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="4.1682270765304565">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="990bacacade8f7dc5b2d9f4d042c3349" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="7.013818025588989">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="990bacacade8f7dc5b2d9f4d042c3349" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="4.297332048416138">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="990bacacade8f7dc5b2d9f4d042c3349" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="4.593717932701111">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="09150042f5350c48cdf9784faaa4b911" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="4.915426015853882">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="09150042f5350c48cdf9784faaa4b911" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="4.210652947425842">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+  <testcase configuration-hash="09150042f5350c48cdf9784faaa4b911" name="testLaunch()" classname="DarkAndLightModeUITestsLaunchTests" time="3.837563991546631">
+   <failure>/Users/vagrant/git/DarkAndLightModeUITests/DarkAndLightModeUITestsLaunchTests.swift:32 - XCTAssertTrue failed</failure>
+  </testcase>
+ </testsuite>
+</testsuites>
diff --git a/test/testdata/ios_xml_output.golden b/test/testdata/ios_xml_output.golden
@@ -1,40 +1,40 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuites>
  <testsuite name="BitriseBasicUITest" tests="4" failures="0" skipped="0" errors="0" time="77.06086790561676">
-  <testcase name="testCollectionView()" classname="BitriseBasicUITest" time="23.776288986206055"></testcase>
-  <testcase name="testInReadTopView()" classname="BitriseBasicUITest" time="34.97628891468048"></testcase>
-  <testcase name="testInterstitialView()" classname="BitriseBasicUITest" time="10.270301938056946"></testcase>
-  <testcase name="testRewardView()" classname="BitriseBasicUITest" time="8.037988066673279"></testcase>
+  <testcase configuration-hash="" name="testCollectionView()" classname="BitriseBasicUITest" time="23.776288986206055"></testcase>
+  <testcase configuration-hash="" name="testInReadTopView()" classname="BitriseBasicUITest" time="34.97628891468048"></testcase>
+  <testcase configuration-hash="" name="testInterstitialView()" classname="BitriseBasicUITest" time="10.270301938056946"></testcase>
+  <testcase configuration-hash="" name="testRewardView()" classname="BitriseBasicUITest" time="8.037988066673279"></testcase>
  </testsuite>
  <testsuite name="BitriseCommanderUITests" tests="1" failures="0" skipped="1" errors="0" time="40.58690905570984">
-  <testcase name="testCommanderUpdate()" classname="BitriseCommanderUITests" time="40.58690905570984">
+  <testcase configuration-hash="" name="testCommanderUpdate()" classname="BitriseCommanderUITests" time="40.58690905570984">
    <skipped></skipped>
   </testcase>
  </testsuite>
  <testsuite name="BitriseComponentUITest" tests="2" failures="0" skipped="0" errors="0" time="33.92029893398285">
-  <testcase name="testAdChoiceIcon()" classname="BitriseComponentUITest" time="15.12886095046997"></testcase>
-  <testcase name="testFullScreenReplay()" classname="BitriseComponentUITest" time="18.79143798351288"></testcase>
+  <testcase configuration-hash="" name="testAdChoiceIcon()" classname="BitriseComponentUITest" time="15.12886095046997"></testcase>
+  <testcase configuration-hash="" name="testFullScreenReplay()" classname="BitriseComponentUITest" time="18.79143798351288"></testcase>
  </testsuite>
  <testsuite name="BitriseOMUITests" tests="3" failures="1" skipped="0" errors="0" time="80.687903881073">
-  <testcase name="testOMBanner()" classname="BitriseOMUITests" time="15.550718903541565">
+  <testcase configuration-hash="" name="testOMBanner()" classname="BitriseOMUITests" time="15.550718903541565">
    <failure>/Users/vagrant/git/BitriseApp/BitriseAppUITests/BitriseOMUITests.swift:70 - XCTAssertTrue failed - No session found</failure>
   </testcase>
-  <testcase name="testOMInterstitial()" classname="BitriseOMUITests" time="26.03164303302765"></testcase>
-  <testcase name="testOMScrollView()" classname="BitriseOMUITests" time="39.105541944503784"></testcase>
+  <testcase configuration-hash="" name="testOMInterstitial()" classname="BitriseOMUITests" time="26.03164303302765"></testcase>
+  <testcase configuration-hash="" name="testOMScrollView()" classname="BitriseOMUITests" time="39.105541944503784"></testcase>
  </testsuite>
  <testsuite name="BitriseTrackingsUITest" tests="2" failures="0" skipped="0" errors="0" time="28.362125039100647">
-  <testcase name="testTrackingsWithAd()" classname="BitriseTrackingsUITest" time="16.5611629486084"></testcase>
-  <testcase name="testTrackingsWithNoAd()" classname="BitriseTrackingsUITest" time="11.800962090492249"></testcase>
+  <testcase configuration-hash="" name="testTrackingsWithAd()" classname="BitriseTrackingsUITest" time="16.5611629486084"></testcase>
+  <testcase configuration-hash="" name="testTrackingsWithNoAd()" classname="BitriseTrackingsUITest" time="11.800962090492249"></testcase>
  </testsuite>
  <testsuite name="bitrErrorUITest" tests="9" failures="0" skipped="0" errors="0" time="152.8058191537857">
-  <testcase name="testbitrError403()" classname="bitrErrorUITest" time="16.062182068824768"></testcase>
-  <testcase name="testbitrError405()" classname="bitrErrorUITest" time="16.12081003189087"></testcase>
-  <testcase name="testbitrError901()" classname="bitrErrorUITest" time="16.46643900871277"></testcase>
-  <testcase name="testbitrError901_2()" classname="bitrErrorUITest" time="16.493804097175598"></testcase>
-  <testcase name="testbitrError901_3()" classname="bitrErrorUITest" time="16.023802995681763"></testcase>
-  <testcase name="testbitrError901_4()" classname="bitrErrorUITest" time="16.05063498020172"></testcase>
-  <testcase name="testbitrError901_5()" classname="bitrErrorUITest" time="16.06726598739624"></testcase>
-  <testcase name="testbitrVideoError402()" classname="bitrErrorUITest" time="23.205875992774963"></testcase>
-  <testcase name="testVpaidError401()" classname="bitrErrorUITest" time="16.315003991127014"></testcase>
+  <testcase configuration-hash="" name="testbitrError403()" classname="bitrErrorUITest" time="16.062182068824768"></testcase>
+  <testcase configuration-hash="" name="testbitrError405()" classname="bitrErrorUITest" time="16.12081003189087"></testcase>
+  <testcase configuration-hash="" name="testbitrError901()" classname="bitrErrorUITest" time="16.46643900871277"></testcase>
+  <testcase configuration-hash="" name="testbitrError901_2()" classname="bitrErrorUITest" time="16.493804097175598"></testcase>
+  <testcase configuration-hash="" name="testbitrError901_3()" classname="bitrErrorUITest" time="16.023802995681763"></testcase>
+  <testcase configuration-hash="" name="testbitrError901_4()" classname="bitrErrorUITest" time="16.05063498020172"></testcase>
+  <testcase configuration-hash="" name="testbitrError901_5()" classname="bitrErrorUITest" time="16.06726598739624"></testcase>
+  <testcase configuration-hash="" name="testbitrVideoError402()" classname="bitrErrorUITest" time="23.205875992774963"></testcase>
+  <testcase configuration-hash="" name="testVpaidError401()" classname="bitrErrorUITest" time="16.315003991127014"></testcase>
  </testsuite>
 </testsuites>