From 8181af3c8d537657dd06bf2cf8db874298d9a29f Mon Sep 17 00:00:00 2001 From: Ivan Dyatlov Date: Mon, 20 Jan 2025 15:57:55 +0000 Subject: [PATCH] Optimize Allure Reporting by removing unnecessary attachments for passed tests * Remove [flaky] prefix as the issue is fixed and flaky test identified by * Don't attach summary for the passed tests * Don't attach and copy device logs for the passed tests (greatly decrease allure report size) * Backport Allure test class label fix from origin * Fix test suite name for JUnit reports * Properly set test case id and test case name variables --- .../marathon/execution/TestResult.kt | 6 ++ .../marathon/report/allure/AllureReporter.kt | 75 +++++++++++-------- .../report/junit/FinalJUnitReporter.kt | 2 +- .../marathon/report/junit/JUnitWriter.kt | 2 +- .../report/logs/LogReportTestEventInflator.kt | 5 ++ 5 files changed, 57 insertions(+), 33 deletions(-) diff --git a/core/src/main/kotlin/com/malinskiy/marathon/execution/TestResult.kt b/core/src/main/kotlin/com/malinskiy/marathon/execution/TestResult.kt index c949d9f89..3f19c61dd 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/execution/TestResult.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/execution/TestResult.kt @@ -17,6 +17,12 @@ data class TestResult( ) { fun durationMillis(): Long = endTime - startTime + val isFailedOrBroken: Boolean + get() = when (status) { + TestStatus.FAILURE, TestStatus.INCOMPLETE -> true + else -> false + } + val isIgnored: Boolean get() = when (status) { TestStatus.IGNORED, TestStatus.ASSUMPTION_FAILURE -> true diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/allure/AllureReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/allure/AllureReporter.kt index acfa6b057..78f808f9d 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/allure/AllureReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/allure/AllureReporter.kt @@ -79,12 +79,7 @@ class AllureReporter( summary: TestSummary? ): io.qameta.allure.model.TestResult { val test = testResult.test - val fullName = if (summary?.isFlaky == true) { - // TODO: remove this when flaky reporting will be fixed (https://github.com/allure-framework/allure2/pull/1135) - "[flaky] " + test.toSafeTestName() - } else { - test.toSafeTestName() - } + val fullName = test.toSafeTestName() val testMethodName = test.method val suite = "${test.pkg}.${test.clazz}" @@ -97,47 +92,40 @@ class AllureReporter( TestStatus.IGNORED -> Status.SKIPPED } - val summaryFile = outputDirectory - .resolve("$uuid-summary.log") - .apply { writeText(testSummaryFormatter.formatTestResultSummary(testResult, summary)) } - - val summaryAttachment = Attachment() - .setName("Summary") - .setSource(summaryFile.relativePathTo(outputDirectory)) - .setType("text/plain") - testResult.attachments.forEach { val linkFile = outputDirectory.resolve(it.file.name).toPath() Files.deleteIfExists(linkFile) Files.createSymbolicLink(linkFile, it.file.toPath()) } - val testAttachments: List = testResult + val testAttachments: MutableList = testResult .attachments .map { Attachment() .setName(it.type.name.lowercase().replaceFirstChar(Char::titlecase)) .setSource(it.file.name) .setType(it.type.toMimeType()) - } + }.toMutableList() - val allAttachments = listOf(summaryAttachment) + testAttachments + attachSummary(summary, uuid, testResult, testAttachments) val allureTestResult = io.qameta.allure.model.TestResult() .setUuid(uuid) .setFullName(fullName) .setName(testMethodName) .setHistoryId(getHistoryId(test)) + .setTestCaseId(fullName) + .setTestCaseName(testMethodName) .setStatus(status) .setStart(testResult.startTime) .setStop(testResult.endTime) - .setAttachments(allAttachments) + .setAttachments(testAttachments) .setParameters(emptyList()) .setLabels( mutableListOf( ResultsUtils.createHostLabel().setValue(device.serialNumber), ResultsUtils.createPackageLabel(test.pkg), - ResultsUtils.createTestClassLabel(test.clazz), + ResultsUtils.createTestClassLabel(suite), ResultsUtils.createTestMethodLabel(test.method), ResultsUtils.createSuiteLabel(suite) ) @@ -154,13 +142,6 @@ class AllureReporter( test.findValue(Description::class.java.canonicalName)?.let { allureTestResult.setDescription(it) } test.findValue(Issue::class.java.canonicalName)?.let { allureTestResult.links.add(ResultsUtils.createIssueLink(it)) } test.findValue(TmsLink::class.java.canonicalName)?.let { allureTestResult.links.add(ResultsUtils.createTmsLink(it)) } - - allureTestResult.labels.add( - ResultsUtils.createLabel( - LAYER, if (test.isApplicationTest()) CLIENT_APPLICATION else CLIENT_COMPONENT - ) - ) - allureTestResult.labels.add(ResultsUtils.createLabel(PLATFORM, ANDROID)) allureTestResult.labels.addAll(ResultsUtils.getProvidedLabels()) allureTestResult.labels.addAll(test.getOptionalLabels()) @@ -168,8 +149,32 @@ class AllureReporter( return allureTestResult } + private fun attachSummary( + summary: TestSummary?, + uuid: String, + testResult: TestResult, + testAttachments: MutableList + ) { + if (summary != null && summary.results.any { it.isFailedOrBroken }) { + + // We must add summary file to Allure only if we had something failed or broken + // If everything has been passed or ignored summary won't give us anything + + val summaryFile = outputDirectory + .resolve("$uuid-summary.log") + .apply { writeText(testSummaryFormatter.formatTestResultSummary(testResult, summary)) } + + val summaryAttachment = Attachment() + .setName("Summary") + .setSource(summaryFile.relativePathTo(outputDirectory)) + .setType("text/plain") + + testAttachments += summaryAttachment + } + } + private fun Test.isApplicationTest(): Boolean = - configuration.appModuleRegexes.any { it.matches(pkg) } + configuration.appModuleRegexes.any { it.matches(componentInfo.name) } private fun getHistoryId(test: Test): String = ResultsUtils.generateMethodSignatureHash(test.clazz, test.method, emptyList()) @@ -184,9 +189,15 @@ class AllureReporter( findValue(Owner::class.java.canonicalName)?.let { list.add(ResultsUtils.createOwnerLabel(it)) } findValue(Lead::class.java.canonicalName)?.let { list.add(ResultsUtils.createLabel(ResultsUtils.LEAD_LABEL_NAME, it)) } findValue("io.qameta.allure.junit4.Tag")?.let { list.add(ResultsUtils.createTagLabel(it)) } - findValue("io.qameta.allure.label.Layer")?.let { list.add(ResultsUtils.createLabel("layer", it)) } - findValue("io.qameta.allure.label.Team")?.let { list.add(ResultsUtils.createLabel("team", it)) } - findValue("io.qameta.allure.label.Component")?.let { list.add(ResultsUtils.createLabel("component", it)) } + findValue("io.qameta.allure.label.Layer") + ?.let { list.add(ResultsUtils.createLabel(LAYER, it)) } + ?: list.add( + ResultsUtils.createLabel( + LAYER, if (isApplicationTest()) CLIENT_APPLICATION else CLIENT_COMPONENT + ) + ) + findValue("io.qameta.allure.label.Team")?.let { list.add(ResultsUtils.createLabel(TEAM, it)) } + findValue("io.qameta.allure.label.Component")?.let { list.add(ResultsUtils.createLabel(COMPONENT, it)) } return list } @@ -202,6 +213,8 @@ class AllureReporter( private companion object { private const val MESSAGE_LINES_COUNT = 3 private const val LAYER = "layer" + private const val TEAM = "team" + private const val COMPONENT = "component" private const val PLATFORM = "platform" private const val ANDROID = "Android" private const val CLIENT_APPLICATION = "Application client" diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/junit/FinalJUnitReporter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/junit/FinalJUnitReporter.kt index 354ec732d..d198871ed 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/junit/FinalJUnitReporter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/junit/FinalJUnitReporter.kt @@ -13,7 +13,7 @@ internal class FinalJUnitReporter(private val jUnitWriter: JUnitWriter) : Report .testEvents .filter { it.final } .forEach { event -> - val summary = summaries[event.testResult.test] + val summary = summaries[event.testResult.test].takeIf { it?.results?.any { it.isFailedOrBroken } == true } jUnitWriter.testFinished(event.poolId, event.device, event.testResult, summary) } } diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/junit/JUnitWriter.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/junit/JUnitWriter.kt index 33ef24b20..607bdfff7 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/junit/JUnitWriter.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/junit/JUnitWriter.kt @@ -52,7 +52,7 @@ class JUnitWriter( writer.document { element("testsuite") { - attribute("name", "common") + attribute("name", "${test.pkg}.${test.clazz}") attribute("tests", "1") attribute("failures", "$failures") attribute("errors", "0") diff --git a/core/src/main/kotlin/com/malinskiy/marathon/report/logs/LogReportTestEventInflator.kt b/core/src/main/kotlin/com/malinskiy/marathon/report/logs/LogReportTestEventInflator.kt index ceb27364b..a0cd3adc6 100644 --- a/core/src/main/kotlin/com/malinskiy/marathon/report/logs/LogReportTestEventInflator.kt +++ b/core/src/main/kotlin/com/malinskiy/marathon/report/logs/LogReportTestEventInflator.kt @@ -11,6 +11,11 @@ import com.malinskiy.marathon.report.logs.LogEvent.Crash class LogReportTestEventInflator(private val logReport: LogReport) : TestEventInflator { override fun inflate(event: TestEvent): TestEvent { + + if (!event.testResult.isFailedOrBroken) { + return event + } + val log = getLog(event.testResult) val additionalAttachments = listOfNotNull( log?.let { Attachment(log.file, AttachmentType.LOG, FileType.LOG) }