Report risky tests separately (#523)

paratestphp · Aug 26, 2020 · e54bea9 · e54bea9
1 parent 6a07c7e
commit e54bea9
Show file tree

Hide file tree

Showing 9 changed files with 125 additions and 28 deletions.
diff --git a/src/Logging/JUnit/Reader.php b/src/Logging/JUnit/Reader.php
@@ -89,14 +89,16 @@ public function getFeedback(): array
         $suites   = $this->isSingle ? $this->suites : $this->suites[0]->suites;
         foreach ($suites as $suite) {
             foreach ($suite->cases as $case) {
-                if (count($case->failures) > 0) {
-                    $feedback[] = 'F';
-                } elseif (count($case->errors) > 0) {
+                if (count($case->errors) > 0) {
                     $feedback[] = 'E';
-                } elseif (count($case->skipped) > 0) {
-                    $feedback[] = 'S';
                 } elseif (count($case->warnings) > 0) {
                     $feedback[] = 'W';
+                } elseif (count($case->failures) > 0) {
+                    $feedback[] = 'F';
+                } elseif (count($case->risky) > 0) {
+                    $feedback[] = 'R';
+                } elseif (count($case->skipped) > 0) {
+                    $feedback[] = 'S';
                 } else {
                     $feedback[] = '.';
                 }
@@ -315,4 +317,22 @@ public function getFailures(): array
 
         return $messages;
     }
+
+    /**
+     * {@inheritDoc}
+     */
+    public function getRisky(): array
+    {
+        $messages = [];
+        $suites   = $this->isSingle ? $this->suites : $this->suites[0]->suites;
+        foreach ($suites as $suite) {
+            foreach ($suite->cases as $case) {
+                foreach ($case->risky as $msg) {
+                    $messages[] = $msg['text'];
+                }
+            }
+        }
+
+        return $messages;
+    }
 }
diff --git a/src/Logging/JUnit/TestCase.php b/src/Logging/JUnit/TestCase.php
@@ -4,9 +4,13 @@
 
 namespace ParaTest\Logging\JUnit;
 
+use PHPUnit\Framework\RiskyTestError;
 use SimpleXMLElement;
 
 use function assert;
+use function class_exists;
+use function is_subclass_of;
+use function iterator_to_array;
 use function trim;
 
 /**
@@ -48,6 +52,9 @@ final class TestCase
     /** @var array<int, array{type: string, text: string}> */
     public $skipped = [];
 
+    /** @var array<int, array{type: string, text: string}> */
+    public $risky = [];
+
     public function __construct(
         string $name,
         string $class,
@@ -82,11 +89,32 @@ public static function caseFromNode(SimpleXMLElement $node): self
         );
 
         $system_output = $node->{'system-out'};
+
+        /** @var SimpleXMLElement[] $errors */
+        $errors = (array) $node->xpath('error');
+        $risky  = [];
+        foreach ($errors as $index => $error) {
+            $attributes = $error->attributes();
+            assert($attributes !== null);
+            $attributes = iterator_to_array($attributes);
+            $type       = (string) $attributes['type'];
+            if (
+                ! class_exists($type)
+                || ! ($type === RiskyTestError::class || is_subclass_of($type, RiskyTestError::class))
+            ) {
+                continue;
+            }
+
+            unset($errors[$index]);
+            $risky[] = $error;
+        }
+
         $defect_groups = [
             'failures' => (array) $node->xpath('failure'),
-            'errors' => (array) $node->xpath('error'),
+            'errors' => $errors,
             'warnings' => (array) $node->xpath('warning'),
             'skipped' => (array) $node->xpath('skipped'),
+            'risky' => $risky,
         ];
 
         foreach ($defect_groups as $group => $defects) {

diff --git a/src/Logging/JUnit/Writer.php b/src/Logging/JUnit/Writer.php
@@ -160,6 +160,7 @@ private function appendCase(DOMElement $suiteNode, TestCase $case): DOMElement
         $this->appendDefects($caseNode, $case->failures, 'failure');
         $this->appendDefects($caseNode, $case->errors, 'error');
         $this->appendDefects($caseNode, $case->warnings, 'warning');
+        $this->appendDefects($caseNode, $case->risky, 'error');
         $this->appendDefects($caseNode, $case->skipped, 'skipped');
 
         return $caseNode;
@@ -228,7 +229,6 @@ private function getSuiteRootAttributes(array $suites): array
             'skipped' => 0,
             'time' => 0,
         ];
-
         foreach ($suites as $suite) {
             $result['tests']      += $suite->tests;
             $result['assertions'] += $suite->assertions;

diff --git a/src/Logging/LogInterpreter.php b/src/Logging/LogInterpreter.php
@@ -120,7 +120,7 @@ public function flattenCases(): array
             ++$dict[$case->file]->tests;
             $dict[$case->file]->assertions += $case->assertions;
             $dict[$case->file]->failures   += count($case->failures);
-            $dict[$case->file]->errors     += count($case->errors);
+            $dict[$case->file]->errors     += count($case->errors) + count($case->risky);
             $dict[$case->file]->warnings   += count($case->warnings);
             $dict[$case->file]->skipped    += count($case->skipped);
             $dict[$case->file]->time       += $case->time;
@@ -216,4 +216,17 @@ public function getFailures(): array
 
         return $messages;
     }
+
+    /**
+     * {@inheritDoc}
+     */
+    public function getRisky(): array
+    {
+        $messages = [];
+        foreach ($this->readers as $reader) {
+            $messages = array_merge($messages, $reader->getRisky());
+        }
+
+        return $messages;
+    }
 }
diff --git a/src/Logging/MetaProviderInterface.php b/src/Logging/MetaProviderInterface.php
@@ -31,4 +31,7 @@ public function getWarnings(): array;
 
     /** @return string[] */
     public function getFailures(): array;
+
+    /** @return string[] */
+    public function getRisky(): array;
 }
diff --git a/src/Runners/PHPUnit/ResultPrinter.php b/src/Runners/PHPUnit/ResultPrinter.php
@@ -167,6 +167,7 @@ public function printResults(): void
             $this->getErrors(),
             $this->getWarnings(),
             $this->getFailures(),
+            $this->getRisky(),
         ]);
 
         $this->output->write($this->getHeader());
@@ -207,16 +208,6 @@ public function getHeader(): string
         return "\n\n" . $resourceUsage . "\n\n";
     }
 
-    /**
-     * Returns warning messages as a string.
-     */
-    public function getWarnings(): string
-    {
-        $warnings = $this->results->getWarnings();
-
-        return $this->getDefects($warnings, 'warning');
-    }
-
     /**
      * Return the footer information reporting success
      * or failure.
@@ -236,6 +227,26 @@ public function getFooter(): string
         return "{$footer}\n";
     }
 
+    /**
+     * Returns error messages.
+     */
+    public function getErrors(): string
+    {
+        $errors = $this->results->getErrors();
+
+        return $this->getDefects($errors, 'error');
+    }
+
+    /**
+     * Returns warning messages as a string.
+     */
+    public function getWarnings(): string
+    {
+        $warnings = $this->results->getWarnings();
+
+        return $this->getDefects($warnings, 'warning');
+    }
+
     /**
      * Returns the failure messages.
      */
@@ -247,13 +258,13 @@ public function getFailures(): string
     }
 
     /**
-     * Returns error messages.
+     * Returns the failure messages.
      */
-    public function getErrors(): string
+    public function getRisky(): string
     {
-        $errors = $this->results->getErrors();
+        $risky = $this->results->getRisky();
 
-        return $this->getDefects($errors, 'error');
+        return $this->getDefects($risky, 'risky');
     }
 
     /**

diff --git a/test/Unit/Logging/JUnit/ReaderTest.php b/test/Unit/Logging/JUnit/ReaderTest.php
@@ -271,7 +271,7 @@ public function testMixedGetFailureMessages(): void
     public function testMixedGetErrorMessages(): void
     {
         $errors = $this->mixed->getErrors();
-        static::assertCount(3, $errors);
+        static::assertCount(1, $errors);
         static::assertSame(
             "UnitTestWithErrorTest::testTruth\nException: Error!!!\n\n" .
                 '/home/brian/Projects/parallel-phpunit/test/fixtures/failing_tests/UnitTestWithErrorTest.php:17',
@@ -306,7 +306,7 @@ public function testSingleGetMessages(): void
      */
     public function testGetMultiErrorsMessages(): void
     {
-        $errors = $this->multi_errors->getErrors();
+        $errors = $this->multi_errors->getRisky();
         static::assertCount(2, $errors);
         static::assertSame(
             "Risky Test\n" .
@@ -327,7 +327,7 @@ public function testGetMultiErrorsMessages(): void
     public function testMixedGetFeedback(): void
     {
         $feedback = $this->mixed->getFeedback();
-        static::assertSame('.F..E.F.WSSE.F.WSSE', implode('', $feedback));
+        static::assertSame('.F..E.F.WSSR.F.WSSR', implode('', $feedback));
     }
 
     public function testRemoveLog(): void

diff --git a/test/Unit/Logging/LogInterpreterTest.php b/test/Unit/Logging/LogInterpreterTest.php
@@ -93,8 +93,6 @@ public function testGetErrorsReturnsArrayOfErrorMessages(): void
         $errors = [
             "UnitTestWithErrorTest::testTruth\nException: Error!!!\n\n/home/brian/Projects/parallel-phpunit/" .
             'test/fixtures/failing_tests/UnitTestWithErrorTest.php:17',
-            'Risky Test',
-            'Risky Test',
         ];
         static::assertSame($errors, $this->interpreter->getErrors());
     }
@@ -122,6 +120,15 @@ public function testGetFailuresReturnsArrayOfFailureMessages(): void
         static::assertSame($failures, $this->interpreter->getFailures());
     }
 
+    public function testGetRiskyReturnsArrayOfErrorMessages(): void
+    {
+        $errors = [
+            'Risky Test',
+            'Risky Test',
+        ];
+        static::assertSame($errors, $this->interpreter->getRisky());
+    }
+
     public function testGetCasesReturnsAllCases(): void
     {
         $cases = $this->interpreter->getCases();

diff --git a/test/Unit/Runners/PHPUnit/ResultPrinterTest.php b/test/Unit/Runners/PHPUnit/ResultPrinterTest.php
@@ -225,6 +225,21 @@ public function testGetFailures(): void
         static::assertSame($eq, $failures);
     }
 
+    public function testGetRisky(): void
+    {
+        $this->printer->addTest($this->mixedSuite);
+
+        $this->prepareReaders();
+
+        $failures = $this->printer->getRisky();
+
+        $eq  = "There were 2 riskys:\n\n";
+        $eq .= "1) Risky Test\n\n";
+        $eq .= "2) Risky Test\n\n";
+
+        static::assertSame($eq, $failures);
+    }
+
     public function testGetFooterWithFailures(): void
     {
         $this->printer->addTest($this->errorSuite);
@@ -272,7 +287,7 @@ public function testPrintFeedbackForMixed(): void
         $this->printer->addTest($this->mixedSuite);
         $this->printer->printFeedback($this->mixedSuite);
         $contents = $this->output->fetch();
-        static::assertSame('.F..E.F.WSSE.F.WSSE', $contents);
+        static::assertSame('.F..E.F.WSSR.F.WSSR', $contents);
     }
 
     public function testPrintFeedbackForMoreThan100Suites(): void