diff --git a/.github/workflows/post-build-selective.yml b/.github/workflows/post-build-selective.yml
index 095c471763c..603a1734cee 100644
--- a/.github/workflows/post-build-selective.yml
+++ b/.github/workflows/post-build-selective.yml
@@ -77,9 +77,8 @@ jobs:
         uses: mikepenz/action-junit-report@v5
         if: always() # always run even if the previous step fails
         with:
-          fail_on_failure: false
-          include_passed: false
+          summary: true
           detailed_summary: true
           annotate_only: true
-          require_tests: false
+          truncate_stack_traces: false
           report_paths: 'out/**/test-report.xml'
diff --git a/main/eval/test/src/mill/eval/EvaluationTests.scala b/main/eval/test/src/mill/eval/EvaluationTests.scala
index 7defb75553a..5a3a77e188e 100644
--- a/main/eval/test/src/mill/eval/EvaluationTests.scala
+++ b/main/eval/test/src/mill/eval/EvaluationTests.scala
@@ -228,6 +228,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite {
         val overridden = os.read(
           checker.evaluator.outPath / "foo.super/BaseModule.json"
         )
+        throw new Exception("boom")
         assert(
           public.contains("base"),
           public.contains("object"),