Improve test reliability (#43)

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Install JDK ${{ matrix.jre }}
         uses: actions/setup-java@v3
         with:
@@ -33,7 +33,7 @@ jobs:
       - name: gradlew build
         run: ./gradlew build
       - name: junit result
-        uses: mikepenz/action-junit-report@v3
+        uses: mikepenz/action-junit-report@v4
         if: always() # always run even if the previous step fails
         with:
           check_name: JUnit ${{ matrix.jre }} ${{ matrix.os }}

selfie-runner-junit5/build.gradle

@@ -33,6 +33,7 @@ dependencies {
 }
 test {
   useJUnitPlatform()
+  getInputs().dir('../undertest-junit5/src/test/kotlin')
 }
 
 // it all needs to get published and formatted

selfie-runner-junit5/src/test/kotlin/com/diffplug/selfie/junit5/Harness.kt

@@ -24,9 +24,9 @@ import javax.xml.xpath.XPathFactory
 import okio.FileSystem
 import okio.Path
 import okio.Path.Companion.toPath
-import org.gradle.internal.impldep.junit.framework.AssertionFailedError
 import org.gradle.tooling.BuildException
 import org.gradle.tooling.GradleConnector
+import org.opentest4j.AssertionFailedError
 import org.w3c.dom.NodeList
 import org.xml.sax.InputSource
 
@@ -174,27 +174,29 @@ open class Harness(subproject: String) {
     }
   }
   fun gradlew(task: String, vararg args: String): AssertionFailedError? {
-    val runner =
-        GradleConnector.newConnector()
-            .forProjectDirectory(subprojectFolder.parent!!.toFile())
-            .connect()
-
-    try {
-      val buildLauncher =
-          runner
-              .newBuild()
-              .forTasks(":${subprojectFolder.name}:$task")
-              .withArguments(
-                  buildList<String> {
-                    addAll(args)
-                    add("--configuration-cache") // ControlWR enabled vs disables is 11s vs 24s
-                    add("--stacktrace")
-                  })
-      buildLauncher.run()
-      return null
-    } catch (e: BuildException) {
-      return parseBuildException(task, e)
-    }
+    return GradleConnector.newConnector()
+        .forProjectDirectory(subprojectFolder.parent!!.toFile())
+        .connect()
+        .use { connection ->
+          try {
+            val buildLauncher =
+                connection
+                    .newBuild()
+                    // .setStandardError(System.err)
+                    // .setStandardOutput(System.out)
+                    .forTasks(":${subprojectFolder.name}:$task")
+                    .withArguments(
+                        buildList<String> {
+                          addAll(args)
+                          add("--configuration-cache") // enabled vs disabled is 11s vs 24s
+                          add("--stacktrace")
+                        })
+            buildLauncher.run()
+            return null
+          } catch (e: BuildException) {
+            return parseBuildException(task, e)
+          }
+        }
   }
 
   /**

undertest-junit5/build.gradle

@@ -27,10 +27,19 @@ test {
 }
 tasks.register('underTest', Test) {
   useJUnitPlatform()
+  testClassesDirs = testing.suites.test.sources.output.classesDirs
+  classpath = testing.suites.test.sources.runtimeClasspath
+  // the snapshots are both output and input, for this harness best if the test just always runs
+  outputs.upToDateWhen { false }
   // defaults to 'write'
   systemProperty 'selfie', findProperty('selfie')
 }
 tasks.register('underTestRead', Test) {
   useJUnitPlatform()
+  testClassesDirs = testing.suites.test.sources.output.classesDirs
+  classpath = testing.suites.test.sources.runtimeClasspath
+  // the snapshots are both output and input, for this harness best if the test just always runs
+  outputs.upToDateWhen { false }
+  // read-only
   systemProperty 'selfie', 'read'
 }