diff --git a/build.gradle b/build.gradle
index 864e73e..3083cb1 100644
--- a/build.gradle
+++ b/build.gradle
@@ -132,18 +132,6 @@ tasks.register('jspecifySamplesTest', Test) {
     inputs.files("${jspecify.projectDir}/samples")
 }
 
-TaskProvider<Test> conformanceTest(String name, String testDirectory, String reportFile) {
-    check.dependsOn(name)
-    return tasks.register(name, Test) {
-        group = 'verification'
-        include '**/ConformanceTest.class'
-        inputs.dir(testDirectory)
-        inputs.files(reportFile)
-        systemProperty("JSpecifyConformanceTest.sourceDirectory", testDirectory)
-        systemProperty("JSpecifyConformanceTest.report", reportFile)
-    }
-}
-
 tasks.register('unzipConformanceTestAssertions', Copy) {
     dependsOn configurations.conformanceTestAssertions
     from zipTree(configurations.conformanceTestAssertions.singleFile)
@@ -151,13 +139,29 @@ tasks.register('unzipConformanceTestAssertions', Copy) {
     into "${buildDir}/conformanceTests"
 }
 
-conformanceTest('conformanceTest', "${buildDir}/conformanceTests/org/jspecify/conformance/tests", 'tests/ConformanceTest-report.txt').configure {
+tasks.register('conformanceTests', Test) {
+    group = 'verification'
     dependsOn 'unzipConformanceTestAssertions', configurations.conformanceTestDeps
+    include '**/ConformanceTest.class'
+
+    // Conformance tests
+    inputs.dir("${buildDir}/conformanceTests")
+    inputs.files("tests/ConformanceTest-report.txt")
     inputs.files(configurations.conformanceTestDeps)
-    systemProperty("JSpecifyConformanceTest.testDeps", configurations.conformanceTestDeps.join(":"))
+    systemProperty("JSpecifyConformanceTest.inputs", "${buildDir}/conformanceTests/org/jspecify/conformance/tests")
+    systemProperty("JSpecifyConformanceTest.report", "tests/ConformanceTest-report.txt")
+    systemProperty("JSpecifyConformanceTest.deps", configurations.conformanceTestDeps.join(":"))
+
+    // Conformance tests run on the samples directory
+    inputs.dir("${jspecify.projectDir}/samples")
+    inputs.files("tests/ConformanceTestOnSamples-report.txt")
+    systemProperty("JSpecifyConformanceTest.samples.inputs", "${jspecify.projectDir}/samples")
+    systemProperty("JSpecifyConformanceTest.samples.report", "tests/ConformanceTestOnSamples-report.txt")
 }
 
-conformanceTest('conformanceTestOnSamples', "${jspecify.projectDir}/samples", 'tests/ConformanceTestOnSamples-report.txt')
+tasks.named('check').configure {
+    dependsOn('conformanceTests')
+}
 
 clean.doFirst {
     delete "${rootDir}/tests/build/"