build(test): a test class that leaves no result fails the build (#9)

build.gradle

@@ -52,3 +52,5 @@ tasks.withType(JavaCompile).configureEach { javaCompiler ->
     // uncomment the next line for build with compiler argument -Xlint:unchecked
     // options.compilerArgs << "-Xlint:unchecked"
 }
+
+apply from: "gradle/test-results-complete.gradle"

gradle/test-results-complete.gradle

@@ -0,0 +1,108 @@
+// Every test class leaves a result, in every test task of this build.
+//
+// A test that ends the test JVM - System.exit(0), as awt-extensions' CloseWindow does when its window is closed -
+// leaves every class after it without a result, and Gradle still reports the task green, because the worker
+// exited with 0 (astrapi69/swing-components#34; the same in astrapi69/mystic-crypt-ui#288). So after each Test task,
+// every concrete compiled class that refers to a JUnit Jupiter, JUnit 4 or TestNG test annotation must have left a
+// result in the task's JUnit XML directory; otherwise the build fails and names the classes.
+//
+// A result is recognised by the file name, the <testsuite> name or the classname of its test cases: none of them
+// alone is reliable (Gradle shortens long file names to a hash, a class-level @DisplayName becomes the suite name).
+// A run narrowed with --tests or a filter is not checked, and says so. A class that is known to leave no result -
+// TestNG writes none for a class whose every test has enabled = false - is declared in the build, with its reason:
+//
+//     ext.testClassesWithoutResult = ['org.example.SomeTest': 'every test has enabled = false (#12)']
+//
+// set before this file is applied.
+//
+// The logic stays inside doLast, without a reference to this script, so that the configuration cache can store it.
+def declaredWithoutResult = (project.findProperty('testClassesWithoutResult') ?: [:]) as Map<String, String>
+tasks.withType(Test).configureEach { Test testTask ->
+    def candidates = testTask.candidateClassFiles
+    def resultsDir = testTask.reports.junitXml.outputLocation
+    def declared = new TreeMap<String, String>(declaredWithoutResult)
+    testTask.doLast { Test task ->
+        def filter = task.filter
+        if (!filter.commandLineIncludePatterns.isEmpty() || !filter.includePatterns.isEmpty()
+            || !filter.excludePatterns.isEmpty()) {
+            task.logger.lifecycle("${task.name}: narrowed by a filter, not checked for missing results")
+            return
+        }
+        // JUnit Jupiter, JUnit 4 and TestNG: a class that refers to one of their test annotations
+        def markers = ['org/junit/jupiter/api/Test', 'org/junit/jupiter/params/ParameterizedTest',
+                       'org/junit/jupiter/api/TestFactory', 'org/junit/jupiter/api/RepeatedTest',
+                       'org/junit/jupiter/api/TestTemplate', 'org/junit/Test', 'org/testng/annotations/Test']
+        def expected = [] as Set
+        candidates.visit { details ->
+            def f = details.file
+            if (details.directory || !f.name.endsWith('.class') || f.name.contains('$')) {
+                return
+            }
+            def bytes = f.bytes
+            def text = new String(bytes, 'ISO-8859-1')
+            if (!markers.any { text.contains(it) }) {
+                return
+            }
+            // the class's access flags follow the constant pool: walk it, then leave out abstract classes and
+            // interfaces, which never run on their own
+            def input = new DataInputStream(new ByteArrayInputStream(bytes))
+            input.readInt(); input.readUnsignedShort(); input.readUnsignedShort()
+            int count = input.readUnsignedShort()
+            for (int i = 1; i < count; i++) {
+                int tag = input.readUnsignedByte()
+                switch (tag) {
+                    case 1: input.skipBytes(input.readUnsignedShort()); break
+                    case 3: case 4: input.skipBytes(4); break
+                    case 5: case 6: input.skipBytes(8); i++; break
+                    case 7: case 8: case 16: case 19: case 20: input.skipBytes(2); break
+                    case 9: case 10: case 11: case 12: case 17: case 18: input.skipBytes(4); break
+                    case 15: input.skipBytes(3); break
+                    default: throw new GradleException("unknown constant pool tag ${tag} in ${f}")
+                }
+            }
+            int access = input.readUnsignedShort()
+            if ((access & 0x0400) != 0 || (access & 0x0200) != 0) {
+                return
+            }
+            expected << (details.relativePath.pathString - ~/\.class$/).replace('/', '.')
+        }
+        def dir = resultsDir.get().asFile
+        def files = (dir.listFiles() ?: [] as File[]).findAll { it.name.startsWith('TEST-') && it.name.endsWith('.xml') }
+        def results = [] as Set
+        files.each { File file ->
+            results << file.name.substring(5, file.name.length() - 4)
+            def head = file.withReader('UTF-8') { reader ->
+                char[] buffer = new char[65536]
+                int read = reader.read(buffer)
+                read > 0 ? new String(buffer, 0, read) : ''
+            }
+            def suite = head =~ /<testsuite\s+name="([^"]+)"/
+            if (suite.find()) {
+                results << suite.group(1)
+            }
+            (head =~ /classname="([^"]+)"/).each { results << it[1] }
+        }
+        // a check that recognises no test class while the task wrote results cannot check, and says so instead
+        // of reporting green; a task without any test class says that, too
+        if (expected.isEmpty()) {
+            if (!files.isEmpty()) {
+                throw new GradleException("${task.name}: ${files.size()} result files, but no compiled class "
+                    + "refers to a test annotation this check knows (JUnit Jupiter, JUnit 4, TestNG), so it "
+                    + "cannot tell whether every test class ran")
+            }
+            task.logger.lifecycle("${task.name}: no test classes, nothing to check")
+            return
+        }
+        def missing = (expected - results).sort()
+        def excused = missing.findAll { declared.containsKey(it) }
+        missing = missing - excused
+        excused.each { task.logger.lifecycle("${task.name}: ${it} leaves no result, as declared: ${declared[it]}") }
+        if (missing) {
+            throw new GradleException("${task.name}: ${missing.size()} of ${expected.size()} test classes left no "
+                + "result, so the test JVM ended before they ran or they were never run (System.exit?): "
+                + missing.join(', '))
+        }
+        task.logger.lifecycle("${task.name}: all ${expected.size() - excused.size()} test classes left a result"
+            + (excused ? ", ${excused.size()} declared without one" : ''))
+    }
+}