From 5b3bcf629ce64ded66c406a300239f3660b51aa1 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 19 Oct 2026 14:25:16 -0400 Subject: [PATCH 1/2] Collect diagnostics when OSGi smoke test application hangs Fail the OSGi smoke test after 2 minutes instead of waiting forever, and collect two thread dumps of the application (plus a system dump on IBM JVMs) into the build reports so they are included in the GitLab job artifacts. Co-Authored-By: Claude Opus 5.5 --- .../smoketest/AbstractOSGiSmokeTest.groovy | 62 ++++++++++++++++++- .../datadog/smoketest/ProcessManager.groovy | 2 +- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy b/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy index 784cb7ae53d..1dfde48753e 100644 --- a/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy +++ b/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy @@ -1,5 +1,7 @@ package datadog.smoketest +import java.util.concurrent.TimeUnit + abstract class AbstractOSGiSmokeTest extends AbstractSmokeTest { @Override @@ -18,6 +20,11 @@ abstract class AbstractOSGiSmokeTest extends AbstractSmokeTest { List command = new ArrayList<>() command.add(javaPath()) command.addAll(defaultJavaProperties) + if (isIBM) { + // Write javacores (SIGQUIT) into the collected reports and take a system dump on SIGUSR2 for hang diagnostics. + command.add("-Xdump:directory=${diagnosticsDirectory().absolutePath}".toString()) + command.add('-Xdump:system:events=user2,request=exclusive+prepwalk') + } command.add("-Ddd.profiling.enabled=false") command.add((String) "-Dorg.osgi.framework.storage=${storageDir}") command.add("-Dorg.osgi.framework.bootdelegation=") @@ -44,7 +51,9 @@ abstract class AbstractOSGiSmokeTest extends AbstractSmokeTest { def "example application runs without errors"() { when: - testedProcess.waitFor() + if (!testedProcess.waitFor(2, TimeUnit.MINUTES)) { + collectDiagnostics() + } then: testedProcess.exitValue() == 0 @@ -53,6 +62,57 @@ abstract class AbstractOSGiSmokeTest extends AbstractSmokeTest { } } + private File diagnosticsDirectory() { + // Share the application log timestamp so retries keep separate dumps that match their logs. + File directory = new File(buildDirectory, "reports/${getClass().simpleName}.${logFileTimestamp}-dumps") + directory.mkdirs() + return directory + } + + /** Collects thread dumps (and a system dump on IBM) of the hanging application, then fails the test. */ + private void collectDiagnostics() { + File directory = diagnosticsDirectory() + AssertionError failure = new AssertionError((Object) "OSGi application did not exit within 2 minutes; diagnostics: ${directory}; application log: ${logFilePath}") + try { + // Java 8 has no public Process.pid() API. + long pid = testedProcess.metaClass.respondsTo(testedProcess, 'pid') ? testedProcess.pid() : testedProcess.@pid + File summary = new File(directory, 'collection.log') + summary << "Application PID: ${pid}\n" + if (isIBM) { + summary << 'IBM JVM: SIGQUIT thread dumps are written as javacore.*.txt and the SIGUSR2 system dump as core.*.dmp in this directory; ' + + 'threads-*.log and system-dump.log only capture the output of the kill commands\n' + } + + File javaHome = new File(System.getProperty('java.home')) + File jcmd = new File(javaHome.name == 'jre' ? javaHome.parentFile : javaHome, 'bin/jcmd') + List threadDump = isIBM ? ['kill', '-3', pid.toString()] : [jcmd.path, pid.toString(), 'Thread.print', '-l'] + def run = { List command, String output -> + Process diagnostic = new ProcessBuilder(command).redirectErrorStream(true).redirectOutput(new File(directory, output)).start() + if (diagnostic.waitFor(15, TimeUnit.SECONDS)) { + summary << "Exited ${diagnostic.exitValue()}: ${command}\n" + } else { + diagnostic.destroyForcibly() + summary << "Timed out: ${command}\n" + } + } + + run(threadDump, 'threads-1.log') + // Space the snapshots so a stuck thread can be told apart from a slow one. + if (!testedProcess.waitFor(10, TimeUnit.SECONDS)) { + run(threadDump, 'threads-2.log') + if (isIBM) { + run(['kill', '-USR2', pid.toString()], 'system-dump.log') + // The VM logs the end of the dump event once the core file is in place; cleanup must not kill it earlier. + outputThreads.processTestLogLines({ it.contains('Processed dump event "user2"') }, TimeUnit.MINUTES.toMillis(2)) + summary << 'System dump event processed\n' + } + } + } catch (Exception e) { + failure.addSuppressed(e) + } + throw failure + } + @Override def logLevel() { return "debug" diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy index 5cb11cd3216..72016a63fac 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/ProcessManager.groovy @@ -57,7 +57,7 @@ abstract class ProcessManager extends Specification { // Added a timestamp to protect logs from being overwritten on retries. @Shared - private String logFileTimestamp = LOG_FILE_TIMESTAMP_FORMATTER.format(Instant.now()) + protected String logFileTimestamp = LOG_FILE_TIMESTAMP_FORMATTER.format(Instant.now()) @Shared private String[] logFilePaths = (0.. From 7b2b647b664f0db16ad208561c7564b3ad6b130d Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 19 Oct 2026 14:33:58 -0400 Subject: [PATCH 2/2] Tidy OSGi smoke test hang diagnostics Use a single constant for the exit timeout, resolve jcmd only for HotSpot, and create the dumps folder only when diagnostics are collected so passing IBM runs no longer leave an empty folder in reports. Co-Authored-By: Claude Opus 5.5 --- .../smoketest/AbstractOSGiSmokeTest.groovy | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy b/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy index 1dfde48753e..38085608aea 100644 --- a/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy +++ b/dd-smoke-tests/osgi/src/test/groovy/datadog/smoketest/AbstractOSGiSmokeTest.groovy @@ -3,6 +3,7 @@ package datadog.smoketest import java.util.concurrent.TimeUnit abstract class AbstractOSGiSmokeTest extends AbstractSmokeTest { + private static final long EXIT_TIMEOUT_MINUTES = 2 @Override ProcessBuilder createProcessBuilder() { @@ -51,7 +52,7 @@ abstract class AbstractOSGiSmokeTest extends AbstractSmokeTest { def "example application runs without errors"() { when: - if (!testedProcess.waitFor(2, TimeUnit.MINUTES)) { + if (!testedProcess.waitFor(EXIT_TIMEOUT_MINUTES, TimeUnit.MINUTES)) { collectDiagnostics() } @@ -64,18 +65,18 @@ abstract class AbstractOSGiSmokeTest extends AbstractSmokeTest { private File diagnosticsDirectory() { // Share the application log timestamp so retries keep separate dumps that match their logs. - File directory = new File(buildDirectory, "reports/${getClass().simpleName}.${logFileTimestamp}-dumps") - directory.mkdirs() - return directory + // Not created here: the IBM JVM creates its dump directory on the first dump, so passing runs leave no folder. + return new File(buildDirectory, "reports/${getClass().simpleName}.${logFileTimestamp}-dumps") } /** Collects thread dumps (and a system dump on IBM) of the hanging application, then fails the test. */ private void collectDiagnostics() { File directory = diagnosticsDirectory() - AssertionError failure = new AssertionError((Object) "OSGi application did not exit within 2 minutes; diagnostics: ${directory}; application log: ${logFilePath}") + AssertionError failure = new AssertionError((Object) "OSGi application did not exit within ${EXIT_TIMEOUT_MINUTES} minutes; diagnostics: ${directory}; application log: ${logFilePath}") try { // Java 8 has no public Process.pid() API. long pid = testedProcess.metaClass.respondsTo(testedProcess, 'pid') ? testedProcess.pid() : testedProcess.@pid + directory.mkdirs() File summary = new File(directory, 'collection.log') summary << "Application PID: ${pid}\n" if (isIBM) { @@ -83,9 +84,14 @@ abstract class AbstractOSGiSmokeTest extends AbstractSmokeTest { 'threads-*.log and system-dump.log only capture the output of the kill commands\n' } - File javaHome = new File(System.getProperty('java.home')) - File jcmd = new File(javaHome.name == 'jre' ? javaHome.parentFile : javaHome, 'bin/jcmd') - List threadDump = isIBM ? ['kill', '-3', pid.toString()] : [jcmd.path, pid.toString(), 'Thread.print', '-l'] + List threadDump + if (isIBM) { + threadDump = ['kill', '-3', pid.toString()] + } else { + File javaHome = new File(System.getProperty('java.home')) + File jcmd = new File(javaHome.name == 'jre' ? javaHome.parentFile : javaHome, 'bin/jcmd') + threadDump = [jcmd.path, pid.toString(), 'Thread.print', '-l'] + } def run = { List command, String output -> Process diagnostic = new ProcessBuilder(command).redirectErrorStream(true).redirectOutput(new File(directory, output)).start() if (diagnostic.waitFor(15, TimeUnit.SECONDS)) {