From 1ecaa96363685543b73d5cdad08902c74ba69fec Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Wed, 23 Sep 2026 11:27:25 -0400 Subject: [PATCH 01/18] Add Azure Durable Functions worker instrumentation --- .../azure-functions-worker-2.7/build.gradle | 26 ++ .../gradle.lockfile | 130 +++++++ .../AzureFunctionsWorkerInstrumentation.java | 106 ++++++ .../worker/DurableFunctionsDecorator.java | 46 +++ .../worker/DurableFunctionsUtils.java | 165 ++++++++ .../worker/TraceContextExtractAdapter.java | 87 +++++ .../groovy/AzureFunctionsWorkerTest.groovy | 357 ++++++++++++++++++ .../chain/FunctionExecutionMiddleware.java | 10 + .../OrchestratorBlockedException.java | 3 + .../ContinueAsNewInterruption.java | 3 + .../OrchestratorBlockedException.java | 3 + .../core/propagation/ExtractedContext.java | 11 + .../propagation/ExtractedContextTest.java | 40 ++ .../instrumentation/api/AgentSpanContext.java | 8 + .../instrumentation/api/TagContext.java | 6 +- settings.gradle.kts | 1 + 16 files changed, 1001 insertions(+), 1 deletion(-) create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/groovy/AzureFunctionsWorkerTest.groovy create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/OrchestratorBlockedException.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/ContinueAsNewInterruption.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/OrchestratorBlockedException.java create mode 100644 dd-trace-core/src/test/java/datadog/trace/core/propagation/ExtractedContextTest.java diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle new file mode 100644 index 00000000000..930115252af --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle @@ -0,0 +1,26 @@ +plugins { + id 'dd-trace-java.module.instrumentation' +} + +muzzle { + pass { + group = 'com.microsoft.azure.functions' + module = 'azure-functions-java-spi' + versions = '[1.0.0,)' + extraDependency 'com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0' + } +} + +addTestSuiteForDir('latestDepTest', 'test') + +dependencies { + compileOnly group: 'com.microsoft.azure.functions', name: 'azure-functions-java-core-library', version: '1.2.0' + compileOnly group: 'com.microsoft.azure.functions', name: 'azure-functions-java-spi', version: '1.0.0' + + testImplementation group: 'com.microsoft.azure.functions', name: 'azure-functions-java-core-library', version: '1.2.0' + testImplementation group: 'com.microsoft.azure.functions', name: 'azure-functions-java-spi', version: '1.0.0' + testImplementation libs.bundles.mockito + + latestDepTestImplementation group: 'com.microsoft.azure.functions', name: 'azure-functions-java-core-library', version: '+' + latestDepTestImplementation group: 'com.microsoft.azure.functions', name: 'azure-functions-java-spi', version: '+' +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile b/dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile new file mode 100644 index 00000000000..a8ef06fa2bb --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile @@ -0,0 +1,130 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +# To regenerate this file, run: ./gradlew :dd-java-agent:instrumentation:azure-functions-worker-2.7:dependencies --write-locks +cafe.cryptography:curve25519-elisabeth:0.1.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +cafe.cryptography:ed25519-elisabeth:0.1.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +ch.qos.logback:logback-classic:1.2.13=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +ch.qos.logback:logback-core:1.2.13=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq.okhttp3:okhttp:3.12.15=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq.okio:okio:1.17.6=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-instrument-java:0.0.5=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:java-dogstatsd-client:4.4.5=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.datadoghq:sketches-java:0.8.3=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.javaparser:javaparser-core:3.25.6=codenarc +com.github.jnr:jffi:1.3.15=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-a64asm:1.0.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-constants:0.10.4=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-enxio:0.32.20=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-ffi:2.2.19=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-posix:3.1.22=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-unixsocket:0.38.25=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-x86asm:1.0.2=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.spotbugs:spotbugs-annotations:4.10.3=compileClasspath,spotbugs +com.github.spotbugs:spotbugs:4.10.3=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,testAnnotationProcessor,testCompileClasspath +com.google.auto.service:auto-service:1.1.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.code.gson:gson:2.14.0=spotbugs +com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.47.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.google.errorprone:error_prone_annotations:2.48.0=spotbugs +com.google.guava:failureaccess:1.0.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.google.guava:guava:32.0.1-jre=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.google.re2j:re2j:1.8=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-core-library:1.3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-spi:1.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-spi:1.1.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath +com.squareup.moshi:moshi:1.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:logging-interceptor:3.12.12=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:okhttp:3.12.12=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.squareup.okio:okio:1.17.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.thoughtworks.qdox:qdox:1.12.1=codenarc +commons-fileupload:commons-fileupload:1.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.21.0=spotbugs +de.thetaphi:forbiddenapis:3.10=compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.leangen.geantyref:geantyref:1.3.16=latestDepTestRuntimeClasspath,testRuntimeClasspath +io.sqreen:libsqreen:17.5.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +javax.servlet:javax.servlet-api:3.1.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +jaxen:jaxen:2.0.6=spotbugs +junit:junit:4.13.2=latestDepTestRuntimeClasspath,testRuntimeClasspath +net.bytebuddy:byte-buddy-agent:1.18.12=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.bytebuddy:byte-buddy:1.18.12=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna-platform:5.8.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +net.java.dev.jna:jna:5.8.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.10=spotbugs +org.apache.ant:ant-antlr:1.10.14=codenarc +org.apache.ant:ant-junit:1.10.14=codenarc +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.logging.log4j:log4j-api:2.26.1=spotbugs +org.apache.logging.log4j:log4j-core:2.26.1=spotbugs +org.apiguardian:apiguardian-api:1.1.2=latestDepTestCompileClasspath,testCompileClasspath +org.checkerframework:checker-qual:3.33.0=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +org.codehaus.groovy:groovy-ant:3.0.23=codenarc +org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc +org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.25=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.codehaus.groovy:groovy-templates:3.0.23=codenarc +org.codehaus.groovy:groovy-xml:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.25=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.codenarc:CodeNarc:3.7.0=codenarc +org.dom4j:dom4j:2.2.0=spotbugs +org.gmetrics:GMetrics:2.1.0=codenarc +org.hamcrest:hamcrest-core:1.3=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.hamcrest:hamcrest:3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.jctools:jctools-core-jdk11:4.0.6=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.jctools:jctools-core:4.0.6=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.jspecify:jspecify:1.0.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:1.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-runner:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-suite-api:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-suite-commons:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.junit:junit-bom:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit:junit-bom:6.1.2=spotbugs +org.mockito:mockito-core:4.4.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.mockito:mockito-junit-jupiter:4.4.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.objenesis:objenesis:3.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.opentest4j:opentest4j:1.3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-analysis:9.7.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.ow2.asm:asm-commons:9.10.1=latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath +org.ow2.asm:asm-tree:9.10.1=latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm-util:9.7.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.ow2.asm:asm:9.10.1=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +org.slf4j:jcl-over-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:jul-to-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:log4j-over-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath +org.slf4j:slf4j-api:1.7.32=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=spotbugsSlf4j +org.slf4j:slf4j-api:2.0.18=spotbugs +org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j +org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testRuntimeClasspath +org.spockframework:spock-bom:2.4-groovy-3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.spockframework:spock-core:2.4-groovy-3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-junit:1.2.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-parser:1.2.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=spotbugs +empty=spotbugsPlugins diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java new file mode 100644 index 00000000000..8c2332ef049 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -0,0 +1,106 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.extractContextAndGetSpanContext; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.spanFromScope; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.AZURE_FUNCTIONS_REQUEST; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import com.microsoft.azure.functions.TraceContext; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; +import datadog.context.ContextScope; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import net.bytebuddy.asm.Advice; + +@AutoService(InstrumenterModule.class) +public final class AzureFunctionsWorkerInstrumentation extends InstrumenterModule.Tracing + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + public AzureFunctionsWorkerInstrumentation() { + super("azure-functions"); + } + + @Override + public String instrumentedType() { + return "com.microsoft.azure.functions.worker.chain.FunctionExecutionMiddleware"; + } + + @Override + public String[] helperClassNames() { + return new String[] { + packageName + ".DurableFunctionsDecorator", + packageName + ".DurableFunctionsUtils", + packageName + ".TraceContextExtractAdapter" + }; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(isPublic()) + .and(named("invoke")) + .and(takesArguments(2)) + .and( + takesArgument( + 0, + named( + "com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext"))) + .and( + takesArgument( + 1, + named( + "com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain"))), + AzureFunctionsWorkerInstrumentation.class.getName() + "$InvokeAdvice"); + } + + public static class InvokeAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static ContextScope onEnter(@Advice.Argument(0) MiddlewareContext context) { + final String trigger = DurableFunctionsUtils.getTrigger(context); + if (trigger == null) { + return null; + } + if ("DurableOrchestration".equals(trigger) + && !DurableFunctionsUtils.shouldTraceOrchestration(context)) { + return null; + } + + final TraceContext traceContext = context.getTraceContext(); + AgentSpanContext.Extracted parent = + traceContext == null + ? null + : extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); + parent = DurableFunctionsUtils.reconcileSamplingPriority(parent, traceContext); + + final AgentSpan span = startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent); + DECORATE.afterStart(span); + DECORATE.onInvoke(span, context.getFunctionName(), trigger); + return activateSpan(span); + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.Enter ContextScope scope, @Advice.Thrown Throwable throwable) { + if (scope != null) { + final AgentSpan span = spanFromScope(scope); + if (!DurableFunctionsUtils.isReplayControlFlow(throwable)) { + DECORATE.onError(span, throwable); + } + DECORATE.beforeFinish(span); + scope.close(); + span.finish(); + } + } + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java new file mode 100644 index 00000000000..f6ec65d193a --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java @@ -0,0 +1,46 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import datadog.trace.api.naming.SpanNaming; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; +import datadog.trace.bootstrap.instrumentation.decorator.BaseDecorator; + +public final class DurableFunctionsDecorator extends BaseDecorator { + public static final DurableFunctionsDecorator DECORATE = new DurableFunctionsDecorator(); + public static final CharSequence AZURE_FUNCTIONS_REQUEST = + UTF8BytesString.create( + SpanNaming.instance().namingSchema().cloud().operationForFaas("azure")); + + private static final CharSequence AZURE_FUNCTIONS = UTF8BytesString.create("azure-functions"); + + private DurableFunctionsDecorator() {} + + @Override + protected String[] instrumentationNames() { + return new String[] {"azure-functions"}; + } + + @Override + protected CharSequence spanType() { + return InternalSpanTypes.SERVERLESS; + } + + @Override + protected CharSequence component() { + return AZURE_FUNCTIONS; + } + + @Override + protected void doAfterStart(AgentSpan span) { + super.doAfterStart(span); + span.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER); + } + + public void onInvoke(AgentSpan span, String functionName, String trigger) { + span.setResourceName(trigger + " " + functionName); + span.setTag("aas.function.name", functionName); + span.setTag("aas.function.trigger", trigger); + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java new file mode 100644 index 00000000000..570cb2df037 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -0,0 +1,165 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; +import static datadog.trace.api.sampling.PrioritySampling.UNSET; + +import com.microsoft.azure.functions.TraceContext; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import java.util.Base64; + +public final class DurableFunctionsUtils { + private static final String ACTIVITY_ANNOTATION = "DurableActivityTrigger"; + private static final String ENTITY_ANNOTATION = "DurableEntityTrigger"; + private static final String ORCHESTRATION_ANNOTATION = "DurableOrchestrationTrigger"; + + // OrchestratorRequest protobuf field numbers. + private static final int PAST_EVENTS_FIELD = 3; + private static final int NEW_EVENTS_FIELD = 4; + + // HistoryEvent protobuf field numbers. + private static final int TASK_FAILED_FIELD = 8; + private static final int SUB_ORCHESTRATION_FAILED_FIELD = 11; + + private DurableFunctionsUtils() {} + + public static String getTrigger(MiddlewareContext context) { + if (context.getParameterName(ORCHESTRATION_ANNOTATION) != null) { + return "DurableOrchestration"; + } + if (context.getParameterName(ACTIVITY_ANNOTATION) != null) { + return "DurableActivity"; + } + if (context.getParameterName(ENTITY_ANNOTATION) != null) { + return "DurableEntity"; + } + return null; + } + + public static AgentSpanContext.Extracted reconcileSamplingPriority( + AgentSpanContext.Extracted parent, TraceContext traceContext) { + if (parent != null + && parent.getSamplingPriority() == SAMPLER_DROP + && traceContext != null + && TraceContextExtractAdapter.datadogSamplingPriority(traceContext.getTracestate()) + == UNSET) { + return parent.withSamplingPriority(UNSET); + } + return parent; + } + + public static boolean isReplayControlFlow(Throwable throwable) { + while (throwable != null) { + final String className = throwable.getClass().getName(); + if ("com.microsoft.durabletask.OrchestratorBlockedException".equals(className) + || "com.microsoft.durabletask.interruption.OrchestratorBlockedException".equals(className) + || "com.microsoft.durabletask.interruption.ContinueAsNewInterruption".equals(className)) { + return true; + } + throwable = throwable.getCause(); + } + return false; + } + + /** + * Returns whether an orchestration invocation should produce a span. + * + *

Durable orchestrators replay from the beginning whenever new history arrives. Those + * executions are implementation details rather than new logical operations, so successful replays + * are suppressed. A replay containing a newly failed activity or sub-orchestration is retained so + * the failure remains visible. If the trigger payload cannot be inspected, tracing fails open. + */ + public static boolean shouldTraceOrchestration(MiddlewareContext context) { + try { + final String parameterName = context.getParameterName(ORCHESTRATION_ANNOTATION); + final Object parameterValue = context.getParameterValue(parameterName); + if (!(parameterValue instanceof String)) { + return true; + } + + final byte[] request = Base64.getDecoder().decode((String) parameterValue); + boolean replay = false; + boolean newFailure = false; + final int[] position = {0}; + while (position[0] < request.length) { + final long tag = readVarint(request, position, request.length); + final int field = (int) (tag >>> 3); + final int wireType = (int) (tag & 7); + if (wireType == 2) { + final long length = readVarint(request, position, request.length); + if (length < 0 || length > request.length - position[0]) { + return true; + } + final int end = position[0] + (int) length; + if (field == PAST_EVENTS_FIELD) { + replay = true; + } else if (field == NEW_EVENTS_FIELD && containsFailureEvent(request, position[0], end)) { + newFailure = true; + } + position[0] = end; + } else { + skipValue(request, position, request.length, wireType); + } + } + return !replay || newFailure; + } catch (Throwable ignored) { + return true; + } + } + + private static boolean containsFailureEvent(byte[] data, int offset, int limit) { + final int[] position = {offset}; + while (position[0] < limit) { + final long tag = readVarint(data, position, limit); + final int field = (int) (tag >>> 3); + final int wireType = (int) (tag & 7); + if (wireType == 2 + && (field == TASK_FAILED_FIELD || field == SUB_ORCHESTRATION_FAILED_FIELD)) { + return true; + } + skipValue(data, position, limit, wireType); + } + return false; + } + + private static long readVarint(byte[] data, int[] position, int limit) { + long value = 0; + for (int shift = 0; shift < 64 && position[0] < limit; shift += 7) { + final int current = data[position[0]++] & 0xff; + value |= (long) (current & 0x7f) << shift; + if ((current & 0x80) == 0) { + return value; + } + } + throw new IllegalArgumentException("Invalid protobuf varint"); + } + + private static void skipValue(byte[] data, int[] position, int limit, int wireType) { + switch (wireType) { + case 0: + readVarint(data, position, limit); + return; + case 1: + if (limit - position[0] < 8) { + throw new IllegalArgumentException("Invalid fixed64 field"); + } + position[0] += 8; + return; + case 2: + final long length = readVarint(data, position, limit); + if (length < 0 || length > limit - position[0]) { + throw new IllegalArgumentException("Invalid length-delimited field"); + } + position[0] += (int) length; + return; + case 5: + if (limit - position[0] < 4) { + throw new IllegalArgumentException("Invalid fixed32 field"); + } + position[0] += 4; + return; + default: + throw new IllegalArgumentException("Unsupported protobuf wire type"); + } + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java new file mode 100644 index 00000000000..d31e210635b --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java @@ -0,0 +1,87 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.api.sampling.PrioritySampling.UNSET; + +import com.microsoft.azure.functions.TraceContext; +import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; + +public final class TraceContextExtractAdapter + implements AgentPropagation.ContextVisitor { + public static final TraceContextExtractAdapter GETTER = new TraceContextExtractAdapter(); + + private static final String TRACEPARENT = "traceparent"; + private static final String TRACESTATE = "tracestate"; + + private TraceContextExtractAdapter() {} + + @Override + public void forEachKey(TraceContext carrier, AgentPropagation.KeyClassifier classifier) { + final String tracestate = carrier.getTracestate(); + String traceparent = carrier.getTraceparent(); + if (datadogSamplingPriority(tracestate) > 0) { + traceparent = setSampledFlag(traceparent); + } + if (traceparent != null && !classifier.accept(TRACEPARENT, traceparent)) { + return; + } + if (tracestate != null) { + classifier.accept(TRACESTATE, tracestate); + } + } + + static int datadogSamplingPriority(String tracestate) { + if (tracestate == null) { + return UNSET; + } + int memberStart = 0; + while (memberStart < tracestate.length()) { + int memberEnd = tracestate.indexOf(',', memberStart); + if (memberEnd < 0) { + memberEnd = tracestate.length(); + } + int start = memberStart; + while (start < memberEnd && tracestate.charAt(start) == ' ') { + start++; + } + if (memberEnd - start >= 3 + && tracestate.charAt(start) == 'd' + && tracestate.charAt(start + 1) == 'd' + && tracestate.charAt(start + 2) == '=') { + return parseSamplingPriority(tracestate, start + 3, memberEnd); + } + memberStart = memberEnd + 1; + } + return UNSET; + } + + private static int parseSamplingPriority(String value, int start, int end) { + while (start < end) { + int fieldEnd = value.indexOf(';', start); + if (fieldEnd < 0 || fieldEnd > end) { + fieldEnd = end; + } + if (fieldEnd - start > 2 && value.charAt(start) == 's' && value.charAt(start + 1) == ':') { + try { + return Integer.parseInt(value.substring(start + 2, fieldEnd)); + } catch (NumberFormatException ignored) { + return UNSET; + } + } + start = fieldEnd + 1; + } + return UNSET; + } + + private static String setSampledFlag(String traceparent) { + if (traceparent == null || traceparent.length() < 55) { + return traceparent; + } + final int high = Character.digit(traceparent.charAt(53), 16); + final int low = Character.digit(traceparent.charAt(54), 16); + if (high < 0 || low < 0 || (low & 1) != 0) { + return traceparent; + } + final char sampledLow = Character.forDigit(low | 1, 16); + return traceparent.substring(0, 54) + sampledLow + traceparent.substring(55); + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/groovy/AzureFunctionsWorkerTest.groovy b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/groovy/AzureFunctionsWorkerTest.groovy new file mode 100644 index 00000000000..37253f6a3b0 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/groovy/AzureFunctionsWorkerTest.groovy @@ -0,0 +1,357 @@ +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP +import static datadog.trace.api.sampling.PrioritySampling.UNSET +import static org.mockito.ArgumentMatchers.anyString +import static org.mockito.Mockito.doAnswer +import static org.mockito.Mockito.mock +import static org.mockito.Mockito.when + +import com.microsoft.azure.functions.TraceContext +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext +import com.microsoft.azure.functions.worker.chain.FunctionExecutionMiddleware +import com.microsoft.durabletask.interruption.ContinueAsNewInterruption +import com.microsoft.durabletask.interruption.OrchestratorBlockedException +import datadog.trace.agent.test.naming.VersionedNamingTestBase +import datadog.trace.api.DDSpanTypes +import datadog.trace.bootstrap.instrumentation.api.AgentTracer +import datadog.trace.bootstrap.instrumentation.api.Tags +import spock.lang.Unroll + +abstract class AzureFunctionsWorkerTest extends VersionedNamingTestBase { + + @Override + String service() { + null + } + + @Unroll + def "creates a span for #trigger"() { + setup: + def context = contextFor(annotation, "MyFunction") + def chain = mock(MiddlewareChain) + + when: + new FunctionExecutionMiddleware().invoke(context, chain) + + then: + assertTraces(1) { + trace(1) { + span { + parent() + operationName operation() + resourceName "$trigger MyFunction" + spanType DDSpanTypes.SERVERLESS + errored false + tags { + defaultTags() + "$Tags.COMPONENT" "azure-functions" + "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" + "aas.function.name" "MyFunction" + "aas.function.trigger" trigger + } + } + } + } + + where: + annotation | trigger + "DurableOrchestrationTrigger" | "DurableOrchestration" + "DurableActivityTrigger" | "DurableActivity" + "DurableEntityTrigger" | "DurableEntity" + } + + def "does not create a span for a non-Durable function"() { + setup: + def context = contextFor(null, "HttpFunction") + + when: + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) + + then: + assertTraces(0) {} + } + + def "does not create a span for a successful orchestration replay"() { + setup: + def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") + // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { taskCompleted: {} } } + when(context.getParameterValue("input")).thenReturn( + Base64.encoder.encodeToString([0x1a, 0x00, 0x22, 0x02, 0x3a, 0x00] as byte[])) + + when: + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) + + then: + assertTraces(0) {} + } + + def "does not treat a failure field with the wrong protobuf wire type as a failure"() { + setup: + def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") + // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { field 8: varint 0 } } + when(context.getParameterValue("input")).thenReturn( + Base64.encoder.encodeToString([0x1a, 0x00, 0x22, 0x02, 0x40, 0x00] as byte[])) + + when: + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) + + then: + assertTraces(0) {} + } + + def "creates a span for an initial orchestration execution"() { + setup: + def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") + // OrchestratorRequest { newEvents: HistoryEvent { executionStarted: {} } } + when(context.getParameterValue("input")).thenReturn( + Base64.encoder.encodeToString([0x22, 0x02, 0x1a, 0x00] as byte[])) + + when: + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) + + then: + assertTraces(1) { + trace(1) { + span { + operationName operation() + resourceName "DurableOrchestration Orchestrator" + spanType DDSpanTypes.SERVERLESS + errored false + tags { + defaultTags() + "$Tags.COMPONENT" "azure-functions" + "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" + "aas.function.name" "Orchestrator" + "aas.function.trigger" "DurableOrchestration" + } + } + } + } + } + + @Unroll + def "creates a span for an orchestration replay with #failure"() { + setup: + def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") + // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { failure: {} } } + when(context.getParameterValue("input")).thenReturn( + Base64.encoder.encodeToString([0x1a, 0x00, 0x22, 0x02, failureTag, 0x00] as byte[])) + + when: + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) + + then: + assertTraces(1) { + trace(1) { + span { + operationName operation() + resourceName "DurableOrchestration Orchestrator" + spanType DDSpanTypes.SERVERLESS + errored false + tags { + defaultTags() + "$Tags.COMPONENT" "azure-functions" + "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" + "aas.function.name" "Orchestrator" + "aas.function.trigger" "DurableOrchestration" + } + } + } + } + + where: + failure | failureTag + "a failed activity" | 0x42 + "a failed sub-orchestration" | 0x5a + } + + @Unroll + def "fails open for an unreadable orchestration payload: #description"() { + setup: + def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") + when(context.getParameterValue("input")).thenReturn(payload) + + when: + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) + + then: + assertTraces(1) { + trace(1) { + span { + operationName operation() + resourceName "DurableOrchestration Orchestrator" + spanType DDSpanTypes.SERVERLESS + errored false + tags { + defaultTags() + "$Tags.COMPONENT" "azure-functions" + "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" + "aas.function.name" "Orchestrator" + "aas.function.trigger" "DurableOrchestration" + } + } + } + } + + where: + description | payload + "non-base64 input" | "not base64" + "truncated protobuf" | Base64.encoder.encodeToString([0x1a, 0x02, 0x00] as byte[]) + "non-string input" | new Object() + } + + def "continues the Azure trace and ignores its implicit sampling rejection"() { + setup: + def context = contextFor("DurableActivityTrigger", "Activity") + def traceContext = mock(TraceContext) + when(traceContext.getTraceparent()).thenReturn( + "00-0000000000000000000000000000002a-000000000000002b-00") + when(traceContext.getTracestate()).thenReturn(null) + when(context.getTraceContext()).thenReturn(traceContext) + def priority = Integer.MIN_VALUE + def chain = mock(MiddlewareChain) + doAnswer { + priority = AgentTracer.activeSpan().spanContext().samplingPriority + null + }.when(chain).doNext(context) + + when: + new FunctionExecutionMiddleware().invoke(context, chain) + + then: + TEST_WRITER.waitForTraces(1) + def span = TEST_WRITER[0][0] + span.traceId.toString() == "42" + span.parentId == 43 + priority == UNSET + span.samplingPriority() == SAMPLER_KEEP + } + + def "honors a Datadog keep decision when Azure clears the W3C sampled flag"() { + setup: + def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") + def traceContext = mock(TraceContext) + when(traceContext.getTraceparent()).thenReturn( + "00-0000000000000000000000000000002a-000000000000002b-00") + when(traceContext.getTracestate()).thenReturn("dd=s:2;o:rum") + when(context.getTraceContext()).thenReturn(traceContext) + def priority = Integer.MIN_VALUE + def chain = mock(MiddlewareChain) + doAnswer { + priority = AgentTracer.activeSpan().spanContext().samplingPriority + null + }.when(chain).doNext(context) + + when: + new FunctionExecutionMiddleware().invoke(context, chain) + + then: + TEST_WRITER.waitForTraces(1) + def span = TEST_WRITER[0][0] + span.traceId.toString() == "42" + span.parentId == 43 + priority > 0 + } + + @Unroll + def "does not mark #controlFlow.class.simpleName as an error"() { + setup: + def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") + def chain = mock(MiddlewareChain) + doAnswer { throw new RuntimeException(controlFlow) }.when(chain).doNext(context) + + when: + new FunctionExecutionMiddleware().invoke(context, chain) + + then: + thrown(RuntimeException) + assertTraces(1) { + trace(1) { + span { + operationName operation() + resourceName "DurableOrchestration Orchestrator" + spanType DDSpanTypes.SERVERLESS + errored false + tags { + defaultTags() + "$Tags.COMPONENT" "azure-functions" + "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" + "aas.function.name" "Orchestrator" + "aas.function.trigger" "DurableOrchestration" + } + } + } + } + + where: + controlFlow << [ + new com.microsoft.durabletask.OrchestratorBlockedException(), + new OrchestratorBlockedException(), + new ContinueAsNewInterruption() + ] + } + + def "marks application failures as errors"() { + setup: + def context = contextFor("DurableActivityTrigger", "Activity") + def chain = mock(MiddlewareChain) + doAnswer { throw new IllegalStateException("failure") }.when(chain).doNext(context) + + when: + new FunctionExecutionMiddleware().invoke(context, chain) + + then: + thrown(IllegalStateException) + assertTraces(1) { + trace(1) { + span { + operationName operation() + resourceName "DurableActivity Activity" + spanType DDSpanTypes.SERVERLESS + errored true + tags { + errorTags IllegalStateException, "failure" + defaultTags() + "$Tags.COMPONENT" "azure-functions" + "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" + "aas.function.name" "Activity" + "aas.function.trigger" "DurableActivity" + } + } + } + } + } + + private static MiddlewareContext contextFor(String annotation, String functionName) { + def context = mock(MiddlewareContext) + when(context.getFunctionName()).thenReturn(functionName) + when(context.getParameterName(anyString())).thenAnswer { + it.arguments[0] == annotation ? "input" : null + } + context + } +} + +class AzureFunctionsWorkerV0ForkedTest extends AzureFunctionsWorkerTest { + @Override + int version() { + 0 + } + + @Override + String operation() { + "dd-tracer-serverless-span" + } +} + +class AzureFunctionsWorkerV1Test extends AzureFunctionsWorkerTest { + @Override + int version() { + 1 + } + + @Override + String operation() { + "azure.functions.invoke" + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java new file mode 100644 index 00000000000..1cda6f23f27 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java @@ -0,0 +1,10 @@ +package com.microsoft.azure.functions.worker.chain; + +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; + +public class FunctionExecutionMiddleware { + public void invoke(MiddlewareContext context, MiddlewareChain chain) throws Exception { + chain.doNext(context); + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/OrchestratorBlockedException.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/OrchestratorBlockedException.java new file mode 100644 index 00000000000..d8839fdf084 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/OrchestratorBlockedException.java @@ -0,0 +1,3 @@ +package com.microsoft.durabletask; + +public class OrchestratorBlockedException extends RuntimeException {} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/ContinueAsNewInterruption.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/ContinueAsNewInterruption.java new file mode 100644 index 00000000000..e540e7519e0 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/ContinueAsNewInterruption.java @@ -0,0 +1,3 @@ +package com.microsoft.durabletask.interruption; + +public class ContinueAsNewInterruption extends RuntimeException {} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/OrchestratorBlockedException.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/OrchestratorBlockedException.java new file mode 100644 index 00000000000..681aa96c4bd --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/OrchestratorBlockedException.java @@ -0,0 +1,3 @@ +package com.microsoft.durabletask.interruption; + +public class OrchestratorBlockedException extends RuntimeException {} diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java index af503e6a6ed..7c48ac9e452 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java @@ -1,5 +1,7 @@ package datadog.trace.core.propagation; +import static datadog.trace.api.sampling.SamplingMechanism.EXTERNAL_OVERRIDE; + import datadog.trace.api.DDTraceId; import datadog.trace.api.TagMap; import datadog.trace.api.TraceConfig; @@ -117,6 +119,15 @@ public PropagationTags getPropagationTags() { return propagationTags; } + @Override + public ExtractedContext withSamplingPriority(final int samplingPriority) { + setSamplingPriority(samplingPriority); + if (propagationTags != null) { + propagationTags.updateTraceSamplingPriority(samplingPriority, EXTERNAL_OVERRIDE); + } + return this; + } + @Override public String toString() { StringBuilder builder = new StringBuilder("ExtractedContext{"); diff --git a/dd-trace-core/src/test/java/datadog/trace/core/propagation/ExtractedContextTest.java b/dd-trace-core/src/test/java/datadog/trace/core/propagation/ExtractedContextTest.java new file mode 100644 index 00000000000..130d5adef33 --- /dev/null +++ b/dd-trace-core/src/test/java/datadog/trace/core/propagation/ExtractedContextTest.java @@ -0,0 +1,40 @@ +package datadog.trace.core.propagation; + +import static datadog.trace.api.TracePropagationStyle.TRACECONTEXT; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; +import static datadog.trace.api.sampling.PrioritySampling.UNSET; +import static datadog.trace.api.sampling.SamplingMechanism.EXTERNAL_OVERRIDE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; + +import datadog.trace.api.DDTraceId; +import org.junit.jupiter.api.Test; + +class ExtractedContextTest { + @Test + void replacesSamplingPriorityWithoutChangingTraceIdentity() { + PropagationTags propagationTags = PropagationTags.factory().empty(); + propagationTags.updateTraceSamplingPriority(SAMPLER_DROP, EXTERNAL_OVERRIDE); + ExtractedContext original = + new ExtractedContext( + DDTraceId.from(42), 43, SAMPLER_DROP, null, propagationTags, TRACECONTEXT); + + ExtractedContext updated = original.withSamplingPriority(UNSET); + + assertSame(original, updated); + assertEquals(original.getTraceId(), updated.getTraceId()); + assertEquals(original.getSpanId(), updated.getSpanId()); + assertEquals(UNSET, updated.getSamplingPriority()); + assertEquals(UNSET, updated.getPropagationTags().getSamplingPriority()); + } + + @Test + void replacesSamplingPriorityWithoutPropagationTags() { + ExtractedContext context = + new ExtractedContext(DDTraceId.from(42), 43, SAMPLER_DROP, null, null, TRACECONTEXT); + + context.withSamplingPriority(UNSET); + + assertEquals(UNSET, context.getSamplingPriority()); + } +} diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java index 6e950925305..fcde079c1ea 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java @@ -72,6 +72,14 @@ default CharSequence getIntegrationName() { boolean isRemote(); interface Extracted extends AgentSpanContext { + /** + * Returns this extracted context with the supplied sampling priority when the implementation + * supports replacing propagation decisions. + */ + default Extracted withSamplingPriority(int samplingPriority) { + return this; + } + /** * Gets the span links related to the other terminated context. * diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/TagContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/TagContext.java index 078ffeb1625..f7098e94aa2 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/TagContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/TagContext.java @@ -33,7 +33,7 @@ public class TagContext implements AgentSpanContext.Extracted { private final HttpHeaders httpHeaders; private final Map baggage; private Baggage w3cBaggage; - private final int samplingPriority; + private int samplingPriority; private final TraceConfig traceConfig; private final TracePropagationStyle propagationStyle; private final DDTraceId traceId; @@ -181,6 +181,10 @@ public final int getSamplingPriority() { return samplingPriority; } + protected final void setSamplingPriority(int samplingPriority) { + this.samplingPriority = samplingPriority; + } + public final Map getBaggage() { return baggage; } diff --git a/settings.gradle.kts b/settings.gradle.kts index e557332879d..182dfc511fc 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -322,6 +322,7 @@ include( ":dd-java-agent:instrumentation:axis2-1.3", ":dd-java-agent:instrumentation:axway-api-7.5", ":dd-java-agent:instrumentation:azure-functions-1.2.2", + ":dd-java-agent:instrumentation:azure-functions-worker-2.7", ":dd-java-agent:instrumentation:beanshell-2.0", ":dd-java-agent:instrumentation:caffeine-1.0", ":dd-java-agent:instrumentation:cdi-1.2", From f6cb9cf240f586a01fed7291b2909a78976ea3ab Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Wed, 23 Sep 2026 13:09:09 -0400 Subject: [PATCH 02/18] Add Azure Functions worker code owner --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d61754b6290..c5c661a393f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -172,6 +172,7 @@ /dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/ @DataDog/apm-serverless /dd-java-agent/instrumentation/azure-functions/ @DataDog/apm-serverless /dd-java-agent/instrumentation/azure-functions-1.2.2/ @DataDog/apm-serverless +/dd-java-agent/instrumentation/azure-functions-worker-2.7/ @DataDog/apm-serverless # @DataDog/apm-lang-platform-java /.editorconfig @DataDog/apm-lang-platform-java From 376c24d8510fef4f062deedbabd056132182f1aa Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Wed, 23 Sep 2026 13:39:55 -0400 Subject: [PATCH 03/18] Convert Azure Functions worker tests to JUnit --- .../groovy/AzureFunctionsWorkerTest.groovy | 357 ------------------ .../test/java/AzureFunctionsWorkerTest.java | 290 ++++++++++++++ .../AzureFunctionsWorkerV0ForkedTest.java | 12 + .../test/java/AzureFunctionsWorkerV1Test.java | 12 + 4 files changed, 314 insertions(+), 357 deletions(-) delete mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/groovy/AzureFunctionsWorkerTest.groovy create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV0ForkedTest.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV1Test.java diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/groovy/AzureFunctionsWorkerTest.groovy b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/groovy/AzureFunctionsWorkerTest.groovy deleted file mode 100644 index 37253f6a3b0..00000000000 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/groovy/AzureFunctionsWorkerTest.groovy +++ /dev/null @@ -1,357 +0,0 @@ -import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP -import static datadog.trace.api.sampling.PrioritySampling.UNSET -import static org.mockito.ArgumentMatchers.anyString -import static org.mockito.Mockito.doAnswer -import static org.mockito.Mockito.mock -import static org.mockito.Mockito.when - -import com.microsoft.azure.functions.TraceContext -import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain -import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext -import com.microsoft.azure.functions.worker.chain.FunctionExecutionMiddleware -import com.microsoft.durabletask.interruption.ContinueAsNewInterruption -import com.microsoft.durabletask.interruption.OrchestratorBlockedException -import datadog.trace.agent.test.naming.VersionedNamingTestBase -import datadog.trace.api.DDSpanTypes -import datadog.trace.bootstrap.instrumentation.api.AgentTracer -import datadog.trace.bootstrap.instrumentation.api.Tags -import spock.lang.Unroll - -abstract class AzureFunctionsWorkerTest extends VersionedNamingTestBase { - - @Override - String service() { - null - } - - @Unroll - def "creates a span for #trigger"() { - setup: - def context = contextFor(annotation, "MyFunction") - def chain = mock(MiddlewareChain) - - when: - new FunctionExecutionMiddleware().invoke(context, chain) - - then: - assertTraces(1) { - trace(1) { - span { - parent() - operationName operation() - resourceName "$trigger MyFunction" - spanType DDSpanTypes.SERVERLESS - errored false - tags { - defaultTags() - "$Tags.COMPONENT" "azure-functions" - "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" - "aas.function.name" "MyFunction" - "aas.function.trigger" trigger - } - } - } - } - - where: - annotation | trigger - "DurableOrchestrationTrigger" | "DurableOrchestration" - "DurableActivityTrigger" | "DurableActivity" - "DurableEntityTrigger" | "DurableEntity" - } - - def "does not create a span for a non-Durable function"() { - setup: - def context = contextFor(null, "HttpFunction") - - when: - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) - - then: - assertTraces(0) {} - } - - def "does not create a span for a successful orchestration replay"() { - setup: - def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") - // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { taskCompleted: {} } } - when(context.getParameterValue("input")).thenReturn( - Base64.encoder.encodeToString([0x1a, 0x00, 0x22, 0x02, 0x3a, 0x00] as byte[])) - - when: - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) - - then: - assertTraces(0) {} - } - - def "does not treat a failure field with the wrong protobuf wire type as a failure"() { - setup: - def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") - // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { field 8: varint 0 } } - when(context.getParameterValue("input")).thenReturn( - Base64.encoder.encodeToString([0x1a, 0x00, 0x22, 0x02, 0x40, 0x00] as byte[])) - - when: - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) - - then: - assertTraces(0) {} - } - - def "creates a span for an initial orchestration execution"() { - setup: - def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") - // OrchestratorRequest { newEvents: HistoryEvent { executionStarted: {} } } - when(context.getParameterValue("input")).thenReturn( - Base64.encoder.encodeToString([0x22, 0x02, 0x1a, 0x00] as byte[])) - - when: - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) - - then: - assertTraces(1) { - trace(1) { - span { - operationName operation() - resourceName "DurableOrchestration Orchestrator" - spanType DDSpanTypes.SERVERLESS - errored false - tags { - defaultTags() - "$Tags.COMPONENT" "azure-functions" - "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" - "aas.function.name" "Orchestrator" - "aas.function.trigger" "DurableOrchestration" - } - } - } - } - } - - @Unroll - def "creates a span for an orchestration replay with #failure"() { - setup: - def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") - // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { failure: {} } } - when(context.getParameterValue("input")).thenReturn( - Base64.encoder.encodeToString([0x1a, 0x00, 0x22, 0x02, failureTag, 0x00] as byte[])) - - when: - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) - - then: - assertTraces(1) { - trace(1) { - span { - operationName operation() - resourceName "DurableOrchestration Orchestrator" - spanType DDSpanTypes.SERVERLESS - errored false - tags { - defaultTags() - "$Tags.COMPONENT" "azure-functions" - "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" - "aas.function.name" "Orchestrator" - "aas.function.trigger" "DurableOrchestration" - } - } - } - } - - where: - failure | failureTag - "a failed activity" | 0x42 - "a failed sub-orchestration" | 0x5a - } - - @Unroll - def "fails open for an unreadable orchestration payload: #description"() { - setup: - def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") - when(context.getParameterValue("input")).thenReturn(payload) - - when: - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain)) - - then: - assertTraces(1) { - trace(1) { - span { - operationName operation() - resourceName "DurableOrchestration Orchestrator" - spanType DDSpanTypes.SERVERLESS - errored false - tags { - defaultTags() - "$Tags.COMPONENT" "azure-functions" - "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" - "aas.function.name" "Orchestrator" - "aas.function.trigger" "DurableOrchestration" - } - } - } - } - - where: - description | payload - "non-base64 input" | "not base64" - "truncated protobuf" | Base64.encoder.encodeToString([0x1a, 0x02, 0x00] as byte[]) - "non-string input" | new Object() - } - - def "continues the Azure trace and ignores its implicit sampling rejection"() { - setup: - def context = contextFor("DurableActivityTrigger", "Activity") - def traceContext = mock(TraceContext) - when(traceContext.getTraceparent()).thenReturn( - "00-0000000000000000000000000000002a-000000000000002b-00") - when(traceContext.getTracestate()).thenReturn(null) - when(context.getTraceContext()).thenReturn(traceContext) - def priority = Integer.MIN_VALUE - def chain = mock(MiddlewareChain) - doAnswer { - priority = AgentTracer.activeSpan().spanContext().samplingPriority - null - }.when(chain).doNext(context) - - when: - new FunctionExecutionMiddleware().invoke(context, chain) - - then: - TEST_WRITER.waitForTraces(1) - def span = TEST_WRITER[0][0] - span.traceId.toString() == "42" - span.parentId == 43 - priority == UNSET - span.samplingPriority() == SAMPLER_KEEP - } - - def "honors a Datadog keep decision when Azure clears the W3C sampled flag"() { - setup: - def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") - def traceContext = mock(TraceContext) - when(traceContext.getTraceparent()).thenReturn( - "00-0000000000000000000000000000002a-000000000000002b-00") - when(traceContext.getTracestate()).thenReturn("dd=s:2;o:rum") - when(context.getTraceContext()).thenReturn(traceContext) - def priority = Integer.MIN_VALUE - def chain = mock(MiddlewareChain) - doAnswer { - priority = AgentTracer.activeSpan().spanContext().samplingPriority - null - }.when(chain).doNext(context) - - when: - new FunctionExecutionMiddleware().invoke(context, chain) - - then: - TEST_WRITER.waitForTraces(1) - def span = TEST_WRITER[0][0] - span.traceId.toString() == "42" - span.parentId == 43 - priority > 0 - } - - @Unroll - def "does not mark #controlFlow.class.simpleName as an error"() { - setup: - def context = contextFor("DurableOrchestrationTrigger", "Orchestrator") - def chain = mock(MiddlewareChain) - doAnswer { throw new RuntimeException(controlFlow) }.when(chain).doNext(context) - - when: - new FunctionExecutionMiddleware().invoke(context, chain) - - then: - thrown(RuntimeException) - assertTraces(1) { - trace(1) { - span { - operationName operation() - resourceName "DurableOrchestration Orchestrator" - spanType DDSpanTypes.SERVERLESS - errored false - tags { - defaultTags() - "$Tags.COMPONENT" "azure-functions" - "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" - "aas.function.name" "Orchestrator" - "aas.function.trigger" "DurableOrchestration" - } - } - } - } - - where: - controlFlow << [ - new com.microsoft.durabletask.OrchestratorBlockedException(), - new OrchestratorBlockedException(), - new ContinueAsNewInterruption() - ] - } - - def "marks application failures as errors"() { - setup: - def context = contextFor("DurableActivityTrigger", "Activity") - def chain = mock(MiddlewareChain) - doAnswer { throw new IllegalStateException("failure") }.when(chain).doNext(context) - - when: - new FunctionExecutionMiddleware().invoke(context, chain) - - then: - thrown(IllegalStateException) - assertTraces(1) { - trace(1) { - span { - operationName operation() - resourceName "DurableActivity Activity" - spanType DDSpanTypes.SERVERLESS - errored true - tags { - errorTags IllegalStateException, "failure" - defaultTags() - "$Tags.COMPONENT" "azure-functions" - "$Tags.SPAN_KIND" "$Tags.SPAN_KIND_SERVER" - "aas.function.name" "Activity" - "aas.function.trigger" "DurableActivity" - } - } - } - } - } - - private static MiddlewareContext contextFor(String annotation, String functionName) { - def context = mock(MiddlewareContext) - when(context.getFunctionName()).thenReturn(functionName) - when(context.getParameterName(anyString())).thenAnswer { - it.arguments[0] == annotation ? "input" : null - } - context - } -} - -class AzureFunctionsWorkerV0ForkedTest extends AzureFunctionsWorkerTest { - @Override - int version() { - 0 - } - - @Override - String operation() { - "dd-tracer-serverless-span" - } -} - -class AzureFunctionsWorkerV1Test extends AzureFunctionsWorkerTest { - @Override - int version() { - 1 - } - - @Override - String operation() { - "azure.functions.invoke" - } -} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java new file mode 100644 index 00000000000..c3b79300cab --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -0,0 +1,290 @@ +import static datadog.trace.agent.test.assertions.SpanMatcher.span; +import static datadog.trace.agent.test.assertions.TagsMatcher.defaultTags; +import static datadog.trace.agent.test.assertions.TagsMatcher.error; +import static datadog.trace.agent.test.assertions.TagsMatcher.tag; +import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; +import static datadog.trace.api.sampling.PrioritySampling.UNSET; +import static datadog.trace.test.junit.utils.assertions.Matchers.is; +import static datadog.trace.test.junit.utils.assertions.Matchers.matches; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.microsoft.azure.functions.TraceContext; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; +import com.microsoft.azure.functions.worker.chain.FunctionExecutionMiddleware; +import com.microsoft.durabletask.interruption.ContinueAsNewInterruption; +import com.microsoft.durabletask.interruption.OrchestratorBlockedException; +import datadog.trace.agent.test.AbstractInstrumentationTest; +import datadog.trace.agent.test.assertions.SpanMatcher; +import datadog.trace.api.DDSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.core.DDSpan; +import java.util.Base64; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.tabletest.junit.TableTest; + +abstract class AzureFunctionsWorkerTest extends AbstractInstrumentationTest { + + abstract String operation(); + + // spotless:off + @TableTest({ + "scenario | annotation | trigger", + "orchestration | DurableOrchestrationTrigger | DurableOrchestration", + "activity | DurableActivityTrigger | DurableActivity", + "entity | DurableEntityTrigger | DurableEntity" + }) + // spotless:on + void createsSpanForDurableTrigger(String annotation, String trigger) throws Exception { + MiddlewareContext context = contextFor(annotation, "MyFunction"); + + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + + assertTraces(trace(durableSpan("MyFunction", trigger))); + } + + @Test + void doesNotCreateSpanForNonDurableFunction() throws Exception { + MiddlewareContext context = contextFor(null, "HttpFunction"); + + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + + assertTraces(); + } + + @Test + void doesNotCreateSpanForSuccessfulOrchestrationReplay() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { taskCompleted: {} } } + when(context.getParameterValue("input")) + .thenReturn( + Base64.getEncoder().encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, 0x3a, 0x00})); + + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + + assertTraces(); + } + + @Test + void doesNotTreatFailureFieldWithWrongProtobufWireTypeAsFailure() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { field 8: varint 0 } } + when(context.getParameterValue("input")) + .thenReturn( + Base64.getEncoder().encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, 0x40, 0x00})); + + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + + assertTraces(); + } + + @Test + void createsSpanForInitialOrchestrationExecution() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + // OrchestratorRequest { newEvents: HistoryEvent { executionStarted: {} } } + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(new byte[] {0x22, 0x02, 0x1a, 0x00})); + + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + + assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + } + + // spotless:off + @TableTest({ + "scenario | failureTag", + "failed activity | 66", + "failed sub-orchestration | 90" + }) + // spotless:on + void createsSpanForOrchestrationReplayWithFailure(int failureTag) throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { failure: {} } } + when(context.getParameterValue("input")) + .thenReturn( + Base64.getEncoder() + .encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, (byte) failureTag, 0x00})); + + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + + assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + } + + @ParameterizedTest(name = "{0}") + // spotless:off + @TableTest({ + "scenario | description | payload", + "non-base64 input | non-base64 input | 'not base64'", + "truncated protobuf | truncated protobuf | GgIA" + }) + // spotless:on + @MethodSource("failsOpenForUnreadableOrchestrationPayloadArguments") + void failsOpenForUnreadableOrchestrationPayload(String description, Object payload) + throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + when(context.getParameterValue("input")).thenReturn(payload); + + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + + assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + } + + static Stream failsOpenForUnreadableOrchestrationPayloadArguments() { + return Stream.of(arguments("non-string input", new Object())); + } + + @Test + void continuesAzureTraceAndIgnoresImplicitSamplingRejection() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); + when(traceContext.getTracestate()).thenReturn(null); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + + writer.waitForTraces(1); + DDSpan span = writer.firstTrace().get(0); + assertEquals("42", span.getTraceId().toString()); + assertEquals(43L, span.getParentId()); + assertEquals(UNSET, priority.get()); + assertEquals(SAMPLER_KEEP, span.samplingPriority()); + } + + @Test + void honorsDatadogKeepDecisionWhenAzureClearsW3cSampledFlag() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); + when(traceContext.getTracestate()).thenReturn("dd=s:2;o:rum"); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + + writer.waitForTraces(1); + DDSpan span = writer.firstTrace().get(0); + assertEquals("42", span.getTraceId().toString()); + assertEquals(43L, span.getParentId()); + assertTrue(priority.get() > 0); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("doesNotMarkReplayControlFlowAsErrorArguments") + void doesNotMarkReplayControlFlowAsError(String description, Throwable controlFlow) + throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new RuntimeException(controlFlow); + }) + .when(chain) + .doNext(context); + + assertThrows( + RuntimeException.class, () -> new FunctionExecutionMiddleware().invoke(context, chain)); + + assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + } + + static Stream doesNotMarkReplayControlFlowAsErrorArguments() { + return Stream.of( + arguments( + "legacy OrchestratorBlockedException", + new com.microsoft.durabletask.OrchestratorBlockedException()), + arguments("OrchestratorBlockedException", new OrchestratorBlockedException()), + arguments("ContinueAsNewInterruption", new ContinueAsNewInterruption())); + } + + @Test + void marksApplicationFailuresAsErrors() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new IllegalStateException("failure"); + }) + .when(chain) + .doNext(context); + + assertThrows( + IllegalStateException.class, + () -> new FunctionExecutionMiddleware().invoke(context, chain)); + + assertTraces( + trace( + span() + .root() + .operationName(Pattern.compile(Pattern.quote(operation()))) + .resourceName("DurableActivity Activity") + .type(DDSpanTypes.SERVERLESS) + .error(true) + .tags( + defaultTags(), + error(IllegalStateException.class, "failure"), + tag(Tags.COMPONENT, matches(Pattern.quote("azure-functions"))), + tag(Tags.SPAN_KIND, is(Tags.SPAN_KIND_SERVER)), + tag("aas.function.name", is("Activity")), + tag("aas.function.trigger", is("DurableActivity"))))); + } + + private SpanMatcher durableSpan(String functionName, String trigger) { + return span() + .root() + .operationName(Pattern.compile(Pattern.quote(operation()))) + .resourceName(trigger + " " + functionName) + .type(DDSpanTypes.SERVERLESS) + .error(false) + .tags( + defaultTags(), + tag(Tags.COMPONENT, matches(Pattern.quote("azure-functions"))), + tag(Tags.SPAN_KIND, is(Tags.SPAN_KIND_SERVER)), + tag("aas.function.name", is(functionName)), + tag("aas.function.trigger", is(trigger))); + } + + private static MiddlewareContext contextFor(String annotation, String functionName) { + MiddlewareContext context = mock(MiddlewareContext.class); + when(context.getFunctionName()).thenReturn(functionName); + when(context.getParameterName(anyString())) + .thenAnswer( + invocation -> Objects.equals(invocation.getArgument(0), annotation) ? "input" : null); + return context; + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV0ForkedTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV0ForkedTest.java new file mode 100644 index 00000000000..e3d36f11ea3 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV0ForkedTest.java @@ -0,0 +1,12 @@ +import static datadog.trace.api.config.TracerConfig.TRACE_SPAN_ATTRIBUTE_SCHEMA; + +import datadog.trace.test.junit.utils.config.WithConfig; + +@WithConfig(key = TRACE_SPAN_ATTRIBUTE_SCHEMA, value = "v0") +class AzureFunctionsWorkerV0ForkedTest extends AzureFunctionsWorkerTest { + + @Override + String operation() { + return "dd-tracer-serverless-span"; + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV1Test.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV1Test.java new file mode 100644 index 00000000000..ce27c2c15ca --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV1Test.java @@ -0,0 +1,12 @@ +import static datadog.trace.api.config.TracerConfig.TRACE_SPAN_ATTRIBUTE_SCHEMA; + +import datadog.trace.test.junit.utils.config.WithConfig; + +@WithConfig(key = TRACE_SPAN_ATTRIBUTE_SCHEMA, value = "v1") +class AzureFunctionsWorkerV1Test extends AzureFunctionsWorkerTest { + + @Override + String operation() { + return "azure.functions.invoke"; + } +} From 12dec943396a3b6a28afb95a9b8d5e6e7899168b Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Wed, 23 Sep 2026 15:21:32 -0400 Subject: [PATCH 04/18] Fix Durable Functions replay edge cases --- .../AzureFunctionsWorkerInstrumentation.java | 38 ++--- .../worker/Base64StringInputStream.java | 34 ++++ .../worker/DurableFunctionsUtils.java | 145 ++++++++++++++---- .../worker/TraceContextExtractAdapter.java | 14 +- .../test/java/AzureFunctionsWorkerTest.java | 58 ++++++- .../api/AgentSpanContextTest.java | 15 ++ 6 files changed, 247 insertions(+), 57 deletions(-) create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/Base64StringInputStream.java create mode 100644 internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContextTest.java diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java index 8c2332ef049..b37e9e335cb 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -1,11 +1,7 @@ package datadog.trace.instrumentation.azure.functions.worker; import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; -import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.extractContextAndGetSpanContext; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.spanFromScope; -import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.AZURE_FUNCTIONS_REQUEST; import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.isPublic; @@ -13,13 +9,11 @@ import static net.bytebuddy.matcher.ElementMatchers.takesArguments; import com.google.auto.service.AutoService; -import com.microsoft.azure.functions.TraceContext; import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; import datadog.context.ContextScope; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; import net.bytebuddy.asm.Advice; @AutoService(InstrumenterModule.class) @@ -40,6 +34,7 @@ public String[] helperClassNames() { return new String[] { packageName + ".DurableFunctionsDecorator", packageName + ".DurableFunctionsUtils", + packageName + ".Base64StringInputStream", packageName + ".TraceContextExtractAdapter" }; } @@ -76,29 +71,30 @@ public static ContextScope onEnter(@Advice.Argument(0) MiddlewareContext context return null; } - final TraceContext traceContext = context.getTraceContext(); - AgentSpanContext.Extracted parent = - traceContext == null - ? null - : extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); - parent = DurableFunctionsUtils.reconcileSamplingPriority(parent, traceContext); - - final AgentSpan span = startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent); - DECORATE.afterStart(span); - DECORATE.onInvoke(span, context.getFunctionName(), trigger); - return activateSpan(span); + return DurableFunctionsUtils.startSpanScope(context, trigger); } @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void onExit( - @Advice.Enter ContextScope scope, @Advice.Thrown Throwable throwable) { - if (scope != null) { - final AgentSpan span = spanFromScope(scope); + @Advice.Argument(0) MiddlewareContext context, + @Advice.Enter ContextScope scope, + @Advice.Thrown Throwable throwable) { + ContextScope activeScope = scope; + if (activeScope == null + && throwable != null + && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { + final String trigger = DurableFunctionsUtils.getTrigger(context); + if ("DurableOrchestration".equals(trigger)) { + activeScope = DurableFunctionsUtils.startSpanScope(context, trigger); + } + } + if (activeScope != null) { + final AgentSpan span = spanFromScope(activeScope); if (!DurableFunctionsUtils.isReplayControlFlow(throwable)) { DECORATE.onError(span, throwable); } DECORATE.beforeFinish(span); - scope.close(); + activeScope.close(); span.finish(); } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/Base64StringInputStream.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/Base64StringInputStream.java new file mode 100644 index 00000000000..9ee4c414e75 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/Base64StringInputStream.java @@ -0,0 +1,34 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import java.io.InputStream; + +/** Exposes an ASCII Base64 string as a stream without copying it into a byte array. */ +public final class Base64StringInputStream extends InputStream { + private final String value; + private int position; + + public Base64StringInputStream(String value) { + this.value = value; + } + + @Override + public int read() { + return position < value.length() ? value.charAt(position++) : -1; + } + + @Override + public int read(byte[] buffer, int offset, int length) { + if (length == 0) { + return 0; + } + if (position >= value.length()) { + return -1; + } + final int count = Math.min(length, value.length() - position); + for (int index = 0; index < count; index++) { + buffer[offset + index] = (byte) value.charAt(position + index); + } + position += count; + return count; + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index 570cb2df037..ff0cbcc3c7f 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -2,10 +2,19 @@ import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; import static datadog.trace.api.sampling.PrioritySampling.UNSET; +import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.extractContextAndGetSpanContext; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.AZURE_FUNCTIONS_REQUEST; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; import com.microsoft.azure.functions.TraceContext; import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; +import datadog.context.ContextScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import java.io.IOException; +import java.io.InputStream; import java.util.Base64; public final class DurableFunctionsUtils { @@ -48,6 +57,20 @@ public static AgentSpanContext.Extracted reconcileSamplingPriority( return parent; } + public static ContextScope startSpanScope(MiddlewareContext context, String trigger) { + final TraceContext traceContext = context.getTraceContext(); + AgentSpanContext.Extracted parent = + traceContext == null + ? null + : extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); + parent = reconcileSamplingPriority(parent, traceContext); + + final AgentSpan span = startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent); + DECORATE.afterStart(span); + DECORATE.onInvoke(span, context.getFunctionName(), trigger); + return activateSpan(span); + } + public static boolean isReplayControlFlow(Throwable throwable) { while (throwable != null) { final String className = throwable.getClass().getName(); @@ -77,28 +100,42 @@ public static boolean shouldTraceOrchestration(MiddlewareContext context) { return true; } - final byte[] request = Base64.getDecoder().decode((String) parameterValue); + final InputStream request = + Base64.getDecoder().wrap(new Base64StringInputStream((String) parameterValue)); + final byte[] skipBuffer = new byte[512]; boolean replay = false; boolean newFailure = false; - final int[] position = {0}; - while (position[0] < request.length) { - final long tag = readVarint(request, position, request.length); + final long[] position = {0}; + while (true) { + final int first = request.read(); + if (first < 0) { + break; + } + position[0]++; + final long tag = readVarint(request, position, Long.MAX_VALUE, first); final int field = (int) (tag >>> 3); final int wireType = (int) (tag & 7); if (wireType == 2) { - final long length = readVarint(request, position, request.length); - if (length < 0 || length > request.length - position[0]) { - return true; - } - final int end = position[0] + (int) length; + final long length = readVarint(request, position, Long.MAX_VALUE); + final long end = checkedEnd(position[0], length); if (field == PAST_EVENTS_FIELD) { replay = true; - } else if (field == NEW_EVENTS_FIELD && containsFailureEvent(request, position[0], end)) { - newFailure = true; + skipFully(request, position, length, skipBuffer); + } else if (field == NEW_EVENTS_FIELD) { + if (containsFailureEvent(request, position, end, skipBuffer)) { + newFailure = true; + } + } else { + skipFully(request, position, length, skipBuffer); + } + if (position[0] != end) { + throw new IllegalArgumentException("Invalid length-delimited field"); } - position[0] = end; } else { - skipValue(request, position, request.length, wireType); + skipValue(request, position, Long.MAX_VALUE, wireType, skipBuffer); + } + if (replay && newFailure) { + return true; } } return !replay || newFailure; @@ -107,59 +144,103 @@ public static boolean shouldTraceOrchestration(MiddlewareContext context) { } } - private static boolean containsFailureEvent(byte[] data, int offset, int limit) { - final int[] position = {offset}; + private static boolean containsFailureEvent( + InputStream data, long[] position, long limit, byte[] skipBuffer) throws IOException { while (position[0] < limit) { final long tag = readVarint(data, position, limit); final int field = (int) (tag >>> 3); final int wireType = (int) (tag & 7); if (wireType == 2 && (field == TASK_FAILED_FIELD || field == SUB_ORCHESTRATION_FAILED_FIELD)) { + skipFully(data, position, limit - position[0], skipBuffer); return true; } - skipValue(data, position, limit, wireType); + skipValue(data, position, limit, wireType, skipBuffer); } return false; } - private static long readVarint(byte[] data, int[] position, int limit) { + private static long readVarint(InputStream data, long[] position, long limit) throws IOException { + if (position[0] >= limit) { + throw new IllegalArgumentException("Invalid protobuf varint"); + } + final int first = data.read(); + if (first < 0) { + throw new IllegalArgumentException("Invalid protobuf varint"); + } + position[0]++; + return readVarint(data, position, limit, first); + } + + private static long readVarint(InputStream data, long[] position, long limit, int first) + throws IOException { long value = 0; - for (int shift = 0; shift < 64 && position[0] < limit; shift += 7) { - final int current = data[position[0]++] & 0xff; + int current = first; + for (int shift = 0; shift < 64; shift += 7) { value |= (long) (current & 0x7f) << shift; if ((current & 0x80) == 0) { return value; } + if (position[0] >= limit) { + break; + } + current = data.read(); + if (current < 0) { + break; + } + position[0]++; } throw new IllegalArgumentException("Invalid protobuf varint"); } - private static void skipValue(byte[] data, int[] position, int limit, int wireType) { + private static void skipValue( + InputStream data, long[] position, long limit, int wireType, byte[] skipBuffer) + throws IOException { switch (wireType) { case 0: readVarint(data, position, limit); return; case 1: - if (limit - position[0] < 8) { - throw new IllegalArgumentException("Invalid fixed64 field"); - } - position[0] += 8; + skipWithinLimit(data, position, limit, 8, skipBuffer, "Invalid fixed64 field"); return; case 2: final long length = readVarint(data, position, limit); - if (length < 0 || length > limit - position[0]) { - throw new IllegalArgumentException("Invalid length-delimited field"); - } - position[0] += (int) length; + skipWithinLimit( + data, position, limit, length, skipBuffer, "Invalid length-delimited field"); return; case 5: - if (limit - position[0] < 4) { - throw new IllegalArgumentException("Invalid fixed32 field"); - } - position[0] += 4; + skipWithinLimit(data, position, limit, 4, skipBuffer, "Invalid fixed32 field"); return; default: throw new IllegalArgumentException("Unsupported protobuf wire type"); } } + + private static void skipWithinLimit( + InputStream data, long[] position, long limit, long length, byte[] skipBuffer, String error) + throws IOException { + if (length < 0 || length > limit - position[0]) { + throw new IllegalArgumentException(error); + } + skipFully(data, position, length, skipBuffer); + } + + private static void skipFully(InputStream data, long[] position, long length, byte[] skipBuffer) + throws IOException { + while (length > 0) { + final int read = data.read(skipBuffer, 0, (int) Math.min(length, skipBuffer.length)); + if (read < 0) { + throw new IllegalArgumentException("Truncated protobuf field"); + } + position[0] += read; + length -= read; + } + } + + private static long checkedEnd(long position, long length) { + if (length < 0 || length > Long.MAX_VALUE - position) { + throw new IllegalArgumentException("Invalid length-delimited field"); + } + return position + length; + } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java index d31e210635b..3ea6c52d457 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java @@ -40,20 +40,28 @@ static int datadogSamplingPriority(String tracestate) { memberEnd = tracestate.length(); } int start = memberStart; - while (start < memberEnd && tracestate.charAt(start) == ' ') { + while (start < memberEnd && isOptionalWhitespace(tracestate.charAt(start))) { start++; } - if (memberEnd - start >= 3 + int end = memberEnd; + while (end > start && isOptionalWhitespace(tracestate.charAt(end - 1))) { + end--; + } + if (end - start >= 3 && tracestate.charAt(start) == 'd' && tracestate.charAt(start + 1) == 'd' && tracestate.charAt(start + 2) == '=') { - return parseSamplingPriority(tracestate, start + 3, memberEnd); + return parseSamplingPriority(tracestate, start + 3, end); } memberStart = memberEnd + 1; } return UNSET; } + private static boolean isOptionalWhitespace(char value) { + return value == ' ' || value == '\t'; + } + private static int parseSamplingPriority(String value, int start, int end) { while (start < end) { int fieldEnd = value.indexOf(';', start); diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index c3b79300cab..6352a6d0b99 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -94,6 +94,26 @@ void doesNotTreatFailureFieldWithWrongProtobufWireTypeAsFailure() throws Excepti assertTraces(); } + @Test + void suppressesReplayWithLargeHistory() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + byte[] request = new byte[4103]; + // OrchestratorRequest { pastEvents: <4096 bytes>, newEvents: taskCompleted {} } + request[0] = 0x1a; + request[1] = (byte) 0x80; + request[2] = 0x20; + request[4099] = 0x22; + request[4100] = 0x02; + request[4101] = 0x3a; + request[4102] = 0x00; + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request)); + + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + + assertTraces(); + } + @Test void createsSpanForInitialOrchestrationExecution() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); @@ -126,6 +146,42 @@ void createsSpanForOrchestrationReplayWithFailure(int failureTag) throws Excepti assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); } + @Test + void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { taskCompleted: {} } } + when(context.getParameterValue("input")) + .thenReturn( + Base64.getEncoder().encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, 0x3a, 0x00})); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new IllegalStateException("replay failure"); + }) + .when(chain) + .doNext(context); + + assertThrows( + IllegalStateException.class, + () -> new FunctionExecutionMiddleware().invoke(context, chain)); + + assertTraces( + trace( + span() + .root() + .operationName(Pattern.compile(Pattern.quote(operation()))) + .resourceName("DurableOrchestration Orchestrator") + .type(DDSpanTypes.SERVERLESS) + .error(true) + .tags( + defaultTags(), + error(IllegalStateException.class, "replay failure"), + tag(Tags.COMPONENT, matches(Pattern.quote("azure-functions"))), + tag(Tags.SPAN_KIND, is(Tags.SPAN_KIND_SERVER)), + tag("aas.function.name", is("Orchestrator")), + tag("aas.function.trigger", is("DurableOrchestration"))))); + } + @ParameterizedTest(name = "{0}") // spotless:off @TableTest({ @@ -183,7 +239,7 @@ void honorsDatadogKeepDecisionWhenAzureClearsW3cSampledFlag() throws Exception { TraceContext traceContext = mock(TraceContext.class); when(traceContext.getTraceparent()) .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); - when(traceContext.getTracestate()).thenReturn("dd=s:2;o:rum"); + when(traceContext.getTracestate()).thenReturn("vendor=value,\tdd=s:2;o:rum\t"); when(context.getTraceContext()).thenReturn(traceContext); AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); MiddlewareChain chain = mock(MiddlewareChain.class); diff --git a/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContextTest.java b/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContextTest.java new file mode 100644 index 00000000000..aea6c034aae --- /dev/null +++ b/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContextTest.java @@ -0,0 +1,15 @@ +package datadog.trace.bootstrap.instrumentation.api; + +import static datadog.trace.api.sampling.PrioritySampling.UNSET; +import static org.junit.jupiter.api.Assertions.assertSame; + +import org.junit.jupiter.api.Test; + +class AgentSpanContextTest { + @Test + void extractedContextReturnsItselfWhenSamplingPriorityCannotBeReplaced() { + AgentSpanContext.Extracted context = NoopSpanContext.INSTANCE; + + assertSame(context, context.withSamplingPriority(UNSET)); + } +} From 8cd74a37b776afd96bf3f3dc510d1ae97c84c9ce Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Wed, 23 Sep 2026 16:36:31 -0400 Subject: [PATCH 05/18] Fix Azure Functions replay error handling --- .../AzureFunctionsWorkerInstrumentation.java | 9 +++- .../worker/DurableFunctionsUtils.java | 18 +++++++- .../test/java/AzureFunctionsWorkerTest.java | 41 +++++++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java index b37e9e335cb..cd0051a7930 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -3,6 +3,7 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.spanFromScope; import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.isPublic; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; @@ -61,13 +62,16 @@ public void methodAdvice(MethodTransformer transformer) { public static class InvokeAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) - public static ContextScope onEnter(@Advice.Argument(0) MiddlewareContext context) { + public static ContextScope onEnter( + @Advice.Argument(0) MiddlewareContext context, + @Advice.Local("startTimeMicros") long startTimeMicros) { final String trigger = DurableFunctionsUtils.getTrigger(context); if (trigger == null) { return null; } if ("DurableOrchestration".equals(trigger) && !DurableFunctionsUtils.shouldTraceOrchestration(context)) { + startTimeMicros = MILLISECONDS.toMicros(System.currentTimeMillis()); return null; } @@ -78,6 +82,7 @@ public static ContextScope onEnter(@Advice.Argument(0) MiddlewareContext context public static void onExit( @Advice.Argument(0) MiddlewareContext context, @Advice.Enter ContextScope scope, + @Advice.Local("startTimeMicros") long startTimeMicros, @Advice.Thrown Throwable throwable) { ContextScope activeScope = scope; if (activeScope == null @@ -85,7 +90,7 @@ public static void onExit( && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { final String trigger = DurableFunctionsUtils.getTrigger(context); if ("DurableOrchestration".equals(trigger)) { - activeScope = DurableFunctionsUtils.startSpanScope(context, trigger); + activeScope = DurableFunctionsUtils.startSpanScope(context, trigger, startTimeMicros); } } if (activeScope != null) { diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index ff0cbcc3c7f..6b48c3051e0 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.Base64; +import java.util.IdentityHashMap; public final class DurableFunctionsUtils { private static final String ACTIVITY_ANNOTATION = "DurableActivityTrigger"; @@ -58,6 +59,11 @@ public static AgentSpanContext.Extracted reconcileSamplingPriority( } public static ContextScope startSpanScope(MiddlewareContext context, String trigger) { + return startSpanScope(context, trigger, 0); + } + + public static ContextScope startSpanScope( + MiddlewareContext context, String trigger, long startTimeMicros) { final TraceContext traceContext = context.getTraceContext(); AgentSpanContext.Extracted parent = traceContext == null @@ -65,14 +71,22 @@ public static ContextScope startSpanScope(MiddlewareContext context, String trig : extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); parent = reconcileSamplingPriority(parent, traceContext); - final AgentSpan span = startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent); + final AgentSpan span = + startTimeMicros > 0 + ? startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent, startTimeMicros) + : startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent); DECORATE.afterStart(span); DECORATE.onInvoke(span, context.getFunctionName(), trigger); return activateSpan(span); } public static boolean isReplayControlFlow(Throwable throwable) { - while (throwable != null) { + if (throwable == null) { + return false; + } + + final IdentityHashMap seen = new IdentityHashMap<>(); + while (throwable != null && seen.put(throwable, Boolean.TRUE) == null) { final String className = throwable.getClass().getName(); if ("com.microsoft.durabletask.OrchestratorBlockedException".equals(className) || "com.microsoft.durabletask.interruption.OrchestratorBlockedException".equals(className) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index 6352a6d0b99..7c5fd9ed982 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -7,7 +7,9 @@ import static datadog.trace.api.sampling.PrioritySampling.UNSET; import static datadog.trace.test.junit.utils.assertions.Matchers.is; import static datadog.trace.test.junit.utils.assertions.Matchers.matches; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -28,9 +30,11 @@ import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.core.DDSpan; +import datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils; import java.util.Base64; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; import java.util.stream.Stream; import org.junit.jupiter.api.Test; @@ -154,8 +158,11 @@ void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { .thenReturn( Base64.getEncoder().encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, 0x3a, 0x00})); MiddlewareChain chain = mock(MiddlewareChain.class); + AtomicLong invocationStartMillis = new AtomicLong(); doAnswer( invocation -> { + invocationStartMillis.set(System.currentTimeMillis()); + Thread.sleep(25); throw new IllegalStateException("replay failure"); }) .when(chain) @@ -165,6 +172,11 @@ void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { IllegalStateException.class, () -> new FunctionExecutionMiddleware().invoke(context, chain)); + writer.waitForTraces(1); + DDSpan errorSpan = writer.firstTrace().get(0); + assertTrue(errorSpan.getStartTime() <= MILLISECONDS.toNanos(invocationStartMillis.get())); + assertTrue(errorSpan.getDurationNano() >= MILLISECONDS.toNanos(20)); + assertTraces( trace( span() @@ -182,6 +194,35 @@ void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { tag("aas.function.trigger", is("DurableOrchestration"))))); } + @Test + void stopsTraversingRepeatedExceptionCause() { + AtomicInteger causeReads = new AtomicInteger(); + Throwable[] cycle = new Throwable[2]; + cycle[0] = + new RuntimeException("first") { + @Override + public Throwable getCause() { + if (causeReads.incrementAndGet() > 2) { + throw new AssertionError("cause traversal did not terminate"); + } + return cycle[1]; + } + }; + cycle[1] = + new RuntimeException("second") { + @Override + public Throwable getCause() { + if (causeReads.incrementAndGet() > 2) { + throw new AssertionError("cause traversal did not terminate"); + } + return cycle[0]; + } + }; + + assertFalse(DurableFunctionsUtils.isReplayControlFlow(cycle[0])); + assertEquals(2, causeReads.get()); + } + @ParameterizedTest(name = "{0}") // spotless:off @TableTest({ From e159a3d17365a64fe336c626e46dfe6d780f4213 Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Wed, 23 Sep 2026 17:29:21 -0400 Subject: [PATCH 06/18] Address Azure Functions review feedback --- .../azure-functions-worker-2.7/build.gradle | 26 ----------- .../build.gradle.kts | 38 ++++++++++++++++ ...tream.java => AsciiStringInputStream.java} | 6 +-- .../AzureFunctionsWorkerInstrumentation.java | 2 +- .../worker/DurableFunctionsUtils.java | 2 +- .../test/java/AzureFunctionsWorkerTest.java | 45 +++++++------------ 6 files changed, 60 insertions(+), 59 deletions(-) delete mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts rename dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/{Base64StringInputStream.java => AsciiStringInputStream.java} (77%) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle deleted file mode 100644 index 930115252af..00000000000 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle +++ /dev/null @@ -1,26 +0,0 @@ -plugins { - id 'dd-trace-java.module.instrumentation' -} - -muzzle { - pass { - group = 'com.microsoft.azure.functions' - module = 'azure-functions-java-spi' - versions = '[1.0.0,)' - extraDependency 'com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0' - } -} - -addTestSuiteForDir('latestDepTest', 'test') - -dependencies { - compileOnly group: 'com.microsoft.azure.functions', name: 'azure-functions-java-core-library', version: '1.2.0' - compileOnly group: 'com.microsoft.azure.functions', name: 'azure-functions-java-spi', version: '1.0.0' - - testImplementation group: 'com.microsoft.azure.functions', name: 'azure-functions-java-core-library', version: '1.2.0' - testImplementation group: 'com.microsoft.azure.functions', name: 'azure-functions-java-spi', version: '1.0.0' - testImplementation libs.bundles.mockito - - latestDepTestImplementation group: 'com.microsoft.azure.functions', name: 'azure-functions-java-core-library', version: '+' - latestDepTestImplementation group: 'com.microsoft.azure.functions', name: 'azure-functions-java-spi', version: '+' -} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts new file mode 100644 index 00000000000..944def40f19 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts @@ -0,0 +1,38 @@ +import groovy.lang.Closure + +plugins { + id("dd-trace-java.module.instrumentation") +} + +muzzle { + pass { + group = "com.microsoft.azure.functions" + module = "azure-functions-java-spi" + versions = "[1.0.0,)" + extraDependency("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + } +} + +fun addTestSuiteForDir(name: String, directory: String) { + (project.extra["addTestSuiteForDir"] as Closure<*>).call(name, directory) +} + +addTestSuiteForDir("latestDepTest", "test") + +dependencies { + compileOnly("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + compileOnly("com.microsoft.azure.functions:azure-functions-java-spi:1.0.0") + + testImplementation("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + testImplementation("com.microsoft.azure.functions:azure-functions-java-spi:1.0.0") + testImplementation(libs.bundles.mockito) + + add( + "latestDepTestImplementation", + "com.microsoft.azure.functions:azure-functions-java-core-library:+" + ) + add( + "latestDepTestImplementation", + "com.microsoft.azure.functions:azure-functions-java-spi:+" + ) +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/Base64StringInputStream.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AsciiStringInputStream.java similarity index 77% rename from dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/Base64StringInputStream.java rename to dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AsciiStringInputStream.java index 9ee4c414e75..e26d6e9803f 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/Base64StringInputStream.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AsciiStringInputStream.java @@ -2,12 +2,12 @@ import java.io.InputStream; -/** Exposes an ASCII Base64 string as a stream without copying it into a byte array. */ -public final class Base64StringInputStream extends InputStream { +/** Exposes an ASCII string as a stream without copying it into a byte array. */ +public final class AsciiStringInputStream extends InputStream { private final String value; private int position; - public Base64StringInputStream(String value) { + public AsciiStringInputStream(String value) { this.value = value; } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java index cd0051a7930..ed827072bb7 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -35,7 +35,7 @@ public String[] helperClassNames() { return new String[] { packageName + ".DurableFunctionsDecorator", packageName + ".DurableFunctionsUtils", - packageName + ".Base64StringInputStream", + packageName + ".AsciiStringInputStream", packageName + ".TraceContextExtractAdapter" }; } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index 6b48c3051e0..3269eede800 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -115,7 +115,7 @@ public static boolean shouldTraceOrchestration(MiddlewareContext context) { } final InputStream request = - Base64.getDecoder().wrap(new Base64StringInputStream((String) parameterValue)); + Base64.getDecoder().wrap(new AsciiStringInputStream((String) parameterValue)); final byte[] skipBuffer = new byte[512]; boolean replay = false; boolean newFailure = false; diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index 7c5fd9ed982..a7b7f59f89c 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -22,8 +22,6 @@ import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain; import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; import com.microsoft.azure.functions.worker.chain.FunctionExecutionMiddleware; -import com.microsoft.durabletask.interruption.ContinueAsNewInterruption; -import com.microsoft.durabletask.interruption.OrchestratorBlockedException; import datadog.trace.agent.test.AbstractInstrumentationTest; import datadog.trace.agent.test.assertions.SpanMatcher; import datadog.trace.api.DDSpanTypes; @@ -47,14 +45,12 @@ abstract class AzureFunctionsWorkerTest extends AbstractInstrumentationTest { abstract String operation(); - // spotless:off @TableTest({ - "scenario | annotation | trigger", + "scenario | annotation | trigger ", "orchestration | DurableOrchestrationTrigger | DurableOrchestration", - "activity | DurableActivityTrigger | DurableActivity", - "entity | DurableEntityTrigger | DurableEntity" + "activity | DurableActivityTrigger | DurableActivity ", + "entity | DurableEntityTrigger | DurableEntity " }) - // spotless:on void createsSpanForDurableTrigger(String annotation, String trigger) throws Exception { MiddlewareContext context = contextFor(annotation, "MyFunction"); @@ -102,10 +98,11 @@ void doesNotTreatFailureFieldWithWrongProtobufWireTypeAsFailure() throws Excepti void suppressesReplayWithLargeHistory() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); byte[] request = new byte[4103]; - // OrchestratorRequest { pastEvents: <4096 bytes>, newEvents: taskCompleted {} } + // Field 3 (pastEvents), length-delimited with a two-byte varint length of 4096. request[0] = 0x1a; request[1] = (byte) 0x80; request[2] = 0x20; + // Field 4 (newEvents), containing HistoryEvent field 7 (taskCompleted) with an empty payload. request[4099] = 0x22; request[4100] = 0x02; request[4101] = 0x3a; @@ -130,13 +127,11 @@ void createsSpanForInitialOrchestrationExecution() throws Exception { assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); } - // spotless:off @TableTest({ "scenario | failureTag", - "failed activity | 66", - "failed sub-orchestration | 90" + "failed activity | 66 ", + "failed sub-orchestration | 90 " }) - // spotless:on void createsSpanForOrchestrationReplayWithFailure(int failureTag) throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { failure: {} } } @@ -224,13 +219,11 @@ public Throwable getCause() { } @ParameterizedTest(name = "{0}") - // spotless:off @TableTest({ - "scenario | description | payload", + "scenario | description | payload ", "non-base64 input | non-base64 input | 'not base64'", - "truncated protobuf | truncated protobuf | GgIA" + "truncated protobuf | truncated protobuf | GgIA " }) - // spotless:on @MethodSource("failsOpenForUnreadableOrchestrationPayloadArguments") void failsOpenForUnreadableOrchestrationPayload(String description, Object payload) throws Exception { @@ -301,10 +294,15 @@ void honorsDatadogKeepDecisionWhenAzureClearsW3cSampledFlag() throws Exception { assertTrue(priority.get() > 0); } - @ParameterizedTest(name = "{0}") - @MethodSource("doesNotMarkReplayControlFlowAsErrorArguments") - void doesNotMarkReplayControlFlowAsError(String description, Throwable controlFlow) + @TableTest({ + "scenario | controlFlowType ", + "legacy OrchestratorBlockedException | com.microsoft.durabletask.OrchestratorBlockedException ", + "OrchestratorBlockedException | com.microsoft.durabletask.interruption.OrchestratorBlockedException", + "ContinueAsNewInterruption | com.microsoft.durabletask.interruption.ContinueAsNewInterruption " + }) + void doesNotMarkReplayControlFlowAsError(Class controlFlowType) throws Exception { + Throwable controlFlow = controlFlowType.getDeclaredConstructor().newInstance(); MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); MiddlewareChain chain = mock(MiddlewareChain.class); doAnswer( @@ -320,15 +318,6 @@ void doesNotMarkReplayControlFlowAsError(String description, Throwable controlFl assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); } - static Stream doesNotMarkReplayControlFlowAsErrorArguments() { - return Stream.of( - arguments( - "legacy OrchestratorBlockedException", - new com.microsoft.durabletask.OrchestratorBlockedException()), - arguments("OrchestratorBlockedException", new OrchestratorBlockedException()), - arguments("ContinueAsNewInterruption", new ContinueAsNewInterruption())); - } - @Test void marksApplicationFailuresAsErrors() throws Exception { MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); From ab2c863fe914bcc54561a51f218559d592aa428d Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Wed, 23 Sep 2026 17:34:54 -0400 Subject: [PATCH 07/18] Sort Azure Functions helper classes --- .../functions/worker/AzureFunctionsWorkerInstrumentation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java index ed827072bb7..f676499197e 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -33,9 +33,9 @@ public String instrumentedType() { @Override public String[] helperClassNames() { return new String[] { + packageName + ".AsciiStringInputStream", packageName + ".DurableFunctionsDecorator", packageName + ".DurableFunctionsUtils", - packageName + ".AsciiStringInputStream", packageName + ".TraceContextExtractAdapter" }; } From 868f14c316528b7423c46b4fb32f8b60f7f6badb Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Thu, 24 Sep 2026 11:24:52 -0400 Subject: [PATCH 08/18] Scope Azure sampling workaround to durable functions --- .../worker/DurableFunctionsUtils.java | 17 +------- .../worker/TraceContextExtractAdapter.java | 6 ++- .../test/java/AzureFunctionsWorkerTest.java | 34 ++++++++++++++-- .../core/propagation/ExtractedContext.java | 11 ----- .../propagation/ExtractedContextTest.java | 40 ------------------- .../instrumentation/api/AgentSpanContext.java | 8 ---- .../instrumentation/api/TagContext.java | 6 +-- .../api/AgentSpanContextTest.java | 15 ------- 8 files changed, 38 insertions(+), 99 deletions(-) delete mode 100644 dd-trace-core/src/test/java/datadog/trace/core/propagation/ExtractedContextTest.java delete mode 100644 internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContextTest.java diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index 3269eede800..665307c77a0 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -1,7 +1,5 @@ package datadog.trace.instrumentation.azure.functions.worker; -import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; -import static datadog.trace.api.sampling.PrioritySampling.UNSET; import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.extractContextAndGetSpanContext; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; @@ -46,18 +44,6 @@ public static String getTrigger(MiddlewareContext context) { return null; } - public static AgentSpanContext.Extracted reconcileSamplingPriority( - AgentSpanContext.Extracted parent, TraceContext traceContext) { - if (parent != null - && parent.getSamplingPriority() == SAMPLER_DROP - && traceContext != null - && TraceContextExtractAdapter.datadogSamplingPriority(traceContext.getTracestate()) - == UNSET) { - return parent.withSamplingPriority(UNSET); - } - return parent; - } - public static ContextScope startSpanScope(MiddlewareContext context, String trigger) { return startSpanScope(context, trigger, 0); } @@ -65,11 +51,10 @@ public static ContextScope startSpanScope(MiddlewareContext context, String trig public static ContextScope startSpanScope( MiddlewareContext context, String trigger, long startTimeMicros) { final TraceContext traceContext = context.getTraceContext(); - AgentSpanContext.Extracted parent = + final AgentSpanContext.Extracted parent = traceContext == null ? null : extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); - parent = reconcileSamplingPriority(parent, traceContext); final AgentSpan span = startTimeMicros > 0 diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java index 3ea6c52d457..d210f9b66c7 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java @@ -18,7 +18,11 @@ private TraceContextExtractAdapter() {} public void forEachKey(TraceContext carrier, AgentPropagation.KeyClassifier classifier) { final String tracestate = carrier.getTracestate(); String traceparent = carrier.getTraceparent(); - if (datadogSamplingPriority(tracestate) > 0) { + final int datadogSamplingPriority = datadogSamplingPriority(tracestate); + if (datadogSamplingPriority == UNSET || datadogSamplingPriority > 0) { + // The Azure host can clear the sampled flag when host telemetry is disabled even though the + // trace identifiers are intended to correlate durable worker invocations. Restore the flag + // only in this durable-trigger extraction path, while preserving explicit Datadog drops. traceparent = setSampledFlag(traceparent); } if (traceparent != null && !classifier.accept(TRACEPARENT, traceparent)) { diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index a7b7f59f89c..de80b4b18a1 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -3,8 +3,8 @@ import static datadog.trace.agent.test.assertions.TagsMatcher.error; import static datadog.trace.agent.test.assertions.TagsMatcher.tag; import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; -import static datadog.trace.api.sampling.PrioritySampling.UNSET; import static datadog.trace.test.junit.utils.assertions.Matchers.is; import static datadog.trace.test.junit.utils.assertions.Matchers.matches; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -240,7 +240,7 @@ static Stream failsOpenForUnreadableOrchestrationPayloadArguments() { } @Test - void continuesAzureTraceAndIgnoresImplicitSamplingRejection() throws Exception { + void continuesAzureTraceWhenHostClearsW3cSampledFlag() throws Exception { MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); TraceContext traceContext = mock(TraceContext.class); when(traceContext.getTraceparent()) @@ -263,10 +263,38 @@ void continuesAzureTraceAndIgnoresImplicitSamplingRejection() throws Exception { DDSpan span = writer.firstTrace().get(0); assertEquals("42", span.getTraceId().toString()); assertEquals(43L, span.getParentId()); - assertEquals(UNSET, priority.get()); + assertEquals(SAMPLER_KEEP, priority.get()); assertEquals(SAMPLER_KEEP, span.samplingPriority()); } + @Test + void honorsDatadogDropDecisionWhenAzureClearsW3cSampledFlag() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); + when(traceContext.getTracestate()).thenReturn("dd=s:0"); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + + assertEquals(SAMPLER_DROP, priority.get()); + writer.waitForTraces(1); + DDSpan span = writer.firstTrace().get(0); + assertEquals("42", span.getTraceId().toString()); + assertEquals(43L, span.getParentId()); + assertEquals(SAMPLER_DROP, span.samplingPriority()); + } + @Test void honorsDatadogKeepDecisionWhenAzureClearsW3cSampledFlag() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); diff --git a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java index 7c48ac9e452..af503e6a6ed 100644 --- a/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java +++ b/dd-trace-core/src/main/java/datadog/trace/core/propagation/ExtractedContext.java @@ -1,7 +1,5 @@ package datadog.trace.core.propagation; -import static datadog.trace.api.sampling.SamplingMechanism.EXTERNAL_OVERRIDE; - import datadog.trace.api.DDTraceId; import datadog.trace.api.TagMap; import datadog.trace.api.TraceConfig; @@ -119,15 +117,6 @@ public PropagationTags getPropagationTags() { return propagationTags; } - @Override - public ExtractedContext withSamplingPriority(final int samplingPriority) { - setSamplingPriority(samplingPriority); - if (propagationTags != null) { - propagationTags.updateTraceSamplingPriority(samplingPriority, EXTERNAL_OVERRIDE); - } - return this; - } - @Override public String toString() { StringBuilder builder = new StringBuilder("ExtractedContext{"); diff --git a/dd-trace-core/src/test/java/datadog/trace/core/propagation/ExtractedContextTest.java b/dd-trace-core/src/test/java/datadog/trace/core/propagation/ExtractedContextTest.java deleted file mode 100644 index 130d5adef33..00000000000 --- a/dd-trace-core/src/test/java/datadog/trace/core/propagation/ExtractedContextTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package datadog.trace.core.propagation; - -import static datadog.trace.api.TracePropagationStyle.TRACECONTEXT; -import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; -import static datadog.trace.api.sampling.PrioritySampling.UNSET; -import static datadog.trace.api.sampling.SamplingMechanism.EXTERNAL_OVERRIDE; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertSame; - -import datadog.trace.api.DDTraceId; -import org.junit.jupiter.api.Test; - -class ExtractedContextTest { - @Test - void replacesSamplingPriorityWithoutChangingTraceIdentity() { - PropagationTags propagationTags = PropagationTags.factory().empty(); - propagationTags.updateTraceSamplingPriority(SAMPLER_DROP, EXTERNAL_OVERRIDE); - ExtractedContext original = - new ExtractedContext( - DDTraceId.from(42), 43, SAMPLER_DROP, null, propagationTags, TRACECONTEXT); - - ExtractedContext updated = original.withSamplingPriority(UNSET); - - assertSame(original, updated); - assertEquals(original.getTraceId(), updated.getTraceId()); - assertEquals(original.getSpanId(), updated.getSpanId()); - assertEquals(UNSET, updated.getSamplingPriority()); - assertEquals(UNSET, updated.getPropagationTags().getSamplingPriority()); - } - - @Test - void replacesSamplingPriorityWithoutPropagationTags() { - ExtractedContext context = - new ExtractedContext(DDTraceId.from(42), 43, SAMPLER_DROP, null, null, TRACECONTEXT); - - context.withSamplingPriority(UNSET); - - assertEquals(UNSET, context.getSamplingPriority()); - } -} diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java index fcde079c1ea..6e950925305 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContext.java @@ -72,14 +72,6 @@ default CharSequence getIntegrationName() { boolean isRemote(); interface Extracted extends AgentSpanContext { - /** - * Returns this extracted context with the supplied sampling priority when the implementation - * supports replacing propagation decisions. - */ - default Extracted withSamplingPriority(int samplingPriority) { - return this; - } - /** * Gets the span links related to the other terminated context. * diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/TagContext.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/TagContext.java index f7098e94aa2..078ffeb1625 100644 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/TagContext.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/TagContext.java @@ -33,7 +33,7 @@ public class TagContext implements AgentSpanContext.Extracted { private final HttpHeaders httpHeaders; private final Map baggage; private Baggage w3cBaggage; - private int samplingPriority; + private final int samplingPriority; private final TraceConfig traceConfig; private final TracePropagationStyle propagationStyle; private final DDTraceId traceId; @@ -181,10 +181,6 @@ public final int getSamplingPriority() { return samplingPriority; } - protected final void setSamplingPriority(int samplingPriority) { - this.samplingPriority = samplingPriority; - } - public final Map getBaggage() { return baggage; } diff --git a/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContextTest.java b/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContextTest.java deleted file mode 100644 index aea6c034aae..00000000000 --- a/internal-api/src/test/java/datadog/trace/bootstrap/instrumentation/api/AgentSpanContextTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package datadog.trace.bootstrap.instrumentation.api; - -import static datadog.trace.api.sampling.PrioritySampling.UNSET; -import static org.junit.jupiter.api.Assertions.assertSame; - -import org.junit.jupiter.api.Test; - -class AgentSpanContextTest { - @Test - void extractedContextReturnsItselfWhenSamplingPriorityCannotBeReplaced() { - AgentSpanContext.Extracted context = NoopSpanContext.INSTANCE; - - assertSame(context, context.withSamplingPriority(UNSET)); - } -} From 675b1ca459e31fe5cd5c6ac5de3a69d0344d4fba Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Thu, 24 Sep 2026 14:35:40 -0400 Subject: [PATCH 09/18] Align Azure worker test fixture with SPI --- .../functions/worker/chain/FunctionExecutionMiddleware.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java index 1cda6f23f27..5b8b9822d87 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java @@ -1,9 +1,11 @@ package com.microsoft.azure.functions.worker.chain; +import com.microsoft.azure.functions.internal.spi.middleware.Middleware; import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain; import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; -public class FunctionExecutionMiddleware { +public class FunctionExecutionMiddleware implements Middleware { + @Override public void invoke(MiddlewareContext context, MiddlewareChain chain) throws Exception { chain.doNext(context); } From 96637fd5557e476f76a254bd43d81fb141d6b708 Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Thu, 24 Sep 2026 15:38:34 -0400 Subject: [PATCH 10/18] Harden durable function replay tracing --- .../AzureFunctionsWorkerInstrumentation.java | 12 ++--- .../worker/DurableFunctionsUtils.java | 11 ++--- .../test/java/AzureFunctionsWorkerTest.java | 44 +++++++++++++++++-- 3 files changed, 51 insertions(+), 16 deletions(-) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java index f676499197e..efe806deff2 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -64,15 +64,17 @@ public static class InvokeAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static ContextScope onEnter( @Advice.Argument(0) MiddlewareContext context, + @Advice.Local("trigger") String trigger, @Advice.Local("startTimeMicros") long startTimeMicros) { - final String trigger = DurableFunctionsUtils.getTrigger(context); + trigger = DurableFunctionsUtils.getTrigger(context); if (trigger == null) { return null; } - if ("DurableOrchestration".equals(trigger) - && !DurableFunctionsUtils.shouldTraceOrchestration(context)) { + if ("DurableOrchestration".equals(trigger)) { startTimeMicros = MILLISECONDS.toMicros(System.currentTimeMillis()); - return null; + if (!DurableFunctionsUtils.shouldTraceOrchestration(context)) { + return null; + } } return DurableFunctionsUtils.startSpanScope(context, trigger); @@ -82,13 +84,13 @@ public static ContextScope onEnter( public static void onExit( @Advice.Argument(0) MiddlewareContext context, @Advice.Enter ContextScope scope, + @Advice.Local("trigger") String trigger, @Advice.Local("startTimeMicros") long startTimeMicros, @Advice.Thrown Throwable throwable) { ContextScope activeScope = scope; if (activeScope == null && throwable != null && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { - final String trigger = DurableFunctionsUtils.getTrigger(context); if ("DurableOrchestration".equals(trigger)) { activeScope = DurableFunctionsUtils.startSpanScope(context, trigger, startTimeMicros); } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index 665307c77a0..aab77d76563 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -127,9 +127,6 @@ public static boolean shouldTraceOrchestration(MiddlewareContext context) { } else { skipFully(request, position, length, skipBuffer); } - if (position[0] != end) { - throw new IllegalArgumentException("Invalid length-delimited field"); - } } else { skipValue(request, position, Long.MAX_VALUE, wireType, skipBuffer); } @@ -138,25 +135,25 @@ public static boolean shouldTraceOrchestration(MiddlewareContext context) { } } return !replay || newFailure; - } catch (Throwable ignored) { + } catch (Exception ignored) { return true; } } private static boolean containsFailureEvent( InputStream data, long[] position, long limit, byte[] skipBuffer) throws IOException { + boolean failure = false; while (position[0] < limit) { final long tag = readVarint(data, position, limit); final int field = (int) (tag >>> 3); final int wireType = (int) (tag & 7); if (wireType == 2 && (field == TASK_FAILED_FIELD || field == SUB_ORCHESTRATION_FAILED_FIELD)) { - skipFully(data, position, limit - position[0], skipBuffer); - return true; + failure = true; } skipValue(data, position, limit, wireType, skipBuffer); } - return false; + return failure; } private static long readVarint(InputStream data, long[] position, long limit) throws IOException { diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index de80b4b18a1..e5269da97a8 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -146,12 +146,38 @@ void createsSpanForOrchestrationReplayWithFailure(int failureTag) throws Excepti } @Test - void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { + void createsSpanWhenFailurePrecedesAnotherHistoryField() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); - // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { taskCompleted: {} } } + // OrchestratorRequest { + // pastEvents: {}, + // newEvents: HistoryEvent { taskFailed: {}, field 20: 5 } + // } when(context.getParameterValue("input")) .thenReturn( - Base64.getEncoder().encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, 0x3a, 0x00})); + Base64.getEncoder() + .encodeToString( + new byte[] {0x1a, 0x00, 0x22, 0x05, 0x42, 0x00, (byte) 0xa0, 0x01, 0x05})); + + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + + assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + } + + @Test + void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { taskCompleted: {} } } + String payload = + Base64.getEncoder().encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, 0x3a, 0x00}); + AtomicLong payloadReadStartMillis = new AtomicLong(); + doAnswer( + invocation -> { + payloadReadStartMillis.set(System.currentTimeMillis()); + Thread.sleep(25); + return payload; + }) + .when(context) + .getParameterValue("input"); MiddlewareChain chain = mock(MiddlewareChain.class); AtomicLong invocationStartMillis = new AtomicLong(); doAnswer( @@ -169,8 +195,9 @@ void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { writer.waitForTraces(1); DDSpan errorSpan = writer.firstTrace().get(0); + assertTrue(errorSpan.getStartTime() <= MILLISECONDS.toNanos(payloadReadStartMillis.get())); assertTrue(errorSpan.getStartTime() <= MILLISECONDS.toNanos(invocationStartMillis.get())); - assertTrue(errorSpan.getDurationNano() >= MILLISECONDS.toNanos(20)); + assertTrue(errorSpan.getDurationNano() >= MILLISECONDS.toNanos(45)); assertTraces( trace( @@ -239,6 +266,15 @@ static Stream failsOpenForUnreadableOrchestrationPayloadArguments() { return Stream.of(arguments("non-string input", new Object())); } + @Test + void doesNotSwallowErrorsWhileInspectingOrchestrationPayload() { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + when(context.getParameterValue("input")).thenThrow(new AssertionError("failure")); + + assertThrows( + AssertionError.class, () -> DurableFunctionsUtils.shouldTraceOrchestration(context)); + } + @Test void continuesAzureTraceWhenHostClearsW3cSampledFlag() throws Exception { MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); From c56b3ed10052522bc5f7571c5425ab0250485753 Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Tue, 29 Sep 2026 11:46:42 -0400 Subject: [PATCH 11/18] Remove Azure Functions sampling override --- .../worker/TraceContextExtractAdapter.java | 77 +---------------- .../test/java/AzureFunctionsWorkerTest.java | 85 ++++++++++++++++++- 2 files changed, 84 insertions(+), 78 deletions(-) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java index d210f9b66c7..19034ecb555 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java @@ -1,7 +1,5 @@ package datadog.trace.instrumentation.azure.functions.worker; -import static datadog.trace.api.sampling.PrioritySampling.UNSET; - import com.microsoft.azure.functions.TraceContext; import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; @@ -16,84 +14,13 @@ private TraceContextExtractAdapter() {} @Override public void forEachKey(TraceContext carrier, AgentPropagation.KeyClassifier classifier) { - final String tracestate = carrier.getTracestate(); - String traceparent = carrier.getTraceparent(); - final int datadogSamplingPriority = datadogSamplingPriority(tracestate); - if (datadogSamplingPriority == UNSET || datadogSamplingPriority > 0) { - // The Azure host can clear the sampled flag when host telemetry is disabled even though the - // trace identifiers are intended to correlate durable worker invocations. Restore the flag - // only in this durable-trigger extraction path, while preserving explicit Datadog drops. - traceparent = setSampledFlag(traceparent); - } + final String traceparent = carrier.getTraceparent(); if (traceparent != null && !classifier.accept(TRACEPARENT, traceparent)) { return; } + final String tracestate = carrier.getTracestate(); if (tracestate != null) { classifier.accept(TRACESTATE, tracestate); } } - - static int datadogSamplingPriority(String tracestate) { - if (tracestate == null) { - return UNSET; - } - int memberStart = 0; - while (memberStart < tracestate.length()) { - int memberEnd = tracestate.indexOf(',', memberStart); - if (memberEnd < 0) { - memberEnd = tracestate.length(); - } - int start = memberStart; - while (start < memberEnd && isOptionalWhitespace(tracestate.charAt(start))) { - start++; - } - int end = memberEnd; - while (end > start && isOptionalWhitespace(tracestate.charAt(end - 1))) { - end--; - } - if (end - start >= 3 - && tracestate.charAt(start) == 'd' - && tracestate.charAt(start + 1) == 'd' - && tracestate.charAt(start + 2) == '=') { - return parseSamplingPriority(tracestate, start + 3, end); - } - memberStart = memberEnd + 1; - } - return UNSET; - } - - private static boolean isOptionalWhitespace(char value) { - return value == ' ' || value == '\t'; - } - - private static int parseSamplingPriority(String value, int start, int end) { - while (start < end) { - int fieldEnd = value.indexOf(';', start); - if (fieldEnd < 0 || fieldEnd > end) { - fieldEnd = end; - } - if (fieldEnd - start > 2 && value.charAt(start) == 's' && value.charAt(start + 1) == ':') { - try { - return Integer.parseInt(value.substring(start + 2, fieldEnd)); - } catch (NumberFormatException ignored) { - return UNSET; - } - } - start = fieldEnd + 1; - } - return UNSET; - } - - private static String setSampledFlag(String traceparent) { - if (traceparent == null || traceparent.length() < 55) { - return traceparent; - } - final int high = Character.digit(traceparent.charAt(53), 16); - final int low = Character.digit(traceparent.charAt(54), 16); - if (high < 0 || low < 0 || (low & 1) != 0) { - return traceparent; - } - final char sampledLow = Character.forDigit(low | 1, 16); - return traceparent.substring(0, 54) + sampledLow + traceparent.substring(55); - } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index e5269da97a8..3c8dfb12256 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -276,7 +276,7 @@ void doesNotSwallowErrorsWhileInspectingOrchestrationPayload() { } @Test - void continuesAzureTraceWhenHostClearsW3cSampledFlag() throws Exception { + void honorsW3cDropDecisionWithoutTracestate() throws Exception { MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); TraceContext traceContext = mock(TraceContext.class); when(traceContext.getTraceparent()) @@ -295,6 +295,34 @@ void continuesAzureTraceWhenHostClearsW3cSampledFlag() throws Exception { new FunctionExecutionMiddleware().invoke(context, chain); + writer.waitForTraces(1); + DDSpan span = writer.firstTrace().get(0); + assertEquals("42", span.getTraceId().toString()); + assertEquals(43L, span.getParentId()); + assertEquals(SAMPLER_DROP, priority.get()); + assertEquals(SAMPLER_DROP, span.samplingPriority()); + } + + @Test + void honorsW3cKeepDecisionWithoutTracestate() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-01"); + when(traceContext.getTracestate()).thenReturn(null); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + writer.waitForTraces(1); DDSpan span = writer.firstTrace().get(0); assertEquals("42", span.getTraceId().toString()); @@ -332,7 +360,58 @@ void honorsDatadogDropDecisionWhenAzureClearsW3cSampledFlag() throws Exception { } @Test - void honorsDatadogKeepDecisionWhenAzureClearsW3cSampledFlag() throws Exception { + void honorsOtelDropDecisionWhenAzureClearsW3cSampledFlag() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); + when(traceContext.getTracestate()).thenReturn("ot=rv:00000000000000;th:8,vendor=value"); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + + assertEquals(SAMPLER_DROP, priority.get()); + writer.waitForTraces(1); + DDSpan span = writer.firstTrace().get(0); + assertEquals("42", span.getTraceId().toString()); + assertEquals(43L, span.getParentId()); + assertEquals(SAMPLER_DROP, span.samplingPriority()); + } + + @Test + void doesNotForceKeepForMalformedDatadogSamplingPriority() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); + when(traceContext.getTracestate()).thenReturn("dd=s:not-a-number"); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + + assertEquals(SAMPLER_DROP, priority.get()); + } + + @Test + void honorsW3cDropWhenDatadogTracestateConflicts() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); TraceContext traceContext = mock(TraceContext.class); when(traceContext.getTraceparent()) @@ -355,7 +434,7 @@ void honorsDatadogKeepDecisionWhenAzureClearsW3cSampledFlag() throws Exception { DDSpan span = writer.firstTrace().get(0); assertEquals("42", span.getTraceId().toString()); assertEquals(43L, span.getParentId()); - assertTrue(priority.get() > 0); + assertEquals(SAMPLER_DROP, priority.get()); } @TableTest({ From 9b6fc61294101efe21de9d1822cfafa34b8d4799 Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Tue, 29 Sep 2026 16:54:20 -0400 Subject: [PATCH 12/18] Avoid rescanning Azure Durable orchestration history --- .../build.gradle.kts | 26 ++ .../gradle.lockfile | 51 ++- .../worker/AsciiStringInputStream.java | 34 -- .../AzureFunctionsWorkerInstrumentation.java | 17 +- .../worker/DurableFunctionsDecorator.java | 6 +- .../worker/DurableFunctionsUtils.java | 192 ++--------- ...eOrchestrationExecutorInstrumentation.java | 79 +++++ ...rchestrationMiddlewareInstrumentation.java | 82 +++++ .../worker/DurableOrchestrationUtils.java | 55 ++++ .../test/java/AzureFunctionsWorkerTest.java | 299 +++++++++++------- .../OrchestratorBlockedException.java | 3 - .../ContinueAsNewInterruption.java | 3 - .../OrchestratorBlockedException.java | 3 - 13 files changed, 512 insertions(+), 338 deletions(-) delete mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AsciiStringInputStream.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java delete mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/OrchestratorBlockedException.java delete mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/ContinueAsNewInterruption.java delete mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/OrchestratorBlockedException.java diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts index 944def40f19..8140ce3ac6d 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts @@ -10,6 +10,24 @@ muzzle { module = "azure-functions-java-spi" versions = "[1.0.0,)" extraDependency("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + extraDependency("com.microsoft:durabletask-azure-functions:1.0.1") + extraDependency("com.microsoft:durabletask-client:1.0.0") + } + pass { + group = "com.microsoft" + module = "durabletask-client" + versions = "[1.0.0,)" + extraDependency("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + extraDependency("com.microsoft.azure.functions:azure-functions-java-spi:1.0.0") + extraDependency("com.microsoft:durabletask-azure-functions:1.0.1") + } + pass { + group = "com.microsoft" + module = "durabletask-azure-functions" + versions = "[1.0.1,)" + extraDependency("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + extraDependency("com.microsoft.azure.functions:azure-functions-java-spi:1.0.0") + extraDependency("com.microsoft:durabletask-client:1.0.0") } } @@ -20,13 +38,21 @@ fun addTestSuiteForDir(name: String, directory: String) { addTestSuiteForDir("latestDepTest", "test") dependencies { + compileOnly("com.google.protobuf:protobuf-java:3.19.2") + compileOnly("com.microsoft:durabletask-client:1.0.0") compileOnly("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") compileOnly("com.microsoft.azure.functions:azure-functions-java-spi:1.0.0") + testImplementation("com.microsoft:durabletask-azure-functions:1.0.1") + testImplementation("com.microsoft:durabletask-client:1.0.0") + testImplementation("com.google.protobuf:protobuf-java:3.19.2") testImplementation("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") testImplementation("com.microsoft.azure.functions:azure-functions-java-spi:1.0.0") testImplementation(libs.bundles.mockito) + add("latestDepTestImplementation", "com.microsoft:durabletask-azure-functions:+") + add("latestDepTestImplementation", "com.microsoft:durabletask-client:+") + add("latestDepTestImplementation", "com.google.protobuf:protobuf-java:+") add( "latestDepTestImplementation", "com.microsoft.azure.functions:azure-functions-java-core-library:+" diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile b/dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile index a8ef06fa2bb..fb7f62d5403 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile @@ -13,6 +13,15 @@ com.datadoghq:dd-instrument-java:0.0.5=buildTimeInstrumentationPlugin,compileCla com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath com.datadoghq:java-dogstatsd-client:4.4.5=latestDepTestRuntimeClasspath,testRuntimeClasspath com.datadoghq:sketches-java:0.8.3=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.12.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.18.3=latestDepTestRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.12.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.18.3=latestDepTestRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.12.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.18.3=latestDepTestRuntimeClasspath +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.12.3=testRuntimeClasspath +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.3=latestDepTestRuntimeClasspath +com.fasterxml.jackson:jackson-bom:2.18.3=latestDepTestRuntimeClasspath com.github.javaparser:javaparser-core:3.25.6=codenarc com.github.jnr:jffi:1.3.15=latestDepTestRuntimeClasspath,testRuntimeClasspath com.github.jnr:jnr-a64asm:1.0.0=latestDepTestRuntimeClasspath,testRuntimeClasspath @@ -22,14 +31,19 @@ com.github.jnr:jnr-ffi:2.2.19=latestDepTestRuntimeClasspath,testRuntimeClasspath com.github.jnr:jnr-posix:3.1.22=latestDepTestRuntimeClasspath,testRuntimeClasspath com.github.jnr:jnr-unixsocket:0.38.25=latestDepTestRuntimeClasspath,testRuntimeClasspath com.github.jnr:jnr-x86asm:1.0.2=latestDepTestRuntimeClasspath,testRuntimeClasspath -com.github.spotbugs:spotbugs-annotations:4.10.3=compileClasspath,spotbugs -com.github.spotbugs:spotbugs:4.10.3=spotbugs +com.github.spotbugs:spotbugs-annotations:4.10.4=compileClasspath,spotbugs +com.github.spotbugs:spotbugs:4.10.4=spotbugs com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.android:annotations:4.1.1.4=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.google.api.grpc:proto-google-common-protos:2.0.1=testRuntimeClasspath +com.google.api.grpc:proto-google-common-protos:2.63.1=latestDepTestRuntimeClasspath com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,testAnnotationProcessor,testCompileClasspath com.google.auto.service:auto-service:1.1.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor com.google.auto:auto-common:1.2.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.code.gson:gson:2.12.1=latestDepTestRuntimeClasspath com.google.code.gson:gson:2.14.0=spotbugs +com.google.code.gson:gson:2.8.9=testRuntimeClasspath com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor com.google.errorprone:error_prone_annotations:2.47.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath com.google.errorprone:error_prone_annotations:2.48.0=spotbugs @@ -40,11 +54,19 @@ com.google.guava:guava:33.6.0-jre=latestDepTestCompileClasspath,latestDepTestRun com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor com.google.j2objc:j2objc-annotations:3.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.google.protobuf:protobuf-java:3.19.2=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.google.protobuf:protobuf-java:4.36.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath com.google.re2j:re2j:1.8=latestDepTestRuntimeClasspath,testRuntimeClasspath com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0=compileClasspath,testCompileClasspath,testRuntimeClasspath com.microsoft.azure.functions:azure-functions-java-core-library:1.3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-library:2.2.0=testRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-library:3.2.3=latestDepTestRuntimeClasspath com.microsoft.azure.functions:azure-functions-java-spi:1.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath com.microsoft.azure.functions:azure-functions-java-spi:1.1.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath +com.microsoft:durabletask-azure-functions:1.0.1=testCompileClasspath,testRuntimeClasspath +com.microsoft:durabletask-azure-functions:1.9.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath +com.microsoft:durabletask-client:1.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.microsoft:durabletask-client:1.9.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath com.squareup.moshi:moshi:1.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath com.squareup.okhttp3:logging-interceptor:3.12.12=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath com.squareup.okhttp3:okhttp:3.12.12=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath @@ -53,8 +75,28 @@ com.thoughtworks.qdox:qdox:1.12.1=codenarc commons-fileupload:commons-fileupload:1.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath commons-io:commons-io:2.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath commons-io:commons-io:2.21.0=spotbugs -de.thetaphi:forbiddenapis:3.10=compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +de.thetaphi:forbiddenapis:3.11=compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.grpc:grpc-api:1.46.0=testRuntimeClasspath +io.grpc:grpc-api:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-context:1.46.0=testRuntimeClasspath +io.grpc:grpc-context:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-core:1.46.0=testRuntimeClasspath +io.grpc:grpc-core:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-netty-shaded:1.46.0=testRuntimeClasspath +io.grpc:grpc-netty-shaded:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-protobuf-lite:1.46.0=testRuntimeClasspath +io.grpc:grpc-protobuf-lite:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-protobuf:1.46.0=testRuntimeClasspath +io.grpc:grpc-protobuf:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-stub:1.46.0=testRuntimeClasspath +io.grpc:grpc-stub:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-util:1.78.0=latestDepTestRuntimeClasspath io.leangen.geantyref:geantyref:1.3.16=latestDepTestRuntimeClasspath,testRuntimeClasspath +io.opentelemetry:opentelemetry-api:1.58.0=latestDepTestRuntimeClasspath +io.opentelemetry:opentelemetry-common:1.58.0=latestDepTestRuntimeClasspath +io.opentelemetry:opentelemetry-context:1.58.0=latestDepTestRuntimeClasspath +io.perfmark:perfmark-api:0.25.0=testRuntimeClasspath +io.perfmark:perfmark-api:0.27.0=latestDepTestRuntimeClasspath io.sqreen:libsqreen:17.5.0=latestDepTestRuntimeClasspath,testRuntimeClasspath javax.servlet:javax.servlet-api:3.1.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath jaxen:jaxen:2.0.6=spotbugs @@ -82,6 +124,8 @@ org.codehaus.groovy:groovy-templates:3.0.23=codenarc org.codehaus.groovy:groovy-xml:3.0.23=codenarc org.codehaus.groovy:groovy:3.0.23=codenarc org.codehaus.groovy:groovy:3.0.25=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.codehaus.mojo:animal-sniffer-annotations:1.19=testRuntimeClasspath +org.codehaus.mojo:animal-sniffer-annotations:1.26=latestDepTestRuntimeClasspath org.codenarc:CodeNarc:3.7.0=codenarc org.dom4j:dom4j:2.2.0=spotbugs org.gmetrics:GMetrics:2.1.0=codenarc @@ -101,7 +145,6 @@ org.junit.platform:junit-platform-runner:1.14.1=latestDepTestRuntimeClasspath,te org.junit.platform:junit-platform-suite-api:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath org.junit.platform:junit-platform-suite-commons:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath org.junit:junit-bom:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -org.junit:junit-bom:6.1.2=spotbugs org.mockito:mockito-core:4.4.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath org.mockito:mockito-junit-jupiter:4.4.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath org.objenesis:objenesis:3.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AsciiStringInputStream.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AsciiStringInputStream.java deleted file mode 100644 index e26d6e9803f..00000000000 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AsciiStringInputStream.java +++ /dev/null @@ -1,34 +0,0 @@ -package datadog.trace.instrumentation.azure.functions.worker; - -import java.io.InputStream; - -/** Exposes an ASCII string as a stream without copying it into a byte array. */ -public final class AsciiStringInputStream extends InputStream { - private final String value; - private int position; - - public AsciiStringInputStream(String value) { - this.value = value; - } - - @Override - public int read() { - return position < value.length() ? value.charAt(position++) : -1; - } - - @Override - public int read(byte[] buffer, int offset, int length) { - if (length == 0) { - return 0; - } - if (position >= value.length()) { - return -1; - } - final int count = Math.min(length, value.length() - position); - for (int index = 0; index < count; index++) { - buffer[offset + index] = (byte) value.charAt(position + index); - } - position += count; - return count; - } -} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java index efe806deff2..f77ad7a6d63 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -33,7 +33,6 @@ public String instrumentedType() { @Override public String[] helperClassNames() { return new String[] { - packageName + ".AsciiStringInputStream", packageName + ".DurableFunctionsDecorator", packageName + ".DurableFunctionsUtils", packageName + ".TraceContextExtractAdapter" @@ -65,16 +64,18 @@ public static class InvokeAdvice { public static ContextScope onEnter( @Advice.Argument(0) MiddlewareContext context, @Advice.Local("trigger") String trigger, + @Advice.Local("orchestrationSpan") AgentSpan orchestrationSpan, @Advice.Local("startTimeMicros") long startTimeMicros) { trigger = DurableFunctionsUtils.getTrigger(context); if (trigger == null) { return null; } if ("DurableOrchestration".equals(trigger)) { - startTimeMicros = MILLISECONDS.toMicros(System.currentTimeMillis()); - if (!DurableFunctionsUtils.shouldTraceOrchestration(context)) { - return null; + orchestrationSpan = DurableFunctionsUtils.onOrchestrationInvoke(context); + if (orchestrationSpan == null) { + startTimeMicros = MILLISECONDS.toMicros(System.currentTimeMillis()); } + return null; } return DurableFunctionsUtils.startSpanScope(context, trigger); @@ -85,8 +86,16 @@ public static void onExit( @Advice.Argument(0) MiddlewareContext context, @Advice.Enter ContextScope scope, @Advice.Local("trigger") String trigger, + @Advice.Local("orchestrationSpan") AgentSpan orchestrationSpan, @Advice.Local("startTimeMicros") long startTimeMicros, @Advice.Thrown Throwable throwable) { + if (orchestrationSpan != null) { + if (throwable != null && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { + DECORATE.onError(orchestrationSpan, throwable); + } + return; + } + ContextScope activeScope = scope; if (activeScope == null && throwable != null diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java index f6ec65d193a..e67300a125a 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java @@ -39,8 +39,10 @@ protected void doAfterStart(AgentSpan span) { } public void onInvoke(AgentSpan span, String functionName, String trigger) { - span.setResourceName(trigger + " " + functionName); - span.setTag("aas.function.name", functionName); + span.setResourceName(functionName == null ? trigger : trigger + " " + functionName); + if (functionName != null) { + span.setTag("aas.function.name", functionName); + } span.setTag("aas.function.trigger", trigger); } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index aab77d76563..904d6bc551c 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -11,9 +11,6 @@ import datadog.context.ContextScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; -import java.io.IOException; -import java.io.InputStream; -import java.util.Base64; import java.util.IdentityHashMap; public final class DurableFunctionsUtils { @@ -21,14 +18,6 @@ public final class DurableFunctionsUtils { private static final String ENTITY_ANNOTATION = "DurableEntityTrigger"; private static final String ORCHESTRATION_ANNOTATION = "DurableOrchestrationTrigger"; - // OrchestratorRequest protobuf field numbers. - private static final int PAST_EVENTS_FIELD = 3; - private static final int NEW_EVENTS_FIELD = 4; - - // HistoryEvent protobuf field numbers. - private static final int TASK_FAILED_FIELD = 8; - private static final int SUB_ORCHESTRATION_FAILED_FIELD = 11; - private DurableFunctionsUtils() {} public static String getTrigger(MiddlewareContext context) { @@ -65,6 +54,30 @@ public static ContextScope startSpanScope( return activateSpan(span); } + public static ContextScope activateTraceContext(MiddlewareContext context) { + if (context.getParameterName(ORCHESTRATION_ANNOTATION) == null) { + return null; + } + final TraceContext traceContext = context.getTraceContext(); + if (traceContext == null) { + return null; + } + + final AgentSpanContext.Extracted extracted = + extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); + final AgentSpan remoteSpan = AgentSpan.fromSpanContext(extracted); + return remoteSpan.isValid() ? activateSpan(remoteSpan) : null; + } + + public static AgentSpan onOrchestrationInvoke(MiddlewareContext context) { + final AgentSpan span = AgentSpan.current(); + if (span != null && "DurableOrchestration".equals(span.getTag("aas.function.trigger"))) { + DECORATE.onInvoke(span, context.getFunctionName(), "DurableOrchestration"); + return span; + } + return null; + } + public static boolean isReplayControlFlow(Throwable throwable) { if (throwable == null) { return false; @@ -82,161 +95,4 @@ public static boolean isReplayControlFlow(Throwable throwable) { } return false; } - - /** - * Returns whether an orchestration invocation should produce a span. - * - *

Durable orchestrators replay from the beginning whenever new history arrives. Those - * executions are implementation details rather than new logical operations, so successful replays - * are suppressed. A replay containing a newly failed activity or sub-orchestration is retained so - * the failure remains visible. If the trigger payload cannot be inspected, tracing fails open. - */ - public static boolean shouldTraceOrchestration(MiddlewareContext context) { - try { - final String parameterName = context.getParameterName(ORCHESTRATION_ANNOTATION); - final Object parameterValue = context.getParameterValue(parameterName); - if (!(parameterValue instanceof String)) { - return true; - } - - final InputStream request = - Base64.getDecoder().wrap(new AsciiStringInputStream((String) parameterValue)); - final byte[] skipBuffer = new byte[512]; - boolean replay = false; - boolean newFailure = false; - final long[] position = {0}; - while (true) { - final int first = request.read(); - if (first < 0) { - break; - } - position[0]++; - final long tag = readVarint(request, position, Long.MAX_VALUE, first); - final int field = (int) (tag >>> 3); - final int wireType = (int) (tag & 7); - if (wireType == 2) { - final long length = readVarint(request, position, Long.MAX_VALUE); - final long end = checkedEnd(position[0], length); - if (field == PAST_EVENTS_FIELD) { - replay = true; - skipFully(request, position, length, skipBuffer); - } else if (field == NEW_EVENTS_FIELD) { - if (containsFailureEvent(request, position, end, skipBuffer)) { - newFailure = true; - } - } else { - skipFully(request, position, length, skipBuffer); - } - } else { - skipValue(request, position, Long.MAX_VALUE, wireType, skipBuffer); - } - if (replay && newFailure) { - return true; - } - } - return !replay || newFailure; - } catch (Exception ignored) { - return true; - } - } - - private static boolean containsFailureEvent( - InputStream data, long[] position, long limit, byte[] skipBuffer) throws IOException { - boolean failure = false; - while (position[0] < limit) { - final long tag = readVarint(data, position, limit); - final int field = (int) (tag >>> 3); - final int wireType = (int) (tag & 7); - if (wireType == 2 - && (field == TASK_FAILED_FIELD || field == SUB_ORCHESTRATION_FAILED_FIELD)) { - failure = true; - } - skipValue(data, position, limit, wireType, skipBuffer); - } - return failure; - } - - private static long readVarint(InputStream data, long[] position, long limit) throws IOException { - if (position[0] >= limit) { - throw new IllegalArgumentException("Invalid protobuf varint"); - } - final int first = data.read(); - if (first < 0) { - throw new IllegalArgumentException("Invalid protobuf varint"); - } - position[0]++; - return readVarint(data, position, limit, first); - } - - private static long readVarint(InputStream data, long[] position, long limit, int first) - throws IOException { - long value = 0; - int current = first; - for (int shift = 0; shift < 64; shift += 7) { - value |= (long) (current & 0x7f) << shift; - if ((current & 0x80) == 0) { - return value; - } - if (position[0] >= limit) { - break; - } - current = data.read(); - if (current < 0) { - break; - } - position[0]++; - } - throw new IllegalArgumentException("Invalid protobuf varint"); - } - - private static void skipValue( - InputStream data, long[] position, long limit, int wireType, byte[] skipBuffer) - throws IOException { - switch (wireType) { - case 0: - readVarint(data, position, limit); - return; - case 1: - skipWithinLimit(data, position, limit, 8, skipBuffer, "Invalid fixed64 field"); - return; - case 2: - final long length = readVarint(data, position, limit); - skipWithinLimit( - data, position, limit, length, skipBuffer, "Invalid length-delimited field"); - return; - case 5: - skipWithinLimit(data, position, limit, 4, skipBuffer, "Invalid fixed32 field"); - return; - default: - throw new IllegalArgumentException("Unsupported protobuf wire type"); - } - } - - private static void skipWithinLimit( - InputStream data, long[] position, long limit, long length, byte[] skipBuffer, String error) - throws IOException { - if (length < 0 || length > limit - position[0]) { - throw new IllegalArgumentException(error); - } - skipFully(data, position, length, skipBuffer); - } - - private static void skipFully(InputStream data, long[] position, long length, byte[] skipBuffer) - throws IOException { - while (length > 0) { - final int read = data.read(skipBuffer, 0, (int) Math.min(length, skipBuffer.length)); - if (read < 0) { - throw new IllegalArgumentException("Truncated protobuf field"); - } - position[0] += read; - length -= read; - } - } - - private static long checkedEnd(long position, long length) { - if (length < 0 || length > Long.MAX_VALUE - position) { - throw new IllegalArgumentException("Invalid length-delimited field"); - } - return position + length; - } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java new file mode 100644 index 00000000000..451a3e343da --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java @@ -0,0 +1,79 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.spanFromScope; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +import com.google.auto.service.AutoService; +import datadog.context.ContextScope; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.agent.tooling.muzzle.Reference; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.util.List; +import net.bytebuddy.asm.Advice; + +@AutoService(InstrumenterModule.class) +public final class DurableOrchestrationExecutorInstrumentation extends InstrumenterModule.Tracing + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + private static final Reference EXECUTOR_REFERENCE = + new Reference.Builder("com.microsoft.durabletask.TaskOrchestrationExecutor").build(); + + public DurableOrchestrationExecutorInstrumentation() { + super("azure-functions"); + } + + @Override + public String instrumentedType() { + return "com.microsoft.durabletask.TaskOrchestrationExecutor"; + } + + @Override + public String[] helperClassNames() { + return new String[] { + packageName + ".DurableFunctionsDecorator", packageName + ".DurableOrchestrationUtils" + }; + } + + @Override + public Reference[] additionalMuzzleReferences() { + return new Reference[] {EXECUTOR_REFERENCE}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(isPublic()) + .and(named("execute")) + .and(takesArgument(0, named("java.util.List"))) + .and(takesArgument(1, named("java.util.List"))), + DurableOrchestrationExecutorInstrumentation.class.getName() + "$ExecuteAdvice"); + } + + public static class ExecuteAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static ContextScope onEnter( + @Advice.Argument(0) List pastEvents, @Advice.Argument(1) List newEvents) { + return DurableOrchestrationUtils.shouldTrace(pastEvents, newEvents) + ? DurableOrchestrationUtils.startSpanScope() + : null; + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.Enter ContextScope scope, @Advice.Thrown Throwable throwable) { + if (scope != null) { + final AgentSpan span = spanFromScope(scope); + DECORATE.onError(span, throwable); + DECORATE.beforeFinish(span); + scope.close(); + span.finish(); + } + } + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java new file mode 100644 index 00000000000..f7d335e4415 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java @@ -0,0 +1,82 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; +import datadog.context.ContextScope; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.agent.tooling.muzzle.Reference; +import net.bytebuddy.asm.Advice; + +@AutoService(InstrumenterModule.class) +public final class DurableOrchestrationMiddlewareInstrumentation extends InstrumenterModule.Tracing + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + private static final Reference MIDDLEWARE_REFERENCE = + new Reference.Builder( + "com.microsoft.durabletask.azurefunctions.internal.middleware.OrchestrationMiddleware") + .build(); + + public DurableOrchestrationMiddlewareInstrumentation() { + super("azure-functions"); + } + + @Override + public String instrumentedType() { + return "com.microsoft.durabletask.azurefunctions.internal.middleware.OrchestrationMiddleware"; + } + + @Override + public String[] helperClassNames() { + return new String[] { + packageName + ".DurableFunctionsDecorator", + packageName + ".DurableFunctionsUtils", + packageName + ".TraceContextExtractAdapter" + }; + } + + @Override + public Reference[] additionalMuzzleReferences() { + return new Reference[] {MIDDLEWARE_REFERENCE}; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(isPublic()) + .and(named("invoke")) + .and(takesArguments(2)) + .and( + takesArgument( + 0, + named( + "com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext"))) + .and( + takesArgument( + 1, + named( + "com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain"))), + DurableOrchestrationMiddlewareInstrumentation.class.getName() + "$InvokeAdvice"); + } + + public static class InvokeAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static ContextScope onEnter(@Advice.Argument(0) MiddlewareContext context) { + return DurableFunctionsUtils.activateTraceContext(context); + } + + @Advice.OnMethodExit(suppress = Throwable.class) + public static void onExit(@Advice.Enter ContextScope scope) { + if (scope != null) { + scope.close(); + } + } + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java new file mode 100644 index 00000000000..7a23bf29a72 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java @@ -0,0 +1,55 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.AZURE_FUNCTIONS_REQUEST; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; + +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.HistoryEvent; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.HistoryEvent.EventTypeCase; +import datadog.context.ContextScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.util.List; + +public final class DurableOrchestrationUtils { + private static final String ORCHESTRATION_TRIGGER = "DurableOrchestration"; + + private DurableOrchestrationUtils() {} + + /** + * Returns whether an orchestration invocation should produce a span. + * + *

The Durable SDK has already decoded the request by this point. Successful replays are + * suppressed, while a replay containing a newly failed activity or sub-orchestration is retained. + * Only new events are inspected, so each history event is examined once instead of rescanning the + * complete history on every replay. + */ + public static boolean shouldTrace(List pastEvents, List newEvents) { + if (pastEvents == null || pastEvents.isEmpty()) { + return true; + } + if (newEvents == null) { + return true; + } + + for (int index = 0; index < newEvents.size(); index++) { + final Object event = newEvents.get(index); + if (!(event instanceof HistoryEvent)) { + return true; + } + final EventTypeCase eventType = ((HistoryEvent) event).getEventTypeCase(); + if (eventType == EventTypeCase.TASKFAILED + || eventType == EventTypeCase.SUBORCHESTRATIONINSTANCEFAILED) { + return true; + } + } + return false; + } + + public static ContextScope startSpanScope() { + final AgentSpan span = startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST); + DECORATE.afterStart(span); + DECORATE.onInvoke(span, null, ORCHESTRATION_TRIGGER); + return activateSpan(span); + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index 3c8dfb12256..05d356cfcf3 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -12,16 +12,25 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import com.microsoft.azure.functions.TraceContext; import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain; import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; import com.microsoft.azure.functions.worker.chain.FunctionExecutionMiddleware; +import com.microsoft.durabletask.azurefunctions.internal.middleware.OrchestrationMiddleware; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.ExecutionStartedEvent; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.HistoryEvent; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.OrchestrationInstance; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.OrchestratorRequest; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.SubOrchestrationInstanceFailedEvent; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskCompletedEvent; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskFailedEvent; import datadog.trace.agent.test.AbstractInstrumentationTest; import datadog.trace.agent.test.assertions.SpanMatcher; import datadog.trace.api.DDSpanTypes; @@ -29,16 +38,16 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.core.DDSpan; import datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils; +import datadog.trace.instrumentation.azure.functions.worker.DurableOrchestrationUtils; +import java.lang.reflect.Constructor; import java.util.Base64; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; -import java.util.stream.Stream; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; import org.tabletest.junit.TableTest; abstract class AzureFunctionsWorkerTest extends AbstractInstrumentationTest { @@ -54,7 +63,15 @@ abstract class AzureFunctionsWorkerTest extends AbstractInstrumentationTest { void createsSpanForDurableTrigger(String annotation, String trigger) throws Exception { MiddlewareContext context = contextFor(annotation, "MyFunction"); - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + if ("DurableOrchestration".equals(trigger)) { + invokeOrchestration( + context, + Collections.emptyList(), + Collections.singletonList(executionStarted()), + mock(MiddlewareChain.class)); + } else { + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + } assertTraces(trace(durableSpan("MyFunction", trigger))); } @@ -71,113 +88,105 @@ void doesNotCreateSpanForNonDurableFunction() throws Exception { @Test void doesNotCreateSpanForSuccessfulOrchestrationReplay() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); - // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { taskCompleted: {} } } - when(context.getParameterValue("input")) - .thenReturn( - Base64.getEncoder().encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, 0x3a, 0x00})); - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + invokeOrchestration( + context, + Collections.singletonList(executionStarted()), + Collections.singletonList(taskCompleted()), + mock(MiddlewareChain.class)); assertTraces(); } @Test - void doesNotTreatFailureFieldWithWrongProtobufWireTypeAsFailure() throws Exception { - MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); - // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { field 8: varint 0 } } - when(context.getParameterValue("input")) - .thenReturn( - Base64.getEncoder().encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, 0x40, 0x00})); + @SuppressWarnings("unchecked") + void doesNotTraversePastEventsForReplayDecision() { + List pastEvents = mock(List.class); + when(pastEvents.isEmpty()).thenReturn(false); - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + assertFalse( + DurableOrchestrationUtils.shouldTrace( + pastEvents, Collections.singletonList(taskCompleted()))); - assertTraces(); + verify(pastEvents).isEmpty(); + verifyNoMoreInteractions(pastEvents); } @Test - void suppressesReplayWithLargeHistory() throws Exception { - MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); - byte[] request = new byte[4103]; - // Field 3 (pastEvents), length-delimited with a two-byte varint length of 4096. - request[0] = 0x1a; - request[1] = (byte) 0x80; - request[2] = 0x20; - // Field 4 (newEvents), containing HistoryEvent field 7 (taskCompleted) with an empty payload. - request[4099] = 0x22; - request[4100] = 0x02; - request[4101] = 0x3a; - request[4102] = 0x00; - when(context.getParameterValue("input")) - .thenReturn(Base64.getEncoder().encodeToString(request)); - - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); - - assertTraces(); + void failsOpenForUnexpectedParsedEvent() { + assertTrue( + DurableOrchestrationUtils.shouldTrace( + Collections.singletonList(executionStarted()), + Collections.singletonList(new Object()))); } @Test void createsSpanForInitialOrchestrationExecution() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); - // OrchestratorRequest { newEvents: HistoryEvent { executionStarted: {} } } - when(context.getParameterValue("input")) - .thenReturn(Base64.getEncoder().encodeToString(new byte[] {0x22, 0x02, 0x1a, 0x00})); - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + invokeOrchestration( + context, + Collections.emptyList(), + Collections.singletonList(executionStarted()), + mock(MiddlewareChain.class)); assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); } @TableTest({ - "scenario | failureTag", - "failed activity | 66 ", - "failed sub-orchestration | 90 " + "scenario | subOrchestration", + "failed activity | false ", + "failed sub-orchestration | true " }) - void createsSpanForOrchestrationReplayWithFailure(int failureTag) throws Exception { + void createsSpanForOrchestrationReplayWithFailure(boolean subOrchestration) throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); - // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { failure: {} } } - when(context.getParameterValue("input")) - .thenReturn( - Base64.getEncoder() - .encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, (byte) failureTag, 0x00})); - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + invokeOrchestration( + context, + Collections.singletonList(executionStarted()), + Collections.singletonList(failureEvent(subOrchestration)), + mock(MiddlewareChain.class)); assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); } @Test - void createsSpanWhenFailurePrecedesAnotherHistoryField() throws Exception { + void marksRetainedOrchestrationFailureAsError() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); - // OrchestratorRequest { - // pastEvents: {}, - // newEvents: HistoryEvent { taskFailed: {}, field 20: 5 } - // } - when(context.getParameterValue("input")) - .thenReturn( - Base64.getEncoder() - .encodeToString( - new byte[] {0x1a, 0x00, 0x22, 0x05, 0x42, 0x00, (byte) 0xa0, 0x01, 0x05})); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new IllegalStateException("activity failure"); + }) + .when(chain) + .doNext(context); - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + invokeOrchestration( + context, + Collections.singletonList(executionStarted()), + Collections.singletonList(failureEvent(false)), + chain); - assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + assertTraces( + trace( + span() + .root() + .operationName(Pattern.compile(Pattern.quote(operation()))) + .resourceName("DurableOrchestration Orchestrator") + .type(DDSpanTypes.SERVERLESS) + .error(true) + .tags( + defaultTags(), + error(IllegalStateException.class, "activity failure"), + tag(Tags.COMPONENT, matches(Pattern.quote("azure-functions"))), + tag(Tags.SPAN_KIND, is(Tags.SPAN_KIND_SERVER)), + tag("aas.function.name", is("Orchestrator")), + tag("aas.function.trigger", is("DurableOrchestration"))))); } @Test void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); - // OrchestratorRequest { pastEvents: {}, newEvents: HistoryEvent { taskCompleted: {} } } - String payload = - Base64.getEncoder().encodeToString(new byte[] {0x1a, 0x00, 0x22, 0x02, 0x3a, 0x00}); - AtomicLong payloadReadStartMillis = new AtomicLong(); - doAnswer( - invocation -> { - payloadReadStartMillis.set(System.currentTimeMillis()); - Thread.sleep(25); - return payload; - }) - .when(context) - .getParameterValue("input"); MiddlewareChain chain = mock(MiddlewareChain.class); AtomicLong invocationStartMillis = new AtomicLong(); doAnswer( @@ -189,15 +198,16 @@ void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { .when(chain) .doNext(context); - assertThrows( - IllegalStateException.class, - () -> new FunctionExecutionMiddleware().invoke(context, chain)); + invokeOrchestration( + context, + Collections.singletonList(executionStarted()), + Collections.singletonList(taskCompleted()), + chain); writer.waitForTraces(1); DDSpan errorSpan = writer.firstTrace().get(0); - assertTrue(errorSpan.getStartTime() <= MILLISECONDS.toNanos(payloadReadStartMillis.get())); assertTrue(errorSpan.getStartTime() <= MILLISECONDS.toNanos(invocationStartMillis.get())); - assertTrue(errorSpan.getDurationNano() >= MILLISECONDS.toNanos(45)); + assertTrue(errorSpan.getDurationNano() >= MILLISECONDS.toNanos(20)); assertTraces( trace( @@ -245,36 +255,6 @@ public Throwable getCause() { assertEquals(2, causeReads.get()); } - @ParameterizedTest(name = "{0}") - @TableTest({ - "scenario | description | payload ", - "non-base64 input | non-base64 input | 'not base64'", - "truncated protobuf | truncated protobuf | GgIA " - }) - @MethodSource("failsOpenForUnreadableOrchestrationPayloadArguments") - void failsOpenForUnreadableOrchestrationPayload(String description, Object payload) - throws Exception { - MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); - when(context.getParameterValue("input")).thenReturn(payload); - - new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); - - assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); - } - - static Stream failsOpenForUnreadableOrchestrationPayloadArguments() { - return Stream.of(arguments("non-string input", new Object())); - } - - @Test - void doesNotSwallowErrorsWhileInspectingOrchestrationPayload() { - MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); - when(context.getParameterValue("input")).thenThrow(new AssertionError("failure")); - - assertThrows( - AssertionError.class, () -> DurableFunctionsUtils.shouldTraceOrchestration(context)); - } - @Test void honorsW3cDropDecisionWithoutTracestate() throws Exception { MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); @@ -428,7 +408,8 @@ void honorsW3cDropWhenDatadogTracestateConflicts() throws Exception { .when(chain) .doNext(context); - new FunctionExecutionMiddleware().invoke(context, chain); + invokeOrchestration( + context, Collections.emptyList(), Collections.singletonList(executionStarted()), chain); writer.waitForTraces(1); DDSpan span = writer.firstTrace().get(0); @@ -438,14 +419,20 @@ void honorsW3cDropWhenDatadogTracestateConflicts() throws Exception { } @TableTest({ - "scenario | controlFlowType ", + "scenario | controlFlowTypeName ", "legacy OrchestratorBlockedException | com.microsoft.durabletask.OrchestratorBlockedException ", "OrchestratorBlockedException | com.microsoft.durabletask.interruption.OrchestratorBlockedException", "ContinueAsNewInterruption | com.microsoft.durabletask.interruption.ContinueAsNewInterruption " }) - void doesNotMarkReplayControlFlowAsError(Class controlFlowType) - throws Exception { - Throwable controlFlow = controlFlowType.getDeclaredConstructor().newInstance(); + @SuppressWarnings("unchecked") + void doesNotMarkReplayControlFlowAsError(String controlFlowTypeName) throws Exception { + Class controlFlowType; + try { + controlFlowType = (Class) Class.forName(controlFlowTypeName); + } catch (ClassNotFoundException ignored) { + return; + } + Throwable controlFlow = instantiateThrowable(controlFlowType); MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); MiddlewareChain chain = mock(MiddlewareChain.class); doAnswer( @@ -455,8 +442,8 @@ void doesNotMarkReplayControlFlowAsError(Class controlFlowT .when(chain) .doNext(context); - assertThrows( - RuntimeException.class, () -> new FunctionExecutionMiddleware().invoke(context, chain)); + invokeOrchestration( + context, Collections.emptyList(), Collections.singletonList(executionStarted()), chain); assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); } @@ -508,6 +495,84 @@ private SpanMatcher durableSpan(String functionName, String trigger) { tag("aas.function.trigger", is(trigger))); } + private static void invokeOrchestration( + MiddlewareContext context, + List pastEvents, + List newEvents, + MiddlewareChain functionChain) + throws Exception { + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("test-instance") + .addAllPastEvents(pastEvents) + .addAllNewEvents(newEvents) + .build(); + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request.toByteArray())); + + MiddlewareChain orchestrationChain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + new FunctionExecutionMiddleware().invoke(context, functionChain); + return null; + }) + .when(orchestrationChain) + .doNext(context); + + new OrchestrationMiddleware().invoke(context, orchestrationChain); + } + + private static HistoryEvent executionStarted() { + return HistoryEvent.newBuilder() + .setExecutionStarted( + ExecutionStartedEvent.newBuilder() + .setName("Orchestrator") + .setOrchestrationInstance( + OrchestrationInstance.newBuilder().setInstanceId("test-instance"))) + .build(); + } + + private static HistoryEvent taskCompleted() { + return HistoryEvent.newBuilder() + .setTaskCompleted(TaskCompletedEvent.getDefaultInstance()) + .build(); + } + + private static HistoryEvent failureEvent(boolean subOrchestration) { + HistoryEvent.Builder event = HistoryEvent.newBuilder(); + if (subOrchestration) { + event.setSubOrchestrationInstanceFailed( + SubOrchestrationInstanceFailedEvent.getDefaultInstance()); + } else { + event.setTaskFailed(TaskFailedEvent.getDefaultInstance()); + } + return event.build(); + } + + private static Throwable instantiateThrowable(Class type) throws Exception { + Constructor constructor = type.getDeclaredConstructors()[0]; + for (Constructor candidate : type.getDeclaredConstructors()) { + if (candidate.getParameterCount() < constructor.getParameterCount()) { + constructor = candidate; + } + } + constructor.setAccessible(true); + Object[] arguments = new Object[constructor.getParameterCount()]; + Class[] parameterTypes = constructor.getParameterTypes(); + for (int index = 0; index < parameterTypes.length; index++) { + if (parameterTypes[index] == String.class) { + arguments[index] = "test"; + } else if (parameterTypes[index] == boolean.class) { + arguments[index] = false; + } else if (parameterTypes[index] == char.class) { + arguments[index] = '\0'; + } else if (parameterTypes[index].isPrimitive()) { + arguments[index] = 0; + } + } + return (Throwable) constructor.newInstance(arguments); + } + private static MiddlewareContext contextFor(String annotation, String functionName) { MiddlewareContext context = mock(MiddlewareContext.class); when(context.getFunctionName()).thenReturn(functionName); diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/OrchestratorBlockedException.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/OrchestratorBlockedException.java deleted file mode 100644 index d8839fdf084..00000000000 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/OrchestratorBlockedException.java +++ /dev/null @@ -1,3 +0,0 @@ -package com.microsoft.durabletask; - -public class OrchestratorBlockedException extends RuntimeException {} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/ContinueAsNewInterruption.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/ContinueAsNewInterruption.java deleted file mode 100644 index e540e7519e0..00000000000 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/ContinueAsNewInterruption.java +++ /dev/null @@ -1,3 +0,0 @@ -package com.microsoft.durabletask.interruption; - -public class ContinueAsNewInterruption extends RuntimeException {} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/OrchestratorBlockedException.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/OrchestratorBlockedException.java deleted file mode 100644 index 681aa96c4bd..00000000000 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/durabletask/interruption/OrchestratorBlockedException.java +++ /dev/null @@ -1,3 +0,0 @@ -package com.microsoft.durabletask.interruption; - -public class OrchestratorBlockedException extends RuntimeException {} From b706092025a3b7dac201ec138b2c86f00d79d02c Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Tue, 29 Sep 2026 21:39:02 -0400 Subject: [PATCH 13/18] Run Azure worker v0 tests with latest dependencies --- .../azure-functions-worker-2.7/build.gradle.kts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts index 8140ce3ac6d..103c3815d4d 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts @@ -5,6 +5,10 @@ plugins { } muzzle { + // FunctionExecutionMiddleware lives in the unpublished Azure Functions Java worker. + // The SPI pass validates its public middleware contract, but cannot check the worker's + // internal class name or invoke signature. The test stub exercises the advice behavior; + // worker compatibility also requires validation against a running Function App. pass { group = "com.microsoft.azure.functions" module = "azure-functions-java-spi" @@ -36,6 +40,7 @@ fun addTestSuiteForDir(name: String, directory: String) { } addTestSuiteForDir("latestDepTest", "test") +(project.extra["addForkedTestTask"] as Closure<*>).call("latestDepTest") dependencies { compileOnly("com.google.protobuf:protobuf-java:3.19.2") From 05516caa6dd7255dca0f355e343f2684cd08f130 Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Tue, 29 Sep 2026 21:45:52 -0400 Subject: [PATCH 14/18] Avoid activating scope for Azure replay error span --- .../AzureFunctionsWorkerInstrumentation.java | 30 ++++++++++--------- .../worker/DurableFunctionsUtils.java | 6 ++-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java index f77ad7a6d63..31f79811a58 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -96,23 +96,25 @@ public static void onExit( return; } - ContextScope activeScope = scope; - if (activeScope == null - && throwable != null - && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { - if ("DurableOrchestration".equals(trigger)) { - activeScope = DurableFunctionsUtils.startSpanScope(context, trigger, startTimeMicros); - } - } - if (activeScope != null) { - final AgentSpan span = spanFromScope(activeScope); - if (!DurableFunctionsUtils.isReplayControlFlow(throwable)) { + if (scope == null) { + if (throwable != null + && "DurableOrchestration".equals(trigger) + && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { + final AgentSpan span = + DurableFunctionsUtils.startInvocationSpan(context, trigger, startTimeMicros); DECORATE.onError(span, throwable); + DECORATE.beforeFinish(span); + span.finish(); } - DECORATE.beforeFinish(span); - activeScope.close(); - span.finish(); + return; + } + final AgentSpan span = spanFromScope(scope); + if (!DurableFunctionsUtils.isReplayControlFlow(throwable)) { + DECORATE.onError(span, throwable); } + DECORATE.beforeFinish(span); + scope.close(); + span.finish(); } } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index 904d6bc551c..45eb28869aa 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -34,10 +34,10 @@ public static String getTrigger(MiddlewareContext context) { } public static ContextScope startSpanScope(MiddlewareContext context, String trigger) { - return startSpanScope(context, trigger, 0); + return activateSpan(startInvocationSpan(context, trigger, 0)); } - public static ContextScope startSpanScope( + public static AgentSpan startInvocationSpan( MiddlewareContext context, String trigger, long startTimeMicros) { final TraceContext traceContext = context.getTraceContext(); final AgentSpanContext.Extracted parent = @@ -51,7 +51,7 @@ public static ContextScope startSpanScope( : startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent); DECORATE.afterStart(span); DECORATE.onInvoke(span, context.getFunctionName(), trigger); - return activateSpan(span); + return span; } public static ContextScope activateTraceContext(MiddlewareContext context) { From 06e2965bf28f3752876b614eed0b1afca129aa48 Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Wed, 30 Sep 2026 10:04:22 -0400 Subject: [PATCH 15/18] Scope Azure durable orchestration tracing to worker invocations --- .../azure-functions-worker-2.7/README.md | 20 ++++ .../AzureFunctionsWorkerInstrumentation.java | 7 +- .../worker/DurableFunctionsDecorator.java | 8 ++ .../worker/DurableFunctionsUtils.java | 11 ++- ...eOrchestrationExecutorInstrumentation.java | 27 ++++-- ...rchestrationMiddlewareInstrumentation.java | 5 +- .../worker/DurableOrchestrationState.java | 38 ++++++++ .../test/java/AzureFunctionsWorkerTest.java | 91 ++++++++++++++++++- 8 files changed, 185 insertions(+), 22 deletions(-) create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/README.md create mode 100644 dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationState.java diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/README.md b/dd-java-agent/instrumentation/azure-functions-worker-2.7/README.md new file mode 100644 index 00000000000..90213a621e6 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/README.md @@ -0,0 +1,20 @@ +# Azure Durable Functions Java tracing + +For correlated host and Java worker traces, enable OpenTelemetry in the Function App's +`host.json`: + +```json +{ + "version": "2.0", + "telemetryMode": "OpenTelemetry" +} +``` + +The instrumentation honors the incoming W3C `traceparent` sampled flag and Datadog +`tracestate` sampling decision. It does not force-keep Durable invocations. If the host or an +upstream service sends a drop decision, the corresponding Durable spans may not be ingested. +Keep the application's sampling configuration consistent across the HTTP starter and Durable +worker invocations; enabling OpenTelemetry does not override an intentional upstream drop. + +See the [Azure Functions OpenTelemetry setup guide](https://learn.microsoft.com/en-us/azure/azure-functions/opentelemetry-howto) +for host configuration details. diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java index 31f79811a58..8803dc79731 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -35,6 +35,7 @@ public String[] helperClassNames() { return new String[] { packageName + ".DurableFunctionsDecorator", packageName + ".DurableFunctionsUtils", + packageName + ".DurableOrchestrationState", packageName + ".TraceContextExtractAdapter" }; } @@ -91,7 +92,7 @@ public static void onExit( @Advice.Thrown Throwable throwable) { if (orchestrationSpan != null) { if (throwable != null && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { - DECORATE.onError(orchestrationSpan, throwable); + DECORATE.onInvocationError(orchestrationSpan, throwable); } return; } @@ -102,7 +103,7 @@ public static void onExit( && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { final AgentSpan span = DurableFunctionsUtils.startInvocationSpan(context, trigger, startTimeMicros); - DECORATE.onError(span, throwable); + DECORATE.onInvocationError(span, throwable); DECORATE.beforeFinish(span); span.finish(); } @@ -110,7 +111,7 @@ public static void onExit( } final AgentSpan span = spanFromScope(scope); if (!DurableFunctionsUtils.isReplayControlFlow(throwable)) { - DECORATE.onError(span, throwable); + DECORATE.onInvocationError(span, throwable); } DECORATE.beforeFinish(span); scope.close(); diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java index e67300a125a..f85f9dc0871 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java @@ -6,6 +6,7 @@ import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.BaseDecorator; +import java.lang.reflect.InvocationTargetException; public final class DurableFunctionsDecorator extends BaseDecorator { public static final DurableFunctionsDecorator DECORATE = new DurableFunctionsDecorator(); @@ -45,4 +46,11 @@ public void onInvoke(AgentSpan span, String functionName, String trigger) { } span.setTag("aas.function.trigger", trigger); } + + public void onInvocationError(AgentSpan span, Throwable throwable) { + while (throwable instanceof InvocationTargetException && throwable.getCause() != null) { + throwable = throwable.getCause(); + } + onError(span, throwable); + } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index 45eb28869aa..5f0920b77b7 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -54,24 +54,25 @@ public static AgentSpan startInvocationSpan( return span; } - public static ContextScope activateTraceContext(MiddlewareContext context) { + public static ContextScope activateOrchestrationContext(MiddlewareContext context) { if (context.getParameterName(ORCHESTRATION_ANNOTATION) == null) { return null; } final TraceContext traceContext = context.getTraceContext(); if (traceContext == null) { - return null; + return DurableOrchestrationState.activate(null); } final AgentSpanContext.Extracted extracted = extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); final AgentSpan remoteSpan = AgentSpan.fromSpanContext(extracted); - return remoteSpan.isValid() ? activateSpan(remoteSpan) : null; + return DurableOrchestrationState.activate(remoteSpan); } public static AgentSpan onOrchestrationInvoke(MiddlewareContext context) { - final AgentSpan span = AgentSpan.current(); - if (span != null && "DurableOrchestration".equals(span.getTag("aas.function.trigger"))) { + final DurableOrchestrationState state = DurableOrchestrationState.current(); + final AgentSpan span = state == null ? null : state.span(); + if (span != null) { DECORATE.onInvoke(span, context.getFunctionName(), "DurableOrchestration"); return span; } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java index 451a3e343da..fbdec809b96 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java @@ -35,7 +35,9 @@ public String instrumentedType() { @Override public String[] helperClassNames() { return new String[] { - packageName + ".DurableFunctionsDecorator", packageName + ".DurableOrchestrationUtils" + packageName + ".DurableFunctionsDecorator", + packageName + ".DurableOrchestrationState", + packageName + ".DurableOrchestrationUtils" }; } @@ -58,19 +60,30 @@ public void methodAdvice(MethodTransformer transformer) { public static class ExecuteAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static ContextScope onEnter( - @Advice.Argument(0) List pastEvents, @Advice.Argument(1) List newEvents) { - return DurableOrchestrationUtils.shouldTrace(pastEvents, newEvents) - ? DurableOrchestrationUtils.startSpanScope() - : null; + @Advice.Argument(0) List pastEvents, + @Advice.Argument(1) List newEvents, + @Advice.Local("state") DurableOrchestrationState state, + @Advice.Local("previousSpan") AgentSpan previousSpan) { + state = DurableOrchestrationState.current(); + if (state == null || !DurableOrchestrationUtils.shouldTrace(pastEvents, newEvents)) { + return null; + } + final ContextScope scope = DurableOrchestrationUtils.startSpanScope(); + previousSpan = state.setSpan(spanFromScope(scope)); + return scope; } @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void onExit( - @Advice.Enter ContextScope scope, @Advice.Thrown Throwable throwable) { + @Advice.Enter ContextScope scope, + @Advice.Local("state") DurableOrchestrationState state, + @Advice.Local("previousSpan") AgentSpan previousSpan, + @Advice.Thrown Throwable throwable) { if (scope != null) { final AgentSpan span = spanFromScope(scope); - DECORATE.onError(span, throwable); + DECORATE.onInvocationError(span, throwable); DECORATE.beforeFinish(span); + state.setSpan(previousSpan); scope.close(); span.finish(); } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java index f7d335e4415..9ee72a0baaf 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java @@ -37,6 +37,7 @@ public String[] helperClassNames() { return new String[] { packageName + ".DurableFunctionsDecorator", packageName + ".DurableFunctionsUtils", + packageName + ".DurableOrchestrationState", packageName + ".TraceContextExtractAdapter" }; } @@ -69,10 +70,10 @@ public void methodAdvice(MethodTransformer transformer) { public static class InvokeAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) public static ContextScope onEnter(@Advice.Argument(0) MiddlewareContext context) { - return DurableFunctionsUtils.activateTraceContext(context); + return DurableFunctionsUtils.activateOrchestrationContext(context); } - @Advice.OnMethodExit(suppress = Throwable.class) + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) public static void onExit(@Advice.Enter ContextScope scope) { if (scope != null) { scope.close(); diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationState.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationState.java new file mode 100644 index 00000000000..d89f96f30ca --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationState.java @@ -0,0 +1,38 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import datadog.context.Context; +import datadog.context.ContextKey; +import datadog.context.ContextScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; + +/** Carries the Azure orchestration invocation and its span through the synchronous worker call. */ +public final class DurableOrchestrationState { + private static final ContextKey KEY = + ContextKey.named("azure-durable-orchestration"); + + private AgentSpan span; + + private DurableOrchestrationState() {} + + public static ContextScope activate(AgentSpan remoteSpan) { + Context context = Context.current(); + if (remoteSpan != null && remoteSpan.isValid()) { + context = context.with(remoteSpan); + } + return context.with(KEY, new DurableOrchestrationState()).attach(); + } + + public static DurableOrchestrationState current() { + return Context.current().get(KEY); + } + + public AgentSpan span() { + return span; + } + + public AgentSpan setSpan(AgentSpan span) { + final AgentSpan previous = this.span; + this.span = span; + return previous; + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index 05d356cfcf3..f749e3d1776 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -10,8 +10,11 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -23,6 +26,7 @@ import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain; import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; import com.microsoft.azure.functions.worker.chain.FunctionExecutionMiddleware; +import com.microsoft.durabletask.OrchestrationRunner; import com.microsoft.durabletask.azurefunctions.internal.middleware.OrchestrationMiddleware; import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.ExecutionStartedEvent; import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.HistoryEvent; @@ -31,15 +35,19 @@ import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.SubOrchestrationInstanceFailedEvent; import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskCompletedEvent; import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskFailedEvent; +import datadog.context.ContextScope; import datadog.trace.agent.test.AbstractInstrumentationTest; import datadog.trace.agent.test.assertions.SpanMatcher; import datadog.trace.api.DDSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.core.DDSpan; import datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils; +import datadog.trace.instrumentation.azure.functions.worker.DurableOrchestrationState; import datadog.trace.instrumentation.azure.functions.worker.DurableOrchestrationUtils; import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.util.Base64; import java.util.Collections; import java.util.List; @@ -133,6 +141,78 @@ void createsSpanForInitialOrchestrationExecution() throws Exception { assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); } + @Test + void doesNotCreateAzureSpanForStandaloneOrchestration() { + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("standalone-instance") + .addNewEvents(executionStarted()) + .build(); + + OrchestrationRunner.loadAndRun(request.toByteArray(), context -> null); + + assertTraces(); + } + + @Test + void closesOrchestrationContextWhenMiddlewareThrows() { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + when(context.getParameterValue("input")).thenReturn("invalid base64!"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-01"); + when(context.getTraceContext()).thenReturn(traceContext); + AgentSpan previousSpan = AgentTracer.activeSpan(); + + assertThrows( + IllegalArgumentException.class, + () -> new OrchestrationMiddleware().invoke(context, mock(MiddlewareChain.class))); + + assertSame(previousSpan, AgentTracer.activeSpan()); + assertNull(DurableOrchestrationState.current()); + assertTraces(); + } + + @Test + void findsOrchestrationSpanWhenAnotherSpanIsActive() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("test-instance") + .addNewEvents(executionStarted()) + .build(); + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request.toByteArray())); + + MiddlewareChain orchestrationChain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + AgentSpan intervening = AgentTracer.startSpan("test", "intervening"); + try (ContextScope scope = AgentTracer.activateSpan(intervening)) { + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + } finally { + intervening.finish(); + } + return null; + }) + .when(orchestrationChain) + .doNext(context); + + new OrchestrationMiddleware().invoke(context, orchestrationChain); + + writer.waitForTraces(1); + List spans = writer.firstTrace(); + assertEquals(2, spans.size()); + DDSpan orchestrationSpan = + spans.stream() + .filter(span -> "DurableOrchestration".equals(span.getTag("aas.function.trigger"))) + .findFirst() + .orElseThrow(AssertionError::new); + assertEquals("Orchestrator", orchestrationSpan.getTag("aas.function.name")); + assertEquals( + "DurableOrchestration Orchestrator", orchestrationSpan.getResourceName().toString()); + } + @TableTest({ "scenario | subOrchestration", "failed activity | false ", @@ -156,7 +236,7 @@ void marksRetainedOrchestrationFailureAsError() throws Exception { MiddlewareChain chain = mock(MiddlewareChain.class); doAnswer( invocation -> { - throw new IllegalStateException("activity failure"); + throw new InvocationTargetException(new IllegalStateException("activity failure")); }) .when(chain) .doNext(context); @@ -193,7 +273,7 @@ void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { invocation -> { invocationStartMillis.set(System.currentTimeMillis()); Thread.sleep(25); - throw new IllegalStateException("replay failure"); + throw new InvocationTargetException(new IllegalStateException("replay failure")); }) .when(chain) .doNext(context); @@ -430,6 +510,7 @@ void doesNotMarkReplayControlFlowAsError(String controlFlowTypeName) throws Exce try { controlFlowType = (Class) Class.forName(controlFlowTypeName); } catch (ClassNotFoundException ignored) { + assumeTrue(false, controlFlowTypeName + " is not available in this Durable Task version"); return; } Throwable controlFlow = instantiateThrowable(controlFlowType); @@ -449,18 +530,18 @@ void doesNotMarkReplayControlFlowAsError(String controlFlowTypeName) throws Exce } @Test - void marksApplicationFailuresAsErrors() throws Exception { + void marksWrappedApplicationFailuresAsErrors() throws Exception { MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); MiddlewareChain chain = mock(MiddlewareChain.class); doAnswer( invocation -> { - throw new IllegalStateException("failure"); + throw new InvocationTargetException(new IllegalStateException("failure")); }) .when(chain) .doNext(context); assertThrows( - IllegalStateException.class, + InvocationTargetException.class, () -> new FunctionExecutionMiddleware().invoke(context, chain)); assertTraces( From 5bd0b58514feef8cff3ea6017295d42b92ff8ae8 Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Wed, 30 Sep 2026 11:12:55 -0400 Subject: [PATCH 16/18] Share Azure durable orchestration state across classloaders --- .../AzureFunctionsWorkerInstrumentation.java | 1 - .../worker/DurableFunctionsUtils.java | 1 + ...eOrchestrationExecutorInstrumentation.java | 5 +- ...rchestrationMiddlewareInstrumentation.java | 1 - .../test/java/AzureFunctionsWorkerTest.java | 60 ++++++++++++++++++- .../api}/DurableOrchestrationState.java | 5 +- 6 files changed, 64 insertions(+), 9 deletions(-) rename {dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker => internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api}/DurableOrchestrationState.java (80%) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java index 8803dc79731..d74c48dd797 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -35,7 +35,6 @@ public String[] helperClassNames() { return new String[] { packageName + ".DurableFunctionsDecorator", packageName + ".DurableFunctionsUtils", - packageName + ".DurableOrchestrationState", packageName + ".TraceContextExtractAdapter" }; } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index 5f0920b77b7..8a47ff7454f 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -11,6 +11,7 @@ import datadog.context.ContextScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.api.DurableOrchestrationState; import java.util.IdentityHashMap; public final class DurableFunctionsUtils { diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java index fbdec809b96..e78be6b2de1 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java @@ -13,6 +13,7 @@ import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.agent.tooling.muzzle.Reference; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.DurableOrchestrationState; import java.util.List; import net.bytebuddy.asm.Advice; @@ -35,9 +36,7 @@ public String instrumentedType() { @Override public String[] helperClassNames() { return new String[] { - packageName + ".DurableFunctionsDecorator", - packageName + ".DurableOrchestrationState", - packageName + ".DurableOrchestrationUtils" + packageName + ".DurableFunctionsDecorator", packageName + ".DurableOrchestrationUtils" }; } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java index 9ee72a0baaf..c6d84d1b440 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java @@ -37,7 +37,6 @@ public String[] helperClassNames() { return new String[] { packageName + ".DurableFunctionsDecorator", packageName + ".DurableFunctionsUtils", - packageName + ".DurableOrchestrationState", packageName + ".TraceContextExtractAdapter" }; } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index f749e3d1776..6efc553c946 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -10,6 +10,7 @@ import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -41,11 +42,14 @@ import datadog.trace.api.DDSpanTypes; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; +import datadog.trace.bootstrap.instrumentation.api.DurableOrchestrationState; import datadog.trace.bootstrap.instrumentation.api.Tags; import datadog.trace.core.DDSpan; import datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils; -import datadog.trace.instrumentation.azure.functions.worker.DurableOrchestrationState; import datadog.trace.instrumentation.azure.functions.worker.DurableOrchestrationUtils; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.Base64; @@ -213,6 +217,60 @@ void findsOrchestrationSpanWhenAnotherSpanIsActive() throws Exception { "DurableOrchestration Orchestrator", orchestrationSpan.getResourceName().toString()); } + @Test + void findsOrchestrationSpanAcrossWorkerClassLoader() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + AgentSpan expectedSpan = mock(AgentSpan.class); + + ClassLoader workerLoader = + new ClassLoader(AzureFunctionsWorkerTest.class.getClassLoader()) { + @Override + protected synchronized Class loadClass(String name, boolean resolve) + throws ClassNotFoundException { + if (!name.equals(DurableFunctionsUtils.class.getName())) { + return super.loadClass(name, resolve); + } + Class loaded = findLoadedClass(name); + if (loaded == null) { + String resource = name.replace('.', '/') + ".class"; + try (InputStream input = getParent().getResourceAsStream(resource)) { + if (input == null) { + throw new ClassNotFoundException(name); + } + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int count; + while ((count = input.read(buffer)) != -1) { + bytes.write(buffer, 0, count); + } + byte[] classBytes = bytes.toByteArray(); + loaded = defineClass(name, classBytes, 0, classBytes.length); + } catch (IOException e) { + throw new ClassNotFoundException(name, e); + } + } + if (resolve) { + resolveClass(loaded); + } + return loaded; + } + }; + + try (ContextScope scope = DurableOrchestrationState.activate(null)) { + DurableOrchestrationState.current().setSpan(expectedSpan); + Class workerUtils = + Class.forName(DurableFunctionsUtils.class.getName(), true, workerLoader); + assertNotSame(DurableFunctionsUtils.class, workerUtils); + + Object actualSpan = + workerUtils + .getMethod("onOrchestrationInvoke", MiddlewareContext.class) + .invoke(null, context); + assertSame(expectedSpan, actualSpan); + } + assertTraces(); + } + @TableTest({ "scenario | subOrchestration", "failed activity | false ", diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationState.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/DurableOrchestrationState.java similarity index 80% rename from dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationState.java rename to internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/DurableOrchestrationState.java index d89f96f30ca..503ede5704c 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationState.java +++ b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/DurableOrchestrationState.java @@ -1,11 +1,10 @@ -package datadog.trace.instrumentation.azure.functions.worker; +package datadog.trace.bootstrap.instrumentation.api; import datadog.context.Context; import datadog.context.ContextKey; import datadog.context.ContextScope; -import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -/** Carries the Azure orchestration invocation and its span through the synchronous worker call. */ +/** Shares an Azure Durable orchestration invocation and its span across worker classloaders. */ public final class DurableOrchestrationState { private static final ContextKey KEY = ContextKey.named("azure-durable-orchestration"); From 374e8667f416d51767ce38749f4a3fbbb856c908 Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Thu, 1 Oct 2026 10:26:13 -0400 Subject: [PATCH 17/18] Harden Azure Durable Functions tracing and classloader isolation --- .../azure/DurableOrchestrationState.java | 61 ++ .../azure/DurableOrchestrationStateTest.java | 61 ++ .../AzureFunctionsWorkerInstrumentation.java | 43 +- .../worker/DurableFunctionsDecorator.java | 1 + .../worker/DurableFunctionsUtils.java | 120 ++-- ...eOrchestrationExecutorInstrumentation.java | 63 +- ...rchestrationMiddlewareInstrumentation.java | 39 +- .../worker/DurableOrchestrationUtils.java | 97 ++- .../test/java/AzureFunctionsWorkerTest.java | 632 ++++++++++++++---- .../api/DurableOrchestrationState.java | 37 - 10 files changed, 892 insertions(+), 262 deletions(-) create mode 100644 dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationState.java create mode 100644 dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationStateTest.java delete mode 100644 internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/DurableOrchestrationState.java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationState.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationState.java new file mode 100644 index 00000000000..7c1b04f4a88 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationState.java @@ -0,0 +1,61 @@ +package datadog.trace.bootstrap.instrumentation.azure; + +import datadog.context.Context; +import datadog.context.ContextKey; +import datadog.context.ContextScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +/** Shares an Azure Durable orchestration invocation and its span across worker classloaders. */ +public final class DurableOrchestrationState { + private static final ContextKey KEY = + ContextKey.named("azure-durable-orchestration"); + + private final AgentSpan parentSpan; + private final String functionName; + private final AtomicReference span = new AtomicReference<>(); + private final AtomicBoolean errorRecorded = new AtomicBoolean(); + + private DurableOrchestrationState(AgentSpan parentSpan, String functionName) { + this.parentSpan = parentSpan; + this.functionName = functionName; + } + + public static ContextScope activate(AgentSpan remoteSpan, String functionName) { + Context context = Context.current(); + final AgentSpan parentSpan = remoteSpan != null && remoteSpan.isValid() ? remoteSpan : null; + if (parentSpan != null) { + context = context.with(parentSpan); + } + return context.with(KEY, new DurableOrchestrationState(parentSpan, functionName)).attach(); + } + + public static DurableOrchestrationState current() { + return Context.current().get(KEY); + } + + public AgentSpan span() { + return span.get(); + } + + public AgentSpan parentSpan() { + return parentSpan; + } + + public String functionName() { + return functionName; + } + + public boolean errorRecorded() { + return errorRecorded.get(); + } + + public void markErrorRecorded() { + errorRecorded.set(true); + } + + public AgentSpan setSpan(AgentSpan span) { + return this.span.getAndSet(span); + } +} diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationStateTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationStateTest.java new file mode 100644 index 00000000000..93afc8c7d68 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationStateTest.java @@ -0,0 +1,61 @@ +package datadog.trace.bootstrap.instrumentation.azure; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.context.Context; +import datadog.context.ContextScope; +import datadog.trace.api.DDTraceId; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import org.junit.jupiter.api.Test; + +class DurableOrchestrationStateTest { + + @Test + void activatesRemoteParentAndRestoresPreviousContext() { + AgentSpanContext spanContext = mock(AgentSpanContext.class); + when(spanContext.getTraceId()).thenReturn(DDTraceId.from(42)); + when(spanContext.getSpanId()).thenReturn(43L); + AgentSpan parent = AgentSpan.fromSpanContext(spanContext); + AgentSpan orchestration = mock(AgentSpan.class); + + assertNull(DurableOrchestrationState.current()); + try (ContextScope scope = DurableOrchestrationState.activate(parent, "Orchestrator")) { + DurableOrchestrationState state = DurableOrchestrationState.current(); + assertSame(parent, state.parentSpan()); + assertSame("Orchestrator", state.functionName()); + assertSame(parent, AgentSpan.fromContext(Context.current())); + assertFalse(state.errorRecorded()); + state.markErrorRecorded(); + assertTrue(state.errorRecorded()); + assertNull(state.span()); + assertNull(state.setSpan(orchestration)); + assertSame(orchestration, state.span()); + assertSame(orchestration, state.setSpan(null)); + assertNull(state.span()); + } + assertNull(DurableOrchestrationState.current()); + } + + @Test + void ignoresMissingAndInvalidRemoteParents() { + AgentSpanContext invalidContext = mock(AgentSpanContext.class); + when(invalidContext.getTraceId()).thenReturn(DDTraceId.ZERO); + when(invalidContext.getSpanId()).thenReturn(0L); + AgentSpan invalidParent = AgentSpan.fromSpanContext(invalidContext); + assertFalse(invalidParent.isValid()); + try (ContextScope scope = DurableOrchestrationState.activate(null, null)) { + assertNull(DurableOrchestrationState.current().parentSpan()); + } + try (ContextScope scope = DurableOrchestrationState.activate(invalidParent, null)) { + assertNull(DurableOrchestrationState.current().parentSpan()); + assertNull(AgentSpan.fromContext(Context.current())); + } + assertNull(DurableOrchestrationState.current()); + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java index d74c48dd797..3146f0e2a3c 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -3,7 +3,7 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.spanFromScope; import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; -import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.ORCHESTRATION_TRIGGER; import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.isPublic; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; @@ -15,6 +15,7 @@ import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; import net.bytebuddy.asm.Advice; @AutoService(InstrumenterModule.class) @@ -30,15 +31,6 @@ public String instrumentedType() { return "com.microsoft.azure.functions.worker.chain.FunctionExecutionMiddleware"; } - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".DurableFunctionsDecorator", - packageName + ".DurableFunctionsUtils", - packageName + ".TraceContextExtractAdapter" - }; - } - @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( @@ -70,10 +62,10 @@ public static ContextScope onEnter( if (trigger == null) { return null; } - if ("DurableOrchestration".equals(trigger)) { - orchestrationSpan = DurableFunctionsUtils.onOrchestrationInvoke(context); + if (ORCHESTRATION_TRIGGER.equals(trigger)) { + orchestrationSpan = DurableFunctionsUtils.onOrchestrationInvoke(); if (orchestrationSpan == null) { - startTimeMicros = MILLISECONDS.toMicros(System.currentTimeMillis()); + startTimeMicros = DurableFunctionsUtils.nowMicros(); } return null; } @@ -98,23 +90,28 @@ public static void onExit( if (scope == null) { if (throwable != null - && "DurableOrchestration".equals(trigger) + && ORCHESTRATION_TRIGGER.equals(trigger) && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { final AgentSpan span = - DurableFunctionsUtils.startInvocationSpan(context, trigger, startTimeMicros); - DECORATE.onInvocationError(span, throwable); - DECORATE.beforeFinish(span); - span.finish(); + DurableFunctionsUtils.startOrchestrationErrorSpan(context, startTimeMicros); + final DurableOrchestrationState state = DurableOrchestrationState.current(); + DurableFunctionsUtils.recordOrchestrationError(span, throwable, state); } return; } final AgentSpan span = spanFromScope(scope); - if (!DurableFunctionsUtils.isReplayControlFlow(throwable)) { - DECORATE.onInvocationError(span, throwable); + try { + if (!DurableFunctionsUtils.isReplayControlFlow(throwable)) { + DECORATE.onInvocationError(span, throwable); + } + DECORATE.beforeFinish(span); + } finally { + try { + scope.close(); + } finally { + span.finish(); + } } - DECORATE.beforeFinish(span); - scope.close(); - span.finish(); } } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java index f85f9dc0871..f4dc6324842 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java @@ -9,6 +9,7 @@ import java.lang.reflect.InvocationTargetException; public final class DurableFunctionsDecorator extends BaseDecorator { + public static final String ORCHESTRATION_TRIGGER = "DurableOrchestration"; public static final DurableFunctionsDecorator DECORATE = new DurableFunctionsDecorator(); public static final CharSequence AZURE_FUNCTIONS_REQUEST = UTF8BytesString.create( diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index 8a47ff7454f..33664abd75b 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -5,14 +5,17 @@ import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.AZURE_FUNCTIONS_REQUEST; import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.ORCHESTRATION_TRIGGER; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; import com.microsoft.azure.functions.TraceContext; import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; import datadog.context.ContextScope; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; -import datadog.trace.bootstrap.instrumentation.api.DurableOrchestrationState; -import java.util.IdentityHashMap; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; +import java.time.Instant; public final class DurableFunctionsUtils { private static final String ACTIVITY_ANNOTATION = "DurableActivityTrigger"; @@ -23,7 +26,7 @@ private DurableFunctionsUtils() {} public static String getTrigger(MiddlewareContext context) { if (context.getParameterName(ORCHESTRATION_ANNOTATION) != null) { - return "DurableOrchestration"; + return ORCHESTRATION_TRIGGER; } if (context.getParameterName(ACTIVITY_ANNOTATION) != null) { return "DurableActivity"; @@ -35,66 +38,107 @@ public static String getTrigger(MiddlewareContext context) { } public static ContextScope startSpanScope(MiddlewareContext context, String trigger) { - return activateSpan(startInvocationSpan(context, trigger, 0)); + return activateSpan( + startInvocationSpan( + context.getFunctionName(), trigger, extractParent(context.getTraceContext()), 0)); } - public static AgentSpan startInvocationSpan( - MiddlewareContext context, String trigger, long startTimeMicros) { - final TraceContext traceContext = context.getTraceContext(); - final AgentSpanContext.Extracted parent = - traceContext == null - ? null - : extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); + public static long nowMicros() { + final Instant now = Instant.now(); + return SECONDS.toMicros(now.getEpochSecond()) + NANOSECONDS.toMicros(now.getNano()); + } + + public static AgentSpan startOrchestrationErrorSpan( + MiddlewareContext context, long startTimeMicros) { + final DurableOrchestrationState state = DurableOrchestrationState.current(); + if (state != null) { + return startOrchestrationSpan(state, startTimeMicros); + } + return startInvocationSpan( + context.getFunctionName(), + ORCHESTRATION_TRIGGER, + extractParent(context.getTraceContext()), + startTimeMicros); + } + public static AgentSpan startOrchestrationSpan( + DurableOrchestrationState state, long startTimeMicros) { + final AgentSpan parentSpan = state.parentSpan(); + return startInvocationSpan( + state.functionName(), + ORCHESTRATION_TRIGGER, + parentSpan == null ? null : parentSpan.spanContext(), + startTimeMicros); + } + + public static AgentSpan startInvocationSpan( + String functionName, String trigger, AgentSpanContext parent, long startTimeMicros) { final AgentSpan span = startTimeMicros > 0 ? startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent, startTimeMicros) : startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent); DECORATE.afterStart(span); - DECORATE.onInvoke(span, context.getFunctionName(), trigger); + DECORATE.onInvoke(span, functionName, trigger); return span; } + public static void recordOrchestrationError( + AgentSpan span, Throwable throwable, DurableOrchestrationState state) { + try { + DECORATE.onInvocationError(span, throwable); + DECORATE.beforeFinish(span); + } finally { + try { + // This path is only used for failures, even if error decoration itself throws. + if (!span.isError()) { + span.setError(true); + } + } finally { + try { + span.finish(nowMicros()); + } finally { + if (state != null && span.isError()) { + state.markErrorRecorded(); + } + } + } + } + } + + private static AgentSpanContext.Extracted extractParent(TraceContext traceContext) { + return traceContext == null + ? null + : extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); + } + public static ContextScope activateOrchestrationContext(MiddlewareContext context) { if (context.getParameterName(ORCHESTRATION_ANNOTATION) == null) { return null; } - final TraceContext traceContext = context.getTraceContext(); - if (traceContext == null) { - return DurableOrchestrationState.activate(null); - } - - final AgentSpanContext.Extracted extracted = - extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); + final AgentSpanContext.Extracted extracted = extractParent(context.getTraceContext()); final AgentSpan remoteSpan = AgentSpan.fromSpanContext(extracted); - return DurableOrchestrationState.activate(remoteSpan); + return DurableOrchestrationState.activate(remoteSpan, context.getFunctionName()); } - public static AgentSpan onOrchestrationInvoke(MiddlewareContext context) { + public static AgentSpan onOrchestrationInvoke() { final DurableOrchestrationState state = DurableOrchestrationState.current(); - final AgentSpan span = state == null ? null : state.span(); - if (span != null) { - DECORATE.onInvoke(span, context.getFunctionName(), "DurableOrchestration"); - return span; - } - return null; + return state == null ? null : state.span(); } public static boolean isReplayControlFlow(Throwable throwable) { if (throwable == null) { return false; } + // The Durable middleware recognizes only the direct cause of the exception it catches. + // A more deeply wrapped interruption is treated by the SDK as a real failure. + final Throwable cause = throwable.getCause(); + return cause != null && isReplayInterruption(cause); + } - final IdentityHashMap seen = new IdentityHashMap<>(); - while (throwable != null && seen.put(throwable, Boolean.TRUE) == null) { - final String className = throwable.getClass().getName(); - if ("com.microsoft.durabletask.OrchestratorBlockedException".equals(className) - || "com.microsoft.durabletask.interruption.OrchestratorBlockedException".equals(className) - || "com.microsoft.durabletask.interruption.ContinueAsNewInterruption".equals(className)) { - return true; - } - throwable = throwable.getCause(); - } - return false; + private static boolean isReplayInterruption(Throwable throwable) { + final String className = throwable.getClass().getName(); + return "com.microsoft.durabletask.OrchestratorBlockedException".equals(className) + || "com.microsoft.durabletask.interruption.OrchestratorBlockedException".equals(className) + || "com.microsoft.durabletask.interruption.ContinueAsNewInterruption".equals(className); } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java index e78be6b2de1..8b29a0adfe5 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java @@ -3,17 +3,19 @@ import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.spanFromScope; import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils.startOrchestrationSpan; import static net.bytebuddy.matcher.ElementMatchers.isMethod; import static net.bytebuddy.matcher.ElementMatchers.isPublic; import static net.bytebuddy.matcher.ElementMatchers.takesArgument; import com.google.auto.service.AutoService; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskFailureDetails; import datadog.context.ContextScope; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.agent.tooling.muzzle.Reference; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.DurableOrchestrationState; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; import java.util.List; import net.bytebuddy.asm.Advice; @@ -33,13 +35,6 @@ public String instrumentedType() { return "com.microsoft.durabletask.TaskOrchestrationExecutor"; } - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".DurableFunctionsDecorator", packageName + ".DurableOrchestrationUtils" - }; - } - @Override public Reference[] additionalMuzzleReferences() { return new Reference[] {EXECUTOR_REFERENCE}; @@ -62,12 +57,17 @@ public static ContextScope onEnter( @Advice.Argument(0) List pastEvents, @Advice.Argument(1) List newEvents, @Advice.Local("state") DurableOrchestrationState state, - @Advice.Local("previousSpan") AgentSpan previousSpan) { + @Advice.Local("previousSpan") AgentSpan previousSpan, + @Advice.Local("startTimeMicros") long startTimeMicros) { state = DurableOrchestrationState.current(); - if (state == null || !DurableOrchestrationUtils.shouldTrace(pastEvents, newEvents)) { + if (state == null) { return null; } - final ContextScope scope = DurableOrchestrationUtils.startSpanScope(); + startTimeMicros = DurableFunctionsUtils.nowMicros(); + if (!DurableOrchestrationUtils.shouldTrace(pastEvents, newEvents)) { + return null; + } + final ContextScope scope = DurableOrchestrationUtils.startSpanScope(state); previousSpan = state.setSpan(spanFromScope(scope)); return scope; } @@ -77,14 +77,43 @@ public static void onExit( @Advice.Enter ContextScope scope, @Advice.Local("state") DurableOrchestrationState state, @Advice.Local("previousSpan") AgentSpan previousSpan, - @Advice.Thrown Throwable throwable) { + @Advice.Local("startTimeMicros") long startTimeMicros, + @Advice.Thrown Throwable throwable, + @Advice.Return Object result) { + if (state == null) { + return; + } if (scope != null) { final AgentSpan span = spanFromScope(scope); - DECORATE.onInvocationError(span, throwable); - DECORATE.beforeFinish(span); - state.setSpan(previousSpan); - scope.close(); - span.finish(); + try { + final TaskFailureDetails failure = + throwable == null ? DurableOrchestrationUtils.failureDetails(result) : null; + DECORATE.onInvocationError(span, throwable); + if (failure != null) { + DurableOrchestrationUtils.onOrchestrationFailure(span, failure); + } + DECORATE.beforeFinish(span); + } finally { + state.setSpan(previousSpan); + try { + scope.close(); + } finally { + span.finish(); + if (span.isError()) { + state.markErrorRecorded(); + } + } + } + return; + } + if (state.errorRecorded()) { + return; + } + final TaskFailureDetails failure = + throwable == null ? DurableOrchestrationUtils.failureDetails(result) : null; + if (failure != null || throwable != null) { + final AgentSpan span = startOrchestrationSpan(state, startTimeMicros); + DurableOrchestrationUtils.recordSdkOrchestrationError(span, throwable, failure, state); } } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java index c6d84d1b440..47b754c7998 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java @@ -12,6 +12,8 @@ import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; import datadog.trace.agent.tooling.muzzle.Reference; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; import net.bytebuddy.asm.Advice; @AutoService(InstrumenterModule.class) @@ -32,15 +34,6 @@ public String instrumentedType() { return "com.microsoft.durabletask.azurefunctions.internal.middleware.OrchestrationMiddleware"; } - @Override - public String[] helperClassNames() { - return new String[] { - packageName + ".DurableFunctionsDecorator", - packageName + ".DurableFunctionsUtils", - packageName + ".TraceContextExtractAdapter" - }; - } - @Override public Reference[] additionalMuzzleReferences() { return new Reference[] {MIDDLEWARE_REFERENCE}; @@ -68,13 +61,33 @@ public void methodAdvice(MethodTransformer transformer) { public static class InvokeAdvice { @Advice.OnMethodEnter(suppress = Throwable.class) - public static ContextScope onEnter(@Advice.Argument(0) MiddlewareContext context) { - return DurableFunctionsUtils.activateOrchestrationContext(context); + public static ContextScope onEnter( + @Advice.Argument(0) MiddlewareContext context, + @Advice.Local("startTimeMicros") long startTimeMicros) { + final ContextScope scope = DurableFunctionsUtils.activateOrchestrationContext(context); + if (scope != null) { + startTimeMicros = DurableFunctionsUtils.nowMicros(); + } + return scope; } @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) - public static void onExit(@Advice.Enter ContextScope scope) { - if (scope != null) { + public static void onExit( + @Advice.Argument(0) MiddlewareContext context, + @Advice.Enter ContextScope scope, + @Advice.Local("startTimeMicros") long startTimeMicros, + @Advice.Thrown Throwable throwable) { + if (scope == null) { + return; + } + try { + final DurableOrchestrationState state = DurableOrchestrationState.current(); + if (throwable != null && state != null && !state.errorRecorded()) { + final AgentSpan span = + DurableFunctionsUtils.startOrchestrationErrorSpan(context, startTimeMicros); + DurableFunctionsUtils.recordOrchestrationError(span, throwable, state); + } + } finally { scope.close(); } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java index 7a23bf29a72..983e6f5091e 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java @@ -1,18 +1,26 @@ package datadog.trace.instrumentation.azure.functions.worker; import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; -import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; -import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.AZURE_FUNCTIONS_REQUEST; -import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils.recordOrchestrationError; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils.startOrchestrationSpan; import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.HistoryEvent; import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.HistoryEvent.EventTypeCase; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.OrchestrationStatus; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.OrchestratorAction; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskFailureDetails; import datadog.context.ContextScope; +import datadog.trace.api.DDTags; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collection; import java.util.List; public final class DurableOrchestrationUtils { - private static final String ORCHESTRATION_TRIGGER = "DurableOrchestration"; + private static volatile Method getActionsMethod; + private static volatile Class missingActionsClass; private DurableOrchestrationUtils() {} @@ -20,9 +28,10 @@ private DurableOrchestrationUtils() {} * Returns whether an orchestration invocation should produce a span. * *

The Durable SDK has already decoded the request by this point. Successful replays are - * suppressed, while a replay containing a newly failed activity or sub-orchestration is retained. - * Only new events are inspected, so each history event is examined once instead of rescanning the - * complete history on every replay. + * suppressed, while a replay containing a newly failed activity, sub-orchestration, or entity + * operation is retained. Only new events are inspected, so each history event is examined once + * instead of rescanning the complete history on every replay. User-created spans are not + * suppressed, because a replay may also execute new work after its last completed await. */ public static boolean shouldTrace(List pastEvents, List newEvents) { if (pastEvents == null || pastEvents.isEmpty()) { @@ -39,17 +48,79 @@ public static boolean shouldTrace(List pastEvents, List newEvents) { } final EventTypeCase eventType = ((HistoryEvent) event).getEventTypeCase(); if (eventType == EventTypeCase.TASKFAILED - || eventType == EventTypeCase.SUBORCHESTRATIONINSTANCEFAILED) { + || eventType == EventTypeCase.SUBORCHESTRATIONINSTANCEFAILED + || "ENTITYOPERATIONFAILED".equals(eventType.name())) { return true; } } return false; } - public static ContextScope startSpanScope() { - final AgentSpan span = startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST); - DECORATE.afterStart(span); - DECORATE.onInvoke(span, null, ORCHESTRATION_TRIGGER); - return activateSpan(span); + public static ContextScope startSpanScope(DurableOrchestrationState state) { + return activateSpan(startOrchestrationSpan(state, 0)); + } + + public static TaskFailureDetails failureDetails(Object result) { + if (result == null) { + return null; + } + try { + // TaskOrchestratorResult is package-private, although its accessor and action types are + // public. + final Class resultClass = result.getClass(); + if (resultClass == missingActionsClass) { + return null; + } + Method method = getActionsMethod; + if (method == null || method.getDeclaringClass() != resultClass) { + method = resultClass.getDeclaredMethod("getActions"); + method.setAccessible(true); + getActionsMethod = method; + } + for (Object value : (Collection) method.invoke(result)) { + if (value instanceof OrchestratorAction) { + final OrchestratorAction action = (OrchestratorAction) value; + if (action.hasCompleteOrchestration() + && action.getCompleteOrchestration().getOrchestrationStatus() + == OrchestrationStatus.ORCHESTRATION_STATUS_FAILED) { + return action.getCompleteOrchestration().getFailureDetails(); + } + } + } + } catch (NoSuchMethodException ignored) { + missingActionsClass = result.getClass(); + } catch (IllegalAccessException | InvocationTargetException | RuntimeException ignored) { + // The result is an internal SDK type; do not interfere with execution if it changes. + } + return null; + } + + public static void recordSdkOrchestrationError( + AgentSpan span, + Throwable throwable, + TaskFailureDetails failure, + DurableOrchestrationState state) { + try { + if (failure != null) { + onOrchestrationFailure(span, failure); + } + } finally { + recordOrchestrationError(span, throwable, state); + } + } + + public static void onOrchestrationFailure(AgentSpan span, TaskFailureDetails details) { + if (!span.isError()) { + span.setError(true); + if (!details.getErrorType().isEmpty()) { + span.setTag(DDTags.ERROR_TYPE, details.getErrorType()); + } + if (!details.getErrorMessage().isEmpty()) { + span.setTag(DDTags.ERROR_MSG, details.getErrorMessage()); + } + if (details.hasStackTrace()) { + span.setTag(DDTags.ERROR_STACK, details.getStackTrace().getValue()); + } + } } } diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index 6efc553c946..c30cecf70b1 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -5,18 +5,31 @@ import static datadog.trace.agent.test.assertions.TraceMatcher.trace; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; import static datadog.trace.test.junit.utils.assertions.Matchers.is; import static datadog.trace.test.junit.utils.assertions.Matchers.matches; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.concurrent.TimeUnit.MICROSECONDS; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyByte; +import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -36,29 +49,35 @@ import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.SubOrchestrationInstanceFailedEvent; import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskCompletedEvent; import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskFailedEvent; +import datadog.appsec.api.blocking.BlockingException; +import datadog.context.Context; import datadog.context.ContextScope; import datadog.trace.agent.test.AbstractInstrumentationTest; import datadog.trace.agent.test.assertions.SpanMatcher; import datadog.trace.api.DDSpanTypes; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; -import datadog.trace.bootstrap.instrumentation.api.AgentTracer; -import datadog.trace.bootstrap.instrumentation.api.DurableOrchestrationState; import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; import datadog.trace.core.DDSpan; +import datadog.trace.instrumentation.azure.functions.worker.AzureFunctionsWorkerInstrumentation; import datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils; +import datadog.trace.instrumentation.azure.functions.worker.DurableOrchestrationExecutorInstrumentation; import datadog.trace.instrumentation.azure.functions.worker.DurableOrchestrationUtils; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Constructor; +import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.Base64; -import java.util.Collections; +import java.util.Collection; import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.tabletest.junit.TableTest; @@ -77,10 +96,7 @@ void createsSpanForDurableTrigger(String annotation, String trigger) throws Exce if ("DurableOrchestration".equals(trigger)) { invokeOrchestration( - context, - Collections.emptyList(), - Collections.singletonList(executionStarted()), - mock(MiddlewareChain.class)); + context, emptyList(), singletonList(executionStarted()), mock(MiddlewareChain.class)); } else { new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); } @@ -103,22 +119,41 @@ void doesNotCreateSpanForSuccessfulOrchestrationReplay() throws Exception { invokeOrchestration( context, - Collections.singletonList(executionStarted()), - Collections.singletonList(taskCompleted()), + singletonList(executionStarted()), + singletonList(taskCompleted()), mock(MiddlewareChain.class)); assertTraces(); } + @Test + void preservesUserSpansFromNewWorkDuringOrchestrationReplay() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + AgentSpan userSpan = startSpan("test", "user-code"); + userSpan.finish(); + return null; + }) + .when(chain) + .doNext(context); + + invokeOrchestration( + context, singletonList(executionStarted()), singletonList(taskCompleted()), chain); + + writer.waitForTraces(1); + assertEquals(1, writer.firstTrace().size()); + assertEquals("user-code", writer.firstTrace().get(0).getOperationName().toString()); + } + @Test @SuppressWarnings("unchecked") void doesNotTraversePastEventsForReplayDecision() { List pastEvents = mock(List.class); when(pastEvents.isEmpty()).thenReturn(false); - assertFalse( - DurableOrchestrationUtils.shouldTrace( - pastEvents, Collections.singletonList(taskCompleted()))); + assertFalse(DurableOrchestrationUtils.shouldTrace(pastEvents, singletonList(taskCompleted()))); verify(pastEvents).isEmpty(); verifyNoMoreInteractions(pastEvents); @@ -128,8 +163,7 @@ void doesNotTraversePastEventsForReplayDecision() { void failsOpenForUnexpectedParsedEvent() { assertTrue( DurableOrchestrationUtils.shouldTrace( - Collections.singletonList(executionStarted()), - Collections.singletonList(new Object()))); + singletonList(executionStarted()), singletonList(new Object()))); } @Test @@ -137,14 +171,33 @@ void createsSpanForInitialOrchestrationExecution() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); invokeOrchestration( - context, - Collections.emptyList(), - Collections.singletonList(executionStarted()), - mock(MiddlewareChain.class)); + context, emptyList(), singletonList(executionStarted()), mock(MiddlewareChain.class)); assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); } + @Test + void orchestrationWithoutRemoteParentDoesNotJoinAmbientTrace() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + AgentSpan unrelated = startSpan("test", "unrelated"); + try (ContextScope scope = activateSpan(unrelated)) { + invokeOrchestration( + context, emptyList(), singletonList(executionStarted()), mock(MiddlewareChain.class)); + } finally { + unrelated.finish(); + } + + writer.waitForTraces(2); + DDSpan orchestration = + writer.stream() + .flatMap(List::stream) + .filter(span -> "DurableOrchestration".equals(span.getTag("aas.function.trigger"))) + .findFirst() + .orElseThrow(AssertionError::new); + assertEquals(0L, orchestration.getParentId()); + assertNotEquals(unrelated.getTraceId(), orchestration.getTraceId()); + } + @Test void doesNotCreateAzureSpanForStandaloneOrchestration() { OrchestratorRequest request = @@ -159,21 +212,130 @@ void doesNotCreateAzureSpanForStandaloneOrchestration() { } @Test - void closesOrchestrationContextWhenMiddlewareThrows() { + void closesOrchestrationContextWhenMiddlewareThrows() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); when(context.getParameterValue("input")).thenReturn("invalid base64!"); TraceContext traceContext = mock(TraceContext.class); when(traceContext.getTraceparent()) .thenReturn("00-0000000000000000000000000000002a-000000000000002b-01"); when(context.getTraceContext()).thenReturn(traceContext); - AgentSpan previousSpan = AgentTracer.activeSpan(); + AgentSpan previousSpan = activeSpan(); assertThrows( IllegalArgumentException.class, () -> new OrchestrationMiddleware().invoke(context, mock(MiddlewareChain.class))); - assertSame(previousSpan, AgentTracer.activeSpan()); + assertSame(previousSpan, activeSpan()); + assertNull(DurableOrchestrationState.current()); + writer.waitForTraces(1); + assertEquals(1, writer.size()); + DDSpan errorSpan = writer.firstTrace().get(0); + assertEquals("42", errorSpan.getTraceId().toString()); + assertEquals(43L, errorSpan.getParentId()); + assertEquals("DurableOrchestration Orchestrator", errorSpan.getResourceName().toString()); + assertTrue(errorSpan.isError()); + } + + @Test + void retainsTraceParentForOrchestrationErrorWithoutMiddlewareState() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-01"); + when(context.getTraceContext()).thenReturn(traceContext); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new IllegalStateException("worker failure"); + }) + .when(chain) + .doNext(context); + + assertThrows( + IllegalStateException.class, + () -> new FunctionExecutionMiddleware().invoke(context, chain)); + + writer.waitForTraces(1); + assertEquals(1, writer.size()); + DDSpan errorSpan = writer.firstTrace().get(0); + assertEquals("42", errorSpan.getTraceId().toString()); + assertEquals(43L, errorSpan.getParentId()); + assertEquals("DurableOrchestration Orchestrator", errorSpan.getResourceName().toString()); + assertTrue(errorSpan.isError()); + } + + @Test + void closesOrchestrationScopeWhenFailureInspectionFailsOpen() throws Exception { + Field actionsMethod = DurableOrchestrationUtils.class.getDeclaredField("getActionsMethod"); + actionsMethod.setAccessible(true); + Object previousMethod = actionsMethod.get(null); + AgentSpan previousSpan = activeSpan(); + try { + try (ContextScope context = DurableOrchestrationState.activate(null, "Orchestrator")) { + DurableOrchestrationState state = DurableOrchestrationState.current(); + ContextScope scope = DurableOrchestrationUtils.startSpanScope(state); + state.setSpan(activeSpan()); + + DurableOrchestrationExecutorInstrumentation.ExecuteAdvice.onExit( + scope, state, null, 0, null, new NullActionsResult()); + + assertNull(state.span()); + assertSame(previousSpan, activeSpan()); + DurableOrchestrationExecutorInstrumentation.ExecuteAdvice.onExit( + null, state, null, DurableFunctionsUtils.nowMicros(), null, new NullActionsResult()); + assertFalse(state.errorRecorded()); + } + } finally { + actionsMethod.set(null, previousMethod); + } assertNull(DurableOrchestrationState.current()); + writer.waitForTraces(1); + assertEquals(1, writer.size()); + } + + @Test + void finishesErrorSpanWhenDecorationThrows() throws Exception { + AgentSpan span = mock(AgentSpan.class); + when(span.addThrowable(any(Throwable.class), anyByte())) + .thenThrow(new BlockingException("blocked")); + when(span.isError()).thenReturn(false, true); + try (ContextScope context = DurableOrchestrationState.activate(null, "Orchestrator")) { + DurableOrchestrationState state = DurableOrchestrationState.current(); + + assertThrows( + BlockingException.class, + () -> + DurableFunctionsUtils.recordOrchestrationError( + span, new IllegalStateException("failure"), state)); + + verify(span).setError(true); + verify(span).finish(anyLong()); + assertTrue(state.errorRecorded()); + } + assertTraces(); + } + + @Test + void closesActivityScopeWhenDecorationThrows() throws Exception { + AgentSpan previousSpan = activeSpan(); + AgentSpan span = mock(AgentSpan.class, CALLS_REAL_METHODS); + when(span.addThrowable(any(Throwable.class), anyByte())) + .thenThrow(new BlockingException("blocked")); + ContextScope scope = Context.current().with(span).attach(); + + assertThrows( + BlockingException.class, + () -> + AzureFunctionsWorkerInstrumentation.InvokeAdvice.onExit( + contextFor("DurableActivityTrigger", "Activity"), + scope, + "DurableActivity", + null, + 0, + new IllegalStateException("failure"))); + + assertSame(previousSpan, activeSpan()); + verify(span).finish(); assertTraces(); } @@ -191,8 +353,8 @@ void findsOrchestrationSpanWhenAnotherSpanIsActive() throws Exception { MiddlewareChain orchestrationChain = mock(MiddlewareChain.class); doAnswer( invocation -> { - AgentSpan intervening = AgentTracer.startSpan("test", "intervening"); - try (ContextScope scope = AgentTracer.activateSpan(intervening)) { + AgentSpan intervening = startSpan("test", "intervening"); + try (ContextScope scope = activateSpan(intervening)) { new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); } finally { intervening.finish(); @@ -219,58 +381,69 @@ void findsOrchestrationSpanWhenAnotherSpanIsActive() throws Exception { @Test void findsOrchestrationSpanAcrossWorkerClassLoader() throws Exception { - MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); AgentSpan expectedSpan = mock(AgentSpan.class); + ClassLoader firstWorker = isolatedWorkerLoader(); + ClassLoader secondWorker = isolatedWorkerLoader(); - ClassLoader workerLoader = - new ClassLoader(AzureFunctionsWorkerTest.class.getClassLoader()) { - @Override - protected synchronized Class loadClass(String name, boolean resolve) - throws ClassNotFoundException { - if (!name.equals(DurableFunctionsUtils.class.getName())) { - return super.loadClass(name, resolve); - } - Class loaded = findLoadedClass(name); - if (loaded == null) { - String resource = name.replace('.', '/') + ".class"; - try (InputStream input = getParent().getResourceAsStream(resource)) { - if (input == null) { - throw new ClassNotFoundException(name); - } - ByteArrayOutputStream bytes = new ByteArrayOutputStream(); - byte[] buffer = new byte[4096]; - int count; - while ((count = input.read(buffer)) != -1) { - bytes.write(buffer, 0, count); - } - byte[] classBytes = bytes.toByteArray(); - loaded = defineClass(name, classBytes, 0, classBytes.length); - } catch (IOException e) { - throw new ClassNotFoundException(name, e); - } - } - if (resolve) { - resolveClass(loaded); - } - return loaded; - } - }; - - try (ContextScope scope = DurableOrchestrationState.activate(null)) { + try (ContextScope scope = DurableOrchestrationState.activate(null, "Orchestrator")) { DurableOrchestrationState.current().setSpan(expectedSpan); - Class workerUtils = - Class.forName(DurableFunctionsUtils.class.getName(), true, workerLoader); - assertNotSame(DurableFunctionsUtils.class, workerUtils); - - Object actualSpan = - workerUtils - .getMethod("onOrchestrationInvoke", MiddlewareContext.class) - .invoke(null, context); - assertSame(expectedSpan, actualSpan); + Class firstUtils = Class.forName(DurableFunctionsUtils.class.getName(), true, firstWorker); + Class secondUtils = + Class.forName(DurableFunctionsUtils.class.getName(), true, secondWorker); + assertNotSame(DurableFunctionsUtils.class, firstUtils); + assertNotSame(firstUtils, secondUtils); + for (Class workerUtils : new Class[] {firstUtils, secondUtils}) { + assertSame( + DurableOrchestrationState.class, + Class.forName( + DurableOrchestrationState.class.getName(), true, workerUtils.getClassLoader())); + assertSame(expectedSpan, workerUtils.getMethod("onOrchestrationInvoke").invoke(null)); + } } assertTraces(); } + @Test + void workerAdviceDoesNotNeedDurableTaskClasses() throws Exception { + assertFalse( + asList(new AzureFunctionsWorkerInstrumentation().helperClassNames()) + .contains(DurableOrchestrationUtils.class.getName())); + ClassLoader workerLoader = isolatedWorkerLoader(true); + assertThrows( + ClassNotFoundException.class, + () -> workerLoader.loadClass("com.microsoft.durabletask.TaskOrchestrationExecutor")); + Class workerAdvice = + Class.forName( + AzureFunctionsWorkerInstrumentation.InvokeAdvice.class.getName(), true, workerLoader); + assertNotSame(AzureFunctionsWorkerInstrumentation.InvokeAdvice.class, workerAdvice); + Method onExit = + workerAdvice.getMethod( + "onExit", + MiddlewareContext.class, + ContextScope.class, + String.class, + AgentSpan.class, + long.class, + Throwable.class); + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + + onExit.invoke( + null, + context, + null, + "DurableOrchestration", + null, + DurableFunctionsUtils.nowMicros(), + new IllegalStateException("worker failure")); + + writer.waitForTraces(1); + assertEquals(1, writer.size()); + assertEquals( + "DurableOrchestration Orchestrator", + writer.firstTrace().get(0).getResourceName().toString()); + assertTrue(writer.firstTrace().get(0).isError()); + } + @TableTest({ "scenario | subOrchestration", "failed activity | false ", @@ -281,8 +454,25 @@ void createsSpanForOrchestrationReplayWithFailure(boolean subOrchestration) thro invokeOrchestration( context, - Collections.singletonList(executionStarted()), - Collections.singletonList(failureEvent(subOrchestration)), + singletonList(executionStarted()), + singletonList(failureEvent(subOrchestration)), + mock(MiddlewareChain.class)); + + assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + } + + @Test + void createsSpanForOrchestrationReplayWithEntityFailure() throws Exception { + assumeTrue( + HistoryEvent.getDescriptor().findFieldByNumber(26) != null, + "Entity operation events require durabletask-client 1.9 or later"); + HistoryEvent entityFailure = HistoryEvent.parseFrom(new byte[] {(byte) 0xd2, 0x01, 0x00}); + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + + invokeOrchestration( + context, + singletonList(executionStarted()), + singletonList(entityFailure), mock(MiddlewareChain.class)); assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); @@ -300,10 +490,7 @@ void marksRetainedOrchestrationFailureAsError() throws Exception { .doNext(context); invokeOrchestration( - context, - Collections.singletonList(executionStarted()), - Collections.singletonList(failureEvent(false)), - chain); + context, singletonList(executionStarted()), singletonList(failureEvent(false)), chain); assertTraces( trace( @@ -322,14 +509,125 @@ void marksRetainedOrchestrationFailureAsError() throws Exception { tag("aas.function.trigger", is("DurableOrchestration"))))); } + @TableTest({ + "scenario | replay", + "initial execution | false ", + "completing replay | true " + }) + void marksSdkResultFailureAsError(boolean replay) throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + when(context.getReturnValue()).thenReturn(new FailingOutput()); + + invokeOrchestration( + context, + replay ? singletonList(executionStarted()) : emptyList(), + singletonList(replay ? taskCompleted() : executionStarted()), + mock(MiddlewareChain.class)); + + writer.waitForTraces(1); + assertEquals(1, writer.size()); + DDSpan span = writer.firstTrace().get(0); + assertEquals("DurableOrchestration Orchestrator", span.getResourceName().toString()); + assertTrue(span.isError()); + assertNotNull(span.getTag("error.type")); + assertTrue(String.valueOf(span.getTag("error.message")).contains("serialization failure")); + } + + @Test + void marksSdkResultFailureWithoutWorkerAdvice() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + when(context.getReturnValue()).thenReturn(new FailingOutput()); + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("test-instance") + .addNewEvents(executionStarted()) + .build(); + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request.toByteArray())); + + new OrchestrationMiddleware().invoke(context, mock(MiddlewareChain.class)); + + writer.waitForTraces(1); + assertEquals(1, writer.size()); + DDSpan span = writer.firstTrace().get(0); + assertEquals("DurableOrchestration Orchestrator", span.getResourceName().toString()); + assertTrue(span.isError()); + assertTrue(String.valueOf(span.getTag("error.message")).contains("serialization failure")); + } + + @Test + void recordsFailureAfterExecutorReturns() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("test-instance") + .addNewEvents(executionStarted()) + .build(); + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request.toByteArray())); + doAnswer( + invocation -> { + throw new IllegalStateException("response failure"); + }) + .when(context) + .updateReturnValue(any()); + + assertThrows( + IllegalStateException.class, + () -> new OrchestrationMiddleware().invoke(context, mock(MiddlewareChain.class))); + + writer.waitForTraces(2); + assertEquals(2, writer.stream().flatMap(List::stream).count()); + List errors = + writer.stream().flatMap(List::stream).filter(DDSpan::isError).collect(Collectors.toList()); + assertEquals(1, errors.size()); + assertEquals("DurableOrchestration Orchestrator", errors.get(0).getResourceName().toString()); + assertEquals("response failure", errors.get(0).getTag("error.message")); + } + + @Test + void doesNotDuplicateErrorWhenResponseFailsAfterFailedExecution() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("test-instance") + .addNewEvents(executionStarted()) + .build(); + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request.toByteArray())); + when(context.getReturnValue()).thenReturn(new FailingOutput()); + doAnswer( + invocation -> { + throw new IllegalStateException("response failure"); + }) + .when(context) + .updateReturnValue(any()); + + assertThrows( + IllegalStateException.class, + () -> new OrchestrationMiddleware().invoke(context, mock(MiddlewareChain.class))); + + writer.waitForTraces(1); + assertEquals(1, writer.stream().flatMap(List::stream).count()); + DDSpan errorSpan = writer.firstTrace().get(0); + assertTrue(errorSpan.isError()); + assertTrue(String.valueOf(errorSpan.getTag("error.message")).contains("serialization failure")); + } + @Test void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn( + "00-0000000000000000000000000000002a-000000000000002b-01", + "00-00000000000000000000000000000063-0000000000000064-01"); + when(context.getTraceContext()).thenReturn(traceContext); MiddlewareChain chain = mock(MiddlewareChain.class); - AtomicLong invocationStartMillis = new AtomicLong(); + AtomicLong invocationStartMicros = new AtomicLong(); doAnswer( invocation -> { - invocationStartMillis.set(System.currentTimeMillis()); + invocationStartMicros.set(DurableFunctionsUtils.nowMicros()); Thread.sleep(25); throw new InvocationTargetException(new IllegalStateException("replay failure")); }) @@ -337,20 +635,19 @@ void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { .doNext(context); invokeOrchestration( - context, - Collections.singletonList(executionStarted()), - Collections.singletonList(taskCompleted()), - chain); + context, singletonList(executionStarted()), singletonList(taskCompleted()), chain); writer.waitForTraces(1); DDSpan errorSpan = writer.firstTrace().get(0); - assertTrue(errorSpan.getStartTime() <= MILLISECONDS.toNanos(invocationStartMillis.get())); + assertEquals("42", errorSpan.getTraceId().toString()); + assertEquals(43L, errorSpan.getParentId()); + assertTrue(errorSpan.getStartTime() <= MICROSECONDS.toNanos(invocationStartMicros.get())); assertTrue(errorSpan.getDurationNano() >= MILLISECONDS.toNanos(20)); assertTraces( trace( span() - .root() + .childOf(43L) .operationName(Pattern.compile(Pattern.quote(operation()))) .resourceName("DurableOrchestration Orchestrator") .type(DDSpanTypes.SERVERLESS) @@ -364,35 +661,6 @@ void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { tag("aas.function.trigger", is("DurableOrchestration"))))); } - @Test - void stopsTraversingRepeatedExceptionCause() { - AtomicInteger causeReads = new AtomicInteger(); - Throwable[] cycle = new Throwable[2]; - cycle[0] = - new RuntimeException("first") { - @Override - public Throwable getCause() { - if (causeReads.incrementAndGet() > 2) { - throw new AssertionError("cause traversal did not terminate"); - } - return cycle[1]; - } - }; - cycle[1] = - new RuntimeException("second") { - @Override - public Throwable getCause() { - if (causeReads.incrementAndGet() > 2) { - throw new AssertionError("cause traversal did not terminate"); - } - return cycle[0]; - } - }; - - assertFalse(DurableFunctionsUtils.isReplayControlFlow(cycle[0])); - assertEquals(2, causeReads.get()); - } - @Test void honorsW3cDropDecisionWithoutTracestate() throws Exception { MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); @@ -405,7 +673,7 @@ void honorsW3cDropDecisionWithoutTracestate() throws Exception { MiddlewareChain chain = mock(MiddlewareChain.class); doAnswer( invocation -> { - priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + priority.set(activeSpan().spanContext().getSamplingPriority()); return null; }) .when(chain) @@ -433,7 +701,7 @@ void honorsW3cKeepDecisionWithoutTracestate() throws Exception { MiddlewareChain chain = mock(MiddlewareChain.class); doAnswer( invocation -> { - priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + priority.set(activeSpan().spanContext().getSamplingPriority()); return null; }) .when(chain) @@ -461,7 +729,7 @@ void honorsDatadogDropDecisionWhenAzureClearsW3cSampledFlag() throws Exception { MiddlewareChain chain = mock(MiddlewareChain.class); doAnswer( invocation -> { - priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + priority.set(activeSpan().spanContext().getSamplingPriority()); return null; }) .when(chain) @@ -489,7 +757,7 @@ void honorsOtelDropDecisionWhenAzureClearsW3cSampledFlag() throws Exception { MiddlewareChain chain = mock(MiddlewareChain.class); doAnswer( invocation -> { - priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + priority.set(activeSpan().spanContext().getSamplingPriority()); return null; }) .when(chain) @@ -517,7 +785,7 @@ void doesNotForceKeepForMalformedDatadogSamplingPriority() throws Exception { MiddlewareChain chain = mock(MiddlewareChain.class); doAnswer( invocation -> { - priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + priority.set(activeSpan().spanContext().getSamplingPriority()); return null; }) .when(chain) @@ -540,14 +808,13 @@ void honorsW3cDropWhenDatadogTracestateConflicts() throws Exception { MiddlewareChain chain = mock(MiddlewareChain.class); doAnswer( invocation -> { - priority.set(AgentTracer.activeSpan().spanContext().getSamplingPriority()); + priority.set(activeSpan().spanContext().getSamplingPriority()); return null; }) .when(chain) .doNext(context); - invokeOrchestration( - context, Collections.emptyList(), Collections.singletonList(executionStarted()), chain); + invokeOrchestration(context, emptyList(), singletonList(executionStarted()), chain); writer.waitForTraces(1); DDSpan span = writer.firstTrace().get(0); @@ -581,12 +848,65 @@ void doesNotMarkReplayControlFlowAsError(String controlFlowTypeName) throws Exce .when(chain) .doNext(context); - invokeOrchestration( - context, Collections.emptyList(), Collections.singletonList(executionStarted()), chain); + invokeOrchestration(context, emptyList(), singletonList(executionStarted()), chain); assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); } + @Test + void marksDeeplyWrappedReplayInterruptionAsRealFailure() throws Exception { + Throwable interruption = replayInterruption(); + assertFalse( + DurableFunctionsUtils.isReplayControlFlow( + new InvocationTargetException(new RuntimeException(interruption)))); + + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new InvocationTargetException(new RuntimeException(interruption)); + }) + .when(chain) + .doNext(context); + + invokeOrchestration(context, emptyList(), singletonList(executionStarted()), chain); + + assertTraces( + trace( + span() + .root() + .operationName(Pattern.compile(Pattern.quote(operation()))) + .resourceName("DurableOrchestration Orchestrator") + .type(DDSpanTypes.SERVERLESS) + .error(true) + .tags( + defaultTags(), + error(RuntimeException.class, interruption.toString()), + tag(Tags.COMPONENT, matches(Pattern.quote("azure-functions"))), + tag(Tags.SPAN_KIND, is(Tags.SPAN_KIND_SERVER)), + tag("aas.function.name", is("Orchestrator")), + tag("aas.function.trigger", is("DurableOrchestration"))))); + } + + @Test + void marksUnwrappedReplayInterruptionAsRealFailure() throws Exception { + Throwable interruption = replayInterruption(); + assertFalse(DurableFunctionsUtils.isReplayControlFlow(interruption)); + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw interruption; + }) + .when(chain) + .doNext(context); + + invokeOrchestration(context, emptyList(), singletonList(executionStarted()), chain); + + writer.waitForTraces(1); + assertTrue(writer.firstTrace().get(0).isError()); + } + @Test void marksWrappedApplicationFailuresAsErrors() throws Exception { MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); @@ -712,6 +1032,76 @@ private static Throwable instantiateThrowable(Class type) t return (Throwable) constructor.newInstance(arguments); } + private static ClassLoader isolatedWorkerLoader() { + return isolatedWorkerLoader(false); + } + + private static ClassLoader isolatedWorkerLoader(boolean hideDurableTask) { + return new ClassLoader(AzureFunctionsWorkerTest.class.getClassLoader()) { + @Override + protected synchronized Class loadClass(String name, boolean resolve) + throws ClassNotFoundException { + if (hideDurableTask && name.startsWith("com.microsoft.durabletask.")) { + throw new ClassNotFoundException(name); + } + if (!name.startsWith("datadog.trace.instrumentation.azure.functions.worker.")) { + return super.loadClass(name, resolve); + } + Class loaded = findLoadedClass(name); + if (loaded == null) { + String resource = name.replace('.', '/') + ".class"; + try (InputStream input = getParent().getResourceAsStream(resource)) { + if (input == null) { + throw new ClassNotFoundException(name); + } + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int count; + while ((count = input.read(buffer)) != -1) { + bytes.write(buffer, 0, count); + } + byte[] classBytes = bytes.toByteArray(); + loaded = defineClass(name, classBytes, 0, classBytes.length); + } catch (IOException e) { + throw new ClassNotFoundException(name, e); + } + } + if (resolve) { + resolveClass(loaded); + } + return loaded; + } + }; + } + + @SuppressWarnings("unchecked") + private static Throwable replayInterruption() throws Exception { + for (String typeName : + new String[] { + "com.microsoft.durabletask.OrchestratorBlockedException", + "com.microsoft.durabletask.interruption.OrchestratorBlockedException" + }) { + try { + return instantiateThrowable((Class) Class.forName(typeName)); + } catch (ClassNotFoundException ignored) { + // The SDK moved the interruption class in newer versions. + } + } + throw new AssertionError("No Durable Task replay interruption class found"); + } + + private static final class FailingOutput { + public String getValue() { + throw new IllegalStateException("serialization failure"); + } + } + + public static final class NullActionsResult { + public Collection getActions() { + return null; + } + } + private static MiddlewareContext contextFor(String annotation, String functionName) { MiddlewareContext context = mock(MiddlewareContext.class); when(context.getFunctionName()).thenReturn(functionName); diff --git a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/DurableOrchestrationState.java b/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/DurableOrchestrationState.java deleted file mode 100644 index 503ede5704c..00000000000 --- a/internal-api/src/main/java/datadog/trace/bootstrap/instrumentation/api/DurableOrchestrationState.java +++ /dev/null @@ -1,37 +0,0 @@ -package datadog.trace.bootstrap.instrumentation.api; - -import datadog.context.Context; -import datadog.context.ContextKey; -import datadog.context.ContextScope; - -/** Shares an Azure Durable orchestration invocation and its span across worker classloaders. */ -public final class DurableOrchestrationState { - private static final ContextKey KEY = - ContextKey.named("azure-durable-orchestration"); - - private AgentSpan span; - - private DurableOrchestrationState() {} - - public static ContextScope activate(AgentSpan remoteSpan) { - Context context = Context.current(); - if (remoteSpan != null && remoteSpan.isValid()) { - context = context.with(remoteSpan); - } - return context.with(KEY, new DurableOrchestrationState()).attach(); - } - - public static DurableOrchestrationState current() { - return Context.current().get(KEY); - } - - public AgentSpan span() { - return span; - } - - public AgentSpan setSpan(AgentSpan span) { - final AgentSpan previous = this.span; - this.span = span; - return previous; - } -} From fb59635366a5ff7175f9e24695a7ea5612ce4cbb Mon Sep 17 00:00:00 2001 From: jcstorms1 Date: Fri, 2 Oct 2026 10:33:22 -0400 Subject: [PATCH 18/18] Simplify Azure Durable Functions muzzle and error span finish --- .../azure/functions/worker/DurableFunctionsUtils.java | 2 +- .../DurableOrchestrationExecutorInstrumentation.java | 9 --------- ...DurableOrchestrationMiddlewareInstrumentation.java | 11 ----------- .../src/test/java/AzureFunctionsWorkerTest.java | 3 +-- 4 files changed, 2 insertions(+), 23 deletions(-) diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java index 33664abd75b..8cda909b0f1 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -95,7 +95,7 @@ public static void recordOrchestrationError( } } finally { try { - span.finish(nowMicros()); + span.finish(); } finally { if (state != null && span.isError()) { state.markErrorRecorded(); diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java index 8b29a0adfe5..a1bb053cfe1 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java @@ -13,7 +13,6 @@ import datadog.context.ContextScope; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.agent.tooling.muzzle.Reference; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; import java.util.List; @@ -23,9 +22,6 @@ public final class DurableOrchestrationExecutorInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - private static final Reference EXECUTOR_REFERENCE = - new Reference.Builder("com.microsoft.durabletask.TaskOrchestrationExecutor").build(); - public DurableOrchestrationExecutorInstrumentation() { super("azure-functions"); } @@ -35,11 +31,6 @@ public String instrumentedType() { return "com.microsoft.durabletask.TaskOrchestrationExecutor"; } - @Override - public Reference[] additionalMuzzleReferences() { - return new Reference[] {EXECUTOR_REFERENCE}; - } - @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java index 47b754c7998..107d1cca55a 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java @@ -11,7 +11,6 @@ import datadog.context.ContextScope; import datadog.trace.agent.tooling.Instrumenter; import datadog.trace.agent.tooling.InstrumenterModule; -import datadog.trace.agent.tooling.muzzle.Reference; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; import net.bytebuddy.asm.Advice; @@ -20,11 +19,6 @@ public final class DurableOrchestrationMiddlewareInstrumentation extends InstrumenterModule.Tracing implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { - private static final Reference MIDDLEWARE_REFERENCE = - new Reference.Builder( - "com.microsoft.durabletask.azurefunctions.internal.middleware.OrchestrationMiddleware") - .build(); - public DurableOrchestrationMiddlewareInstrumentation() { super("azure-functions"); } @@ -34,11 +28,6 @@ public String instrumentedType() { return "com.microsoft.durabletask.azurefunctions.internal.middleware.OrchestrationMiddleware"; } - @Override - public Reference[] additionalMuzzleReferences() { - return new Reference[] {MIDDLEWARE_REFERENCE}; - } - @Override public void methodAdvice(MethodTransformer transformer) { transformer.applyAdvice( diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java index c30cecf70b1..03965303ab7 100644 --- a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -27,7 +27,6 @@ import static org.junit.jupiter.api.Assumptions.assumeTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyByte; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.CALLS_REAL_METHODS; import static org.mockito.Mockito.doAnswer; @@ -309,7 +308,7 @@ void finishesErrorSpanWhenDecorationThrows() throws Exception { span, new IllegalStateException("failure"), state)); verify(span).setError(true); - verify(span).finish(anyLong()); + verify(span).finish(); assertTrue(state.errorRecorded()); } assertTraces();