diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 61524f9355c..f6bd693e0c4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -170,14 +170,15 @@ /utils/flare-utils/ @DataDog/apm-sdk-capabilities # @DataDog/apm-serverless -/dd-trace-core/src/*/*/datadog/trace/lambda/ @DataDog/apm-serverless -/utils/container-utils/ @DataDog/apm-serverless -**/InferredProxy* @DataDog/apm-serverless -/dd-java-agent/instrumentation/**/*InferredProxy* @DataDog/apm-serverless -/dd-java-agent/instrumentation/aws-java/ @DataDog/apm-serverless -/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/ @DataDog/apm-serverless -/dd-java-agent/instrumentation/azure-functions/ @DataDog/apm-serverless -/dd-java-agent/instrumentation/azure-functions-1.2.2/ @DataDog/apm-serverless +/dd-trace-core/src/*/*/datadog/trace/lambda/ @DataDog/apm-serverless +/utils/container-utils/ @DataDog/apm-serverless +**/InferredProxy* @DataDog/apm-serverless +/dd-java-agent/instrumentation/**/*InferredProxy* @DataDog/apm-serverless +/dd-java-agent/instrumentation/aws-java/ @DataDog/apm-serverless +/dd-java-agent/instrumentation/aws-java/aws-java-lambda-handler-1.2/ @DataDog/apm-serverless +/dd-java-agent/instrumentation/azure-functions/ @DataDog/apm-serverless +/dd-java-agent/instrumentation/azure-functions-1.2.2/ @DataDog/apm-serverless +/dd-java-agent/instrumentation/azure-functions-worker-2.7/ @DataDog/apm-serverless # @DataDog/apm-lang-platform-java /.agents/skills/fix-continuation-leakage/ @DataDog/apm-lang-platform-java diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationState.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationState.java new file mode 100644 index 00000000000..7c1b04f4a88 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationState.java @@ -0,0 +1,61 @@ +package datadog.trace.bootstrap.instrumentation.azure; + +import datadog.context.Context; +import datadog.context.ContextKey; +import datadog.context.ContextScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +/** Shares an Azure Durable orchestration invocation and its span across worker classloaders. */ +public final class DurableOrchestrationState { + private static final ContextKey KEY = + ContextKey.named("azure-durable-orchestration"); + + private final AgentSpan parentSpan; + private final String functionName; + private final AtomicReference span = new AtomicReference<>(); + private final AtomicBoolean errorRecorded = new AtomicBoolean(); + + private DurableOrchestrationState(AgentSpan parentSpan, String functionName) { + this.parentSpan = parentSpan; + this.functionName = functionName; + } + + public static ContextScope activate(AgentSpan remoteSpan, String functionName) { + Context context = Context.current(); + final AgentSpan parentSpan = remoteSpan != null && remoteSpan.isValid() ? remoteSpan : null; + if (parentSpan != null) { + context = context.with(parentSpan); + } + return context.with(KEY, new DurableOrchestrationState(parentSpan, functionName)).attach(); + } + + public static DurableOrchestrationState current() { + return Context.current().get(KEY); + } + + public AgentSpan span() { + return span.get(); + } + + public AgentSpan parentSpan() { + return parentSpan; + } + + public String functionName() { + return functionName; + } + + public boolean errorRecorded() { + return errorRecorded.get(); + } + + public void markErrorRecorded() { + errorRecorded.set(true); + } + + public AgentSpan setSpan(AgentSpan span) { + return this.span.getAndSet(span); + } +} diff --git a/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationStateTest.java b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationStateTest.java new file mode 100644 index 00000000000..93afc8c7d68 --- /dev/null +++ b/dd-java-agent/agent-bootstrap/src/test/java/datadog/trace/bootstrap/instrumentation/azure/DurableOrchestrationStateTest.java @@ -0,0 +1,61 @@ +package datadog.trace.bootstrap.instrumentation.azure; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import datadog.context.Context; +import datadog.context.ContextScope; +import datadog.trace.api.DDTraceId; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import org.junit.jupiter.api.Test; + +class DurableOrchestrationStateTest { + + @Test + void activatesRemoteParentAndRestoresPreviousContext() { + AgentSpanContext spanContext = mock(AgentSpanContext.class); + when(spanContext.getTraceId()).thenReturn(DDTraceId.from(42)); + when(spanContext.getSpanId()).thenReturn(43L); + AgentSpan parent = AgentSpan.fromSpanContext(spanContext); + AgentSpan orchestration = mock(AgentSpan.class); + + assertNull(DurableOrchestrationState.current()); + try (ContextScope scope = DurableOrchestrationState.activate(parent, "Orchestrator")) { + DurableOrchestrationState state = DurableOrchestrationState.current(); + assertSame(parent, state.parentSpan()); + assertSame("Orchestrator", state.functionName()); + assertSame(parent, AgentSpan.fromContext(Context.current())); + assertFalse(state.errorRecorded()); + state.markErrorRecorded(); + assertTrue(state.errorRecorded()); + assertNull(state.span()); + assertNull(state.setSpan(orchestration)); + assertSame(orchestration, state.span()); + assertSame(orchestration, state.setSpan(null)); + assertNull(state.span()); + } + assertNull(DurableOrchestrationState.current()); + } + + @Test + void ignoresMissingAndInvalidRemoteParents() { + AgentSpanContext invalidContext = mock(AgentSpanContext.class); + when(invalidContext.getTraceId()).thenReturn(DDTraceId.ZERO); + when(invalidContext.getSpanId()).thenReturn(0L); + AgentSpan invalidParent = AgentSpan.fromSpanContext(invalidContext); + assertFalse(invalidParent.isValid()); + try (ContextScope scope = DurableOrchestrationState.activate(null, null)) { + assertNull(DurableOrchestrationState.current().parentSpan()); + } + try (ContextScope scope = DurableOrchestrationState.activate(invalidParent, null)) { + assertNull(DurableOrchestrationState.current().parentSpan()); + assertNull(AgentSpan.fromContext(Context.current())); + } + assertNull(DurableOrchestrationState.current()); + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/README.md b/dd-java-agent/instrumentation/azure-functions-worker-2.7/README.md new file mode 100644 index 00000000000..90213a621e6 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/README.md @@ -0,0 +1,20 @@ +# Azure Durable Functions Java tracing + +For correlated host and Java worker traces, enable OpenTelemetry in the Function App's +`host.json`: + +```json +{ + "version": "2.0", + "telemetryMode": "OpenTelemetry" +} +``` + +The instrumentation honors the incoming W3C `traceparent` sampled flag and Datadog +`tracestate` sampling decision. It does not force-keep Durable invocations. If the host or an +upstream service sends a drop decision, the corresponding Durable spans may not be ingested. +Keep the application's sampling configuration consistent across the HTTP starter and Durable +worker invocations; enabling OpenTelemetry does not override an intentional upstream drop. + +See the [Azure Functions OpenTelemetry setup guide](https://learn.microsoft.com/en-us/azure/azure-functions/opentelemetry-howto) +for host configuration details. diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts new file mode 100644 index 00000000000..103c3815d4d --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/build.gradle.kts @@ -0,0 +1,69 @@ +import groovy.lang.Closure + +plugins { + id("dd-trace-java.module.instrumentation") +} + +muzzle { + // FunctionExecutionMiddleware lives in the unpublished Azure Functions Java worker. + // The SPI pass validates its public middleware contract, but cannot check the worker's + // internal class name or invoke signature. The test stub exercises the advice behavior; + // worker compatibility also requires validation against a running Function App. + pass { + group = "com.microsoft.azure.functions" + module = "azure-functions-java-spi" + versions = "[1.0.0,)" + extraDependency("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + extraDependency("com.microsoft:durabletask-azure-functions:1.0.1") + extraDependency("com.microsoft:durabletask-client:1.0.0") + } + pass { + group = "com.microsoft" + module = "durabletask-client" + versions = "[1.0.0,)" + extraDependency("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + extraDependency("com.microsoft.azure.functions:azure-functions-java-spi:1.0.0") + extraDependency("com.microsoft:durabletask-azure-functions:1.0.1") + } + pass { + group = "com.microsoft" + module = "durabletask-azure-functions" + versions = "[1.0.1,)" + extraDependency("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + extraDependency("com.microsoft.azure.functions:azure-functions-java-spi:1.0.0") + extraDependency("com.microsoft:durabletask-client:1.0.0") + } +} + +fun addTestSuiteForDir(name: String, directory: String) { + (project.extra["addTestSuiteForDir"] as Closure<*>).call(name, directory) +} + +addTestSuiteForDir("latestDepTest", "test") +(project.extra["addForkedTestTask"] as Closure<*>).call("latestDepTest") + +dependencies { + compileOnly("com.google.protobuf:protobuf-java:3.19.2") + compileOnly("com.microsoft:durabletask-client:1.0.0") + compileOnly("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + compileOnly("com.microsoft.azure.functions:azure-functions-java-spi:1.0.0") + + testImplementation("com.microsoft:durabletask-azure-functions:1.0.1") + testImplementation("com.microsoft:durabletask-client:1.0.0") + testImplementation("com.google.protobuf:protobuf-java:3.19.2") + testImplementation("com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0") + testImplementation("com.microsoft.azure.functions:azure-functions-java-spi:1.0.0") + testImplementation(libs.bundles.mockito) + + add("latestDepTestImplementation", "com.microsoft:durabletask-azure-functions:+") + add("latestDepTestImplementation", "com.microsoft:durabletask-client:+") + add("latestDepTestImplementation", "com.google.protobuf:protobuf-java:+") + add( + "latestDepTestImplementation", + "com.microsoft.azure.functions:azure-functions-java-core-library:+" + ) + add( + "latestDepTestImplementation", + "com.microsoft.azure.functions:azure-functions-java-spi:+" + ) +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile b/dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile new file mode 100644 index 00000000000..fb7f62d5403 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/gradle.lockfile @@ -0,0 +1,173 @@ +# This is a Gradle generated file for dependency locking. +# Manual edits can break the build and are not advised. +# This file is expected to be part of source control. +# To regenerate this file, run: ./gradlew :dd-java-agent:instrumentation:azure-functions-worker-2.7:dependencies --write-locks +cafe.cryptography:curve25519-elisabeth:0.1.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +cafe.cryptography:ed25519-elisabeth:0.1.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +ch.qos.logback:logback-classic:1.2.13=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +ch.qos.logback:logback-core:1.2.13=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.blogspot.mydailyjava:weak-lock-free:0.17=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq.okhttp3:okhttp:3.12.15=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq.okio:okio:1.17.6=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-instrument-java:0.0.5=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:dd-javac-plugin-client:0.2.2=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +com.datadoghq:java-dogstatsd-client:4.4.5=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.datadoghq:sketches-java:0.8.3=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.12.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-annotations:2.18.3=latestDepTestRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.12.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-core:2.18.3=latestDepTestRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.12.3=testRuntimeClasspath +com.fasterxml.jackson.core:jackson-databind:2.18.3=latestDepTestRuntimeClasspath +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.12.3=testRuntimeClasspath +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.3=latestDepTestRuntimeClasspath +com.fasterxml.jackson:jackson-bom:2.18.3=latestDepTestRuntimeClasspath +com.github.javaparser:javaparser-core:3.25.6=codenarc +com.github.jnr:jffi:1.3.15=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-a64asm:1.0.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-constants:0.10.4=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-enxio:0.32.20=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-ffi:2.2.19=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-posix:3.1.22=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-unixsocket:0.38.25=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.jnr:jnr-x86asm:1.0.2=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.github.spotbugs:spotbugs-annotations:4.10.4=compileClasspath,spotbugs +com.github.spotbugs:spotbugs:4.10.4=spotbugs +com.github.stephenc.jcip:jcip-annotations:1.0-1=spotbugs +com.google.android:annotations:4.1.1.4=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.google.api.grpc:proto-google-common-protos:2.0.1=testRuntimeClasspath +com.google.api.grpc:proto-google-common-protos:2.63.1=latestDepTestRuntimeClasspath +com.google.auto.service:auto-service-annotations:1.1.1=annotationProcessor,compileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,testAnnotationProcessor,testCompileClasspath +com.google.auto.service:auto-service:1.1.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.auto:auto-common:1.2.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.code.findbugs:jsr305:3.0.2=annotationProcessor,compileClasspath,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,spotbugs,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.code.gson:gson:2.12.1=latestDepTestRuntimeClasspath +com.google.code.gson:gson:2.14.0=spotbugs +com.google.code.gson:gson:2.8.9=testRuntimeClasspath +com.google.errorprone:error_prone_annotations:2.18.0=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.errorprone:error_prone_annotations:2.47.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.google.errorprone:error_prone_annotations:2.48.0=spotbugs +com.google.guava:failureaccess:1.0.1=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.guava:failureaccess:1.0.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.google.guava:guava:32.0.1-jre=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.guava:guava:33.6.0-jre=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=annotationProcessor,latestDepTestAnnotationProcessor,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testAnnotationProcessor,testCompileClasspath,testRuntimeClasspath +com.google.j2objc:j2objc-annotations:2.8=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +com.google.j2objc:j2objc-annotations:3.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.google.protobuf:protobuf-java:3.19.2=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.google.protobuf:protobuf-java:4.36.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath +com.google.re2j:re2j:1.8=latestDepTestRuntimeClasspath,testRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-core-library:1.2.0=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-core-library:1.3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-library:2.2.0=testRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-library:3.2.3=latestDepTestRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-spi:1.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.microsoft.azure.functions:azure-functions-java-spi:1.1.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath +com.microsoft:durabletask-azure-functions:1.0.1=testCompileClasspath,testRuntimeClasspath +com.microsoft:durabletask-azure-functions:1.9.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath +com.microsoft:durabletask-client:1.0.0=compileClasspath,testCompileClasspath,testRuntimeClasspath +com.microsoft:durabletask-client:1.9.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath +com.squareup.moshi:moshi:1.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:logging-interceptor:3.12.12=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.squareup.okhttp3:okhttp:3.12.12=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.squareup.okio:okio:1.17.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +com.thoughtworks.qdox:qdox:1.12.1=codenarc +commons-fileupload:commons-fileupload:1.5=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +commons-io:commons-io:2.21.0=spotbugs +de.thetaphi:forbiddenapis:3.11=compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.grpc:grpc-api:1.46.0=testRuntimeClasspath +io.grpc:grpc-api:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-context:1.46.0=testRuntimeClasspath +io.grpc:grpc-context:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-core:1.46.0=testRuntimeClasspath +io.grpc:grpc-core:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-netty-shaded:1.46.0=testRuntimeClasspath +io.grpc:grpc-netty-shaded:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-protobuf-lite:1.46.0=testRuntimeClasspath +io.grpc:grpc-protobuf-lite:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-protobuf:1.46.0=testRuntimeClasspath +io.grpc:grpc-protobuf:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-stub:1.46.0=testRuntimeClasspath +io.grpc:grpc-stub:1.78.0=latestDepTestRuntimeClasspath +io.grpc:grpc-util:1.78.0=latestDepTestRuntimeClasspath +io.leangen.geantyref:geantyref:1.3.16=latestDepTestRuntimeClasspath,testRuntimeClasspath +io.opentelemetry:opentelemetry-api:1.58.0=latestDepTestRuntimeClasspath +io.opentelemetry:opentelemetry-common:1.58.0=latestDepTestRuntimeClasspath +io.opentelemetry:opentelemetry-context:1.58.0=latestDepTestRuntimeClasspath +io.perfmark:perfmark-api:0.25.0=testRuntimeClasspath +io.perfmark:perfmark-api:0.27.0=latestDepTestRuntimeClasspath +io.sqreen:libsqreen:17.5.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +javax.servlet:javax.servlet-api:3.1.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +jaxen:jaxen:2.0.6=spotbugs +junit:junit:4.13.2=latestDepTestRuntimeClasspath,testRuntimeClasspath +net.bytebuddy:byte-buddy-agent:1.18.12=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.bytebuddy:byte-buddy:1.18.12=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testCompileClasspath,testRuntimeClasspath +net.java.dev.jna:jna-platform:5.8.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +net.java.dev.jna:jna:5.8.0=latestDepTestRuntimeClasspath,testRuntimeClasspath +net.sf.saxon:Saxon-HE:12.10=spotbugs +org.apache.ant:ant-antlr:1.10.14=codenarc +org.apache.ant:ant-junit:1.10.14=codenarc +org.apache.bcel:bcel:6.12.0=spotbugs +org.apache.commons:commons-lang3:3.20.0=spotbugs +org.apache.commons:commons-text:1.15.0=spotbugs +org.apache.logging.log4j:log4j-api:2.26.1=spotbugs +org.apache.logging.log4j:log4j-core:2.26.1=spotbugs +org.apiguardian:apiguardian-api:1.1.2=latestDepTestCompileClasspath,testCompileClasspath +org.checkerframework:checker-qual:3.33.0=annotationProcessor,latestDepTestAnnotationProcessor,testAnnotationProcessor +org.codehaus.groovy:groovy-ant:3.0.23=codenarc +org.codehaus.groovy:groovy-docgenerator:3.0.23=codenarc +org.codehaus.groovy:groovy-groovydoc:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.23=codenarc +org.codehaus.groovy:groovy-json:3.0.25=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.codehaus.groovy:groovy-templates:3.0.23=codenarc +org.codehaus.groovy:groovy-xml:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.23=codenarc +org.codehaus.groovy:groovy:3.0.25=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.codehaus.mojo:animal-sniffer-annotations:1.19=testRuntimeClasspath +org.codehaus.mojo:animal-sniffer-annotations:1.26=latestDepTestRuntimeClasspath +org.codenarc:CodeNarc:3.7.0=codenarc +org.dom4j:dom4j:2.2.0=spotbugs +org.gmetrics:GMetrics:2.1.0=codenarc +org.hamcrest:hamcrest-core:1.3=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.hamcrest:hamcrest:3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.jctools:jctools-core-jdk11:4.0.6=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.jctools:jctools-core:4.0.6=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.jspecify:jspecify:1.0.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-api:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-engine:5.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter-params:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.jupiter:junit-jupiter:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-commons:1.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-engine:1.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-launcher:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-runner:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-suite-api:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.junit.platform:junit-platform-suite-commons:1.14.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.junit:junit-bom:5.14.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.mockito:mockito-core:4.4.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.mockito:mockito-junit-jupiter:4.4.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.objenesis:objenesis:3.3=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.opentest4j:opentest4j:1.3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.ow2.asm:asm-analysis:9.10.1=spotbugs +org.ow2.asm:asm-analysis:9.7.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.ow2.asm:asm-commons:9.10.1=latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath +org.ow2.asm:asm-tree:9.10.1=latestDepTestRuntimeClasspath,spotbugs,testRuntimeClasspath +org.ow2.asm:asm-util:9.10.1=spotbugs +org.ow2.asm:asm-util:9.7.1=latestDepTestRuntimeClasspath,testRuntimeClasspath +org.ow2.asm:asm:9.10.1=buildTimeInstrumentationPlugin,compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,spotbugs,testCompileClasspath,testRuntimeClasspath +org.slf4j:jcl-over-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:jul-to-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:log4j-over-slf4j:1.7.30=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:1.7.30=buildTimeInstrumentationPlugin,compileClasspath,muzzleBootstrap,muzzleTooling,runtimeClasspath +org.slf4j:slf4j-api:1.7.32=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.slf4j:slf4j-api:2.0.17=spotbugsSlf4j +org.slf4j:slf4j-api:2.0.18=spotbugs +org.slf4j:slf4j-simple:2.0.17=spotbugsSlf4j +org.snakeyaml:snakeyaml-engine:2.9=buildTimeInstrumentationPlugin,latestDepTestRuntimeClasspath,muzzleTooling,runtimeClasspath,testRuntimeClasspath +org.spockframework:spock-bom:2.4-groovy-3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.spockframework:spock-core:2.4-groovy-3.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-junit:1.2.2=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.tabletest:tabletest-parser:1.2.1=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +org.xmlresolver:xmlresolver:5.3.3=spotbugs +empty=spotbugsPlugins diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java new file mode 100644 index 00000000000..3146f0e2a3c --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/AzureFunctionsWorkerInstrumentation.java @@ -0,0 +1,117 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.spanFromScope; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.ORCHESTRATION_TRIGGER; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; +import datadog.context.ContextScope; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; +import net.bytebuddy.asm.Advice; + +@AutoService(InstrumenterModule.class) +public final class AzureFunctionsWorkerInstrumentation extends InstrumenterModule.Tracing + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + public AzureFunctionsWorkerInstrumentation() { + super("azure-functions"); + } + + @Override + public String instrumentedType() { + return "com.microsoft.azure.functions.worker.chain.FunctionExecutionMiddleware"; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(isPublic()) + .and(named("invoke")) + .and(takesArguments(2)) + .and( + takesArgument( + 0, + named( + "com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext"))) + .and( + takesArgument( + 1, + named( + "com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain"))), + AzureFunctionsWorkerInstrumentation.class.getName() + "$InvokeAdvice"); + } + + public static class InvokeAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static ContextScope onEnter( + @Advice.Argument(0) MiddlewareContext context, + @Advice.Local("trigger") String trigger, + @Advice.Local("orchestrationSpan") AgentSpan orchestrationSpan, + @Advice.Local("startTimeMicros") long startTimeMicros) { + trigger = DurableFunctionsUtils.getTrigger(context); + if (trigger == null) { + return null; + } + if (ORCHESTRATION_TRIGGER.equals(trigger)) { + orchestrationSpan = DurableFunctionsUtils.onOrchestrationInvoke(); + if (orchestrationSpan == null) { + startTimeMicros = DurableFunctionsUtils.nowMicros(); + } + return null; + } + + return DurableFunctionsUtils.startSpanScope(context, trigger); + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.Argument(0) MiddlewareContext context, + @Advice.Enter ContextScope scope, + @Advice.Local("trigger") String trigger, + @Advice.Local("orchestrationSpan") AgentSpan orchestrationSpan, + @Advice.Local("startTimeMicros") long startTimeMicros, + @Advice.Thrown Throwable throwable) { + if (orchestrationSpan != null) { + if (throwable != null && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { + DECORATE.onInvocationError(orchestrationSpan, throwable); + } + return; + } + + if (scope == null) { + if (throwable != null + && ORCHESTRATION_TRIGGER.equals(trigger) + && !DurableFunctionsUtils.isReplayControlFlow(throwable)) { + final AgentSpan span = + DurableFunctionsUtils.startOrchestrationErrorSpan(context, startTimeMicros); + final DurableOrchestrationState state = DurableOrchestrationState.current(); + DurableFunctionsUtils.recordOrchestrationError(span, throwable, state); + } + return; + } + final AgentSpan span = spanFromScope(scope); + try { + if (!DurableFunctionsUtils.isReplayControlFlow(throwable)) { + DECORATE.onInvocationError(span, throwable); + } + DECORATE.beforeFinish(span); + } finally { + try { + scope.close(); + } finally { + span.finish(); + } + } + } + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java new file mode 100644 index 00000000000..f4dc6324842 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsDecorator.java @@ -0,0 +1,57 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import datadog.trace.api.naming.SpanNaming; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; +import datadog.trace.bootstrap.instrumentation.decorator.BaseDecorator; +import java.lang.reflect.InvocationTargetException; + +public final class DurableFunctionsDecorator extends BaseDecorator { + public static final String ORCHESTRATION_TRIGGER = "DurableOrchestration"; + public static final DurableFunctionsDecorator DECORATE = new DurableFunctionsDecorator(); + public static final CharSequence AZURE_FUNCTIONS_REQUEST = + UTF8BytesString.create( + SpanNaming.instance().namingSchema().cloud().operationForFaas("azure")); + + private static final CharSequence AZURE_FUNCTIONS = UTF8BytesString.create("azure-functions"); + + private DurableFunctionsDecorator() {} + + @Override + protected String[] instrumentationNames() { + return new String[] {"azure-functions"}; + } + + @Override + protected CharSequence spanType() { + return InternalSpanTypes.SERVERLESS; + } + + @Override + protected CharSequence component() { + return AZURE_FUNCTIONS; + } + + @Override + protected void doAfterStart(AgentSpan span) { + super.doAfterStart(span); + span.setTag(Tags.SPAN_KIND, Tags.SPAN_KIND_SERVER); + } + + public void onInvoke(AgentSpan span, String functionName, String trigger) { + span.setResourceName(functionName == null ? trigger : trigger + " " + functionName); + if (functionName != null) { + span.setTag("aas.function.name", functionName); + } + span.setTag("aas.function.trigger", trigger); + } + + public void onInvocationError(AgentSpan span, Throwable throwable) { + while (throwable instanceof InvocationTargetException && throwable.getCause() != null) { + throwable = throwable.getCause(); + } + onError(span, throwable); + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java new file mode 100644 index 00000000000..8cda909b0f1 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableFunctionsUtils.java @@ -0,0 +1,144 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.bootstrap.instrumentation.api.AgentPropagation.extractContextAndGetSpanContext; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.AZURE_FUNCTIONS_REQUEST; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.ORCHESTRATION_TRIGGER; +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import com.microsoft.azure.functions.TraceContext; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; +import datadog.context.ContextScope; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; +import java.time.Instant; + +public final class DurableFunctionsUtils { + private static final String ACTIVITY_ANNOTATION = "DurableActivityTrigger"; + private static final String ENTITY_ANNOTATION = "DurableEntityTrigger"; + private static final String ORCHESTRATION_ANNOTATION = "DurableOrchestrationTrigger"; + + private DurableFunctionsUtils() {} + + public static String getTrigger(MiddlewareContext context) { + if (context.getParameterName(ORCHESTRATION_ANNOTATION) != null) { + return ORCHESTRATION_TRIGGER; + } + if (context.getParameterName(ACTIVITY_ANNOTATION) != null) { + return "DurableActivity"; + } + if (context.getParameterName(ENTITY_ANNOTATION) != null) { + return "DurableEntity"; + } + return null; + } + + public static ContextScope startSpanScope(MiddlewareContext context, String trigger) { + return activateSpan( + startInvocationSpan( + context.getFunctionName(), trigger, extractParent(context.getTraceContext()), 0)); + } + + public static long nowMicros() { + final Instant now = Instant.now(); + return SECONDS.toMicros(now.getEpochSecond()) + NANOSECONDS.toMicros(now.getNano()); + } + + public static AgentSpan startOrchestrationErrorSpan( + MiddlewareContext context, long startTimeMicros) { + final DurableOrchestrationState state = DurableOrchestrationState.current(); + if (state != null) { + return startOrchestrationSpan(state, startTimeMicros); + } + return startInvocationSpan( + context.getFunctionName(), + ORCHESTRATION_TRIGGER, + extractParent(context.getTraceContext()), + startTimeMicros); + } + + public static AgentSpan startOrchestrationSpan( + DurableOrchestrationState state, long startTimeMicros) { + final AgentSpan parentSpan = state.parentSpan(); + return startInvocationSpan( + state.functionName(), + ORCHESTRATION_TRIGGER, + parentSpan == null ? null : parentSpan.spanContext(), + startTimeMicros); + } + + public static AgentSpan startInvocationSpan( + String functionName, String trigger, AgentSpanContext parent, long startTimeMicros) { + final AgentSpan span = + startTimeMicros > 0 + ? startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent, startTimeMicros) + : startSpan("azure-functions", AZURE_FUNCTIONS_REQUEST, parent); + DECORATE.afterStart(span); + DECORATE.onInvoke(span, functionName, trigger); + return span; + } + + public static void recordOrchestrationError( + AgentSpan span, Throwable throwable, DurableOrchestrationState state) { + try { + DECORATE.onInvocationError(span, throwable); + DECORATE.beforeFinish(span); + } finally { + try { + // This path is only used for failures, even if error decoration itself throws. + if (!span.isError()) { + span.setError(true); + } + } finally { + try { + span.finish(); + } finally { + if (state != null && span.isError()) { + state.markErrorRecorded(); + } + } + } + } + } + + private static AgentSpanContext.Extracted extractParent(TraceContext traceContext) { + return traceContext == null + ? null + : extractContextAndGetSpanContext(traceContext, TraceContextExtractAdapter.GETTER); + } + + public static ContextScope activateOrchestrationContext(MiddlewareContext context) { + if (context.getParameterName(ORCHESTRATION_ANNOTATION) == null) { + return null; + } + final AgentSpanContext.Extracted extracted = extractParent(context.getTraceContext()); + final AgentSpan remoteSpan = AgentSpan.fromSpanContext(extracted); + return DurableOrchestrationState.activate(remoteSpan, context.getFunctionName()); + } + + public static AgentSpan onOrchestrationInvoke() { + final DurableOrchestrationState state = DurableOrchestrationState.current(); + return state == null ? null : state.span(); + } + + public static boolean isReplayControlFlow(Throwable throwable) { + if (throwable == null) { + return false; + } + // The Durable middleware recognizes only the direct cause of the exception it catches. + // A more deeply wrapped interruption is treated by the SDK as a real failure. + final Throwable cause = throwable.getCause(); + return cause != null && isReplayInterruption(cause); + } + + private static boolean isReplayInterruption(Throwable throwable) { + final String className = throwable.getClass().getName(); + return "com.microsoft.durabletask.OrchestratorBlockedException".equals(className) + || "com.microsoft.durabletask.interruption.OrchestratorBlockedException".equals(className) + || "com.microsoft.durabletask.interruption.ContinueAsNewInterruption".equals(className); + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java new file mode 100644 index 00000000000..a1bb053cfe1 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationExecutorInstrumentation.java @@ -0,0 +1,111 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static datadog.trace.bootstrap.instrumentation.api.Java8BytecodeBridge.spanFromScope; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsDecorator.DECORATE; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils.startOrchestrationSpan; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; + +import com.google.auto.service.AutoService; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskFailureDetails; +import datadog.context.ContextScope; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; +import java.util.List; +import net.bytebuddy.asm.Advice; + +@AutoService(InstrumenterModule.class) +public final class DurableOrchestrationExecutorInstrumentation extends InstrumenterModule.Tracing + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + public DurableOrchestrationExecutorInstrumentation() { + super("azure-functions"); + } + + @Override + public String instrumentedType() { + return "com.microsoft.durabletask.TaskOrchestrationExecutor"; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(isPublic()) + .and(named("execute")) + .and(takesArgument(0, named("java.util.List"))) + .and(takesArgument(1, named("java.util.List"))), + DurableOrchestrationExecutorInstrumentation.class.getName() + "$ExecuteAdvice"); + } + + public static class ExecuteAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static ContextScope onEnter( + @Advice.Argument(0) List pastEvents, + @Advice.Argument(1) List newEvents, + @Advice.Local("state") DurableOrchestrationState state, + @Advice.Local("previousSpan") AgentSpan previousSpan, + @Advice.Local("startTimeMicros") long startTimeMicros) { + state = DurableOrchestrationState.current(); + if (state == null) { + return null; + } + startTimeMicros = DurableFunctionsUtils.nowMicros(); + if (!DurableOrchestrationUtils.shouldTrace(pastEvents, newEvents)) { + return null; + } + final ContextScope scope = DurableOrchestrationUtils.startSpanScope(state); + previousSpan = state.setSpan(spanFromScope(scope)); + return scope; + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.Enter ContextScope scope, + @Advice.Local("state") DurableOrchestrationState state, + @Advice.Local("previousSpan") AgentSpan previousSpan, + @Advice.Local("startTimeMicros") long startTimeMicros, + @Advice.Thrown Throwable throwable, + @Advice.Return Object result) { + if (state == null) { + return; + } + if (scope != null) { + final AgentSpan span = spanFromScope(scope); + try { + final TaskFailureDetails failure = + throwable == null ? DurableOrchestrationUtils.failureDetails(result) : null; + DECORATE.onInvocationError(span, throwable); + if (failure != null) { + DurableOrchestrationUtils.onOrchestrationFailure(span, failure); + } + DECORATE.beforeFinish(span); + } finally { + state.setSpan(previousSpan); + try { + scope.close(); + } finally { + span.finish(); + if (span.isError()) { + state.markErrorRecorded(); + } + } + } + return; + } + if (state.errorRecorded()) { + return; + } + final TaskFailureDetails failure = + throwable == null ? DurableOrchestrationUtils.failureDetails(result) : null; + if (failure != null || throwable != null) { + final AgentSpan span = startOrchestrationSpan(state, startTimeMicros); + DurableOrchestrationUtils.recordSdkOrchestrationError(span, throwable, failure, state); + } + } + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java new file mode 100644 index 00000000000..107d1cca55a --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationMiddlewareInstrumentation.java @@ -0,0 +1,84 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.agent.tooling.bytebuddy.matcher.NameMatchers.named; +import static net.bytebuddy.matcher.ElementMatchers.isMethod; +import static net.bytebuddy.matcher.ElementMatchers.isPublic; +import static net.bytebuddy.matcher.ElementMatchers.takesArgument; +import static net.bytebuddy.matcher.ElementMatchers.takesArguments; + +import com.google.auto.service.AutoService; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; +import datadog.context.ContextScope; +import datadog.trace.agent.tooling.Instrumenter; +import datadog.trace.agent.tooling.InstrumenterModule; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; +import net.bytebuddy.asm.Advice; + +@AutoService(InstrumenterModule.class) +public final class DurableOrchestrationMiddlewareInstrumentation extends InstrumenterModule.Tracing + implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice { + + public DurableOrchestrationMiddlewareInstrumentation() { + super("azure-functions"); + } + + @Override + public String instrumentedType() { + return "com.microsoft.durabletask.azurefunctions.internal.middleware.OrchestrationMiddleware"; + } + + @Override + public void methodAdvice(MethodTransformer transformer) { + transformer.applyAdvice( + isMethod() + .and(isPublic()) + .and(named("invoke")) + .and(takesArguments(2)) + .and( + takesArgument( + 0, + named( + "com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext"))) + .and( + takesArgument( + 1, + named( + "com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain"))), + DurableOrchestrationMiddlewareInstrumentation.class.getName() + "$InvokeAdvice"); + } + + public static class InvokeAdvice { + @Advice.OnMethodEnter(suppress = Throwable.class) + public static ContextScope onEnter( + @Advice.Argument(0) MiddlewareContext context, + @Advice.Local("startTimeMicros") long startTimeMicros) { + final ContextScope scope = DurableFunctionsUtils.activateOrchestrationContext(context); + if (scope != null) { + startTimeMicros = DurableFunctionsUtils.nowMicros(); + } + return scope; + } + + @Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class) + public static void onExit( + @Advice.Argument(0) MiddlewareContext context, + @Advice.Enter ContextScope scope, + @Advice.Local("startTimeMicros") long startTimeMicros, + @Advice.Thrown Throwable throwable) { + if (scope == null) { + return; + } + try { + final DurableOrchestrationState state = DurableOrchestrationState.current(); + if (throwable != null && state != null && !state.errorRecorded()) { + final AgentSpan span = + DurableFunctionsUtils.startOrchestrationErrorSpan(context, startTimeMicros); + DurableFunctionsUtils.recordOrchestrationError(span, throwable, state); + } + } finally { + scope.close(); + } + } + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java new file mode 100644 index 00000000000..983e6f5091e --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/DurableOrchestrationUtils.java @@ -0,0 +1,126 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils.recordOrchestrationError; +import static datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils.startOrchestrationSpan; + +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.HistoryEvent; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.HistoryEvent.EventTypeCase; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.OrchestrationStatus; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.OrchestratorAction; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskFailureDetails; +import datadog.context.ContextScope; +import datadog.trace.api.DDTags; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Collection; +import java.util.List; + +public final class DurableOrchestrationUtils { + private static volatile Method getActionsMethod; + private static volatile Class missingActionsClass; + + private DurableOrchestrationUtils() {} + + /** + * Returns whether an orchestration invocation should produce a span. + * + *

The Durable SDK has already decoded the request by this point. Successful replays are + * suppressed, while a replay containing a newly failed activity, sub-orchestration, or entity + * operation is retained. Only new events are inspected, so each history event is examined once + * instead of rescanning the complete history on every replay. User-created spans are not + * suppressed, because a replay may also execute new work after its last completed await. + */ + public static boolean shouldTrace(List pastEvents, List newEvents) { + if (pastEvents == null || pastEvents.isEmpty()) { + return true; + } + if (newEvents == null) { + return true; + } + + for (int index = 0; index < newEvents.size(); index++) { + final Object event = newEvents.get(index); + if (!(event instanceof HistoryEvent)) { + return true; + } + final EventTypeCase eventType = ((HistoryEvent) event).getEventTypeCase(); + if (eventType == EventTypeCase.TASKFAILED + || eventType == EventTypeCase.SUBORCHESTRATIONINSTANCEFAILED + || "ENTITYOPERATIONFAILED".equals(eventType.name())) { + return true; + } + } + return false; + } + + public static ContextScope startSpanScope(DurableOrchestrationState state) { + return activateSpan(startOrchestrationSpan(state, 0)); + } + + public static TaskFailureDetails failureDetails(Object result) { + if (result == null) { + return null; + } + try { + // TaskOrchestratorResult is package-private, although its accessor and action types are + // public. + final Class resultClass = result.getClass(); + if (resultClass == missingActionsClass) { + return null; + } + Method method = getActionsMethod; + if (method == null || method.getDeclaringClass() != resultClass) { + method = resultClass.getDeclaredMethod("getActions"); + method.setAccessible(true); + getActionsMethod = method; + } + for (Object value : (Collection) method.invoke(result)) { + if (value instanceof OrchestratorAction) { + final OrchestratorAction action = (OrchestratorAction) value; + if (action.hasCompleteOrchestration() + && action.getCompleteOrchestration().getOrchestrationStatus() + == OrchestrationStatus.ORCHESTRATION_STATUS_FAILED) { + return action.getCompleteOrchestration().getFailureDetails(); + } + } + } + } catch (NoSuchMethodException ignored) { + missingActionsClass = result.getClass(); + } catch (IllegalAccessException | InvocationTargetException | RuntimeException ignored) { + // The result is an internal SDK type; do not interfere with execution if it changes. + } + return null; + } + + public static void recordSdkOrchestrationError( + AgentSpan span, + Throwable throwable, + TaskFailureDetails failure, + DurableOrchestrationState state) { + try { + if (failure != null) { + onOrchestrationFailure(span, failure); + } + } finally { + recordOrchestrationError(span, throwable, state); + } + } + + public static void onOrchestrationFailure(AgentSpan span, TaskFailureDetails details) { + if (!span.isError()) { + span.setError(true); + if (!details.getErrorType().isEmpty()) { + span.setTag(DDTags.ERROR_TYPE, details.getErrorType()); + } + if (!details.getErrorMessage().isEmpty()) { + span.setTag(DDTags.ERROR_MSG, details.getErrorMessage()); + } + if (details.hasStackTrace()) { + span.setTag(DDTags.ERROR_STACK, details.getStackTrace().getValue()); + } + } + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java new file mode 100644 index 00000000000..19034ecb555 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/main/java/datadog/trace/instrumentation/azure/functions/worker/TraceContextExtractAdapter.java @@ -0,0 +1,26 @@ +package datadog.trace.instrumentation.azure.functions.worker; + +import com.microsoft.azure.functions.TraceContext; +import datadog.trace.bootstrap.instrumentation.api.AgentPropagation; + +public final class TraceContextExtractAdapter + implements AgentPropagation.ContextVisitor { + public static final TraceContextExtractAdapter GETTER = new TraceContextExtractAdapter(); + + private static final String TRACEPARENT = "traceparent"; + private static final String TRACESTATE = "tracestate"; + + private TraceContextExtractAdapter() {} + + @Override + public void forEachKey(TraceContext carrier, AgentPropagation.KeyClassifier classifier) { + final String traceparent = carrier.getTraceparent(); + if (traceparent != null && !classifier.accept(TRACEPARENT, traceparent)) { + return; + } + final String tracestate = carrier.getTracestate(); + if (tracestate != null) { + classifier.accept(TRACESTATE, tracestate); + } + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java new file mode 100644 index 00000000000..03965303ab7 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerTest.java @@ -0,0 +1,1112 @@ +import static datadog.trace.agent.test.assertions.SpanMatcher.span; +import static datadog.trace.agent.test.assertions.TagsMatcher.defaultTags; +import static datadog.trace.agent.test.assertions.TagsMatcher.error; +import static datadog.trace.agent.test.assertions.TagsMatcher.tag; +import static datadog.trace.agent.test.assertions.TraceMatcher.trace; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_DROP; +import static datadog.trace.api.sampling.PrioritySampling.SAMPLER_KEEP; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activateSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.activeSpan; +import static datadog.trace.bootstrap.instrumentation.api.AgentTracer.startSpan; +import static datadog.trace.test.junit.utils.assertions.Matchers.is; +import static datadog.trace.test.junit.utils.assertions.Matchers.matches; +import static java.util.Arrays.asList; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.concurrent.TimeUnit.MICROSECONDS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyByte; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import com.microsoft.azure.functions.TraceContext; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; +import com.microsoft.azure.functions.worker.chain.FunctionExecutionMiddleware; +import com.microsoft.durabletask.OrchestrationRunner; +import com.microsoft.durabletask.azurefunctions.internal.middleware.OrchestrationMiddleware; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.ExecutionStartedEvent; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.HistoryEvent; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.OrchestrationInstance; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.OrchestratorRequest; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.SubOrchestrationInstanceFailedEvent; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskCompletedEvent; +import com.microsoft.durabletask.implementation.protobuf.OrchestratorService.TaskFailedEvent; +import datadog.appsec.api.blocking.BlockingException; +import datadog.context.Context; +import datadog.context.ContextScope; +import datadog.trace.agent.test.AbstractInstrumentationTest; +import datadog.trace.agent.test.assertions.SpanMatcher; +import datadog.trace.api.DDSpanTypes; +import datadog.trace.bootstrap.instrumentation.api.AgentSpan; +import datadog.trace.bootstrap.instrumentation.api.Tags; +import datadog.trace.bootstrap.instrumentation.azure.DurableOrchestrationState; +import datadog.trace.core.DDSpan; +import datadog.trace.instrumentation.azure.functions.worker.AzureFunctionsWorkerInstrumentation; +import datadog.trace.instrumentation.azure.functions.worker.DurableFunctionsUtils; +import datadog.trace.instrumentation.azure.functions.worker.DurableOrchestrationExecutorInstrumentation; +import datadog.trace.instrumentation.azure.functions.worker.DurableOrchestrationUtils; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Base64; +import java.util.Collection; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import org.tabletest.junit.TableTest; + +abstract class AzureFunctionsWorkerTest extends AbstractInstrumentationTest { + + abstract String operation(); + + @TableTest({ + "scenario | annotation | trigger ", + "orchestration | DurableOrchestrationTrigger | DurableOrchestration", + "activity | DurableActivityTrigger | DurableActivity ", + "entity | DurableEntityTrigger | DurableEntity " + }) + void createsSpanForDurableTrigger(String annotation, String trigger) throws Exception { + MiddlewareContext context = contextFor(annotation, "MyFunction"); + + if ("DurableOrchestration".equals(trigger)) { + invokeOrchestration( + context, emptyList(), singletonList(executionStarted()), mock(MiddlewareChain.class)); + } else { + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + } + + assertTraces(trace(durableSpan("MyFunction", trigger))); + } + + @Test + void doesNotCreateSpanForNonDurableFunction() throws Exception { + MiddlewareContext context = contextFor(null, "HttpFunction"); + + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + + assertTraces(); + } + + @Test + void doesNotCreateSpanForSuccessfulOrchestrationReplay() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + + invokeOrchestration( + context, + singletonList(executionStarted()), + singletonList(taskCompleted()), + mock(MiddlewareChain.class)); + + assertTraces(); + } + + @Test + void preservesUserSpansFromNewWorkDuringOrchestrationReplay() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + AgentSpan userSpan = startSpan("test", "user-code"); + userSpan.finish(); + return null; + }) + .when(chain) + .doNext(context); + + invokeOrchestration( + context, singletonList(executionStarted()), singletonList(taskCompleted()), chain); + + writer.waitForTraces(1); + assertEquals(1, writer.firstTrace().size()); + assertEquals("user-code", writer.firstTrace().get(0).getOperationName().toString()); + } + + @Test + @SuppressWarnings("unchecked") + void doesNotTraversePastEventsForReplayDecision() { + List pastEvents = mock(List.class); + when(pastEvents.isEmpty()).thenReturn(false); + + assertFalse(DurableOrchestrationUtils.shouldTrace(pastEvents, singletonList(taskCompleted()))); + + verify(pastEvents).isEmpty(); + verifyNoMoreInteractions(pastEvents); + } + + @Test + void failsOpenForUnexpectedParsedEvent() { + assertTrue( + DurableOrchestrationUtils.shouldTrace( + singletonList(executionStarted()), singletonList(new Object()))); + } + + @Test + void createsSpanForInitialOrchestrationExecution() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + + invokeOrchestration( + context, emptyList(), singletonList(executionStarted()), mock(MiddlewareChain.class)); + + assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + } + + @Test + void orchestrationWithoutRemoteParentDoesNotJoinAmbientTrace() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + AgentSpan unrelated = startSpan("test", "unrelated"); + try (ContextScope scope = activateSpan(unrelated)) { + invokeOrchestration( + context, emptyList(), singletonList(executionStarted()), mock(MiddlewareChain.class)); + } finally { + unrelated.finish(); + } + + writer.waitForTraces(2); + DDSpan orchestration = + writer.stream() + .flatMap(List::stream) + .filter(span -> "DurableOrchestration".equals(span.getTag("aas.function.trigger"))) + .findFirst() + .orElseThrow(AssertionError::new); + assertEquals(0L, orchestration.getParentId()); + assertNotEquals(unrelated.getTraceId(), orchestration.getTraceId()); + } + + @Test + void doesNotCreateAzureSpanForStandaloneOrchestration() { + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("standalone-instance") + .addNewEvents(executionStarted()) + .build(); + + OrchestrationRunner.loadAndRun(request.toByteArray(), context -> null); + + assertTraces(); + } + + @Test + void closesOrchestrationContextWhenMiddlewareThrows() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + when(context.getParameterValue("input")).thenReturn("invalid base64!"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-01"); + when(context.getTraceContext()).thenReturn(traceContext); + AgentSpan previousSpan = activeSpan(); + + assertThrows( + IllegalArgumentException.class, + () -> new OrchestrationMiddleware().invoke(context, mock(MiddlewareChain.class))); + + assertSame(previousSpan, activeSpan()); + assertNull(DurableOrchestrationState.current()); + writer.waitForTraces(1); + assertEquals(1, writer.size()); + DDSpan errorSpan = writer.firstTrace().get(0); + assertEquals("42", errorSpan.getTraceId().toString()); + assertEquals(43L, errorSpan.getParentId()); + assertEquals("DurableOrchestration Orchestrator", errorSpan.getResourceName().toString()); + assertTrue(errorSpan.isError()); + } + + @Test + void retainsTraceParentForOrchestrationErrorWithoutMiddlewareState() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-01"); + when(context.getTraceContext()).thenReturn(traceContext); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new IllegalStateException("worker failure"); + }) + .when(chain) + .doNext(context); + + assertThrows( + IllegalStateException.class, + () -> new FunctionExecutionMiddleware().invoke(context, chain)); + + writer.waitForTraces(1); + assertEquals(1, writer.size()); + DDSpan errorSpan = writer.firstTrace().get(0); + assertEquals("42", errorSpan.getTraceId().toString()); + assertEquals(43L, errorSpan.getParentId()); + assertEquals("DurableOrchestration Orchestrator", errorSpan.getResourceName().toString()); + assertTrue(errorSpan.isError()); + } + + @Test + void closesOrchestrationScopeWhenFailureInspectionFailsOpen() throws Exception { + Field actionsMethod = DurableOrchestrationUtils.class.getDeclaredField("getActionsMethod"); + actionsMethod.setAccessible(true); + Object previousMethod = actionsMethod.get(null); + AgentSpan previousSpan = activeSpan(); + try { + try (ContextScope context = DurableOrchestrationState.activate(null, "Orchestrator")) { + DurableOrchestrationState state = DurableOrchestrationState.current(); + ContextScope scope = DurableOrchestrationUtils.startSpanScope(state); + state.setSpan(activeSpan()); + + DurableOrchestrationExecutorInstrumentation.ExecuteAdvice.onExit( + scope, state, null, 0, null, new NullActionsResult()); + + assertNull(state.span()); + assertSame(previousSpan, activeSpan()); + DurableOrchestrationExecutorInstrumentation.ExecuteAdvice.onExit( + null, state, null, DurableFunctionsUtils.nowMicros(), null, new NullActionsResult()); + assertFalse(state.errorRecorded()); + } + } finally { + actionsMethod.set(null, previousMethod); + } + assertNull(DurableOrchestrationState.current()); + writer.waitForTraces(1); + assertEquals(1, writer.size()); + } + + @Test + void finishesErrorSpanWhenDecorationThrows() throws Exception { + AgentSpan span = mock(AgentSpan.class); + when(span.addThrowable(any(Throwable.class), anyByte())) + .thenThrow(new BlockingException("blocked")); + when(span.isError()).thenReturn(false, true); + try (ContextScope context = DurableOrchestrationState.activate(null, "Orchestrator")) { + DurableOrchestrationState state = DurableOrchestrationState.current(); + + assertThrows( + BlockingException.class, + () -> + DurableFunctionsUtils.recordOrchestrationError( + span, new IllegalStateException("failure"), state)); + + verify(span).setError(true); + verify(span).finish(); + assertTrue(state.errorRecorded()); + } + assertTraces(); + } + + @Test + void closesActivityScopeWhenDecorationThrows() throws Exception { + AgentSpan previousSpan = activeSpan(); + AgentSpan span = mock(AgentSpan.class, CALLS_REAL_METHODS); + when(span.addThrowable(any(Throwable.class), anyByte())) + .thenThrow(new BlockingException("blocked")); + ContextScope scope = Context.current().with(span).attach(); + + assertThrows( + BlockingException.class, + () -> + AzureFunctionsWorkerInstrumentation.InvokeAdvice.onExit( + contextFor("DurableActivityTrigger", "Activity"), + scope, + "DurableActivity", + null, + 0, + new IllegalStateException("failure"))); + + assertSame(previousSpan, activeSpan()); + verify(span).finish(); + assertTraces(); + } + + @Test + void findsOrchestrationSpanWhenAnotherSpanIsActive() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("test-instance") + .addNewEvents(executionStarted()) + .build(); + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request.toByteArray())); + + MiddlewareChain orchestrationChain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + AgentSpan intervening = startSpan("test", "intervening"); + try (ContextScope scope = activateSpan(intervening)) { + new FunctionExecutionMiddleware().invoke(context, mock(MiddlewareChain.class)); + } finally { + intervening.finish(); + } + return null; + }) + .when(orchestrationChain) + .doNext(context); + + new OrchestrationMiddleware().invoke(context, orchestrationChain); + + writer.waitForTraces(1); + List spans = writer.firstTrace(); + assertEquals(2, spans.size()); + DDSpan orchestrationSpan = + spans.stream() + .filter(span -> "DurableOrchestration".equals(span.getTag("aas.function.trigger"))) + .findFirst() + .orElseThrow(AssertionError::new); + assertEquals("Orchestrator", orchestrationSpan.getTag("aas.function.name")); + assertEquals( + "DurableOrchestration Orchestrator", orchestrationSpan.getResourceName().toString()); + } + + @Test + void findsOrchestrationSpanAcrossWorkerClassLoader() throws Exception { + AgentSpan expectedSpan = mock(AgentSpan.class); + ClassLoader firstWorker = isolatedWorkerLoader(); + ClassLoader secondWorker = isolatedWorkerLoader(); + + try (ContextScope scope = DurableOrchestrationState.activate(null, "Orchestrator")) { + DurableOrchestrationState.current().setSpan(expectedSpan); + Class firstUtils = Class.forName(DurableFunctionsUtils.class.getName(), true, firstWorker); + Class secondUtils = + Class.forName(DurableFunctionsUtils.class.getName(), true, secondWorker); + assertNotSame(DurableFunctionsUtils.class, firstUtils); + assertNotSame(firstUtils, secondUtils); + for (Class workerUtils : new Class[] {firstUtils, secondUtils}) { + assertSame( + DurableOrchestrationState.class, + Class.forName( + DurableOrchestrationState.class.getName(), true, workerUtils.getClassLoader())); + assertSame(expectedSpan, workerUtils.getMethod("onOrchestrationInvoke").invoke(null)); + } + } + assertTraces(); + } + + @Test + void workerAdviceDoesNotNeedDurableTaskClasses() throws Exception { + assertFalse( + asList(new AzureFunctionsWorkerInstrumentation().helperClassNames()) + .contains(DurableOrchestrationUtils.class.getName())); + ClassLoader workerLoader = isolatedWorkerLoader(true); + assertThrows( + ClassNotFoundException.class, + () -> workerLoader.loadClass("com.microsoft.durabletask.TaskOrchestrationExecutor")); + Class workerAdvice = + Class.forName( + AzureFunctionsWorkerInstrumentation.InvokeAdvice.class.getName(), true, workerLoader); + assertNotSame(AzureFunctionsWorkerInstrumentation.InvokeAdvice.class, workerAdvice); + Method onExit = + workerAdvice.getMethod( + "onExit", + MiddlewareContext.class, + ContextScope.class, + String.class, + AgentSpan.class, + long.class, + Throwable.class); + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + + onExit.invoke( + null, + context, + null, + "DurableOrchestration", + null, + DurableFunctionsUtils.nowMicros(), + new IllegalStateException("worker failure")); + + writer.waitForTraces(1); + assertEquals(1, writer.size()); + assertEquals( + "DurableOrchestration Orchestrator", + writer.firstTrace().get(0).getResourceName().toString()); + assertTrue(writer.firstTrace().get(0).isError()); + } + + @TableTest({ + "scenario | subOrchestration", + "failed activity | false ", + "failed sub-orchestration | true " + }) + void createsSpanForOrchestrationReplayWithFailure(boolean subOrchestration) throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + + invokeOrchestration( + context, + singletonList(executionStarted()), + singletonList(failureEvent(subOrchestration)), + mock(MiddlewareChain.class)); + + assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + } + + @Test + void createsSpanForOrchestrationReplayWithEntityFailure() throws Exception { + assumeTrue( + HistoryEvent.getDescriptor().findFieldByNumber(26) != null, + "Entity operation events require durabletask-client 1.9 or later"); + HistoryEvent entityFailure = HistoryEvent.parseFrom(new byte[] {(byte) 0xd2, 0x01, 0x00}); + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + + invokeOrchestration( + context, + singletonList(executionStarted()), + singletonList(entityFailure), + mock(MiddlewareChain.class)); + + assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + } + + @Test + void marksRetainedOrchestrationFailureAsError() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new InvocationTargetException(new IllegalStateException("activity failure")); + }) + .when(chain) + .doNext(context); + + invokeOrchestration( + context, singletonList(executionStarted()), singletonList(failureEvent(false)), chain); + + assertTraces( + trace( + span() + .root() + .operationName(Pattern.compile(Pattern.quote(operation()))) + .resourceName("DurableOrchestration Orchestrator") + .type(DDSpanTypes.SERVERLESS) + .error(true) + .tags( + defaultTags(), + error(IllegalStateException.class, "activity failure"), + tag(Tags.COMPONENT, matches(Pattern.quote("azure-functions"))), + tag(Tags.SPAN_KIND, is(Tags.SPAN_KIND_SERVER)), + tag("aas.function.name", is("Orchestrator")), + tag("aas.function.trigger", is("DurableOrchestration"))))); + } + + @TableTest({ + "scenario | replay", + "initial execution | false ", + "completing replay | true " + }) + void marksSdkResultFailureAsError(boolean replay) throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + when(context.getReturnValue()).thenReturn(new FailingOutput()); + + invokeOrchestration( + context, + replay ? singletonList(executionStarted()) : emptyList(), + singletonList(replay ? taskCompleted() : executionStarted()), + mock(MiddlewareChain.class)); + + writer.waitForTraces(1); + assertEquals(1, writer.size()); + DDSpan span = writer.firstTrace().get(0); + assertEquals("DurableOrchestration Orchestrator", span.getResourceName().toString()); + assertTrue(span.isError()); + assertNotNull(span.getTag("error.type")); + assertTrue(String.valueOf(span.getTag("error.message")).contains("serialization failure")); + } + + @Test + void marksSdkResultFailureWithoutWorkerAdvice() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + when(context.getReturnValue()).thenReturn(new FailingOutput()); + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("test-instance") + .addNewEvents(executionStarted()) + .build(); + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request.toByteArray())); + + new OrchestrationMiddleware().invoke(context, mock(MiddlewareChain.class)); + + writer.waitForTraces(1); + assertEquals(1, writer.size()); + DDSpan span = writer.firstTrace().get(0); + assertEquals("DurableOrchestration Orchestrator", span.getResourceName().toString()); + assertTrue(span.isError()); + assertTrue(String.valueOf(span.getTag("error.message")).contains("serialization failure")); + } + + @Test + void recordsFailureAfterExecutorReturns() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("test-instance") + .addNewEvents(executionStarted()) + .build(); + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request.toByteArray())); + doAnswer( + invocation -> { + throw new IllegalStateException("response failure"); + }) + .when(context) + .updateReturnValue(any()); + + assertThrows( + IllegalStateException.class, + () -> new OrchestrationMiddleware().invoke(context, mock(MiddlewareChain.class))); + + writer.waitForTraces(2); + assertEquals(2, writer.stream().flatMap(List::stream).count()); + List errors = + writer.stream().flatMap(List::stream).filter(DDSpan::isError).collect(Collectors.toList()); + assertEquals(1, errors.size()); + assertEquals("DurableOrchestration Orchestrator", errors.get(0).getResourceName().toString()); + assertEquals("response failure", errors.get(0).getTag("error.message")); + } + + @Test + void doesNotDuplicateErrorWhenResponseFailsAfterFailedExecution() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("test-instance") + .addNewEvents(executionStarted()) + .build(); + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request.toByteArray())); + when(context.getReturnValue()).thenReturn(new FailingOutput()); + doAnswer( + invocation -> { + throw new IllegalStateException("response failure"); + }) + .when(context) + .updateReturnValue(any()); + + assertThrows( + IllegalStateException.class, + () -> new OrchestrationMiddleware().invoke(context, mock(MiddlewareChain.class))); + + writer.waitForTraces(1); + assertEquals(1, writer.stream().flatMap(List::stream).count()); + DDSpan errorSpan = writer.firstTrace().get(0); + assertTrue(errorSpan.isError()); + assertTrue(String.valueOf(errorSpan.getTag("error.message")).contains("serialization failure")); + } + + @Test + void createsErrorSpanWhenSuppressedOrchestrationReplayFails() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn( + "00-0000000000000000000000000000002a-000000000000002b-01", + "00-00000000000000000000000000000063-0000000000000064-01"); + when(context.getTraceContext()).thenReturn(traceContext); + MiddlewareChain chain = mock(MiddlewareChain.class); + AtomicLong invocationStartMicros = new AtomicLong(); + doAnswer( + invocation -> { + invocationStartMicros.set(DurableFunctionsUtils.nowMicros()); + Thread.sleep(25); + throw new InvocationTargetException(new IllegalStateException("replay failure")); + }) + .when(chain) + .doNext(context); + + invokeOrchestration( + context, singletonList(executionStarted()), singletonList(taskCompleted()), chain); + + writer.waitForTraces(1); + DDSpan errorSpan = writer.firstTrace().get(0); + assertEquals("42", errorSpan.getTraceId().toString()); + assertEquals(43L, errorSpan.getParentId()); + assertTrue(errorSpan.getStartTime() <= MICROSECONDS.toNanos(invocationStartMicros.get())); + assertTrue(errorSpan.getDurationNano() >= MILLISECONDS.toNanos(20)); + + assertTraces( + trace( + span() + .childOf(43L) + .operationName(Pattern.compile(Pattern.quote(operation()))) + .resourceName("DurableOrchestration Orchestrator") + .type(DDSpanTypes.SERVERLESS) + .error(true) + .tags( + defaultTags(), + error(IllegalStateException.class, "replay failure"), + tag(Tags.COMPONENT, matches(Pattern.quote("azure-functions"))), + tag(Tags.SPAN_KIND, is(Tags.SPAN_KIND_SERVER)), + tag("aas.function.name", is("Orchestrator")), + tag("aas.function.trigger", is("DurableOrchestration"))))); + } + + @Test + void honorsW3cDropDecisionWithoutTracestate() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); + when(traceContext.getTracestate()).thenReturn(null); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + + writer.waitForTraces(1); + DDSpan span = writer.firstTrace().get(0); + assertEquals("42", span.getTraceId().toString()); + assertEquals(43L, span.getParentId()); + assertEquals(SAMPLER_DROP, priority.get()); + assertEquals(SAMPLER_DROP, span.samplingPriority()); + } + + @Test + void honorsW3cKeepDecisionWithoutTracestate() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-01"); + when(traceContext.getTracestate()).thenReturn(null); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + + writer.waitForTraces(1); + DDSpan span = writer.firstTrace().get(0); + assertEquals("42", span.getTraceId().toString()); + assertEquals(43L, span.getParentId()); + assertEquals(SAMPLER_KEEP, priority.get()); + assertEquals(SAMPLER_KEEP, span.samplingPriority()); + } + + @Test + void honorsDatadogDropDecisionWhenAzureClearsW3cSampledFlag() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); + when(traceContext.getTracestate()).thenReturn("dd=s:0"); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + + assertEquals(SAMPLER_DROP, priority.get()); + writer.waitForTraces(1); + DDSpan span = writer.firstTrace().get(0); + assertEquals("42", span.getTraceId().toString()); + assertEquals(43L, span.getParentId()); + assertEquals(SAMPLER_DROP, span.samplingPriority()); + } + + @Test + void honorsOtelDropDecisionWhenAzureClearsW3cSampledFlag() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); + when(traceContext.getTracestate()).thenReturn("ot=rv:00000000000000;th:8,vendor=value"); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + + assertEquals(SAMPLER_DROP, priority.get()); + writer.waitForTraces(1); + DDSpan span = writer.firstTrace().get(0); + assertEquals("42", span.getTraceId().toString()); + assertEquals(43L, span.getParentId()); + assertEquals(SAMPLER_DROP, span.samplingPriority()); + } + + @Test + void doesNotForceKeepForMalformedDatadogSamplingPriority() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); + when(traceContext.getTracestate()).thenReturn("dd=s:not-a-number"); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + new FunctionExecutionMiddleware().invoke(context, chain); + + assertEquals(SAMPLER_DROP, priority.get()); + } + + @Test + void honorsW3cDropWhenDatadogTracestateConflicts() throws Exception { + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + TraceContext traceContext = mock(TraceContext.class); + when(traceContext.getTraceparent()) + .thenReturn("00-0000000000000000000000000000002a-000000000000002b-00"); + when(traceContext.getTracestate()).thenReturn("vendor=value,\tdd=s:2;o:rum\t"); + when(context.getTraceContext()).thenReturn(traceContext); + AtomicInteger priority = new AtomicInteger(Integer.MIN_VALUE); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + priority.set(activeSpan().spanContext().getSamplingPriority()); + return null; + }) + .when(chain) + .doNext(context); + + invokeOrchestration(context, emptyList(), singletonList(executionStarted()), chain); + + writer.waitForTraces(1); + DDSpan span = writer.firstTrace().get(0); + assertEquals("42", span.getTraceId().toString()); + assertEquals(43L, span.getParentId()); + assertEquals(SAMPLER_DROP, priority.get()); + } + + @TableTest({ + "scenario | controlFlowTypeName ", + "legacy OrchestratorBlockedException | com.microsoft.durabletask.OrchestratorBlockedException ", + "OrchestratorBlockedException | com.microsoft.durabletask.interruption.OrchestratorBlockedException", + "ContinueAsNewInterruption | com.microsoft.durabletask.interruption.ContinueAsNewInterruption " + }) + @SuppressWarnings("unchecked") + void doesNotMarkReplayControlFlowAsError(String controlFlowTypeName) throws Exception { + Class controlFlowType; + try { + controlFlowType = (Class) Class.forName(controlFlowTypeName); + } catch (ClassNotFoundException ignored) { + assumeTrue(false, controlFlowTypeName + " is not available in this Durable Task version"); + return; + } + Throwable controlFlow = instantiateThrowable(controlFlowType); + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new RuntimeException(controlFlow); + }) + .when(chain) + .doNext(context); + + invokeOrchestration(context, emptyList(), singletonList(executionStarted()), chain); + + assertTraces(trace(durableSpan("Orchestrator", "DurableOrchestration"))); + } + + @Test + void marksDeeplyWrappedReplayInterruptionAsRealFailure() throws Exception { + Throwable interruption = replayInterruption(); + assertFalse( + DurableFunctionsUtils.isReplayControlFlow( + new InvocationTargetException(new RuntimeException(interruption)))); + + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new InvocationTargetException(new RuntimeException(interruption)); + }) + .when(chain) + .doNext(context); + + invokeOrchestration(context, emptyList(), singletonList(executionStarted()), chain); + + assertTraces( + trace( + span() + .root() + .operationName(Pattern.compile(Pattern.quote(operation()))) + .resourceName("DurableOrchestration Orchestrator") + .type(DDSpanTypes.SERVERLESS) + .error(true) + .tags( + defaultTags(), + error(RuntimeException.class, interruption.toString()), + tag(Tags.COMPONENT, matches(Pattern.quote("azure-functions"))), + tag(Tags.SPAN_KIND, is(Tags.SPAN_KIND_SERVER)), + tag("aas.function.name", is("Orchestrator")), + tag("aas.function.trigger", is("DurableOrchestration"))))); + } + + @Test + void marksUnwrappedReplayInterruptionAsRealFailure() throws Exception { + Throwable interruption = replayInterruption(); + assertFalse(DurableFunctionsUtils.isReplayControlFlow(interruption)); + MiddlewareContext context = contextFor("DurableOrchestrationTrigger", "Orchestrator"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw interruption; + }) + .when(chain) + .doNext(context); + + invokeOrchestration(context, emptyList(), singletonList(executionStarted()), chain); + + writer.waitForTraces(1); + assertTrue(writer.firstTrace().get(0).isError()); + } + + @Test + void marksWrappedApplicationFailuresAsErrors() throws Exception { + MiddlewareContext context = contextFor("DurableActivityTrigger", "Activity"); + MiddlewareChain chain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + throw new InvocationTargetException(new IllegalStateException("failure")); + }) + .when(chain) + .doNext(context); + + assertThrows( + InvocationTargetException.class, + () -> new FunctionExecutionMiddleware().invoke(context, chain)); + + assertTraces( + trace( + span() + .root() + .operationName(Pattern.compile(Pattern.quote(operation()))) + .resourceName("DurableActivity Activity") + .type(DDSpanTypes.SERVERLESS) + .error(true) + .tags( + defaultTags(), + error(IllegalStateException.class, "failure"), + tag(Tags.COMPONENT, matches(Pattern.quote("azure-functions"))), + tag(Tags.SPAN_KIND, is(Tags.SPAN_KIND_SERVER)), + tag("aas.function.name", is("Activity")), + tag("aas.function.trigger", is("DurableActivity"))))); + } + + private SpanMatcher durableSpan(String functionName, String trigger) { + return span() + .root() + .operationName(Pattern.compile(Pattern.quote(operation()))) + .resourceName(trigger + " " + functionName) + .type(DDSpanTypes.SERVERLESS) + .error(false) + .tags( + defaultTags(), + tag(Tags.COMPONENT, matches(Pattern.quote("azure-functions"))), + tag(Tags.SPAN_KIND, is(Tags.SPAN_KIND_SERVER)), + tag("aas.function.name", is(functionName)), + tag("aas.function.trigger", is(trigger))); + } + + private static void invokeOrchestration( + MiddlewareContext context, + List pastEvents, + List newEvents, + MiddlewareChain functionChain) + throws Exception { + OrchestratorRequest request = + OrchestratorRequest.newBuilder() + .setInstanceId("test-instance") + .addAllPastEvents(pastEvents) + .addAllNewEvents(newEvents) + .build(); + when(context.getParameterValue("input")) + .thenReturn(Base64.getEncoder().encodeToString(request.toByteArray())); + + MiddlewareChain orchestrationChain = mock(MiddlewareChain.class); + doAnswer( + invocation -> { + new FunctionExecutionMiddleware().invoke(context, functionChain); + return null; + }) + .when(orchestrationChain) + .doNext(context); + + new OrchestrationMiddleware().invoke(context, orchestrationChain); + } + + private static HistoryEvent executionStarted() { + return HistoryEvent.newBuilder() + .setExecutionStarted( + ExecutionStartedEvent.newBuilder() + .setName("Orchestrator") + .setOrchestrationInstance( + OrchestrationInstance.newBuilder().setInstanceId("test-instance"))) + .build(); + } + + private static HistoryEvent taskCompleted() { + return HistoryEvent.newBuilder() + .setTaskCompleted(TaskCompletedEvent.getDefaultInstance()) + .build(); + } + + private static HistoryEvent failureEvent(boolean subOrchestration) { + HistoryEvent.Builder event = HistoryEvent.newBuilder(); + if (subOrchestration) { + event.setSubOrchestrationInstanceFailed( + SubOrchestrationInstanceFailedEvent.getDefaultInstance()); + } else { + event.setTaskFailed(TaskFailedEvent.getDefaultInstance()); + } + return event.build(); + } + + private static Throwable instantiateThrowable(Class type) throws Exception { + Constructor constructor = type.getDeclaredConstructors()[0]; + for (Constructor candidate : type.getDeclaredConstructors()) { + if (candidate.getParameterCount() < constructor.getParameterCount()) { + constructor = candidate; + } + } + constructor.setAccessible(true); + Object[] arguments = new Object[constructor.getParameterCount()]; + Class[] parameterTypes = constructor.getParameterTypes(); + for (int index = 0; index < parameterTypes.length; index++) { + if (parameterTypes[index] == String.class) { + arguments[index] = "test"; + } else if (parameterTypes[index] == boolean.class) { + arguments[index] = false; + } else if (parameterTypes[index] == char.class) { + arguments[index] = '\0'; + } else if (parameterTypes[index].isPrimitive()) { + arguments[index] = 0; + } + } + return (Throwable) constructor.newInstance(arguments); + } + + private static ClassLoader isolatedWorkerLoader() { + return isolatedWorkerLoader(false); + } + + private static ClassLoader isolatedWorkerLoader(boolean hideDurableTask) { + return new ClassLoader(AzureFunctionsWorkerTest.class.getClassLoader()) { + @Override + protected synchronized Class loadClass(String name, boolean resolve) + throws ClassNotFoundException { + if (hideDurableTask && name.startsWith("com.microsoft.durabletask.")) { + throw new ClassNotFoundException(name); + } + if (!name.startsWith("datadog.trace.instrumentation.azure.functions.worker.")) { + return super.loadClass(name, resolve); + } + Class loaded = findLoadedClass(name); + if (loaded == null) { + String resource = name.replace('.', '/') + ".class"; + try (InputStream input = getParent().getResourceAsStream(resource)) { + if (input == null) { + throw new ClassNotFoundException(name); + } + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int count; + while ((count = input.read(buffer)) != -1) { + bytes.write(buffer, 0, count); + } + byte[] classBytes = bytes.toByteArray(); + loaded = defineClass(name, classBytes, 0, classBytes.length); + } catch (IOException e) { + throw new ClassNotFoundException(name, e); + } + } + if (resolve) { + resolveClass(loaded); + } + return loaded; + } + }; + } + + @SuppressWarnings("unchecked") + private static Throwable replayInterruption() throws Exception { + for (String typeName : + new String[] { + "com.microsoft.durabletask.OrchestratorBlockedException", + "com.microsoft.durabletask.interruption.OrchestratorBlockedException" + }) { + try { + return instantiateThrowable((Class) Class.forName(typeName)); + } catch (ClassNotFoundException ignored) { + // The SDK moved the interruption class in newer versions. + } + } + throw new AssertionError("No Durable Task replay interruption class found"); + } + + private static final class FailingOutput { + public String getValue() { + throw new IllegalStateException("serialization failure"); + } + } + + public static final class NullActionsResult { + public Collection getActions() { + return null; + } + } + + private static MiddlewareContext contextFor(String annotation, String functionName) { + MiddlewareContext context = mock(MiddlewareContext.class); + when(context.getFunctionName()).thenReturn(functionName); + when(context.getParameterName(anyString())) + .thenAnswer( + invocation -> Objects.equals(invocation.getArgument(0), annotation) ? "input" : null); + return context; + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV0ForkedTest.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV0ForkedTest.java new file mode 100644 index 00000000000..e3d36f11ea3 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV0ForkedTest.java @@ -0,0 +1,12 @@ +import static datadog.trace.api.config.TracerConfig.TRACE_SPAN_ATTRIBUTE_SCHEMA; + +import datadog.trace.test.junit.utils.config.WithConfig; + +@WithConfig(key = TRACE_SPAN_ATTRIBUTE_SCHEMA, value = "v0") +class AzureFunctionsWorkerV0ForkedTest extends AzureFunctionsWorkerTest { + + @Override + String operation() { + return "dd-tracer-serverless-span"; + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV1Test.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV1Test.java new file mode 100644 index 00000000000..ce27c2c15ca --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/AzureFunctionsWorkerV1Test.java @@ -0,0 +1,12 @@ +import static datadog.trace.api.config.TracerConfig.TRACE_SPAN_ATTRIBUTE_SCHEMA; + +import datadog.trace.test.junit.utils.config.WithConfig; + +@WithConfig(key = TRACE_SPAN_ATTRIBUTE_SCHEMA, value = "v1") +class AzureFunctionsWorkerV1Test extends AzureFunctionsWorkerTest { + + @Override + String operation() { + return "azure.functions.invoke"; + } +} diff --git a/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java new file mode 100644 index 00000000000..5b8b9822d87 --- /dev/null +++ b/dd-java-agent/instrumentation/azure-functions-worker-2.7/src/test/java/com/microsoft/azure/functions/worker/chain/FunctionExecutionMiddleware.java @@ -0,0 +1,12 @@ +package com.microsoft.azure.functions.worker.chain; + +import com.microsoft.azure.functions.internal.spi.middleware.Middleware; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareChain; +import com.microsoft.azure.functions.internal.spi.middleware.MiddlewareContext; + +public class FunctionExecutionMiddleware implements Middleware { + @Override + public void invoke(MiddlewareContext context, MiddlewareChain chain) throws Exception { + chain.doNext(context); + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 468d89c5cb4..67fbb3ad7a0 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -323,6 +323,7 @@ include( ":dd-java-agent:instrumentation:axis2-1.3", ":dd-java-agent:instrumentation:axway-api-7.5", ":dd-java-agent:instrumentation:azure-functions-1.2.2", + ":dd-java-agent:instrumentation:azure-functions-worker-2.7", ":dd-java-agent:instrumentation:beanshell-2.0", ":dd-java-agent:instrumentation:caffeine-1.0", ":dd-java-agent:instrumentation:cdi-1.2",