diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java b/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java index 4542e816c92..710418789ba 100644 --- a/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/main/java/datadog/trace/instrumentation/karate/KarateTracingHook.java @@ -115,7 +115,9 @@ public boolean beforeScenario(ScenarioRuntime sr) { if (Config.get().isCiVisibilityTestSkippingEnabled() || Config.get().isCiVisibilityTestManagementEnabled()) { TestIdentifier skippableTest = KarateUtils.toTestIdentifier(scenario); - SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(skippableTest); + SkipReason skipReason = + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason( + skippableTest, TestSourceData.UNKNOWN, categories); if (skipReason != null && !(skipReason == SkipReason.ITR diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy index 4cafd545c09..3ffb1d4a80c 100644 --- a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/groovy/KarateTest.groovy @@ -27,6 +27,33 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") class KarateTest extends CiVisibilityInstrumentationTest { + def "test automation exclusions preserve a blocking failure and execute once"() { + given: + def id = new TestFQN("[org/example/test_automation_excluded] automation exclusions", "protected_failure") + givenDisabledTests([id]) + givenQuarantinedTests([id]) + givenAttemptToFixTests([id]) + givenFlakyRetryEnabled(true) + givenFlakyTests([id]) + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests([]) + + when: + runTests([TestAutomationExcludedKarate], false) + + then: + TEST_WRITER.toList().flatten().count { it.getTag("test.name") == "protected_failure" } == 1 + def testSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "protected_failure" } + testSpan != null + testSpan.getTag("test.test_management.no_auto_retry") == true + testSpan.getTag("test.test_management.prevent_failure_suppression") == true + testSpan.getTag("test.test_management.is_test_disabled") == true + testSpan.getTag("test.test_management.is_quarantined") == true + testSpan.getTag("test.test_management.is_attempt_to_fix") == true + testSpan.getTag("test.test_management.attempt_to_fix_passed") == null + } + + def "test #testcaseName"() { Assumptions.assumeTrue(assumption) diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestAutomationExcludedKarate.java b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestAutomationExcludedKarate.java new file mode 100644 index 00000000000..574462995be --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/TestAutomationExcludedKarate.java @@ -0,0 +1,11 @@ +package org.example; + +import com.intuit.karate.junit5.Karate; + +public class TestAutomationExcludedKarate { + + @Karate.Test + public Karate testFailed() { + return Karate.run("classpath:org/example/test_automation_excluded.feature"); + } +} diff --git a/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_automation_excluded.feature b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_automation_excluded.feature new file mode 100644 index 00000000000..77dc5f12a0d --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-1.0/src/test/java/org/example/test_automation_excluded.feature @@ -0,0 +1,5 @@ +Feature: automation exclusions + + @datadog_no_auto_retry @datadog_prevent_failure_suppression + Scenario: protected_failure + * assert false diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java b/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java index 84a5adbd524..ead16307d1f 100644 --- a/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java @@ -124,7 +124,9 @@ private boolean beforeScenario(ScenarioRunEvent event) { if (Config.get().isCiVisibilityTestSkippingEnabled() || Config.get().isCiVisibilityTestManagementEnabled()) { TestIdentifier skippableTest = KarateUtils.toTestIdentifier(scenario); - SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(skippableTest); + SkipReason skipReason = + TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason( + skippableTest, TestSourceData.UNKNOWN, categories); if (skipReason != null && !(skipReason == SkipReason.ITR diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy index a36885edbf4..c3a0e9b3876 100644 --- a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/groovy/KarateV2Test.groovy @@ -21,6 +21,33 @@ import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass @DisableTestTrace(reason = "avoid self-tracing") class KarateV2Test extends CiVisibilityInstrumentationTest { + def "test automation exclusions preserve a blocking failure and execute once"() { + given: + def id = new TestFQN("[org/example/test_automation_excluded] automation exclusions", "protected_failure") + givenDisabledTests([id]) + givenQuarantinedTests([id]) + givenAttemptToFixTests([id]) + givenFlakyRetryEnabled(true) + givenFlakyTests([id]) + givenEarlyFlakinessDetectionEnabled(true) + givenKnownTests([]) + + when: + runTests([TestAutomationExcludedKarate], false) + + then: + TEST_WRITER.toList().flatten().count { it.getTag("test.name") == "protected_failure" } == 1 + def testSpan = TEST_WRITER.toList().flatten().find { it.getTag("test.name") == "protected_failure" } + testSpan != null + testSpan.getTag("test.test_management.no_auto_retry") == true + testSpan.getTag("test.test_management.prevent_failure_suppression") == true + testSpan.getTag("test.test_management.is_test_disabled") == true + testSpan.getTag("test.test_management.is_quarantined") == true + testSpan.getTag("test.test_management.is_attempt_to_fix") == true + testSpan.getTag("test.test_management.attempt_to_fix_passed") == null + } + + def "test #testcaseName"() { runTests(tests, success) diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestAutomationExcludedKarate.java b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestAutomationExcludedKarate.java new file mode 100644 index 00000000000..5a9e8ee0a1d --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/TestAutomationExcludedKarate.java @@ -0,0 +1,17 @@ +package org.example; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.intuit.karate.Results; +import com.intuit.karate.Runner; +import org.junit.jupiter.api.Test; + +public class TestAutomationExcludedKarate { + + @Test + public void test() { + Results results = + Runner.path("classpath:org/example/test_automation_excluded.feature").parallel(1); + assertEquals(0, results.getFailCount(), results.getErrorMessages()); + } +} diff --git a/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_automation_excluded.feature b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_automation_excluded.feature new file mode 100644 index 00000000000..77dc5f12a0d --- /dev/null +++ b/dd-java-agent/instrumentation/karate/karate-2.0/src/test/java/org/example/test_automation_excluded.feature @@ -0,0 +1,5 @@ +Feature: automation exclusions + + @datadog_no_auto_retry @datadog_prevent_failure_suppression + Scenario: protected_failure + * assert false diff --git a/docs/test-automation-exclusions.md b/docs/test-automation-exclusions.md index cb2294435df..33fb0e8474a 100644 --- a/docs/test-automation-exclusions.md +++ b/docs/test-automation-exclusions.md @@ -54,3 +54,10 @@ For either the JUnit4 or JUnit Platform Cucumber runner, tag a Scenario or Scenario Outline with `@datadog_no_auto_retry` and/or `@datadog_prevent_failure_suppression`. These names correspond to the public `TestAutomationExclusions` constants. Tags apply to every outline example. + +## Karate + +Tag the Scenario or Scenario Outline with `@datadog_no_auto_retry` and/or +`@datadog_prevent_failure_suppression` (the names from the public +`TestAutomationExclusions` constants). Both Karate 1.x and 2.x resolve these +scenario tags before Datadog skipping and retry policy selection.