From ed9ca00c921499075e8fa446cf85a1807752f198 Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Tue, 20 Oct 2026 20:21:13 +0200 Subject: [PATCH 1/2] test: Remove the io.sentry.systemtest plugin and sentry-system-test-support With the runner gone, nothing applies the io.sentry.systemtest convention plugin or depends on sentry-system-test-support, which held the HTTP and GraphQL clients and the TestHelper the system tests used. The envelope matchers it delegated to stay in sentry-test-support. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../kotlin/io.sentry.systemtest.gradle.kts | 38 --- .../io/sentry/gradle/SystemTestExtension.kt | 17 - build.gradle.kts | 5 +- sentry-system-test-support/build.gradle.kts | 44 --- .../src/main/graphql/greeting.graphql | 3 - .../src/main/graphql/project.graphql | 11 - .../src/main/graphql/schema.graphqls | 70 ---- .../src/main/graphql/task.graphql | 16 - .../io/sentry/systemtest/ResponseTypes.kt | 7 - .../systemtest/graphql/GraphqlTestClient.kt | 38 --- .../util/LoggingInsecureRestClient.kt | 83 ----- .../sentry/systemtest/util/RestTestClient.kt | 115 ------- .../systemtest/util/SentryMockServerClient.kt | 35 -- .../io/sentry/systemtest/util/TestHelper.kt | 298 ------------------ settings.gradle.kts | 1 - 15 files changed, 2 insertions(+), 779 deletions(-) delete mode 100644 build-logic/src/main/kotlin/io.sentry.systemtest.gradle.kts delete mode 100644 build-logic/src/main/kotlin/io/sentry/gradle/SystemTestExtension.kt delete mode 100644 sentry-system-test-support/build.gradle.kts delete mode 100644 sentry-system-test-support/src/main/graphql/greeting.graphql delete mode 100644 sentry-system-test-support/src/main/graphql/project.graphql delete mode 100644 sentry-system-test-support/src/main/graphql/schema.graphqls delete mode 100644 sentry-system-test-support/src/main/graphql/task.graphql delete mode 100644 sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/ResponseTypes.kt delete mode 100644 sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/graphql/GraphqlTestClient.kt delete mode 100644 sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/LoggingInsecureRestClient.kt delete mode 100644 sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/RestTestClient.kt delete mode 100644 sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/SentryMockServerClient.kt delete mode 100644 sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/TestHelper.kt diff --git a/build-logic/src/main/kotlin/io.sentry.systemtest.gradle.kts b/build-logic/src/main/kotlin/io.sentry.systemtest.gradle.kts deleted file mode 100644 index a21079e1336..00000000000 --- a/build-logic/src/main/kotlin/io.sentry.systemtest.gradle.kts +++ /dev/null @@ -1,38 +0,0 @@ -import io.sentry.gradle.SystemTestExtension -import org.gradle.api.tasks.ClasspathNormalizer - -val systemTest = extensions.create("sentrySystemTest") - -// The sample system tests launch the packaged app (war/shadowJar/bootJar) from build/libs as a -// separate process, so the archive is a real input even though it is not on the test classpath. -// Agent-based samples are additionally launched with -javaagent:, another runtime -// input not on the classpath. See test/system-test-runner.py. -tasks.matching { it.name == "systemTest" }.configureEach { - val archiveTask = - listOf("war", "shadowJar", "bootJar").firstOrNull { it in tasks.names } - ?: throw GradleException( - "io.sentry.systemtest is applied to $path but none of war/shadowJar/bootJar " + - "exist to provide the launched app archive for the systemTest task" - ) - // Declaring the archive as an input also wires the dependency on its producing task. - inputs - .files(tasks.named(archiveTask)) - .withPropertyName("appArchive") - .withNormalizer(ClasspathNormalizer::class.java) - - if (systemTest.usesOpenTelemetryAgent.get()) { - // The runner builds the agent and launches the app with -javaagent before invoking this task, - // so the agent jar is tracked for content only (by path, no cross-project task dependency): a - // change to it makes systemTest out of date even though it runs outside the test JVM. - val version = providers.gradleProperty("versionName").get() - inputs - .files( - rootProject.layout.projectDirectory.file( - "sentry-opentelemetry/sentry-opentelemetry-agent/build/libs/" + - "sentry-opentelemetry-agent-$version.jar" - ) - ) - .withPropertyName("openTelemetryAgent") - .withNormalizer(ClasspathNormalizer::class.java) - } -} diff --git a/build-logic/src/main/kotlin/io/sentry/gradle/SystemTestExtension.kt b/build-logic/src/main/kotlin/io/sentry/gradle/SystemTestExtension.kt deleted file mode 100644 index 9111ce17b1f..00000000000 --- a/build-logic/src/main/kotlin/io/sentry/gradle/SystemTestExtension.kt +++ /dev/null @@ -1,17 +0,0 @@ -package io.sentry.gradle - -import org.gradle.api.provider.Property - -/** Configuration for the `io.sentry.systemtest` convention plugin. */ -abstract class SystemTestExtension { - /** - * Set to `true` for samples that the system-test runner launches with the Sentry OpenTelemetry - * Java agent (`-javaagent`). The agent jar is then tracked as a `systemTest` input so the task - * re-runs when the agent changes, even though it is started outside the test JVM. - */ - abstract val usesOpenTelemetryAgent: Property - - init { - usesOpenTelemetryAgent.convention(false) - } -} diff --git a/build.gradle.kts b/build.gradle.kts index 898b429cb5d..3aceb123fba 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -87,8 +87,7 @@ apiValidation { "test-app-size", "sentry-samples-netflix-dgs", "sentry-samples-console-otlp", - "sentry-test-support", - "sentry-system-test-support" + "sentry-test-support" ) ) } @@ -126,7 +125,7 @@ subprojects { } } - if (!this.name.contains("sample") && !this.name.contains("integration-tests") && this.name != "sentry-system-test-support" && this.name != "sentry-test-support") { + if (!this.name.contains("sample") && !this.name.contains("integration-tests") && this.name != "sentry-test-support") { apply() apply() diff --git a/sentry-system-test-support/build.gradle.kts b/sentry-system-test-support/build.gradle.kts deleted file mode 100644 index ecfdc6011f8..00000000000 --- a/sentry-system-test-support/build.gradle.kts +++ /dev/null @@ -1,44 +0,0 @@ -plugins { - `java-library` - id("io.sentry.javadoc") - alias(libs.plugins.kotlin.jvm) - alias(libs.plugins.errorprone) - alias(libs.plugins.gradle.versions) - id("com.apollographql.apollo") version "4.1.1" -} - -tasks.withType().configureEach { - compilerOptions.languageVersion = org.jetbrains.kotlin.gradle.dsl.KotlinVersion.KOTLIN_1_9 - compilerOptions.apiVersion = org.jetbrains.kotlin.gradle.dsl.KotlinVersion.KOTLIN_1_9 -} - -dependencies { - api(projects.sentry) - api(projects.sentryTestSupport) - api(libs.apollo4.kotlin) - - compileOnly(libs.jetbrains.annotations) - compileOnly(libs.nopen.annotations) - - compileOnly(libs.springboot.starter.web) - - implementation(libs.jackson.databind) - implementation(libs.jackson.kotlin) - implementation(libs.okhttp) - - errorprone(libs.errorprone.core) - errorprone(libs.nopen.checker) - - // tests - implementation(kotlin(Config.kotlinStdLib)) - implementation(libs.kotlin.test.junit) - implementation(libs.mockito.kotlin) -} - -apollo { - service("service") { - srcDir("src/main/graphql") - packageName.set("io.sentry.samples.graphql") - outputDirConnection { connectToKotlinSourceSet("main") } - } -} diff --git a/sentry-system-test-support/src/main/graphql/greeting.graphql b/sentry-system-test-support/src/main/graphql/greeting.graphql deleted file mode 100644 index 06c866a65fa..00000000000 --- a/sentry-system-test-support/src/main/graphql/greeting.graphql +++ /dev/null @@ -1,3 +0,0 @@ -query GreetingQuery($name: String!) { - greeting(name: $name) -} diff --git a/sentry-system-test-support/src/main/graphql/project.graphql b/sentry-system-test-support/src/main/graphql/project.graphql deleted file mode 100644 index bff62ed2c2c..00000000000 --- a/sentry-system-test-support/src/main/graphql/project.graphql +++ /dev/null @@ -1,11 +0,0 @@ -query ProjectQuery($slug: ID!) { - project(slug: $slug) { - slug - name - status - } -} - -mutation AddProjectMutation($slug: ID!) { - addProject(slug: $slug) -} diff --git a/sentry-system-test-support/src/main/graphql/schema.graphqls b/sentry-system-test-support/src/main/graphql/schema.graphqls deleted file mode 100644 index d76aca4756a..00000000000 --- a/sentry-system-test-support/src/main/graphql/schema.graphqls +++ /dev/null @@ -1,70 +0,0 @@ -type Query { - greeting(name: String! = "Spring"): String! - project(slug: ID!): Project - tasks(projectSlug: ID!): [Task] -} - -type Mutation { - addProject(slug: ID!): String! -} - -type Subscription { - notifyNewTask(projectSlug: ID!): Task -} - -""" A Project in the Spring portfolio """ -type Project { - """ Unique string id used in URLs """ - slug: ID! - """ Project name """ - name: String - """ URL of the git repository """ - repositoryUrl: String! - """ Current support status """ - status: ProjectStatus! -} - -""" A task """ -type Task { - """ ID """ - id: String! - """ Name """ - name: String! - """ ID of the Assignee """ - assigneeId: String - """ Assignee """ - assignee: Assignee - """ ID of the Creator """ - creatorId: String - """ Creator """ - creator: Creator -} - -""" An Assignee """ -type Assignee { - """ ID """ - id: String! - """ Name """ - name: String! -} - -""" An Creator """ -type Creator { - """ ID """ - id: String! - """ Name """ - name: String! -} - -enum ProjectStatus { - """ Actively supported by the Spring team """ - ACTIVE - """ Supported by the community """ - COMMUNITY - """ Prototype, not officially supported yet """ - INCUBATING - """ Project being retired, in maintenance mode """ - ATTIC - """ End-Of-Lifed """ - EOL -} diff --git a/sentry-system-test-support/src/main/graphql/task.graphql b/sentry-system-test-support/src/main/graphql/task.graphql deleted file mode 100644 index 11ae18574d3..00000000000 --- a/sentry-system-test-support/src/main/graphql/task.graphql +++ /dev/null @@ -1,16 +0,0 @@ -query TasksAndAssigneesQuery($slug: ID!) { - tasks(projectSlug: $slug) { - id - name - assigneeId - assignee { - id - name - } - creatorId - creator { - id - name - } - } -} diff --git a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/ResponseTypes.kt b/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/ResponseTypes.kt deleted file mode 100644 index 4c9a4d903ad..00000000000 --- a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/ResponseTypes.kt +++ /dev/null @@ -1,7 +0,0 @@ -package io.sentry.systemtest - -data class Todo(val id: Long, val title: String, val completed: Boolean) - -data class Person(val firstName: String, val lastName: String) { - override fun toString(): String = "Person{firstName='$firstName', lastName='$lastName'}" -} diff --git a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/graphql/GraphqlTestClient.kt b/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/graphql/GraphqlTestClient.kt deleted file mode 100644 index c1b0a409bda..00000000000 --- a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/graphql/GraphqlTestClient.kt +++ /dev/null @@ -1,38 +0,0 @@ -package io.sentry.systemtest.graphql - -import com.apollographql.apollo.ApolloClient -import com.apollographql.apollo.api.ApolloResponse -import com.apollographql.apollo.api.Mutation -import com.apollographql.apollo.api.Query -import io.sentry.samples.graphql.AddProjectMutation -import io.sentry.samples.graphql.GreetingQuery -import io.sentry.samples.graphql.ProjectQuery -import io.sentry.samples.graphql.TasksAndAssigneesQuery -import kotlinx.coroutines.runBlocking - -class GraphqlTestClient(backendUrl: String) { - val apollo = - ApolloClient.Builder() - .serverUrl("$backendUrl/graphql") - .addHttpHeader("Authorization", "Basic dXNlcjpwYXNzd29yZA==") - .build() - - fun greet(name: String): ApolloResponse? = executeQuery(GreetingQuery(name)) - - fun project(slug: String): ApolloResponse? = executeQuery(ProjectQuery(slug)) - - fun tasksAndAssignees(slug: String): ApolloResponse? = - executeQuery(TasksAndAssigneesQuery(slug)) - - fun addProject(slug: String): ApolloResponse? = - executeMutation(AddProjectMutation(slug)) - - private fun executeQuery(query: Query): ApolloResponse? = runBlocking { - apollo.query(query).execute() - } - - private fun executeMutation(mutation: Mutation): ApolloResponse? = - runBlocking { - apollo.mutation(mutation).execute() - } -} diff --git a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/LoggingInsecureRestClient.kt b/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/LoggingInsecureRestClient.kt deleted file mode 100644 index 8c4f27345f8..00000000000 --- a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/LoggingInsecureRestClient.kt +++ /dev/null @@ -1,83 +0,0 @@ -package io.sentry.systemtest.util - -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import java.util.concurrent.TimeUnit -import okhttp3.Credentials -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.RequestBody -import okhttp3.RequestBody.Companion.toRequestBody -import okhttp3.Response -import org.slf4j.LoggerFactory - -open class LoggingInsecureRestClient { - val logger = LoggerFactory.getLogger(LoggingInsecureRestClient::class.java) - var lastKnownStatusCode: Int? = null - - protected inline fun callTyped( - requestBuilder: Request.Builder, - useAuth: Boolean, - extraHeaders: Map? = null, - ): T? { - val response = call(requestBuilder, useAuth, extraHeaders) - val responseBody = response?.body?.string() - if (response?.isSuccessful != true) { - return null - } - if (T::class == String::class) { - return responseBody as? T - } - return responseBody?.let { objectMapper().readValue(it, T::class.java) } - } - - protected fun call( - requestBuilder: Request.Builder, - useAuth: Boolean, - extraHeaders: Map? = null, - ): Response? { - try { - val request = - requestBuilder - .also { originalRequest -> - var modifiedRequest = originalRequest - - if (useAuth) { - modifiedRequest = - modifiedRequest.header("Authorization", Credentials.basic("user", "password")) - } - - extraHeaders?.forEach { key, value -> - modifiedRequest = modifiedRequest.header(key, value) - } - - modifiedRequest - } - .build() - val call = client().newCall(request) - val response = call.execute() - lastKnownStatusCode = response.code - return response - } catch (e: Exception) { - lastKnownStatusCode = null - logger.error("Request failed", e) - return null - } - } - - protected fun client(): OkHttpClient = - OkHttpClient.Builder() - .callTimeout(60, TimeUnit.SECONDS) - .connectTimeout(60, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS) - .writeTimeout(60, TimeUnit.SECONDS) - .build() - - protected fun objectMapper(): ObjectMapper = jacksonObjectMapper() - - protected fun toRequestBody(o: Any?): RequestBody { - val stringValue = objectMapper().writeValueAsString(o) - return stringValue.toRequestBody("application/json; charset=utf-8".toMediaType()) - } -} diff --git a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/RestTestClient.kt b/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/RestTestClient.kt deleted file mode 100644 index ac2c69f5ed2..00000000000 --- a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/RestTestClient.kt +++ /dev/null @@ -1,115 +0,0 @@ -package io.sentry.systemtest.util - -import io.sentry.systemtest.Person -import io.sentry.systemtest.Todo -import okhttp3.Request - -class RestTestClient(private val backendBaseUrl: String) : LoggingInsecureRestClient() { - fun getPerson(id: Long): Person? { - val request = Request.Builder().url("$backendBaseUrl/person/$id") - - return callTyped(request, true) - } - - fun createPerson(person: Person, extraHeaders: Map? = null): Person? { - val request = Request.Builder().url("$backendBaseUrl/person/").post(toRequestBody(person)) - - return callTyped(request, true, extraHeaders) - } - - fun getPersonDistributedTracing(id: Long, extraHeaders: Map? = null): Person? { - val request = Request.Builder().url("$backendBaseUrl/tracing/$id") - - return callTyped(request, true, extraHeaders) - } - - fun createPersonDistributedTracing( - person: Person, - extraHeaders: Map? = null, - ): Person? { - val request = Request.Builder().url("$backendBaseUrl/tracing/").post(toRequestBody(person)) - - return callTyped(request, true, extraHeaders) - } - - fun getTodo(id: Long): Todo? { - val request = Request.Builder().url("$backendBaseUrl/todo/$id") - - return callTyped(request, true) - } - - fun getTodoWebclient(id: Long): Todo? { - val request = Request.Builder().url("$backendBaseUrl/todo-webclient/$id") - - return callTyped(request, true) - } - - fun getTodoRestClient(id: Long): Todo? { - val request = Request.Builder().url("$backendBaseUrl/todo-restclient/$id") - - return callTyped(request, true) - } - - fun getTodoOkHttp(id: Long): Todo? { - val request = Request.Builder().url("$backendBaseUrl/todo-okhttp/$id") - - return callTyped(request, true) - } - - fun getCachedTodo(id: Long): Todo? { - val request = Request.Builder().url("$backendBaseUrl/cache/$id") - - return callTyped(request, true) - } - - fun saveCachedTodo(todo: Todo): Todo? { - val request = Request.Builder().url("$backendBaseUrl/cache/").post(toRequestBody(todo)) - - return callTyped(request, true) - } - - fun deleteCachedTodo(id: Long) { - val request = Request.Builder().url("$backendBaseUrl/cache/$id").delete() - - call(request, true) - } - - fun checkFeatureFlag(flagKey: String): FeatureFlagResponse? { - val request = Request.Builder().url("$backendBaseUrl/feature-flag/check/$flagKey") - - return callTyped(request, true) - } - - fun errorWithFeatureFlag(flagKey: String): String? { - val request = Request.Builder().url("$backendBaseUrl/feature-flag/error/$flagKey") - - val response = call(request, true) - return response?.body?.string() - } - - fun produceKafkaMessage(message: String = "hello from sentry!"): String? { - val request = Request.Builder().url("$backendBaseUrl/kafka/produce?message=$message") - - return callTyped(request, true) - } - - fun getCountMetric(): String? { - val request = Request.Builder().url("$backendBaseUrl/metric/count") - - return callTyped(request, true) - } - - fun getGaugeMetric(value: Long): String? { - val request = Request.Builder().url("$backendBaseUrl/metric/gauge/$value") - - return callTyped(request, true) - } - - fun getDistributionMetric(value: Long): String? { - val request = Request.Builder().url("$backendBaseUrl/metric/distribution/$value") - - return callTyped(request, true) - } -} - -data class FeatureFlagResponse(val flagKey: String, val value: Boolean) diff --git a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/SentryMockServerClient.kt b/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/SentryMockServerClient.kt deleted file mode 100644 index 389f3bb5b80..00000000000 --- a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/SentryMockServerClient.kt +++ /dev/null @@ -1,35 +0,0 @@ -package io.sentry.systemtest.util - -import okhttp3.Request - -class SentryMockServerClient(private val baseUrl: String) : LoggingInsecureRestClient() { - fun getEnvelopeCount(): EnvelopeCounts { - val request = Request.Builder().url("$baseUrl/envelope-count") - - return callTyped(request, false)!! - } - - fun reset() { - val request = Request.Builder().url("$baseUrl/reset") - - call(request, false) - } - - fun getEnvelopes(): EnvelopesReceived { - val request = Request.Builder().url("$baseUrl/envelopes-received") - - return callTyped(request, false)!! - } -} - -class EnvelopeCounts { - val envelopes: Long? = null - - override fun toString(): String = "EnvelopeCounts{envelopes=$envelopes}" -} - -class EnvelopesReceived { - val envelopes: List? = null - - override fun toString(): String = "EnvelopesReceived{envelopes=$envelopes}" -} diff --git a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/TestHelper.kt b/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/TestHelper.kt deleted file mode 100644 index 8ef74cae3c9..00000000000 --- a/sentry-system-test-support/src/main/kotlin/io/sentry/systemtest/util/TestHelper.kt +++ /dev/null @@ -1,298 +0,0 @@ -package io.sentry.systemtest.util - -import com.apollographql.apollo.api.ApolloResponse -import com.apollographql.apollo.api.Operation -import io.sentry.JsonSerializer -import io.sentry.ProfileChunk -import io.sentry.SentryEnvelope -import io.sentry.SentryEnvelopeHeader -import io.sentry.SentryEvent -import io.sentry.SentryLogEvents -import io.sentry.SentryMetricsEvents -import io.sentry.SentryOptions -import io.sentry.protocol.FeatureFlag -import io.sentry.protocol.SentrySpan -import io.sentry.protocol.SentryTransaction -import io.sentry.systemtest.graphql.GraphqlTestClient -import io.sentry.test.event -import io.sentry.test.hasExceptionMessage -import io.sentry.test.hasFlag -import io.sentry.test.hasLogWithAttribute -import io.sentry.test.hasLogWithBody -import io.sentry.test.hasMetric -import io.sentry.test.hasMetricWithAttribute -import io.sentry.test.hasOp -import io.sentry.test.hasSpanWithOp -import io.sentry.test.hasSpanWithOpAndDescription -import io.sentry.test.hasTraceId -import io.sentry.test.logs -import io.sentry.test.metrics -import io.sentry.test.profileChunk -import io.sentry.test.transaction -import java.io.File -import java.io.PrintWriter -import kotlin.test.assertEquals -import kotlin.test.assertFalse -import kotlin.test.assertNotNull -import kotlin.test.assertTrue - -class TestHelper(backendUrl: String) { - val restClient: RestTestClient - val graphqlClient: GraphqlTestClient - val sentryClient: SentryMockServerClient - val jsonSerializer: JsonSerializer - val dsn = "http://502f25099c204a2fbf4cb16edc5975d1@localhost:8000/0" - - var envelopeCounts: EnvelopeCounts? = null - - init { - restClient = RestTestClient(backendUrl) - sentryClient = SentryMockServerClient("http://localhost:8000") - graphqlClient = GraphqlTestClient(backendUrl) - jsonSerializer = JsonSerializer(SentryOptions.empty()) - } - - fun snapshotEnvelopeCount() { - envelopeCounts = sentryClient.getEnvelopeCount() - } - - fun ensureEnvelopeCountIncreased() { - Thread.sleep(1000) - val envelopeCountsAfter = sentryClient.getEnvelopeCount() - assertTrue(envelopeCountsAfter!!.envelopes!! > envelopeCounts!!.envelopes!!) - } - - fun ensureEnvelopeReceived(retryCount: Int = 1, callback: ((String) -> Boolean)) { - val envelopes = sentryClient.getEnvelopes() - assertNotNull(envelopes.envelopes) - envelopes.envelopes.forEach { envelopeString -> - val didMatch = callback(envelopeString) - if (didMatch) { - return - } - } - if (retryCount <= 0) { - throw RuntimeException("Unable to find matching envelope received by relay") - } else { - Thread.sleep(10000) - ensureEnvelopeReceived(retryCount - 1, callback) - } - } - - fun ensureNoEnvelopeReceived(callback: ((String) -> Boolean)) { - Thread.sleep(10000) - val envelopes = sentryClient.getEnvelopes() - - if (envelopes.envelopes.isNullOrEmpty()) { - return - } - - envelopes.envelopes.forEach { envelopeString -> - val didMatch = callback(envelopeString) - if (didMatch) { - throw RuntimeException("Found unexpected matching envelope received by relay") - } - } - } - - fun ensureTransactionReceived(callback: ((SentryTransaction, SentryEnvelopeHeader) -> Boolean)) { - ensureEnvelopeReceived { envelopeString -> checkIfTransactionMatches(envelopeString, callback) } - } - - fun ensureProfileChunkReceived(callback: ((ProfileChunk, SentryEnvelopeHeader) -> Boolean)) { - ensureEnvelopeReceived { envelopeString -> checkIfProfileMatches(envelopeString, callback) } - } - - fun ensureNoTransactionReceived( - callback: ((SentryTransaction, SentryEnvelopeHeader) -> Boolean) - ) { - ensureNoEnvelopeReceived { envelopeString -> - checkIfTransactionMatches(envelopeString, callback) - } - } - - fun ensureLogsReceived(callback: ((SentryLogEvents, SentryEnvelopeHeader) -> Boolean)) { - ensureEnvelopeReceived { envelopeString -> checkIfLogsMatch(envelopeString, callback) } - } - - private fun checkIfLogsMatch( - envelopeString: String, - callback: ((SentryLogEvents, SentryEnvelopeHeader) -> Boolean), - ): Boolean { - val envelope = deserialize(envelopeString) ?: return false - val logs = envelope.logs() ?: return false - return callback(logs, envelope.header) - } - - fun doesContainMetric( - metrics: SentryMetricsEvents, - name: String, - type: String, - value: Double, - unit: String? = null, - ): Boolean = metrics.hasMetric(name, type, value, unit) - - fun ensureMetricsReceived(callback: ((SentryMetricsEvents, SentryEnvelopeHeader) -> Boolean)) { - ensureEnvelopeReceived { envelopeString -> checkIfMetricsMatch(envelopeString, callback) } - } - - private fun checkIfMetricsMatch( - envelopeString: String, - callback: ((SentryMetricsEvents, SentryEnvelopeHeader) -> Boolean), - ): Boolean { - val envelope = deserialize(envelopeString) ?: return false - val metrics = envelope.metrics() ?: return false - return callback(metrics, envelope.header) - } - - fun doesContainLogWithBody(logs: SentryLogEvents, body: String): Boolean = - logs.hasLogWithBody(body) - - fun doesLogWithBodyHaveAttribute( - logs: SentryLogEvents, - body: String, - attributeKey: String, - attributeValue: Any?, - ): Boolean = logs.hasLogWithAttribute(body, attributeKey, attributeValue) - - fun doesMetricHaveAttribute( - metrics: SentryMetricsEvents, - metricName: String, - attributeKey: String, - attributeValue: Any?, - ): Boolean = metrics.hasMetricWithAttribute(metricName, attributeKey, attributeValue) - - private fun checkIfTransactionMatches( - envelopeString: String, - callback: ((SentryTransaction, SentryEnvelopeHeader) -> Boolean), - ): Boolean { - val envelope = deserialize(envelopeString) ?: return false - val transaction = envelope.transaction() ?: return false - return callback(transaction, envelope.header) - } - - private fun checkIfProfileMatches( - envelopeString: String, - callback: ((ProfileChunk, SentryEnvelopeHeader) -> Boolean), - ): Boolean { - val envelope = deserialize(envelopeString) ?: return false - val chunk = envelope.profileChunk() ?: return false - return callback(chunk, envelope.header) - } - - fun ensureErrorReceived(callback: ((SentryEvent) -> Boolean)) { - ensureEnvelopeReceived(retryCount = 3) { envelopeString -> - val error = deserialize(envelopeString)?.event() ?: return@ensureEnvelopeReceived false - val callbackResult = callback(error) - if (!callbackResult) { - println("found an error event but it did not match:") - logObject(error) - } - callbackResult - } - } - - fun ensureTransactionWithSpanReceived(callback: ((SentrySpan) -> Boolean)) { - ensureTransactionReceived { transaction, envelopeHeader -> - transaction.spans.forEach { span -> - val callbackResult = callback(span) - if (callbackResult) { - return@ensureTransactionReceived true - } - } - false - } - } - - fun reset() { - sentryClient.reset() - } - - fun logObject(obj: Any?) { - obj ?: return - // Flush rather than close: closing the writer would close System.out. - PrintWriter(System.out).apply { - jsonSerializer.serialize(obj, this) - flush() - } - println() - } - - private fun deserialize(envelopeString: String): SentryEnvelope? = - jsonSerializer.deserializeEnvelope(envelopeString.byteInputStream()) - - fun ensureNoErrors(response: ApolloResponse?) { - response ?: throw RuntimeException("no response") - assertFalse(response.hasErrors()) - } - - fun ensureErrorCount(response: ApolloResponse?, errorCount: Int) { - response ?: throw RuntimeException("no response") - assertEquals(errorCount, response.errors?.size) - } - - fun doesTransactionContainSpanWithOp(transaction: SentryTransaction, op: String): Boolean = - transaction.hasSpanWithOp(op) - - fun doesTransactionContainSpanWithOpAndDescription( - transaction: SentryTransaction, - op: String, - description: String, - ): Boolean = transaction.hasSpanWithOpAndDescription(op, description) - - fun doesTransactionHaveTraceId(transaction: SentryTransaction, traceId: String): Boolean = - transaction.hasTraceId(traceId) - - fun doesTransactionHaveOp(transaction: SentryTransaction, op: String): Boolean = - transaction.hasOp(op) - - fun doesTransactionHave( - transaction: SentryTransaction, - op: String, - featureFlag: FeatureFlag? = null, - ): Boolean = transaction.hasOp(op, featureFlag) - - fun doesTransactionHaveSpanWith( - transaction: SentryTransaction, - op: String, - featureFlag: FeatureFlag? = null, - noFeatureFlags: Boolean = false, - ): Boolean = transaction.hasSpanWithOp(op, featureFlag, noFeatureFlags) - - fun doesEventHaveExceptionMessage(event: SentryEvent, expectedMessage: String): Boolean = - event.hasExceptionMessage(expectedMessage) - - fun doesEventHaveFlag(event: SentryEvent, flag: String, result: Boolean): Boolean = - event.hasFlag(flag, result) - - fun findJar(prefix: String, inDir: String = "build/libs"): File { - val buildDir = File(inDir) - val jarFiles = - buildDir.listFiles { _, name -> name.startsWith(prefix) && name.endsWith(".jar") }?.toList() - ?: emptyList() - - if (jarFiles.isEmpty()) { - throw AssertionError("No JAR found in ${buildDir.absolutePath}") - } - - return jarFiles.maxOf { it } - } - - fun launch(jar: File, env: Map, enableOtelAutoConfig: Boolean = false): Process { - val processBuilderList = mutableListOf("java", "--add-opens", "java.base/java.lang=ALL-UNNAMED") - - if (enableOtelAutoConfig) { - processBuilderList.add("-Dotel.java.global-autoconfigure.enabled=true") - } - - processBuilderList.add("-jar") - processBuilderList.add(jar.absolutePath) - - val processBuilder = - ProcessBuilder(processBuilderList).inheritIO() // forward i/o to current process - - processBuilder.environment().putAll(env) - - return processBuilder.start() - } -} diff --git a/settings.gradle.kts b/settings.gradle.kts index 413ab5fd59b..5947411dd56 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -59,7 +59,6 @@ include( "sentry-apollo-3", "sentry-apollo-4", "sentry-apollo-5", - "sentry-system-test-support", "sentry-test-support", "sentry-log4j2", "sentry-logback", From ff26ba9f865bb895fbbdc921d27d31fd98e4bf0c Mon Sep 17 00:00:00 2001 From: Nelson Osacky Date: Tue, 27 Oct 2026 09:05:31 +0100 Subject: [PATCH 2/2] docs: [Sample Tests 40] Describe the in-module end-to-end tests instead of the system tests (#6332) * docs: Describe the in-module end-to-end tests instead of the system tests The e2e_tests rule now covers MockSentryServer, where each kind of end-to-end test lives, the Spring suites per flavor, the OpenTelemetry agent fixture, how to run them and the gotchas the migration found. The test skill runs integration suites by their source set instead of the Python runner, the new_module rule no longer asks for sample tests, and AGENTS.md drops sentry-system-test-support. Co-Authored-By: Claude Opus 5.5 (1M context) * Merge the audit fixes from the branches below, and document the test client and log4j2 suite The context's TestRestTemplate adds trace headers of its own whenever Sentry customizes the RestTemplateBuilder, not only in OpenTelemetry mode, and the Spring Boot modules have an integrationTestLog4j2 suite. Co-Authored-By: Claude Opus 5.5 (1M context) --- .claude/skills/test/SKILL.md | 38 +++++++--------- .cursor/rules/e2e_tests.mdc | 84 ++++++++++++++++++++++++++++++------ .cursor/rules/new_module.mdc | 4 +- AGENTS.md | 9 ++-- 4 files changed, 92 insertions(+), 43 deletions(-) diff --git a/.claude/skills/test/SKILL.md b/.claude/skills/test/SKILL.md index d7b28f6d260..0a066c76de0 100644 --- a/.claude/skills/test/SKILL.md +++ b/.claude/skills/test/SKILL.md @@ -1,13 +1,14 @@ --- name: test -description: Run tests for a specific SDK module. Use when asked to "run tests", "test module", "run unit tests", "run system tests", "run e2e tests", or test a specific class. Auto-detects unit vs system tests. Supports interactive mode. +description: Run tests for a specific SDK module. Use when asked to "run tests", "test module", "run unit tests", "run integration tests", "run e2e tests", or test a specific class. Auto-detects unit tests vs integration test suites. Supports interactive mode. allowed-tools: Bash, Read, Glob, AskUserQuestion argument-hint: [interactive] [test-class-filter] --- # Run Tests -Run tests for a specific module. Auto-detects whether to run unit tests or system tests. +Run tests for a specific module. Auto-detects whether to run unit tests or an integration test +suite. ## Step 0: Check for Interactive Mode @@ -19,7 +20,7 @@ In interactive mode, use AskUserQuestion at decision points as described in the The argument can be either: - A **file path** (e.g., `@sentry/src/test/java/io/sentry/ScopesTest.kt`) -- A **module name** (e.g., `sentry-android-core`, `sentry-samples-spring-boot-4`) +- A **module name** (e.g., `sentry-android-core`, `sentry-spring-boot-4`) - A **module name + test filter** (e.g., `sentry ScopesTest`) Extract the module name and optional test class filter from the argument. @@ -30,10 +31,13 @@ Extract the module name and optional test class filter from the argument. | Signal | Test Type | |--------|-----------| -| Path contains `sentry-samples/` | System test | -| Module name starts with `sentry-samples-` | System test | +| Path contains `/src/integrationTest*/` or `/src/agentFixture/` | Integration test suite | +| Path contains `sentry-samples/`, or module name starts with `sentry-samples-` | None: samples have no tests | | Everything else | Unit test | +For a sample, tell the user that samples have no tests, and that the end-to-end tests for what +the sample shows live in the module it uses (see the `e2e_tests` rule). + ## Step 3a: Run Unit Tests Determine the Gradle test task: @@ -60,30 +64,20 @@ Without a filter: ./gradlew '::' --info ``` -## Step 3b: Run System Tests - -System tests require the Python-based test runner which manages a mock Sentry server and sample app lifecycle. - -1. Ensure the Python venv exists: -```bash -test -d .venv || make setupPython -``` +## Step 3b: Run an Integration Test Suite -2. Extract the sample module name. For file paths like `sentry-samples//src/...`, the sample module is the directory name (e.g., `sentry-samples-spring`). +The suite's task is named after its source set directory: `src/integrationTestWebflux/...` runs +with `integrationTestWebflux`. The OpenTelemetry agent suite runs in two tasks, one per init mode, +so for `src/integrationTestOtelAgent/` or `src/agentFixture/` run both: -3. Run the system test: ```bash -.venv/bin/python test/system-test-runner.py test --module +./gradlew '::integrationTestOtelAgent' '::integrationTestOtelAgentWithoutAutoInit' ``` -This starts the mock Sentry server, starts the sample app (Spring Boot/Tomcat/CLI), runs tests via `./gradlew :sentry-samples::systemTest`, and cleans up afterwards. - -To run **every** system test instead of one module, use the Makefile targets — they also create the -venv for you: +Otherwise, with or without a test class filter: ```bash -make systemTest # all system tests (--all) -make systemTestInteractive # pick the setups to run (--interactive) +./gradlew '::' --tests="**" ``` ## Step 4: Report Results diff --git a/.cursor/rules/e2e_tests.mdc b/.cursor/rules/e2e_tests.mdc index 17e088774b5..1785e61b5c1 100644 --- a/.cursor/rules/e2e_tests.mdc +++ b/.cursor/rules/e2e_tests.mdc @@ -3,26 +3,82 @@ alwaysApply: false description: Java SDK End to End Tests --- -# Java SDK End to End Tests (System Tests) +# Java SDK End to End Tests -The samples in the `sentry-samples` directory are used to run end to end tests against them. +End-to-end tests live in the module whose code they test, not in `sentry-samples`. Samples have no +tests. A test runs the code the way an app would and asserts what reaches Sentry over the real HTTP +transport. -There is a python script (`system-test-runner.py`) that can be used to run one (using `--module SAMPLE_NAME`) or all (using `--all`) system tests. +## MockSentryServer -The script has an interactive mode (`-i`) which allows selection of test setups to execute, whether to run the tests or just prepare infrastructure for testing from IDE. +`MockSentryServer` (`sentry-test-support`) is an in-process HTTP server on a free port. Point the +SDK at `server.dsn`, then: -The tests run a mock Sentry server via `system-test-sentry-server.py`. Any system under test will then have a DSN set that reflects this local mock server like `http://502f25099c204a2fbf4cb16edc5975d1@localhost:8000/0`. -By using this local DSN, the system under test sends events to the local mock server. -The tests can then use `TestHelper` to assert envelopes that were received by the mock server. -`TestHelper` uses HTTP requests to retrieve the JSON payload of the received events and deserialize them back to objects for easier assertion. -Tests can then assert events, transactions, logs etc. similar to how they would appear in `beforeSend` and similar callbacks. +- `awaitEnvelope(timeoutMillis) { predicate }` polls for a matching envelope and fails with what it + received on timeout. +- `assertNoEnvelope(millis) { predicate }` checks that nothing matching arrives. +- `MockSentryServer { ... }` takes a hook that runs before each check while waiting, for example to + flush an OpenTelemetry tracer provider. -`TestHelper` has a lot of helper methods for asserting, e.g. by span name, log body etc. +`EnvelopeMatchers.kt` has the accessors and matchers: `event()`, `transaction()`, `logs()`, +`metrics()`, `profileChunk()`, `hasSpanWithOp`, `hasFlag`, `hasLogWithAttribute`, `hasMetric` and +so on. -The end to end tests either expect the system under test to either be running on a server or call `java -jar` to execute a CLI system under test. +Call `Sentry.flush()` before awaiting when Sentry runs in the test's JVM: logs and metrics are +batched. Envelopes that arrive late can outlive a server's `reset()`, so tests that share a server +match on values only they produce. -For Spring Boot, we spin up the Spring Boot server. The tests then send requests to that server and assert what is sent to Sentry. +## Where the tests are -End to end tests are also executed on CI using a matrix build, as defined in `.github/workflows/system-tests-backend.yml`. +- **Core and plain integrations** use their module's `src/test`. Examples: `SentryEndToEndTest` + (`sentry`), `SentryAppenderIntegrationTest` (`sentry-logback`, `sentry-log4j2`, loading the + appender from XML), `SentryHandlerIntegrationTest` (`sentry-jul`), + `ContinuousProfilerEndToEndTest` (`sentry-async-profiler`), `SentryKafkaEndToEndTest` + (`sentry-kafka`), `AgentlessEndToEndTest` and `OtlpEndToEndTest` (`sentry-opentelemetry-*`). +- **Spring Boot** (`sentry-spring-boot`, `sentry-spring-boot-jakarta`, `sentry-spring-boot-4`) has + one JVM test suite per flavor, because a flavor is a classpath: Sentry's auto-configuration and + OpenTelemetry mode switch on which classes are present. + - `integrationTest`: servlet app. + - `integrationTestWebflux`: reactive app. + - `integrationTestOtelNoAgent`: `sentry-opentelemetry-agentless-spring`. + - `integrationTestOtelAgent` and `integrationTestOtelAgentWithoutAutoInit`: the built agent. + - `integrationTestOtlp` (Boot 4): `sentry-opentelemetry-otlp-spring`. + - `integrationTestLog4j2` (Boot 3, 4): Log4j2 instead of Logback, for the auto-configured + Log4j2 appender. -Some of the samples are tested in multiple ways, e.g. with OpenTelemetry Agent auto init turned on and off. + Each suite has a base class with `@SpringBootTest(RANDOM_PORT)` and a `@DynamicPropertySource` + that sets `sentry.dsn`. Subclasses share one context and one server. +- **Plain Spring** (`sentry-spring`, `sentry-spring-jakarta`, `sentry-spring-7`) has an + `integrationTest` suite that starts the app in an embedded Tomcat, the way a WAR starts. Spring + Boot stays off its classpath, because `SentrySpringServletContainerInitializer` behaves + differently with it. + +## The OpenTelemetry agent + +The agent must not run in the test's JVM, because it would instrument the test's own HTTP client +and server. The Spring Boot modules have an `agentFixture` source set with the app. `AgentApp` +starts it once per test JVM in a child process, with `ChildJvmApp` and the agent jar passed through +`ChildJvmTestArguments`. The suite's task runs with the agent initializing Sentry, and +`integrationTestOtelAgentWithoutAutoInit` runs the same tests with the Spring Boot starter doing +it. Gradle runs the two in parallel. + +## Running them + +- The `spring-boot-{2,3,4}-matrix.yml` workflows run the suites against each supported Spring Boot + version, by setting the version in `gradle/libs.versions.toml`. The suites aren't part of + `check`, so a new suite only runs on CI once it's added to its matrix. + +## Gotchas + +- Don't use the context's `TestRestTemplate`. It comes from the `RestTemplateBuilder` that Sentry + customizes, and that the OpenTelemetry starter instruments, so it adds or replaces trace headers + on the requests a test sends. Build one outside the context. +- Sentry exports OpenTelemetry spans in batches every 5 seconds. In-process, flush the tracer + provider from the server's hook. With the agent this isn't possible, so each test waits for a + batch and negative checks wait at least 6 seconds. +- Sentry is global, so a second live Spring context in an OpenTelemetry suite takes it over. Keep + one live context per JVM (`@DirtiesContext` when a test needs its own). +- The async profiler rejects a traces path with a `.` anywhere in its absolute form. Put the path + in `java.io.tmpdir`. +- Kafka uses an in-JVM broker (`@EmbeddedKafka` or spring-kafka-test's broker classes), so no + Docker is needed. diff --git a/.cursor/rules/new_module.mdc b/.cursor/rules/new_module.mdc index d499acf03a0..b9fdde3df6a 100644 --- a/.cursor/rules/new_module.mdc +++ b/.cursor/rules/new_module.mdc @@ -69,8 +69,8 @@ static { addIntegrationToSdkVersion("{IntegrationName}"); ``` -7. If adding a JVM sample, add E2E (system) tests, following the structure we have in the existing JVM examples. - The test should then be added to `test/system-test-runner.py` and `.github/workflows/system-tests-backend.yml`. +7. Samples have no tests. Add end-to-end coverage of what the sample shows to the module it uses, + against `MockSentryServer`, as described in the `e2e_tests` rule. `sentry-bom` and the root `build.gradle.kts` need no change — they iterate over subprojects. diff --git a/AGENTS.md b/AGENTS.md index badfba1b3e4..1ffec6c02b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -20,7 +20,7 @@ rule file in `.cursor/rules/`: | `continuous_profiling_jvm` | `sentry-async-profiler`, `IContinuousProfiler`, `ProfileChunk`, JFR files, `ProfileLifecycle` | | `opentelemetry` | `sentry-opentelemetry-*`, agent vs agentless, span processing, sampling, context propagation | | `new_module` | Adding a new integration or sample module | -| `e2e_tests` | System tests, sample applications, `system-test-runner.py`, mock Sentry server | +| `e2e_tests` | End-to-end tests, `MockSentryServer`, integration test suites, the OpenTelemetry agent fixture | Rules can be combined — a tracing scope issue may need both `scopes` and `opentelemetry`. There is no rule for Android profiling yet; read the `sentry-android-core` profiling code @@ -55,7 +55,7 @@ The project uses **Gradle** with Kotlin DSL. Key build files: ``` To run tests, use the `test` skill rather than composing the Gradle invocation by hand — it -resolves the per-module test task and the unit-test vs system-test split for you. +resolves the per-module test task, including integration test suites, for you. ### Android-Specific Commands ```bash @@ -109,8 +109,7 @@ The repository is organized into multiple modules: - **Other**: `sentry-spotlight`, `sentry-kotlin-extensions`, `sentry-android-distribution` ### Utility Modules -- **`sentry-test-support`** - Shared test utilities -- **`sentry-system-test-support`** - System testing infrastructure +- **`sentry-test-support`** - Shared test utilities, including `MockSentryServer` and envelope matchers - **`sentry-samples`** - Example applications - **`sentry-bom`** - Bill of Materials for dependency management @@ -158,7 +157,7 @@ PR description why the broad catch is necessary. ### Testing Requirements - Write comprehensive unit tests for new features - Android modules require both unit tests and instrumented tests where applicable -- System tests validate end-to-end functionality with sample applications +- End-to-end tests run in the module they test, against `MockSentryServer`. Samples have no tests - **Assertions**: For new unit tests, prefer [Google Truth](https://truth.dev/) (`com.google.common.truth.Truth.assertThat`) over `kotlin.test`/JUnit assertions for its readable, fluent API. Keep using `kotlin.test` for test structure (`@Test`, `assertFailsWith`). See `sentry/src/test/java/io/sentry/DsnTest.kt` for the style. Don't rewrite existing `kotlin.test` assertions solely to switch libraries. - Truth is wired into the `sentry` module. When adding Truth-based tests to another module, add `testImplementation(libs.google.truth)` to that module's `build.gradle.kts`.