Add virtual thread friendly, blocking, Smithy HTTP client that supports HTTP 1.1 and HTTP 2 bidirectional streaming

.github/workflows/fuzz-testing.yml

@@ -59,6 +59,17 @@ jobs:
         run: |
           ./gradlew :rulesengine:test --tests "*FuzzTest*" -Djazzer.max_duration=1h --stacktrace
 
+      - name: Run HPACK fuzz tests
+        env:
+          JAZZER_FUZZ: "1"
+        run: |
+          ./gradlew :http:http-hpack:test --tests "*FuzzTest*" -Djazzer.max_duration=1h --stacktrace
+
+      - name: Run HTTP client fuzz tests
+        env:
+          JAZZER_FUZZ: "1"
+        run: |
+          ./gradlew :http:http-client:test --tests "*FuzzTest*" -Djazzer.max_duration=1h --stacktrace
       - name: Save fuzz corpus cache
         uses: actions/cache/save@v6
         if: always()

.gitignore

@@ -1,6 +1,8 @@
 # Ignore Gradle project-specific cache directory
 .gradle
 
+.tool-versions
+
 # Ignore kotlin cache dir
 .kotlin
 
@@ -28,3 +30,5 @@ mise.toml
 .claude/settings.local.json
 
 **/bin
+
+benchmarks/e2e-benchmarks/profiles

benchmarks/e2e-benchmarks/README.md

@@ -1,8 +1,6 @@
 # smithy-java end-to-end benchmark runner
 
-A workload-driven runner that exercises the smithy-java SDK against live AWS
-services. It implements the same workload spec as the Java SDK v2 reference
-runner so results are directly comparable across SDKs.
+A small fixed-workload runner that exercises the smithy-java SDK against live AWS services.
 
 ## Scope
 
@@ -13,20 +11,13 @@ runner so results are directly comparable across SDKs.
 | S3       | PutObject | Throughput|
 | S3       | GetObject | Throughput|
 
-Only the **synchronous** client mode is supported. smithy-java does not
-generate async clients today, so a `--client async` argument is accepted but
-prints a warning and proceeds with the sync client. Throughput tests still
-push the SDK through a thread pool, which is the SDK's idiomatic concurrency
-mechanism.
-
 ## Build
 
 ```bash
 ./gradlew :benchmarks:e2e-benchmarks:shadowJar
 ```
 
-The jar lands at
-`benchmarks/e2e-benchmarks/build/libs/smithy-java-e2e-benchmark-runner.jar`.
+The jar lands at `benchmarks/e2e-benchmarks/build/libs/smithy-java-e2e-benchmark-runner.jar`.
 
 The build pulls AWS service models from Maven (`software.amazon.api.models:dynamodb`
 and `software.amazon.api.models:s3`) and codegens the typed DynamoDB and S3
@@ -37,16 +28,102 @@ don't collide.
 
 ```bash
 java -jar build/libs/smithy-java-e2e-benchmark-runner.jar \
-     workloads/ddb-getitem-1KiB-latency-benchmark.json us-east-1
+     --operation s3-put \
+     --bucket my-bench--use1-az4--x-s3 \
+     --region us-east-1
+```
+
+Operations:
+
+| Operation | Workload |
+|-----------|----------|
+| `s3-put`  | S3 PutObject, 256 KiB body, concurrent |
+| `s3-get`  | S3 GetObject, 256 KiB body, concurrent |
+| `ddb-put` | DynamoDB PutItem, 1 KiB item, sequential |
+| `ddb-get` | DynamoDB GetItem, 1 KiB item, sequential |
+
+Flags:
+
+| Flag | Notes |
+|------|-------|
+| `--operation <name>` | One of `s3-put`, `s3-get`, `ddb-put`, `ddb-get` |
+| `--bucket <name>` | S3 Express directory bucket: `<base>--<az>--x-s3` |
+| `--table <name>`  | Must have a `String` partition key named `pk` |
+| `--region <region>` | Use the region the bucket / table lives in |
+| `--client sync\|async` | Accepted for compatibility; smithy-java only supports sync |
+| `--key-prefix <prefix>` | DynamoDB key prefix, default `item-` |
+| `--s3-key-prefix <prefix>` | S3 key prefix, default `objects/256KiB/` |
+
+Common system properties:
+
+| Property | Default |
+|----------|---------|
+| `e2e.batch.actions` | `10000` for S3, `1000` for DynamoDB |
+| `e2e.warmup.batches` | `2` |
+| `e2e.measurement.batches` | `3` |
+| `e2e.collectMetrics` | `true` |
+| `e2e.object.size` | `262144` |
+| `e2e.data.length` | `1024` |
+| `e2e.ddb.createTable` | `true` |
+| `e2e.ddb.deleteTable` | `true` |
+| `e2e.ddb.readCapacityUnits` | `5000` |
+| `e2e.ddb.writeCapacityUnits` | `5000` |
+
+## Provision the resources
+
+Run from an EC2 instance in the same AZ as the bucket. The instance role
+needs `s3:CreateSession`, `s3express:*`, and `dynamodb:GetItem`/`PutItem`
+on the resources below.
+
+S3 Express directory bucket (replace `my-bench` and `use1-az4` with your
+own base name and availability zone):
+
+```bash
+aws s3api create-bucket \
+  --bucket my-bench--use1-az4--x-s3 \
+  --region us-east-1 \
+  --create-bucket-configuration '{
+    "Location": {"Type": "AvailabilityZone", "Name": "use1-az4"},
+    "Bucket":   {"Type": "Directory", "DataRedundancy": "SingleAvailabilityZone"}
+  }'
 ```
 
-Region passed on the command line is ignored — the workload JSON is the
-source of truth.
+The S3 download workload reads keys named `objects/256KiB/<index>`. Pre-seed
+them by running the upload workload once first.
 
-You can point the runner at any v1 workload JSON.
+The DynamoDB workloads create a fresh provisioned table by default using
+`--table` as the base name. The actual table name gets a run-specific suffix,
+uses a `String` partition key named `pk`, defaults to `5000` RCU / `5000` WCU,
+and is deleted after the run. The GetItem workload seeds its keys before warmup.
+
+To reuse an existing table instead:
+
+```bash
+java -De2e.ddb.createTable=false -jar build/libs/smithy-java-e2e-benchmark-runner.jar \
+     --operation ddb-get --table my-bench-table --region us-east-1
+```
+
+## Cleanup
+
+```bash
+aws s3 rm s3://my-bench--use1-az4--x-s3 --recursive --region us-east-1
+aws s3api delete-bucket --bucket my-bench--use1-az4--x-s3 --region us-east-1
+aws dynamodb delete-table --table-name my-bench-table --region us-east-1
+```
+
+## Concurrency
+
+Throughput benchmarks dispatch each batch action onto a virtual thread,
+gated by a semaphore. The cap is `cores × multiplier`, where `multiplier`
+defaults to `4` and can be overridden:
+
+```bash
+java -De2e.concurrency.multiplier=16 -jar build/libs/smithy-java-e2e-benchmark-runner.jar \
+     --operation s3-put --bucket my-bench--use1-az4--x-s3 --region us-east-1
+```
 
 ## Credentials
 
-Credentials come from the AWS SDK v2 default credential provider chain
-(env vars, profile file, container roles, IMDS). Override via the standard
-SDK v2 environment variables.
+Credentials are resolved from EC2 IMDSv2 directly. The benchmark is meant
+to run on EC2 against directory buckets in the same AZ. For local testing
+or non-EC2 hosts, supply credentials by extending the `Clients` factory.

benchmarks/e2e-benchmarks/build.gradle.kts

@@ -1,3 +1,4 @@
+// Note: Not published
 plugins {
     id("smithy-java.java-conventions")
     id("com.gradleup.shadow")
@@ -7,9 +8,6 @@ plugins {
 
 description = "End-to-end SDK benchmarks against live AWS services (DynamoDB GetItem/PutItem latency, S3 GetObject/PutObject throughput)."
 
-// Not published. Mirrors the Java SDK v2 reference runner and reads the same
-// workload JSON files so results are directly comparable.
-
 application {
     mainClass.set("software.amazon.smithy.java.benchmarks.e2e.WorkloadRunner")
 }
@@ -19,6 +17,11 @@ dependencies {
     smithyBuild(project(":codegen:codegen-plugin"))
     smithyBuild(project(":client:client-core"))
     smithyBuild(project(":client:client-waiters"))
+    // Needed at codegen time so RulesEngineBuilder loads AwsRulesExtension via ServiceLoader.
+    smithyBuild(project(":aws:client:aws-client-rulesengine"))
+    // Codegen needs the plugin class on its classpath so `Class.forName(...)` resolves it
+    // when wiring it into the generated client.
+    smithyBuild(project(":aws:aws-sigv4-s3express"))
 
     // AWS service models pulled from Maven (https://github.com/aws/api-models-aws).
     // The smithy-base plugin only loads models from sources + runtimeClasspath
@@ -48,6 +51,7 @@ dependencies {
 
     // AWS-specific runtime: SigV4, AWS protocols, AWS endpoints.
     implementation(project(":aws:aws-sigv4"))
+    implementation(project(":aws:aws-sigv4-s3express"))
     implementation(project(":aws:aws-auth-api"))
     implementation(project(":aws:client:aws-client-core"))
     implementation(project(":aws:client:aws-client-http"))
@@ -65,18 +69,18 @@ dependencies {
     implementation(libs.smithy.aws.traits)
     implementation(libs.smithy.model)
 
-    // smithy-java native credential chain — covers env vars, system props,
-    // shared config, web identity token, and ECS container slots out of the
-    // box; pulling in aws-credentials-imds adds the EC2 instance-metadata
-    // provider on top of it. Both modules register their providers via
-    // ServiceLoader, so just having them on the classpath is enough.
+    // smithy-java credential chain
     implementation(project(":aws:aws-credential-chain"))
     implementation(project(":aws:aws-credentials-imds"))
+
+    // Alternate transports — selected at runtime via -De2e.transport=smithy|smithy-boringssl
+    implementation(project(":client:client-http-smithy"))
+    implementation(project(":client:client-http-boringssl"))
 }
 
 // Two projections so that DynamoDB and S3 generate into different namespaces
-// and don't collide. Each projection filters down to just the ONE service
-// it wants — the model JAR for s3 only has the s3 model, but the projection
+// and don't collide. Each projection filters down to just the one service
+// it wants; the model JAR for s3 only has the s3 model, but the projection
 // makes the intent explicit and gives us a stable name.
 val codegenProjections = listOf("dynamodb-client", "s3-client")
 
@@ -104,7 +108,8 @@ tasks.named<Copy>("processResources") {
     dependsOn("smithyBuild")
 }
 
-// The shaded jar is the self-contained artifact users invoke to run a workload.
+// The shaded jar is what users invoke from run-benchmark.py, mirroring
+// `java -jar runners/java-workload-runner/target/workload-runner-1.0.0.jar`.
 tasks.named<com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar>("shadowJar") {
     archiveBaseName.set("smithy-java-e2e-benchmark-runner")
     archiveClassifier.set("")
@@ -123,7 +128,7 @@ tasks.named("assemble") {
     dependsOn("shadowJar")
 }
 
-// Don't run benchmarks under `./gradlew check` — they hit live AWS.
+// Don't run benchmarks under `./gradlew check`, they hit live AWS.
 tasks.named("check") {
     enabled = true
 }

benchmarks/e2e-benchmarks/smithy-build.json

@@ -22,7 +22,12 @@
           "transport": {
             "http-java": {}
           },
-          "modes": ["client"]
+          "modes": ["client"],
+          "runtimeTraits": [
+            "smithy.rules#contextParam",
+            "smithy.rules#staticContextParams",
+            "smithy.rules#operationContextParams"
+          ]
         }
       }
     },
@@ -47,7 +52,12 @@
           "transport": {
             "http-java": {}
           },
-          "modes": ["client"]
+          "modes": ["client"],
+          "runtimeTraits": [
+            "smithy.rules#contextParam",
+            "smithy.rules#staticContextParams",
+            "smithy.rules#operationContextParams"
+          ]
         }
       }
     }

benchmarks/e2e-benchmarks/src/main/java/software/amazon/smithy/java/benchmarks/e2e/ActionExecutor.java

@@ -6,7 +6,6 @@
 package software.amazon.smithy.java.benchmarks.e2e;
 
 import java.io.IOException;
-import java.io.OutputStream;
 import java.io.UncheckedIOException;
 import java.util.Map;
 import software.amazon.smithy.java.benchmarks.e2e.dynamodb.client.DynamoDBClient;
@@ -19,10 +18,7 @@
 import software.amazon.smithy.java.io.datastream.DataStream;
 
 /**
- * Wraps the smithy-java DynamoDB and S3 clients with the four operations the
- * benchmark exercises. Smithy-java currently only generates synchronous
- * clients, so the throughput tests achieve concurrency by submitting work to
- * a thread pool from {@link WorkloadRunner}.
+ * Wraps the smithy-java DynamoDB and S3 clients with the four operations the benchmark exercises.
  */
 final class ActionExecutor {
 
@@ -51,9 +47,8 @@ void getItem(String tableName, Map<String, AttributeValue> key) {
     }
 
     void putObject(String bucket, String key, int objectSize) {
-        // The reference runner reuses a single in-memory body across all
-        // uploads. DataStream.ofBytes wraps the array without copying, so
-        // each call is a thin handle over the same buffer.
+        // The reference runner reuses a single in-memory body across all uploads.
+        // DataStream.ofBytes wraps the array without copying, so each call is a thin handle over the same buffer.
         var body = DataStream.ofBytes(payload, 0, objectSize, "application/octet-stream");
         s3.putObject(PutObjectInput.builder()
                 .bucket(bucket)
@@ -68,19 +63,14 @@ void getObject(String bucket, String key) {
                 .bucket(bucket)
                 .key(key)
                 .build());
-        // Drain the body so the download time is what we measure, not just
-        // the response headers. writeTo(nullOutputStream) walks the stream
-        // without forcing a single-contiguous-ByteBuffer materialization, so
-        // we exercise the SDK's most efficient consume path (multi-chunk
-        // delivery from the wire is fed straight through with no stitch).
+        // discard() drains and releases the underlying source.
         var body = output.getBody();
         if (body != null) {
             try {
-                body.writeTo(OutputStream.nullOutputStream());
+                body.discard();
             } catch (IOException e) {
-                throw new UncheckedIOException("Failed to drain S3 GetObject body", e);
+                throw new UncheckedIOException("Failed to discard S3 GetObject body", e);
             }
         }
     }
-
 }