diff --git a/Cargo.lock b/Cargo.lock index f2aee9f..6219530 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,9 +34,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "applevisor" @@ -65,6 +65,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayvec" version = "0.7.6" @@ -461,9 +467,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.131.0" +version = "1.132.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe1b8c5282bf859170836045296b3cd710b7573aceb909498366bb508a41058e" +checksum = "5575840a3a6b11f6011463ebe359320dfe5b67babb5e9b06fed6ddf809a9ab40" dependencies = [ "aws-credential-types", "aws-runtime", @@ -1161,9 +1167,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -1236,6 +1242,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "config" +version = "0.15.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b85f248a4de22d204ceabc6299d89d2c70fbd7f09fea53c06c852369652d8139" +dependencies = [ + "pathdiff", + "serde_core", + "winnow", + "yaml-rust2", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1547,7 +1565,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -1734,11 +1752,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi 5.3.0", "wasip2", - "wasm-bindgen", ] [[package]] @@ -1748,11 +1764,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -1774,9 +1792,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -1834,6 +1852,15 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +[[package]] +name = "hashlink" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "heck" version = "0.5.0" @@ -2476,6 +2503,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2525,15 +2558,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -2555,9 +2579,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.2" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", ] @@ -2584,15 +2608,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.2", "lru-slab", - "rand 0.9.4", + "rand", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -2639,16 +2664,6 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" -[[package]] -name = "rand" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" -dependencies = [ - "rand_chacha", - "rand_core 0.9.5", -] - [[package]] name = "rand" version = "0.10.1" @@ -2660,16 +2675,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", -] - [[package]] name = "rand_core" version = "0.6.4" @@ -2681,18 +2686,18 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.5" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" -dependencies = [ - "getrandom 0.3.4", -] +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] -name = "rand_core" -version = "0.10.1" +name = "rand_pcg" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] [[package]] name = "redox_syscall" @@ -2843,7 +2848,7 @@ dependencies = [ [[package]] name = "rustack-apigatewayv2-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "base64", @@ -2851,7 +2856,7 @@ dependencies = [ "chrono", "dashmap", "http 1.4.0", - "rand 0.10.1", + "rand", "reqwest 0.12.28", "rustack-apigatewayv2-http", "rustack-apigatewayv2-model", @@ -2866,7 +2871,7 @@ dependencies = [ [[package]] name = "rustack-apigatewayv2-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "http 1.4.0", @@ -2876,6 +2881,7 @@ dependencies = [ "hyper-util", "rustack-apigatewayv2-model", "rustack-auth", + "rustack-core", "serde", "serde_json", "thiserror 2.0.18", @@ -2897,7 +2903,7 @@ dependencies = [ [[package]] name = "rustack-auth" -version = "0.9.0" +version = "0.10.0" dependencies = [ "base64", "chrono", @@ -2916,7 +2922,7 @@ dependencies = [ [[package]] name = "rustack-cli" -version = "0.9.1" +version = "0.10.0" dependencies = [ "anyhow", "async-trait", @@ -2987,12 +2993,12 @@ dependencies = [ [[package]] name = "rustack-cloudfront-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "chrono", "dashmap", "parking_lot", - "rand 0.10.1", + "rand", "rustack-cloudfront-model", "rustack-core", "serde", @@ -3004,7 +3010,7 @@ dependencies = [ [[package]] name = "rustack-cloudfront-dataplane" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "dashmap", @@ -3016,6 +3022,7 @@ dependencies = [ "reqwest 0.13.2", "rustack-cloudfront-core", "rustack-cloudfront-model", + "rustack-core", "rustack-s3-core", "rustack-s3-model", "serde", @@ -3028,7 +3035,7 @@ dependencies = [ [[package]] name = "rustack-cloudfront-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "base64", "bytes", @@ -3043,6 +3050,7 @@ dependencies = [ "rustack-auth", "rustack-cloudfront-core", "rustack-cloudfront-model", + "rustack-core", "thiserror 2.0.18", "tokio", "tracing", @@ -3061,7 +3069,7 @@ dependencies = [ [[package]] name = "rustack-cloudwatch-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "async-trait", @@ -3085,7 +3093,7 @@ dependencies = [ [[package]] name = "rustack-cloudwatch-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "ciborium", @@ -3097,6 +3105,7 @@ dependencies = [ "hyper-util", "rustack-auth", "rustack-cloudwatch-model", + "rustack-core", "serde", "serde_json", "tokio", @@ -3117,22 +3126,29 @@ dependencies = [ [[package]] name = "rustack-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", + "bytes", "chrono", + "config", "dashmap", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", "serde", "serde_json", "thiserror 2.0.18", "tokio", "tracing", + "typed-builder", + "url", "uuid", ] [[package]] name = "rustack-dynamodb-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "bytes", @@ -3147,13 +3163,14 @@ dependencies = [ "serde", "serde_json", "thiserror 2.0.18", + "tokio", "tracing", "uuid", ] [[package]] name = "rustack-dynamodb-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "chrono", @@ -3164,6 +3181,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-dynamodb-model", "serde", "serde_json", @@ -3187,7 +3205,7 @@ dependencies = [ [[package]] name = "rustack-dynamodbstreams-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "bytes", @@ -3210,7 +3228,7 @@ dependencies = [ [[package]] name = "rustack-dynamodbstreams-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "crc32fast", @@ -3220,6 +3238,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-dynamodbstreams-model", "serde", "serde_json", @@ -3243,7 +3262,7 @@ dependencies = [ [[package]] name = "rustack-events-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "async-trait", @@ -3267,7 +3286,7 @@ dependencies = [ [[package]] name = "rustack-events-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "crc32fast", @@ -3277,6 +3296,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-events-model", "serde", "serde_json", @@ -3288,7 +3308,7 @@ dependencies = [ [[package]] name = "rustack-events-model" -version = "0.9.0" +version = "0.10.0" dependencies = [ "http 1.4.0", "serde", @@ -3297,7 +3317,7 @@ dependencies = [ [[package]] name = "rustack-iam-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "bytes", @@ -3305,7 +3325,7 @@ dependencies = [ "dashmap", "http 1.4.0", "percent-encoding", - "rand 0.10.1", + "rand", "rustack-auth", "rustack-core", "rustack-iam-http", @@ -3319,7 +3339,7 @@ dependencies = [ [[package]] name = "rustack-iam-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "form_urlencoded", @@ -3329,6 +3349,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-iam-model", "serde", "tokio", @@ -3377,7 +3398,7 @@ dependencies = [ "base64", "bytes", "futures", - "rand 0.10.1", + "rand", "reqwest 0.13.2", "rustack-lambda-core", "rustack-lambda-model", @@ -3395,7 +3416,7 @@ dependencies = [ [[package]] name = "rustack-kinesis-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "base64", @@ -3404,7 +3425,7 @@ dependencies = [ "dashmap", "http 1.4.0", "md-5", - "rand 0.10.1", + "rand", "rustack-auth", "rustack-core", "rustack-kinesis-http", @@ -3420,7 +3441,7 @@ dependencies = [ [[package]] name = "rustack-kinesis-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "crc32fast", @@ -3430,6 +3451,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-kinesis-model", "serde", "serde_json", @@ -3453,7 +3475,7 @@ dependencies = [ [[package]] name = "rustack-kms-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "aws-lc-rs", @@ -3462,7 +3484,7 @@ dependencies = [ "chrono", "dashmap", "http 1.4.0", - "rand 0.10.1", + "rand", "rustack-auth", "rustack-core", "rustack-kms-http", @@ -3476,7 +3498,7 @@ dependencies = [ [[package]] name = "rustack-kms-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "crc32fast", @@ -3486,6 +3508,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-kms-model", "serde", "serde_json", @@ -3509,7 +3532,7 @@ dependencies = [ [[package]] name = "rustack-lambda-core" -version = "0.9.1" +version = "0.10.0" dependencies = [ "anyhow", "async-trait", @@ -3542,13 +3565,19 @@ dependencies = [ name = "rustack-lambda-echo-bootstrap" version = "0.9.0" dependencies = [ + "base64", + "rustack-lambda-core", + "rustack-lambda-model", "serde_json", + "tempfile", + "tokio", "ureq", + "zip", ] [[package]] name = "rustack-lambda-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "http 1.4.0", @@ -3557,6 +3586,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-lambda-model", "serde", "serde_json", @@ -3578,7 +3608,7 @@ dependencies = [ [[package]] name = "rustack-logs-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "bytes", @@ -3600,7 +3630,7 @@ dependencies = [ [[package]] name = "rustack-logs-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "crc32fast", @@ -3610,6 +3640,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-logs-model", "serde", "serde_json", @@ -3630,7 +3661,7 @@ dependencies = [ [[package]] name = "rustack-s3-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "base64", @@ -3648,7 +3679,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.10.1", + "rand", "rustack-auth", "rustack-core", "rustack-s3-model", @@ -3667,7 +3698,7 @@ dependencies = [ [[package]] name = "rustack-s3-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "chrono", @@ -3682,6 +3713,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustack-auth", + "rustack-core", + "rustack-s3-core", "rustack-s3-model", "rustack-s3-xml", "sha2 0.11.0", @@ -3694,7 +3727,7 @@ dependencies = [ [[package]] name = "rustack-s3-model" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "chrono", @@ -3704,7 +3737,7 @@ dependencies = [ [[package]] name = "rustack-s3-xml" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "chrono", @@ -3716,14 +3749,14 @@ dependencies = [ [[package]] name = "rustack-secretsmanager-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "base64", "bytes", "chrono", "dashmap", "http 1.4.0", - "rand 0.10.1", + "rand", "rustack-auth", "rustack-core", "rustack-secretsmanager-http", @@ -3736,7 +3769,7 @@ dependencies = [ [[package]] name = "rustack-secretsmanager-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "crc32fast", @@ -3746,6 +3779,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-secretsmanager-model", "serde", "serde_json", @@ -3769,7 +3803,7 @@ dependencies = [ [[package]] name = "rustack-ses-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "base64", @@ -3793,7 +3827,7 @@ dependencies = [ [[package]] name = "rustack-ses-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "form_urlencoded", @@ -3803,6 +3837,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-ses-model", "serde", "serde_json", @@ -3824,7 +3859,7 @@ dependencies = [ [[package]] name = "rustack-sns-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "async-trait", @@ -3850,7 +3885,7 @@ dependencies = [ [[package]] name = "rustack-sns-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "form_urlencoded", @@ -3860,6 +3895,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-sns-model", "serde", "tokio", @@ -3879,7 +3915,7 @@ dependencies = [ [[package]] name = "rustack-sqs-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "base64", @@ -3905,7 +3941,7 @@ dependencies = [ [[package]] name = "rustack-sqs-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "crc32fast", @@ -3915,6 +3951,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-sqs-model", "serde", "serde_json", @@ -3937,7 +3974,7 @@ dependencies = [ [[package]] name = "rustack-ssm-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "base64", @@ -3959,7 +3996,7 @@ dependencies = [ [[package]] name = "rustack-ssm-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "crc32fast", @@ -3969,6 +4006,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-ssm-model", "serde", "serde_json", @@ -3989,14 +4027,14 @@ dependencies = [ [[package]] name = "rustack-sts-core" -version = "0.9.0" +version = "0.10.0" dependencies = [ "base64", "bytes", "chrono", "dashmap", "http 1.4.0", - "rand 0.10.1", + "rand", "rustack-core", "rustack-sts-http", "rustack-sts-model", @@ -4008,7 +4046,7 @@ dependencies = [ [[package]] name = "rustack-sts-http" -version = "0.9.0" +version = "0.10.0" dependencies = [ "bytes", "form_urlencoded", @@ -4018,6 +4056,7 @@ dependencies = [ "hyper", "hyper-util", "rustack-auth", + "rustack-core", "rustack-sts-model", "serde", "tokio", @@ -4061,7 +4100,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -4119,7 +4158,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -4407,9 +4446,9 @@ dependencies = [ [[package]] name = "spin" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" [[package]] name = "spki" @@ -4737,7 +4776,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -5347,7 +5386,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -5642,6 +5681,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -5748,6 +5796,17 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" +[[package]] +name = "yaml-rust2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b36710ce3a279cfce8465dbab826f161675a262950b922cb2c3663852dfe9eb0" +dependencies = [ + "arraydeque", + "encoding_rs", + "hashlink", +] + [[package]] name = "yoke" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 176901c..aba1168 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,64 +29,64 @@ collapsible_if = "allow" [workspace.dependencies] # Internal crates -rustack-core = { version = "0.9.0", path = "crates/rustack-core" } -rustack-auth = { version = "0.9.0", path = "crates/rustack-auth" } -rustack-s3-core = { version = "0.9.0", path = "crates/rustack-s3-core" } -rustack-s3-http = { version = "0.9.0", path = "crates/rustack-s3-http" } -rustack-s3-model = { version = "0.9.0", path = "crates/rustack-s3-model" } -rustack-s3-xml = { version = "0.9.0", path = "crates/rustack-s3-xml" } +rustack-core = { version = "0.10.0", path = "crates/rustack-core" } +rustack-auth = { version = "0.10.0", path = "crates/rustack-auth" } +rustack-s3-core = { version = "0.10.0", path = "crates/rustack-s3-core" } +rustack-s3-http = { version = "0.10.0", path = "crates/rustack-s3-http" } +rustack-s3-model = { version = "0.10.0", path = "crates/rustack-s3-model" } +rustack-s3-xml = { version = "0.10.0", path = "crates/rustack-s3-xml" } rustack-dynamodb-model = { version = "0.9.0", path = "crates/rustack-dynamodb-model" } -rustack-dynamodb-http = { version = "0.9.0", path = "crates/rustack-dynamodb-http" } -rustack-dynamodb-core = { version = "0.9.0", path = "crates/rustack-dynamodb-core" } +rustack-dynamodb-http = { version = "0.10.0", path = "crates/rustack-dynamodb-http" } +rustack-dynamodb-core = { version = "0.10.0", path = "crates/rustack-dynamodb-core" } rustack-sqs-model = { version = "0.9.0", path = "crates/rustack-sqs-model" } -rustack-sqs-http = { version = "0.9.0", path = "crates/rustack-sqs-http" } -rustack-sqs-core = { version = "0.9.0", path = "crates/rustack-sqs-core" } +rustack-sqs-http = { version = "0.10.0", path = "crates/rustack-sqs-http" } +rustack-sqs-core = { version = "0.10.0", path = "crates/rustack-sqs-core" } rustack-ssm-model = { version = "0.9.0", path = "crates/rustack-ssm-model" } -rustack-ssm-http = { version = "0.9.0", path = "crates/rustack-ssm-http" } -rustack-ssm-core = { version = "0.9.0", path = "crates/rustack-ssm-core" } +rustack-ssm-http = { version = "0.10.0", path = "crates/rustack-ssm-http" } +rustack-ssm-core = { version = "0.10.0", path = "crates/rustack-ssm-core" } rustack-sns-model = { version = "0.9.0", path = "crates/rustack-sns-model" } -rustack-sns-http = { version = "0.9.0", path = "crates/rustack-sns-http" } -rustack-sns-core = { version = "0.9.0", path = "crates/rustack-sns-core" } +rustack-sns-http = { version = "0.10.0", path = "crates/rustack-sns-http" } +rustack-sns-core = { version = "0.10.0", path = "crates/rustack-sns-core" } rustack-lambda-model = { version = "0.9.0", path = "crates/rustack-lambda-model" } -rustack-lambda-http = { version = "0.9.0", path = "crates/rustack-lambda-http" } -rustack-lambda-core = { version = "0.9.1", path = "crates/rustack-lambda-core" } -rustack-events-model = { version = "0.9.0", path = "crates/rustack-events-model" } +rustack-lambda-http = { version = "0.10.0", path = "crates/rustack-lambda-http" } +rustack-lambda-core = { version = "0.10.0", path = "crates/rustack-lambda-core" } +rustack-events-model = { version = "0.10.0", path = "crates/rustack-events-model" } rustack-logs-model = { version = "0.9.0", path = "crates/rustack-logs-model" } -rustack-logs-http = { version = "0.9.0", path = "crates/rustack-logs-http" } -rustack-logs-core = { version = "0.9.0", path = "crates/rustack-logs-core" } -rustack-events-http = { version = "0.9.0", path = "crates/rustack-events-http" } -rustack-events-core = { version = "0.9.0", path = "crates/rustack-events-core" } +rustack-logs-http = { version = "0.10.0", path = "crates/rustack-logs-http" } +rustack-logs-core = { version = "0.10.0", path = "crates/rustack-logs-core" } +rustack-events-http = { version = "0.10.0", path = "crates/rustack-events-http" } +rustack-events-core = { version = "0.10.0", path = "crates/rustack-events-core" } rustack-kms-model = { version = "0.9.0", path = "crates/rustack-kms-model" } -rustack-kms-http = { version = "0.9.0", path = "crates/rustack-kms-http" } -rustack-kms-core = { version = "0.9.0", path = "crates/rustack-kms-core" } +rustack-kms-http = { version = "0.10.0", path = "crates/rustack-kms-http" } +rustack-kms-core = { version = "0.10.0", path = "crates/rustack-kms-core" } rustack-kinesis-model = { version = "0.9.0", path = "crates/rustack-kinesis-model" } -rustack-kinesis-http = { version = "0.9.0", path = "crates/rustack-kinesis-http" } -rustack-kinesis-core = { version = "0.9.0", path = "crates/rustack-kinesis-core" } +rustack-kinesis-http = { version = "0.10.0", path = "crates/rustack-kinesis-http" } +rustack-kinesis-core = { version = "0.10.0", path = "crates/rustack-kinesis-core" } rustack-secretsmanager-model = { version = "0.9.0", path = "crates/rustack-secretsmanager-model" } -rustack-secretsmanager-http = { version = "0.9.0", path = "crates/rustack-secretsmanager-http" } -rustack-secretsmanager-core = { version = "0.9.0", path = "crates/rustack-secretsmanager-core" } +rustack-secretsmanager-http = { version = "0.10.0", path = "crates/rustack-secretsmanager-http" } +rustack-secretsmanager-core = { version = "0.10.0", path = "crates/rustack-secretsmanager-core" } rustack-ses-model = { version = "0.9.0", path = "crates/rustack-ses-model" } -rustack-ses-http = { version = "0.9.0", path = "crates/rustack-ses-http" } -rustack-ses-core = { version = "0.9.0", path = "crates/rustack-ses-core" } +rustack-ses-http = { version = "0.10.0", path = "crates/rustack-ses-http" } +rustack-ses-core = { version = "0.10.0", path = "crates/rustack-ses-core" } rustack-apigatewayv2-model = { version = "0.9.0", path = "crates/rustack-apigatewayv2-model" } -rustack-apigatewayv2-http = { version = "0.9.0", path = "crates/rustack-apigatewayv2-http" } -rustack-apigatewayv2-core = { version = "0.9.0", path = "crates/rustack-apigatewayv2-core" } +rustack-apigatewayv2-http = { version = "0.10.0", path = "crates/rustack-apigatewayv2-http" } +rustack-apigatewayv2-core = { version = "0.10.0", path = "crates/rustack-apigatewayv2-core" } rustack-cloudwatch-model = { version = "0.9.0", path = "crates/rustack-cloudwatch-model" } -rustack-cloudwatch-http = { version = "0.9.0", path = "crates/rustack-cloudwatch-http" } -rustack-cloudwatch-core = { version = "0.9.0", path = "crates/rustack-cloudwatch-core" } +rustack-cloudwatch-http = { version = "0.10.0", path = "crates/rustack-cloudwatch-http" } +rustack-cloudwatch-core = { version = "0.10.0", path = "crates/rustack-cloudwatch-core" } rustack-dynamodbstreams-model = { version = "0.9.0", path = "crates/rustack-dynamodbstreams-model" } -rustack-dynamodbstreams-http = { version = "0.9.0", path = "crates/rustack-dynamodbstreams-http" } -rustack-dynamodbstreams-core = { version = "0.9.0", path = "crates/rustack-dynamodbstreams-core" } +rustack-dynamodbstreams-http = { version = "0.10.0", path = "crates/rustack-dynamodbstreams-http" } +rustack-dynamodbstreams-core = { version = "0.10.0", path = "crates/rustack-dynamodbstreams-core" } rustack-iam-model = { version = "0.9.0", path = "crates/rustack-iam-model" } -rustack-iam-http = { version = "0.9.0", path = "crates/rustack-iam-http" } -rustack-iam-core = { version = "0.9.0", path = "crates/rustack-iam-core" } +rustack-iam-http = { version = "0.10.0", path = "crates/rustack-iam-http" } +rustack-iam-core = { version = "0.10.0", path = "crates/rustack-iam-core" } rustack-sts-model = { version = "0.9.0", path = "crates/rustack-sts-model" } -rustack-sts-http = { version = "0.9.0", path = "crates/rustack-sts-http" } -rustack-sts-core = { version = "0.9.0", path = "crates/rustack-sts-core" } +rustack-sts-http = { version = "0.10.0", path = "crates/rustack-sts-http" } +rustack-sts-core = { version = "0.10.0", path = "crates/rustack-sts-core" } rustack-cloudfront-model = { version = "0.9.0", path = "crates/rustack-cloudfront-model" } -rustack-cloudfront-http = { version = "0.9.0", path = "crates/rustack-cloudfront-http" } -rustack-cloudfront-core = { version = "0.9.0", path = "crates/rustack-cloudfront-core" } -rustack-cloudfront-dataplane = { version = "0.9.0", path = "crates/rustack-cloudfront-dataplane" } +rustack-cloudfront-http = { version = "0.10.0", path = "crates/rustack-cloudfront-http" } +rustack-cloudfront-core = { version = "0.10.0", path = "crates/rustack-cloudfront-core" } +rustack-cloudfront-dataplane = { version = "0.10.0", path = "crates/rustack-cloudfront-dataplane" } squib = "0.2.0" # Async runtime @@ -113,11 +113,15 @@ http = "1.4" http-body = "1.0" http-body-util = "0.1" +# Validated runtime configuration (YAML only). +config = { version = "0.15.25", default-features = false, features = ["yaml"] } +url = "2.5.8" + # Serialization serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" ciborium = "0.2" -quick-xml = "0.39" +quick-xml = "0.41" zstd = { version = "0.13.3", default-features = false } # Error handling diff --git a/Dockerfile b/Dockerfile index 937e706..c74760e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,181 +1,42 @@ # syntax=docker/dockerfile:1 -# --------------------------------------------------------------------------- -# Builder stage: compile a fully static binary with musl -# --------------------------------------------------------------------------- -FROM rust:1.94-slim AS builder - -RUN apt-get update && apt-get install -y musl-tools && rm -rf /var/lib/apt/lists/* - +# Retain the repository-selected toolchain without depending on a local +# rust-toolchain.toml (maintainers may delete it). rust:1.95 matches the +# repository pin; rustup adds the musl targets to that toolchain. +FROM rust:1.95-slim AS builder +RUN apt-get update && apt-get install -y --no-install-recommends musl-tools ca-certificates \ + && rm -rf /var/lib/apt/lists/* WORKDIR /src - -# Copy the toolchain file first so rustup installs the correct version, -# then add musl targets to the resolved toolchain. -COPY rust-toolchain.toml ./ RUN rustup target add x86_64-unknown-linux-musl aarch64-unknown-linux-musl - -# Copy manifests to cache dependency compilation. COPY Cargo.toml Cargo.lock ./ -COPY apps/rustack/Cargo.toml apps/rustack/Cargo.toml -COPY crates/rustack-auth/Cargo.toml crates/rustack-auth/Cargo.toml -COPY crates/rustack-core/Cargo.toml crates/rustack-core/Cargo.toml -COPY crates/rustack-s3-core/Cargo.toml crates/rustack-s3-core/Cargo.toml -COPY crates/rustack-s3-http/Cargo.toml crates/rustack-s3-http/Cargo.toml -COPY crates/rustack-s3-model/Cargo.toml crates/rustack-s3-model/Cargo.toml -COPY crates/rustack-s3-xml/Cargo.toml crates/rustack-s3-xml/Cargo.toml -COPY crates/rustack-dynamodb-model/Cargo.toml crates/rustack-dynamodb-model/Cargo.toml -COPY crates/rustack-dynamodb-http/Cargo.toml crates/rustack-dynamodb-http/Cargo.toml -COPY crates/rustack-dynamodb-core/Cargo.toml crates/rustack-dynamodb-core/Cargo.toml -COPY crates/rustack-dynamodbstreams-model/Cargo.toml crates/rustack-dynamodbstreams-model/Cargo.toml -COPY crates/rustack-dynamodbstreams-http/Cargo.toml crates/rustack-dynamodbstreams-http/Cargo.toml -COPY crates/rustack-dynamodbstreams-core/Cargo.toml crates/rustack-dynamodbstreams-core/Cargo.toml -COPY crates/rustack-sqs-model/Cargo.toml crates/rustack-sqs-model/Cargo.toml -COPY crates/rustack-sqs-http/Cargo.toml crates/rustack-sqs-http/Cargo.toml -COPY crates/rustack-sqs-core/Cargo.toml crates/rustack-sqs-core/Cargo.toml -COPY crates/rustack-ssm-model/Cargo.toml crates/rustack-ssm-model/Cargo.toml -COPY crates/rustack-ssm-http/Cargo.toml crates/rustack-ssm-http/Cargo.toml -COPY crates/rustack-ssm-core/Cargo.toml crates/rustack-ssm-core/Cargo.toml -COPY crates/rustack-sns-model/Cargo.toml crates/rustack-sns-model/Cargo.toml -COPY crates/rustack-sns-http/Cargo.toml crates/rustack-sns-http/Cargo.toml -COPY crates/rustack-sns-core/Cargo.toml crates/rustack-sns-core/Cargo.toml -COPY crates/rustack-lambda-model/Cargo.toml crates/rustack-lambda-model/Cargo.toml -COPY crates/rustack-lambda-http/Cargo.toml crates/rustack-lambda-http/Cargo.toml -COPY crates/rustack-lambda-core/Cargo.toml crates/rustack-lambda-core/Cargo.toml -COPY crates/rustack-events-model/Cargo.toml crates/rustack-events-model/Cargo.toml -COPY crates/rustack-events-http/Cargo.toml crates/rustack-events-http/Cargo.toml -COPY crates/rustack-events-core/Cargo.toml crates/rustack-events-core/Cargo.toml -COPY crates/rustack-logs-model/Cargo.toml crates/rustack-logs-model/Cargo.toml -COPY crates/rustack-logs-http/Cargo.toml crates/rustack-logs-http/Cargo.toml -COPY crates/rustack-logs-core/Cargo.toml crates/rustack-logs-core/Cargo.toml -COPY crates/rustack-kms-model/Cargo.toml crates/rustack-kms-model/Cargo.toml -COPY crates/rustack-kms-http/Cargo.toml crates/rustack-kms-http/Cargo.toml -COPY crates/rustack-kms-core/Cargo.toml crates/rustack-kms-core/Cargo.toml -COPY crates/rustack-kinesis-model/Cargo.toml crates/rustack-kinesis-model/Cargo.toml -COPY crates/rustack-kinesis-http/Cargo.toml crates/rustack-kinesis-http/Cargo.toml -COPY crates/rustack-kinesis-core/Cargo.toml crates/rustack-kinesis-core/Cargo.toml -COPY crates/rustack-secretsmanager-model/Cargo.toml crates/rustack-secretsmanager-model/Cargo.toml -COPY crates/rustack-secretsmanager-http/Cargo.toml crates/rustack-secretsmanager-http/Cargo.toml -COPY crates/rustack-secretsmanager-core/Cargo.toml crates/rustack-secretsmanager-core/Cargo.toml -COPY crates/rustack-ses-model/Cargo.toml crates/rustack-ses-model/Cargo.toml -COPY crates/rustack-ses-http/Cargo.toml crates/rustack-ses-http/Cargo.toml -COPY crates/rustack-ses-core/Cargo.toml crates/rustack-ses-core/Cargo.toml -COPY crates/rustack-apigatewayv2-model/Cargo.toml crates/rustack-apigatewayv2-model/Cargo.toml -COPY crates/rustack-apigatewayv2-http/Cargo.toml crates/rustack-apigatewayv2-http/Cargo.toml -COPY crates/rustack-apigatewayv2-core/Cargo.toml crates/rustack-apigatewayv2-core/Cargo.toml -COPY crates/rustack-cloudwatch-model/Cargo.toml crates/rustack-cloudwatch-model/Cargo.toml -COPY crates/rustack-cloudwatch-http/Cargo.toml crates/rustack-cloudwatch-http/Cargo.toml -COPY crates/rustack-cloudwatch-core/Cargo.toml crates/rustack-cloudwatch-core/Cargo.toml -COPY crates/rustack-iam-model/Cargo.toml crates/rustack-iam-model/Cargo.toml -COPY crates/rustack-iam-http/Cargo.toml crates/rustack-iam-http/Cargo.toml -COPY crates/rustack-iam-core/Cargo.toml crates/rustack-iam-core/Cargo.toml -COPY crates/rustack-sts-model/Cargo.toml crates/rustack-sts-model/Cargo.toml -COPY crates/rustack-sts-http/Cargo.toml crates/rustack-sts-http/Cargo.toml -COPY crates/rustack-sts-core/Cargo.toml crates/rustack-sts-core/Cargo.toml -COPY tests/integration/Cargo.toml tests/integration/Cargo.toml - -# Create stub sources so cargo can resolve the workspace. -RUN mkdir -p apps/rustack/src && echo 'fn main() {}' > apps/rustack/src/main.rs \ - && mkdir -p crates/rustack-auth/src && echo '//! stub' > crates/rustack-auth/src/lib.rs \ - && mkdir -p crates/rustack-core/src && echo '//! stub' > crates/rustack-core/src/lib.rs \ - && mkdir -p crates/rustack-s3-core/src && echo '//! stub' > crates/rustack-s3-core/src/lib.rs \ - && mkdir -p crates/rustack-s3-http/src && echo '//! stub' > crates/rustack-s3-http/src/lib.rs \ - && mkdir -p crates/rustack-s3-model/src && echo '//! stub' > crates/rustack-s3-model/src/lib.rs \ - && mkdir -p crates/rustack-s3-xml/src && echo '//! stub' > crates/rustack-s3-xml/src/lib.rs \ - && mkdir -p crates/rustack-dynamodb-model/src && echo '//! stub' > crates/rustack-dynamodb-model/src/lib.rs \ - && mkdir -p crates/rustack-dynamodb-http/src && echo '//! stub' > crates/rustack-dynamodb-http/src/lib.rs \ - && mkdir -p crates/rustack-dynamodb-core/src && echo '//! stub' > crates/rustack-dynamodb-core/src/lib.rs \ - && mkdir -p crates/rustack-dynamodbstreams-model/src && echo '//! stub' > crates/rustack-dynamodbstreams-model/src/lib.rs \ - && mkdir -p crates/rustack-dynamodbstreams-http/src && echo '//! stub' > crates/rustack-dynamodbstreams-http/src/lib.rs \ - && mkdir -p crates/rustack-dynamodbstreams-core/src && echo '//! stub' > crates/rustack-dynamodbstreams-core/src/lib.rs \ - && mkdir -p crates/rustack-sqs-model/src && echo '//! stub' > crates/rustack-sqs-model/src/lib.rs \ - && mkdir -p crates/rustack-sqs-http/src && echo '//! stub' > crates/rustack-sqs-http/src/lib.rs \ - && mkdir -p crates/rustack-sqs-core/src && echo '//! stub' > crates/rustack-sqs-core/src/lib.rs \ - && mkdir -p crates/rustack-ssm-model/src && echo '//! stub' > crates/rustack-ssm-model/src/lib.rs \ - && mkdir -p crates/rustack-ssm-http/src && echo '//! stub' > crates/rustack-ssm-http/src/lib.rs \ - && mkdir -p crates/rustack-ssm-core/src && echo '//! stub' > crates/rustack-ssm-core/src/lib.rs \ - && mkdir -p crates/rustack-sns-model/src && echo '//! stub' > crates/rustack-sns-model/src/lib.rs \ - && mkdir -p crates/rustack-sns-http/src && echo '//! stub' > crates/rustack-sns-http/src/lib.rs \ - && mkdir -p crates/rustack-sns-core/src && echo '//! stub' > crates/rustack-sns-core/src/lib.rs \ - && mkdir -p crates/rustack-lambda-model/src && echo '//! stub' > crates/rustack-lambda-model/src/lib.rs \ - && mkdir -p crates/rustack-lambda-http/src && echo '//! stub' > crates/rustack-lambda-http/src/lib.rs \ - && mkdir -p crates/rustack-lambda-core/src && echo '//! stub' > crates/rustack-lambda-core/src/lib.rs \ - && mkdir -p crates/rustack-events-model/src && echo '//! stub' > crates/rustack-events-model/src/lib.rs \ - && mkdir -p crates/rustack-events-http/src && echo '//! stub' > crates/rustack-events-http/src/lib.rs \ - && mkdir -p crates/rustack-events-core/src && echo '//! stub' > crates/rustack-events-core/src/lib.rs \ - && mkdir -p crates/rustack-logs-model/src && echo '//! stub' > crates/rustack-logs-model/src/lib.rs \ - && mkdir -p crates/rustack-logs-http/src && echo '//! stub' > crates/rustack-logs-http/src/lib.rs \ - && mkdir -p crates/rustack-logs-core/src && echo '//! stub' > crates/rustack-logs-core/src/lib.rs \ - && mkdir -p crates/rustack-kms-model/src && echo '//! stub' > crates/rustack-kms-model/src/lib.rs \ - && mkdir -p crates/rustack-kms-http/src && echo '//! stub' > crates/rustack-kms-http/src/lib.rs \ - && mkdir -p crates/rustack-kms-core/src && echo '//! stub' > crates/rustack-kms-core/src/lib.rs \ - && mkdir -p crates/rustack-kinesis-model/src && echo '//! stub' > crates/rustack-kinesis-model/src/lib.rs \ - && mkdir -p crates/rustack-kinesis-http/src && echo '//! stub' > crates/rustack-kinesis-http/src/lib.rs \ - && mkdir -p crates/rustack-kinesis-core/src && echo '//! stub' > crates/rustack-kinesis-core/src/lib.rs \ - && mkdir -p crates/rustack-secretsmanager-model/src && echo '//! stub' > crates/rustack-secretsmanager-model/src/lib.rs \ - && mkdir -p crates/rustack-secretsmanager-http/src && echo '//! stub' > crates/rustack-secretsmanager-http/src/lib.rs \ - && mkdir -p crates/rustack-secretsmanager-core/src && echo '//! stub' > crates/rustack-secretsmanager-core/src/lib.rs \ - && mkdir -p crates/rustack-ses-model/src && echo '//! stub' > crates/rustack-ses-model/src/lib.rs \ - && mkdir -p crates/rustack-ses-http/src && echo '//! stub' > crates/rustack-ses-http/src/lib.rs \ - && mkdir -p crates/rustack-ses-core/src && echo '//! stub' > crates/rustack-ses-core/src/lib.rs \ - && mkdir -p crates/rustack-apigatewayv2-model/src && echo '//! stub' > crates/rustack-apigatewayv2-model/src/lib.rs \ - && mkdir -p crates/rustack-apigatewayv2-http/src && echo '//! stub' > crates/rustack-apigatewayv2-http/src/lib.rs \ - && mkdir -p crates/rustack-apigatewayv2-core/src && echo '//! stub' > crates/rustack-apigatewayv2-core/src/lib.rs \ - && mkdir -p crates/rustack-cloudwatch-model/src && echo '//! stub' > crates/rustack-cloudwatch-model/src/lib.rs \ - && mkdir -p crates/rustack-cloudwatch-http/src && echo '//! stub' > crates/rustack-cloudwatch-http/src/lib.rs \ - && mkdir -p crates/rustack-cloudwatch-core/src && echo '//! stub' > crates/rustack-cloudwatch-core/src/lib.rs \ - && mkdir -p crates/rustack-iam-model/src && echo '//! stub' > crates/rustack-iam-model/src/lib.rs \ - && mkdir -p crates/rustack-iam-http/src && echo '//! stub' > crates/rustack-iam-http/src/lib.rs \ - && mkdir -p crates/rustack-iam-core/src && echo '//! stub' > crates/rustack-iam-core/src/lib.rs \ - && mkdir -p crates/rustack-sts-model/src && echo '//! stub' > crates/rustack-sts-model/src/lib.rs \ - && mkdir -p crates/rustack-sts-http/src && echo '//! stub' > crates/rustack-sts-http/src/lib.rs \ - && mkdir -p crates/rustack-sts-core/src && echo '//! stub' > crates/rustack-sts-core/src/lib.rs \ - && mkdir -p tests/integration/src && echo '//! stub' > tests/integration/src/lib.rs - -# Pre-build dependencies (cached layer). -ARG TARGETARCH -RUN --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/src/target \ - if [ "$TARGETARCH" = "arm64" ]; then \ - RUST_TARGET=aarch64-unknown-linux-musl; \ - else \ - RUST_TARGET=x86_64-unknown-linux-musl; \ - fi && \ - cargo build --release --target "$RUST_TARGET" -p rustack-cli 2>/dev/null || true - -# Copy real source code. COPY crates/ crates/ COPY apps/ apps/ COPY tests/ tests/ -# Touch source files so cargo knows they changed. -RUN find crates/ apps/ tests/ -name '*.rs' -exec touch {} + - -# Build the actual binary. +ARG TARGETARCH RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/src/target \ - if [ "$TARGETARCH" = "arm64" ]; then \ - RUST_TARGET=aarch64-unknown-linux-musl; \ - else \ - RUST_TARGET=x86_64-unknown-linux-musl; \ - fi && \ - cargo build --release --target "$RUST_TARGET" -p rustack-cli && \ + case "$TARGETARCH" in \ + arm64) RUST_TARGET=aarch64-unknown-linux-musl ;; \ + amd64) RUST_TARGET=x86_64-unknown-linux-musl ;; \ + *) echo "unsupported target architecture" >&2; exit 1 ;; \ + esac && \ + cargo build --locked --release --target "$RUST_TARGET" -p rustack-cli --bin rustack && \ cp "/src/target/$RUST_TARGET/release/rustack" /rustack +RUN mkdir -p /runtime/tmp /runtime/data && chmod 1777 /runtime/tmp -# --------------------------------------------------------------------------- -# Runtime stage: scratch image with just the binary -# --------------------------------------------------------------------------- FROM scratch - COPY --from=builder /rustack /rustack -COPY --from=builder /tmp /tmp - +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=builder --chown=65532:65532 /runtime/tmp /tmp +COPY --from=builder --chown=65532:65532 /runtime/data /data +WORKDIR /data +USER 65532:65532 +# Containers opt in to all-interface binding; bare binaries default to loopback. ENV GATEWAY_LISTEN=0.0.0.0:4566 ENV LOG_LEVEL=info ENV SERVICES= - EXPOSE 4566 - -HEALTHCHECK --interval=2s --timeout=3s --start-period=1s --retries=3 \ +HEALTHCHECK --interval=2s --timeout=4s --start-period=1s --retries=3 \ CMD ["/rustack", "--health-check"] - ENTRYPOINT ["/rustack"] diff --git a/Makefile b/Makefile index 4e66eba..ea71e02 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ deny: @cargo deny check run: - @cargo run -p rustack + @cargo run -p rustack-cli --bin rustack release: @cargo release tag --execute diff --git a/README.md b/README.md index 42f69c2..8f26c11 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Rustack -A high-performance, LocalStack-compatible AWS service emulator written in Rust. **18 services, 779 routed operations, <1s startup, ~8 MB Docker image.** +A local AWS service emulator written in Rust, with **18 service adapters**. Routed operation counts describe API surface—not full AWS semantics, authorization enforcement, execution support, or persistence. See the [capability and safety guide](docs/runtime-hardening.md) before choosing a test workload. ## Install @@ -12,13 +12,13 @@ cargo install rustack-cli cargo install --git https://github.com/tyrchen/rustack rustack-cli # Or use Docker -docker run -p 4566:4566 ghcr.io/tyrchen/rustack:latest +docker run --rm -p 127.0.0.1:4566:4566 ghcr.io/tyrchen/rustack:latest ``` ## Quick Start ```bash -# Start the server (all 18 services on port 4566) +# Start the server on loopback:4566 (all compiled services; Lambda execution disabled) rustack # Or start with specific services only @@ -42,10 +42,12 @@ services: rustack: image: ghcr.io/tyrchen/rustack:latest ports: - - "4566:4566" + - "127.0.0.1:4566:4566" environment: - SERVICES=s3,dynamodb,sqs,lambda - LOG_LEVEL=info + - GATEWAY_LISTEN=0.0.0.0:4566 + - RUSTACK_ADVERTISED_ENDPOINT=http://rustack:4566 app: build: . @@ -60,21 +62,17 @@ services: ## Why Rustack? -| | Rustack | LocalStack | -|---|---|---| -| **Language** | Rust (static binary) | Python | -| **Docker image** | ~8 MB (scratch) | ~475 MB / ~1.88 GB on disk | -| **Startup time** | < 1 second | 10-45s (S3 only); up to 2 min (all) | -| **Memory (idle)** | ~10 MB | ~750 MB minimum | -| **Services** | 18 | 80+ | -| **Operations** | 600+ | More per service, but behind paywall | -| **CI cold start** | Pull + ready in ~3s | Pull + ready in 30-90s | -| **Auth** | SigV4 + SigV2 + presigned URLs | SigV4 (Pro for IAM enforcement) | -| **License** | MIT, fully open source | Registration-required; free tier limited | - -## Supported Services - -| Service | Operations | Protocol | +- A single Rust binary for local SDK, CLI and infrastructure tests. +- Select services at build time and startup; inspect live readiness and capabilities. +- MIT licensed. Local emulation deliberately omits production IAM enforcement and isolation. + +Startup time, memory and image size depend on build features, workload, platform and snapshot size; historical measurements are not guarantees for this revision. + +## Service API Surface + +The counts below describe routed operations. **Metadata CRUD is not evidence that execution, delivery, policy enforcement or restart persistence is implemented.** Use the [capability and safety guide](docs/runtime-hardening.md) and `GET /_rustack/capabilities` for those boundaries. + +| Service | Routed operations | Protocol | |---------|-----------|----------| | **S3** | 71 | REST XML | | **DynamoDB** | 24 | awsJson 1.0 | @@ -284,11 +282,14 @@ DescribeStream, GetShardIterator, GetRecords, ListStreams ## Configuration -All settings are controlled via environment variables: +Set `RUSTACK_CONFIG` to a YAML file; environment values override the corresponding YAML environment entries. Unknown YAML keys, invalid values, unknown/uncompiled services, and strict signature validation without credentials fail startup. See [the complete configuration and migration guide](docs/runtime-hardening.md). | Variable | Default | Description | |----------|---------|-------------| -| `GATEWAY_LISTEN` | `0.0.0.0:4566` | Bind address and port | +| `GATEWAY_LISTEN` | `127.0.0.1:4566` | Bind address and port; container image explicitly binds `0.0.0.0` | +| `RUSTACK_CONFIG` | *(unset)* | YAML configuration file | +| `RUSTACK_ADVERTISED_ENDPOINT` | *(derived from bind address)* | Public local HTTP(S) authority for generated URLs | +| `LAMBDA_EXECUTOR` | `disabled` | Explicit `native`/`squib` selection required for execution; Docker unsupported | | `SERVICES` | *(empty = all)* | Comma-separated list of services to enable | | `LOG_LEVEL` | `info` | Log level (`error`, `warn`, `info`, `debug`, `trace`) | | `RUST_LOG` | | Fine-grained tracing filter (overrides `LOG_LEVEL`) | @@ -309,7 +310,7 @@ SERVICES=s3,dynamodb,sqs rustack **Compile-time** — exclude services from the binary entirely: ```bash -cargo build -p rustack --no-default-features --features s3,dynamodb +cargo build -p rustack-cli --bin rustack --no-default-features --features s3,dynamodb ``` Available features: `s3`, `dynamodb`, `dynamodbstreams`, `sqs`, `ssm`, `sns`, `lambda`, `events`, `logs`, `kms`, `kinesis`, `secretsmanager`, `ses`, `apigatewayv2`, `cloudfront`, `cloudfront-dataplane`, `cloudwatch`, `iam`, `sts` @@ -366,7 +367,7 @@ Each service follows the same three-crate pattern. The unified server binary (`r ## Development -**Prerequisites:** Rust 1.93+ (pinned in `rust-toolchain.toml`) +**Prerequisites:** the repository-selected stable Rust toolchain, plus nightly rustfmt. Use an explicit `cargo +` when intentionally keeping a local toolchain override; do not infer compatibility from old minimum-version claims. ```bash make build # Compile all crates diff --git a/apps/rustack/Cargo.toml b/apps/rustack/Cargo.toml index 7a3649d..9f9f8ea 100644 --- a/apps/rustack/Cargo.toml +++ b/apps/rustack/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-cli" description = "A high-performance, LocalStack-compatible AWS service emulator written in Rust — 18 services, <1s startup, ~8 MB Docker image" -version = "0.9.1" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -121,7 +121,7 @@ rustack-cloudfront-dataplane = { workspace = true, optional = true } tokio = { workspace = true } # HTTP -hyper = { workspace = true } +hyper = { workspace = true, features = ["client"] } hyper-util = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } diff --git a/apps/rustack/src/events_bridge.rs b/apps/rustack/src/events_bridge.rs index 6ca5af5..cfd4208 100644 --- a/apps/rustack/src/events_bridge.rs +++ b/apps/rustack/src/events_bridge.rs @@ -1,96 +1,132 @@ -//! Bridge between EventBridge and SQS for target delivery. -//! -//! Implements the [`TargetDelivery`] trait from `rustack-events-core` by wrapping -//! the actual SQS provider. This bridge lives in the server binary to avoid -//! a direct dependency from `rustack-events-core` to `rustack-sqs-core`. - +//! Application-owned EventBridge → SQS delivery bridge. use std::sync::Arc; use async_trait::async_trait; -use rustack_events_core::delivery::{DeliveryError, TargetDelivery}; +use rustack_events_core::delivery::{DeliveryError, Target, TargetDelivery}; use rustack_sqs_core::provider::RustackSqs; use rustack_sqs_model::input::SendMessageInput; -/// Production target delivery that routes events to SQS queues. +/// Actual in-process SQS target delivery, preserving all FIFO parameters. #[derive(Debug)] pub struct LocalTargetDelivery { sqs: Arc, - account_id: String, - host: String, - port: u16, } - impl LocalTargetDelivery { - /// Create a new delivery bridge wrapping the given SQS provider. - pub fn new(sqs: Arc, account_id: String, host: String, port: u16) -> Self { - Self { - sqs, - account_id, - host, - port, - } + /// Connect the bridge to an enabled SQS provider. + pub fn new(sqs: Arc) -> Self { + Self { sqs } } - - /// Convert an SQS queue ARN to a queue URL. - /// - /// ARN format: `arn:aws:sqs:{region}:{account}:{queue_name}` - /// URL format: `http://{host}:{port}/{account}/{queue_name}` - fn arn_to_queue_url(&self, queue_arn: &str) -> String { - let parts: Vec<&str> = queue_arn.split(':').collect(); - if parts.len() >= 6 { - let account = parts[4]; - let queue_name = parts[5]; - format!("http://{}:{}/{account}/{queue_name}", self.host, self.port) - } else { - format!( - "http://{}:{}/{}/{}", - self.host, self.port, self.account_id, queue_arn - ) - } + fn arn_to_queue_url(&self, arn: &str) -> Result { + self.sqs + .queue_url_for_arn(arn) + .map_err(|error| DeliveryError::InvalidArn(error.to_string())) } } - #[async_trait] impl TargetDelivery for LocalTargetDelivery { - async fn deliver(&self, target_arn: &str, event_json: &str) -> Result<(), DeliveryError> { - if target_arn.contains(":sqs:") { - let queue_url = self.arn_to_queue_url(target_arn); - let input = SendMessageInput { - queue_url, - message_body: event_json.to_string(), - ..SendMessageInput::default() - }; - self.sqs - .send_message(input) - .await - .map_err(|e| DeliveryError::TargetError(e.to_string()))?; - Ok(()) - } else { - tracing::debug!( - target_arn = %target_arn, - "unsupported target type, event not delivered" - ); - Ok(()) + fn validate(&self, target: &Target) -> Result<(), DeliveryError> { + if let ["arn", _, service, _, _, _] = target.arn.split(':').collect::>().as_slice() { + if *service != "sqs" { + return Err(DeliveryError::Unsupported( + "Only SQS targets are executable".into(), + )); + } } + self.arn_to_queue_url(&target.arn)?; + let fifo = std::path::Path::new(&target.arn) + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("fifo")); + if fifo && target.sqs_parameters.is_none() { + return Err(DeliveryError::Unsupported( + "FIFO target requires SqsParameters.MessageGroupId".into(), + )); + } + if !fifo && target.sqs_parameters.is_some() { + return Err(DeliveryError::Unsupported( + "SqsParameters requires a FIFO queue".into(), + )); + } + Ok(()) + } + async fn deliver(&self, target: &Target, event_json: &str) -> Result<(), DeliveryError> { + self.validate(target)?; + self.sqs + .send_message(SendMessageInput { + queue_url: self.arn_to_queue_url(&target.arn)?, + message_body: event_json.to_owned(), + message_group_id: target + .sqs_parameters + .as_ref() + .map(|parameters| parameters.message_group_id.clone()), + ..Default::default() + }) + .await + .map_err(|error| DeliveryError::TargetError(error.to_string()))?; + Ok(()) } } #[cfg(test)] mod tests { + use rustack_events_core::{config::EventsConfig, provider::RustackEvents}; use rustack_sqs_core::config::SqsConfig; + use rustack_sqs_model::input::{CreateQueueInput, ReceiveMessageInput}; use super::*; - #[test] - fn test_should_convert_arn_to_queue_url() { - let delivery = LocalTargetDelivery { - sqs: Arc::new(RustackSqs::new(SqsConfig::default())), - account_id: "000000000000".to_string(), - host: "localhost".to_string(), - port: 4566, - }; - - let url = delivery.arn_to_queue_url("arn:aws:sqs:us-east-1:000000000000:my-queue"); - assert_eq!(url, "http://localhost:4566/000000000000/my-queue"); + #[tokio::test] + async fn test_should_roundtrip_sqs_parameters_and_deliver_fifo_group() { + let sqs = Arc::new(RustackSqs::new(SqsConfig::default())); + let queue = sqs.create_queue(serde_json::from_value::(serde_json::json!({ + "QueueName":"events.fifo", "Attributes":{"FifoQueue":"true","ContentBasedDeduplication":"true"} + })).unwrap()).await.unwrap().queue_url.unwrap(); + let bridge = Arc::new(LocalTargetDelivery::new(sqs.clone())); + let events = RustackEvents::new(EventsConfig::default(), bridge); + events + .handle_put_rule( + serde_json::from_value( + serde_json::json!({"Name":"rule", "EventPattern":"{\"source\":[\"test\"]}"}), + ) + .unwrap(), + ) + .unwrap(); + let target = serde_json::json!({"Id":"fifo","Arn":"arn:aws:sqs:us-east-1:000000000000:events.fifo", "SqsParameters":{"MessageGroupId":"g"}}); + let result = events + .handle_put_targets( + serde_json::from_value( + serde_json::json!({"Rule":"rule","Targets":[target.clone()]}), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(result.failed_entry_count, 0); + let listed = events + .handle_list_targets_by_rule( + &serde_json::from_value(serde_json::json!({"Rule":"rule"})).unwrap(), + ) + .unwrap(); + assert_eq!(serde_json::to_value(&listed.targets[0]).unwrap(), target); + let result = events.handle_put_events(&serde_json::from_value(serde_json::json!({"Entries":[{"Source":"test","DetailType":"test","Detail":"{}"}]})).unwrap()).unwrap(); + assert_eq!(result.failed_entry_count, 0); + events.quiesce().await.unwrap(); + let messages = sqs + .receive_message(ReceiveMessageInput { + queue_url: queue, + message_system_attribute_names: vec!["All".into()], + ..Default::default() + }) + .await + .unwrap() + .messages; + assert_eq!(messages.len(), 1); + assert_eq!( + messages[0] + .attributes + .get("MessageGroupId") + .map(String::as_str), + Some("g") + ); + assert_eq!(events.delivery_stats().delivered, 1); + sqs.shutdown_all().await; } } diff --git a/apps/rustack/src/gateway.rs b/apps/rustack/src/gateway.rs index 6f5bf71..2f87ec8 100644 --- a/apps/rustack/src/gateway.rs +++ b/apps/rustack/src/gateway.rs @@ -1,109 +1,345 @@ -//! Gateway service that routes requests to registered AWS services. -//! -//! The gateway holds an ordered list of [`ServiceRouter`] implementations and -//! dispatches each request to the first router whose [`matches`](ServiceRouter::matches) -//! method returns `true`. If no router matches, a 404 response is returned. -//! -//! Health-check endpoints (`/_localstack/health`, `/_health`, `/health`) are -//! intercepted at the gateway level and return a combined status for all -//! registered services. - -use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; - -use hyper::{body::Incoming, service::Service}; - -use crate::service::{GatewayBody, ServiceRouter, gateway_body_from_string}; - -/// Gateway that routes incoming HTTP requests to registered service routers. -/// -/// Services are tried in registration order; the first whose -/// [`ServiceRouter::matches`] returns `true` handles the request. Register -/// specific services (DynamoDB, etc.) before catch-all services (S3). -pub struct GatewayService { +//! Gateway admission, routing, and explicit runtime health/capability reporting. + +use std::{ + convert::Infallible, + future::Future, + io, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + task::{Context, Poll}, + time::Duration, +}; + +use bytes::Bytes; +use http::{ + Method, Response, StatusCode, + header::{CONTENT_TYPE, RETRY_AFTER}, +}; +use http_body_util::BodyExt; +use hyper::{ + body::{Body, Frame, Incoming, SizeHint}, + service::Service, +}; +use rustack_core::{ + http::{BodyBudget, BudgetedBody}, + settings::RuntimeBudgets, +}; +use serde::Serialize; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +use crate::{ + runtime::RuntimeWorkers, + service::{GatewayBody, ServiceRouter, gateway_body_from_string}, +}; + +/// Process state is independent from whether any service supports snapshots. +#[derive(Debug)] +pub(crate) struct RuntimeStatus { + ready: AtomicBool, + workers: Arc, +} + +impl RuntimeStatus { + pub(crate) fn drain(&self) { + self.ready.store(false, Ordering::Release); + } +} + +/// Service capability description: registration is not full AWS compatibility. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct Capability { + name: &'static str, + status: &'static str, + snapshot: &'static str, + execution: &'static str, +} + +/// Routable services with shared process-level in-flight admission. +#[derive(Clone)] +pub(crate) struct GatewayService { services: Arc>>, + admission: Arc, + budgets: RuntimeBudgets, + state: Arc, + health_only: bool, } impl GatewayService { - /// Create a new gateway from a list of service routers. - pub fn new(services: Vec>) -> Self { + pub(crate) fn new(services: Vec>) -> Self { + let budgets = rustack_core::settings::budgets(); Self { services: Arc::new(services), + admission: Arc::new(Semaphore::new(budgets.requests)), + budgets, + state: Arc::new(RuntimeStatus { + ready: AtomicBool::new(true), + workers: Arc::new(RuntimeWorkers::default()), + }), + health_only: false, } } - /// Return the names of all registered services. - pub fn service_names(&self) -> Vec<&'static str> { - self.services.iter().map(|s| s.name()).collect() + pub(crate) fn with_workers(mut self, workers: Arc) -> Self { + self.state = Arc::new(RuntimeStatus { + ready: AtomicBool::new(true), + workers, + }); + self } -} -impl Clone for GatewayService { - fn clone(&self) -> Self { - Self { - services: Arc::clone(&self.services), + pub(crate) fn service_names(&self) -> Vec<&'static str> { + self.services.iter().map(|service| service.name()).collect() + } + + pub(crate) fn state(&self) -> Arc { + Arc::clone(&self.state) + } + + /// Reserved connections only serve diagnostics, never additional business load. + pub(crate) fn health_only(mut self) -> Self { + self.health_only = true; + self + } + + fn diagnostics(&self, req: &http::Request) -> Option> { + let path = req.uri().path(); + if !(is_health_check(req.method(), path) + || matches!(*req.method(), Method::GET | Method::HEAD) + && path == "/_rustack/capabilities") + { + return None; + } + let workers = self.state.workers.diagnostics(); + let ready = + self.state.ready.load(Ordering::Acquire) && workers.ready && !self.services.is_empty(); + let status = if path == "/_health/live" || path == "/minio/health/live" || ready { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }; + let draining = !self.state.ready.load(Ordering::Acquire); + let mut response = if path == "/_rustack/capabilities" { + let capabilities: Vec = self + .services + .iter() + .map(|service| capability(service.name())) + .collect(); + json_response( + status, + &serde_json::json!({"version": env!("CARGO_PKG_VERSION"), "ready": ready, "services": capabilities, "workers": workers, "localEndpoints": {"apiGateway": "/_aws/execute-api/{apiId}/{stage}", "lambda": "/lambda-url/{functionName}/", "cloudFront": "/_aws/cloudfront/{distributionId}/"}, "limitations": ["No multi-tenant isolation or production IAM enforcement", "Snapshot resources-only services do not retain messages", "routed does not imply complete operation semantics"]}), + ) + } else { + let services: std::collections::BTreeMap<_, _> = self + .services + .iter() + .map(|service| { + ( + service.name(), + service_status(service.name(), draining, &workers), + ) + }) + .collect(); + json_response( + status, + &serde_json::json!({"version": env!("CARGO_PKG_VERSION"), "ready": ready, "services": services, "workers": workers}), + ) + }; + if *req.method() == Method::HEAD { + *response.body_mut() = gateway_body_from_string(""); } + Some(response) } } impl Service> for GatewayService { - type Response = http::Response; + type Response = Response; type Error = Infallible; type Future = Pin> + Send>>; fn call(&self, req: http::Request) -> Self::Future { - // Intercept health checks at the gateway level. - if is_health_check(req.method(), req.uri().path()) { - let services = Arc::clone(&self.services); - return Box::pin(async move { - let names: Vec<&str> = services.iter().map(|s| s.name()).collect(); - Ok(health_check_response(&names)) + if let Some(response) = self.diagnostics(&req) { + return Box::pin(async { Ok(response) }); + } + if self.health_only || !self.state.ready.load(Ordering::Acquire) { + return Box::pin(async { + Ok(unavailable( + "runtime is draining or connection capacity is full", + )) }); } + let Ok(permit) = Arc::clone(&self.admission).try_acquire_owned() else { + return Box::pin(async { Ok(unavailable("request capacity is full")) }); + }; + let selected = self.services.iter().find(|service| service.matches(&req)); + // Object streams and synchronous invocations exceed the control deadline. + let request_seconds = match selected.map(|service| service.name()) { + Some("s3") => self.budgets.s3_body_total_seconds, + Some("lambda") + if req.uri().path().ends_with("/invocations") + || req.uri().path().starts_with("/lambda-url/") => + { + self.budgets.lambda_invoke_seconds + } + _ => self.budgets.request_seconds, + }; + let service_future = selected.map(|service| service.call(req)); + let budget = self.budgets.clone(); + Box::pin(async move { + let response = match service_future { + Some(future) => { + match tokio::time::timeout(Duration::from_secs(request_seconds), future).await { + Ok(Ok(response)) => response, + Err(_) => json_response( + StatusCode::GATEWAY_TIMEOUT, + &serde_json::json!({"error": "request deadline exceeded"}), + ), + Ok(Err(never)) => match never {}, + } + } + None => json_response( + StatusCode::NOT_FOUND, + &serde_json::json!({"error": "no service matched the request"}), + ), + }; + let Ok(response_budget) = BodyBudget::new( + budget.s3_object_body_bytes, + Duration::from_secs(budget.s3_body_total_seconds), + Duration::from_secs(budget.body_idle_seconds), + ) else { + return Ok(json_response( + StatusCode::INTERNAL_SERVER_ERROR, + &serde_json::json!({"error": "invalid response budget"}), + )); + }; + Ok(response.map(|body| { + PermitBody { + inner: BudgetedBody::new(body, response_budget), + permit: Some(permit), + } + .boxed() + })) + }) + } +} + +struct PermitBody { + inner: BudgetedBody, + permit: Option, +} + +impl Body for PermitBody { + type Data = Bytes; + type Error = io::Error; - // Route to the first matching service. - for svc in self.services.iter() { - if svc.matches(&req) { - return svc.call(req); + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + let this = self.get_mut(); + match Pin::new(&mut this.inner).poll_frame(cx) { + Poll::Ready(None) => { + this.permit.take(); + Poll::Ready(None) + } + Poll::Ready(Some(Err(error))) => { + this.permit.take(); + Poll::Ready(Some(Err(io::Error::other(error)))) } + other => other.map(|frame| frame.map(|result| result.map_err(io::Error::other))), } + } + fn is_end_stream(&self) -> bool { + self.inner.is_end_stream() + } + fn size_hint(&self) -> SizeHint { + self.inner.size_hint() + } +} - // No service matched — return a 404. - Box::pin(async { - Ok(http::Response::builder() - .status(http::StatusCode::NOT_FOUND) - .header("Content-Type", "application/json") - .body(gateway_body_from_string( - r#"{"error":"no service matched the request"}"#, - )) - .expect("static 404 response should be valid")) - }) +fn capability(name: &'static str) -> Capability { + let snapshot = crate::snapshot::snapshot_coverage(name); + let execution = match name { + "lambda" => "explicit backend required; native is trusted-code-only", + "sns" | "events" => "SQS targets only; target availability validated", + _ => "partial AWS compatibility; see service capability documentation", + }; + Capability { + name, + status: "partial", + snapshot, + execution, } } -/// Check if the request is a health check probe. -fn is_health_check(method: &http::Method, path: &str) -> bool { - *method == http::Method::GET - && (path == "/_localstack/health" - || path == "/_health" - || path == "/health" - || path == "/minio/health/live" - || path == "/minio/health/ready" - || path == "/minio/health/cluster") +/// Derive the health state of one registered service. Disabled is an explicit +/// capability (e.g. Lambda without an execution backend), not a failure. +fn service_status( + name: &'static str, + draining: bool, + workers: &crate::runtime::WorkerDiagnostics, +) -> &'static str { + if draining { + return "draining"; + } + if let Some(status) = workers.services.get(name) { + return status; + } + if name == "lambda" + && rustack_core::settings::var("LAMBDA_EXECUTOR") + .ok() + .as_deref() + == Some("disabled") + { + return "disabled"; + } + "running" } -/// Produce a health check response listing all registered services. -fn health_check_response(service_names: &[&str]) -> http::Response { - let entries: Vec = service_names - .iter() - .map(|name| format!(r#""{name}":"running""#)) - .collect(); - let body = format!(r#"{{"services":{{{}}}}}"#, entries.join(",")); - - http::Response::builder() - .status(http::StatusCode::OK) - .header("Content-Type", "application/json") - .body(gateway_body_from_string(body)) - .expect("static health response should be valid") +fn unavailable(message: &str) -> Response { + let mut response = json_response( + StatusCode::SERVICE_UNAVAILABLE, + &serde_json::json!({"error": message}), + ); + response + .headers_mut() + .insert(RETRY_AFTER, http::HeaderValue::from_static("1")); + response +} + +fn json_response(status: StatusCode, value: &serde_json::Value) -> Response { + let (status, body) = match serde_json::to_string(value) { + Ok(body) => (status, body), + Err(_) => ( + StatusCode::INTERNAL_SERVER_ERROR, + "{\"error\":\"diagnostic serialization failed\"}".to_owned(), + ), + }; + let mut response = Response::new(gateway_body_from_string(body)); + *response.status_mut() = status; + response.headers_mut().insert( + CONTENT_TYPE, + http::HeaderValue::from_static("application/json"), + ); + response +} + +fn is_health_check(method: &Method, path: &str) -> bool { + matches!(*method, Method::GET | Method::HEAD) + && matches!( + path, + "/_localstack/health" + | "/_health" + | "/health" + | "/_health/live" + | "/_health/ready" + | "/minio/health/live" + | "/minio/health/ready" + | "/minio/health/cluster" + ) } #[cfg(test)] @@ -111,38 +347,32 @@ mod tests { use super::*; #[test] - fn test_should_detect_health_check_paths() { - assert!(is_health_check(&http::Method::GET, "/_localstack/health")); - assert!(is_health_check(&http::Method::GET, "/_health")); - assert!(is_health_check(&http::Method::GET, "/health")); - assert!(!is_health_check(&http::Method::POST, "/_health")); - assert!(!is_health_check(&http::Method::GET, "/mybucket")); + fn test_should_support_head_and_distinct_health_paths() { + assert!(is_health_check(&Method::HEAD, "/_health/ready")); + assert!(is_health_check(&Method::GET, "/_health/live")); + assert!(!is_health_check(&Method::POST, "/_health")); + assert!(!is_health_check(&Method::GET, "/bucket")); } #[test] - fn test_should_produce_health_check_response_with_both_services() { - let names = vec!["s3", "dynamodb"]; - let resp = health_check_response(&names); - assert_eq!(resp.status(), http::StatusCode::OK); - assert_eq!( - resp.headers() - .get("Content-Type") - .and_then(|v| v.to_str().ok()), - Some("application/json"), - ); + fn test_should_report_snapshot_limits_without_full_support_claim() { + assert_eq!(capability("sqs").snapshot, "resources-only"); + assert_eq!(capability("sns").snapshot, "unsupported"); + assert_eq!(capability("lambda").status, "partial"); } - #[test] - fn test_should_produce_health_check_response_with_single_service() { - let names = vec!["dynamodb"]; - let resp = health_check_response(&names); - assert_eq!(resp.status(), http::StatusCode::OK); - } - - #[test] - fn test_should_produce_health_check_response_with_no_services() { - let names: Vec<&str> = vec![]; - let resp = health_check_response(&names); - assert_eq!(resp.status(), http::StatusCode::OK); + #[tokio::test] + async fn test_should_keep_permit_until_response_body_is_consumed() + -> Result<(), Box> { + let admission = Arc::new(Semaphore::new(1)); + let permit = Arc::clone(&admission).acquire_owned().await?; + let body = PermitBody { + inner: BudgetedBody::new(gateway_body_from_string("data"), BodyBudget::control()), + permit: Some(permit), + }; + assert_eq!(admission.available_permits(), 0); + let _ = body.collect().await?; + assert_eq!(admission.available_permits(), 1); + Ok(()) } } diff --git a/apps/rustack/src/handler.rs b/apps/rustack/src/handler.rs index a37fadb..00e437b 100644 --- a/apps/rustack/src/handler.rs +++ b/apps/rustack/src/handler.rs @@ -25,6 +25,16 @@ use rustack_s3_model::{ pub struct RustackHandler(pub RustackS3); impl S3Handler for RustackHandler { + fn handle_staged_upload( + &self, + mut parts: http::request::Parts, + upload: std::sync::Arc, + ctx: RoutingContext, + ) -> Pin, S3Error>> + Send>> { + parts.extensions.insert(upload); + self.handle_operation(ctx.operation, parts, Bytes::new(), ctx) + } + // This function dispatches all S3 operations via a match expression. Each arm // is a single-line delegation, so the overall line count is proportional to // the number of S3 operations rather than logic complexity. @@ -319,16 +329,43 @@ impl S3Handler for RustackHandler { // Object CRUD // --------------------------------------------------------------- S3Operation::PutObject => { - dispatch_output(&parts, bucket, key, query_params, body, |input| { - provider.handle_put_object(input) - }) + let upload = parts + .extensions + .get::>() + .cloned(); + dispatch_output( + &parts, + bucket, + key, + query_params, + body, + |input| async move { + match upload { + Some(upload) => { + provider.handle_put_object_staged(input, upload).await + } + None => provider.handle_put_object(input).await, + } + }, + ) .await } S3Operation::GetObject => { - dispatch_output(&parts, bucket, key, query_params, body, |input| { - provider.handle_get_object(input) - }) - .await + let input = rustack_s3_model::input::GetObjectInput::from_s3_request( + &parts, + bucket, + key, + query_params, + body, + )?; + let (output, staged) = provider.handle_get_object_streaming(input).await?; + let mut response = output.into_s3_response()?; + if let Some(staged) = staged { + *response.body_mut() = S3ResponseBody::from_staged(staged) + .await + .map_err(|error| S3Error::internal_error(error.to_string()))?; + } + Ok(response) } S3Operation::HeadObject => { dispatch_output(&parts, bucket, key, query_params, body, |input| { @@ -454,9 +491,25 @@ impl S3Handler for RustackHandler { .await } S3Operation::UploadPart => { - dispatch_output(&parts, bucket, key, query_params, body, |input| { - provider.handle_upload_part(input) - }) + let upload = parts + .extensions + .get::>() + .cloned(); + dispatch_output( + &parts, + bucket, + key, + query_params, + body, + |input| async move { + match upload { + Some(upload) => { + provider.handle_upload_part_staged(input, upload).await + } + None => provider.handle_upload_part(input).await, + } + }, + ) .await } S3Operation::UploadPartCopy => { diff --git a/apps/rustack/src/main.rs b/apps/rustack/src/main.rs index e9c3cee..095708e 100644 --- a/apps/rustack/src/main.rs +++ b/apps/rustack/src/main.rs @@ -23,7 +23,7 @@ //! //! | Variable | Default | Description | //! |----------|---------|-------------| -//! | `GATEWAY_LISTEN` | `0.0.0.0:4566` | Bind address | +//! | `GATEWAY_LISTEN` | `127.0.0.1:4566` | Bind address | //! | `SERVICES` | *(empty = all)* | Comma-separated list of services to enable | //! | `_SKIP_SIGNATURE_VALIDATION` | `true` | Skip SigV4 verification for `` | //! | `S3_DOMAIN` | `s3.localhost.localstack.cloud` | Virtual hosting domain | @@ -38,16 +38,17 @@ mod gateway; mod handler; #[cfg(all(feature = "lambda", feature = "s3"))] mod lambda_s3_bridge; +mod runtime; mod service; mod snapshot; #[cfg(feature = "sns")] mod sns_bridge; -use std::{net::SocketAddr, sync::Arc}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; use anyhow::{Context, Result}; use hyper_util::{ - rt::{TokioExecutor, TokioIo}, + rt::{TokioExecutor, TokioIo, TokioTimer}, server::conn::auto::Builder as HttpConnBuilder, }; #[cfg(feature = "apigatewayv2")] @@ -72,6 +73,7 @@ use rustack_cloudwatch_core::handler::RustackCloudWatchHandler; use rustack_cloudwatch_core::provider::RustackCloudWatch; #[cfg(feature = "cloudwatch")] use rustack_cloudwatch_http::service::{CloudWatchHttpConfig, CloudWatchHttpService}; +use rustack_core::settings; #[cfg(feature = "dynamodb")] use rustack_dynamodb_core::config::DynamoDBConfig; #[cfg(feature = "dynamodb")] @@ -202,7 +204,7 @@ use rustack_sts_core::handler::RustackStsHandler; use rustack_sts_core::provider::RustackSts; #[cfg(feature = "sts")] use rustack_sts_http::service::{StsHttpConfig, StsHttpService}; -use tokio::net::TcpListener; +use tokio::{net::TcpListener, sync::Semaphore, task::JoinSet, time::Instant}; use tracing::{info, warn}; use tracing_subscriber::EnvFilter; @@ -212,6 +214,7 @@ use crate::events_bridge::LocalTargetDelivery; use crate::sns_bridge::RustackSqsPublisher; use crate::{ gateway::GatewayService, + runtime::RuntimeWorkers, service::ServiceRouter, snapshot::{RuntimeProviders, SnapshotConfig}, }; @@ -250,9 +253,9 @@ enum CliAction { /// Classify a sequence of CLI arguments (including argv\[0\]) into a [`CliAction`]. /// -/// Precedence: `--help` > `--version` > `--health-check`. Positional args are -/// ignored (the binary takes no positional arguments today). Any other token -/// that starts with `-` is treated as an unknown flag. +/// Precedence: `--help` > `--version` > `--health-check`. The binary takes no +/// positional arguments: any unexpected token (including unknown flags) is +/// rejected as an unknown flag. fn classify_args(args: I) -> CliAction where I: IntoIterator, @@ -277,7 +280,14 @@ where match arg.as_str() { "-h" | "--help" => return CliAction::Help, "-v" | "--version" => return CliAction::Version, - "--health-check" => action = CliAction::HealthCheck, + "--health-check" => { + if !matches!(action, CliAction::Run { snapshot: None }) { + return CliAction::UnknownFlag( + "conflicting --health-check/--snapshot options".to_owned(), + ); + } + action = CliAction::HealthCheck; + } "--snapshot" => { let Some(name) = iter.next() else { return CliAction::MissingFlagValue("--snapshot".to_owned()); @@ -285,8 +295,13 @@ where if name.starts_with('-') { return CliAction::MissingFlagValue("--snapshot".to_owned()); } - if let CliAction::Run { snapshot } = &mut action { - *snapshot = Some(name); + match &mut action { + CliAction::Run { snapshot } if snapshot.is_none() => *snapshot = Some(name), + _ => { + return CliAction::UnknownFlag( + "duplicate or conflicting --snapshot option".to_owned(), + ); + } } } other if other.starts_with("--snapshot=") => { @@ -296,14 +311,18 @@ where if name.is_empty() { return CliAction::MissingFlagValue("--snapshot".to_owned()); } - if let CliAction::Run { snapshot } = &mut action { - *snapshot = Some(name.to_owned()); + match &mut action { + CliAction::Run { snapshot } if snapshot.is_none() => { + *snapshot = Some(name.to_owned()); + } + _ => { + return CliAction::UnknownFlag( + "duplicate or conflicting --snapshot option".to_owned(), + ); + } } } - other if other.starts_with('-') => { - return CliAction::UnknownFlag(other.to_string()); - } - _ => {} + other => return CliAction::UnknownFlag(other.to_string()), } } action @@ -326,7 +345,7 @@ fn help_text() -> String { --snapshot Load snapshot before serving and save it on shutdown\n\ \n\ ENVIRONMENT:\n \ - GATEWAY_LISTEN Bind address (default: 0.0.0.0:4566)\n \ + GATEWAY_LISTEN Bind address (default: 127.0.0.1:4566)\n \ SERVICES Comma-separated list of services to enable (default: all compiled-in)\n \ LOG_LEVEL Log level filter (default: info)\n \ RUST_LOG Fine-grained tracing filter (overrides LOG_LEVEL)\n \ @@ -347,13 +366,14 @@ fn version_text() -> String { /// Initialize the tracing subscriber. /// -/// Uses `RUST_LOG` if set, otherwise falls back to the `LOG_LEVEL` config value. +/// Uses the validated `RUST_LOG` setting if present, otherwise falls back to the +/// `LOG_LEVEL` config value. The facade value is authoritative so YAML-set +/// filters work without exporting them into the real process environment. fn init_tracing(log_level: &str) -> Result<()> { - let filter = if std::env::var("RUST_LOG").is_ok() { - EnvFilter::from_default_env() - } else { - EnvFilter::try_new(log_level) - .with_context(|| format!("invalid log level filter: {log_level}"))? + let filter = match settings::var("RUST_LOG") { + Ok(filter) => EnvFilter::try_new(filter).with_context(|| "invalid RUST_LOG filter")?, + Err(_) => EnvFilter::try_new(log_level) + .with_context(|| format!("invalid log level filter: {log_level}"))?, }; tracing_subscriber::fmt() @@ -608,69 +628,93 @@ fn build_sts_http_config(config: &StsConfig) -> StsHttpConfig { fn build_credential_provider() -> Option> { use rustack_auth::StaticCredentialProvider; - let access_key = std::env::var("ACCESS_KEY") - .or_else(|_| std::env::var("AWS_ACCESS_KEY_ID")) + let access_key = settings::var("ACCESS_KEY") + .or_else(|_| settings::var("AWS_ACCESS_KEY_ID")) .ok()?; - let secret_key = std::env::var("SECRET_KEY") - .or_else(|_| std::env::var("AWS_SECRET_ACCESS_KEY")) + let secret_key = settings::var("SECRET_KEY") + .or_else(|_| settings::var("AWS_SECRET_ACCESS_KEY")) .ok()?; - info!( - access_key = %access_key, - "configured credential provider from environment" - ); + info!("configured credential provider (values redacted)"); Some(Arc::new(StaticCredentialProvider::new(vec![( access_key, secret_key, )]))) } -/// Run the accept loop, serving connections until a shutdown signal is received. -async fn serve(listener: TcpListener, service: GatewayService) -> Result<()> { +/// Accept bounded connections, then return the shared shutdown deadline and drain result. +async fn serve(listener: TcpListener, service: GatewayService) -> Result<(Instant, Result<()>)> { + let budgets = settings::budgets(); let graceful = hyper_util::server::graceful::GracefulShutdown::new(); - let http = HttpConnBuilder::new(TokioExecutor::new()); - + let mut http = HttpConnBuilder::new(TokioExecutor::new()); + http.http1() + .timer(TokioTimer::new()) + .header_read_timeout(Duration::from_secs(budgets.header_seconds)); + http.http2().max_concurrent_streams( + u32::try_from(budgets.requests).context("HTTP/2 request budget exceeds u32")?, + ); + let connections = Arc::new(Semaphore::new(budgets.connections)); + // Eight reserved diagnostic-only sockets remain available under business saturation. + let diagnostics = Arc::new(Semaphore::new(8)); + let mut tasks = JoinSet::new(); + #[cfg(unix)] + let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .context("register SIGTERM handler")?; let shutdown = async { - tokio::signal::ctrl_c().await.ok(); - info!("received shutdown signal, draining connections"); + #[cfg(unix)] + tokio::select! { + result = tokio::signal::ctrl_c() => result.context("wait for SIGINT"), + _ = terminate.recv() => Ok(()), + } + #[cfg(not(unix))] + tokio::signal::ctrl_c().await.context("wait for shutdown") }; - tokio::pin!(shutdown); - loop { tokio::select! { + biased; + result = &mut shutdown => { result?; break; } + result = tasks.join_next(), if !tasks.is_empty() => { + if let Some(Err(error)) = result { warn!(error = %error, "connection task failed"); } + } result = listener.accept() => { - let (stream, peer_addr) = match result { - Ok(conn) => conn, - Err(e) => { - warn!(error = %e, "failed to accept connection"); - continue; - } + let (stream, peer_addr) = result.context("accept gateway connection")?; + let (permit, svc) = if let Ok(permit) = Arc::clone(&connections).try_acquire_owned() { + (permit, service.clone()) + } else if let Ok(permit) = Arc::clone(&diagnostics).try_acquire_owned() { + (permit, service.clone().health_only()) + } else { + warn!(peer = %peer_addr, "gateway rejected connection: business and diagnostic capacity both exhausted"); + drop(stream); + continue; }; - - let svc = service.clone(); - let conn = http.serve_connection(TokioIo::new(stream), svc); - let conn = graceful.watch(conn.into_owned()); - - tokio::spawn(async move { - if let Err(e) = conn.await { - warn!(peer_addr = %peer_addr, error = %e, "connection error"); - } + let conn = graceful.watch(http.serve_connection(TokioIo::new(stream), svc).into_owned()); + tasks.spawn(async move { + let _permit = permit; + if let Err(error) = conn.await { warn!(%peer_addr, error = %error, "connection error"); } }); } - - () = &mut shutdown => { - info!("shutting down gracefully"); - break; + } + } + service.state().drain(); + drop(listener); + let deadline = Instant::now() + Duration::from_secs(budgets.shutdown_seconds); + info!("stopped ingress; draining connections"); + let drain_deadline = deadline - shutdown_cleanup_reserve(); + let drained = tokio::time::timeout_at(drain_deadline, graceful.shutdown()) + .await + .context("HTTP drain deadline exceeded"); + if drained.is_err() { + tasks.abort_all(); + } + while let Some(result) = tasks.join_next().await { + if let Err(error) = result { + if !error.is_cancelled() { + warn!(error = %error, "connection task failed during drain"); } } } - - // Wait for in-flight requests to complete. - graceful.shutdown().await; - info!("all connections drained, exiting"); - - Ok(()) + Ok((deadline, drained)) } /// Check whether a service name was compiled into this binary. @@ -699,7 +743,7 @@ fn is_compiled_in(name: &str) -> bool { /// /// If `SERVICES` is unset or empty, returns all compiled-in services. fn parse_enabled_services() -> Vec { - let raw = std::env::var("SERVICES").unwrap_or_default(); + let raw = settings::var("SERVICES").unwrap_or_default(); parse_services_value(&raw) } @@ -775,37 +819,72 @@ fn parse_services_value(raw: &str) -> Vec { } } -/// Perform a health check by connecting to the gateway and requesting the health endpoint. -/// -/// Exits with code 0 if the response is 200 OK and contains at least one -/// running service, 1 otherwise. +/// Structured `/health/ready` payload the probe requires. +#[derive(serde::Deserialize)] +struct ProbeStatus { + ready: bool, + services: std::collections::BTreeMap, +} + +/// Probe real HTTP status and structured readiness with one absolute deadline. async fn run_health_check(addr: &str) -> Result<()> { - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpStream, + let probe = async { + let stream = tokio::net::TcpStream::connect(addr) + .await + .with_context(|| format!("connect health endpoint {addr}"))?; + let (mut sender, connection) = hyper::client::conn::http1::Builder::new() + .max_buf_size(65_536) + .handshake::<_, http_body_util::Empty>(TokioIo::new(stream)) + .await + .context("initialize health HTTP connection")?; + let mut tasks = JoinSet::new(); + tasks.spawn(connection); + let mut request = http::Request::new(http_body_util::Empty::new()); + *request.uri_mut() = http::Uri::from_static("/_health/ready"); + request.headers_mut().insert( + http::header::HOST, + http::HeaderValue::from_str(addr).context("invalid health host")?, + ); + let response = sender + .send_request(request) + .await + .context("request health status")?; + if response.status() != http::StatusCode::OK { + anyhow::bail!("health endpoint returned {}", response.status()); + } + let budget = rustack_core::http::BodyBudget::new( + 65_536, + Duration::from_secs(3), + Duration::from_secs(3), + )?; + let bytes = rustack_core::http::collect_body(response.into_body(), budget) + .await + .context("read bounded health response")?; + let status: ProbeStatus = + serde_json::from_slice(&bytes).context("parse health response JSON")?; + if !status.ready + || status.services.is_empty() + || status + .services + .values() + .any(|state| !matches!(state.as_str(), "running" | "disabled")) + { + anyhow::bail!("runtime is not ready"); + } + drop(sender); + tasks.abort_all(); + while let Some(result) = tasks.join_next().await { + if let Err(error) = result { + if !error.is_cancelled() { + return Err(error).context("health connection task failed"); + } + } + } + Ok(()) }; - - let mut stream = TcpStream::connect(addr) + tokio::time::timeout(Duration::from_secs(3), probe) .await - .with_context(|| format!("cannot connect to {addr}"))?; - - let request = - format!("GET /_localstack/health HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\n\r\n"); - stream.write_all(request.as_bytes()).await?; - // Do not half-close the write side: the HTTP request is self-framing - // (GET with no body) and the Connection: close header tells hyper not - // to expect further requests. Hyper will close after responding, - // which gives read_to_string its EOF. - - let mut response = String::new(); - stream.read_to_string(&mut response).await?; - - // Accept any 200 response that reports at least one running service. - if response.contains("200 OK") && response.contains("\"running\"") { - Ok(()) - } else { - anyhow::bail!("unhealthy response from {addr}") - } + .context("health probe deadline exceeded")? } /// Read the gateway listen address from the environment. @@ -813,25 +892,27 @@ async fn run_health_check(addr: &str) -> Result<()> { /// Checks `GATEWAY_LISTEN` (the canonical var) and falls back to the /// S3Config default when S3 is compiled in. fn gateway_listen_addr() -> String { - std::env::var("GATEWAY_LISTEN").unwrap_or_else(|_| "0.0.0.0:4566".to_string()) + settings::var("GATEWAY_LISTEN").unwrap_or_else(|_| "127.0.0.1:4566".to_string()) } /// Read the log level from the environment. fn log_level() -> String { - std::env::var("LOG_LEVEL").unwrap_or_else(|_| "info".to_string()) + settings::var("LOG_LEVEL").unwrap_or_else(|_| "info".to_string()) } /// Built runtime services and provider handles. struct Runtime { services: Vec>, providers: RuntimeProviders, + workers: RuntimeWorkers, } /// Build all enabled service routers based on environment configuration. #[allow(clippy::too_many_lines)] -fn build_services(is_enabled: impl Fn(&str) -> bool) -> Runtime { +fn build_services(is_enabled: impl Fn(&str) -> bool) -> Result { let mut services: Vec> = Vec::new(); let mut providers = RuntimeProviders::default(); + let mut workers = RuntimeWorkers::default(); // ----- DynamoDB + DynamoDB Streams (register before S3: S3 is the catch-all) ----- #[cfg(feature = "dynamodb")] @@ -870,6 +951,7 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Runtime { providers.register_dynamodb_streams(Arc::clone(store)); } providers.register_dynamodb(Arc::clone(&dynamodb_provider)); + workers.dynamodb = Some(Arc::clone(&dynamodb_provider)); let dynamodb_handler = RustackDynamoDBHandler::new(Arc::clone(&dynamodb_provider)); let dynamodb_http_config = build_dynamodb_http_config(&dynamodb_config); let dynamodb_service = @@ -903,6 +985,7 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Runtime { ); let sqs_provider = Arc::new(RustackSqs::new(sqs_config.clone())); providers.register_sqs(Arc::clone(&sqs_provider)); + workers.sqs = Some(Arc::clone(&sqs_provider)); let sqs_handler = RustackSqsHandler::new(Arc::clone(&sqs_provider)); let sqs_http_config = build_sqs_http_config(&sqs_config); let sqs_service = SqsHttpService::new(Arc::new(sqs_handler), sqs_http_config); @@ -1008,15 +1091,13 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Runtime { ); let sqs_publisher: Arc = if let Some(ref sqs) = sqs_provider_arc { - Arc::new(RustackSqsPublisher::new( - Arc::clone(sqs), - sns_config.clone(), - )) + Arc::new(RustackSqsPublisher::new(Arc::clone(sqs))) } else { - Arc::new(rustack_sns_core::publisher::NoopSqsPublisher) + Arc::new(rustack_sns_core::publisher::UnavailableSqsPublisher) }; - let sns_provider = RustackSns::new(sns_config.clone(), sqs_publisher); - let sns_handler = RustackSnsHandler::new(Arc::new(sns_provider)); + let sns_provider = Arc::new(RustackSns::new(sns_config.clone(), sqs_publisher)); + workers.sns = Some(Arc::clone(&sns_provider)); + let sns_handler = RustackSnsHandler::new(sns_provider); let sns_http_config = build_sns_http_config(&sns_config); let sns_service = SnsHttpService::new(Arc::new(sns_handler), sns_http_config); services.push(Box::new(service::SnsServiceRouter::new(sns_service))); @@ -1032,17 +1113,13 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Runtime { ); let delivery: Arc = if let Some(ref sqs) = sqs_provider_arc { - Arc::new(LocalTargetDelivery::new( - Arc::clone(sqs), - events_config.account_id.clone(), - events_config.host.clone(), - events_config.port, - )) + Arc::new(LocalTargetDelivery::new(Arc::clone(sqs))) } else { - Arc::new(rustack_events_core::delivery::NoopTargetDelivery) + Arc::new(rustack_events_core::delivery::UnavailableTargetDelivery) }; - let events_provider = RustackEvents::new(events_config.clone(), delivery); - let events_handler = RustackEventsHandler::new(Arc::new(events_provider)); + let events_provider = Arc::new(RustackEvents::new(events_config.clone(), delivery)); + workers.events = Some(Arc::clone(&events_provider)); + let events_handler = RustackEventsHandler::new(events_provider); let events_http_config = build_events_http_config(&events_config); let events_service = EventsHttpService::new(Arc::new(events_handler), events_http_config); services.push(Box::new(service::EventsServiceRouter::new(events_service))); @@ -1121,7 +1198,7 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Runtime { apigatewayv2_skip_signature_validation = apigw_config.skip_signature_validation, "initializing API Gateway v2 service", ); - let apigw_provider = Arc::new(RustackApiGatewayV2::new(apigw_config.clone())); + let apigw_provider = Arc::new(RustackApiGatewayV2::new(apigw_config.clone())?); providers.register_apigatewayv2(Arc::clone(&apigw_provider)); let apigw_handler = RustackApiGatewayV2Handler::new(Arc::clone(&apigw_provider)); let apigw_http_config = build_apigatewayv2_http_config(&apigw_config); @@ -1158,7 +1235,7 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Runtime { // ----- Lambda (register before S3: S3 is the catch-all) ----- #[cfg(feature = "lambda")] if is_enabled("lambda") { - let lambda_config = LambdaConfig::from_env(); + let lambda_config = LambdaConfig::from_env()?; info!( lambda_skip_signature_validation = lambda_config.skip_signature_validation, lambda_docker_enabled = lambda_config.docker_enabled, @@ -1180,6 +1257,7 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Runtime { let lambda_provider = Arc::new(RustackLambda::new(lambda_config.clone()).with_code_fetcher(code_fetcher)); providers.register_lambda(Arc::clone(&lambda_provider)); + workers.lambda = Some(Arc::clone(&lambda_provider)); let lambda_handler = RustackLambdaHandler::new(Arc::clone(&lambda_provider)); let lambda_http_config = build_lambda_http_config(&lambda_config); let lambda_service = LambdaHttpService::new(Arc::new(lambda_handler), lambda_http_config); @@ -1210,13 +1288,11 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Runtime { if let Some(s3) = s3_provider_arc.as_ref() { builder = builder.s3(Arc::clone(s3)); } - match builder.build() { - Ok(plane) => { - providers.register_cloudfront_cache(plane.clone()); - services.push(Box::new(service::CloudFrontDataPlaneRouter::new(plane))); - } - Err(e) => warn!(error = %e, "failed to initialise CloudFront data plane"), - } + let plane = builder + .build() + .map_err(|error| anyhow::anyhow!("initialize CloudFront data plane: {error}"))?; + providers.register_cloudfront_cache(plane.clone()); + services.push(Box::new(service::CloudFrontDataPlaneRouter::new(plane))); } services.push(Box::new(service::CloudFrontServiceRouter::new(cf_service))); @@ -1232,14 +1308,33 @@ fn build_services(is_enabled: impl Fn(&str) -> bool) -> Runtime { services.push(Box::new(service::S3ServiceRouter::new(s3_service))); } - Runtime { + Ok(Runtime { services, providers, - } + workers, + }) +} + +fn shutdown_cleanup_reserve() -> Duration { + (Duration::from_secs(settings::budgets().shutdown_seconds) / 10).min(Duration::from_secs(1)) +} + +fn main() -> Result<()> { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .context("create runtime")?; + let result = runtime.block_on(run()); + // The supervisor has already joined normal work and reaped children. A blocked + // filesystem spawn_blocking cannot be aborted; do not let Tokio's destructor + // turn a reported shutdown timeout into an unbounded process hang. + runtime.shutdown_timeout(Duration::ZERO); + result } -#[tokio::main] -async fn main() -> Result<()> { +// Sequential supervisor steps stay explicit rather than being folded into helpers. +#[allow(clippy::too_many_lines)] +async fn run() -> Result<()> { // Parse CLI flags once and dispatch. Help / version are handled before // any tracing or config work so they're cheap and side-effect free. let snapshot_name = match classify_args(std::env::args()) { @@ -1262,14 +1357,25 @@ async fn main() -> Result<()> { std::process::exit(2); } CliAction::HealthCheck => { - let listen_addr = gateway_listen_addr(); - let addr = listen_addr.replace("0.0.0.0", "127.0.0.1"); - let healthy = run_health_check(&addr).await.is_ok(); - std::process::exit(i32::from(!healthy)); + let config = settings::initialize().await?; + let mut addr = config.listen(); + if addr.ip().is_unspecified() { + addr.set_ip(if addr.is_ipv6() { + std::net::Ipv6Addr::LOCALHOST.into() + } else { + std::net::Ipv4Addr::LOCALHOST.into() + }); + } + if let Err(error) = run_health_check(&addr.to_string()).await { + eprintln!("health check failed: {error:#}"); + std::process::exit(1); + } + return Ok(()); } CliAction::Run { snapshot } => snapshot, }; + let config = settings::initialize().await?; let listen_addr = gateway_listen_addr(); let log = log_level(); init_tracing(&log)?; @@ -1277,14 +1383,16 @@ async fn main() -> Result<()> { let enabled = parse_enabled_services(); - // Warn about services that are requested but not compiled in. for name in &enabled { if !is_compiled_in(name) { - warn!(service = %name, "requested service is not compiled in, skipping"); + anyhow::bail!( + "requested service '{name}' is unknown or not compiled in; see rustack --help" + ); } } - - let runtime = build_services(|name| enabled.iter().any(|s| s == name) && is_compiled_in(name)); + config.validate_credentials(&enabled)?; + info!(settings = ?config, "validated runtime configuration"); + let runtime = build_services(|name| enabled.iter().any(|s| s == name))?; if runtime.services.is_empty() { anyhow::bail!( @@ -1293,15 +1401,22 @@ async fn main() -> Result<()> { ); } + let _snapshot_lease = match snapshot_config.as_ref() { + Some(config) => Some(config.acquire_lease().await?), + None => None, + }; if let Some(config) = snapshot_config.as_ref() { + info!(coverage = ?runtime.providers.coverage(), "snapshot includes only declared state; other services and SQS messages are not persisted"); config.load(&runtime.providers).await?; } let Runtime { services, providers, + workers, } = runtime; - let gateway = GatewayService::new(services); + let workers = Arc::new(workers); + let gateway = GatewayService::new(services).with_workers(Arc::clone(&workers)); let service_names = gateway.service_names(); let addr: SocketAddr = listen_addr @@ -1319,22 +1434,125 @@ async fn main() -> Result<()> { "starting Rustack Server", ); - let serve_result = serve(listener, gateway).await; - let save_result = if let Some(config) = snapshot_config.as_ref() { - config.save(&providers, VERSION).await - } else { - Ok(()) + let (deadline, serve_result) = match serve(listener, gateway).await { + Ok(outcome) => outcome, + Err(error) => ( + Instant::now() + Duration::from_secs(settings::budgets().shutdown_seconds), + Err(error), + ), }; - providers.shutdown().await; - - serve_result?; - save_result + let work_deadline = deadline - shutdown_cleanup_reserve(); + let save_result = async { + serve_result?; + workers + .quiesce(work_deadline.saturating_duration_since(Instant::now())) + .await?; + if let Some(config) = snapshot_config.as_ref() { + tokio::time::timeout_at(work_deadline, config.save(&providers, VERSION)) + .await + .context("snapshot save deadline exceeded")??; + } + Result::<()>::Ok(()) + } + .await; + let stop_result = async { + match tokio::time::timeout_at(deadline, workers.shutdown()).await { + Ok(result) => result.context("runtime shutdown failed")?, + Err(_) => anyhow::bail!( + "resource cleanup incomplete: shutdown deadline exceeded; dropping providers now \ + (kill_on_drop reaps remaining children)" + ), + } + Result::<()>::Ok(()) + } + .await; + save_result?; + stop_result?; + Ok(()) } #[cfg(test)] mod tests { + use tokio::io::AsyncWriteExt; + use super::*; + async fn probe_fixture(status: &str, body: &str) -> Result> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: \ + {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let mut servers = JoinSet::new(); + servers.spawn(async move { + let (mut stream, _) = listener.accept().await?; + stream.write_all(response.as_bytes()).await + }); + let result = run_health_check(&address.to_string()).await; + while let Some(joined) = servers.join_next().await { + joined??; + } + Ok(result) + } + + #[tokio::test] + async fn test_should_parse_health_status_not_substrings() -> Result<()> { + let valid = r#"{"ready":true,"services":{"s3":"running"}}"#; + probe_fixture("200 OK", valid).await??; + assert!(probe_fixture("500 Error", valid).await?.is_err()); + assert!(probe_fixture("200 OK", "200 OK running").await?.is_err()); + assert!( + probe_fixture("200 OK", r#"{"ready":false,"services":{"s3":"running"}}"#) + .await? + .is_err() + ); + assert!( + probe_fixture("200 OK", r#"{"ready":true,"services":{}}"#) + .await? + .is_err() + ); + assert!(probe_fixture("200 OK", &"x".repeat(65_537)).await?.is_err()); + Ok(()) + } + + #[tokio::test] + async fn test_should_bound_health_probe_without_response_eof() -> Result<()> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + let mut servers = JoinSet::new(); + servers.spawn(async move { + let (stream, _) = listener.accept().await?; + let _held_stream = stream; + std::future::pending::<()>().await; + Result::<()>::Ok(()) + }); + let result = tokio::time::timeout( + Duration::from_secs(5), + run_health_check(&address.to_string()), + ) + .await?; + assert!(result.is_err()); + servers.abort_all(); + while let Some(joined) = servers.join_next().await { + assert!(joined.is_err_and(|error| error.is_cancelled())); + } + Ok(()) + } + + #[test] + fn test_should_reject_positionals_and_conflicting_runtime_flags() { + for args in [ + vec!["rustack", "typo"], + vec!["rustack", "--snapshot", "one", "--snapshot", "two"], + vec!["rustack", "--health-check", "--snapshot", "dev"], + vec!["rustack", "--snapshot=dev", "--health-check"], + ] { + assert!(matches!(classify_args(args), CliAction::UnknownFlag(_))); + } + } + #[test] fn test_should_parse_services_value_default() { // Empty string = all compiled-in services. diff --git a/apps/rustack/src/runtime.rs b/apps/rustack/src/runtime.rs new file mode 100644 index 0000000..1b251c9 --- /dev/null +++ b/apps/rustack/src/runtime.rs @@ -0,0 +1,177 @@ +//! Runtime worker ownership independent of snapshot participation. + +use std::{collections::BTreeMap, fmt, sync::Arc, time::Duration}; + +use anyhow::{Context, Result}; +use serde::Serialize; + +/// Handles retained until quiescence, snapshot publication, and final shutdown. +#[derive(Default)] +pub(crate) struct RuntimeWorkers { + #[cfg(feature = "dynamodb")] + pub(crate) dynamodb: Option>, + #[cfg(feature = "events")] + pub(crate) events: Option>, + #[cfg(feature = "sns")] + pub(crate) sns: Option>, + #[cfg(feature = "lambda")] + pub(crate) lambda: Option>, + #[cfg(feature = "sqs")] + pub(crate) sqs: Option>, +} + +impl fmt::Debug for RuntimeWorkers { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RuntimeWorkers") + .finish_non_exhaustive() + } +} + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct DeliverySummary { + accepted: u64, + delivered: u64, + failed: u64, + rejected: u64, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct WorkerDiagnostics { + pub(crate) ready: bool, + pub(crate) services: BTreeMap<&'static str, &'static str>, + delivery: BTreeMap<&'static str, DeliverySummary>, +} + +impl RuntimeWorkers { + /// Report live supervisor state, independently of historical delivery failures. + pub(crate) fn diagnostics(&self) -> WorkerDiagnostics { + let readiness: &[Option<(&'static str, bool)>] = &[ + #[cfg(feature = "events")] + self.events + .as_ref() + .map(|provider| ("events", provider.is_ready())), + #[cfg(feature = "sns")] + self.sns + .as_ref() + .map(|provider| ("sns", provider.is_ready())), + #[cfg(feature = "dynamodb")] + self.dynamodb + .as_ref() + .map(|provider| ("dynamodb", provider.is_ready())), + #[cfg(feature = "sqs")] + self.sqs + .as_ref() + .map(|provider| ("sqs", provider.is_ready())), + ]; + let deliveries: &[Option<(&'static str, DeliverySummary)>] = &[ + #[cfg(feature = "events")] + self.events.as_ref().map(|provider| { + let stats = provider.delivery_stats(); + ( + "events", + DeliverySummary { + accepted: stats.accepted, + delivered: stats.delivered, + failed: stats.failed, + rejected: stats.rejected, + }, + ) + }), + #[cfg(feature = "sns")] + self.sns.as_ref().map(|provider| { + let stats = provider.delivery_stats(); + ( + "sns", + DeliverySummary { + accepted: stats.accepted, + delivered: stats.delivered, + failed: stats.failed, + rejected: stats.rejected, + }, + ) + }), + ]; + WorkerDiagnostics { + ready: readiness.iter().flatten().all(|(_, ready)| *ready), + services: readiness + .iter() + .flatten() + .map(|(name, ready)| (*name, if *ready { "running" } else { "failed" })) + .collect(), + delivery: deliveries.iter().flatten().copied().collect(), + } + } + + /// Stop new cross-service work before collecting the supported snapshot state. + pub(crate) async fn quiesce(&self, remaining: Duration) -> Result<()> { + let deadline = tokio::time::Instant::now() + remaining; + #[cfg(feature = "events")] + if let Some(events) = &self.events { + tokio::time::timeout_at(deadline, events.quiesce()) + .await + .context("EventBridge quiesce deadline exceeded")??; + } + #[cfg(feature = "sns")] + if let Some(sns) = &self.sns { + tokio::time::timeout_at(deadline, sns.quiesce()) + .await + .context("SNS quiesce deadline exceeded")?; + } + #[cfg(feature = "dynamodb")] + if let Some(dynamodb) = &self.dynamodb { + tokio::time::timeout_at(deadline, dynamodb.quiesce()) + .await + .context("DynamoDB quiesce deadline exceeded")??; + } + #[cfg(feature = "lambda")] + if let Some(lambda) = &self.lambda { + lambda + .quiesce(deadline.saturating_duration_since(tokio::time::Instant::now())) + .await?; + } + #[cfg(feature = "sqs")] + if let Some(sqs) = &self.sqs { + tokio::time::timeout_at(deadline, sqs.quiesce()) + .await + .context("SQS quiesce deadline exceeded")??; + } + Ok(()) + } + + /// Final resource destruction. Call only after the snapshot decision. + pub(crate) async fn shutdown(&self) -> Result<()> { + // After the snapshot decision, stop independent resource owners together: + // a failed queue worker must not delay killing Lambda child processes. + let (events_result, (), (), ()) = tokio::join!( + async { + #[cfg(feature = "events")] + if let Some(events) = &self.events { + events.shutdown().await?; + } + Result::<()>::Ok(()) + }, + async { + #[cfg(feature = "sns")] + if let Some(sns) = &self.sns { + sns.shutdown().await; + } + }, + async { + #[cfg(feature = "lambda")] + if let Some(lambda) = &self.lambda { + lambda.shutdown().await; + } + }, + async { + #[cfg(feature = "sqs")] + if let Some(sqs) = &self.sqs { + sqs.shutdown_all().await; + } + }, + ); + events_result + } +} diff --git a/apps/rustack/src/service.rs b/apps/rustack/src/service.rs index 984018b..ed7e62e 100644 --- a/apps/rustack/src/service.rs +++ b/apps/rustack/src/service.rs @@ -25,6 +25,36 @@ pub fn gateway_body_from_string(s: impl Into) -> GatewayBody { .boxed() } +/// Preserve arbitrary data-plane bytes without lossy UTF-8 conversion. +#[cfg(feature = "apigatewayv2")] +pub fn gateway_body_from_bytes(bytes: Bytes) -> GatewayBody { + Full::new(bytes) + .map_err(|never: Infallible| match never {}) + .boxed() +} + +#[cfg(any(feature = "apigatewayv2", feature = "cloudfront-dataplane"))] +fn body_error_response(error: &rustack_core::http::BodyReadError) -> http::Response { + use rustack_core::http::BodyReadError; + let status = match &error { + BodyReadError::TooLarge => http::StatusCode::PAYLOAD_TOO_LARGE, + BodyReadError::Deadline | BodyReadError::Idle => http::StatusCode::REQUEST_TIMEOUT, + BodyReadError::Transport(_) => http::StatusCode::BAD_REQUEST, + BodyReadError::InvalidBudget | BodyReadError::Allocation(_) => { + http::StatusCode::INTERNAL_SERVER_ERROR + } + }; + let mut response = http::Response::new(gateway_body_from_string( + serde_json::json!({"message": error.to_string()}).to_string(), + )); + *response.status_mut() = status; + response.headers_mut().insert( + http::header::CONTENT_TYPE, + http::HeaderValue::from_static("application/json"), + ); + response +} + /// A routable AWS service registered with the gateway. /// /// Implementors declare which requests they handle (via [`matches`](ServiceRouter::matches)) @@ -1117,25 +1147,14 @@ mod apigatewayv2_router { }; // Collect request body. - let body_bytes: Bytes = match http_body_util::BodyExt::collect(req.into_body()) - .await - .map(http_body_util::Collected::to_bytes) + let body_bytes: Bytes = match rustack_core::http::collect_body( + req.into_body(), + rustack_core::http::BodyBudget::control(), + ) + .await { Ok(b) => b, - Err(e) => { - let body = - serde_json::json!({"message": format!("Failed to read body: {e}")}); - let resp = http::Response::builder() - .status(http::StatusCode::BAD_REQUEST) - .header("content-type", "application/json") - .body(gateway_body_from_string(body.to_string())) - .unwrap_or_else(|_| { - http::Response::new(gateway_body_from_string( - "Bad Request".to_owned(), - )) - }); - return Ok(resp); - } + Err(error) => return Ok(super::body_error_response(&error)), }; match handle_execution( @@ -1153,7 +1172,7 @@ mod apigatewayv2_router { let (parts, body) = resp.into_parts(); Ok(http::Response::from_parts( parts, - gateway_body_from_string(String::from_utf8_lossy(&body).into_owned()), + super::gateway_body_from_bytes(body), )) } Err(e) => { @@ -1618,9 +1637,14 @@ mod cloudfront_dataplane_router { let plane = self.plane.clone(); Box::pin(async move { let (parts, incoming) = req.into_parts(); - let body_bytes: Bytes = match incoming.collect().await { - Ok(c) => c.to_bytes(), - Err(_) => Bytes::new(), + let body_bytes: Bytes = match rustack_core::http::collect_body( + incoming, + rustack_core::http::BodyBudget::control(), + ) + .await + { + Ok(body) => body, + Err(error) => return Ok(super::body_error_response(&error)), }; let resp = plane .handle_request( diff --git a/apps/rustack/src/snapshot.rs b/apps/rustack/src/snapshot.rs index 28f8581..0249428 100644 --- a/apps/rustack/src/snapshot.rs +++ b/apps/rustack/src/snapshot.rs @@ -117,17 +117,17 @@ impl RuntimeProviders { self.services.push(Arc::new(service)); } - /// Stop stateful provider background workers after snapshot save. - pub(crate) async fn shutdown(&self) { - for service in &self.services { - if let Err(error) = service.shutdown().await { - warn!( - service = service.service_name(), - error = %error, - "snapshot service shutdown failed", - ); - } - } + /// Explicit persistence coverage; workers are owned by the runtime supervisor. + pub(crate) fn coverage(&self) -> Vec<(&'static str, &'static str)> { + self.services + .iter() + .map(|service| { + ( + service.service_name(), + snapshot_coverage(service.service_name()), + ) + }) + .collect() } fn services(&self) -> &[Arc] { @@ -135,6 +135,18 @@ impl RuntimeProviders { } } +/// Human-facing coverage is not the binary archive's resource/data encoding kind. +pub(crate) fn snapshot_coverage(service: &str) -> &'static str { + match service { + "s3" | "dynamodb" => "resources-and-data", + "dynamodbstreams" => "resources-and-records", + "lambda" => "resources-and-code", + "cloudfront-cache" | "cloudfront-dataplane" => "cache-data", + "sqs" | "ssm" | "iam" | "apigatewayv2" | "cloudfront" => "resources-only", + _ => "unsupported", + } +} + #[async_trait] trait SnapshotService: Send + Sync { fn service_name(&self) -> &'static str; @@ -146,10 +158,6 @@ trait SnapshotService: Send + Sync { async fn save_meta(&self, data_staging_dir: &Path) -> Result>; async fn load_meta(&self, state_cbor: &[u8], data_staging_dir: &Path) -> Result<()>; - - async fn shutdown(&self) -> Result<()> { - Ok(()) - } } #[cfg(feature = "s3")] @@ -253,11 +261,6 @@ impl SnapshotService for SqsSnapshotService { self.provider.import_snapshot(snapshot).await?; Ok(()) } - - async fn shutdown(&self) -> Result<()> { - self.provider.shutdown_all().await; - Ok(()) - } } #[cfg(feature = "ssm")] @@ -327,11 +330,6 @@ impl SnapshotService for LambdaSnapshotService { self.provider.import_snapshot(snapshot).await?; Ok(()) } - - async fn shutdown(&self) -> Result<()> { - self.provider.shutdown().await; - Ok(()) - } } #[cfg(feature = "apigatewayv2")] @@ -422,7 +420,45 @@ pub(crate) struct SnapshotConfig { root: PathBuf, } +/// Advisory file lock retained for the full lifetime of a named snapshot runtime. +#[derive(Debug)] +#[allow(clippy::disallowed_types)] // OS advisory lock requires the blocking std File API. +pub(crate) struct SnapshotLease { + _file: std::fs::File, +} + impl SnapshotConfig { + /// Acquire an OS-released exclusive lease before loading or serving a named snapshot. + /// + /// The blocking std file API is required for advisory locking and is isolated + /// inside a blocking worker; tokio offers no equivalent lock surface. + #[allow(clippy::disallowed_types, clippy::disallowed_methods)] + pub(crate) async fn acquire_lease(&self) -> Result { + let root = self.root.clone(); + let name = self.name.as_str().to_owned(); + tokio::task::spawn_blocking(move || { + std::fs::create_dir_all(&root).context("create snapshot root for lease")?; + let lock_path = root.join(format!(".{name}.lock")); + if std::fs::symlink_metadata(&lock_path) + .is_ok_and(|metadata| metadata.file_type().is_symlink()) + { + bail!("snapshot lock path must not be a symbolic link"); + } + let file = std::fs::OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&lock_path) + .context("open snapshot lease")?; + file.try_lock() + .context("snapshot is already owned by another runtime")?; + Ok(SnapshotLease { _file: file }) + }) + .await + .context("snapshot lease task failed")? + } + /// Build snapshot configuration from a CLI-provided name. /// /// # Errors @@ -446,6 +482,8 @@ impl SnapshotConfig { pub(crate) async fn load(&self, providers: &RuntimeProviders) -> Result<()> { let started = Instant::now(); let dir = self.snapshot_dir(); + cleanup_stale_staging(&self.root, self.name.as_str()).await?; + recover_snapshot(&self.root, &dir, self.name.as_str()).await?; if !path_exists(&dir).await? { info!(snapshot = %self.name.as_str(), path = %dir.display(), "snapshot not found, starting empty"); return Ok(()); @@ -453,6 +491,14 @@ impl SnapshotConfig { let manifest_path = dir.join(MANIFEST_FILE); let manifest = read_manifest(&manifest_path).await?; + if manifest.snapshot_name != self.name.as_str() { + bail!( + "snapshot manifest names {} but was requested as {}; refusing to import a renamed \ + or copied directory", + manifest.snapshot_name, + self.name.as_str() + ); + } if manifest.schema_version != SNAPSHOT_SCHEMA_VERSION { bail!( "unsupported snapshot schema version {} in {}", @@ -794,7 +840,7 @@ async fn read_manifest(path: &Path) -> Result { } async fn record_snapshot_timing(metric: &str, elapsed: Duration) { - let Ok(path) = std::env::var(SNAPSHOT_PERF_FILE_ENV) else { + let Ok(path) = rustack_core::settings::var(SNAPSHOT_PERF_FILE_ENV) else { return; }; let line = format!("{metric}={}\n", elapsed.as_millis()); @@ -813,7 +859,7 @@ async fn record_snapshot_timing(metric: &str, elapsed: Duration) { } fn snapshot_root() -> PathBuf { - std::env::var(SNAPSHOT_ROOT_ENV) + rustack_core::settings::var(SNAPSHOT_ROOT_ENV) .map_or_else(|_| PathBuf::from(DEFAULT_SNAPSHOT_ROOT), PathBuf::from) } @@ -841,41 +887,172 @@ async fn remove_dir_if_exists(path: &Path) -> Result<()> { } async fn replace_directory(temp: &Path, target: &Path, root: &Path, name: &str) -> Result<()> { - let suffix = unique_suffix()?; - let backup = root.join(format!(".{name}.bak.{suffix}")); - remove_dir_if_exists(&backup).await?; - + recover_snapshot(root, target, name).await?; + validate_generation(temp, name) + .await + .context("validate prepared snapshot")?; + sync_tree(temp).await?; + let backup = root.join(format!(".{name}.previous")); let had_target = path_exists(target).await?; if had_target { - fs::rename(target, &backup).await.with_context(|| { - format!( - "failed to move existing snapshot {} to backup {}", - target.display(), - backup.display() - ) - })?; + fs::rename(target, &backup) + .await + .context("preserve previous committed snapshot")?; + sync_directory(root).await?; + } + if let Err(error) = fs::rename(temp, target).await { + if had_target { + fs::rename(&backup, target).await.with_context(|| { + format!( + "snapshot publication failed ({error}); recovery also failed; previous \ + snapshot retained at {}", + backup.display() + ) + })?; + sync_directory(root).await?; + } + return Err(error).context("publish prepared snapshot"); } + sync_directory(root).await?; + if had_target { + remove_dir_if_exists(&backup).await?; + sync_directory(root).await?; + } + Ok(()) +} - match fs::rename(temp, target).await { - Ok(()) => { - if had_target { - remove_dir_if_exists(&backup).await?; +/// Recover an interrupted publish, including the legacy randomly named backups. +async fn recover_snapshot(root: &Path, target: &Path, name: &str) -> Result<()> { + if !path_exists(root).await? { + return Ok(()); + } + let previous = root.join(format!(".{name}.previous")); + let backup = if path_exists(&previous).await? { + Some(previous) + } else if !path_exists(target).await? { + let mut entries = fs::read_dir(root) + .await + .context("inspect snapshot recovery candidates")?; + let prefix = format!(".{name}.bak."); + let mut candidate = None; + while let Some(entry) = entries.next_entry().await? { + if entry.file_name().to_string_lossy().starts_with(&prefix) { + if candidate.is_some() { + bail!( + "multiple legacy snapshot backups require explicit operator recovery; \ + refusing empty startup" + ); + } + candidate = Some(entry.path()); } - Ok(()) } - Err(error) => { - if had_target { - let _ = fs::rename(&backup, target).await; + candidate + } else { + None + }; + let Some(backup) = backup else { + return Ok(()); + }; + if path_exists(target).await? && validate_generation(target, name).await.is_ok() { + remove_dir_if_exists(&backup).await?; + sync_directory(root).await?; + return Ok(()); + } + validate_generation(&backup, name) + .await + .context("previous snapshot is not a valid recovery generation")?; + if path_exists(target).await? { + let failed = root.join(format!(".{name}.failed.{}", unique_suffix()?)); + fs::rename(target, failed) + .await + .context("retain invalid snapshot for diagnosis")?; + } + fs::rename(&backup, target) + .await + .context("recover previous committed snapshot")?; + sync_directory(root).await?; + warn!( + snapshot = name, + "recovered previous committed snapshot after interrupted publication" + ); + Ok(()) +} + +async fn cleanup_stale_staging(root: &Path, name: &str) -> Result<()> { + let mut entries = fs::read_dir(root) + .await + .context("inspect snapshot staging leftovers")?; + let prefixes = [ + format!(".{name}.tmp."), + format!(".{name}.{LOAD_STAGING_PREFIX}."), + ]; + while let Some(entry) = entries.next_entry().await? { + let file_name = entry.file_name().to_string_lossy().into_owned(); + if prefixes.iter().any(|prefix| file_name.starts_with(prefix)) + && entry.file_type().await?.is_dir() + { + remove_dir_if_exists(&entry.path()).await?; + } + } + Ok(()) +} + +async fn validate_generation(directory: &Path, name: &str) -> Result<()> { + let manifest = read_manifest(&directory.join(MANIFEST_FILE)).await?; + if manifest.schema_version != SNAPSHOT_SCHEMA_VERSION || manifest.snapshot_name != name { + bail!("snapshot generation name/schema mismatch"); + } + for entry in manifest.services.values() { + let meta = read_archive( + &snapshot_child(directory, &entry.meta_file)?, + ArchiveKind::ServiceMeta, + ) + .await?; + get_required_section(&meta, SECTION_STATE_CBOR)?; + if let Some(data) = &entry.data_file { + read_archive(&snapshot_child(directory, data)?, ArchiveKind::ServiceData).await?; + } + } + Ok(()) +} + +async fn sync_tree(root: &Path) -> Result<()> { + let mut pending = vec![root.to_owned()]; + let mut directories = Vec::new(); + while let Some(directory) = pending.pop() { + let mut entries = fs::read_dir(&directory).await?; + while let Some(entry) = entries.next_entry().await? { + let kind = entry.file_type().await?; + if kind.is_symlink() { + bail!("snapshot generation contains a symbolic link"); + } + if kind.is_dir() { + pending.push(entry.path()); + } else if kind.is_file() { + fs::File::open(entry.path()).await?.sync_all().await?; + } else { + bail!("snapshot generation contains a non-regular entry"); } - Err(error).with_context(|| { - format!( - "failed to replace snapshot {} with {}", - target.display(), - temp.display() - ) - }) } + directories.push(directory); + } + for directory in directories.into_iter().rev() { + sync_directory(&directory).await?; } + Ok(()) +} + +async fn sync_directory(directory: &Path) -> Result<()> { + #[cfg(unix)] + fs::File::open(directory) + .await + .context("open snapshot directory for durability")? + .sync_all() + .await + .context("sync snapshot directory")?; + #[cfg(not(unix))] + let _ = directory; + Ok(()) } fn unique_suffix() -> Result { @@ -924,6 +1101,131 @@ mod tests { assert!(path.is_ok()); } + async fn generation(root: &Path, folder: &str, version: &str) -> Result { + let directory = root.join(folder); + fs::create_dir_all(&directory).await?; + write_manifest( + &directory.join(MANIFEST_FILE), + &SnapshotManifest::new("dev", version)?, + ) + .await?; + Ok(directory) + } + + #[tokio::test] + async fn test_should_recover_gap_between_snapshot_renames() -> Result<()> { + let root = tempfile::tempdir()?; + let target = generation(root.path(), "dev", "old").await?; + fs::rename(&target, root.path().join(".dev.previous")).await?; + recover_snapshot(root.path(), &target, "dev").await?; + assert_eq!( + read_manifest(&target.join(MANIFEST_FILE)) + .await? + .rustack_version, + "old" + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_keep_valid_new_generation_after_publish_crash() -> Result<()> { + let root = tempfile::tempdir()?; + let target = generation(root.path(), "dev", "new").await?; + let previous = generation(root.path(), ".dev.previous", "old").await?; + recover_snapshot(root.path(), &target, "dev").await?; + assert_eq!( + read_manifest(&target.join(MANIFEST_FILE)) + .await? + .rustack_version, + "new" + ); + assert!(!path_exists(&previous).await?); + Ok(()) + } + + #[tokio::test] + async fn test_should_recover_old_when_published_generation_is_invalid() -> Result<()> { + let root = tempfile::tempdir()?; + let target = generation(root.path(), "dev", "new").await?; + generation(root.path(), ".dev.previous", "old").await?; + fs::write(target.join(MANIFEST_FILE), b"corrupt").await?; + recover_snapshot(root.path(), &target, "dev").await?; + assert_eq!( + read_manifest(&target.join(MANIFEST_FILE)) + .await? + .rustack_version, + "old" + ); + Ok(()) + } + + #[tokio::test] + async fn test_should_refuse_empty_start_with_invalid_recovery_generation() -> Result<()> { + let root = tempfile::tempdir()?; + fs::create_dir(root.path().join(".dev.previous")).await?; + let config = SnapshotConfig { + root: root.path().to_owned(), + name: SnapshotName::try_from("dev".to_owned())?, + }; + assert!(config.load(&RuntimeProviders::default()).await.is_err()); + Ok(()) + } + + #[tokio::test] + async fn test_should_recover_legacy_backup_and_refuse_ambiguous_backups() -> Result<()> { + let root = tempfile::tempdir()?; + let target = root.path().join("dev"); + generation(root.path(), ".dev.bak.legacy", "old").await?; + recover_snapshot(root.path(), &target, "dev").await?; + fs::rename(&target, root.path().join(".dev.bak.one")).await?; + generation(root.path(), ".dev.bak.two", "other").await?; + assert!(recover_snapshot(root.path(), &target, "dev").await.is_err()); + Ok(()) + } + + #[tokio::test] + async fn test_should_exclusively_lease_named_snapshot_until_owner_drops() -> Result<()> { + let root = tempfile::tempdir()?; + let config = SnapshotConfig { + root: root.path().to_owned(), + name: SnapshotName::try_from("dev".to_owned())?, + }; + let lease = config.acquire_lease().await?; + assert!(config.acquire_lease().await.is_err()); + drop(lease); + let _next = config.acquire_lease().await?; + Ok(()) + } + + #[tokio::test] + async fn test_should_publish_valid_generation_and_preserve_old_on_invalid_prepare() -> Result<()> + { + let root = tempfile::tempdir()?; + let target = generation(root.path(), "dev", "old").await?; + let staged = generation(root.path(), ".dev.tmp.new", "new").await?; + replace_directory(&staged, &target, root.path(), "dev").await?; + assert_eq!( + read_manifest(&target.join(MANIFEST_FILE)) + .await? + .rustack_version, + "new" + ); + let invalid = root.path().join(".dev.tmp.invalid"); + fs::create_dir(&invalid).await?; + assert!( + replace_directory(&invalid, &target, root.path(), "dev") + .await + .is_err() + ); + assert_eq!( + read_manifest(&target.join(MANIFEST_FILE)) + .await? + .rustack_version, + "new" + ); + Ok(()) + } + #[tokio::test] async fn test_should_round_trip_manifest_archive() -> Result<()> { let dir = tempfile::tempdir()?; diff --git a/apps/rustack/src/sns_bridge.rs b/apps/rustack/src/sns_bridge.rs index 570ff86..16a8d2c 100644 --- a/apps/rustack/src/sns_bridge.rs +++ b/apps/rustack/src/sns_bridge.rs @@ -1,82 +1,54 @@ -//! Bridge between SNS and SQS for fan-out delivery. -//! -//! Implements the [`SqsPublisher`] trait from `rustack-sns-core` by wrapping -//! the actual SQS provider. This bridge lives in the server binary to avoid -//! a direct dependency from `rustack-sns-core` to `rustack-sqs-core`. - +//! Application-owned SNS → SQS bridge; no core-to-core dependency. use std::sync::Arc; use async_trait::async_trait; -use rustack_sns_core::{ - config::SnsConfig, - publisher::{DeliveryError, SqsPublisher}, -}; +use rustack_sns_core::publisher::{DeliveryError, SqsPublisher}; use rustack_sqs_core::provider::RustackSqs; use rustack_sqs_model::input::SendMessageInput; -/// Production SQS publisher that delegates to the SQS provider. +/// Production publisher using the SQS provider's authoritative ARN/URL scope. #[derive(Debug)] pub struct RustackSqsPublisher { sqs: Arc, - account_id: String, - host: String, - port: u16, } - impl RustackSqsPublisher { - /// Create a new publisher wrapping the given SQS provider. - pub fn new(sqs: Arc, config: SnsConfig) -> Self { - Self { - sqs, - account_id: config.account_id, - host: config.host, - port: config.port, - } - } - - /// Convert an SQS queue ARN to a queue URL. - /// - /// ARN format: `arn:aws:sqs:{region}:{account}:{queue_name}` - /// URL format: `http://{host}:{port}/{account}/{queue_name}` - fn arn_to_queue_url(&self, queue_arn: &str) -> String { - let parts: Vec<&str> = queue_arn.split(':').collect(); - if parts.len() >= 6 { - let account = parts[4]; - let queue_name = parts[5]; - format!("http://{}:{}/{account}/{queue_name}", self.host, self.port) - } else { - // Fallback: use the ARN as-is (shouldn't happen with valid ARNs). - format!( - "http://{}:{}/{}/{}", - self.host, self.port, self.account_id, queue_arn - ) - } + /// Connect to an enabled SQS provider. + pub fn new(sqs: Arc) -> Self { + Self { sqs } } } - #[async_trait] impl SqsPublisher for RustackSqsPublisher { + fn validate(&self, arn: &str) -> Result<(), DeliveryError> { + self.sqs + .queue_url_for_arn(arn) + .map(|_| ()) + .map_err(|error| DeliveryError::Unsupported(error.to_string())) + } async fn send_message( &self, queue_arn: &str, - message_body: &str, - message_group_id: Option<&str>, - message_deduplication_id: Option<&str>, + body: &str, + group: Option<&str>, + dedup: Option<&str>, ) -> Result<(), DeliveryError> { - let queue_url = self.arn_to_queue_url(queue_arn); - let input = SendMessageInput { - queue_url, - message_body: message_body.to_string(), - message_group_id: message_group_id.map(String::from), - message_deduplication_id: message_deduplication_id.map(String::from), - ..SendMessageInput::default() - }; + self.validate(queue_arn)?; + let queue_url = self + .sqs + .queue_url_for_arn(queue_arn) + .map_err(|error| DeliveryError::Unsupported(error.to_string()))?; self.sqs - .send_message(input) + .send_message(SendMessageInput { + queue_url, + message_body: body.to_owned(), + message_group_id: group.map(str::to_owned), + message_deduplication_id: dedup.map(str::to_owned), + ..Default::default() + }) .await - .map_err(|e| DeliveryError::SqsDeliveryFailed { - queue_arn: queue_arn.to_string(), - reason: e.to_string(), + .map_err(|error| DeliveryError::SqsDeliveryFailed { + queue_arn: queue_arn.to_owned(), + reason: error.to_string(), })?; Ok(()) } @@ -84,20 +56,67 @@ impl SqsPublisher for RustackSqsPublisher { #[cfg(test)] mod tests { + use rustack_sns_core::{config::SnsConfig, provider::RustackSns}; use rustack_sqs_core::config::SqsConfig; + use rustack_sqs_model::input::{CreateQueueInput, ReceiveMessageInput}; use super::*; - #[test] - fn test_should_convert_arn_to_queue_url() { - let publisher = RustackSqsPublisher { - sqs: Arc::new(RustackSqs::new(SqsConfig::default())), - account_id: "000000000000".to_string(), - host: "localhost".to_string(), - port: 4566, - }; - - let url = publisher.arn_to_queue_url("arn:aws:sqs:us-east-1:000000000000:my-queue"); - assert_eq!(url, "http://localhost:4566/000000000000/my-queue"); + #[tokio::test] + async fn test_should_deliver_sns_fifo_to_real_queue_with_identity() { + let sqs = Arc::new(RustackSqs::new(SqsConfig::default())); + let queue = sqs + .create_queue( + serde_json::from_value::(serde_json::json!({ + "QueueName":"sns.fifo","Attributes":{"FifoQueue":"true"} + })) + .unwrap(), + ) + .await + .unwrap() + .queue_url + .unwrap(); + let sns = RustackSns::new( + SnsConfig::default(), + Arc::new(RustackSqsPublisher::new(sqs.clone())), + ); + let topic = sns + .create_topic( + serde_json::from_value( + serde_json::json!({"Name":"topic.fifo","Attributes":{"FifoTopic":"true"}}), + ) + .unwrap(), + ) + .unwrap() + .topic_arn; + sns.subscribe(serde_json::from_value(serde_json::json!({"TopicArn":topic,"Protocol":"sqs","Endpoint":"arn:aws:sqs:us-east-1:000000000000:sns.fifo"})).unwrap()).unwrap(); + sns.publish(serde_json::from_value(serde_json::json!({"TopicArn":topic,"Message":"payload","MessageGroupId":"g","MessageDeduplicationId":"d"})).unwrap()).await.unwrap(); + sns.quiesce().await; + let messages = sqs + .receive_message(ReceiveMessageInput { + queue_url: queue, + message_system_attribute_names: vec!["All".into()], + ..Default::default() + }) + .await + .unwrap() + .messages; + assert_eq!(messages.len(), 1); + assert_eq!( + messages[0] + .attributes + .get("MessageGroupId") + .map(String::as_str), + Some("g") + ); + assert_eq!( + messages[0] + .attributes + .get("MessageDeduplicationId") + .map(String::as_str), + Some("d") + ); + assert_eq!(sns.delivery_stats().delivered, 1); + sqs.shutdown_all().await; } } diff --git a/codegen/Cargo.lock b/codegen/Cargo.lock index 3ca8768..b8f117e 100644 --- a/codegen/Cargo.lock +++ b/codegen/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "equivalent" @@ -67,8 +67,8 @@ dependencies = [ ] [[package]] -name = "ruststack-codegen" -version = "0.1.0" +name = "rustack-codegen" +version = "0.1.1" dependencies = [ "anyhow", "heck", @@ -122,11 +122,11 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.9" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -142,44 +142,42 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.23" +version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "serde", + "indexmap", + "serde_core", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_parser", + "toml_writer", + "winnow 0.7.15", ] [[package]] name = "toml_datetime" -version = "0.6.11" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ - "serde", + "serde_core", ] [[package]] -name = "toml_edit" -version = "0.22.27" +name = "toml_parser" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", - "winnow", + "winnow 1.0.4", ] [[package]] -name = "toml_write" -version = "0.1.2" +name = "toml_writer" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" [[package]] name = "unicode-ident" @@ -192,9 +190,12 @@ name = "winnow" version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" [[package]] name = "zmij" diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 970a1b4..018d838 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustack-codegen" -version = "0.1.0" +version = "0.1.1" edition = "2024" [dependencies] diff --git a/codegen/src/codegen.rs b/codegen/src/codegen.rs index 7f292a0..0331563 100644 --- a/codegen/src/codegen.rs +++ b/codegen/src/codegen.rs @@ -1255,6 +1255,12 @@ fn generate_error_s3(config: &ServiceConfig) -> Result { // Error struct writeln!(out, "/// An {prefix} error response.")?; + writeln!(out, "///")?; + writeln!( + out, + "/// Immutable resource and request-ID strings omit spare capacity to keep errors" + )?; + writeln!(out, "/// compact when returned by value.")?; writeln!(out, "#[derive(Debug)]")?; writeln!(out, "pub struct {prefix}Error {{")?; writeln!(out, " /// The error code.")?; @@ -1262,9 +1268,9 @@ fn generate_error_s3(config: &ServiceConfig) -> Result { writeln!(out, " /// A human-readable error message.")?; writeln!(out, " pub message: String,")?; writeln!(out, " /// The resource that caused the error.")?; - writeln!(out, " pub resource: Option,")?; + writeln!(out, " pub resource: Option>,")?; writeln!(out, " /// The request ID.")?; - writeln!(out, " pub request_id: Option,")?; + writeln!(out, " pub request_id: Option>,")?; writeln!(out, " /// The HTTP status code.")?; writeln!(out, " pub status_code: http::StatusCode,")?; writeln!(out, " /// The underlying source error, if any.")?; @@ -1353,7 +1359,10 @@ fn generate_error_s3(config: &ServiceConfig) -> Result { out, " pub fn with_resource(mut self, resource: impl Into) -> Self {{" )?; - writeln!(out, " self.resource = Some(resource.into());")?; + writeln!( + out, + " self.resource = Some(resource.into().into_boxed_str());" + )?; writeln!(out, " self")?; writeln!(out, " }}")?; writeln!(out)?; @@ -1364,7 +1373,10 @@ fn generate_error_s3(config: &ServiceConfig) -> Result { out, " pub fn with_request_id(mut self, request_id: impl Into) -> Self {{" )?; - writeln!(out, " self.request_id = Some(request_id.into());")?; + writeln!( + out, + " self.request_id = Some(request_id.into().into_boxed_str());" + )?; writeln!(out, " self")?; writeln!(out, " }}")?; writeln!(out)?; diff --git a/crates/rustack-apigatewayv2-core/Cargo.toml b/crates/rustack-apigatewayv2-core/Cargo.toml index 408bee5..b3ef5b9 100644 --- a/crates/rustack-apigatewayv2-core/Cargo.toml +++ b/crates/rustack-apigatewayv2-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-apigatewayv2-core" description = "API Gateway v2 core business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-apigatewayv2-core/src/config.rs b/crates/rustack-apigatewayv2-core/src/config.rs index fb9bd9d..3cf3760 100644 --- a/crates/rustack-apigatewayv2-core/src/config.rs +++ b/crates/rustack-apigatewayv2-core/src/config.rs @@ -1,6 +1,6 @@ //! API Gateway v2 service configuration. -use std::env; +use rustack_core::settings as env; /// API Gateway v2 service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-apigatewayv2-core/src/execution/http_proxy.rs b/crates/rustack-apigatewayv2-core/src/execution/http_proxy.rs index e68dd7a..af5b974 100644 --- a/crates/rustack-apigatewayv2-core/src/execution/http_proxy.rs +++ b/crates/rustack-apigatewayv2-core/src/execution/http_proxy.rs @@ -1,6 +1,7 @@ //! HTTP proxy integration for API Gateway v2. //! //! Forwards requests to the configured HTTP endpoint and returns the response. +//! Redirects are returned verbatim; upstream authority is never selected by Location. use bytes::Bytes; @@ -23,11 +24,51 @@ pub async fn handle_http_proxy( ApiGatewayV2ServiceError::Internal("HTTP integration has no URI".to_owned()) })?; - let target_url = format!("{base_uri}{path}"); + let base = reqwest::Url::parse(base_uri).map_err(|_| { + ApiGatewayV2ServiceError::BadRequest("Invalid HTTP integration URL".to_owned()) + })?; + if !matches!(base.scheme(), "http" | "https") + || base.host_str().is_none() + || !base.username().is_empty() + || base.password().is_some() + || base.fragment().is_some() + { + return Err(ApiGatewayV2ServiceError::BadRequest( + "HTTP integration requires an HTTP(S) authority without credentials or fragments" + .to_owned(), + )); + } + let target_url = reqwest::Url::parse(&format!("{base_uri}{path}")).map_err(|_| { + ApiGatewayV2ServiceError::BadRequest("Invalid HTTP integration request URL".to_owned()) + })?; + if target_url.origin() != base.origin() || target_url.fragment().is_some() { + return Err(ApiGatewayV2ServiceError::BadRequest( + "HTTP request must preserve integration authority".to_owned(), + )); + } let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes()) .map_err(|e| ApiGatewayV2ServiceError::BadRequest(format!("Invalid HTTP method: {e}")))?; - let mut request = provider.http_client().request(reqwest_method, &target_url); + let body_total_seconds = rustack_core::settings::budgets().body_total_seconds; + let total = match integration.timeout_in_millis { + Some(milliseconds) => { + if !(1..=30_000).contains(&milliseconds) { + return Err(ApiGatewayV2ServiceError::BadRequest( + "HTTP integration timeout must be between 1 and 30000 milliseconds".to_owned(), + )); + } + let milliseconds = u64::try_from(milliseconds).map_err(|_| { + ApiGatewayV2ServiceError::BadRequest("Invalid integration timeout".to_owned()) + })?; + std::time::Duration::from_millis(milliseconds) + } + // Snapshot-imported or legacy records without a stored timeout still get a + // wall-clock bound instead of an idle-only deadline. + None => std::time::Duration::from_secs(body_total_seconds.min(30)), + } + .min(std::time::Duration::from_secs(body_total_seconds)); + let mut request = provider.http_client().request(reqwest_method, target_url); + request = request.timeout(total); // Forward headers (skip host header) for (name, value) in headers { @@ -42,17 +83,40 @@ pub async fn handle_http_proxy( request = request.body(body.to_vec()); } - let response = request.send().await.map_err(|e| { + let mut response = request.send().await.map_err(|e| { ApiGatewayV2ServiceError::IntegrationError(format!("HTTP proxy request failed: {e}")) })?; let status = response.status().as_u16(); let resp_headers = response.headers().clone(); - let resp_body = response.bytes().await.map_err(|e| { - ApiGatewayV2ServiceError::IntegrationError(format!( - "Failed to read HTTP proxy response: {e}" - )) + let budgets = rustack_core::settings::budgets(); + let max_body = usize::try_from(budgets.upstream_body_bytes).map_err(|_| { + ApiGatewayV2ServiceError::Internal("invalid upstream byte budget".to_owned()) })?; + let mut resp_body = Vec::new(); + let idle = std::time::Duration::from_secs(budgets.body_idle_seconds); + let mut idle_deadline = tokio::time::Instant::now() + idle; + let total_deadline = tokio::time::Instant::now() + total; + while let Some(chunk) = + tokio::time::timeout_at(total_deadline.min(idle_deadline), response.chunk()) + .await + .map_err(|_| { + ApiGatewayV2ServiceError::IntegrationError( + "HTTP proxy response deadline exceeded".to_owned(), + ) + })? + .map_err(|e| { + ApiGatewayV2ServiceError::IntegrationError(format!( + "Failed to read HTTP proxy response: {e}" + )) + })? + { + if !chunk.is_empty() { + idle_deadline = tokio::time::Instant::now() + idle; + } + rustack_core::http::append_bounded(&mut resp_body, &chunk, max_body as u64) + .map_err(|error| ApiGatewayV2ServiceError::IntegrationError(error.to_string()))?; + } let mut builder = http::Response::builder().status(status); for (name, value) in &resp_headers { @@ -60,6 +124,123 @@ pub async fn handle_http_proxy( } builder - .body(Bytes::from(resp_body.to_vec())) + .body(Bytes::from(resp_body)) .map_err(|e| ApiGatewayV2ServiceError::Internal(format!("Failed to build response: {e}"))) } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; + + use super::*; + + fn integration(uri: &str) -> IntegrationRecord { + serde_json::from_value(serde_json::json!({ + "integrationId": "test", "integrationType": "HTTP_PROXY", "integrationUri": uri, + "requestParameters": {}, "requestTemplates": {}, "responseParameters": {}, + "apiGatewayManaged": false, "timeoutInMillis": 100, + })) + .unwrap() + } + + #[tokio::test] + async fn test_should_ignore_environment_proxy_in_isolated_process() { + let mut child = tokio::process::Command::new(std::env::current_exe().unwrap()); + child.args(["--exact", "execution::http_proxy::tests::test_should_return_all_redirects_without_contacting_second_origin"]) + .env("HTTP_PROXY", "http://127.0.0.1:9").env("HTTPS_PROXY", "http://127.0.0.1:9") + .env("ALL_PROXY", "http://127.0.0.1:9").env("http_proxy", "http://127.0.0.1:9") + .env("https_proxy", "http://127.0.0.1:9").env("all_proxy", "http://127.0.0.1:9") + .env_remove("NO_PROXY").env_remove("no_proxy").kill_on_drop(true); + let output = tokio::time::timeout(Duration::from_secs(10), child.output()) + .await + .unwrap() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stdout) + ); + } + + #[tokio::test] + async fn test_should_return_all_redirects_without_contacting_second_origin() { + let provider = + RustackApiGatewayV2::new(crate::config::ApiGatewayV2Config::default()).unwrap(); + for status in [301, 302, 303, 307, 308] { + for style in ["absolute", "relative", "scheme-relative", "ipv6", "mapped"] { + let first = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let second = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let location = match style { + "relative" => "/second".to_owned(), + "scheme-relative" => format!("//{}/second", second.local_addr().unwrap()), + "ipv6" => "http://[::1]:9/second".to_owned(), + "mapped" => "http://[::ffff:127.0.0.1]:9/second".to_owned(), + _ => format!("http://{}/second", second.local_addr().unwrap()), + }; + let record = integration(&format!("http://{}", first.local_addr().unwrap())); + let wire = format!( + "HTTP/1.1 {status} Redirect\r\nLocation: {location}\r\nContent-Length: \ + 5\r\nConnection: close\r\n\r\nfirst" + ); + let server = async { + let (mut socket, _) = first.accept().await.unwrap(); + let mut buffer = [0; 4096]; + assert!(socket.read(&mut buffer).await.unwrap() > 0); + socket.write_all(wire.as_bytes()).await.unwrap(); + }; + let headers = http::HeaderMap::new(); + let request = + handle_http_proxy(&provider, &record, &http::Method::GET, "/", &headers, &[]); + let ((), result) = tokio::join!(server, request); + let response = result.unwrap(); + assert_eq!(response.status().as_u16(), status); + assert_eq!( + response.headers().get("location").unwrap(), + location.as_str() + ); + assert_eq!(response.body().as_ref(), b"first"); + assert!( + tokio::time::timeout(Duration::from_millis(2), second.accept()) + .await + .is_err() + ); + } + } + } + + #[tokio::test] + async fn test_should_apply_total_deadline_to_upstream_body_without_eof() { + let provider = + RustackApiGatewayV2::new(crate::config::ApiGatewayV2Config::default()).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let record = integration(&format!("http://{}", listener.local_addr().unwrap())); + let server = async { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buffer = [0; 4096]; + assert!(socket.read(&mut buffer).await.unwrap() > 0); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nx") + .await + .unwrap(); + assert_eq!( + tokio::time::timeout(Duration::from_secs(1), socket.read(&mut buffer)) + .await + .unwrap() + .unwrap(), + 0 + ); + }; + let headers = http::HeaderMap::new(); + let request = handle_http_proxy(&provider, &record, &http::Method::GET, "/", &headers, &[]); + let ((), result) = tokio::join!(server, request); + assert!(matches!( + result, + Err(ApiGatewayV2ServiceError::IntegrationError(_)) + )); + } +} diff --git a/crates/rustack-apigatewayv2-core/src/provider.rs b/crates/rustack-apigatewayv2-core/src/provider.rs index 60d1b7d..ce1eaa2 100644 --- a/crates/rustack-apigatewayv2-core/src/provider.rs +++ b/crates/rustack-apigatewayv2-core/src/provider.rs @@ -45,13 +45,28 @@ pub struct RustackApiGatewayV2 { )] impl RustackApiGatewayV2 { /// Create a new provider with the given configuration. - #[must_use] - pub fn new(config: ApiGatewayV2Config) -> Self { - Self { + /// + /// # Errors + /// Returns an error if the explicitly isolated upstream client cannot be built. + pub fn new(config: ApiGatewayV2Config) -> Result { + let http_client = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .no_proxy() + .connect_timeout(std::time::Duration::from_secs(5)) + .timeout(std::time::Duration::from_secs( + rustack_core::settings::budgets().body_total_seconds.min(30), + )) + .build() + .map_err(|e| { + ApiGatewayV2ServiceError::Internal(format!( + "Failed to build HTTP proxy client: {e}" + )) + })?; + Ok(Self { store: ApiStore::new(), config: Arc::new(config), - http_client: reqwest::Client::new(), - } + http_client, + }) } /// Returns a reference to the API store. diff --git a/crates/rustack-apigatewayv2-http/Cargo.toml b/crates/rustack-apigatewayv2-http/Cargo.toml index 3a76212..015ab11 100644 --- a/crates/rustack-apigatewayv2-http/Cargo.toml +++ b/crates/rustack-apigatewayv2-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-apigatewayv2-http" description = "API Gateway v2 HTTP service layer for Rustack (restJson1 protocol)" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-apigatewayv2-model = { workspace = true } rustack-auth = { workspace = true } hyper = { workspace = true } diff --git a/crates/rustack-apigatewayv2-http/src/service.rs b/crates/rustack-apigatewayv2-http/src/service.rs index b6e6a56..235d458 100644 --- a/crates/rustack-apigatewayv2-http/src/service.rs +++ b/crates/rustack-apigatewayv2-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_apigatewayv2_model::error::ApiGatewayV2Error; @@ -124,19 +123,17 @@ async fn process_request( }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = ApiGatewayV2Error::with_message( - rustack_apigatewayv2_model::error::ApiGatewayV2ErrorCode::AccessDeniedException, - auth_err.to_string(), - ); - return wrap_error_response(&err); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let err = ApiGatewayV2Error::with_message( + rustack_apigatewayv2_model::error::ApiGatewayV2ErrorCode::AccessDeniedException, + auth_err.to_string(), + ); + return wrap_error_response(&err); } // 5. Dispatch to handler. @@ -174,11 +171,13 @@ fn wrap_error_response(error: &ApiGatewayV2Error) -> http::Response Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| ApiGatewayV2Error::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = ApiGatewayV2Error::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every API Gateway v2 response. diff --git a/crates/rustack-auth/Cargo.toml b/crates/rustack-auth/Cargo.toml index 841df2a..2ed9e4f 100644 --- a/crates/rustack-auth/Cargo.toml +++ b/crates/rustack-auth/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-auth" description = "AWS Signature Version 2 and 4 request authentication for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-auth/src/error.rs b/crates/rustack-auth/src/error.rs index 5220b53..11349eb 100644 --- a/crates/rustack-auth/src/error.rs +++ b/crates/rustack-auth/src/error.rs @@ -6,6 +6,13 @@ /// Errors that can occur during AWS Signature Version 4 authentication. #[derive(Debug, thiserror::Error)] pub enum AuthError { + /// Strict authentication requires a credential provider. + #[error("Strict authentication requires a credential provider")] + MissingCredentialProvider, + + /// The payload declaration is malformed or does not match the actual body. + #[error("Invalid or mismatching payload SHA256")] + InvalidPayloadHash, /// The `Authorization` header is missing from the request. #[error("Missing Authorization header")] MissingAuthHeader, diff --git a/crates/rustack-auth/src/lib.rs b/crates/rustack-auth/src/lib.rs index 7165970..435bd06 100644 --- a/crates/rustack-auth/src/lib.rs +++ b/crates/rustack-auth/src/lib.rs @@ -39,6 +39,8 @@ pub mod canonical; pub mod credentials; pub mod error; +pub mod mode; +pub use mode::AuthMode; pub mod presigned; pub mod sigv2; pub mod sigv4; diff --git a/crates/rustack-auth/src/mode.rs b/crates/rustack-auth/src/mode.rs new file mode 100644 index 0000000..6243fa5 --- /dev/null +++ b/crates/rustack-auth/src/mode.rs @@ -0,0 +1,73 @@ +//! Fail-closed resolution of legacy HTTP authentication configuration. + +use crate::{AuthError, AuthResult, CredentialProvider, verify_sigv4}; + +/// Valid authentication states at an HTTP trust boundary. +#[derive(Clone, Copy)] +#[non_exhaustive] +pub enum AuthMode<'a> { + /// Explicit local development mode; no signature validation. + Development, + /// Every request must authenticate using this credential provider. + Required(&'a dyn CredentialProvider), +} + +impl std::fmt::Debug for AuthMode<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Development => "Development", + Self::Required(_) => "Required([REDACTED])", + }) + } +} + +impl<'a> AuthMode<'a> { + /// Resolve legacy service configuration without permitting strict-mode fallback. + /// + /// # Errors + /// Returns an error if strict mode has no credential provider. + pub fn resolve( + skip_signature_validation: bool, + provider: Option<&'a dyn CredentialProvider>, + ) -> Result { + if skip_signature_validation { + Ok(Self::Development) + } else { + provider + .map(Self::Required) + .ok_or(AuthError::MissingCredentialProvider) + } + } + + /// Verify an ordinary, fully received request in the selected mode. + /// + /// # Errors + /// Returns the signature or payload validation failure in required mode. + pub fn verify( + self, + parts: &http::request::Parts, + actual_body_hash: &str, + ) -> Result, AuthError> { + match self { + Self::Development => Ok(None), + Self::Required(provider) => verify_sigv4(parts, actual_body_hash, provider).map(Some), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_reject_strict_without_provider() { + assert!(matches!( + AuthMode::resolve(false, None), + Err(AuthError::MissingCredentialProvider) + )); + assert!(matches!( + AuthMode::resolve(true, None), + Ok(AuthMode::Development) + )); + } +} diff --git a/crates/rustack-auth/src/presigned.rs b/crates/rustack-auth/src/presigned.rs index 924ab15..713ec84 100644 --- a/crates/rustack-auth/src/presigned.rs +++ b/crates/rustack-auth/src/presigned.rs @@ -173,7 +173,7 @@ pub fn verify_presigned( "{method}\n{canonical_uri}\n{canonical_query}\n{canonical_headers}\n\n{signed_headers_str}\n{UNSIGNED_PAYLOAD}" ); - debug!(canonical_request, "Built presigned canonical request"); + debug!("Built presigned canonical request"); // Hash the canonical request. let canonical_hash = hex::encode(Sha256::digest(canonical_request.as_bytes())); @@ -206,11 +206,7 @@ pub fn verify_presigned( signed_headers: parsed.signed_headers, }) } else { - debug!( - expected = %expected_signature, - provided = %parsed.signature, - "Presigned URL signature mismatch" - ); + debug!("Presigned URL signature mismatch"); Err(AuthError::SignatureDoesNotMatch) } } diff --git a/crates/rustack-auth/src/sigv2.rs b/crates/rustack-auth/src/sigv2.rs index 4a932e3..384af58 100644 --- a/crates/rustack-auth/src/sigv2.rs +++ b/crates/rustack-auth/src/sigv2.rs @@ -61,7 +61,7 @@ pub fn verify_sigv2( let string_to_sign = build_string_to_sign(parts); - debug!(string_to_sign = ?string_to_sign, "Built SigV2 string to sign"); + debug!("Built SigV2 string to sign"); let expected_signature = compute_sigv2_signature(&secret_key, &string_to_sign); @@ -78,11 +78,7 @@ pub fn verify_sigv2( signed_headers: Vec::new(), }) } else { - debug!( - expected = %expected_signature, - provided = %provided_signature, - "SigV2 signature mismatch" - ); + debug!("SigV2 signature mismatch"); Err(AuthError::SignatureDoesNotMatch) } } diff --git a/crates/rustack-auth/src/sigv4.rs b/crates/rustack-auth/src/sigv4.rs index 2e0afd3..7079b07 100644 --- a/crates/rustack-auth/src/sigv4.rs +++ b/crates/rustack-auth/src/sigv4.rs @@ -211,6 +211,39 @@ pub fn verify_sigv4( body_hash: &str, credential_provider: &dyn CredentialProvider, ) -> Result { + verify_sigv4_with_policy(parts, body_hash, credential_provider, false) +} + +/// Verify S3 SigV4, allowing the explicit unsigned-payload protocol exception. +/// +/// This verifies the seed signature only for streaming markers. Callers MUST additionally +/// verify every chunk and signed trailer with [`StreamingVerifier`] before publishing data. +/// # Errors +/// Returns an authentication error for invalid signatures or payload declarations. +pub fn verify_s3_sigv4( + parts: &http::request::Parts, + body_hash: &str, + credential_provider: &dyn CredentialProvider, +) -> Result { + verify_sigv4_with_policy(parts, body_hash, credential_provider, true) +} + +fn verify_sigv4_with_policy( + parts: &http::request::Parts, + body_hash: &str, + credential_provider: &dyn CredentialProvider, + allow_unsigned: bool, +) -> Result { + let payload_hash = validated_payload_hash(parts, body_hash, allow_unsigned)?; + if parts + .headers + .get_all(http::header::AUTHORIZATION) + .iter() + .count() + > 1 + { + return Err(AuthError::InvalidAuthHeader); + } // Extract and parse the Authorization header. let auth_header = parts .headers @@ -219,7 +252,7 @@ pub fn verify_sigv4( .to_str() .map_err(|_| AuthError::InvalidAuthHeader)?; - debug!(auth_header, "Parsing SigV4 authorization header"); + debug!("Parsing SigV4 authorization header"); let parsed = parse_authorization_header(auth_header)?; @@ -246,15 +279,6 @@ pub fn verify_sigv4( let signed_header_refs: Vec<&str> = parsed.signed_headers.iter().map(String::as_str).collect(); let header_pairs: Vec<(&str, &str)> = collect_signed_headers(parts, &signed_header_refs)?; - // Use the x-amz-content-sha256 header value (what the client signed with) - // rather than the recomputed body hash. This is critical because the client - // may use STREAMING-* placeholders or compute the hash before encoding. - let payload_hash = parts - .headers - .get("x-amz-content-sha256") - .and_then(|v| v.to_str().ok()) - .unwrap_or(body_hash); - let canonical_request = build_canonical_request( method, uri, @@ -292,15 +316,122 @@ pub fn verify_sigv4( signed_headers: parsed.signed_headers, }) } else { - debug!( - expected = %expected_signature, - provided = %parsed.signature, - "Signature mismatch" - ); + debug!("Signature mismatch"); Err(AuthError::SignatureDoesNotMatch) } } +/// SigV4 streaming HMAC chain, including terminal chunks and signed trailers. +pub struct StreamingVerifier { + key: Vec, + timestamp: String, + scope: String, + previous: String, +} + +impl std::fmt::Debug for StreamingVerifier { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("StreamingVerifier([REDACTED])") + } +} + +impl StreamingVerifier { + /// Authenticate the seed and initialize the chunk chain. + /// # Errors + /// Returns seed signature or header validation failures. + pub fn new( + parts: &http::request::Parts, + provider: &dyn CredentialProvider, + ) -> Result { + verify_s3_sigv4(parts, &hash_payload(b""), provider)?; + let header = extract_header_value(parts, "authorization")?; + let parsed = parse_authorization_header(&header)?; + let secret = provider.get_secret_key(&parsed.access_key_id)?; + Ok(Self { + key: derive_signing_key(&secret, &parsed.date, &parsed.region, &parsed.service), + timestamp: extract_header_value(parts, "x-amz-date")?, + scope: format!( + "{}/{}/{}/aws4_request", + parsed.date, parsed.region, parsed.service + ), + previous: parsed.signature, + }) + } + + /// Verify a data chunk, including the final zero-size chunk. + /// # Errors + /// Rejects a missing, malformed or mismatching chunk signature. + pub fn verify_chunk(&mut self, actual_sha256: &str, signature: &str) -> Result<(), AuthError> { + let text = format!( + "AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}", + self.timestamp, + self.scope, + self.previous, + hash_payload(b""), + actual_sha256 + ); + self.verify_next(&text, signature) + } + + /// Verify the canonical declared trailer block after the terminal chunk. + /// # Errors + /// Returns an error if the trailer signature does not match. + pub fn verify_trailer(&mut self, canonical: &str, signature: &str) -> Result<(), AuthError> { + let text = format!( + "AWS4-HMAC-SHA256-TRAILER\n{}\n{}\n{}\n{}", + self.timestamp, + self.scope, + self.previous, + hash_payload(canonical.as_bytes()) + ); + self.verify_next(&text, signature) + } + + fn verify_next(&mut self, text: &str, signature: &str) -> Result<(), AuthError> { + let expected = compute_signature(&self.key, text); + if !bool::from(expected.as_bytes().ct_eq(signature.as_bytes())) { + return Err(AuthError::SignatureDoesNotMatch); + } + self.previous = expected; + Ok(()) + } +} + +fn validated_payload_hash<'a>( + parts: &'a http::request::Parts, + actual: &'a str, + allow_unsigned: bool, +) -> Result<&'a str, AuthError> { + let mut values = parts.headers.get_all("x-amz-content-sha256").iter(); + let Some(value) = values.next() else { + return Ok(actual); + }; + if values.next().is_some() { + return Err(AuthError::InvalidPayloadHash); + } + let declared = value.to_str().map_err(|_| AuthError::InvalidPayloadHash)?; + if allow_unsigned + && matches!( + declared, + "UNSIGNED-PAYLOAD" + | "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + | "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" + | "STREAMING-UNSIGNED-PAYLOAD-TRAILER" + ) + { + return Ok(declared); + } + if declared.len() != 64 + || !declared + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + || !bool::from(declared.as_bytes().ct_eq(actual.as_bytes())) + { + return Err(AuthError::InvalidPayloadHash); + } + Ok(actual) +} + /// Extract a header value as a string from the request parts. fn extract_header_value(parts: &http::request::Parts, name: &str) -> Result { parts @@ -459,6 +590,120 @@ mod tests { ); } + fn signed_request(payload_hash: &str, declared: bool) -> http::request::Parts { + let (mut parts, ()) = http::Request::builder() + .method("POST") + .uri("/") + .header("host", "localhost") + .header("x-amz-date", "20130524T000000Z") + .body(()) + .unwrap() + .into_parts(); + let headers = [("host", "localhost"), ("x-amz-date", "20130524T000000Z")]; + let signed = ["host", "x-amz-date"]; + let canonical = build_canonical_request("POST", "/", "", &headers, &signed, payload_hash); + let text = build_string_to_sign( + "20130524T000000Z", + "20130524/us-east-1/s3/aws4_request", + &hash_payload(canonical.as_bytes()), + ); + let signature = compute_signature( + &derive_signing_key(TEST_SECRET_KEY, "20130524", "us-east-1", "s3"), + &text, + ); + parts.headers.insert( + "authorization", + format!( + "AWS4-HMAC-SHA256 \ + Credential={TEST_ACCESS_KEY}/20130524/us-east-1/s3/aws4_request,\ + SignedHeaders=host;x-amz-date,Signature={signature}" + ) + .parse() + .unwrap(), + ); + if declared { + parts + .headers + .insert("x-amz-content-sha256", payload_hash.parse().unwrap()); + } + parts + } + + #[test] + fn test_should_bind_actual_payload_even_with_unsigned_hash_header() { + let provider = test_credential_provider(); + let original = hash_payload(b"original"); + for declared in [false, true] { + let mut parts = signed_request(&original, declared); + assert!(verify_sigv4(&parts, &original, &provider).is_ok()); + assert!(verify_sigv4(&parts, &hash_payload(b"modified"), &provider).is_err()); + parts + .headers + .insert("x-amz-content-sha256", original.parse().unwrap()); + assert!(matches!( + verify_sigv4(&parts, &hash_payload(b"modified"), &provider), + Err(AuthError::InvalidPayloadHash) + )); + } + } + + #[test] + fn test_should_reject_duplicate_malformed_and_protocol_hashes() { + let provider = test_credential_provider(); + let actual = hash_payload(b"original"); + let mut parts = signed_request(&actual, true); + parts + .headers + .append("x-amz-content-sha256", actual.parse().unwrap()); + assert!(matches!( + verify_sigv4(&parts, &actual, &provider), + Err(AuthError::InvalidPayloadHash) + )); + for value in [ + "invalid", + "UNSIGNED-PAYLOAD", + "STREAMING-UNSIGNED-PAYLOAD-TRAILER", + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", + "UNSIGNED-PAYLOAD-UNKNOWN", + ] { + let parts = signed_request(value, true); + assert!(matches!( + verify_sigv4(&parts, &actual, &provider), + Err(AuthError::InvalidPayloadHash) + )); + } + let parts = signed_request("UNSIGNED-PAYLOAD", true); + assert!(verify_s3_sigv4(&parts, &actual, &provider).is_ok()); + } + + #[test] + fn test_should_verify_streaming_chunk_chain_and_reject_tampering() { + let provider = test_credential_provider(); + let parts = signed_request("STREAMING-AWS4-HMAC-SHA256-PAYLOAD", true); + let mut verifier = StreamingVerifier::new(&parts, &provider).unwrap(); + let digest = hash_payload(b"chunk"); + let text = format!( + "AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}", + verifier.timestamp, + verifier.scope, + verifier.previous, + hash_payload(b""), + digest + ); + let signature = compute_signature(&verifier.key, &text); + assert!( + verifier + .verify_chunk(&hash_payload(b"wrong"), &signature) + .is_err() + ); + assert!(verifier.verify_chunk(&digest, &signature).is_ok()); + assert!( + verifier.verify_chunk(&digest, &signature).is_err(), + "a chunk cannot be replayed at the next chain position" + ); + assert!(!format!("{verifier:?}").contains(&signature)); + } + #[test] fn test_should_verify_sigv4_success() { let provider = test_credential_provider(); diff --git a/crates/rustack-cloudfront-core/Cargo.toml b/crates/rustack-cloudfront-core/Cargo.toml index 01f7c40..e356a75 100644 --- a/crates/rustack-cloudfront-core/Cargo.toml +++ b/crates/rustack-cloudfront-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-cloudfront-core" description = "CloudFront business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-cloudfront-core/src/config.rs b/crates/rustack-cloudfront-core/src/config.rs index 6f657dc..854cf8d 100644 --- a/crates/rustack-cloudfront-core/src/config.rs +++ b/crates/rustack-cloudfront-core/src/config.rs @@ -49,31 +49,31 @@ impl CloudFrontConfig { pub fn from_env() -> Self { let mut cfg = Self::default(); - if let Ok(v) = std::env::var("CLOUDFRONT_SKIP_SIGNATURE_VALIDATION") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_SKIP_SIGNATURE_VALIDATION") { cfg.skip_signature_validation = parse_bool(&v).unwrap_or(cfg.skip_signature_validation); } - if let Ok(v) = std::env::var("AWS_DEFAULT_REGION") { + if let Ok(v) = rustack_core::settings::var("AWS_DEFAULT_REGION") { cfg.default_region = v; } - if let Ok(v) = - std::env::var("ACCOUNT_ID").or_else(|_| std::env::var("CLOUDFRONT_ACCOUNT_ID")) + if let Ok(v) = rustack_core::settings::var("ACCOUNT_ID") + .or_else(|_| rustack_core::settings::var("CLOUDFRONT_ACCOUNT_ID")) { cfg.account_id = v; } - if let Ok(v) = std::env::var("CLOUDFRONT_DOMAIN_SUFFIX") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_DOMAIN_SUFFIX") { cfg.domain_suffix = v; } - if let Ok(v) = std::env::var("CLOUDFRONT_DISTRIBUTION_PROPAGATION_MS") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_DISTRIBUTION_PROPAGATION_MS") { if let Ok(ms) = v.parse::() { cfg.distribution_propagation = Duration::from_millis(ms); } } - if let Ok(v) = std::env::var("CLOUDFRONT_INVALIDATION_PROPAGATION_MS") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_INVALIDATION_PROPAGATION_MS") { if let Ok(ms) = v.parse::() { cfg.invalidation_propagation = Duration::from_millis(ms); } } - if let Ok(v) = std::env::var("CLOUDFRONT_DETERMINISTIC_IDS") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_DETERMINISTIC_IDS") { cfg.deterministic_ids = parse_bool(&v).unwrap_or(false); } diff --git a/crates/rustack-cloudfront-dataplane/Cargo.toml b/crates/rustack-cloudfront-dataplane/Cargo.toml index b062db1..cbd5997 100644 --- a/crates/rustack-cloudfront-dataplane/Cargo.toml +++ b/crates/rustack-cloudfront-dataplane/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-cloudfront-dataplane" description = "CloudFront minimal pass-through data plane for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-cloudfront-core = { workspace = true } rustack-cloudfront-model = { workspace = true } rustack-s3-core = { workspace = true } diff --git a/crates/rustack-cloudfront-dataplane/src/config.rs b/crates/rustack-cloudfront-dataplane/src/config.rs index f07ee4b..1803fb1 100644 --- a/crates/rustack-cloudfront-dataplane/src/config.rs +++ b/crates/rustack-cloudfront-dataplane/src/config.rs @@ -39,32 +39,32 @@ impl DataPlaneConfig { #[must_use] pub fn from_env() -> Self { let mut cfg = Self::default(); - if let Ok(v) = std::env::var("CLOUDFRONT_DOMAIN_SUFFIX") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_DOMAIN_SUFFIX") { cfg.domain_suffix = v; } - if let Ok(v) = std::env::var("CLOUDFRONT_FAIL_ON_FUNCTION") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_FAIL_ON_FUNCTION") { cfg.fail_on_function = matches!( v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on" ); } - if let Ok(v) = std::env::var("CLOUDFRONT_FORWARD_USER_METADATA") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_FORWARD_USER_METADATA") { cfg.forward_user_metadata = matches!( v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on" ); } - if let Ok(v) = std::env::var("CLOUDFRONT_MAX_UPSTREAM_BODY_BYTES") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_MAX_UPSTREAM_BODY_BYTES") { if let Ok(n) = v.parse() { cfg.max_upstream_body_bytes = n; } } - if let Ok(v) = std::env::var("CLOUDFRONT_HTTP_ORIGIN_TIMEOUT_MS") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_HTTP_ORIGIN_TIMEOUT_MS") { if let Ok(ms) = v.parse() { cfg.http_origin_timeout = Duration::from_millis(ms); } } - if let Ok(v) = std::env::var("CLOUDFRONT_DIVERGENCE_LOG_INTERVAL_MS") { + if let Ok(v) = rustack_core::settings::var("CLOUDFRONT_DIVERGENCE_LOG_INTERVAL_MS") { if let Ok(ms) = v.parse() { cfg.divergence_log_interval = Duration::from_millis(ms); } diff --git a/crates/rustack-cloudfront-dataplane/src/dispatch.rs b/crates/rustack-cloudfront-dataplane/src/dispatch.rs index 67c021f..25136cb 100644 --- a/crates/rustack-cloudfront-dataplane/src/dispatch.rs +++ b/crates/rustack-cloudfront-dataplane/src/dispatch.rs @@ -97,6 +97,7 @@ pub async fn dispatch_s3_origin( inbound_headers: &HeaderMap, origin_custom_headers: &[CustomHeader], forward_user_metadata: bool, + max_body: usize, ) -> Result, DataPlaneError> { let joined = concat_origin_path(origin_path, request_path); let key = joined.trim_start_matches('/').to_owned(); @@ -116,8 +117,31 @@ pub async fn dispatch_s3_origin( .map(str::to_owned), ..GetObjectInput::default() }; - let out = s3.handle_get_object(input).await.map_err(to_dp_error)?; - let body = out.body.map(|b| b.data).unwrap_or_default(); + let (out, staged) = s3 + .handle_get_object_streaming(input) + .await + .map_err(to_dp_error)?; + let limit = + (max_body as u64).min(rustack_core::settings::budgets().upstream_body_bytes); + let body = if let Some(staged) = staged { + if staged.length > limit { + return Err(DataPlaneError::PayloadTooLarge( + "S3 origin body exceeds proxy byte budget".to_owned(), + )); + } + staged + .collect_bounded(limit) + .await + .map_err(|error| DataPlaneError::Internal(error.to_string()))? + } else { + let body = out.body.map(|body| body.data).unwrap_or_default(); + if body.len() as u64 > limit { + return Err(DataPlaneError::PayloadTooLarge( + "S3 origin body exceeds proxy byte budget".to_owned(), + )); + } + body + }; let mut builder = Response::builder().status(StatusCode::OK); if let Some(ct) = out.content_type { builder = builder.header(http::header::CONTENT_TYPE, ct); @@ -254,6 +278,18 @@ pub async fn dispatch_http_origin( format!("{scheme}://{}:{effective_port}{joined}", origin.domain_name) }; + let url = reqwest::Url::parse(&url) + .map_err(|_| DataPlaneError::Internal("invalid configured origin URL".to_owned()))?; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.fragment().is_some() + { + return Err(DataPlaneError::Internal( + "origin URL requires an HTTP(S) authority without credentials or fragments".to_owned(), + )); + } let mut upstream = filter_inbound_headers(inbound_headers); apply_custom_headers(&mut upstream, &origin.custom_headers); @@ -261,13 +297,13 @@ pub async fn dispatch_http_origin( .map_err(|e| DataPlaneError::Internal(format!("invalid method: {e}")))?; let req = client - .request(reqwest_method, &url) + .request(reqwest_method, url) .headers(translate_headers_to_reqwest(&upstream)) .body(body.to_vec()) .build() .map_err(|e| DataPlaneError::Internal(format!("build reqwest: {e}")))?; - let resp = client.execute(req).await.map_err(|e| { + let mut resp = client.execute(req).await.map_err(|e| { if e.is_timeout() { DataPlaneError::OriginServerError { status: 504, @@ -284,17 +320,39 @@ pub async fn dispatch_http_origin( let status = StatusCode::from_u16(resp.status().as_u16()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); let upstream_headers = translate_headers_from_reqwest(resp.headers()); - let body_bytes = resp - .bytes() + let budgets = rustack_core::settings::budgets(); + let max_body = + max_body + .min(usize::try_from(budgets.upstream_body_bytes).map_err(|_| { + DataPlaneError::Internal("invalid upstream byte budget".to_owned()) + })?); + let mut body_bytes = Vec::new(); + let idle = std::time::Duration::from_secs(budgets.body_idle_seconds); + let mut idle_deadline = tokio::time::Instant::now() + idle; + while let Some(chunk) = tokio::time::timeout_at(idle_deadline, resp.chunk()) .await - .map_err(|e| DataPlaneError::Internal(format!("read body: {e}")))?; - if body_bytes.len() > max_body { - return Err(DataPlaneError::PayloadTooLarge(format!( - "upstream body {} bytes exceeds cap {}", - body_bytes.len(), - max_body - ))); + .map_err(|_| DataPlaneError::OriginServerError { + status: 504, + message: "origin body idle deadline exceeded".to_owned(), + })? + .map_err(|e| DataPlaneError::OriginServerError { + status: 502, + message: format!("read origin body: {e}"), + })? + { + if !chunk.is_empty() { + idle_deadline = tokio::time::Instant::now() + idle; + } + rustack_core::http::append_bounded(&mut body_bytes, &chunk, max_body as u64).map_err( + |error| match error { + rustack_core::http::BodyReadError::TooLarge => { + DataPlaneError::PayloadTooLarge("upstream body exceeds byte budget".to_owned()) + } + error => DataPlaneError::Internal(error.to_string()), + }, + )?; } + let body_bytes = Bytes::from(body_bytes); let mut builder = Response::builder().status(status); for (k, v) in upstream_headers.iter() { diff --git a/crates/rustack-cloudfront-dataplane/src/plane.rs b/crates/rustack-cloudfront-dataplane/src/plane.rs index c06bb8a..afc3261 100644 --- a/crates/rustack-cloudfront-dataplane/src/plane.rs +++ b/crates/rustack-cloudfront-dataplane/src/plane.rs @@ -83,10 +83,26 @@ impl DataPlaneBuilder { /// Construct the `DataPlane`. pub fn build(self) -> Result { let cf = self.cloudfront.ok_or("CloudFront provider is required")?; + if self.config.max_upstream_body_bytes == 0 + || self.config.max_upstream_body_bytes > 64 * 1024 * 1024 + || self.config.http_origin_timeout.is_zero() + || self.config.http_origin_timeout > std::time::Duration::from_secs(30) + { + return Err("invalid CloudFront upstream byte or time budget"); + } let divergence = DivergenceTracker::new(self.config.divergence_log_interval); #[cfg(feature = "http-origin")] let http_client = reqwest::Client::builder() - .timeout(self.config.http_origin_timeout) + .redirect(reqwest::redirect::Policy::none()) + .no_proxy() + .connect_timeout(std::time::Duration::from_secs(5)) + .timeout( + self.config + .http_origin_timeout + .min(std::time::Duration::from_secs( + rustack_core::settings::budgets().body_total_seconds, + )), + ) .build() .map_err(|_| "failed to build reqwest client")?; Ok(DataPlane { @@ -279,11 +295,12 @@ impl DataPlane { &headers, &origin.custom_headers, self.config.forward_user_metadata, + self.config.max_upstream_body_bytes, ) .await { Ok(r) => r, - Err(e) => return self.handle_origin_error(&dist.config, e).await, + Err(e) => return Self::handle_origin_error(&dist.config, &e), } } #[cfg(feature = "http-origin")] @@ -300,7 +317,7 @@ impl DataPlane { .await { Ok(r) => r, - Err(e) => return self.handle_origin_error(&dist.config, e).await, + Err(e) => return Self::handle_origin_error(&dist.config, &e), } } #[cfg(not(feature = "http-origin"))] @@ -343,11 +360,7 @@ impl DataPlane { response } - async fn handle_origin_error( - &self, - config: &DistributionConfig, - err: DataPlaneError, - ) -> Response { + fn handle_origin_error(config: &DistributionConfig, err: &DataPlaneError) -> Response { let status = err.http_status(); // Check CustomErrorResponses. for cer in &config.custom_error_responses { @@ -369,10 +382,10 @@ impl DataPlane { } return builder .body(Bytes::from_static(b"")) - .unwrap_or_else(|_| error_response(&err)); + .unwrap_or_else(|_| error_response(err)); } } - error_response(&err) + error_response(err) } fn check_divergence( @@ -463,6 +476,101 @@ fn xml_escape(s: &str) -> String { mod tests { use super::*; + #[cfg(feature = "http-origin")] + #[tokio::test] + async fn test_should_ignore_environment_proxy_in_isolated_process() { + let mut child = tokio::process::Command::new(std::env::current_exe().unwrap()); + child + .args([ + "--exact", + "plane::tests::test_should_preserve_redirects_and_bound_upstream_chunks", + ]) + .env("HTTP_PROXY", "http://127.0.0.1:9") + .env("HTTPS_PROXY", "http://127.0.0.1:9") + .env("ALL_PROXY", "http://127.0.0.1:9") + .env("http_proxy", "http://127.0.0.1:9") + .env("https_proxy", "http://127.0.0.1:9") + .env("all_proxy", "http://127.0.0.1:9") + .env_remove("NO_PROXY") + .env_remove("no_proxy") + .kill_on_drop(true); + let output = tokio::time::timeout(Duration::from_secs(10), child.output()) + .await + .unwrap() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stdout) + ); + } + + #[cfg(feature = "http-origin")] + #[tokio::test] + async fn test_should_preserve_redirects_and_bound_upstream_chunks() { + use rustack_cloudfront_model::types::{CustomOriginConfig, Origin}; + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; + let cf = Arc::new(RustackCloudFront::new( + rustack_cloudfront_core::config::CloudFrontConfig::default(), + )); + let plane = DataPlane::builder().cloudfront(cf).build().unwrap(); + for status in [301, 302, 303, 307, 308, 200] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let forbidden = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let location = format!("http://{}/secret", forbidden.local_addr().unwrap()); + let origin = Origin { + domain_name: "127.0.0.1".to_owned(), + custom_origin_config: Some(CustomOriginConfig { + http_port: i32::from(listener.local_addr().unwrap().port()), + origin_protocol_policy: "http-only".to_owned(), + ..CustomOriginConfig::default() + }), + ..Origin::default() + }; + let wire = format!( + "HTTP/1.1 {status} Fixture\r\nLocation: {location}\r\nTransfer-Encoding: \ + chunked\r\nConnection: close\r\n\r\n3\r\nabc\r\n1\r\nd\r\n0\r\n\r\n" + ); + let server = async { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut buffer = [0; 4096]; + assert!(socket.read(&mut buffer).await.unwrap() > 0); + socket.write_all(wire.as_bytes()).await.unwrap(); + }; + let headers = HeaderMap::new(); + let limit = if status == 200 { 3 } else { 4 }; + let request = crate::dispatch::dispatch_http_origin( + &plane.http_client, + &origin, + "/", + &Method::GET, + &headers, + Bytes::new(), + limit, + ); + let ((), result) = tokio::join!(server, request); + if status == 200 { + assert!(matches!(result, Err(DataPlaneError::PayloadTooLarge(_)))); + } else { + let response = result.unwrap(); + assert_eq!(response.status().as_u16(), status); + assert_eq!( + response.headers().get("location").unwrap(), + location.as_str() + ); + assert_eq!(response.body().as_ref(), b"abcd"); + } + assert!( + tokio::time::timeout(Duration::from_millis(2), forbidden.accept()) + .await + .is_err() + ); + } + } + #[test] fn test_parse_path_based() { let (id, path) = DataPlane::parse_path_based("/_aws/cloudfront/e1abcd/index.html").unwrap(); diff --git a/crates/rustack-cloudfront-http/Cargo.toml b/crates/rustack-cloudfront-http/Cargo.toml index fd56e26..1a733bf 100644 --- a/crates/rustack-cloudfront-http/Cargo.toml +++ b/crates/rustack-cloudfront-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-cloudfront-http" description = "CloudFront HTTP layer (restXml) for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-cloudfront-core = { workspace = true } rustack-cloudfront-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-cloudfront-http/src/lib.rs b/crates/rustack-cloudfront-http/src/lib.rs index b57013f..66cb898 100644 --- a/crates/rustack-cloudfront-http/src/lib.rs +++ b/crates/rustack-cloudfront-http/src/lib.rs @@ -1,3 +1,4 @@ +#![forbid(unsafe_code)] #![allow(clippy::must_use_candidate)] #![allow(clippy::manual_let_else)] #![allow(clippy::too_many_lines)] diff --git a/crates/rustack-cloudfront-http/src/service.rs b/crates/rustack-cloudfront-http/src/service.rs index 9b63878..5d52cc3 100644 --- a/crates/rustack-cloudfront-http/src/service.rs +++ b/crates/rustack-cloudfront-http/src/service.rs @@ -3,7 +3,7 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::{BodyExt, Full}; +use http_body_util::Full; use hyper::body::{Body, Frame, Incoming}; use crate::{ @@ -50,9 +50,7 @@ impl Body for HttpBody { self: Pin<&mut Self>, cx: &mut std::task::Context<'_>, ) -> std::task::Poll, Self::Error>>> { - // SAFETY: we never move `inner` out. - let inner = unsafe { self.map_unchecked_mut(|s| &mut s.inner) }; - inner.poll_frame(cx) + Pin::new(&mut self.get_mut().inner).poll_frame(cx) } } @@ -134,19 +132,36 @@ impl hyper::service::Service> async fn serve( req: http::Request, handler: &H, - _config: &CloudFrontHttpConfig, + config: &CloudFrontHttpConfig, request_id: String, ) -> http::Response { let (parts, body) = req.into_parts(); - let body_bytes = match body.collect().await { - Ok(c) => c.to_bytes(), - Err(e) => { - let err = rustack_cloudfront_model::CloudFrontError::Internal(format!( - "failed to read body: {e}" - )); - return error_response(&err, &request_id); - } - }; + let body_bytes = + match rustack_core::http::collect_body(body, rustack_core::http::BodyBudget::control()) + .await + { + Ok(c) => c, + Err(e) => { + let err = rustack_cloudfront_model::CloudFrontError::Internal(format!( + "failed to read body: {e}" + )); + let mut response = error_response(&err, &request_id); + *response.status_mut() = e.status_code(); + return response; + } + }; + + if let Err(err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body_bytes))) + { + return error_response( + &rustack_cloudfront_model::CloudFrontError::AccessDenied(err.to_string()), + &request_id, + ); + } let route = match resolve(&parts.method, &parts.uri) { Ok(r) => r, diff --git a/crates/rustack-cloudwatch-core/Cargo.toml b/crates/rustack-cloudwatch-core/Cargo.toml index eb46f04..43a0d36 100644 --- a/crates/rustack-cloudwatch-core/Cargo.toml +++ b/crates/rustack-cloudwatch-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-cloudwatch-core" description = "CloudWatch Metrics business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-cloudwatch-core/src/config.rs b/crates/rustack-cloudwatch-core/src/config.rs index c4780aa..b240f9d 100644 --- a/crates/rustack-cloudwatch-core/src/config.rs +++ b/crates/rustack-cloudwatch-core/src/config.rs @@ -1,6 +1,6 @@ //! CloudWatch service configuration. -use std::env; +use rustack_core::settings as env; /// CloudWatch service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-cloudwatch-http/Cargo.toml b/crates/rustack-cloudwatch-http/Cargo.toml index 7121c64..20e13ca 100644 --- a/crates/rustack-cloudwatch-http/Cargo.toml +++ b/crates/rustack-cloudwatch-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-cloudwatch-http" description = "CloudWatch Metrics HTTP service layer for Rustack (awsQuery + rpcv2Cbor protocols)" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-cloudwatch-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-cloudwatch-http/src/service.rs b/crates/rustack-cloudwatch-http/src/service.rs index a944363..200680a 100644 --- a/crates/rustack-cloudwatch-http/src/service.rs +++ b/crates/rustack-cloudwatch-http/src/service.rs @@ -9,7 +9,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_cloudwatch_model::{error::CloudWatchError, operations::CloudWatchOperation}; @@ -232,19 +231,18 @@ async fn process_request( }; // Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = CloudWatchError::with_message( - rustack_cloudwatch_model::error::CloudWatchErrorCode::InternalServiceFault, - auth_err.to_string(), - ); - return make_error_response(&err, request_id, protocol); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let mut err = CloudWatchError::with_message( + rustack_cloudwatch_model::error::CloudWatchErrorCode::InternalServiceFault, + auth_err.to_string(), + ); + err.status_code = http::StatusCode::FORBIDDEN; + return make_error_response(&err, request_id, protocol); } // Dispatch to handler. @@ -288,11 +286,13 @@ fn json_error_response( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| CloudWatchError::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = CloudWatchError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every CloudWatch response. diff --git a/crates/rustack-core/Cargo.toml b/crates/rustack-core/Cargo.toml index 840c7eb..9474e0c 100644 --- a/crates/rustack-core/Cargo.toml +++ b/crates/rustack-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-core" description = "Core types, configuration, and state management for Rustack" -version.workspace = true +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,14 @@ repository.workspace = true workspace = true [dependencies] +config = { workspace = true } +url = { workspace = true } +typed-builder = { workspace = true } +bytes = { workspace = true } +http = { workspace = true } +http-body = { workspace = true } +http-body-util = { workspace = true } +tokio = { workspace = true } anyhow = { workspace = true } chrono = { workspace = true } dashmap = { workspace = true } diff --git a/crates/rustack-core/src/http.rs b/crates/rustack-core/src/http.rs new file mode 100644 index 0000000..6debd1f --- /dev/null +++ b/crates/rustack-core/src/http.rs @@ -0,0 +1,394 @@ +//! Source-level HTTP byte budgets and progress/absolute deadlines. +//! +//! Limits apply before frames reach parsers or collectors, independent of Content-Length. + +use std::{ + error::Error, + future::Future, + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; + +use bytes::{Buf, Bytes}; +use http_body::{Body, Frame, SizeHint}; +use http_body_util::BodyExt; +use tokio::time::{Instant, Sleep, sleep_until}; + +/// Default maximum aggregate control-plane request size. +pub const CONTROL_BODY_LIMIT: u64 = 16 * 1024 * 1024; +/// Maximum Lambda code-upload JSON envelope. +pub const LAMBDA_CODE_BODY_LIMIT: u64 = 96 * 1024 * 1024; +/// Default maximum upstream response size. +pub const UPSTREAM_BODY_LIMIT: u64 = 64 * 1024 * 1024; +/// Maximum S3 object stream size (not an aggregate allocation budget). +pub const S3_OBJECT_BODY_LIMIT: u64 = 5 * 1024 * 1024 * 1024; + +/// Validated byte and time budget for one HTTP body. +#[derive(Debug, Clone, Copy)] +pub struct BodyBudget { + max_bytes: u64, + total: Duration, + idle: Duration, +} + +impl BodyBudget { + /// Build a nonzero, bounded body budget. + /// + /// # Errors + /// Rejects zero values, sizes above 5 GiB, or deadlines above one hour. + pub fn new(max_bytes: u64, total: Duration, idle: Duration) -> Result { + if max_bytes == 0 + || max_bytes > S3_OBJECT_BODY_LIMIT + || total.is_zero() + || idle.is_zero() + || total > Duration::from_hours(1) + || idle > total + { + return Err(BodyReadError::InvalidBudget); + } + Ok(Self { + max_bytes, + total, + idle, + }) + } + + /// Default control-plane limits: 16 MiB, 30 seconds total, 5 seconds idle. + #[must_use] + pub fn control() -> Self { + let settings = crate::settings::budgets(); + Self { + max_bytes: settings.control_body_bytes, + total: Duration::from_secs(settings.body_total_seconds), + idle: Duration::from_secs(settings.body_idle_seconds), + } + } + + /// Code-upload limits: 96 MiB with the control-plane deadlines. + #[must_use] + pub fn lambda_code() -> Self { + Self { + max_bytes: crate::settings::budgets().lambda_code_body_bytes, + ..Self::control() + } + } + + /// Configured streaming object budget, with independent decoded-byte enforcement. + #[must_use] + pub fn s3_object() -> Self { + let settings = crate::settings::budgets(); + Self { + max_bytes: settings.s3_object_body_bytes, + total: Duration::from_secs(settings.s3_body_total_seconds), + idle: Duration::from_secs(settings.body_idle_seconds), + } + } + + /// Encoded S3 input budget including bounded chunk/trailer overhead. + #[must_use] + pub fn s3_encoded() -> Self { + let budget = Self::s3_object(); + Self { + max_bytes: budget + .max_bytes + .saturating_mul(2) + .saturating_add(CONTROL_BODY_LIMIT), + ..budget + } + } + + /// Tighten the byte limit without relaxing either deadline. + #[must_use] + pub fn capped(self, max_bytes: std::num::NonZeroU64) -> Self { + Self { + max_bytes: self.max_bytes.min(max_bytes.get()), + ..self + } + } + + /// Maximum accepted data bytes. + #[must_use] + pub const fn max_bytes(self) -> u64 { + self.max_bytes + } +} + +/// Terminal body read failures; no partial body is returned. +#[derive(Debug, thiserror::Error)] +pub enum BodyReadError { + /// Invalid operator budget configuration. + #[error("invalid HTTP body budget")] + InvalidBudget, + /// Actual data bytes exceed the budget. + #[error("HTTP body exceeds byte budget")] + TooLarge, + /// Absolute body deadline elapsed. + #[error("HTTP body total deadline exceeded")] + Deadline, + /// No data progress within the idle budget. + #[error("HTTP body idle deadline exceeded")] + Idle, + /// Source transport failed. + #[error("HTTP body transport failed: {0}")] + Transport(#[source] Box), + /// Bounded allocation could not be reserved. + #[error("HTTP body allocation failed")] + Allocation(#[source] std::collections::TryReserveError), +} + +impl BodyReadError { + /// HTTP status for a protocol adapter's native error envelope. + #[must_use] + pub const fn status_code(&self) -> http::StatusCode { + match self { + Self::TooLarge => http::StatusCode::PAYLOAD_TOO_LARGE, + Self::Idle | Self::Deadline => http::StatusCode::REQUEST_TIMEOUT, + Self::Transport(_) => http::StatusCode::BAD_REQUEST, + Self::InvalidBudget | Self::Allocation(_) => http::StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +/// Body adapter checking every frame before passing ownership to its consumer. +#[derive(Debug)] +pub struct BudgetedBody { + inner: Pin>, + remaining: u64, + total: Pin>, + idle: Pin>, + idle_duration: Duration, + finished: bool, +} + +impl BudgetedBody { + /// Wrap a source. Timers start immediately and are not reset by empty frames. + pub fn new(body: B, budget: BodyBudget) -> Self { + let now = Instant::now(); + Self { + inner: Box::pin(body), + remaining: budget.max_bytes, + total: Box::pin(sleep_until(now + budget.total)), + idle: Box::pin(sleep_until(now + budget.idle)), + idle_duration: budget.idle, + finished: false, + } + } +} + +impl Body for BudgetedBody +where + B: Body, + B::Error: Error + Send + Sync + 'static, +{ + type Data = B::Data; + type Error = BodyReadError; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + let this = self.get_mut(); + if this.finished { + return Poll::Ready(None); + } + let timeout = if this.total.as_mut().poll(cx).is_ready() { + Some(BodyReadError::Deadline) + } else if this.idle.as_mut().poll(cx).is_ready() { + Some(BodyReadError::Idle) + } else { + None + }; + if let Some(error) = timeout { + this.finished = true; + return Poll::Ready(Some(Err(error))); + } + match this.inner.as_mut().poll_frame(cx) { + Poll::Ready(Some(Ok(frame))) => { + if let Some(data) = frame.data_ref() { + let size = data.remaining() as u64; + let Some(remaining) = this.remaining.checked_sub(size) else { + this.finished = true; + return Poll::Ready(Some(Err(BodyReadError::TooLarge))); + }; + this.remaining = remaining; + if size != 0 { + this.idle + .as_mut() + .reset(Instant::now() + this.idle_duration); + } + } + Poll::Ready(Some(Ok(frame))) + } + Poll::Ready(Some(Err(error))) => { + this.finished = true; + Poll::Ready(Some(Err(BodyReadError::Transport(Box::new(error))))) + } + Poll::Ready(None) => { + this.finished = true; + Poll::Ready(None) + } + Poll::Pending => Poll::Pending, + } + } + + fn is_end_stream(&self) -> bool { + self.finished + } + + fn size_hint(&self) -> SizeHint { + let mut hint = SizeHint::new(); + hint.set_upper(self.remaining); + hint + } +} + +/// Collect only after enforcing the budget on each source frame. +/// +/// # Errors +/// Returns overflow, timeout or transport errors, discarding partial bytes. +pub async fn collect_body(body: B, budget: BodyBudget) -> Result +where + B: Body, + B::Error: Error + Send + Sync + 'static, +{ + let mut body = BudgetedBody::new(body, budget); + let mut bytes = Vec::new(); + while let Some(frame) = body.frame().await { + if let Ok(data) = frame?.into_data() { + append_bounded(&mut bytes, &data, budget.max_bytes)?; + } + } + Ok(Bytes::from(bytes)) +} + +/// Append bytes without letting geometric buffer growth exceed the byte budget. +/// # Errors +/// Rejects excess bytes before copying or returns a bounded allocation failure. +pub fn append_bounded(buffer: &mut Vec, data: &[u8], limit: u64) -> Result<(), BodyReadError> { + let needed = buffer + .len() + .checked_add(data.len()) + .ok_or(BodyReadError::TooLarge)?; + let limit = usize::try_from(limit).map_err(|_| BodyReadError::InvalidBudget)?; + if needed > limit { + return Err(BodyReadError::TooLarge); + } + if needed > buffer.capacity() { + let target = buffer.capacity().saturating_mul(2).max(needed).min(limit); + buffer + .try_reserve_exact(target.saturating_sub(buffer.len())) + .map_err(BodyReadError::Allocation)?; + } + buffer.extend_from_slice(data); + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::convert::Infallible; + + use http_body_util::Full; + + use super::*; + + #[tokio::test] + async fn test_should_limit_actual_frames_before_collection() { + let budget = BodyBudget::new(3, Duration::from_secs(1), Duration::from_secs(1)).unwrap(); + assert_eq!( + collect_body(Full::new(Bytes::from_static(b"abc")), budget) + .await + .unwrap(), + "abc" + ); + assert!(matches!( + collect_body(Full::new(Bytes::from_static(b"abcd")), budget).await, + Err(BodyReadError::TooLarge) + )); + } + + #[derive(Debug)] + struct PendingBody; + impl Body for PendingBody { + type Data = Bytes; + type Error = Infallible; + fn poll_frame( + self: Pin<&mut Self>, + _: &mut Context<'_>, + ) -> Poll, Infallible>>> { + Poll::Pending + } + } + + #[tokio::test] + async fn test_should_expire_pending_body_without_eof() { + let budget = BodyBudget::new(3, Duration::from_secs(1), Duration::from_millis(5)).unwrap(); + assert!(matches!( + collect_body(PendingBody, budget).await, + Err(BodyReadError::Idle) + )); + let budget = + BodyBudget::new(3, Duration::from_millis(5), Duration::from_millis(5)).unwrap(); + assert!(matches!( + collect_body(PendingBody, budget).await, + Err(BodyReadError::Deadline) + )); + } + + #[test] + fn test_should_keep_aggregate_capacity_inside_budget() { + let mut buffer = Vec::new(); + for _ in 0..15 { + append_bounded(&mut buffer, &[0; 100], 1500).unwrap(); + assert!(buffer.capacity() <= 1500); + } + let capacity = buffer.capacity(); + assert!(matches!( + append_bounded(&mut buffer, &[0], 1500), + Err(BodyReadError::TooLarge) + )); + assert_eq!(buffer.capacity(), capacity); + } + + #[derive(Debug)] + struct DripBody { + timer: Pin>, + } + impl Body for DripBody { + type Data = Bytes; + type Error = Infallible; + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Infallible>>> { + let this = self.get_mut(); + if this.timer.as_mut().poll(cx).is_pending() { + return Poll::Pending; + } + this.timer + .as_mut() + .reset(Instant::now() + Duration::from_millis(2)); + Poll::Ready(Some(Ok(Frame::data(Bytes::from_static(b"x"))))) + } + } + + #[tokio::test] + async fn test_should_not_reset_total_deadline_on_progress() { + let budget = + BodyBudget::new(1000, Duration::from_millis(20), Duration::from_millis(10)).unwrap(); + let body = DripBody { + timer: Box::pin(sleep_until(Instant::now())), + }; + assert!(matches!( + collect_body(body, budget).await, + Err(BodyReadError::Deadline) + )); + } + + #[test] + fn test_should_reject_invalid_budgets() { + assert!(BodyBudget::new(0, Duration::from_secs(1), Duration::from_secs(1)).is_err()); + assert!(BodyBudget::new(1, Duration::ZERO, Duration::ZERO).is_err()); + assert!(BodyBudget::new(1, Duration::from_secs(1), Duration::from_secs(2)).is_err()); + } +} diff --git a/crates/rustack-core/src/lib.rs b/crates/rustack-core/src/lib.rs index da7379a..9a3f364 100644 --- a/crates/rustack-core/src/lib.rs +++ b/crates/rustack-core/src/lib.rs @@ -6,6 +6,8 @@ mod config; mod error; +pub mod http; +pub mod settings; mod state; mod types; diff --git a/crates/rustack-core/src/settings.rs b/crates/rustack-core/src/settings.rs new file mode 100644 index 0000000..d30fb02 --- /dev/null +++ b/crates/rustack-core/src/settings.rs @@ -0,0 +1,703 @@ +//! Validated, immutable YAML and environment configuration shared by services. +//! +//! The binary installs settings once before constructing providers. Library users +//! may use [`ValidatedSettings::parse`] without modifying process globals. Values +//! from the process environment override YAML, and diagnostics never print values. + +pub use std::env::VarError; +use std::{collections::BTreeMap, env, fmt, net::SocketAddr, sync::OnceLock, time::Duration}; + +use config::{Config, File, FileFormat}; +use serde::Deserialize; +use serde_json::Value; +use tokio::io::AsyncReadExt; +use url::Url; + +const MAX_CONFIG_BYTES: u64 = 1024 * 1024; +const SERVICE_NAMES: &[&str] = &[ + "S3", + "DYNAMODB", + "DYNAMODBSTREAMS", + "SQS", + "SSM", + "SNS", + "LAMBDA", + "EVENTS", + "LOGS", + "KMS", + "KINESIS", + "SECRETSMANAGER", + "SES", + "APIGATEWAYV2", + "CLOUDWATCH", + "IAM", + "STS", + "CLOUDFRONT", +]; +const KEYS: &[&str] = &[ + "GATEWAY_LISTEN", + "GATEWAY_HOST", + "GATEWAY_PORT", + "RUSTACK_ADVERTISED_ENDPOINT", + "SERVICES", + "DEFAULT_REGION", + "AWS_DEFAULT_REGION", + "DEFAULT_ACCOUNT_ID", + "ACCOUNT_ID", + "ACCESS_KEY", + "SECRET_KEY", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "LOG_LEVEL", + "RUST_LOG", + "PERSISTENCE", + "DATA_DIR", + "S3_VIRTUAL_HOSTING", + "S3_DOMAIN", + "S3_MAX_MEMORY_OBJECT_SIZE", + "LAMBDA_DOCKER_ENABLED", + "LAMBDA_EXECUTOR", + "LAMBDA_MAX_WARM_INSTANCES", + "LAMBDA_IDLE_TIMEOUT_SECS", + "LAMBDA_INIT_TIMEOUT_SECS", + "LAMBDA_SQUIB_INSTANCE_ID", + "LAMBDA_SQUIB_CONFIG_FILE", + "LAMBDA_SQUIB_VSOCK_PATH", + "LAMBDA_SQUIB_STAGE_PORT", + "LAMBDA_SQUIB_CONNECT_TIMEOUT_MS", + "LAMBDA_SQUIB_RESPONSE_LIMIT_BYTES", + "LAMBDA_SQUIB_RUN_BUDGET_SECS", + "LAMBDA_SQUIB_SHUTDOWN_TIMEOUT_MS", + "RUSTACK_WORKSPACE_ROOT", + "RUSTACK_SNAPSHOT_DIR", + "RUSTACK_SNAPSHOT_PERF_FILE", + "EVENTS_HOST", + "EVENTS_PORT", + "LOGS_HOST", + "LOGS_PORT", + "CLOUDFRONT_ACCOUNT_ID", + "CLOUDFRONT_DOMAIN_SUFFIX", + "CLOUDFRONT_DISTRIBUTION_PROPAGATION_MS", + "CLOUDFRONT_INVALIDATION_PROPAGATION_MS", + "CLOUDFRONT_DETERMINISTIC_IDS", + "CLOUDFRONT_FAIL_ON_FUNCTION", + "CLOUDFRONT_FORWARD_USER_METADATA", + "CLOUDFRONT_MAX_UPSTREAM_BODY_BYTES", + "CLOUDFRONT_HTTP_ORIGIN_TIMEOUT_MS", + "CLOUDFRONT_DIVERGENCE_LOG_INTERVAL_MS", + "CLOUDWATCH_MAX_RETENTION_SECONDS", + "CLOUDWATCH_MAX_POINTS_PER_SERIES", + "DYNAMODBSTREAMS_MAX_RECORDS_PER_SHARD", + "DYNAMODBSTREAMS_MAX_RECORD_AGE_SECONDS", + "SES_REQUIRE_VERIFIED_IDENTITY", + "SES_MAX_24_HOUR_SEND", + "SES_MAX_SEND_RATE", +]; + +static SETTINGS: OnceLock = OnceLock::new(); + +/// Positive, bounded runtime resource limits. YAML names are camelCase. +#[derive(Debug, Clone, Deserialize, typed_builder::TypedBuilder)] +#[non_exhaustive] +#[serde(default, rename_all = "camelCase", deny_unknown_fields)] +pub struct RuntimeBudgets { + /// Maximum live HTTP connections. + pub connections: usize, + /// Maximum concurrently active business requests, including response bodies. + pub requests: usize, + /// HTTP header deadline in seconds. + pub header_seconds: u64, + /// Control-plane operation deadline in seconds. + pub request_seconds: u64, + /// Synchronous Lambda execution deadline in seconds (may exceed 900 while an Invoke runs). + pub lambda_invoke_seconds: u64, + /// Total graceful shutdown deadline in seconds. + pub shutdown_seconds: u64, + /// Maximum aggregate control-plane body bytes. + pub control_body_bytes: u64, + /// Maximum Lambda ZIP JSON envelope bytes. + pub lambda_code_body_bytes: u64, + /// Maximum upstream response bytes. + pub upstream_body_bytes: u64, + /// Body inactivity deadline in seconds. + pub body_idle_seconds: u64, + /// Control-plane body total deadline in seconds. + pub body_total_seconds: u64, + /// S3 streaming body total deadline in seconds. + pub s3_body_total_seconds: u64, + /// Maximum S3 object bytes, without aggregate collection. + pub s3_object_body_bytes: u64, +} + +impl Default for RuntimeBudgets { + fn default() -> Self { + Self { + connections: 256, + requests: 128, + header_seconds: 5, + request_seconds: 30, + lambda_invoke_seconds: 930, + shutdown_seconds: 30, + control_body_bytes: 16 * 1024 * 1024, + lambda_code_body_bytes: 96 * 1024 * 1024, + upstream_body_bytes: 64 * 1024 * 1024, + body_idle_seconds: 5, + body_total_seconds: 30, + s3_body_total_seconds: 3600, + s3_object_body_bytes: 5 * 1024 * 1024 * 1024, + } + } +} + +impl RuntimeBudgets { + /// Validate budgets before allocating semaphore capacity or timers. + /// + /// # Errors + /// Rejects zero, excessive, or inconsistent resource deadlines and sizes. + pub fn validate(&self) -> Result<(), SettingsError> { + for (key, value, max) in [ + ("budgets.connections", self.connections as u64, 65_536), + ("budgets.requests", self.requests as u64, 65_536), + ("budgets.headerSeconds", self.header_seconds, 300), + ("budgets.requestSeconds", self.request_seconds, 3600), + ( + "budgets.lambdaInvokeSeconds", + self.lambda_invoke_seconds, + 3600, + ), + ("budgets.shutdownSeconds", self.shutdown_seconds, 3600), + ( + "budgets.controlBodyBytes", + self.control_body_bytes, + 16 * 1024 * 1024, + ), + ( + "budgets.lambdaCodeBodyBytes", + self.lambda_code_body_bytes, + 96 * 1024 * 1024, + ), + ( + "budgets.upstreamBodyBytes", + self.upstream_body_bytes, + 64 * 1024 * 1024, + ), + ("budgets.bodyIdleSeconds", self.body_idle_seconds, 300), + ("budgets.bodyTotalSeconds", self.body_total_seconds, 3600), + ( + "budgets.s3BodyTotalSeconds", + self.s3_body_total_seconds, + 3600, + ), + ( + "budgets.s3ObjectBodyBytes", + self.s3_object_body_bytes, + 5 * 1024 * 1024 * 1024, + ), + ] { + if value == 0 || value > max { + return Err(invalid(key, "outside supported nonzero range")); + } + } + if self.body_idle_seconds > self.body_total_seconds + || self.body_idle_seconds > self.s3_body_total_seconds + { + return Err(invalid( + "budgets.bodyIdleSeconds", + "idle deadline must not exceed total deadline", + )); + } + Ok(()) + } +} + +#[derive(Deserialize, Default)] +#[serde(default, rename_all = "camelCase", deny_unknown_fields)] +struct Document { + environment: BTreeMap, + budgets: RuntimeBudgets, + advertised_endpoint: Option, +} + +/// Configuration failure without secret-bearing value diagnostics. +#[derive(Debug, thiserror::Error)] +pub enum SettingsError { + /// Invalid or unknown operator setting. + #[error("invalid configuration key {key}: {reason}")] + Invalid { + /// Name of the invalid key, never its value. + key: String, + /// Non-sensitive explanation. + reason: &'static str, + }, + /// YAML cannot be parsed according to the configuration schema. + #[error("invalid YAML configuration schema (values redacted)")] + Document, + /// Configuration source IO failed. + #[error("cannot read configuration source")] + Io(#[source] std::io::Error), + /// Config loading exceeded its deadline. + #[error("configuration read deadline exceeded")] + Timeout, + /// Reinitializing the process configuration is not supported. + #[error("runtime configuration was already installed")] + AlreadyInstalled, +} + +fn invalid(key: &str, reason: &'static str) -> SettingsError { + SettingsError::Invalid { + key: key.to_owned(), + reason, + } +} + +/// Fully validated settings. Debug prints keys, not potentially secret values. +#[derive(Clone)] +pub struct ValidatedSettings { + values: BTreeMap, + budgets: RuntimeBudgets, + listen: SocketAddr, + advertised_endpoint: String, +} + +impl fmt::Debug for ValidatedSettings { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ValidatedSettings") + .field("keys", &self.values.keys().collect::>()) + .field("budgets", &self.budgets) + .field("listen", &self.listen) + .finish_non_exhaustive() + } +} + +impl ValidatedSettings { + /// Parse YAML and explicit environment overrides, without touching globals. + /// + /// # Errors + /// Rejects unknown YAML keys, invalid scalars, unsafe ranges, and invalid addresses. + pub fn parse(yaml: &str, overrides: &BTreeMap) -> Result { + let document: Document = if yaml.trim().is_empty() { + Document::default() + } else { + Config::builder() + .add_source(File::from_str(yaml, FileFormat::Yaml)) + .build() + .map_err(|_| SettingsError::Document)? + .try_deserialize() + .map_err(|_| SettingsError::Document)? + }; + document.budgets.validate()?; + let mut values = BTreeMap::new(); + for (key, value) in document.environment { + if !known_key(&key) { + return Err(invalid(&key, "unknown setting")); + } + let text = match value { + Value::String(text) => text, + Value::Bool(value) => value.to_string(), + Value::Number(value) => value.to_string(), + _ => return Err(invalid(&key, "expected a scalar value")), + }; + values.insert(key, text); + } + for (key, value) in overrides { + if known_key(key) { + values.insert(key.clone(), value.clone()); + } + } + for (key, value) in &mut values { + validate_value(key, value)?; + } + normalize_aliases( + &mut values, + &["DEFAULT_REGION", "AWS_DEFAULT_REGION"], + Some("us-east-1"), + )?; + normalize_aliases( + &mut values, + &["DEFAULT_ACCOUNT_ID", "ACCOUNT_ID", "CLOUDFRONT_ACCOUNT_ID"], + Some("000000000000"), + )?; + normalize_aliases(&mut values, &["ACCESS_KEY", "AWS_ACCESS_KEY_ID"], None)?; + normalize_aliases(&mut values, &["SECRET_KEY", "AWS_SECRET_ACCESS_KEY"], None)?; + let listen: SocketAddr = values + .get("GATEWAY_LISTEN") + .map_or("127.0.0.1:4566", String::as_str) + .parse() + .map_err(|_| invalid("GATEWAY_LISTEN", "expected IP address and nonzero port"))?; + if listen.port() == 0 { + return Err(invalid("GATEWAY_LISTEN", "port must not be zero")); + } + values + .entry("GATEWAY_LISTEN".to_owned()) + .or_insert_with(|| listen.to_string()); + let default_host = if listen.ip().is_unspecified() { + "localhost".to_owned() + } else { + listen.ip().to_string() + }; + let host = values + .entry("GATEWAY_HOST".to_owned()) + .or_insert(default_host) + .clone(); + let port = values + .entry("GATEWAY_PORT".to_owned()) + .or_insert_with(|| listen.port().to_string()) + .clone(); + let host = if host.contains(':') && !host.starts_with('[') { + format!("[{host}]") + } else { + host + }; + values.insert("GATEWAY_HOST".to_owned(), host.clone()); + let advertised_endpoint = values + .get("RUSTACK_ADVERTISED_ENDPOINT") + .cloned() + .or(document.advertised_endpoint) + .unwrap_or_else(|| format!("http://{host}:{port}")); + let endpoint = validate_endpoint(&advertised_endpoint)?; + let endpoint_host = endpoint + .host_str() + .ok_or_else(|| invalid("advertisedEndpoint", "missing host"))?; + let endpoint_port = endpoint + .port_or_known_default() + .ok_or_else(|| invalid("advertisedEndpoint", "missing port"))?; + values.insert("GATEWAY_HOST".to_owned(), endpoint_host.to_owned()); + values.insert("GATEWAY_PORT".to_owned(), endpoint_port.to_string()); + let advertised_endpoint = endpoint.origin().ascii_serialization(); + Ok(Self { + values, + budgets: document.budgets, + listen, + advertised_endpoint, + }) + } + + /// Effective immutable resource budgets. + #[must_use] + pub fn budgets(&self) -> &RuntimeBudgets { + &self.budgets + } + + /// Effective bind address, including the validated nonzero port. + #[must_use] + pub const fn listen(&self) -> SocketAddr { + self.listen + } + + /// Explicit local public endpoint, never derived from an incoming Host header. + #[must_use] + pub fn advertised_endpoint(&self) -> &str { + &self.advertised_endpoint + } + + /// Lookup a validated value for provider construction. + #[must_use] + pub fn get(&self, key: &str) -> Option<&str> { + self.values.get(key).map(String::as_str) + } + + /// Reject strict authentication without a complete static credential pair. + /// + /// # Errors + /// Returns a redacted error if any enabled strict service lacks credentials. + pub fn validate_credentials(&self, enabled: &[String]) -> Result<(), SettingsError> { + let strict = enabled.iter().any(|service| { + self.get(&format!( + "{}_SKIP_SIGNATURE_VALIDATION", + service.to_ascii_uppercase() + )) == Some("false") + }); + if strict { + let access = self + .get("ACCESS_KEY") + .or_else(|| self.get("AWS_ACCESS_KEY_ID")); + let secret = self + .get("SECRET_KEY") + .or_else(|| self.get("AWS_SECRET_ACCESS_KEY")); + if access.is_none_or(str::is_empty) || secret.is_none_or(str::is_empty) { + return Err(invalid( + "credentials", + "strict signature validation requires both access and secret keys", + )); + } + } + Ok(()) + } +} + +fn known_key(key: &str) -> bool { + KEYS.contains(&key) + || key + .strip_suffix("_SKIP_SIGNATURE_VALIDATION") + .is_some_and(|prefix| SERVICE_NAMES.contains(&prefix)) +} + +fn validate_value(key: &str, value: &mut String) -> Result<(), SettingsError> { + if value.is_empty() && key != "SERVICES" { + return Err(invalid(key, "must not be empty")); + } + if value.len() > 4096 || value.chars().any(char::is_control) { + return Err(invalid( + key, + "exceeds byte limit or contains control characters", + )); + } + let boolean = key.ends_with("_SKIP_SIGNATURE_VALIDATION") + || matches!( + key, + "PERSISTENCE" + | "S3_VIRTUAL_HOSTING" + | "LAMBDA_DOCKER_ENABLED" + | "CLOUDFRONT_DETERMINISTIC_IDS" + | "CLOUDFRONT_FAIL_ON_FUNCTION" + | "CLOUDFRONT_FORWARD_USER_METADATA" + | "SES_REQUIRE_VERIFIED_IDENTITY" + ); + if boolean { + *value = match value.to_ascii_lowercase().as_str() { + "true" | "yes" | "on" | "1" => "true".to_owned(), + "false" | "no" | "off" | "0" => "false".to_owned(), + _ => return Err(invalid(key, "expected boolean")), + }; + } else if key == "LAMBDA_EXECUTOR" { + if !matches!( + value.as_str(), + "disabled" | "native" | "auto" | "docker" | "squib" + ) { + return Err(invalid(key, "unknown executor")); + } + } else if key.ends_with("ACCOUNT_ID") || key == "ACCOUNT_ID" { + if value.len() != 12 || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(invalid(key, "expected 12 ASCII digits")); + } + } else if key.ends_with("REGION") { + if value.len() > 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + { + return Err(invalid(key, "invalid region identifier")); + } + } else if key.ends_with("_PORT") { + let port = value + .parse::() + .map_err(|_| invalid(key, "expected valid port"))?; + if port == 0 { + return Err(invalid(key, "port must not be zero")); + } + } else if key == "SES_MAX_24_HOUR_SEND" || key == "SES_MAX_SEND_RATE" { + let number = value + .parse::() + .map_err(|_| invalid(key, "expected finite positive quota"))?; + if !number.is_finite() || number <= 0.0 || number > 1_000_000_000.0 { + return Err(invalid(key, "outside supported quota range")); + } + } else if key.ends_with("_MS") + || key.ends_with("_SECS") + || key.ends_with("_SECONDS") + || key.contains("_MAX_") + || key.ends_with("_LIMIT_BYTES") + { + let number = value + .parse::() + .map_err(|_| invalid(key, "expected bounded nonnegative integer"))?; + let zero_allowed = key.contains("PROPAGATION") || key == "S3_MAX_MEMORY_OBJECT_SIZE"; + let max = if key.ends_with("_MS") { + 86_400_000 + } else if key.ends_with("_SECS") || key.ends_with("_SECONDS") { + 31_536_000 + } else { + 5 * 1024 * 1024 * 1024 + }; + if (!zero_allowed && number == 0) || number > max { + return Err(invalid(key, "outside supported range")); + } + } else if key.ends_with("_HOST") || key.ends_with("_DOMAIN") || key.ends_with("_DOMAIN_SUFFIX") + { + if value.len() > 253 + || !value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b".-:[]".contains(&byte)) + { + return Err(invalid(key, "invalid hostname or address")); + } + } + Ok(()) +} + +fn normalize_aliases( + values: &mut BTreeMap, + keys: &[&str], + default: Option<&str>, +) -> Result<(), SettingsError> { + let mut selected = None; + for key in keys { + if let Some(value) = values.get(*key) { + if selected.as_ref().is_some_and(|current| current != value) { + return Err(invalid(key, "conflicting aliases")); + } + selected = Some(value.clone()); + } + } + if let Some(selected) = selected.or_else(|| default.map(str::to_owned)) { + for key in keys { + values.insert((*key).to_owned(), selected.clone()); + } + } + Ok(()) +} + +fn validate_endpoint(endpoint: &str) -> Result { + if endpoint.len() > 300 + || endpoint + .bytes() + .any(|byte| byte.is_ascii_control() || byte == b' ') + { + return Err(invalid( + "advertisedEndpoint", + "invalid URL length or characters", + )); + } + let parsed = Url::parse(endpoint).map_err(|_| invalid("advertisedEndpoint", "invalid URL"))?; + if !matches!(parsed.scheme(), "http" | "https") + || parsed.host_str().is_none() + || parsed.port() == Some(0) + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.path() != "/" + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(invalid( + "advertisedEndpoint", + "expected http(s) authority without credentials, path, query or fragment", + )); + } + Ok(parsed) +} + +/// Read bounded YAML and freeze process configuration before providers start. +/// +/// # Errors +/// Returns configuration, IO, timeout, or duplicate-install errors. +pub async fn initialize() -> Result<&'static ValidatedSettings, SettingsError> { + let yaml = match env::var("RUSTACK_CONFIG") { + Ok(path) => { + let read = async { + let file = tokio::fs::File::open(path) + .await + .map_err(SettingsError::Io)?; + let mut text = String::new(); + file.take(MAX_CONFIG_BYTES + 1) + .read_to_string(&mut text) + .await + .map_err(SettingsError::Io)?; + if text.len() as u64 > MAX_CONFIG_BYTES { + return Err(invalid("RUSTACK_CONFIG", "configuration exceeds 1 MiB")); + } + Ok(text) + }; + tokio::time::timeout(Duration::from_secs(5), read) + .await + .map_err(|_| SettingsError::Timeout)?? + } + Err(VarError::NotPresent) => String::new(), + Err(_) => return Err(invalid("RUSTACK_CONFIG", "expected UTF-8 path")), + }; + let mut overrides = BTreeMap::new(); + for key in KEYS.iter().map(|key| (*key).to_owned()).chain( + SERVICE_NAMES + .iter() + .map(|key| format!("{key}_SKIP_SIGNATURE_VALIDATION")), + ) { + match env::var(&key) { + Ok(value) => { + overrides.insert(key, value); + } + Err(VarError::NotPresent) => {} + Err(_) => return Err(invalid(&key, "expected UTF-8 value")), + } + } + let settings = ValidatedSettings::parse(&yaml, &overrides)?; + SETTINGS + .set(settings) + .map_err(|_| SettingsError::AlreadyInstalled)?; + SETTINGS.get().ok_or(SettingsError::AlreadyInstalled) +} + +/// Get an effective value; library use without installation reads the environment. +/// +/// # Errors +/// Matches `std::env::var` for absent or non-Unicode values. +pub fn var(key: &str) -> Result { + match SETTINGS.get() { + Some(settings) => settings + .get(key) + .map(str::to_owned) + .ok_or(VarError::NotPresent), + None => env::var(key), + } +} + +/// Effective budgets, or defaults for independently constructed library services. +#[must_use] +pub fn budgets() -> RuntimeBudgets { + SETTINGS + .get() + .map_or_else(RuntimeBudgets::default, |settings| settings.budgets.clone()) +} + +/// Installed public endpoint, when configured by the application. +#[must_use] +pub fn advertised_endpoint() -> Option<&'static str> { + SETTINGS.get().map(ValidatedSettings::advertised_endpoint) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_should_apply_environment_over_yaml_and_derive_ipv6_endpoint() + -> Result<(), SettingsError> { + let overrides = BTreeMap::from([("GATEWAY_LISTEN".to_owned(), "[::1]:4567".to_owned())]); + let config = ValidatedSettings::parse( + "environment:\n GATEWAY_LISTEN: 127.0.0.1:4566\n", + &overrides, + )?; + assert_eq!(config.get("GATEWAY_PORT"), Some("4567")); + assert_eq!(config.advertised_endpoint(), "http://[::1]:4567"); + Ok(()) + } + + #[test] + fn test_should_reject_unknown_keys_bad_values_and_zero_budgets() { + for yaml in [ + "environment:\n LAMBDA_EXECUTOR: disable", + "environment:\n GATEWAY_PORT: 0", + "environment:\n S3_SKIP_SIGNATURE_VALIDATION: maybe", + "environment:\n DYNAMDB_SKIP_SIGNATURE_VALIDATION: true", + "budgets:\n requests: 0", + "unknown: true", + ] { + assert!(ValidatedSettings::parse(yaml, &BTreeMap::new()).is_err()); + } + } + + #[test] + fn test_should_fail_closed_without_credentials_and_redact_debug() -> Result<(), SettingsError> { + let config = ValidatedSettings::parse( + "environment:\n SQS_SKIP_SIGNATURE_VALIDATION: false\n ACCESS_KEY: sample\n \ + SECRET_KEY: sensitive-value", + &BTreeMap::new(), + )?; + config.validate_credentials(&["sqs".to_owned()])?; + assert!(!format!("{config:?}").contains("sensitive-value")); + let missing = ValidatedSettings::parse( + "environment:\n SQS_SKIP_SIGNATURE_VALIDATION: false", + &BTreeMap::new(), + )?; + assert!(missing.validate_credentials(&["sqs".to_owned()]).is_err()); + Ok(()) + } +} diff --git a/crates/rustack-dynamodb-core/Cargo.toml b/crates/rustack-dynamodb-core/Cargo.toml index ee312c3..4e83058 100644 --- a/crates/rustack-dynamodb-core/Cargo.toml +++ b/crates/rustack-dynamodb-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-dynamodb-core" description = "DynamoDB business logic and storage engine for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -27,3 +27,4 @@ uuid = { workspace = true } tracing = { workspace = true } thiserror = { workspace = true } anyhow = { workspace = true } +tokio = { workspace = true } diff --git a/crates/rustack-dynamodb-core/src/config.rs b/crates/rustack-dynamodb-core/src/config.rs index c27799d..4787d41 100644 --- a/crates/rustack-dynamodb-core/src/config.rs +++ b/crates/rustack-dynamodb-core/src/config.rs @@ -1,6 +1,6 @@ //! DynamoDB configuration. -use std::env; +use rustack_core::settings as env; /// DynamoDB service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-dynamodb-core/src/coordination.rs b/crates/rustack-dynamodb-core/src/coordination.rs new file mode 100644 index 0000000..ca8a69b --- /dev/null +++ b/crates/rustack-dynamodb-core/src/coordination.rs @@ -0,0 +1,114 @@ +//! Cancellation-safe admission for synchronous provider tasks. +//! +//! The semaphore contains no data state. Its permit is moved into the blocking +//! closure so HTTP cancellation cannot make a running operation disappear from +//! the shutdown barrier. +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + thread::panicking, +}; + +use parking_lot::MutexGuard; +use rustack_dynamodb_model::error::{DynamoDBError, DynamoDBErrorCode}; +use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore}; + +const REQUEST_CAPACITY: usize = 128; + +#[derive(Debug)] +pub(crate) struct RequestTracker { + permits: Arc, + wake: Arc, + failed: AtomicBool, +} +impl Default for RequestTracker { + fn default() -> Self { + Self { + permits: Arc::new(Semaphore::new(REQUEST_CAPACITY)), + wake: Arc::new(Notify::new()), + failed: AtomicBool::new(false), + } + } +} +impl RequestTracker { + pub fn is_ready(&self) -> bool { + !self.permits.is_closed() + } + pub fn ensure_healthy(&self) -> Result<(), DynamoDBError> { + if self.failed.load(Ordering::Acquire) { + Err(DynamoDBError::internal_error("DynamoDB operation panicked")) + } else { + Ok(()) + } + } + pub fn guard<'a>( + &'a self, + guard: MutexGuard<'a, ()>, + ) -> Result, DynamoDBError> { + self.ensure_open()?; + Ok(OperationGuard { + _guard: guard, + tracker: self, + }) + } + pub fn admit(&self) -> Result { + let permit = Arc::clone(&self.permits) + .try_acquire_owned() + .map_err(|_| Self::unavailable())?; + Ok(RequestPermit { + permit: Some(permit), + wake: Arc::clone(&self.wake), + }) + } + pub fn ensure_open(&self) -> Result<(), DynamoDBError> { + if self.permits.is_closed() { + Err(Self::unavailable()) + } else { + Ok(()) + } + } + fn unavailable() -> DynamoDBError { + DynamoDBError::with_message( + DynamoDBErrorCode::RequestLimitExceeded, + "DynamoDB is full or quiescing", + ) + } + pub async fn quiesce(&self) { + self.permits.close(); + loop { + let notified = self.wake.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.permits.available_permits() == REQUEST_CAPACITY { + return; + } + notified.await; + } + } +} + +pub(crate) struct OperationGuard<'a> { + _guard: MutexGuard<'a, ()>, + tracker: &'a RequestTracker, +} +impl Drop for OperationGuard<'_> { + fn drop(&mut self) { + if panicking() { + self.tracker.failed.store(true, Ordering::Release); + self.tracker.permits.close(); + } + } +} + +pub(crate) struct RequestPermit { + permit: Option, + wake: Arc, +} +impl Drop for RequestPermit { + fn drop(&mut self) { + drop(self.permit.take()); + self.wake.notify_waiters(); + } +} diff --git a/crates/rustack-dynamodb-core/src/handler.rs b/crates/rustack-dynamodb-core/src/handler.rs index 20c2cc6..ee886bb 100644 --- a/crates/rustack-dynamodb-core/src/handler.rs +++ b/crates/rustack-dynamodb-core/src/handler.rs @@ -11,6 +11,7 @@ use rustack_dynamodb_model::{ input::{CreateTableInput, DescribeTableInput}, operations::DynamoDBOperation, }; +use tokio::task::spawn_blocking; use crate::provider::RustackDynamoDB; @@ -39,7 +40,17 @@ impl DynamoDBHandler for RustackDynamoDBHandler { >, > { let provider = Arc::clone(&self.provider); - Box::pin(async move { dispatch(provider.as_ref(), op, &body) }) + Box::pin(async move { + let permit = provider.requests.admit()?; + spawn_blocking(move || { + let _permit = permit; + dispatch(provider.as_ref(), op, &body) + }) + .await + .map_err(|error| { + DynamoDBError::internal_error(format!("DynamoDB operation task failed: {error}")) + })? + }) } } diff --git a/crates/rustack-dynamodb-core/src/lib.rs b/crates/rustack-dynamodb-core/src/lib.rs index ca1ada1..1bb03f6 100644 --- a/crates/rustack-dynamodb-core/src/lib.rs +++ b/crates/rustack-dynamodb-core/src/lib.rs @@ -2,6 +2,7 @@ #![allow(missing_docs, clippy::doc_markdown, clippy::module_name_repetitions)] pub mod config; +mod coordination; pub mod error; pub mod expression; pub mod handler; diff --git a/crates/rustack-dynamodb-core/src/provider.rs b/crates/rustack-dynamodb-core/src/provider.rs index 769c217..a1ff0a0 100644 --- a/crates/rustack-dynamodb-core/src/provider.rs +++ b/crates/rustack-dynamodb-core/src/provider.rs @@ -3,11 +3,15 @@ use std::{ collections::{HashMap, HashSet}, sync::Arc, + time::{Duration, Instant}, }; +use dashmap::DashMap; +use parking_lot::Mutex; +use rustack_auth::hash_payload; use rustack_dynamodb_model::{ AttributeValue, - error::DynamoDBError, + error::{DynamoDBError, DynamoDBErrorCode}, input::{ BatchGetItemInput, BatchWriteItemInput, CreateTableInput, DeleteItemInput, DeleteTableInput, DescribeContinuousBackupsInput, DescribeEndpointsInput, @@ -31,12 +35,14 @@ use rustack_dynamodb_model::{ ContinuousBackupsDescription, ContinuousBackupsStatus, ExpectedAttributeValue, ItemResponse, KeyType, PointInTimeRecoveryDescription, PointInTimeRecoverySpecification, PointInTimeRecoveryStatus, ReturnValue, ScalarAttributeType, Select, TableStatus, - TimeToLiveDescription, + TimeToLiveDescription, TransactWriteItem, }, }; +use tokio::task::spawn_blocking; use crate::{ config::DynamoDBConfig, + coordination::{OperationGuard, RequestTracker}, error::{expression_error_to_dynamodb, storage_error_to_dynamodb}, expression::{ AttributePath, EvalContext, PathElement, UpdateExpr, collect_names_from_expr, @@ -49,6 +55,10 @@ use crate::{ KeyAttribute, KeySchema, PrimaryKey, SortKeyCondition, SortableAttributeValue, TableStorage, calculate_item_size, extract_primary_key, partition_key_segment, }, + stream::{ + ChangeEvent, ChangeEventName, NoopStreamEmitter, NoopStreamLifecycle, StreamEmitter, + StreamLifecycle, + }, }; /// Maximum item size in bytes (400 KB). @@ -167,7 +177,11 @@ fn validate_number_string(s: &str) -> Result<(), DynamoDBError> { let leading_zeros = (all_digits.len() - trimmed_leading.len()) as i64; // Actual magnitude = explicit_exp - frac_digits + all_digits.len() - leading_zeros - 1 #[allow(clippy::cast_possible_wrap)] - let magnitude = explicit_exp - frac_digits + all_digits.len() as i64 - leading_zeros - 1; + let magnitude = explicit_exp + .saturating_sub(frac_digits) + .saturating_add(all_digits.len() as i64) + .saturating_sub(leading_zeros) + .saturating_sub(1); if magnitude > 125 { return Err(DynamoDBError::validation( @@ -250,6 +264,19 @@ fn validate_key_not_empty( ) -> Result<(), DynamoDBError> { for ka in std::iter::once(&key_schema.partition_key).chain(key_schema.sort_key.iter()) { if let Some(val) = item.get(&ka.name) { + let limit = if ka.name == key_schema.partition_key.name { + 2048 + } else { + 1024 + }; + let length = match val { + AttributeValue::S(value) => value.len(), + AttributeValue::B(value) => value.len(), + _ => 0, + }; + if length > limit { + return Err(DynamoDBError::validation("Key exceeds maximum byte length")); + } match val { AttributeValue::S(s) if s.is_empty() => { return Err(DynamoDBError::validation(format!( @@ -457,16 +484,30 @@ fn validate_parallel_scan( } } +/// A successful transaction retained for the full idempotency window. +#[derive(Debug)] +pub(crate) struct CompletedTransaction { + fingerprint: String, + completed_at: Instant, + output: TransactWriteItemsOutput, +} + /// Main DynamoDB provider implementing all operations. pub struct RustackDynamoDB { /// Service state owning all tables. - pub state: Arc, + pub(crate) state: Arc, + /// Serializes complete provider operations, never held across await. + pub(crate) operation_gate: Arc>, + pub(crate) requests: RequestTracker, + /// Completed tokens; admission and mutation are protected by operation_gate. + pub(crate) tokens: DashMap, + token_capacity: usize, /// Configuration. pub config: Arc, /// Stream emitter for change data capture. - emitter: Arc, + emitter: Arc, /// Stream lifecycle manager. - lifecycle: Arc, + lifecycle: Arc, } impl std::fmt::Debug for RustackDynamoDB { @@ -484,9 +525,13 @@ impl RustackDynamoDB { pub fn new(config: DynamoDBConfig) -> Self { Self { state: Arc::new(DynamoDBServiceState::new()), + operation_gate: Arc::new(Mutex::new(())), + requests: RequestTracker::default(), + tokens: DashMap::new(), + token_capacity: 1024, config: Arc::new(config), - emitter: Arc::new(crate::stream::NoopStreamEmitter), - lifecycle: Arc::new(crate::stream::NoopStreamLifecycle), + emitter: Arc::new(NoopStreamEmitter), + lifecycle: Arc::new(NoopStreamLifecycle), } } @@ -494,19 +539,52 @@ impl RustackDynamoDB { /// /// Called by the server binary to wire in the DynamoDB Streams /// implementation. - pub fn set_emitter(&mut self, emitter: Arc) { + pub fn set_emitter(&mut self, emitter: Arc) { self.emitter = emitter; } /// Set the stream lifecycle manager. /// /// Called by the server binary to wire in stream creation/deletion. - pub fn set_lifecycle(&mut self, lifecycle: Arc) { + pub fn set_lifecycle(&mut self, lifecycle: Arc) { self.lifecycle = lifecycle; } - /// Reset all state (for testing). + /// Whether request admission is open and no provider operation has panicked. + #[must_use] + pub fn is_ready(&self) -> bool { + self.requests.is_ready() + } + + fn begin_operation(&self) -> Result, DynamoDBError> { + self.requests.guard(self.operation_gate.lock()) + } + + /// Close request admission and drain blocking work, including cancelled HTTP requests. + /// + /// Snapshot export remains available after this terminal consistency barrier. + /// The runtime must apply its remaining overall shutdown deadline. + /// + /// # Errors + /// Returns an error if the blocking consistency barrier cannot be joined. + pub async fn quiesce(&self) -> Result<(), DynamoDBError> { + self.requests.quiesce().await; + self.requests.ensure_healthy()?; + let gate = Arc::clone(&self.operation_gate); + spawn_blocking(move || { + let _operation = gate.lock(); + }) + .await + .map_err(|error| { + DynamoDBError::internal_error(format!("DynamoDB quiesce failed: {error}")) + })?; + self.requests.ensure_healthy() + } + + /// Reset all state (administrative operation). pub fn reset(&self) { + let _operation = self.operation_gate.lock(); + self.tokens.clear(); self.state.reset(); } } @@ -522,6 +600,7 @@ impl RustackDynamoDB { &self, input: CreateTableInput, ) -> Result { + let _operation = self.begin_operation()?; // Validate table name. validate_table_name(&input.table_name)?; @@ -625,6 +704,7 @@ impl RustackDynamoDB { &self, input: DeleteTableInput, ) -> Result { + let _operation = self.begin_operation()?; let table = self.state.delete_table(&input.table_name)?; self.lifecycle.on_table_deleted(&table.name); Ok(DeleteTableOutput { @@ -638,6 +718,7 @@ impl RustackDynamoDB { &self, input: DescribeTableInput, ) -> Result { + let _operation = self.begin_operation()?; let table = self.state.require_table(&input.table_name)?; let mut desc = table.to_description(); @@ -658,6 +739,7 @@ impl RustackDynamoDB { &self, input: ListTablesInput, ) -> Result { + let _operation = self.begin_operation()?; // Validate limit: must be 1-100 if specified. if let Some(limit) = input.limit { if !(1..=100).contains(&limit) { @@ -705,6 +787,7 @@ impl RustackDynamoDB { &self, input: UpdateTableInput, ) -> Result { + let _operation = self.begin_operation()?; let table = self.state.require_table(&input.table_name)?; // For our in-memory emulator, UpdateTable is accepted but most changes @@ -727,6 +810,7 @@ impl RustackDynamoDB { /// Handle `PutItem`. #[allow(clippy::too_many_lines)] pub fn handle_put_item(&self, mut input: PutItemInput) -> Result { + let _operation = self.begin_operation()?; validate_table_name(&input.table_name)?; let table = self.state.require_table(&input.table_name)?; @@ -837,13 +921,13 @@ impl RustackDynamoDB { .is_some_and(|s| s.stream_enabled) { let event_name = if old.is_some() { - crate::stream::ChangeEventName::Modify + ChangeEventName::Modify } else { - crate::stream::ChangeEventName::Insert + ChangeEventName::Insert }; let keys = extract_key_attributes(&new_item, &table.key_schema_elements); let size = calculate_item_size(&new_item); - self.emitter.emit(crate::stream::ChangeEvent { + self.emitter.emit(ChangeEvent { table_name: table.name.clone(), event_name, keys, @@ -869,6 +953,7 @@ impl RustackDynamoDB { /// Handle `GetItem`. #[allow(clippy::needless_pass_by_value)] pub fn handle_get_item(&self, mut input: GetItemInput) -> Result { + let _operation = self.begin_operation()?; validate_table_name(&input.table_name)?; let table = self.state.require_table(&input.table_name)?; @@ -943,6 +1028,7 @@ impl RustackDynamoDB { &self, mut input: DeleteItemInput, ) -> Result { + let _operation = self.begin_operation()?; validate_table_name(&input.table_name)?; let table = self.state.require_table(&input.table_name)?; @@ -1039,9 +1125,9 @@ impl RustackDynamoDB { { let keys = extract_key_attributes(old_item, &table.key_schema_elements); let size = calculate_item_size(old_item); - self.emitter.emit(crate::stream::ChangeEvent { + self.emitter.emit(ChangeEvent { table_name: table.name.clone(), - event_name: crate::stream::ChangeEventName::Remove, + event_name: ChangeEventName::Remove, keys, old_image: Some(old_item.clone()), new_image: None, @@ -1069,6 +1155,7 @@ impl RustackDynamoDB { &self, mut input: UpdateItemInput, ) -> Result { + let _operation = self.begin_operation()?; validate_table_name(&input.table_name)?; let table = self.state.require_table(&input.table_name)?; @@ -1300,6 +1387,8 @@ impl RustackDynamoDB { }); } + validate_numbers_in_item(&item)?; + validate_item_no_empty_sets(&item)?; // Validate updated item size. let size = calculate_item_size(&item); if size > MAX_ITEM_SIZE_BYTES { @@ -1321,12 +1410,12 @@ impl RustackDynamoDB { .is_some_and(|s| s.stream_enabled) { let event_name = if existing.is_some() || old_item.is_some() { - crate::stream::ChangeEventName::Modify + ChangeEventName::Modify } else { - crate::stream::ChangeEventName::Insert + ChangeEventName::Insert }; let keys = extract_key_attributes(&item, &table.key_schema_elements); - self.emitter.emit(crate::stream::ChangeEvent { + self.emitter.emit(ChangeEvent { table_name: table.name.clone(), event_name, keys, @@ -1363,6 +1452,7 @@ impl RustackDynamoDB { /// Handle `Query`. #[allow(clippy::needless_pass_by_value, clippy::too_many_lines)] pub fn handle_query(&self, mut input: QueryInput) -> Result { + let _operation = self.begin_operation()?; let table = self.state.require_table(&input.table_name)?; // Validate Select parameter. @@ -1659,6 +1749,7 @@ impl RustackDynamoDB { /// Handle `Scan`. #[allow(clippy::needless_pass_by_value, clippy::too_many_lines)] pub fn handle_scan(&self, mut input: ScanInput) -> Result { + let _operation = self.begin_operation()?; let table = self.state.require_table(&input.table_name)?; let has_atg = input @@ -1831,6 +1922,7 @@ impl RustackDynamoDB { &self, input: BatchGetItemInput, ) -> Result { + let _operation = self.begin_operation()?; // Enforce 100-item limit across all tables. let total_keys: usize = input.request_items.values().map(|ka| ka.keys.len()).sum(); if total_keys > 100 { @@ -1912,6 +2004,7 @@ impl RustackDynamoDB { &self, input: BatchWriteItemInput, ) -> Result { + let _operation = self.begin_operation()?; // Enforce 25-item limit across all tables. let total_writes: usize = input.request_items.values().map(Vec::len).sum(); if total_writes > 25 { @@ -1938,7 +2031,14 @@ impl RustackDynamoDB { detect_duplicate_keys(&table.key_schema, key_items.into_iter())?; for wr in write_requests { + if wr.put_request.is_some() == wr.delete_request.is_some() { + return Err(DynamoDBError::validation( + "Each batch write must contain exactly one action", + )); + } if let Some(ref put) = wr.put_request { + validate_numbers_in_item(&put.item)?; + validate_item_no_empty_sets(&put.item)?; validate_key_not_empty(&table.key_schema, &put.item)?; let size = calculate_item_size(&put.item); if size > MAX_ITEM_SIZE_BYTES { @@ -1951,66 +2051,39 @@ impl RustackDynamoDB { extract_primary_key(&table.key_schema, &put.item) .map_err(storage_error_to_dynamodb)?; } else if let Some(ref del) = wr.delete_request { + validate_key_only_has_key_attrs(&table.key_schema, &del.key)?; + validate_key_not_empty(&table.key_schema, &del.key)?; + validate_numbers_in_item(&del.key)?; extract_primary_key(&table.key_schema, &del.key) .map_err(storage_error_to_dynamodb)?; } } } - // Execution pass: all validations passed, now execute writes. + let mut prepared = Vec::with_capacity(total_writes); for (table_name, write_requests) in &input.request_items { let table = self.state.require_table(table_name)?; - let stream_enabled = table - .stream_specification - .as_ref() - .is_some_and(|s| s.stream_enabled); - - for wr in write_requests { - if let Some(ref put) = wr.put_request { - let old = table - .storage - .put_item(put.item.clone()) - .map_err(storage_error_to_dynamodb)?; - - if stream_enabled { - let event_name = if old.is_some() { - crate::stream::ChangeEventName::Modify - } else { - crate::stream::ChangeEventName::Insert - }; - let keys = extract_key_attributes(&put.item, &table.key_schema_elements); - let size = calculate_item_size(&put.item); - self.emitter.emit(crate::stream::ChangeEvent { - table_name: table.name.clone(), - event_name, - keys, - old_image: old, - new_image: Some(put.item.clone()), - size_bytes: size, - }); - } - } else if let Some(ref del) = wr.delete_request { - let pk = extract_primary_key(&table.key_schema, &del.key) - .map_err(storage_error_to_dynamodb)?; - let old = table.storage.delete_item(&pk); - - if stream_enabled { - if let Some(ref old_item) = old { - let keys = extract_key_attributes(old_item, &table.key_schema_elements); - let size = calculate_item_size(old_item); - self.emitter.emit(crate::stream::ChangeEvent { - table_name: table.name.clone(), - event_name: crate::stream::ChangeEventName::Remove, - keys, - old_image: Some(old_item.clone()), - new_image: None, - size_bytes: size, - }); - } - } - } + for request in write_requests { + let (item, new_image) = if let Some(put) = &request.put_request { + (&put.item, Some(put.item.clone())) + } else if let Some(delete) = &request.delete_request { + (&delete.key, None) + } else { + return Err(DynamoDBError::validation("Batch write action is missing")); + }; + let key = extract_primary_key(&table.key_schema, item) + .map_err(storage_error_to_dynamodb)?; + let old_image = table.storage.get_item(&key); + prepared.push(PreparedChange { + table: Arc::clone(&table), + key, + old_image, + new_image, + mutate: true, + }); } } + self.commit_prepared(prepared); Ok(BatchWriteItemOutput { unprocessed_items: HashMap::new(), @@ -2048,6 +2121,7 @@ impl RustackDynamoDB { &self, input: TagResourceInput, ) -> Result { + let _operation = self.begin_operation()?; let table_name = Self::resolve_table_from_arn(&input.resource_arn)?; let table = self.state.require_table(table_name)?; @@ -2100,6 +2174,7 @@ impl RustackDynamoDB { &self, input: UntagResourceInput, ) -> Result { + let _operation = self.begin_operation()?; let table_name = Self::resolve_table_from_arn(&input.resource_arn)?; let table = self.state.require_table(table_name)?; @@ -2116,6 +2191,7 @@ impl RustackDynamoDB { &self, input: ListTagsOfResourceInput, ) -> Result { + let _operation = self.begin_operation()?; let table_name = Self::resolve_table_from_arn(&input.resource_arn)?; let table = self.state.require_table(table_name)?; @@ -2138,6 +2214,7 @@ impl RustackDynamoDB { &self, input: UpdateTimeToLiveInput, ) -> Result { + let _operation = self.begin_operation()?; validate_table_name(&input.table_name)?; let table = self.state.require_table(&input.table_name)?; @@ -2161,6 +2238,7 @@ impl RustackDynamoDB { &self, input: DescribeTimeToLiveInput, ) -> Result { + let _operation = self.begin_operation()?; validate_table_name(&input.table_name)?; let table = self.state.require_table(&input.table_name)?; @@ -2191,6 +2269,7 @@ impl RustackDynamoDB { &self, input: DescribeContinuousBackupsInput, ) -> Result { + let _operation = self.begin_operation()?; validate_table_name(&input.table_name)?; let table = self.state.require_table(&input.table_name)?; let pitr = table.point_in_time_recovery.read().clone(); @@ -2206,6 +2285,7 @@ impl RustackDynamoDB { &self, input: UpdateContinuousBackupsInput, ) -> Result { + let _operation = self.begin_operation()?; validate_table_name(&input.table_name)?; let table = self.state.require_table(&input.table_name)?; @@ -2281,6 +2361,7 @@ impl RustackDynamoDB { &self, _input: DescribeLimitsInput, ) -> Result { + let _operation = self.begin_operation()?; Ok(DescribeLimitsOutput { account_max_read_capacity_units: Some(80_000), account_max_write_capacity_units: Some(80_000), @@ -2297,6 +2378,7 @@ impl RustackDynamoDB { &self, _input: DescribeEndpointsInput, ) -> Result { + let _operation = self.begin_operation()?; let address = format!("dynamodb.{}.amazonaws.com", self.config.default_region); Ok(DescribeEndpointsOutput { endpoints: vec![Endpoint { @@ -2314,6 +2396,105 @@ impl RustackDynamoDB { /// Maximum number of items in a transaction. const MAX_TRANSACT_ITEMS: usize = 100; +struct TransactionParts<'a> { + table_name: &'a str, + key_map: &'a HashMap, + condition: Option<&'a str>, + names: &'a Option>, + values: &'a Option>, + failure_return: Option<&'a str>, +} + +impl<'a> TryFrom<&'a TransactWriteItem> for TransactionParts<'a> { + type Error = DynamoDBError; + fn try_from(action: &'a TransactWriteItem) -> Result { + if let Some(put) = &action.put { + Ok(Self { + table_name: &put.table_name, + key_map: &put.item, + condition: put.condition_expression.as_deref(), + names: &put.expression_attribute_names, + values: &put.expression_attribute_values, + failure_return: put.return_values_on_condition_check_failure.as_deref(), + }) + } else if let Some(update) = &action.update { + Ok(Self { + table_name: &update.table_name, + key_map: &update.key, + condition: update.condition_expression.as_deref(), + names: &update.expression_attribute_names, + values: &update.expression_attribute_values, + failure_return: update.return_values_on_condition_check_failure.as_deref(), + }) + } else if let Some(delete) = &action.delete { + Ok(Self { + table_name: &delete.table_name, + key_map: &delete.key, + condition: delete.condition_expression.as_deref(), + names: &delete.expression_attribute_names, + values: &delete.expression_attribute_values, + failure_return: delete.return_values_on_condition_check_failure.as_deref(), + }) + } else if let Some(check) = &action.condition_check { + Ok(Self { + table_name: &check.table_name, + key_map: &check.key, + condition: Some(&check.condition_expression), + names: &check.expression_attribute_names, + values: &check.expression_attribute_values, + failure_return: check.return_values_on_condition_check_failure.as_deref(), + }) + } else { + Err(DynamoDBError::validation("Transaction action is missing")) + } + } +} + +struct PreparedChange { + table: Arc, + key: PrimaryKey, + old_image: Option>, + new_image: Option>, + mutate: bool, +} + +impl PreparedChange { + fn size_bytes(&self) -> u64 { + self.new_image + .as_ref() + .or(self.old_image.as_ref()) + .map_or(0, calculate_item_size) + } + + fn emit(self, emitter: &dyn StreamEmitter) { + if !self.mutate + || !self + .table + .stream_specification + .as_ref() + .is_some_and(|spec| spec.stream_enabled) + { + return; + } + let Some(image) = self.new_image.as_ref().or(self.old_image.as_ref()) else { + return; + }; + let event_name = match (&self.old_image, &self.new_image) { + (_, None) => ChangeEventName::Remove, + (None, Some(_)) => ChangeEventName::Insert, + (Some(_), Some(_)) => ChangeEventName::Modify, + }; + emitter.emit(ChangeEvent { + table_name: self.table.name.clone(), + event_name, + keys: extract_key_attributes(image, &self.table.key_schema_elements), + size_bytes: calculate_item_size(image), + old_image: self.old_image, + new_image: self.new_image, + }); + } +} + impl RustackDynamoDB { /// Handle `TransactGetItems`. #[allow(clippy::needless_pass_by_value)] @@ -2321,6 +2502,7 @@ impl RustackDynamoDB { &self, input: TransactGetItemsInput, ) -> Result { + let _operation = self.begin_operation()?; if input.transact_items.is_empty() { return Err(DynamoDBError::validation( "1 validation error detected: Value null at 'transactItems' failed to satisfy \ @@ -2339,7 +2521,11 @@ impl RustackDynamoDB { for transact_item in &input.transact_items { let get = &transact_item.get; + validate_table_name(&get.table_name)?; let table = self.state.require_table(&get.table_name)?; + validate_key_only_has_key_attrs(&table.key_schema, &get.key)?; + validate_key_types(&table.key_schema, &get.key)?; + validate_key_not_empty(&table.key_schema, &get.key)?; let pk = extract_primary_key(&table.key_schema, &get.key) .map_err(storage_error_to_dynamodb)?; @@ -2385,6 +2571,13 @@ impl RustackDynamoDB { &self, input: TransactWriteItemsInput, ) -> Result { + let _operation = self.begin_operation()?; + let fingerprint = self.admit_transaction_token(&input)?; + if let Some(token) = input.client_request_token.as_ref() { + if let Some(completed) = self.tokens.get(token) { + return Ok(completed.output.clone()); + } + } if input.transact_items.is_empty() { return Err(DynamoDBError::validation( "1 validation error detected: Value null at 'transactItems' failed to satisfy \ @@ -2478,118 +2671,182 @@ impl RustackDynamoDB { return Err(DynamoDBError::transaction_cancelled(cancellation_reasons)); } - // Phase 3: Apply all writes. + // Prepare computes every final value without touching visible storage. + let mut prepared = Vec::with_capacity(input.transact_items.len()); + let mut total_size = 0u64; for item in &input.transact_items { - if let Some(ref put) = item.put { - let table = self.state.require_table(&put.table_name)?; - let old = table - .storage - .put_item(put.item.clone()) - .map_err(storage_error_to_dynamodb)?; - - if table - .stream_specification - .as_ref() - .is_some_and(|s| s.stream_enabled) - { - let event_name = if old.is_some() { - crate::stream::ChangeEventName::Modify - } else { - crate::stream::ChangeEventName::Insert - }; - let keys = extract_key_attributes(&put.item, &table.key_schema_elements); - let size = calculate_item_size(&put.item); - self.emitter.emit(crate::stream::ChangeEvent { - table_name: table.name.clone(), - event_name, - keys, - old_image: old, - new_image: Some(put.item.clone()), - size_bytes: size, - }); - } - } else if let Some(ref del) = item.delete { - let table = self.state.require_table(&del.table_name)?; - let pk = extract_primary_key(&table.key_schema, &del.key) - .map_err(storage_error_to_dynamodb)?; - let old = table.storage.delete_item(&pk); - - if table - .stream_specification - .as_ref() - .is_some_and(|s| s.stream_enabled) - { - if let Some(ref old_item) = old { - let keys = extract_key_attributes(old_item, &table.key_schema_elements); - let size = calculate_item_size(old_item); - self.emitter.emit(crate::stream::ChangeEvent { - table_name: table.name.clone(), - event_name: crate::stream::ChangeEventName::Remove, - keys, - old_image: Some(old_item.clone()), - new_image: None, - size_bytes: size, - }); - } - } - } else if let Some(ref upd) = item.update { - let table = self.state.require_table(&upd.table_name)?; - let pk = extract_primary_key(&table.key_schema, &upd.key) - .map_err(storage_error_to_dynamodb)?; - let existing = table.storage.get_item(&pk); - let current = existing.clone().unwrap_or_else(|| upd.key.clone()); - - let names = upd.expression_attribute_names.as_ref(); - let values = upd.expression_attribute_values.as_ref(); - let empty_names = HashMap::new(); - let empty_values = HashMap::new(); - let names_ref = names.unwrap_or(&empty_names); - let values_ref = values.unwrap_or(&empty_values); + let change = self.prepare_transaction_change(item)?; + total_size = total_size.saturating_add(change.size_bytes()); + if total_size > 4 * 1024 * 1024 { + return Err(DynamoDBError::validation("Transaction exceeds 4 MiB")); + } + prepared.push(change); + } - let parsed = - parse_update(&upd.update_expression).map_err(expression_error_to_dynamodb)?; - let ctx = EvalContext { - item: ¤t, - names: names_ref, - values: values_ref, - }; - let updated = ctx - .apply_update(&parsed) - .map_err(expression_error_to_dynamodb)?; + self.commit_prepared(prepared); + let output = TransactWriteItemsOutput { + consumed_capacity: Vec::new(), + item_collection_metrics: HashMap::new(), + }; + if let (Some(token), Some(fingerprint)) = (input.client_request_token, fingerprint) { + self.tokens.insert( + token, + CompletedTransaction { + fingerprint, + completed_at: Instant::now(), + output: output.clone(), + }, + ); + } + Ok(output) + } - let old = table + /// Commit fully prepared data before publishing any stream notifications. + fn commit_prepared(&self, prepared: Vec) { + for change in &prepared { + if !change.mutate { + continue; + } + if let Some(item) = &change.new_image { + change + .table .storage - .put_item(updated.clone()) - .map_err(storage_error_to_dynamodb)?; + .put_prepared(change.key.clone(), item.clone()); + } else { + change.table.storage.delete_item(&change.key); + } + } + for change in prepared { + change.emit(self.emitter.as_ref()); + } + } - if table - .stream_specification - .as_ref() - .is_some_and(|s| s.stream_enabled) - { - let event_name = if existing.is_some() { - crate::stream::ChangeEventName::Modify - } else { - crate::stream::ChangeEventName::Insert - }; - let keys = extract_key_attributes(&updated, &table.key_schema_elements); - let size = calculate_item_size(&updated); - self.emitter.emit(crate::stream::ChangeEvent { - table_name: table.name.clone(), - event_name, - keys, - old_image: old, - new_image: Some(updated), - size_bytes: size, - }); - } + fn admit_transaction_token( + &self, + input: &TransactWriteItemsInput, + ) -> Result, DynamoDBError> { + let Some(token) = &input.client_request_token else { + return Ok(None); + }; + if token.is_empty() || token.len() > 36 { + return Err(DynamoDBError::validation( + "ClientRequestToken must contain 1..36 bytes", + )); + } + let mut fingerprint = serde_json::to_value(input) + .map_err(|error| DynamoDBError::internal_error(error.to_string()))?; + if let Some(object) = fingerprint.as_object_mut() { + object.remove("ClientRequestToken"); + } + fingerprint.sort_all_objects(); + let canonical = serde_json::to_vec(&fingerprint) + .map_err(|error| DynamoDBError::internal_error(error.to_string()))?; + if canonical.len() > 8 * 1024 * 1024 { + return Err(DynamoDBError::validation( + "Transaction request exceeds 8 MiB", + )); + } + let fingerprint = hash_payload(&canonical); + self.tokens + .retain(|_, entry| entry.completed_at.elapsed() < Duration::from_mins(10)); + if let Some(entry) = self.tokens.get(token) { + if entry.fingerprint != fingerprint { + return Err(DynamoDBError::with_message( + DynamoDBErrorCode::IdempotentParameterMismatchException, + "ClientRequestToken was already used with different parameters", + )); } - // ConditionCheck: no mutation needed. + } else if self.tokens.len() >= self.token_capacity { + return Err(DynamoDBError::with_message( + DynamoDBErrorCode::RequestLimitExceeded, + "Transaction token capacity is full; retry after the idempotency window expires", + )); } + Ok(Some(fingerprint)) + } - Ok(TransactWriteItemsOutput { - consumed_capacity: Vec::new(), - item_collection_metrics: HashMap::new(), + fn prepare_transaction_change( + &self, + action: &TransactWriteItem, + ) -> Result { + let TransactionParts { + table_name, + key_map, + condition, + names, + values, + failure_return, + } = TransactionParts::try_from(action)?; + validate_table_name(table_name)?; + let table = self.state.require_table(table_name)?; + validate_key_types(&table.key_schema, key_map)?; + validate_key_not_empty(&table.key_schema, key_map)?; + if action.put.is_none() { + validate_key_only_has_key_attrs(&table.key_schema, key_map)?; + } + validate_return_values_on_condition_check_failure(failure_return)?; + validate_condition_not_empty(condition)?; + let empty_names = HashMap::new(); + let empty_values = HashMap::new(); + let names = names.as_ref().unwrap_or(&empty_names); + let values = values.as_ref().unwrap_or(&empty_values); + validate_numbers_in_item(values)?; + validate_no_empty_sets(values)?; + let mut used_names = HashSet::new(); + let mut used_values = HashSet::new(); + if let Some(condition) = condition { + let parsed = parse_condition(condition).map_err(expression_error_to_dynamodb)?; + collect_names_from_expr(&parsed, &mut used_names); + collect_values_from_expr(&parsed, &mut used_values); + } + let key = + extract_primary_key(&table.key_schema, key_map).map_err(storage_error_to_dynamodb)?; + let old_image = table.storage.get_item(&key); + let new_image = if let Some(put) = &action.put { + Some(put.item.clone()) + } else if let Some(update) = &action.update { + let parsed = + parse_update(&update.update_expression).map_err(expression_error_to_dynamodb)?; + validate_update_paths(&parsed, &table.key_schema, names)?; + collect_names_from_update(&parsed, &mut used_names); + collect_values_from_update(&parsed, &mut used_values); + let current = old_image.as_ref().unwrap_or(key_map); + let updated = EvalContext { + item: current, + names, + values, + } + .apply_update(&parsed) + .map_err(expression_error_to_dynamodb)?; + Some(updated) + } else { + None + }; + validate_no_unused_names(names, &used_names)?; + validate_no_unused_values(values, &used_values)?; + if let Some(item) = &new_image { + validate_numbers_in_item(item)?; + validate_item_no_empty_sets(item)?; + validate_key_types(&table.key_schema, item)?; + validate_key_not_empty(&table.key_schema, item)?; + if extract_primary_key(&table.key_schema, item).map_err(storage_error_to_dynamodb)? + != key + { + return Err(DynamoDBError::validation( + "Transaction update cannot modify primary key", + )); + } + if calculate_item_size(item) > MAX_ITEM_SIZE_BYTES { + return Err(DynamoDBError::validation("Item exceeds 400 KiB")); + } + } + let mutate = action.condition_check.is_none(); + Ok(PreparedChange { + table, + key, + old_image, + new_image, + mutate, }) } @@ -2599,7 +2856,7 @@ impl RustackDynamoDB { /// or a `CancellationReason` if the condition fails. fn evaluate_transact_write_condition( &self, - item: &rustack_dynamodb_model::types::TransactWriteItem, + item: &TransactWriteItem, ) -> Result<(), CancellationReason> { if let Some(ref cc) = item.condition_check { self.evaluate_condition_for_key( @@ -4817,6 +5074,10 @@ fn gsi_build_last_key( // Tests // --------------------------------------------------------------------------- +#[cfg(test)] +#[path = "transaction_tests.rs"] +mod transaction_tests; + #[cfg(test)] mod tests { use rustack_dynamodb_model::{ diff --git a/crates/rustack-dynamodb-core/src/snapshot.rs b/crates/rustack-dynamodb-core/src/snapshot.rs index 4d19476..d0ce9b3 100644 --- a/crates/rustack-dynamodb-core/src/snapshot.rs +++ b/crates/rustack-dynamodb-core/src/snapshot.rs @@ -15,7 +15,7 @@ use thiserror::Error; use crate::{ provider::RustackDynamoDB, - state::DynamoDBTable, + state::{DynamoDBServiceState, DynamoDBTable}, storage::{KeySchema, StorageError, TableStorage}, }; @@ -96,6 +96,7 @@ impl RustackDynamoDB { /// Export DynamoDB tables and items into a snapshot. #[must_use] pub fn export_snapshot(&self) -> DynamoDBSnapshot { + let _operation = self.operation_gate.lock(); let tables = self .state .snapshot_tables() @@ -131,8 +132,8 @@ impl RustackDynamoDB { /// /// Returns an error if a stored item cannot be restored into its table. pub fn import_snapshot(&self, snapshot: DynamoDBSnapshot) -> Result<(), DynamoDBSnapshotError> { - self.reset(); - + let _operation = self.operation_gate.lock(); + let replacement = DynamoDBServiceState::new(); for table_snapshot in snapshot.tables { let table_name = table_snapshot.name.clone(); let storage = TableStorage::new(table_snapshot.key_schema.clone()); @@ -168,7 +169,7 @@ impl RustackDynamoDB { storage, }; - self.state.create_table(table).map_err(|source| { + replacement.create_table(table).map_err(|source| { DynamoDBSnapshotError::RestoreTable { table: table_name, source: Box::new(source), @@ -176,6 +177,8 @@ impl RustackDynamoDB { })?; } + self.state.replace_from(replacement); + self.tokens.clear(); Ok(()) } } diff --git a/crates/rustack-dynamodb-core/src/state.rs b/crates/rustack-dynamodb-core/src/state.rs index 8f5d523..c19edab 100644 --- a/crates/rustack-dynamodb-core/src/state.rs +++ b/crates/rustack-dynamodb-core/src/state.rs @@ -83,6 +83,14 @@ impl DynamoDBServiceState { names } + /// Publish a validated replacement while the provider operation gate is held. + pub(crate) fn replace_from(&self, replacement: Self) { + self.tables.clear(); + for (name, table) in replacement.tables { + self.tables.insert(name, table); + } + } + /// Reset all state (remove all tables). pub fn reset(&self) { self.tables.clear(); diff --git a/crates/rustack-dynamodb-core/src/storage.rs b/crates/rustack-dynamodb-core/src/storage.rs index 520d687..b9dbf24 100644 --- a/crates/rustack-dynamodb-core/src/storage.rs +++ b/crates/rustack-dynamodb-core/src/storage.rs @@ -510,6 +510,15 @@ impl TableStorage { item: HashMap, ) -> Result>, StorageError> { let primary_key = extract_primary_key(&self.key_schema, &item)?; + Ok(self.put_prepared(primary_key, item)) + } + + /// Commit an item whose key and value were validated during prepare. + pub(crate) fn put_prepared( + &self, + primary_key: PrimaryKey, + item: HashMap, + ) -> Option> { let sort_key = primary_key .sort_key .unwrap_or(SortableAttributeValue::Sentinel); @@ -528,12 +537,12 @@ impl TableStorage { self.total_size.fetch_add(new_size, AtomicOrdering::Relaxed); self.total_size.fetch_sub(old_size, AtomicOrdering::Relaxed); debug!(old_size, new_size, "replaced existing item"); - Ok(Some(old.attributes)) + Some(old.attributes) } else { self.item_count.fetch_add(1, AtomicOrdering::Relaxed); self.total_size.fetch_add(new_size, AtomicOrdering::Relaxed); debug!(new_size, "inserted new item"); - Ok(None) + None } } diff --git a/crates/rustack-dynamodb-core/src/stream.rs b/crates/rustack-dynamodb-core/src/stream.rs index b2b76fa..f81ba45 100644 --- a/crates/rustack-dynamodb-core/src/stream.rs +++ b/crates/rustack-dynamodb-core/src/stream.rs @@ -67,8 +67,11 @@ pub struct ChangeEvent { pub trait StreamEmitter: Send + Sync + 'static { /// Emit a change event for a successful write operation. /// - /// This method must not block. If the stream is disabled for the table, - /// the implementation should silently discard the event. + /// Called after all transaction items have committed, while the provider's + /// operation gate is held. Implementations must not block, panic, or reenter + /// the DynamoDB provider (including snapshot APIs). They may synchronously + /// append to their independently owned stream store or enqueue a notification. + /// If the stream is disabled for the table, discard the event. fn emit(&self, event: ChangeEvent); } @@ -88,7 +91,9 @@ impl StreamEmitter for NoopStreamEmitter { /// A lifecycle manager for DynamoDB Streams. /// /// Observes DynamoDB table creation/update/deletion and manages -/// corresponding streams in the `StreamStore`. +/// corresponding streams in the `StreamStore`. Like `StreamEmitter`, callbacks +/// run under the provider operation gate and must not block, panic, or reenter +/// the DynamoDB provider. pub trait StreamLifecycle: Send + Sync + 'static { /// Called after a successful `CreateTable` or `UpdateTable` that enables streaming. /// Returns the stream ARN. diff --git a/crates/rustack-dynamodb-core/src/transaction_tests.rs b/crates/rustack-dynamodb-core/src/transaction_tests.rs new file mode 100644 index 0000000..d9aff9b --- /dev/null +++ b/crates/rustack-dynamodb-core/src/transaction_tests.rs @@ -0,0 +1,382 @@ +//! Atomicity, isolation and full-window idempotency regressions. +use std::sync::{ + Barrier, + atomic::{AtomicUsize, Ordering}, +}; + +use super::*; + +#[derive(Default)] +struct CountingEmitter(AtomicUsize); +impl crate::stream::StreamEmitter for CountingEmitter { + fn emit(&self, _: crate::stream::ChangeEvent) { + self.0.fetch_add(1, Ordering::Relaxed); + } +} + +fn provider() -> (RustackDynamoDB, Arc) { + let mut provider = RustackDynamoDB::new(DynamoDBConfig::default()); + let emitter = Arc::new(CountingEmitter::default()); + provider.set_emitter(emitter.clone()); + provider + .handle_create_table( + serde_json::from_value(serde_json::json!({ + "TableName":"TestTable", "KeySchema":[{"AttributeName":"pk","KeyType":"HASH"}], + "AttributeDefinitions":[{"AttributeName":"pk","AttributeType":"S"}], + "BillingMode":"PAY_PER_REQUEST", + "StreamSpecification":{"StreamEnabled":true,"StreamViewType":"NEW_AND_OLD_IMAGES"} + })) + .unwrap(), + ) + .unwrap(); + (provider, emitter) +} + +#[test] +fn test_should_reserve_token_capacity_through_inflight_commit_and_replay() { + struct PausedEmitter { + entered: Barrier, + release: Barrier, + count: AtomicUsize, + } + impl crate::stream::StreamEmitter for PausedEmitter { + fn emit(&self, _: crate::stream::ChangeEvent) { + if self.count.fetch_add(1, Ordering::Relaxed) == 0 { + self.entered.wait(); + self.release.wait(); + } + } + } + let (mut provider, _) = provider(); + provider.token_capacity = 1; + let emitter = Arc::new(PausedEmitter { + entered: Barrier::new(2), + release: Barrier::new(2), + count: AtomicUsize::new(0), + }); + provider.set_emitter(emitter.clone()); + std::thread::scope(|scope| { + let first = scope.spawn(|| provider.handle_transact_write_items(increment("first"))); + emitter.entered.wait(); + let retry = scope.spawn(|| provider.handle_transact_write_items(increment("first"))); + let excess = scope.spawn(|| provider.handle_transact_write_items(increment("excess"))); + emitter.release.wait(); + first.join().unwrap().unwrap(); + retry.join().unwrap().unwrap(); + assert_eq!( + excess.join().unwrap().unwrap_err().code, + rustack_dynamodb_model::error::DynamoDBErrorCode::RequestLimitExceeded + ); + }); + assert_eq!(emitter.count.load(Ordering::Relaxed), 1); + assert_eq!( + get(&provider, "counter").get("n"), + Some(&AttributeValue::N("1".into())) + ); +} + +#[test] +fn test_should_allow_only_one_competing_conditional_transaction() { + let (provider, emitter) = provider(); + let start = Barrier::new(2); + let winners = AtomicUsize::new(0); + std::thread::scope(|scope| { + for _ in 0..2 { + scope.spawn(|| { + start.wait(); + let request = transaction(serde_json::json!({"TransactItems":[{"Put":{ + "TableName":"TestTable", "Item":{"pk":{"S":"unique"}}, + "ConditionExpression":"attribute_not_exists(pk)" + }}]})); + if provider.handle_transact_write_items(request).is_ok() { + winners.fetch_add(1, Ordering::Relaxed); + } + }); + } + }); + assert_eq!(winners.load(Ordering::Relaxed), 1); + assert_eq!(emitter.0.load(Ordering::Relaxed), 1); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_should_drain_cancelled_http_blocking_operation_before_snapshot() { + use bytes::Bytes; + use rustack_dynamodb_http::dispatch::DynamoDBHandler; + use rustack_dynamodb_model::operations::DynamoDBOperation; + + use crate::handler::RustackDynamoDBHandler; + struct PausedEmitter { + entered: tokio::sync::Notify, + release: Barrier, + } + impl crate::stream::StreamEmitter for PausedEmitter { + fn emit(&self, _: crate::stream::ChangeEvent) { + self.entered.notify_one(); + self.release.wait(); + } + } + let (mut provider, _) = provider(); + let emitter = Arc::new(PausedEmitter { + entered: tokio::sync::Notify::new(), + release: Barrier::new(2), + }); + provider.set_emitter(emitter.clone()); + let provider = Arc::new(provider); + let handler = RustackDynamoDBHandler::new(provider.clone()); + let request = tokio::spawn(handler.handle_operation( + DynamoDBOperation::PutItem, + Bytes::from_static(br#"{"TableName":"TestTable","Item":{"pk":{"S":"accepted"}}}"#), + )); + emitter.entered.notified().await; + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + assert!( + tokio::time::timeout(Duration::from_millis(10), provider.quiesce()) + .await + .is_err() + ); + assert!(provider.requests.admit().is_err()); + emitter.release.wait(); + tokio::time::timeout(Duration::from_secs(1), provider.quiesce()) + .await + .unwrap() + .unwrap(); + let snapshot = provider.export_snapshot(); + assert_eq!(snapshot.tables[0].items.len(), 1); + assert_eq!( + snapshot.tables[0].items[0].get("pk"), + Some(&AttributeValue::S("accepted".into())) + ); + assert!( + provider + .handle_get_item(GetItemInput { + table_name: "TestTable".into(), + ..Default::default() + }) + .is_err() + ); +} + +#[test] +fn test_should_reject_late_batch_value_error_without_data_or_stream_commit() { + let (provider, emitter) = provider(); + let input = serde_json::from_value(serde_json::json!({"RequestItems":{"TestTable":[ + {"PutRequest":{"Item":{"pk":{"S":"first"}}}}, + {"PutRequest":{"Item":{"pk":{"S":"second"},"n":{"N":"NaN"}}}} + ]}})) + .unwrap(); + assert!(provider.handle_batch_write_item(input).is_err()); + assert!(get(&provider, "first").is_empty()); + assert_eq!(emitter.0.load(Ordering::Relaxed), 0); +} + +#[test] +fn test_should_discard_prepared_prefix_when_later_item_exceeds_size_limit() { + let (provider, emitter) = provider(); + let input = transaction(serde_json::json!({"TransactItems":[ + {"Put":{"TableName":"TestTable","Item":{"pk":{"S":"first"}}}}, + {"Put":{"TableName":"TestTable","Item":{"pk":{"S":"second"},"payload":{"S":"x".repeat(400 * 1024)}}}} + ]})); + assert!(provider.handle_transact_write_items(input).is_err()); + assert!(get(&provider, "first").is_empty()); + assert_eq!(emitter.0.load(Ordering::Relaxed), 0); +} + +fn transaction(value: serde_json::Value) -> TransactWriteItemsInput { + serde_json::from_value(value).unwrap() +} + +fn increment(token: &str) -> TransactWriteItemsInput { + transaction( + serde_json::json!({"ClientRequestToken":token,"TransactItems":[{"Update":{ + "TableName":"TestTable", "Key":{"pk":{"S":"counter"}}, + "UpdateExpression":"ADD n :one", "ExpressionAttributeValues":{":one":{"N":"1"}} + }}]}), + ) +} + +fn get(provider: &RustackDynamoDB, key: &str) -> HashMap { + provider + .handle_get_item(GetItemInput { + table_name: "TestTable".into(), + key: HashMap::from([("pk".into(), AttributeValue::S(key.into()))]), + ..Default::default() + }) + .unwrap() + .item + .unwrap_or_default() +} + +#[test] +fn test_should_discard_all_prepared_writes_and_streams_on_late_errors() { + for update in [ + "SET", + "SET pk = :one", + "SET n = missing + :one", + "ADD n :one", + ] { + let (provider, emitter) = provider(); + let values = if update == "ADD n :one" { + serde_json::json!({":one":{"S":"bad"}}) + } else if update == "SET" { + serde_json::json!({}) + } else { + serde_json::json!({":one":{"N":"1"}}) + }; + let request = transaction(serde_json::json!({"TransactItems":[ + {"Put":{"TableName":"TestTable","Item":{"pk":{"S":"first"}}}}, + {"Update":{"TableName":"TestTable","Key":{"pk":{"S":"second"}}, + "UpdateExpression":update,"ExpressionAttributeValues":values}} + ]})); + assert!( + provider.handle_transact_write_items(request).is_err(), + "{update}" + ); + assert!(get(&provider, "first").is_empty()); + assert_eq!(emitter.0.load(Ordering::Relaxed), 0); + } +} + +#[test] +fn test_should_preserve_successful_tokens_at_capacity_until_full_window_expires() { + let (mut provider, emitter) = provider(); + provider.token_capacity = 1; + provider + .handle_transact_write_items(increment("first")) + .unwrap(); + let err = provider + .handle_transact_write_items(increment("second")) + .unwrap_err(); + assert_eq!( + err.code, + rustack_dynamodb_model::error::DynamoDBErrorCode::RequestLimitExceeded + ); + provider + .handle_transact_write_items(increment("first")) + .unwrap(); + assert_eq!( + get(&provider, "counter").get("n"), + Some(&AttributeValue::N("1".into())) + ); + let mut mismatch = increment("first"); + mismatch.return_consumed_capacity = + Some(rustack_dynamodb_model::types::ReturnConsumedCapacity::Total); + assert_eq!( + provider + .handle_transact_write_items(mismatch) + .unwrap_err() + .code, + rustack_dynamodb_model::error::DynamoDBErrorCode::IdempotentParameterMismatchException + ); + assert_eq!(emitter.0.load(Ordering::Relaxed), 1); + provider.tokens.get_mut("first").unwrap().completed_at = + Instant::now().checked_sub(Duration::from_mins(10)).unwrap(); + provider + .handle_transact_write_items(increment("first")) + .unwrap(); + assert_eq!( + get(&provider, "counter").get("n"), + Some(&AttributeValue::N("2".into())) + ); +} + +#[test] +fn test_should_execute_concurrent_same_token_only_once() { + let (provider, emitter) = provider(); + let provider = Arc::new(provider); + let barrier = Arc::new(Barrier::new(8)); + std::thread::scope(|scope| { + for _ in 0..8 { + let provider = &provider; + let barrier = &barrier; + scope.spawn(move || { + barrier.wait(); + provider + .handle_transact_write_items(increment("same")) + .unwrap(); + }); + } + }); + assert_eq!( + get(&provider, "counter").get("n"), + Some(&AttributeValue::N("1".into())) + ); + assert_eq!(emitter.0.load(Ordering::Relaxed), 1); +} + +#[test] +fn test_should_coordinate_ordinary_conditional_writes_with_transactions() { + let (provider, emitter) = provider(); + let barrier = Barrier::new(2); + let winners = AtomicUsize::new(0); + std::thread::scope(|scope| { + scope.spawn(|| { + barrier.wait(); + let result = provider.handle_transact_write_items(transaction( + serde_json::json!({"TransactItems":[ + {"Put":{"TableName":"TestTable","Item":{"pk":{"S":"unique"}}, + "ConditionExpression":"attribute_not_exists(pk)"}} + ]}), + )); + if result.is_ok() { + winners.fetch_add(1, Ordering::Relaxed); + } + }); + scope.spawn(|| { + barrier.wait(); + let result = provider.handle_put_item( + serde_json::from_value(serde_json::json!({ + "TableName":"TestTable","Item":{"pk":{"S":"unique"}}, + "ConditionExpression":"attribute_not_exists(pk)" + })) + .unwrap(), + ); + if result.is_ok() { + winners.fetch_add(1, Ordering::Relaxed); + } + }); + }); + assert_eq!(winners.load(Ordering::Relaxed), 1); + assert_eq!(emitter.0.load(Ordering::Relaxed), 1); +} + +#[test] +fn test_should_never_observe_torn_transaction_reads() { + let (provider, _) = provider(); + let write = |n| { + transaction(serde_json::json!({"TransactItems":[ + {"Put":{"TableName":"TestTable","Item":{"pk":{"S":"a"},"n":{"N":format!("{n}")}}}}, + {"Put":{"TableName":"TestTable","Item":{"pk":{"S":"b"},"n":{"N":format!("{n}")}}}} + ]})) + }; + provider.handle_transact_write_items(write(0)).unwrap(); + let barrier = Barrier::new(2); + std::thread::scope(|scope| { + scope.spawn(|| { + barrier.wait(); + for n in 1..100 { + provider.handle_transact_write_items(write(n)).unwrap(); + } + }); + scope.spawn(|| { + barrier.wait(); + for _ in 0..100 { + let output = provider + .handle_transact_get_items( + serde_json::from_value(serde_json::json!({"TransactItems":[ + {"Get":{"TableName":"TestTable","Key":{"pk":{"S":"a"}}}}, + {"Get":{"TableName":"TestTable","Key":{"pk":{"S":"b"}}}} + ]})) + .unwrap(), + ) + .unwrap() + .responses + .unwrap(); + assert_eq!( + output[0].item.as_ref().unwrap().get("n"), + output[1].item.as_ref().unwrap().get("n") + ); + } + }); + }); +} diff --git a/crates/rustack-dynamodb-http/Cargo.toml b/crates/rustack-dynamodb-http/Cargo.toml index 1e31bd7..b6ebda9 100644 --- a/crates/rustack-dynamodb-http/Cargo.toml +++ b/crates/rustack-dynamodb-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-dynamodb-http" description = "DynamoDB HTTP service layer for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-dynamodb-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-dynamodb-http/src/service.rs b/crates/rustack-dynamodb-http/src/service.rs index d99077f..0d678e5 100644 --- a/crates/rustack-dynamodb-http/src/service.rs +++ b/crates/rustack-dynamodb-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_dynamodb_model::error::DynamoDBError; @@ -128,19 +127,17 @@ async fn process_request( }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = DynamoDBError::with_message( - rustack_dynamodb_model::error::DynamoDBErrorCode::AccessDeniedException, - auth_err.to_string(), - ); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let err = DynamoDBError::with_message( + rustack_dynamodb_model::error::DynamoDBErrorCode::AccessDeniedException, + auth_err.to_string(), + ); + return error_to_response(&err, request_id); } // 5. Dispatch to handler. @@ -152,11 +149,13 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| DynamoDBError::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = DynamoDBError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every DynamoDB response. diff --git a/crates/rustack-dynamodbstreams-core/Cargo.toml b/crates/rustack-dynamodbstreams-core/Cargo.toml index 4ea5674..fa17a97 100644 --- a/crates/rustack-dynamodbstreams-core/Cargo.toml +++ b/crates/rustack-dynamodbstreams-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-dynamodbstreams-core" description = "DynamoDB Streams business logic and storage engine for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-dynamodbstreams-core/src/config.rs b/crates/rustack-dynamodbstreams-core/src/config.rs index 8b3425c..464b801 100644 --- a/crates/rustack-dynamodbstreams-core/src/config.rs +++ b/crates/rustack-dynamodbstreams-core/src/config.rs @@ -1,6 +1,6 @@ //! DynamoDB Streams configuration. -use std::env; +use rustack_core::settings as env; /// DynamoDB Streams service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-dynamodbstreams-http/Cargo.toml b/crates/rustack-dynamodbstreams-http/Cargo.toml index d0320bd..e7be0ea 100644 --- a/crates/rustack-dynamodbstreams-http/Cargo.toml +++ b/crates/rustack-dynamodbstreams-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-dynamodbstreams-http" description = "DynamoDB Streams HTTP service layer for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-dynamodbstreams-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-dynamodbstreams-http/src/service.rs b/crates/rustack-dynamodbstreams-http/src/service.rs index 6453235..279f013 100644 --- a/crates/rustack-dynamodbstreams-http/src/service.rs +++ b/crates/rustack-dynamodbstreams-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_dynamodbstreams_model::error::DynamoDBStreamsError; @@ -128,16 +127,15 @@ async fn process_request( }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = DynamoDBStreamsError::internal_error(auth_err.to_string()); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let mut err = DynamoDBStreamsError::internal_error(auth_err.to_string()); + err.status_code = http::StatusCode::FORBIDDEN; + return error_to_response(&err, request_id); } // 5. Dispatch to handler. @@ -149,12 +147,12 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) .map_err(|e| { - DynamoDBStreamsError::internal_error(format!("Failed to read request body: {e}")) + let mut err = DynamoDBStreamsError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err }) } diff --git a/crates/rustack-events-core/Cargo.toml b/crates/rustack-events-core/Cargo.toml index 843d104..b49f1e3 100644 --- a/crates/rustack-events-core/Cargo.toml +++ b/crates/rustack-events-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-events-core" description = "EventBridge business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-events-core/src/config.rs b/crates/rustack-events-core/src/config.rs index 7fbab3a..f2c9af3 100644 --- a/crates/rustack-events-core/src/config.rs +++ b/crates/rustack-events-core/src/config.rs @@ -1,6 +1,6 @@ //! EventBridge service configuration. -use std::env; +use rustack_core::settings as env; /// EventBridge service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-events-core/src/delivery.rs b/crates/rustack-events-core/src/delivery.rs index 26f1b7c..185ba5a 100644 --- a/crates/rustack-events-core/src/delivery.rs +++ b/crates/rustack-events-core/src/delivery.rs @@ -1,40 +1,250 @@ -//! Target delivery abstraction for EventBridge. +//! Typed target delivery and bounded, drainable dispatch. //! -//! The `TargetDelivery` trait defines how matched events are delivered to -//! targets. Implementations may deliver to SQS, SNS, Lambda, or other -//! services. The `NoopTargetDelivery` implementation logs and discards events. +//! `async-trait` is required for object-safe `Arc` bridges. +use std::{ + sync::{ + Arc, OnceLock, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + time::Duration, +}; use async_trait::async_trait; +pub use rustack_events_model::types::Target; +use tokio::sync::{mpsc, watch}; -/// Errors that can occur during target delivery. +/// Explicit target delivery failure. #[derive(Debug, thiserror::Error)] pub enum DeliveryError { - /// The target ARN is malformed or unsupported. + /// Invalid resource identifier. #[error("Invalid target ARN: {0}")] InvalidArn(String), - /// Delivery to the target failed. + /// Unsupported execution capability or parameter. + #[error("Unsupported target delivery: {0}")] + Unsupported(String), + /// Required runtime service is disabled or unavailable. + #[error("Target service unavailable: {0}")] + Unavailable(String), + /// Target rejected the delivery. #[error("Target delivery failed: {0}")] TargetError(String), } -/// Trait for delivering matched events to targets. -/// -/// This trait uses `async-trait` because it requires object safety for -/// dynamic dispatch via `Arc`. +/// Bridge implemented in the application, never by depending on another core. #[async_trait] pub trait TargetDelivery: Send + Sync + std::fmt::Debug + 'static { - /// Deliver an event (as a JSON string) to the specified target ARN. - async fn deliver(&self, target_arn: &str, event_json: &str) -> Result<(), DeliveryError>; + /// Validate configuration against the bridge's actual capabilities. + fn validate(&self, target: &Target) -> Result<(), DeliveryError>; + /// Deliver the complete target configuration and transformed JSON body. + async fn deliver(&self, target: &Target, event_json: &str) -> Result<(), DeliveryError>; } -/// A no-op delivery implementation that logs events but does not deliver them. +/// Explicit unavailable dependency, suitable for metadata-only runtime wiring. #[derive(Debug)] -pub struct NoopTargetDelivery; - +pub struct UnavailableTargetDelivery; #[async_trait] -impl TargetDelivery for NoopTargetDelivery { - async fn deliver(&self, target_arn: &str, _event_json: &str) -> Result<(), DeliveryError> { - tracing::debug!(target_arn = %target_arn, "NoopTargetDelivery: event not delivered"); +impl TargetDelivery for UnavailableTargetDelivery { + fn validate(&self, _: &Target) -> Result<(), DeliveryError> { + Err(DeliveryError::Unavailable("SQS is not enabled".into())) + } + async fn deliver(&self, target: &Target, _: &str) -> Result<(), DeliveryError> { + self.validate(target) + } +} + +/// Cumulative dispatch counters. Accepted counts target attempts, not API calls. +#[derive(Debug, Clone, Copy, Default)] +pub struct DeliveryStats { + /// Accepted target attempts. + pub accepted: u64, + /// Successfully delivered target attempts. + pub delivered: u64, + /// Failed attempts, including timeout and panic. + pub failed: u64, + /// Rejected event batches (capacity, lifecycle or payload budget). + pub rejected: u64, +} + +#[derive(Debug, Default)] +struct Counters { + accepted: AtomicU64, + delivered: AtomicU64, + failed: AtomicU64, + rejected: AtomicU64, +} + +#[derive(Debug)] +pub(crate) struct DeliveryJob { + pub target: Target, + pub body: String, +} + +#[cfg(test)] +#[path = "delivery_tests.rs"] +mod tests; + +enum Command { + #[cfg(test)] + CrashWorker, + Deliver(Vec), + Quiesce, +} +struct Worker { + sender: mpsc::Sender, + stopped: watch::Receiver>, +} + +pub(crate) struct DeliveryQueue { + bridge: Arc, + worker: OnceLock, + closing: Arc, + counters: Arc, +} + +impl DeliveryQueue { + pub fn new(bridge: Arc) -> Self { + Self { + bridge, + worker: OnceLock::new(), + closing: Arc::new(AtomicBool::new(false)), + counters: Arc::new(Counters::default()), + } + } + + pub fn submit(&self, jobs: Vec) -> Result<(), DeliveryError> { + if jobs.len() > 128 || jobs.iter().map(|job| job.body.len()).sum::() > 1024 * 1024 { + self.counters.rejected.fetch_add(1, Ordering::Relaxed); + return Err(DeliveryError::TargetError( + "Event fanout exceeds 128 targets or 1 MiB".into(), + )); + } + let runtime = tokio::runtime::Handle::try_current() + .map_err(|_| DeliveryError::Unavailable("Async runtime is not running".into()))?; + let worker = self.worker.get_or_init(|| { + let (sender, receiver) = mpsc::channel(128); + let (finished, stopped) = watch::channel(None); + let bridge = Arc::clone(&self.bridge); + let counters = Arc::clone(&self.counters); + let closing = Arc::clone(&self.closing); + // One supervisor per provider; it observes the worker's panic/result. + runtime.spawn(async move { + let result = tokio::spawn(run_worker(receiver, bridge, counters, closing)).await; + let _ = finished.send(Some(result.is_ok())); + }); + Worker { sender, stopped } + }); + let count = u64::try_from(jobs.len()).unwrap_or(u64::MAX); + if self.closing.load(Ordering::Acquire) + || worker.sender.try_send(Command::Deliver(jobs)).is_err() + { + self.counters.rejected.fetch_add(1, Ordering::Relaxed); + return Err(DeliveryError::Unavailable( + "Delivery queue is full or quiescing".into(), + )); + } + self.counters.accepted.fetch_add(count, Ordering::Relaxed); Ok(()) } + + pub fn is_ready(&self) -> bool { + !self.closing.load(Ordering::Acquire) + && self.worker.get().is_none_or(|worker| { + !worker.sender.is_closed() + && worker.stopped.borrow().is_none() + && worker.stopped.has_changed().is_ok() + }) + } + + /// Test-only supervision fault injection: crash the worker and observe failure. + #[cfg(test)] + pub fn inject_crash(&self) { + if let Some(worker) = self.worker.get() { + let _ = worker.sender.try_send(Command::CrashWorker); + } + } + + pub fn stats(&self) -> DeliveryStats { + DeliveryStats { + accepted: self.counters.accepted.load(Ordering::Relaxed), + delivered: self.counters.delivered.load(Ordering::Relaxed), + failed: self.counters.failed.load(Ordering::Relaxed), + rejected: self.counters.rejected.load(Ordering::Relaxed), + } + } + + pub async fn quiesce(&self) -> Result<(), DeliveryError> { + self.closing.store(true, Ordering::Release); + let Some(worker) = self.worker.get() else { + return Ok(()); + }; + // A full channel already wakes the worker, which checks closing before each receive. + let _ = worker.sender.try_send(Command::Quiesce); + let mut stopped = worker.stopped.clone(); + loop { + if let Some(success) = *stopped.borrow_and_update() { + return if success { + Ok(()) + } else { + Err(DeliveryError::TargetError( + "Delivery worker panicked".into(), + )) + }; + } + stopped + .changed() + .await + .map_err(|_| DeliveryError::Unavailable("Delivery supervisor stopped".into()))?; + } + } +} + +async fn run_worker( + mut receiver: mpsc::Receiver, + bridge: Arc, + counters: Arc, + closing: Arc, +) { + loop { + if closing.load(Ordering::Acquire) { + receiver.close(); + } + let Some(command) = receiver.recv().await else { + break; + }; + match command { + #[cfg(test)] + Command::CrashWorker => panic!("injected worker panic"), + Command::Quiesce => receiver.close(), + Command::Deliver(jobs) => { + for job in jobs { + let bridge = Arc::clone(&bridge); + let mut task = + tokio::spawn(async move { bridge.deliver(&job.target, &job.body).await }); + let success = + match tokio::time::timeout(Duration::from_secs(5), &mut task).await { + Ok(Ok(Ok(()))) => true, + Ok(Ok(Err(error))) => { + tracing::warn!(error = %error, "Event target delivery failed"); + false + } + Ok(Err(error)) => { + tracing::error!(error = %error, "Event target task failed"); + false + } + Err(_) => { + task.abort(); + let _ = task.await; + tracing::warn!("Event target delivery timed out"); + false + } + }; + if success { + counters.delivered.fetch_add(1, Ordering::Relaxed); + } else { + counters.failed.fetch_add(1, Ordering::Relaxed); + } + } + } + } + } } diff --git a/crates/rustack-events-core/src/delivery_tests.rs b/crates/rustack-events-core/src/delivery_tests.rs new file mode 100644 index 0000000..b4edd26 --- /dev/null +++ b/crates/rustack-events-core/src/delivery_tests.rs @@ -0,0 +1,181 @@ +//! Bounded acceptance, real worker lifecycle, and explicit failure tests. +use super::*; +use crate::{config::EventsConfig, provider::RustackEvents}; + +#[derive(Debug)] +struct Recording(mpsc::Sender); +#[async_trait] +impl TargetDelivery for Recording { + fn validate(&self, _: &Target) -> Result<(), DeliveryError> { + Ok(()) + } + async fn deliver(&self, _: &Target, body: &str) -> Result<(), DeliveryError> { + self.0 + .send(body.to_owned()) + .await + .map_err(|_| DeliveryError::Unavailable("Recorder closed".into())) + } +} +fn target() -> Target { + serde_json::from_value(serde_json::json!({"Id":"target","Arn":"arn:aws:sqs:us-east-1:000000000000:q.fifo","SqsParameters":{"MessageGroupId":"g"}})).unwrap() +} +fn job(body: &str) -> Vec { + vec![DeliveryJob { + target: target(), + body: body.into(), + }] +} + +#[tokio::test] +async fn test_should_resume_waiting_for_drain_after_quiesce_future_is_cancelled() { + let (sender, mut receiver) = mpsc::channel(1); + sender.send("occupied".into()).await.unwrap(); + let queue = DeliveryQueue::new(Arc::new(Recording(sender))); + queue.submit(job("accepted")).unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(1), queue.quiesce()) + .await + .is_err() + ); + assert!(queue.submit(job("late")).is_err()); + assert_eq!(receiver.recv().await.as_deref(), Some("occupied")); + tokio::time::timeout(Duration::from_secs(1), queue.quiesce()) + .await + .unwrap() + .unwrap(); + assert_eq!(receiver.recv().await.as_deref(), Some("accepted")); + assert_eq!(queue.stats().delivered, 1); +} + +#[tokio::test] +async fn test_should_deliver_in_order_and_reject_after_quiesce() { + let (sender, mut receiver) = mpsc::channel(8); + let queue = DeliveryQueue::new(Arc::new(Recording(sender))); + for n in 0..5 { + queue.submit(job(&n.to_string())).unwrap(); + } + queue.quiesce().await.unwrap(); + queue.quiesce().await.unwrap(); + for n in 0..5 { + assert_eq!(receiver.recv().await.unwrap(), n.to_string()); + } + assert!(queue.submit(job("late")).is_err()); + assert_eq!(queue.stats().delivered, 5); + assert_eq!(queue.stats().failed, 0); +} + +#[tokio::test] +async fn test_should_bound_acceptance_and_quiesce_even_when_channel_was_full() { + let (sender, _receiver) = mpsc::channel(1); + let queue = DeliveryQueue::new(Arc::new(Recording(sender))); + // This current-thread task does not yield, so the worker has not consumed a slot. + for _ in 0..128 { + queue.submit(Vec::new()).unwrap(); + } + assert!(queue.submit(Vec::new()).is_err()); + tokio::time::timeout(Duration::from_secs(1), queue.quiesce()) + .await + .unwrap() + .unwrap(); + assert_eq!(queue.stats().rejected, 1); +} + +#[tokio::test] +async fn test_should_record_unavailable_delivery_as_failed_not_delivered() { + let queue = DeliveryQueue::new(Arc::new(UnavailableTargetDelivery)); + queue.submit(job("event")).unwrap(); + queue.quiesce().await.unwrap(); + assert_eq!(queue.stats().accepted, 1); + assert_eq!(queue.stats().delivered, 0); + assert_eq!(queue.stats().failed, 1); +} + +#[test] +fn test_should_allow_metadata_targets_without_runtime_dependency() { + let provider = RustackEvents::new(EventsConfig::default(), Arc::new(UnavailableTargetDelivery)); + provider + .handle_put_rule( + serde_json::from_value(serde_json::json!({"Name":"rule","EventPattern":"{}"})).unwrap(), + ) + .unwrap(); + let output = provider + .handle_put_targets( + serde_json::from_value(serde_json::json!({"Rule":"rule","Targets":[target()]})) + .unwrap(), + ) + .unwrap(); + assert_eq!( + output.failed_entry_count, 0, + "metadata configuration must not require a runtime delivery bridge" + ); + let listed = provider + .handle_list_targets_by_rule( + &serde_json::from_value(serde_json::json!({"Rule":"rule"})).unwrap(), + ) + .unwrap(); + assert_eq!(listed.targets.len(), 1); + // Execution support is the bridge's contract: unavailable delivery is an + // explicit terminal failure, never a silent configuration success. + let unavailable = UnavailableTargetDelivery; + assert!( + unavailable + .validate( + &serde_json::from_value( + serde_json::json!({"Id":"t","Arn":"arn:aws:sqs:us-east-1:000000000000:q"}) + ) + .unwrap() + ) + .is_err() + ); +} + +#[derive(Debug)] +struct Panicking; +#[async_trait] +impl TargetDelivery for Panicking { + fn validate(&self, _: &Target) -> Result<(), DeliveryError> { + Ok(()) + } + async fn deliver(&self, _: &Target, _: &str) -> Result<(), DeliveryError> { + panic!("injected bridge panic") + } +} +#[tokio::test] +async fn test_should_observe_bridge_panic_and_keep_worker_drainable() { + let queue = DeliveryQueue::new(Arc::new(Panicking)); + queue.submit(job("event")).unwrap(); + queue.quiesce().await.unwrap(); + assert_eq!(queue.stats().failed, 1); +} + +#[tokio::test] +async fn test_should_observe_worker_crash_and_report_not_ready() { + let queue = DeliveryQueue::new(Arc::new(Panicking)); + queue.submit(Vec::new()).unwrap(); + queue.inject_crash(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while queue.is_ready() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!queue.is_ready(), "crashed worker must report not ready"); + assert!(queue.submit(job("late")).is_err()); + let error = tokio::time::timeout(Duration::from_secs(2), queue.quiesce()) + .await + .unwrap() + .expect_err("quiesce must surface the worker crash"); + assert!(matches!(error, DeliveryError::TargetError(_))); +} + +#[tokio::test] +async fn test_should_timeout_a_stalled_target_with_observable_terminal_failure() { + let (sender, _receiver) = mpsc::channel(1); + sender.send("occupy".into()).await.unwrap(); + let queue = DeliveryQueue::new(Arc::new(Recording(sender))); + queue.submit(job("event")).unwrap(); + tokio::time::timeout(Duration::from_secs(6), queue.quiesce()) + .await + .unwrap() + .unwrap(); + assert_eq!(queue.stats().failed, 1); + assert_eq!(queue.stats().delivered, 0); +} diff --git a/crates/rustack-events-core/src/pattern/engine.rs b/crates/rustack-events-core/src/pattern/engine.rs index 3c82cc4..52fad4d 100644 --- a/crates/rustack-events-core/src/pattern/engine.rs +++ b/crates/rustack-events-core/src/pattern/engine.rs @@ -99,10 +99,7 @@ fn all_fields_are_exists_false(fields: &[FieldMatcher]) -> bool { fn navigate_path<'a>(event: &'a Value, path: &[String]) -> Option<&'a Value> { let mut current = event; for segment in path { - match current.get(segment.as_str()) { - Some(next) => current = next, - None => return None, - } + current = current.get(segment.as_str())?; } Some(current) } diff --git a/crates/rustack-events-core/src/provider.rs b/crates/rustack-events-core/src/provider.rs index 441588d..5968751 100644 --- a/crates/rustack-events-core/src/provider.rs +++ b/crates/rustack-events-core/src/provider.rs @@ -25,10 +25,14 @@ use rustack_events_model::{ PutRuleOutput, PutTargetsOutput, RemovePermissionOutput, RemoveTargetsOutput, TagResourceOutput, TestEventPatternOutput, UntagResourceOutput, UpdateEventBusOutput, }, - types::{EventBus, InputTransformer, PutEventsResultEntry, Rule, Tag, Target}, + types::{EventBus, PutEventsResultEntry, Rule, Tag, Target}, }; -use crate::{config::EventsConfig, delivery::TargetDelivery, pattern::EventPattern}; +use crate::{ + config::EventsConfig, + delivery::{DeliveryError, DeliveryJob, DeliveryQueue, DeliveryStats, TargetDelivery}, + pattern::EventPattern, +}; /// Maximum number of entries per `PutEvents` call. const MAX_PUT_EVENTS_ENTRIES: usize = 10; @@ -77,21 +81,7 @@ struct RuleState { created_at: String, } -#[derive(Clone)] -struct TargetState { - id: String, - arn: String, - role_arn: Option, - input_path: Option, - input: Option, - input_transformer: Option, -} - -#[derive(Clone)] -struct InputTransformerState { - input_paths_map: HashMap, - input_template: String, -} +type TargetState = Target; // --------------------------------------------------------------------------- // Provider @@ -103,6 +93,7 @@ pub struct RustackEvents { config: EventsConfig, buses: DashMap, delivery: Arc, + delivery_queue: DeliveryQueue, /// Phase 3: Archive metadata storage (key = archive name). archives: DashMap, /// Phase 3: Connection metadata storage (key = connection name). @@ -137,6 +128,7 @@ impl RustackEvents { let provider = Self { config, buses: DashMap::new(), + delivery_queue: DeliveryQueue::new(Arc::clone(&delivery)), delivery, archives: DashMap::new(), connections: DashMap::new(), @@ -148,6 +140,34 @@ impl RustackEvents { provider } + /// Whether admission is open and the lazily started delivery worker is healthy. + #[must_use] + pub fn is_ready(&self) -> bool { + self.delivery_queue.is_ready() + } + + /// Return cumulative delivery outcomes; API acceptance is not delivery success. + #[must_use] + pub fn delivery_stats(&self) -> DeliveryStats { + self.delivery_queue.stats() + } + + /// Stop new events and drain accepted delivery batches. + /// + /// # Errors + /// Reports a failed worker/supervisor. The runtime supplies the total shutdown deadline. + pub async fn quiesce(&self) -> Result<(), DeliveryError> { + self.delivery_queue.quiesce().await + } + + /// Shut down delivery, idempotently, without altering event bus resources. + /// + /// # Errors + /// Reports the same worker failure as `quiesce`. + pub async fn shutdown(&self) -> Result<(), DeliveryError> { + self.quiesce().await + } + fn create_default_bus(&self) { let arn = format!( "arn:aws:events:{}:{}:event-bus/default", @@ -684,22 +704,19 @@ impl RustackEvents { continue; } - let transformer = target.input_transformer.map(|it| InputTransformerState { - input_paths_map: it.input_paths_map, - input_template: it.input_template, - }); - - rule.targets.insert( - target.id.clone(), - TargetState { - id: target.id, - arn: target.arn, - role_arn: target.role_arn, - input_path: target.input_path, - input: target.input, - input_transformer: transformer, - }, - ); + // Metadata configuration is allowed for any structurally valid target + // ARN (AWS stores rules for many target services). Runtime support is + // the delivery bridge's job: unsupported/unavailable targets fail + // explicitly and observably when an event is actually dispatched. + if let Err(error) = validate_target(&target) { + failed_entries.push(rustack_events_model::types::PutTargetsResultEntry { + target_id: Some(target.id.clone()), + error_code: Some("ValidationException".to_owned()), + error_message: Some(error.to_string()), + }); + continue; + } + rule.targets.insert(target.id.clone(), target); } #[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)] @@ -760,7 +777,7 @@ impl RustackEvents { let page_size = resolve_page_size(input.limit); - let mut targets: Vec = rule.targets.values().map(target_state_to_model).collect(); + let mut targets: Vec = rule.targets.values().cloned().collect(); targets.sort_by(|a, b| a.id.cmp(&b.id)); @@ -789,6 +806,8 @@ impl RustackEvents { /// Handle `PutEvents`. Routes events through pattern matching to targets /// and delivers them asynchronously via spawned tasks. + // The per-event routing loop is intentionally explicit about its entry outcomes. + #[allow(clippy::too_many_lines)] pub fn handle_put_events( &self, input: &PutEventsInput, @@ -803,6 +822,19 @@ impl RustackEvents { let mut failed_count = 0i32; for entry in &input.entries { + if entry + .detail + .as_ref() + .is_some_and(|detail| detail.len() > 256 * 1024) + { + failed_count += 1; + result_entries.push(PutEventsResultEntry { + event_id: None, + error_code: Some("ValidationException".into()), + error_message: Some("Detail exceeds 256 KiB".into()), + }); + continue; + } let source = entry.source.as_deref().unwrap_or(""); let detail_type = entry.detail_type.as_deref().unwrap_or(""); let detail = entry.detail.as_deref().unwrap_or("{}"); @@ -827,9 +859,10 @@ impl RustackEvents { let event_id = envelope["id"].as_str().unwrap_or_default().to_owned(); + let mut jobs = Vec::new(); // Route through matching rules in the bus. if let Some(bus) = self.buses.get(&bus_name) { - for rule in bus.rules.values() { + 'rules: for rule in bus.rules.values() { if rule.state != "ENABLED" { continue; } @@ -844,22 +877,51 @@ impl RustackEvents { if matched { for target in rule.targets.values() { let event_json = Self::apply_input_transform(target, &envelope); - let delivery = Arc::clone(&self.delivery); - let target_arn = target.arn.clone(); - tokio::spawn(async move { - if let Err(e) = delivery.deliver(&target_arn, &event_json).await { - tracing::warn!( - target_arn = %target_arn, - error = %e, - "Failed to deliver event to target", - ); - } + jobs.push(DeliveryJob { + target: target.clone(), + body: event_json, }); + if jobs.len() > 128 + || jobs.iter().map(|job| job.body.len()).sum::() + > 1024 * 1024 + { + break 'rules; + } } } } } + jobs.sort_by(|a, b| { + a.target + .arn + .cmp(&b.target.arn) + .then(a.target.id.cmp(&b.target.id)) + }); + // Targets are configurable as metadata for any service ARN; execution + // support is enforced at dispatch time so an unsupported target yields + // an explicit event failure instead of a silent drop. + if let Some(error) = jobs + .iter() + .find_map(|job| self.delivery.validate(&job.target).err()) + { + failed_count += 1; + result_entries.push(PutEventsResultEntry { + event_id: None, + error_code: Some("InvalidTarget".to_owned()), + error_message: Some(error.to_string()), + }); + continue; + } + if let Err(error) = self.delivery_queue.submit(jobs) { + failed_count += 1; + result_entries.push(PutEventsResultEntry { + event_id: None, + error_code: Some("InternalFailure".to_owned()), + error_message: Some(error.to_string()), + }); + continue; + } result_entries.push(PutEventsResultEntry { event_id: Some(event_id), error_code: None, @@ -1953,27 +2015,83 @@ fn apply_json_path(value: &serde_json::Value, path: &str) -> serde_json::Value { current.clone() } -/// Convert internal `TargetState` to the model `Target` type. -fn target_state_to_model(t: &TargetState) -> Target { - Target { - id: t.id.clone(), - arn: t.arn.clone(), - role_arn: t.role_arn.clone(), - input: t.input.clone(), - input_path: t.input_path.clone(), - input_transformer: t.input_transformer.as_ref().map(|it| InputTransformer { - input_paths_map: it.input_paths_map.clone(), - input_template: it.input_template.clone(), - }), - run_command_parameters: None, - ecs_parameters: None, - batch_parameters: None, - sqs_parameters: None, - http_parameters: None, - redshift_data_parameters: None, - sage_maker_pipeline_parameters: None, - dead_letter_config: None, - retry_policy: None, - app_sync_parameters: None, +fn validate_target(target: &Target) -> Result<(), DeliveryError> { + if target.id.is_empty() + || target.id.len() > 64 + || !target + .id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b".-_".contains(&b)) + || target.arn.len() > 1600 + { + return Err(DeliveryError::InvalidArn( + "Invalid target ID or ARN length".into(), + )); + } + if target.role_arn.is_some() + || target.retry_policy.is_some() + || target.dead_letter_config.is_some() + || target.run_command_parameters.is_some() + || target.ecs_parameters.is_some() + || target.batch_parameters.is_some() + || target.kinesis_parameters.is_some() + || target.http_parameters.is_some() + || target.redshift_data_parameters.is_some() + || target.sage_maker_pipeline_parameters.is_some() + || target.app_sync_parameters.is_some() + { + return Err(DeliveryError::Unsupported( + "Only SQS target and input transformation parameters are executable".into(), + )); } + if let Some(parameters) = &target.sqs_parameters { + let group = ¶meters.message_group_id; + if group.is_empty() || group.len() > 128 || !group.bytes().all(|b| b.is_ascii_graphic()) { + return Err(DeliveryError::Unsupported( + "MessageGroupId must contain 1..128 printable ASCII bytes".into(), + )); + } + } + if [ + target.input.is_some(), + target.input_path.is_some(), + target.input_transformer.is_some(), + ] + .into_iter() + .filter(|present| *present) + .count() + > 1 + { + return Err(DeliveryError::Unsupported( + "Input, InputPath and InputTransformer are mutually exclusive".into(), + )); + } + if let Some(input) = &target.input { + if input.len() > 8192 || serde_json::from_str::(input).is_err() { + return Err(DeliveryError::Unsupported( + "Input must be JSON of at most 8192 bytes".into(), + )); + } + } + if target + .input_path + .as_ref() + .is_some_and(|path| path.len() > 256 || !path.starts_with('$')) + { + return Err(DeliveryError::Unsupported("Invalid InputPath".into())); + } + if let Some(transformer) = &target.input_transformer { + if transformer.input_template.len() > 8192 + || transformer.input_paths_map.len() > 100 + || transformer + .input_paths_map + .iter() + .any(|(key, path)| key.len() > 256 || path.len() > 256 || !path.starts_with('$')) + { + return Err(DeliveryError::Unsupported( + "InputTransformer exceeds supported bounds".into(), + )); + } + } + Ok(()) } diff --git a/crates/rustack-events-http/Cargo.toml b/crates/rustack-events-http/Cargo.toml index affb139..949c6d5 100644 --- a/crates/rustack-events-http/Cargo.toml +++ b/crates/rustack-events-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-events-http" description = "EventBridge HTTP service layer for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-events-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-events-http/src/service.rs b/crates/rustack-events-http/src/service.rs index 36ab09a..28bb930 100644 --- a/crates/rustack-events-http/src/service.rs +++ b/crates/rustack-events-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_events_model::error::EventsError; @@ -126,19 +125,18 @@ async fn process_request( }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = EventsError::with_message( - rustack_events_model::error::EventsErrorCode::ValidationException, - auth_err.to_string(), - ); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let mut err = EventsError::with_message( + rustack_events_model::error::EventsErrorCode::ValidationException, + auth_err.to_string(), + ); + err.status_code = http::StatusCode::FORBIDDEN; + return error_to_response(&err, request_id); } // 5. Dispatch to handler. @@ -150,11 +148,13 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| EventsError::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = EventsError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every EventBridge response. diff --git a/crates/rustack-events-model/Cargo.toml b/crates/rustack-events-model/Cargo.toml index c3aea31..f7f5466 100644 --- a/crates/rustack-events-model/Cargo.toml +++ b/crates/rustack-events-model/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-events-model" description = "EventBridge model types for Rustack" -version.workspace = true +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-events-model/src/types.rs b/crates/rustack-events-model/src/types.rs index c1beeaf..32b463d 100644 --- a/crates/rustack-events-model/src/types.rs +++ b/crates/rustack-events-model/src/types.rs @@ -63,9 +63,13 @@ pub struct Target { #[serde(skip_serializing_if = "Option::is_none")] pub batch_parameters: Option, + /// Parameters for a Kinesis target (retained for entry-level capability rejection). + #[serde(skip_serializing_if = "Option::is_none")] + pub kinesis_parameters: Option, + /// Parameters for an SQS queue target. #[serde(skip_serializing_if = "Option::is_none")] - pub sqs_parameters: Option, + pub sqs_parameters: Option, /// Parameters for an HTTP endpoint target. #[serde(skip_serializing_if = "Option::is_none")] @@ -92,6 +96,14 @@ pub struct Target { pub app_sync_parameters: Option, } +/// Complete AWS EventBridge parameters for SQS targets. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "PascalCase", deny_unknown_fields)] +pub struct SqsParameters { + /// FIFO message group identifier (1..128 ASCII printable non-space bytes). + pub message_group_id: String, +} + // --------------------------------------------------------------------------- // Input transformer // --------------------------------------------------------------------------- diff --git a/crates/rustack-iam-core/Cargo.toml b/crates/rustack-iam-core/Cargo.toml index 09e6728..7c40f49 100644 --- a/crates/rustack-iam-core/Cargo.toml +++ b/crates/rustack-iam-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-iam-core" description = "IAM business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-iam-core/src/config.rs b/crates/rustack-iam-core/src/config.rs index 26f1fe3..43f16ff 100644 --- a/crates/rustack-iam-core/src/config.rs +++ b/crates/rustack-iam-core/src/config.rs @@ -1,6 +1,6 @@ //! IAM service configuration. -use std::env; +use rustack_core::settings as env; /// IAM service configuration. /// diff --git a/crates/rustack-iam-http/Cargo.toml b/crates/rustack-iam-http/Cargo.toml index 3e40929..d35ca6d 100644 --- a/crates/rustack-iam-http/Cargo.toml +++ b/crates/rustack-iam-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-iam-http" description = "IAM HTTP service layer for Rustack (awsQuery protocol)" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-iam-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-iam-http/src/service.rs b/crates/rustack-iam-http/src/service.rs index 233b5d2..d8052f4 100644 --- a/crates/rustack-iam-http/src/service.rs +++ b/crates/rustack-iam-http/src/service.rs @@ -7,7 +7,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_iam_model::error::IamError; @@ -140,16 +139,14 @@ async fn process_request( }; // 5. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = IamError::invalid_security(auth_err.to_string()); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let err = IamError::invalid_security(auth_err.to_string()); + return error_to_response(&err, request_id); } // 6. Dispatch to handler (pass raw body so handler can re-parse as needed). @@ -161,11 +158,13 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| IamError::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = IamError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every IAM response. diff --git a/crates/rustack-kinesis-core/Cargo.toml b/crates/rustack-kinesis-core/Cargo.toml index ef9aa39..85a5555 100644 --- a/crates/rustack-kinesis-core/Cargo.toml +++ b/crates/rustack-kinesis-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-kinesis-core" description = "Kinesis business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-kinesis-core/src/config.rs b/crates/rustack-kinesis-core/src/config.rs index 7ef82f4..3d1b17a 100644 --- a/crates/rustack-kinesis-core/src/config.rs +++ b/crates/rustack-kinesis-core/src/config.rs @@ -1,6 +1,6 @@ //! Kinesis service configuration. -use std::env; +use rustack_core::settings as env; /// Kinesis service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-kinesis-http/Cargo.toml b/crates/rustack-kinesis-http/Cargo.toml index 784d5bc..36f06d4 100644 --- a/crates/rustack-kinesis-http/Cargo.toml +++ b/crates/rustack-kinesis-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-kinesis-http" description = "Kinesis HTTP service layer for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-kinesis-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-kinesis-http/src/service.rs b/crates/rustack-kinesis-http/src/service.rs index 93666f8..d36eecb 100644 --- a/crates/rustack-kinesis-http/src/service.rs +++ b/crates/rustack-kinesis-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_kinesis_model::error::KinesisError; @@ -126,19 +125,18 @@ async fn process_request( }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = KinesisError::with_message( - rustack_kinesis_model::error::KinesisErrorCode::InternalFailureException, - auth_err.to_string(), - ); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let mut err = KinesisError::with_message( + rustack_kinesis_model::error::KinesisErrorCode::InternalFailureException, + auth_err.to_string(), + ); + err.status_code = http::StatusCode::FORBIDDEN; + return error_to_response(&err, request_id); } // 5. Dispatch to handler. @@ -150,11 +148,13 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| KinesisError::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = KinesisError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every Kinesis response. diff --git a/crates/rustack-kms-core/Cargo.toml b/crates/rustack-kms-core/Cargo.toml index a3adfe0..02f1427 100644 --- a/crates/rustack-kms-core/Cargo.toml +++ b/crates/rustack-kms-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-kms-core" description = "KMS business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-kms-core/src/config.rs b/crates/rustack-kms-core/src/config.rs index 08f5f04..974889c 100644 --- a/crates/rustack-kms-core/src/config.rs +++ b/crates/rustack-kms-core/src/config.rs @@ -1,6 +1,6 @@ //! KMS service configuration. -use std::env; +use rustack_core::settings as env; /// KMS service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-kms-http/Cargo.toml b/crates/rustack-kms-http/Cargo.toml index af4ae56..79e4993 100644 --- a/crates/rustack-kms-http/Cargo.toml +++ b/crates/rustack-kms-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-kms-http" description = "KMS HTTP service layer for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-kms-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-kms-http/src/service.rs b/crates/rustack-kms-http/src/service.rs index af93d78..57fa274 100644 --- a/crates/rustack-kms-http/src/service.rs +++ b/crates/rustack-kms-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_kms_model::error::KmsError; @@ -126,19 +125,18 @@ async fn process_request( }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = KmsError::with_message( - rustack_kms_model::error::KmsErrorCode::KMSInternalException, - auth_err.to_string(), - ); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let mut err = KmsError::with_message( + rustack_kms_model::error::KmsErrorCode::KMSInternalException, + auth_err.to_string(), + ); + err.status_code = http::StatusCode::FORBIDDEN; + return error_to_response(&err, request_id); } // 5. Dispatch to handler. @@ -150,11 +148,13 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| KmsError::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = KmsError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every KMS response. diff --git a/crates/rustack-lambda-core/Cargo.toml b/crates/rustack-lambda-core/Cargo.toml index 3369a23..181dcfc 100644 --- a/crates/rustack-lambda-core/Cargo.toml +++ b/crates/rustack-lambda-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-lambda-core" description = "Lambda business logic for Rustack" -version = "0.9.1" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-lambda-core/src/config.rs b/crates/rustack-lambda-core/src/config.rs index 2bf757f..ed8fbf3 100644 --- a/crates/rustack-lambda-core/src/config.rs +++ b/crates/rustack-lambda-core/src/config.rs @@ -42,55 +42,103 @@ impl LambdaConfig { /// - `GATEWAY_HOST` (default: `localhost`) /// - `GATEWAY_PORT` (default: `4566`) /// - `LAMBDA_DOCKER_ENABLED` (default: `false` — legacy alias for `LAMBDA_EXECUTOR=docker`). - /// - `LAMBDA_EXECUTOR` (default: `auto`; `docker` if `LAMBDA_DOCKER_ENABLED=true`). Accepts - /// `disabled`, `auto`, `native`, `docker`, `squib`. Auto uses Squib for Zip functions on - /// macOS and native execution otherwise. The native backend runs `provided.*` bootstraps + /// - `LAMBDA_EXECUTOR` (default: `disabled`; unsupported Docker settings fail explicitly). + /// Accepts `disabled`, `auto`, `native`, `docker`, `squib`. Auto uses Squib for Zip functions + /// on macOS and native execution otherwise. The native backend runs `provided.*` bootstraps /// (Rust / Go / C++) directly on the host with no Docker requirement. /// - `LAMBDA_MAX_WARM_INSTANCES` (default: `1`) /// - `LAMBDA_IDLE_TIMEOUT_SECS` (default: `600`) /// - `LAMBDA_INIT_TIMEOUT_SECS` (default: `5`) /// - `LAMBDA_SQUIB_*` variables documented by [`SquibExecutorConfig`]. - #[must_use] - pub fn from_env() -> Self { - let docker_enabled = env_bool("LAMBDA_DOCKER_ENABLED", false); - let executor = env::var("LAMBDA_EXECUTOR") - .ok() - .and_then(|raw| raw.parse::().ok()) - .unwrap_or(if docker_enabled { - ExecutorBackend::Docker - } else { - ExecutorBackend::Auto - }); - Self { - skip_signature_validation: env_bool("LAMBDA_SKIP_SIGNATURE_VALIDATION", true), - default_region: env::var("DEFAULT_REGION").unwrap_or_else(|_| "us-east-1".to_owned()), - account_id: env::var("DEFAULT_ACCOUNT_ID") - .unwrap_or_else(|_| "000000000000".to_owned()), - host: env::var("GATEWAY_HOST").unwrap_or_else(|_| "localhost".to_owned()), - port: env::var("GATEWAY_PORT") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(4566), - docker_enabled, + pub fn from_env() -> Result { + Self::from_lookup(|key| match rustack_core::settings::var(key) { + Ok(value) => Some(value), + Err(env::VarError::NotPresent) => None, + Err(_) => Some(String::new()), + }) + } + + /// Parse an explicit environment lookup without changing process globals. + pub fn from_lookup( + lookup: impl Fn(&str) -> Option, + ) -> Result { + let error = |key: &str| crate::error::LambdaServiceError::InvalidParameter { + message: format!("Invalid Lambda configuration: {key}"), + }; + let boolean = + |key: &str, default: bool| -> Result { + match lookup(key).as_deref() { + None => Ok(default), + Some("true" | "1" | "yes") => Ok(true), + Some("false" | "0" | "no") => Ok(false), + Some(_) => Err(error(key)), + } + }; + let number = + |key: &str, default: u64, max: u64| -> Result { + let value = match lookup(key) { + None => default, + Some(raw) => raw.parse().map_err(|_| error(key))?, + }; + if value == 0 || value > max { + return Err(error(key)); + } + Ok(value) + }; + let docker_enabled = boolean("LAMBDA_DOCKER_ENABLED", false)?; + let executor = match lookup("LAMBDA_EXECUTOR") { + Some(raw) => raw.parse().map_err(|_| error("LAMBDA_EXECUTOR"))?, + None if docker_enabled => ExecutorBackend::Docker, + None => ExecutorBackend::Disabled, + }; + if docker_enabled && executor != ExecutorBackend::Docker { + return Err(error("conflicting Docker/executor settings")); + } + if executor == ExecutorBackend::Docker { + return Err(error("Docker execution is not supported by this build")); + } + let mut config = Self { executor, - max_warm_instances: env::var("LAMBDA_MAX_WARM_INSTANCES") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(1), - idle_timeout: Duration::from_secs( - env::var("LAMBDA_IDLE_TIMEOUT_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(600), - ), - init_timeout: Duration::from_secs( - env::var("LAMBDA_INIT_TIMEOUT_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(5), - ), - squib: SquibExecutorConfig::from_env(), + docker_enabled, + skip_signature_validation: boolean("LAMBDA_SKIP_SIGNATURE_VALIDATION", true)?, + port: u16::try_from(number("GATEWAY_PORT", 4566, 65_535)?) + .map_err(|_| error("GATEWAY_PORT"))?, + max_warm_instances: usize::try_from(number("LAMBDA_MAX_WARM_INSTANCES", 1, 1)?) + .map_err(|_| error("LAMBDA_MAX_WARM_INSTANCES"))?, + idle_timeout: Duration::from_secs(number("LAMBDA_IDLE_TIMEOUT_SECS", 600, 86_400)?), + init_timeout: Duration::from_secs(number("LAMBDA_INIT_TIMEOUT_SECS", 5, 900)?), + ..Self::default() + }; + if let Some(value) = lookup("DEFAULT_REGION") { + if value.is_empty() + || value.len() > 64 + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-') + { + return Err(error("DEFAULT_REGION")); + } + config.default_region = value; + } + if let Some(value) = lookup("DEFAULT_ACCOUNT_ID") { + if value.len() != 12 || !value.bytes().all(|b| b.is_ascii_digit()) { + return Err(error("DEFAULT_ACCOUNT_ID")); + } + config.account_id = value; } + if let Some(value) = lookup("GATEWAY_HOST") { + if value.is_empty() + || value.len() > 253 + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b".-:[]".contains(&b)) + { + return Err(error("GATEWAY_HOST")); + } + config.host = value; + } + config.squib = SquibExecutorConfig::from_env_reader(lookup)?; + Ok(config) } } @@ -112,16 +160,42 @@ impl Default for LambdaConfig { } } -fn env_bool(key: &str, default: bool) -> bool { - env::var(key).map_or(default, |v| { - v.eq_ignore_ascii_case("true") || v.eq_ignore_ascii_case("yes") || v == "1" - }) -} - #[cfg(test)] mod tests { use super::*; + #[test] + fn test_should_reject_invalid_configuration_without_fallback() { + for (key, value) in [ + ("LAMBDA_EXECUTOR", "disable"), + ("LAMBDA_SKIP_SIGNATURE_VALIDATION", "maybe"), + ("GATEWAY_PORT", "0"), + ("GATEWAY_PORT", "65536"), + ("LAMBDA_INIT_TIMEOUT_SECS", "-1"), + ("LAMBDA_MAX_WARM_INSTANCES", "999"), + ("LAMBDA_SQUIB_STAGE_PORT", "oops"), + ("LAMBDA_SQUIB_CONFIG_FILE", ""), + ("DEFAULT_ACCOUNT_ID", "../account"), + ] { + assert!( + LambdaConfig::from_lookup(|name| (name == key).then(|| value.into())).is_err(), + "{key}" + ); + } + assert_eq!( + LambdaConfig::from_lookup(|_| None).unwrap().executor, + ExecutorBackend::Disabled + ); + assert!( + LambdaConfig::from_lookup(|key| match key { + "LAMBDA_EXECUTOR" => Some("native".into()), + "LAMBDA_DOCKER_ENABLED" => Some("true".into()), + _ => None, + }) + .is_err() + ); + } + #[test] fn test_should_create_default_config() { let config = LambdaConfig::default(); diff --git a/crates/rustack-lambda-core/src/error.rs b/crates/rustack-lambda-core/src/error.rs index 6badab5..3a5637b 100644 --- a/crates/rustack-lambda-core/src/error.rs +++ b/crates/rustack-lambda-core/src/error.rs @@ -8,6 +8,16 @@ use rustack_lambda_model::error::{LambdaError, LambdaErrorCode}; /// Internal service error type for Lambda operations. #[derive(Debug, thiserror::Error)] pub enum LambdaServiceError { + /// Invocation or accepted Event capacity is exhausted. + #[error("Lambda concurrency or accepted-work capacity exhausted")] + TooManyRequests, + /// Artifact filesystem operation failed. + #[error("Lambda artifact IO failed: {source}")] + ArtifactIo { + /// Underlying filesystem failure. + #[source] + source: std::io::Error, + }, /// Function does not exist. #[error("Function not found: {name}")] FunctionNotFound { @@ -113,6 +123,13 @@ pub enum LambdaServiceError { impl From for LambdaError { fn from(err: LambdaServiceError) -> Self { match err { + LambdaServiceError::TooManyRequests => LambdaError::new( + LambdaErrorCode::TooManyRequestsException, + "Lambda concurrency or accepted-work capacity exhausted", + ), + LambdaServiceError::ArtifactIo { .. } => { + LambdaError::service_error("Lambda artifact IO failed") + } LambdaServiceError::FunctionNotFound { ref name } => { LambdaError::resource_not_found(format!( "Function not found: arn:aws:lambda:us-east-1:000000000000:function:{name}" diff --git a/crates/rustack-lambda-core/src/executor/error.rs b/crates/rustack-lambda-core/src/executor/error.rs index 98b5ab1..78719d7 100644 --- a/crates/rustack-lambda-core/src/executor/error.rs +++ b/crates/rustack-lambda-core/src/executor/error.rs @@ -39,7 +39,11 @@ pub enum ExecutorError { impl From for LambdaServiceError { fn from(e: ExecutorError) -> Self { match e { - ExecutorError::Disabled => Self::DockerNotAvailable, + ExecutorError::Disabled => Self::ResourceNotReady { + message: "Lambda execution is disabled; explicitly select a trusted execution \ + backend" + .into(), + }, ExecutorError::Unsupported(m) | ExecutorError::InvalidCode(m) | ExecutorError::InitFailed(m) diff --git a/crates/rustack-lambda-core/src/executor/instance.rs b/crates/rustack-lambda-core/src/executor/instance.rs index 62c8047..573e287 100644 --- a/crates/rustack-lambda-core/src/executor/instance.rs +++ b/crates/rustack-lambda-core/src/executor/instance.rs @@ -7,15 +7,14 @@ //! versions. use std::{ - collections::HashMap, sync::Arc, time::{Duration, Instant}, }; use async_trait::async_trait; -use parking_lot::Mutex; -use tokio::sync::oneshot; -use tracing::{debug, warn}; +use dashmap::DashMap; +use tokio::sync::{OwnedSemaphorePermit, Semaphore, oneshot}; +use tracing::debug; use super::{ error::ExecutorError, @@ -31,6 +30,11 @@ pub(crate) type PoolKey = (String, String); /// Object-safe so the pool can hold `Arc`. #[async_trait] pub(crate) trait InstanceBackend: Send + Sync + std::fmt::Debug { + /// Whether another OS execution resource can start without retiring an idle instance. + fn has_capacity(&self) -> bool { + true + } + /// Spawn a bootstrap pointing at `runtime_api_addr` for the given function. /// Returns a handle the pool will keep alive until the instance is reaped. async fn spawn( @@ -77,6 +81,8 @@ struct Instance { api: RuntimeApiHandle, backend: BackendHandle, last_used: Instant, + idle_permit: Option, + init_error: Option>, } /// Pool of warm instances per `(function, qualifier)` key. @@ -90,7 +96,8 @@ pub(crate) struct InstancePool { max_warm: usize, idle_timeout: Duration, init_timeout: Duration, - pools: Mutex>>, + pools: DashMap>, + idle_capacity: Arc, } impl InstancePool { @@ -105,12 +112,32 @@ impl InstancePool { max_warm, idle_timeout, init_timeout, - pools: Mutex::new(HashMap::new()), + pools: DashMap::new(), + idle_capacity: Arc::new(Semaphore::new(32)), } } pub(crate) fn key(req: &InvokeRequest) -> PoolKey { - (req.function_name.clone(), req.qualifier.clone()) + let mut environment: Vec<_> = req.environment.iter().collect(); + environment.sort(); + let identity = format!( + "{:?}", + ( + &req.qualifier, + &req.code_root, + &req.image_uri, + &req.runtime, + &req.handler, + &req.architectures, + environment, + req.timeout, + req.memory_mb + ) + ); + ( + req.function_name.clone(), + crate::storage::compute_sha256(identity.as_bytes()), + ) } /// Run a single invocation against an acquired (or freshly spawned) instance. @@ -137,7 +164,16 @@ impl InstancePool { .await .map_err(|e| ExecutorError::Io(e.to_string()))?; - let result = match tokio::time::timeout(req.timeout, resp_rx).await { + let response = async { + match instance.init_error.take() { + Some(mut init) => tokio::select! { + response = resp_rx => response, + error = &mut init => error.map(RuntimeResult::InitError), + }, + None => resp_rx.await, + } + }; + let result = match tokio::time::timeout(req.timeout, response).await { Ok(Ok(r)) => r, Ok(Err(_)) => { // Bootstrap died before responding. @@ -152,8 +188,10 @@ impl InstancePool { } }; - instance.last_used = Instant::now(); - self.release(key, instance); + if !matches!(result, RuntimeResult::InitError(_)) { + instance.last_used = Instant::now(); + self.release(key, instance); + } match result { RuntimeResult::Success(payload) => Ok(InvokeResponse { @@ -178,52 +216,57 @@ impl InstancePool { } fn try_acquire(&self, key: &PoolKey) -> Option { - let mut pools = self.pools.lock(); - let bucket = pools.get_mut(key)?; - bucket.pop() + let mut bucket = self.pools.get_mut(&key.0)?; + let Some(index) = bucket.iter().position(|(revision, _)| revision == &key.1) else { + bucket.clear(); + return None; + }; + let (_, mut instance) = bucket.swap_remove(index); + instance.idle_permit.take(); + Some(instance) } - fn release(&self, key: PoolKey, instance: Instance) { - let mut pools = self.pools.lock(); - let bucket = pools.entry(key).or_default(); - if bucket.len() >= self.max_warm { - // Pool full — drop (kill) on background task, don't block. - tokio::spawn(async move { - drop(instance); - }); - } else { - bucket.push(instance); + fn release(&self, key: PoolKey, mut instance: Instance) { + let Ok(permit) = Arc::clone(&self.idle_capacity).try_acquire_owned() else { + return; + }; + let mut bucket = self.pools.entry(key.0).or_default(); + if bucket.len() < self.max_warm.min(1) { + instance.idle_permit = Some(permit); + bucket.push((key.1, instance)); } } async fn spawn_new(&self, req: &InvokeRequest) -> Result { + if !self.backend.has_capacity() { + for mut bucket in self.pools.iter_mut() { + if let Some((_, instance)) = bucket.pop() { + drop(instance); + break; + } + } + } let api = runtime_api::start() .await .map_err(|e| ExecutorError::Io(format!("bind runtime api: {e}")))?; let addr = api.addr(); - let mut init_err_rx = api.take_init_error_rx().await; + let init_error = api.take_init_error_rx().await; // Race: backend spawn + first /next poll. We don't observe /next here // directly — we rely on either submit landing on a polling bootstrap // OR an `/init/error` arriving. To keep liveness, spawn the backend // within the init window and watch the init-error channel for a // fast-fail signal. - let backend = self.backend.spawn(req, addr).await?; + let backend = tokio::time::timeout(self.init_timeout, self.backend.spawn(req, addr)) + .await + .map_err(|_| ExecutorError::Timeout(self.init_timeout))??; let inst = Instance { api, backend, last_used: Instant::now(), + idle_permit: None, + init_error, }; - // If the bootstrap failed init, surface that promptly rather than - // waiting for the invocation timeout. - if let Some(rx) = init_err_rx.take() { - let init_timeout = self.init_timeout; - tokio::spawn(async move { - if let Ok(Ok(body)) = tokio::time::timeout(init_timeout, rx).await { - warn!(error = %String::from_utf8_lossy(&body), "lambda bootstrap reported init error"); - } - }); - } debug!(function = %req.function_name, addr = %addr, "spawned new lambda instance"); Ok(inst) } @@ -233,30 +276,18 @@ impl InstancePool { let now = Instant::now(); let idle = self.idle_timeout; let mut killed = 0usize; - let mut pools = self.pools.lock(); - for bucket in pools.values_mut() { - let mut keep = Vec::with_capacity(bucket.len()); - while let Some(inst) = bucket.pop() { - if now.duration_since(inst.last_used) > idle { - killed += 1; - drop(inst); - } else { - keep.push(inst); - } - } - *bucket = keep; + for mut bucket in self.pools.iter_mut() { + let before = bucket.len(); + bucket.retain(|(_, instance)| now.duration_since(instance.last_used) <= idle); + killed += before.saturating_sub(bucket.len()); } + self.pools.retain(|_, bucket| !bucket.is_empty()); killed } /// Drain and kill every instance in every pool. pub(crate) fn shutdown(&self) { - let pools = std::mem::take(&mut *self.pools.lock()); - for (_, bucket) in pools { - for inst in bucket { - drop(inst); - } - } + self.pools.clear(); } } diff --git a/crates/rustack-lambda-core/src/executor/mod.rs b/crates/rustack-lambda-core/src/executor/mod.rs index 43fff44..aec3e86 100644 --- a/crates/rustack-lambda-core/src/executor/mod.rs +++ b/crates/rustack-lambda-core/src/executor/mod.rs @@ -1,14 +1,13 @@ //! Lambda function execution engine. //! -//! Converts the stubbed `Invoke` echo path into real execution by routing -//! every request through an `Executor` trait. Backends include: +//! Routes execution through an object-safe backend with explicit availability. //! -//! - [`NoopExecutor`] — preserves the legacy "echo back the payload" behavior; used in unit tests -//! and when `LAMBDA_EXECUTOR=disabled`. -//! - [`AutoExecutor`] — defaults macOS Zip Lambdas to Squib and otherwise uses native execution. -//! - `NativeExecutor` (Phase 3) — spawns `provided.*` bootstraps directly on the host. -//! - `DockerExecutor` (Phase 4) — runs any supported runtime in an AWS Lambda base image. -//! - `SquibExecutor` — runs `arm64` Zip functions through a Squib microVM guest agent. +//! - [`NoopExecutor`] rejects execution in Disabled/unsupported Docker modes. +//! - [`AutoExecutor`] explicitly opts into Squib on macOS Zip functions, native otherwise. +//! - [`NativeExecutor`] runs trusted host-matching bootstraps without isolation. +//! - [`SquibExecutor`] runs arm64 Zip functions through a microVM guest agent. +//! +//! Docker execution is not supported; it never falls back to native or successful echo. //! //! All backends share a single in-process Lambda Runtime API server (Phase 2) //! so the bootstrap-side protocol is identical to AWS. @@ -37,6 +36,11 @@ pub use types::{ExecutorBackend, InvokeRequest, InvokeResponse, PackageType}; /// Backend that turns an [`InvokeRequest`] into an [`InvokeResponse`]. #[async_trait] pub trait Executor: std::fmt::Debug + Send + Sync + 'static { + /// Check whether execution is enabled before accepting asynchronous work. + fn available(&self) -> Result<(), ExecutorError> { + Ok(()) + } + /// Run the function and return its response. async fn invoke(&self, req: InvokeRequest) -> Result; diff --git a/crates/rustack-lambda-core/src/executor/native.rs b/crates/rustack-lambda-core/src/executor/native.rs index 68500f9..3831011 100644 --- a/crates/rustack-lambda-core/src/executor/native.rs +++ b/crates/rustack-lambda-core/src/executor/native.rs @@ -5,11 +5,16 @@ //! match the host. The auto executor keeps macOS Zip functions on Squib and //! leaves unsupported image packages for the future Docker backend. -use std::{io::Read as _, net::SocketAddr, path::Path, process::Stdio, sync::Arc, time::Duration}; +use std::{net::SocketAddr, path::Path, process::Stdio, sync::Arc, time::Duration}; use async_trait::async_trait; use parking_lot::Mutex as PMutex; -use tokio::{io::AsyncReadExt, process::Command, sync::watch}; +use tokio::{ + io::AsyncReadExt, + process::Command, + sync::{Semaphore, oneshot, watch}, + task::JoinSet, +}; use tracing::{debug, warn}; use super::{ @@ -23,13 +28,28 @@ pub struct NativeExecutor { pool: Arc, cancel_tx: watch::Sender, reaper: PMutex>>, + processes: Arc>>, +} + +impl Drop for NativeExecutor { + fn drop(&mut self) { + self.pool.shutdown(); + if let Some(reaper) = self.reaper.lock().take() { + reaper.abort(); + } + self.processes.lock().abort_all(); + } } impl NativeExecutor { /// Build a new native executor. #[must_use] pub fn new(max_warm: usize, idle_timeout: Duration, init_timeout: Duration) -> Self { - let backend = Arc::new(NativeBackend); + let processes = Arc::new(PMutex::new(JoinSet::new())); + let backend = Arc::new(NativeBackend { + processes: Arc::clone(&processes), + capacity: Arc::new(Semaphore::new(32)), + }); let pool = Arc::new(InstancePool::new( backend, max_warm, @@ -42,6 +62,7 @@ impl NativeExecutor { pool, cancel_tx, reaper: PMutex::new(Some(reaper)), + processes, } } } @@ -59,13 +80,8 @@ impl Executor for NativeExecutor { .as_ref() .ok_or_else(|| ExecutorError::InvalidCode("missing code root".to_owned()))?; let bootstrap = code_root.join("bootstrap"); - if !bootstrap.exists() { - return Err(ExecutorError::InvalidCode(format!( - "no bootstrap at {}", - bootstrap.display() - ))); - } - if !bootstrap_runs_on_host(&bootstrap, &req.architectures) { + ensure_executable(&bootstrap).await?; + if !bootstrap_runs_on_host(&bootstrap, &req.architectures).await? { return Err(ExecutorError::Unsupported(format!( "bootstrap {} cannot run on host {}/{}; use docker backend", bootstrap.display(), @@ -73,7 +89,6 @@ impl Executor for NativeExecutor { std::env::consts::ARCH, ))); } - ensure_executable(&bootstrap)?; self.pool.invoke(req).await } @@ -81,17 +96,31 @@ impl Executor for NativeExecutor { let _ = self.cancel_tx.send(true); let reaper = self.reaper.lock().take(); if let Some(r) = reaper { - let _ = r.await; + if let Err(error) = r.await { + warn!(%error, "Lambda idle reaper failed"); + } } self.pool.shutdown(); + let mut processes = std::mem::take(&mut *self.processes.lock()); + while let Some(result) = processes.join_next().await { + if let Err(error) = result { + warn!(%error, "Lambda process supervisor failed"); + } + } } } #[derive(Debug)] -struct NativeBackend; +struct NativeBackend { + processes: Arc>>, + capacity: Arc, +} #[async_trait] impl InstanceBackend for NativeBackend { + fn has_capacity(&self) -> bool { + self.capacity.available_permits() != 0 + } async fn spawn( &self, req: &InvokeRequest, @@ -120,35 +149,45 @@ impl InstanceBackend for NativeBackend { // Drain stdout/stderr into a small ring so we don't fill pipe buffers. let log_buf = Arc::new(PMutex::new(LogTail::with_capacity(4 * 1024))); + let mut logs = JoinSet::new(); if let Some(stdout) = child.stdout.take() { - tokio::spawn(drain_to_buf(stdout, Arc::clone(&log_buf), "stdout")); + logs.spawn(drain_to_buf(stdout, Arc::clone(&log_buf), "stdout")); } if let Some(stderr) = child.stderr.take() { - tokio::spawn(drain_to_buf(stderr, Arc::clone(&log_buf), "stderr")); + logs.spawn(drain_to_buf(stderr, log_buf, "stderr")); } - + let (cancel, cancelled) = oneshot::channel(); + let mut processes = self.processes.lock(); + while let Some(result) = processes.try_join_next() { + if let Err(error) = result { + warn!(%error, "Lambda process supervisor failed"); + } + } + processes.spawn(async move { + tokio::select! { + _ = cancelled => { if let Err(error) = child.kill().await { warn!(%error, "Lambda process cancellation failed"); } }, + result = child.wait() => { if let Err(error) = result { warn!(%error, "Lambda process wait failed"); } }, + } + logs.abort_all(); + while let Some(result) = logs.join_next().await { if let Err(error) = result { if !error.is_cancelled() { warn!(%error, "Lambda log task failed"); } } } + }); Ok(BackendHandle::new(NativeHandle { - child: Some(child), - log: log_buf, + cancel: Some(cancel), })) } } #[derive(Debug)] struct NativeHandle { - child: Option, - #[allow(dead_code)] - log: Arc>, + cancel: Option>, } impl BackendHandleObj for NativeHandle { fn kill(&mut self) { - if let Some(mut child) = self.child.take() { - // start_kill is sync and non-blocking; reaper task does final wait. - let _: Result<(), std::io::Error> = child.start_kill(); - tokio::spawn(async move { - let _ = child.wait().await; - }); + if let Some(cancel) = self.cancel.take() { + if cancel.send(()).is_err() { + debug!("Lambda process already exited"); + } } } } @@ -159,13 +198,11 @@ impl Drop for NativeHandle { } } -/// Best-effort bootstrap-arch check. -/// -/// Synchronous std::fs is intentional — the file is tiny (4 bytes read) and -/// we'd otherwise need to await inside a hot path. The disallowed-types lint -/// is allowed locally for the same reason. -#[allow(clippy::disallowed_types)] -fn bootstrap_runs_on_host(path: &Path, declared_archs: &[String]) -> bool { +/// Check declared architecture and executable format with fallible asynchronous IO. +async fn bootstrap_runs_on_host( + path: &Path, + declared_archs: &[String], +) -> Result { // Architecture check first — declared `architectures` must include the // host arch. let host_arch = match std::env::consts::ARCH { @@ -174,16 +211,15 @@ fn bootstrap_runs_on_host(path: &Path, declared_archs: &[String]) -> bool { other => other, }; if !declared_archs.iter().any(|a| a == host_arch) { - return false; + return Ok(false); } - // Magic-byte check matches the OS. - let Ok(mut f) = std::fs::File::open(path) else { - return false; - }; + let mut file = tokio::fs::File::open(path) + .await + .map_err(|error| ExecutorError::Io(format!("Open bootstrap: {error}")))?; let mut hdr = [0u8; 4]; - if f.read_exact(&mut hdr).is_err() { - return false; - } + file.read_exact(&mut hdr) + .await + .map_err(|error| ExecutorError::InvalidCode(format!("Read bootstrap header: {error}")))?; let elf = hdr == [0x7f, b'E', b'L', b'F']; let macho = hdr == [0xCF, 0xFA, 0xED, 0xFE] || hdr == [0xFE, 0xED, 0xFA, 0xCE] @@ -191,36 +227,26 @@ fn bootstrap_runs_on_host(path: &Path, declared_archs: &[String]) -> bool { || hdr == [0xCA, 0xFE, 0xBA, 0xBE]; let host_is_macos = std::env::consts::OS == "macos"; let host_is_linux = std::env::consts::OS == "linux"; - if elf && host_is_linux { - return true; - } - if macho && host_is_macos { - return true; - } - false + Ok((elf && host_is_linux) || (macho && host_is_macos)) } -/// Mark the file +x if it isn't already. No-op on non-unix. -/// -/// std::fs is intentional: a single sync stat + chmod is cheaper than the -/// async runtime overhead and runs once per cold start. +/// Require a regular executable bootstrap without modifying immutable artifacts. #[cfg(unix)] -#[allow(clippy::disallowed_methods)] -fn ensure_executable(path: &Path) -> Result<(), ExecutorError> { +async fn ensure_executable(path: &Path) -> Result<(), ExecutorError> { use std::os::unix::fs::PermissionsExt as _; - let meta = std::fs::metadata(path) - .map_err(|e| ExecutorError::Io(format!("stat {}: {e}", path.display())))?; - let mut perms = meta.permissions(); - if perms.mode() & 0o111 == 0 { - perms.set_mode(perms.mode() | 0o755); - std::fs::set_permissions(path, perms) - .map_err(|e| ExecutorError::Io(format!("chmod {}: {e}", path.display())))?; + let meta = tokio::fs::symlink_metadata(path) + .await + .map_err(|error| ExecutorError::Io(format!("Inspect bootstrap: {error}")))?; + if meta.file_type().is_symlink() || !meta.is_file() || meta.permissions().mode() & 0o111 == 0 { + return Err(ExecutorError::InvalidCode( + "bootstrap must be a regular executable file".into(), + )); } Ok(()) } #[cfg(not(unix))] -fn ensure_executable(_path: &Path) -> Result<(), ExecutorError> { +async fn ensure_executable(_path: &Path) -> Result<(), ExecutorError> { Ok(()) } diff --git a/crates/rustack-lambda-core/src/executor/noop.rs b/crates/rustack-lambda-core/src/executor/noop.rs index ac4b87e..7d052c7 100644 --- a/crates/rustack-lambda-core/src/executor/noop.rs +++ b/crates/rustack-lambda-core/src/executor/noop.rs @@ -1,80 +1,39 @@ -//! No-op executor that echoes the request payload. -//! -//! Preserves the legacy stub behavior so downstream tests and the -//! `LAMBDA_EXECUTOR=disabled` mode continue to work without any process or -//! container being started. - +//! Disabled execution backend. Never executes or fabricates a successful result. use async_trait::async_trait; -use bytes::Bytes; -use serde_json::json; use super::{Executor, ExecutorError, InvokeRequest, InvokeResponse}; -/// Echoes the request payload back wrapped in a fake API Gateway response -/// shape — identical to the old hard-coded stub. +/// Explicit unavailable backend for metadata-only operation. #[derive(Debug, Default, Clone)] -pub struct NoopExecutor; - +pub struct NoopExecutor { + unsupported_docker: bool, +} impl NoopExecutor { - /// Construct a new no-op executor. + /// Construct a disabled executor. #[must_use] pub fn new() -> Self { - Self + Self::default() + } + pub(crate) fn docker_unavailable() -> Self { + Self { + unsupported_docker: true, + } + } + fn error(&self) -> ExecutorError { + if self.unsupported_docker { + ExecutorError::Unsupported("Docker execution is not implemented in this build".into()) + } else { + ExecutorError::Disabled + } } } - #[async_trait] impl Executor for NoopExecutor { - async fn invoke(&self, req: InvokeRequest) -> Result { - // Mirror the old echo body shape so callers depending on it (e.g. - // existing integration tests) don't notice the indirection. - let body = json!({ - "statusCode": 200, - "body": String::from_utf8_lossy(&req.payload), - }); - let bytes = serde_json::to_vec(&body).map_err(|e| ExecutorError::Io(e.to_string()))?; - Ok(InvokeResponse::success(Bytes::from(bytes), req.qualifier)) - } - - async fn shutdown(&self) {} -} - -#[cfg(test)] -mod tests { - use std::{collections::HashMap, time::Duration}; - - use super::*; - use crate::executor::PackageType; - - fn req(payload: &str) -> InvokeRequest { - InvokeRequest { - function_arn: "arn".into(), - function_name: "fn".into(), - qualifier: "$LATEST".into(), - runtime: None, - handler: None, - architectures: vec!["x86_64".into()], - package_type: PackageType::Zip, - code_root: None, - code_zip: None, - image_uri: None, - environment: HashMap::new(), - timeout: Duration::from_secs(3), - memory_mb: 128, - payload: Bytes::from(payload.to_owned()), - capture_logs: false, - } + fn available(&self) -> Result<(), ExecutorError> { + Err(self.error()) } - - #[tokio::test] - async fn test_should_echo_payload_in_legacy_shape() { - let exec = NoopExecutor::new(); - let resp = exec.invoke(req("hello")).await.unwrap(); - assert_eq!(resp.status, 200); - assert!(resp.function_error.is_none()); - let body: serde_json::Value = serde_json::from_slice(&resp.payload).unwrap(); - assert_eq!(body["statusCode"], 200); - assert_eq!(body["body"], "hello"); - assert_eq!(resp.executed_version, "$LATEST"); + async fn invoke(&self, _req: InvokeRequest) -> Result { + Err(self.error()) } + async fn shutdown(&self) {} } diff --git a/crates/rustack-lambda-core/src/executor/runtime_api.rs b/crates/rustack-lambda-core/src/executor/runtime_api.rs index 0e817e5..c32e7b9 100644 --- a/crates/rustack-lambda-core/src/executor/runtime_api.rs +++ b/crates/rustack-lambda-core/src/executor/runtime_api.rs @@ -22,7 +22,7 @@ use std::{ use bytes::Bytes; use dashmap::DashMap; -use http_body_util::{BodyExt, Full}; +use http_body_util::{BodyExt, Full, Limited}; use hyper::{ Method, Request, Response, StatusCode, body::Incoming, server::conn::http1, service::service_fn, }; @@ -30,7 +30,7 @@ use hyper_util::rt::TokioIo; use tokio::{ net::TcpListener, sync::{Mutex, mpsc, oneshot, watch}, - task::JoinHandle, + task::{JoinHandle, JoinSet}, }; use tracing::{debug, warn}; @@ -140,8 +140,10 @@ pub async fn start() -> std::io::Result { }); let accept_task = tokio::spawn(async move { + let mut connections = JoinSet::new(); loop { tokio::select! { + result = connections.join_next(), if !connections.is_empty() => { if let Some(Err(error)) = result { warn!(%error, "Runtime API connection task failed"); } } _ = shutdown_rx.changed() => { debug!("runtime api: shutdown signalled, exiting accept loop"); break; @@ -154,8 +156,9 @@ pub async fn start() -> std::io::Result { continue; } }; + if connections.len() >= 4 { drop(stream); continue; } let state = Arc::clone(&state); - tokio::spawn(async move { + connections.spawn(async move { let io = TokioIo::new(stream); let svc = service_fn(move |req| { let state = Arc::clone(&state); @@ -172,6 +175,12 @@ pub async fn start() -> std::io::Result { } } } + connections.abort_all(); + while let Some(result) = connections.join_next().await { + if let Err(error) = result { + debug!(%error, "Runtime API connection stopped"); + } + } }); Ok(RuntimeApiHandle { @@ -321,10 +330,20 @@ fn unix_secs_lower32() -> u32 { } async fn collect(body: Incoming) -> Result { - body.collect() - .await - .map(http_body_util::Collected::to_bytes) - .map_err(|_| StatusCode::BAD_REQUEST) + tokio::time::timeout( + std::time::Duration::from_secs(5), + Limited::new(body, 6 * 1024 * 1024).collect(), + ) + .await + .map_err(|_| StatusCode::REQUEST_TIMEOUT)? + .map(http_body_util::Collected::to_bytes) + .map_err(|error| { + if error.is::() { + StatusCode::PAYLOAD_TOO_LARGE + } else { + StatusCode::BAD_REQUEST + } + }) } fn simple(status: StatusCode, body: &[u8]) -> Response> { diff --git a/crates/rustack-lambda-core/src/executor/squib.rs b/crates/rustack-lambda-core/src/executor/squib.rs index ad2e9b8..fd65648 100644 --- a/crates/rustack-lambda-core/src/executor/squib.rs +++ b/crates/rustack-lambda-core/src/executor/squib.rs @@ -60,44 +60,73 @@ pub struct SquibExecutorConfig { impl SquibExecutorConfig { /// Read Squib executor configuration from environment variables. - #[must_use] - pub fn from_env() -> Self { - Self::from_env_reader(|key| env::var(key).ok()) + pub fn from_env() -> Result { + Self::from_env_reader(|key| match rustack_core::settings::var(key) { + Ok(value) => Some(value), + Err(env::VarError::NotPresent) => None, + Err(_) => Some(String::new()), + }) } - pub(crate) fn from_env_reader(mut read: impl FnMut(&str) -> Option) -> Self { - let instance_id = read("LAMBDA_SQUIB_INSTANCE_ID") - .as_deref() - .and_then(non_empty_string) - .unwrap_or_else(|| DEFAULT_INSTANCE_ID.to_owned()); - Self { - config_file: read("LAMBDA_SQUIB_CONFIG_FILE") - .as_deref() - .and_then(non_empty_string) - .map(PathBuf::from) - .or_else(|| Some(default_config_file())), - vsock_path: read("LAMBDA_SQUIB_VSOCK_PATH") - .as_deref() - .and_then(non_empty_string) - .map(PathBuf::from) - .or_else(|| Some(default_vsock_path())), - instance_id, - stage_port: read("LAMBDA_SQUIB_STAGE_PORT") - .and_then(|v| v.parse().ok()) - .unwrap_or(DEFAULT_STAGE_PORT), - connect_timeout: read("LAMBDA_SQUIB_CONNECT_TIMEOUT_MS") - .and_then(|v| v.parse::().ok()) - .map_or(DEFAULT_CONNECT_TIMEOUT, Duration::from_millis), - response_limit_bytes: read("LAMBDA_SQUIB_RESPONSE_LIMIT_BYTES") - .and_then(|v| v.parse().ok()) - .unwrap_or(DEFAULT_RESPONSE_LIMIT_BYTES), - run_budget: read("LAMBDA_SQUIB_RUN_BUDGET_SECS") - .and_then(|v| v.parse::().ok()) - .map_or(DEFAULT_RUN_BUDGET, Duration::from_secs), - shutdown_timeout: read("LAMBDA_SQUIB_SHUTDOWN_TIMEOUT_MS") - .and_then(|v| v.parse::().ok()) - .map_or(DEFAULT_SHUTDOWN_TIMEOUT, Duration::from_millis), + pub(crate) fn from_env_reader( + mut read: impl FnMut(&str) -> Option, + ) -> Result { + let mut config = Self::default(); + for (key, target) in [ + ("LAMBDA_SQUIB_CONFIG_FILE", &mut config.config_file), + ("LAMBDA_SQUIB_VSOCK_PATH", &mut config.vsock_path), + ] { + if let Some(value) = read(key) { + if value.is_empty() || value.len() > 4096 || value.contains('\0') { + return Err(ExecutorError::Unsupported(format!("Invalid {key}"))); + } + *target = Some(PathBuf::from(value)); + } + } + if let Some(value) = read("LAMBDA_SQUIB_INSTANCE_ID") { + if value.is_empty() + || value.len() > 64 + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') + { + return Err(ExecutorError::Unsupported( + "Invalid LAMBDA_SQUIB_INSTANCE_ID".into(), + )); + } + config.instance_id = value; } + let mut number = |key: &str, default: u64, max: u64| -> Result { + let value = match read(key) { + Some(raw) => raw + .parse() + .map_err(|_| ExecutorError::Unsupported(format!("Invalid {key}")))?, + None => default, + }; + if value == 0 || value > max { + return Err(ExecutorError::Unsupported(format!("Invalid {key}"))); + } + Ok(value) + }; + config.stage_port = u32::try_from(number( + "LAMBDA_SQUIB_STAGE_PORT", + 5003, + u64::from(u32::MAX), + )?) + .map_err(|error| ExecutorError::Unsupported(error.to_string()))?; + config.connect_timeout = + Duration::from_millis(number("LAMBDA_SQUIB_CONNECT_TIMEOUT_MS", 15_000, 900_000)?); + config.response_limit_bytes = usize::try_from(number( + "LAMBDA_SQUIB_RESPONSE_LIMIT_BYTES", + DEFAULT_RESPONSE_LIMIT_BYTES as u64, + DEFAULT_RESPONSE_LIMIT_BYTES as u64, + )?) + .map_err(|error| ExecutorError::Unsupported(error.to_string()))?; + config.run_budget = + Duration::from_secs(number("LAMBDA_SQUIB_RUN_BUDGET_SECS", 86_400, 604_800)?); + config.shutdown_timeout = + Duration::from_millis(number("LAMBDA_SQUIB_SHUTDOWN_TIMEOUT_MS", 10000, 30000)?); + Ok(config) } fn required_config_file(&self) -> Result<&Path, ExecutorError> { @@ -137,6 +166,22 @@ impl Default for SquibExecutorConfig { pub struct SquibExecutor { config: SquibExecutorConfig, runtime: Mutex>, + poisoned: std::sync::atomic::AtomicBool, +} + +struct RuntimeLease<'a> { + runtime: Option, + poisoned: &'a std::sync::atomic::AtomicBool, +} +impl Drop for RuntimeLease<'_> { + fn drop(&mut self) { + if self.runtime.is_some() { + self.poisoned + .store(true, std::sync::atomic::Ordering::Release); + // Squib's RAII handle requests VM shutdown. Never admit another guest after + // cancellation. + } + } } impl SquibExecutor { @@ -146,15 +191,11 @@ impl SquibExecutor { Self { config, runtime: Mutex::new(None), + poisoned: std::sync::atomic::AtomicBool::new(false), } } - async fn ensure_runtime(&self) -> Result<(), ExecutorError> { - let mut runtime = self.runtime.lock().await; - if runtime.is_some() { - return Ok(()); - } - + async fn start_runtime(&self) -> Result { let config_file = self.config.required_config_file()?; ensure_config_file_exists(config_file).await?; @@ -169,8 +210,7 @@ impl SquibExecutor { .spawn() .await .map_err(|err| ExecutorError::InitFailed(format!("start Squib runtime: {err}")))?; - *runtime = Some(squib); - Ok(()) + Ok(squib) } async fn invoke_guest(&self, req: InvokeRequest) -> Result { @@ -214,11 +254,34 @@ impl SquibExecutor { #[async_trait] impl Executor for SquibExecutor { + fn available(&self) -> Result<(), ExecutorError> { + if self.poisoned.load(std::sync::atomic::Ordering::Acquire) { + return Err(ExecutorError::Unsupported( + "Squib guest execution was cancelled; restart the runtime before invoking again" + .into(), + )); + } + Ok(()) + } + async fn invoke(&self, req: InvokeRequest) -> Result { validate_request(&req)?; self.config.required_vsock_path()?; - self.ensure_runtime().await?; - self.invoke_guest(req).await + let mut runtime = self.runtime.lock().await; + self.available()?; + let vm = match runtime.take() { + Some(vm) => vm, + None => self.start_runtime().await?, + }; + let mut lease = RuntimeLease { + runtime: Some(vm), + poisoned: &self.poisoned, + }; + let result = self.invoke_guest(req).await; + if result.is_ok() { + *runtime = lease.runtime.take(); + } + result } async fn shutdown(&self) { @@ -571,15 +634,6 @@ fn default_artifact_dir() -> PathBuf { ) } -fn non_empty_string(value: &str) -> Option { - let trimmed = value.trim(); - if trimmed.is_empty() { - None - } else { - Some(trimmed.to_owned()) - } -} - #[cfg(test)] mod tests { use std::time::Duration; @@ -598,7 +652,8 @@ mod tests { "LAMBDA_SQUIB_RUN_BUDGET_SECS" => Some("60".to_owned()), "LAMBDA_SQUIB_SHUTDOWN_TIMEOUT_MS" => Some("2500".to_owned()), _ => None, - }); + }) + .unwrap(); assert_eq!(config.config_file, Some(PathBuf::from("/tmp/vm.json"))); assert_eq!(config.vsock_path, Some(PathBuf::from("/tmp/vsock.sock"))); diff --git a/crates/rustack-lambda-core/src/executor/types.rs b/crates/rustack-lambda-core/src/executor/types.rs index 5168ae5..5ad0457 100644 --- a/crates/rustack-lambda-core/src/executor/types.rs +++ b/crates/rustack-lambda-core/src/executor/types.rs @@ -7,7 +7,7 @@ use bytes::Bytes; /// Selects which execution backend the provider should construct at startup. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ExecutorBackend { - /// Legacy echo behavior — no real process or container is started. + /// Metadata-only operation: execution fails explicitly and starts no process. #[default] Disabled, /// Pick the most appropriate backend per invocation: Squib for macOS Zip diff --git a/crates/rustack-lambda-core/src/lib.rs b/crates/rustack-lambda-core/src/lib.rs index ec4800c..1ed6361 100644 --- a/crates/rustack-lambda-core/src/lib.rs +++ b/crates/rustack-lambda-core/src/lib.rs @@ -8,6 +8,8 @@ //! - **Resolver**: Function name/ARN parsing and version resolution //! - **Config**: Service configuration from environment variables +#![forbid(unsafe_code)] + pub mod code; pub mod config; pub mod error; @@ -16,3 +18,7 @@ pub mod handler; pub mod provider; pub mod resolver; pub mod storage; +#[cfg(test)] +#[path = "../../../tests/lambda-fixtures/minimal_zip.rs"] +mod test_zip; +mod work; diff --git a/crates/rustack-lambda-core/src/provider.rs b/crates/rustack-lambda-core/src/provider.rs index 657d424..9bd56d5 100644 --- a/crates/rustack-lambda-core/src/provider.rs +++ b/crates/rustack-lambda-core/src/provider.rs @@ -84,12 +84,12 @@ pub enum InvokeKind { /// Build the executor backend for the given configuration. /// -/// `Disabled` preserves legacy echo behavior. `Auto` chooses the concrete -/// backend per invocation. `Docker` is not yet wired and falls back with a -/// warning. +/// `Disabled` and unsupported Docker selections fail execution explicitly. +/// `Auto` is an explicit operator opt-in and chooses the concrete backend per invocation. fn build_executor(config: &LambdaConfig) -> Arc { match config.executor { ExecutorBackend::Disabled => Arc::new(NoopExecutor::new()), + ExecutorBackend::Docker => Arc::new(NoopExecutor::docker_unavailable()), ExecutorBackend::Auto => Arc::new(AutoExecutor::new( config.max_warm_instances, config.idle_timeout, @@ -102,14 +102,6 @@ fn build_executor(config: &LambdaConfig) -> Arc { config.init_timeout, )), ExecutorBackend::Squib => Arc::new(SquibExecutor::new(config.squib.clone())), - ExecutorBackend::Docker => { - tracing::warn!( - "LAMBDA_EXECUTOR=docker requested but the Docker backend is not wired in this \ - build yet; falling back to no-op executor (echoes payload). Use \ - LAMBDA_EXECUTOR=native for real execution of Rust/Go provided.* lambdas." - ); - Arc::new(NoopExecutor::new()) - } } } @@ -140,6 +132,7 @@ pub struct RustackLambda { config: LambdaConfig, executor: Arc, code_fetcher: Arc, + work: crate::work::WorkManager, } /// Serializable Lambda provider snapshot. @@ -186,6 +179,7 @@ impl RustackLambda { config, executor, code_fetcher: Arc::new(UnavailableS3CodeFetcher), + work: crate::work::WorkManager::new(), } } @@ -202,6 +196,7 @@ impl RustackLambda { config, executor, code_fetcher: Arc::new(UnavailableS3CodeFetcher), + work: crate::work::WorkManager::new(), } } @@ -220,6 +215,7 @@ impl RustackLambda { config, executor, code_fetcher: Arc::new(UnavailableS3CodeFetcher), + work: crate::work::WorkManager::new(), } } @@ -242,9 +238,33 @@ impl RustackLambda { /// Stop all warm executor instances. Wired into the rustack server's /// graceful shutdown path. pub async fn shutdown(&self) { + if let Err(error) = self.work.quiesce(Duration::ZERO).await { + tracing::error!(%error, "Lambda shutdown cancelled unfinished work"); + } self.executor.shutdown().await; } + /// Stop accepting invocation work and drain it before snapshot export. + /// + /// # Errors + /// Returns an error after cancelling remaining tasks when the deadline expires. + pub async fn quiesce(&self, timeout: Duration) -> Result<(), LambdaServiceError> { + let deadline = tokio::time::Instant::now() + timeout; + let cleanup_budget = (timeout / 10).min(Duration::from_secs(1)); + let result = self + .work + .quiesce(timeout.saturating_sub(cleanup_budget)) + .await; + if result.is_err() + && tokio::time::timeout_at(deadline, self.executor.shutdown()) + .await + .is_err() + { + tracing::error!("Lambda executor cancellation deadline exceeded"); + } + result + } + /// Export a point-in-time snapshot of Lambda resources. #[must_use] pub fn export_snapshot(&self) -> LambdaSnapshot { @@ -309,6 +329,7 @@ impl RustackLambda { input: CreateFunctionInput, ) -> Result { let name = &input.function_name; + crate::resolver::FunctionName::parse(name)?; if name.is_empty() || name.len() > 140 { return Err(LambdaServiceError::InvalidParameter { message: "Function name must be between 1 and 140 characters".to_owned(), @@ -381,7 +402,7 @@ impl RustackLambda { } } - if self.store.contains(name) { + if self.store.contains(name)? { return Err(LambdaServiceError::ResourceConflict { message: format!("Function already exist: {name}"), }); @@ -523,6 +544,9 @@ impl RustackLambda { ) -> Result { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; let qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(q) = qualifier { + crate::resolver::Qualifier::parse(q)?; + } let record = self.get_record(&name)?; let version = resolve_version(&record, qualifier)?; @@ -560,6 +584,9 @@ impl RustackLambda { ) -> Result { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; let qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(q) = qualifier { + crate::resolver::Qualifier::parse(q)?; + } let record = self.get_record(&name)?; let version = resolve_version(&record, qualifier)?; @@ -607,33 +634,48 @@ impl RustackLambda { let (name, _) = resolve_function_ref(function_ref)?; let should_publish = input.publish.unwrap_or(false); + let before = self.get_record(&name)?; + if input.architectures.as_ref().is_some_and(|architectures| { + architectures.len() != 1 + || architectures + .iter() + .any(|architecture| architecture != "arm64" && architecture != "x86_64") + }) { + return Err(invalid_parameter( + "Exactly one supported architecture is required", + )); + } let (code_sha256, code_size, zip_bytes, code_path, image_uri) = self.process_code(&name, "$LATEST", code_source).await?; - self.store.update(&name, |record| { - let now = now_iso8601(); - record.latest.code_sha256 = code_sha256; - record.latest.code_size = code_size; - record.latest.zip_bytes = zip_bytes; - record.latest.code_path = code_path; - record.latest.image_uri = image_uri; - record.latest.last_modified = now; - record.latest.revision_id = uuid::Uuid::new_v4().to_string(); - - if let Some(archs) = input.architectures.clone() { - record.latest.architectures = archs; - } - })?; - - // If publish=true, publish a new version. - let config = if should_publish { - let publish_input = PublishVersionInput::default(); - self.publish_version(&name, &publish_input)? - } else { - let record = self.get_record(&name)?; - self.build_function_configuration(&record, &record.latest) - }; + let config = + self.store + .update_if_revision(&name, Some(&before.latest.revision_id), |record| { + let now = now_iso8601(); + record.latest.code_sha256 = code_sha256; + record.latest.code_size = code_size; + record.latest.zip_bytes = zip_bytes; + record.latest.code_path = code_path; + record.latest.image_uri = image_uri; + record.latest.last_modified = now; + record.latest.revision_id = uuid::Uuid::new_v4().to_string(); + + if let Some(archs) = input.architectures.clone() { + record.latest.architectures = archs; + } + if should_publish { + let number = record.next_version; + record.next_version = number.saturating_add(1); + let mut published = record.latest.clone(); + published.version = number.to_string(); + let configuration = self.build_function_configuration(record, &published); + record.versions.insert(number, published); + configuration + } else { + self.build_function_configuration(record, &record.latest) + } + })?; info!(function_name = %name, "updated Lambda function code"); Ok(config) @@ -766,29 +808,36 @@ impl RustackLambda { ) -> Result<(), LambdaServiceError> { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; let qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(q) = qualifier { + crate::resolver::Qualifier::parse(q)?; + } - // If qualifier is specified, delete that specific version/alias. + self.store.validate_root().await?; if let Some(q) = qualifier { - if q != "$LATEST" { - // Try to delete a published version. - if let Ok(version_num) = q.parse::() { - self.store.update(&name, |record| { - record.versions.remove(&version_num); - })?; - return Ok(()); - } - // Otherwise it might be an alias -- but DeleteFunction with - // alias qualifier is not a standard API operation; ignore. + let version_num = q.parse::().map_err(|_| { + invalid_parameter("DeleteFunction qualifier must be a published numeric version") + })?; + let removed = self + .store + .update(&name, |record| record.versions.remove(&version_num))?; + if removed.is_none() { + return Err(LambdaServiceError::VersionNotFound { + function_name: name, + version: q.into(), + }); } + return Ok(()); } + // Validate storage before changing metadata. + self.store.validate_root().await?; // Delete the entire function. - if self.store.remove(&name).is_none() { + if self.store.remove(&name)?.is_none() { return Err(LambdaServiceError::FunctionNotFound { name: name.clone() }); } // Clean up code directory. - self.store.cleanup_code(&name).await; + self.store.cleanup_code(&name).await?; info!(function_name = %name, "deleted Lambda function"); Ok(()) @@ -837,7 +886,7 @@ impl RustackLambda { /// Resolves the target version, validates the payload size, then routes /// to the configured [`Executor`]. `DryRun` short-circuits before the /// executor is touched. `Event` returns immediately with a synthetic - /// request id; the actual run happens on a detached `tokio::spawn`. + /// request id only after a slot in the bounded, supervised work set is acquired. pub async fn invoke( &self, function_ref: &str, @@ -846,18 +895,26 @@ impl RustackLambda { invocation_type: InvokeKind, ) -> Result { // Validate synchronous payload size (Appendix C: 6 MB). - if payload.len() > MAX_SYNC_PAYLOAD { + let payload_limit = if invocation_type == InvokeKind::Event { + 1024 * 1024 + } else { + MAX_SYNC_PAYLOAD + }; + if payload.len() > payload_limit { let payload_len = payload.len(); return Err(LambdaServiceError::RequestTooLarge { message: format!( - "Request payload size {payload_len} exceeds the synchronous invoke limit of \ - {MAX_SYNC_PAYLOAD} bytes", + "Request payload size {payload_len} exceeds the invoke limit of \ + {payload_limit} bytes", ), }); } let (name, ref_qualifier) = resolve_function_ref(function_ref)?; let qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(q) = qualifier { + crate::resolver::Qualifier::parse(q)?; + } // Validate function exists and qualifier resolves. let record = self.get_record(&name)?; @@ -867,27 +924,37 @@ impl RustackLambda { return Ok(InvokeOutcome::DryRun); } + if self.config.executor == ExecutorBackend::Docker { + return Err(LambdaServiceError::Internal { + message: "Docker execution is not implemented in this build".into(), + }); + } + self.store.validate_root().await?; + if let Some(path) = &version.code_path { + self.store.validate_artifact(path).await?; + } let req = self.build_invoke_request(&record, version, payload); + let (response, handle) = self + .work + .submit( + Arc::clone(&self.executor), + req, + record.reserved_concurrent_executions, + invocation_type == InvokeKind::Event, + ) + .await?; if invocation_type == InvokeKind::Event { - let executor = Arc::clone(&self.executor); - let request_id = uuid::Uuid::new_v4().to_string(); - let function_name = req.function_name.clone(); - let rid = request_id.clone(); - tokio::spawn(async move { - if let Err(err) = executor.invoke(req).await { - tracing::warn!( - function = %function_name, - request_id = %rid, - error = %err, - "async lambda invocation failed" - ); - } + return Ok(InvokeOutcome::Async { + request_id: uuid::Uuid::new_v4().to_string(), }); - return Ok(InvokeOutcome::Async { request_id }); } - - let response = self.executor.invoke(req).await?; + let _cancel = crate::work::CancelOnDrop(handle); + let response = response + .await + .map_err(|error| LambdaServiceError::Internal { + message: format!("Lambda worker cancelled: {error}"), + })??; Ok(InvokeOutcome::Sync(response)) } @@ -1288,7 +1355,10 @@ impl RustackLambda { input: &AddPermissionInput, ) -> Result { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; - let _qualifier = qualifier.or(ref_qualifier.as_deref()); + let selected_qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(value) = selected_qualifier { + crate::resolver::Qualifier::parse(value)?; + } // Validate required fields per AWS API. let sid = match &input.statement_id { @@ -1361,7 +1431,10 @@ impl RustackLambda { qualifier: Option<&str>, ) -> Result<(), LambdaServiceError> { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; - let _qualifier = qualifier.or(ref_qualifier.as_deref()); + let selected_qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(value) = selected_qualifier { + crate::resolver::Qualifier::parse(value)?; + } self.store.update(&name, |record| { let initial_len = record.policy.statements.len(); @@ -1384,7 +1457,10 @@ impl RustackLambda { qualifier: Option<&str>, ) -> Result { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; - let _qualifier = qualifier.or(ref_qualifier.as_deref()); + let selected_qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(value) = selected_qualifier { + crate::resolver::Qualifier::parse(value)?; + } let record = self.get_record(&name)?; @@ -1511,14 +1587,25 @@ impl RustackLambda { input: CreateFunctionUrlConfigInput, ) -> Result { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; - let _qualifier = qualifier.or(ref_qualifier.as_deref()); + let selected_qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(value) = selected_qualifier { + crate::resolver::Qualifier::parse(value)?; + } let now = now_iso8601(); // Use local URL format for development: http://{host}:{port}/lambda-url/{name}/ - let function_url = format!( - "http://{}:{}/lambda-url/{name}/", - self.config.host, self.config.port, + let endpoint = rustack_core::settings::advertised_endpoint().map_or_else( + || { + let host = if self.config.host.contains(':') && !self.config.host.starts_with('[') { + format!("[{}]", self.config.host) + } else { + self.config.host.clone() + }; + format!("http://{host}:{}", self.config.port) + }, + str::to_owned, ); + let function_url = format!("{}/lambda-url/{name}/", endpoint.trim_end_matches('/')); let function_arn_str = function_arn(&self.config.default_region, &self.config.account_id, &name); @@ -1566,7 +1653,10 @@ impl RustackLambda { qualifier: Option<&str>, ) -> Result { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; - let _qualifier = qualifier.or(ref_qualifier.as_deref()); + let selected_qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(value) = selected_qualifier { + crate::resolver::Qualifier::parse(value)?; + } let record = self.get_record(&name)?; let url_config = @@ -1599,7 +1689,10 @@ impl RustackLambda { input: &UpdateFunctionUrlConfigInput, ) -> Result { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; - let _qualifier = qualifier.or(ref_qualifier.as_deref()); + let selected_qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(value) = selected_qualifier { + crate::resolver::Qualifier::parse(value)?; + } let function_arn_str = function_arn(&self.config.default_region, &self.config.account_id, &name); @@ -1648,7 +1741,10 @@ impl RustackLambda { qualifier: Option<&str>, ) -> Result<(), LambdaServiceError> { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; - let _qualifier = qualifier.or(ref_qualifier.as_deref()); + let selected_qualifier = qualifier.or(ref_qualifier.as_deref()); + if let Some(value) = selected_qualifier { + crate::resolver::Qualifier::parse(value)?; + } self.store .update(&name, |record| -> Result<(), LambdaServiceError> { @@ -2433,6 +2529,7 @@ impl RustackLambda { ) -> Result { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; let qualifier = qualifier.or(ref_qualifier.as_deref()).unwrap_or("$LATEST"); + crate::resolver::Qualifier::parse(qualifier)?; let record = self.get_record(&name)?; let fn_arn = self.build_qualified_arn(&name, qualifier); let now = chrono::Utc::now(); @@ -2465,6 +2562,7 @@ impl RustackLambda { ) -> Result { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; let qualifier = qualifier.or(ref_qualifier.as_deref()).unwrap_or("$LATEST"); + crate::resolver::Qualifier::parse(qualifier)?; let record = self.get_record(&name)?; let config_record = record.event_invoke_configs.get(qualifier).ok_or_else(|| { @@ -2488,6 +2586,7 @@ impl RustackLambda { ) -> Result { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; let qualifier = qualifier.or(ref_qualifier.as_deref()).unwrap_or("$LATEST"); + crate::resolver::Qualifier::parse(qualifier)?; let _ = self.get_record(&name)?; let fn_arn = self.build_qualified_arn(&name, qualifier); let now = chrono::Utc::now(); @@ -2531,6 +2630,7 @@ impl RustackLambda { ) -> Result<(), LambdaServiceError> { let (name, ref_qualifier) = resolve_function_ref(function_ref)?; let qualifier = qualifier.or(ref_qualifier.as_deref()).unwrap_or("$LATEST"); + crate::resolver::Qualifier::parse(qualifier)?; let _ = self.get_record(&name)?; self.store.update(&name, |rec| { @@ -2572,7 +2672,7 @@ impl RustackLambda { /// Get a function record by name, returning `FunctionNotFound` if absent. fn get_record(&self, name: &str) -> Result { self.store - .get(name) + .get(name)? .ok_or(LambdaServiceError::FunctionNotFound { name: name.to_owned(), }) @@ -2933,7 +3033,8 @@ mod tests { fn sample_create_input(name: &str) -> CreateFunctionInput { use base64::Engine; - let zip_data = base64::engine::general_purpose::STANDARD.encode(b"PK\x03\x04fake"); + let zip_data = base64::engine::general_purpose::STANDARD + .encode(crate::test_zip::minimal_zip(b"fixture")); CreateFunctionInput { function_name: name.to_owned(), runtime: Some("python3.12".to_owned()), @@ -3029,9 +3130,9 @@ mod tests { #[tokio::test] async fn test_should_create_function_with_s3_code() { - let provider = provider_with_fetcher(FakeCodeFetcher(FakeFetchResult::Ok( - Bytes::from_static(b"PK\x03\x04fake-s3-code"), - ))); + let provider = provider_with_fetcher(FakeCodeFetcher(FakeFetchResult::Ok(Bytes::from( + crate::test_zip::minimal_zip(b"s3-code"), + )))); let config = provider .create_function(s3_code_input("s3-func")) @@ -3054,7 +3155,7 @@ mod tests { assert!(record.latest.code_path.is_some(), "code_path must be set"); assert_eq!( record.latest.zip_bytes.as_deref(), - Some(b"PK\x03\x04fake-s3-code".as_slice()), + Some(crate::test_zip::minimal_zip(b"s3-code").as_slice()), ); } @@ -3161,8 +3262,10 @@ mod tests { let provider = test_provider(); let mut input = s3_code_input("s3-func"); - input.code.zip_file = - Some(base64::engine::general_purpose::STANDARD.encode(b"PK\x03\x04fake")); + input.code.zip_file = Some( + base64::engine::general_purpose::STANDARD + .encode(crate::test_zip::minimal_zip(b"fixture")), + ); let err = provider .create_function(input) .await @@ -3281,9 +3384,9 @@ mod tests { #[tokio::test] async fn test_should_update_function_code_from_s3() { - let provider = provider_with_fetcher(FakeCodeFetcher(FakeFetchResult::Ok( - Bytes::from_static(b"PK\x03\x04fake-s3-update"), - ))); + let provider = provider_with_fetcher(FakeCodeFetcher(FakeFetchResult::Ok(Bytes::from( + crate::test_zip::minimal_zip(b"s3-update"), + )))); // Start from an inline zip, then switch to an S3 package. provider .create_function(sample_create_input("s3-func")) @@ -3303,10 +3406,13 @@ mod tests { .unwrap(); assert!(config.code_sha256.is_some()); let sha = config.code_sha256.as_ref().unwrap(); - assert_eq!(sha, &compute_sha256(b"PK\x03\x04fake-s3-update"),); + assert_eq!( + sha, + &compute_sha256(&crate::test_zip::minimal_zip(b"s3-update")), + ); assert_eq!( config.code_size, - Some(i64::try_from(b"PK\x03\x04fake-s3-update".len()).unwrap()) + Some(i64::try_from(crate::test_zip::minimal_zip(b"s3-update").len()).unwrap()) ); } @@ -3408,7 +3514,8 @@ mod tests { .await .unwrap(); - let new_zip = base64::engine::general_purpose::STANDARD.encode(b"PK\x03\x04new-code"); + let new_zip = base64::engine::general_purpose::STANDARD + .encode(crate::test_zip::minimal_zip(b"new-code")); let input = UpdateFunctionCodeInput { zip_file: Some(new_zip), ..Default::default() @@ -3473,46 +3580,33 @@ mod tests { } #[tokio::test] - async fn test_should_invoke_sync_via_noop_executor() { - // Default test provider uses ExecutorBackend::Disabled => NoopExecutor, - // which echoes the request body back wrapped in a fake API GW shape. + async fn test_should_reject_sync_execution_when_disabled() { let provider = test_provider(); provider .create_function(sample_create_input("my-func")) .await .unwrap(); - let outcome = provider - .invoke("my-func", None, b"{\"hi\":1}", InvokeKind::RequestResponse) + let error = provider + .invoke("my-func", None, b"{}", InvokeKind::RequestResponse) .await - .unwrap(); - let resp = match outcome { - InvokeOutcome::Sync(r) => r, - other => panic!("expected Sync, got {other:?}"), - }; - assert_eq!(resp.status, 200); - assert_eq!(resp.executed_version, "$LATEST"); - let body: serde_json::Value = serde_json::from_slice(&resp.payload).unwrap(); - assert_eq!(body["statusCode"], 200); - assert_eq!(body["body"], "{\"hi\":1}"); + .unwrap_err(); + assert!(matches!(error, LambdaServiceError::ResourceNotReady { .. })); } #[tokio::test] - async fn test_should_invoke_event_returns_async_outcome() { + async fn test_should_reject_event_before_acceptance_when_disabled() { let provider = test_provider(); provider .create_function(sample_create_input("my-func")) .await .unwrap(); - let outcome = provider + let error = provider .invoke("my-func", None, b"{}", InvokeKind::Event) .await - .unwrap(); - assert!( - matches!(outcome, InvokeOutcome::Async { .. }), - "expected Async, got {outcome:?}" - ); + .unwrap_err(); + assert!(matches!(error, LambdaServiceError::ResourceNotReady { .. })); } #[tokio::test] @@ -3541,7 +3635,8 @@ mod tests { async fn test_should_reject_create_without_runtime_for_zip() { use base64::Engine; let provider = test_provider(); - let zip_data = base64::engine::general_purpose::STANDARD.encode(b"PK\x03\x04fake"); + let zip_data = base64::engine::general_purpose::STANDARD + .encode(crate::test_zip::minimal_zip(b"fixture")); let input = CreateFunctionInput { function_name: "my-func".to_owned(), runtime: None, @@ -3636,7 +3731,8 @@ mod tests { fn test_should_publish_and_get_layer_version() { use base64::Engine; let provider = test_provider(); - let zip_data = base64::engine::general_purpose::STANDARD.encode(b"PK\x03\x04layer"); + let zip_data = base64::engine::general_purpose::STANDARD + .encode(crate::test_zip::minimal_zip(b"layer")); let input = PublishLayerVersionInput { description: Some("Test layer".to_owned()), content: Some(rustack_lambda_model::types::LayerVersionContentInput { diff --git a/crates/rustack-lambda-core/src/resolver.rs b/crates/rustack-lambda-core/src/resolver.rs index e9033be..01afab3 100644 --- a/crates/rustack-lambda-core/src/resolver.rs +++ b/crates/rustack-lambda-core/src/resolver.rs @@ -25,11 +25,99 @@ use crate::{ pub fn resolve_function_ref( function_ref: &str, ) -> Result<(String, Option), LambdaServiceError> { + let result = resolve_raw(function_ref)?; + FunctionName::parse(&result.0)?; + if let Some(q) = &result.1 { + Qualifier::parse(q)?; + } + Ok(result) +} + +/// Validated logical function name, never a filesystem component. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FunctionName(String); +impl FunctionName { + /// Validate an AWS function name. + pub fn parse(value: &str) -> Result { + if value.is_empty() + || value.len() > 64 + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') + { + return Err(LambdaServiceError::InvalidParameter { + message: "Invalid function name".into(), + }); + } + Ok(Self(value.to_owned())) + } +} + +/// Validated full Lambda ARN with separately validated name and qualifier. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FunctionArn { + name: FunctionName, + qualifier: Option, +} +impl FunctionArn { + /// Parse a complete Lambda function ARN. + /// + /// # Errors + /// Rejects malformed ARN components, names, and qualifiers. + pub fn parse(value: &str) -> Result { + if value.len() > 256 { + return Err(LambdaServiceError::InvalidArn { + arn: "ARN exceeds 256 bytes".into(), + }); + } + let (name, qualifier) = parse_arn(value)?; + Ok(Self { + name: FunctionName::parse(&name)?, + qualifier: qualifier.as_deref().map(Qualifier::parse).transpose()?, + }) + } +} + +/// Validated version or alias selector. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Qualifier(String); +impl Qualifier { + /// Validate a version or alias. + pub fn parse(value: &str) -> Result { + if value != "$LATEST" + && (value.is_empty() + || value.len() > 128 + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-') + || (value.bytes().all(|b| b.is_ascii_digit()) + && value.parse::().map_or(true, |n| n == 0))) + { + return Err(LambdaServiceError::InvalidParameter { + message: "Invalid qualifier".into(), + }); + } + Ok(Self(value.to_owned())) + } +} + +fn resolve_raw(function_ref: &str) -> Result<(String, Option), LambdaServiceError> { + if function_ref.len() > 256 { + return Err(LambdaServiceError::InvalidParameter { + message: "Function reference exceeds 256 bytes".into(), + }); + } if function_ref.starts_with("arn:") { - parse_arn(function_ref) + let arn = FunctionArn::parse(function_ref)?; + Ok((arn.name.0, arn.qualifier.map(|qualifier| qualifier.0))) } else if let Some((left, right)) = function_ref.split_once(':') { // Handle partial ARN: `{account}:function:{name}[:{qualifier}]` if let Some(rest) = right.strip_prefix("function:") { + if left.len() != 12 || !left.bytes().all(|b| b.is_ascii_digit()) { + return Err(LambdaServiceError::InvalidArn { + arn: function_ref.to_owned(), + }); + } if let Some((name, qualifier)) = rest.split_once(':') { Ok((name.to_owned(), Some(qualifier.to_owned()))) } else { @@ -51,18 +139,21 @@ fn parse_arn(arn: &str) -> Result<(String, Option), LambdaServiceError> let parts: Vec<&str> = arn.split(':').collect(); // Unqualified ARN: arn:aws:lambda:region:account:function:name = 7 parts // Qualified ARN: arn:aws:lambda:region:account:function:name:qualifier = 8 parts - if parts.len() < 7 || parts[0] != "arn" || parts[2] != "lambda" || parts[5] != "function" { + if !matches!(parts.as_slice(), ["arn", "aws" | "aws-cn" | "aws-us-gov", "lambda", region, account, "function", _, ..] if !region.is_empty() && region.len() <= 64 && region.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') && account.len() == 12 && account.bytes().all(|b| b.is_ascii_digit())) + || !(7..=8).contains(&parts.len()) + { return Err(LambdaServiceError::InvalidArn { arn: arn.to_owned(), }); } - let name = parts[6].to_owned(); - let qualifier = if parts.len() >= 8 && !parts[7].is_empty() { - Some(parts[7].to_owned()) - } else { - None - }; + let name = parts + .get(6) + .ok_or_else(|| LambdaServiceError::InvalidArn { + arn: arn.to_owned(), + })? + .to_string(); + let qualifier = parts.get(7).map(|value| (*value).to_owned()); Ok((name, qualifier)) } @@ -84,6 +175,9 @@ pub fn resolve_version<'a>( function: &'a FunctionRecord, qualifier: Option<&str>, ) -> Result<&'a VersionRecord, LambdaServiceError> { + if let Some(q) = qualifier { + Qualifier::parse(q)?; + } match qualifier { None | Some("$LATEST") => Ok(&function.latest), Some(q) => { @@ -261,6 +355,27 @@ mod tests { } } + #[test] + fn test_should_reject_invalid_names_arns_and_qualifiers() { + for reference in [ + "../escape", + "/absolute", + "a%2fb", + "a\\b", + "a:", + "a:0", + "a:../prod", + "a:prod:extra", + "bad:function:foo", + "arn:aws:lambda:us-east-1:123:function:foo", + "arn:aws:lambda:us-east-1:123456789012:function:foo:prod:extra", + "arn:aws:lambda:us-east-1:123456789012:function:foo:", + ] { + assert!(resolve_function_ref(reference).is_err(), "{reference}"); + } + assert!(resolve_version(&make_function(), Some("../escape")).is_err()); + } + // ---- resolve_function_ref tests ---- #[test] diff --git a/crates/rustack-lambda-core/src/storage.rs b/crates/rustack-lambda-core/src/storage.rs index 2492bb1..c1b6e7c 100644 --- a/crates/rustack-lambda-core/src/storage.rs +++ b/crates/rustack-lambda-core/src/storage.rs @@ -29,6 +29,39 @@ pub struct FunctionStore { functions: DashMap, /// Root directory for storing extracted code. code_dir: PathBuf, + /// Own completed artifacts until the store is dropped; in-flight revisions remain readable. + artifacts: DashMap, + staging: std::sync::Arc, +} + +/// Owns a completed artifact and its resolved filesystem boundary. +#[derive(Debug)] +struct Artifact { + directory: Option, + root: PathBuf, + extracted: PathBuf, +} + +impl Drop for Artifact { + #[allow(clippy::disallowed_methods)] // Drop has no async context; cleanup is bounded to a store-owned directory and failures are reported. + fn drop(&mut self) { + let Some(directory) = self.directory.take() else { + return; + }; + let path = directory.keep(); + let safe = std::fs::canonicalize(&self.root).is_ok_and(|root| root == self.root) + && std::fs::symlink_metadata(&self.root) + .is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink()) + && std::fs::symlink_metadata(&path) + .is_ok_and(|meta| meta.is_dir() && !meta.file_type().is_symlink()); + if !safe { + tracing::error!("Refusing cleanup of replaced Lambda artifact root"); + return; + } + if let Err(error) = std::fs::remove_dir_all(path) { + tracing::error!(%error, "Lambda artifact cleanup failed"); + } + } } /// Complete record for a Lambda function. @@ -794,11 +827,87 @@ impl VersionRecordSnapshot { } impl FunctionStore { + fn validate_record(&self, record: &FunctionRecord) -> Result<(), LambdaServiceError> { + let invalid = |message: &str| LambdaServiceError::InvalidParameter { + message: message.into(), + }; + crate::resolver::FunctionName::parse(&record.name)?; + crate::resolver::FunctionArn::parse(&record.arn)?; + let (name, qualifier) = crate::resolver::resolve_function_ref(&record.arn)?; + if name != record.name || qualifier.is_some() { + return Err(invalid("Function ARN mismatch")); + } + if record.latest.version != "$LATEST" + || record.next_version == 0 + || record.next_version == u64::MAX + || record + .versions + .last_key_value() + .is_some_and(|(version, _)| *version >= record.next_version) + || record + .reserved_concurrent_executions + .is_some_and(|n| !(0..=1000).contains(&n)) + { + return Err(invalid("Invalid function version or concurrency")); + } + for (number, version) in &record.versions { + if *number == 0 || version.version != number.to_string() { + return Err(invalid("Published version number mismatch")); + } + } + for (name, alias) in &record.aliases { + crate::resolver::Qualifier::parse(name)?; + if alias.name != *name { + return Err(invalid("Alias name mismatch")); + } + crate::resolver::Qualifier::parse(&alias.function_version)?; + } + for (qualifier, config) in &record.event_invoke_configs { + crate::resolver::Qualifier::parse(qualifier)?; + if config.qualifier != *qualifier { + return Err(invalid("Event qualifier mismatch")); + } + } + for version in std::iter::once(&record.latest).chain(record.versions.values()) { + crate::resolver::Qualifier::parse(&version.version)?; + if !(1..=900).contains(&version.timeout) + || !(128..=10240).contains(&version.memory_size) + || version.architectures.len() != 1 + || version + .architectures + .iter() + .any(|architecture| architecture != "arm64" && architecture != "x86_64") + || version.environment.len() > 128 + || version + .environment + .iter() + .any(|(key, value)| key.len() > 256 || value.len() > 4096) + { + return Err(invalid("Invalid execution configuration")); + } + if let Some(path) = &version.code_path { + if !self.artifacts.contains_key(path) { + return Err(invalid("Code path is not an owned immutable artifact")); + } + } + if let Some(bytes) = &version.zip_bytes { + if version.code_sha256 != compute_sha256(bytes) + || version.code_size != bytes.len() as u64 + { + return Err(invalid("Deployment package metadata mismatch")); + } + } + } + Ok(()) + } + /// Create a new function store with the given code storage directory. pub fn new(code_dir: impl Into) -> Self { Self { functions: DashMap::new(), code_dir: code_dir.into(), + artifacts: DashMap::new(), + staging: std::sync::Arc::new(tokio::sync::Semaphore::new(4)), } } @@ -815,6 +924,7 @@ impl FunctionStore { /// Returns `ResourceConflict` if a function with the same name already exists. pub fn insert(&self, record: FunctionRecord) -> Result<(), LambdaServiceError> { use dashmap::mapref::entry::Entry; + self.validate_record(&record)?; match self.functions.entry(record.name.clone()) { Entry::Occupied(_) => Err(LambdaServiceError::ResourceConflict { message: format!("Function already exist: {}", record.name), @@ -827,15 +937,15 @@ impl FunctionStore { } /// Get a clone of a function record by name. - #[must_use] - pub fn get(&self, name: &str) -> Option { - self.functions.get(name).map(|r| r.value().clone()) + pub fn get(&self, name: &str) -> Result, LambdaServiceError> { + crate::resolver::FunctionName::parse(name)?; + Ok(self.functions.get(name).map(|r| r.value().clone())) } /// Check whether a function exists. - #[must_use] - pub fn contains(&self, name: &str) -> bool { - self.functions.contains_key(name) + pub fn contains(&self, name: &str) -> Result { + crate::resolver::FunctionName::parse(name)?; + Ok(self.functions.contains_key(name)) } /// Mutate a function record in place. @@ -849,8 +959,41 @@ impl FunctionStore { where F: FnOnce(&mut FunctionRecord) -> R, { + self.update_if_revision(name, None, f) + } + + /// Atomically replace a validated record only if its latest revision still matches. + /// + /// # Errors + /// Rejects invalid names/state, missing functions, and concurrent updates/recreation. + pub fn update_if_revision( + &self, + name: &str, + expected: Option<&str>, + f: F, + ) -> Result + where + F: FnOnce(&mut FunctionRecord) -> R, + { + crate::resolver::FunctionName::parse(name)?; match self.functions.get_mut(name) { - Some(mut entry) => Ok(f(entry.value_mut())), + Some(mut entry) => { + if expected.is_some_and(|revision| revision != entry.latest.revision_id) { + return Err(LambdaServiceError::ResourceConflict { + message: "Function changed while deployment was staged".into(), + }); + } + let mut candidate = entry.value().clone(); + let result = f(&mut candidate); + if candidate.name != name { + return Err(LambdaServiceError::InvalidParameter { + message: "Function identity is immutable".into(), + }); + } + self.validate_record(&candidate)?; + *entry = candidate; + Ok(result) + } None => Err(LambdaServiceError::FunctionNotFound { name: name.to_owned(), }), @@ -860,9 +1003,9 @@ impl FunctionStore { /// Remove a function from the store. /// /// Returns the removed record, or `None` if it did not exist. - #[must_use] - pub fn remove(&self, name: &str) -> Option { - self.functions.remove(name).map(|(_, v)| v) + pub fn remove(&self, name: &str) -> Result, LambdaServiceError> { + crate::resolver::FunctionName::parse(name)?; + Ok(self.functions.remove(name).map(|(_, v)| v)) } /// List all function records. @@ -909,42 +1052,46 @@ impl FunctionStore { &self, snapshot: FunctionStoreSnapshot, ) -> Result<(), LambdaServiceError> { - self.functions.clear(); - match tokio::fs::remove_dir_all(&self.code_dir).await { - Ok(()) => {} - Err(error) if error.kind() == ErrorKind::NotFound => {} - Err(error) => { - return Err(LambdaServiceError::Internal { - message: format!("Failed to clear Lambda code directory: {error}"), + self.validate_root().await?; + let mut staged = BTreeMap::new(); + for function in snapshot.functions { + crate::resolver::FunctionName::parse(&function.name)?; + let (name, qualifier) = crate::resolver::resolve_function_ref(&function.arn)?; + if name != function.name || qualifier.is_some() { + return Err(LambdaServiceError::InvalidParameter { + message: "Snapshot function ARN mismatch".into(), }); } - } - tokio::fs::create_dir_all(&self.code_dir) - .await - .map_err(|error| LambdaServiceError::Internal { - message: format!("Failed to create Lambda code directory: {error}"), - })?; - - for function in snapshot.functions { + for (alias, record) in &function.aliases { + crate::resolver::Qualifier::parse(alias)?; + crate::resolver::Qualifier::parse(&record.function_version)?; + } let record = function.into_record(self).await?; - self.functions.insert(record.name.clone(), record); + self.validate_record(&record)?; + if staged.insert(record.name.clone(), record).is_some() { + return Err(LambdaServiceError::InvalidParameter { + message: "Duplicate snapshot function".into(), + }); + } + } + self.functions.clear(); + for (name, record) in staged { + self.functions.insert(name, record); } Ok(()) } /// Store zip code bytes for a function version and extract them. /// - /// Writes the raw zip bytes to `{code_dir}/{function_name}/{version}/code.zip` - /// and unpacks the contents into `{code_dir}/{function_name}/{version}/extracted/`. + /// Writes ZIP bytes and extracted contents to a new private, internal-ID artifact. + /// Logical names and versions never participate in filesystem paths. /// Returns the **extracted directory** (which is what the executor needs as /// the code root, e.g. for `provided.*` it must contain a `bootstrap` /// binary), along with the base64-encoded SHA-256 and the code size. /// /// Unix file modes from the zip are preserved so executable bits stick. - /// Best-effort: if the bytes are not a valid zip (some early tests use a - /// stub `PK\x03\x04...` blob), the raw zip is still written but extraction - /// is silently skipped — the returned path simply won't contain an - /// executable, which the executor surfaces as a clear error at invoke time. + /// Every ZIP/IO error propagates. Failed staging never modifies old artifacts. + /// Completed artifacts remain owned by the store until shutdown/drop. /// /// # Errors /// @@ -957,50 +1104,35 @@ impl FunctionStore { version: &str, zip_bytes: &[u8], ) -> Result<(PathBuf, String, u64), LambdaServiceError> { - let dir = self.code_dir.join(function_name).join(version); - tokio::fs::create_dir_all(&dir) - .await - .map_err(|e| LambdaServiceError::Internal { - message: format!("Failed to create code directory: {e}"), - })?; - - let zip_path = dir.join("code.zip"); - tokio::fs::write(&zip_path, zip_bytes) - .await - .map_err(|e| LambdaServiceError::Internal { - message: format!("Failed to write code zip: {e}"), - })?; - - let extracted = dir.join("extracted"); - // Wipe any prior extraction (UpdateFunctionCode). - if extracted.exists() { - tokio::fs::remove_dir_all(&extracted).await.map_err(|e| { - LambdaServiceError::Internal { - message: format!("Failed to clear extracted dir: {e}"), - } - })?; + crate::resolver::FunctionName::parse(function_name)?; + crate::resolver::Qualifier::parse(version)?; + if zip_bytes.len() > 50 * 1024 * 1024 { + return Err(LambdaServiceError::InvalidZipFile { + message: "ZIP exceeds 50 MiB".into(), + }); } - tokio::fs::create_dir_all(&extracted) - .await - .map_err(|e| LambdaServiceError::Internal { - message: format!("Failed to create extracted dir: {e}"), + self.validate_root().await?; + let permit = std::sync::Arc::clone(&self.staging) + .try_acquire_owned() + .map_err(|_| LambdaServiceError::ResourceNotReady { + message: "ZIP staging capacity exhausted".into(), })?; - - let extract_to = extracted.clone(); + let root = self.code_dir.clone(); let bytes_owned = zip_bytes.to_vec(); - let extract_result = tokio::task::spawn_blocking(move || { - extract_zip(&bytes_owned, &extract_to, MAX_EXTRACTED_SIZE) - }) - .await - .map_err(|e| LambdaServiceError::Internal { - message: format!("zip extraction task join error: {e}"), - })?; - // A non-zip blob (test stub) is tolerated; a path-traversal attempt is not. - if let Err(err) = extract_result { - if matches!(err, LambdaServiceError::InvalidZipFile { .. }) { - return Err(err); - } - } + let task = tokio::task::spawn_blocking(move || { + let _permit = permit; + stage_zip(&root, &bytes_owned) + }); + let artifact = tokio::time::timeout(std::time::Duration::from_secs(30), task) + .await + .map_err(|_| LambdaServiceError::Internal { + message: "ZIP staging deadline exceeded".into(), + })? + .map_err(|e| LambdaServiceError::Internal { + message: format!("ZIP extraction task failed: {e}"), + })??; + let extracted = artifact.extracted.clone(); + self.artifacts.insert(extracted.clone(), artifact); let sha256 = compute_sha256(zip_bytes); let code_size = zip_bytes.len() as u64; @@ -1008,15 +1140,130 @@ impl FunctionStore { Ok((extracted, sha256, code_size)) } - /// Clean up code directory for a function. + /// Validate logical retirement of a function's code. /// - /// Removes the `{code_dir}/{function_name}` directory tree. - pub async fn cleanup_code(&self, function_name: &str) { - let dir = self.code_dir.join(function_name); - if dir.exists() { - let _ = tokio::fs::remove_dir_all(&dir).await; + /// Immutable artifacts are retained until store drop so published/in-flight + /// references remain valid. No name-derived directory is ever deleted. + pub async fn cleanup_code(&self, function_name: &str) -> Result<(), LambdaServiceError> { + crate::resolver::FunctionName::parse(function_name)?; + self.validate_root().await + } + + /// Check an immutable artifact before handing it to an executor. + pub async fn validate_artifact(&self, path: &Path) -> Result<(), LambdaServiceError> { + self.validate_root().await?; + let owned_root = self + .artifacts + .get(path) + .map(|artifact| artifact.root.clone()) + .ok_or_else(|| LambdaServiceError::InvalidParameter { + message: "Unowned Lambda artifact".into(), + })?; + let root = tokio::fs::canonicalize(&self.code_dir) + .await + .map_err(|error| LambdaServiceError::Internal { + message: format!("Resolve artifact root: {error}"), + })?; + if root != owned_root { + return Err(LambdaServiceError::InvalidParameter { + message: "Artifact root identity changed".into(), + }); + } + let mut current = path; + while current != root { + let meta = tokio::fs::symlink_metadata(current) + .await + .map_err(|error| LambdaServiceError::Internal { + message: format!("Inspect artifact: {error}"), + })?; + if meta.file_type().is_symlink() { + return Err(LambdaServiceError::InvalidParameter { + message: "Symlink in Lambda artifact".into(), + }); + } + current = current + .parent() + .ok_or_else(|| LambdaServiceError::InvalidParameter { + message: "Artifact outside storage root".into(), + })?; + } + let resolved = + tokio::fs::canonicalize(path) + .await + .map_err(|error| LambdaServiceError::Internal { + message: format!("Resolve artifact: {error}"), + })?; + if !resolved.starts_with(root) { + return Err(LambdaServiceError::InvalidParameter { + message: "Artifact escapes storage root".into(), + }); + } + let bootstrap = path.join("bootstrap"); + match tokio::fs::symlink_metadata(bootstrap).await { + Ok(meta) if meta.file_type().is_symlink() => { + Err(LambdaServiceError::InvalidParameter { + message: "Bootstrap cannot be a symlink".into(), + }) + } + Ok(_) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(LambdaServiceError::Internal { + message: format!("Inspect bootstrap: {error}"), + }), } } + + /// Reject a symlink at the operator-selected artifact root. + pub async fn validate_root(&self) -> Result<(), LambdaServiceError> { + match tokio::fs::symlink_metadata(&self.code_dir).await { + Ok(meta) if meta.file_type().is_symlink() || !meta.is_dir() => { + Err(LambdaServiceError::InvalidParameter { + message: "Lambda artifact root must be a real directory".into(), + }) + } + Ok(_) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(LambdaServiceError::Internal { + message: format!("Inspect artifact root: {error}"), + }), + } + } +} + +/// Stage a package in a freshly created private directory on a blocking worker. +#[allow(clippy::disallowed_methods, clippy::disallowed_types)] // Synchronous filesystem work is isolated to spawn_blocking. +fn stage_zip(root: &Path, bytes: &[u8]) -> Result { + let io_error = |source| LambdaServiceError::ArtifactIo { source }; + std::fs::create_dir_all(root).map_err(io_error)?; + if std::fs::symlink_metadata(root) + .map_err(io_error)? + .file_type() + .is_symlink() + { + return Err(LambdaServiceError::InvalidParameter { + message: "Artifact root is a symlink".into(), + }); + } + let root = std::fs::canonicalize(root).map_err(io_error)?; + let artifact = tempfile::Builder::new() + .prefix("artifact-") + .tempdir_in(&root) + .map_err(io_error)?; + let extracted = artifact.path().join("extracted"); + let result = (|| { + std::fs::create_dir(&extracted).map_err(io_error)?; + std::fs::write(artifact.path().join("code.zip"), bytes).map_err(io_error)?; + extract_zip(bytes, &extracted, MAX_EXTRACTED_SIZE) + })(); + if let Err(error) = result { + artifact.close().map_err(io_error)?; + return Err(error); + } + Ok(Artifact { + directory: Some(artifact), + root, + extracted, + }) } /// Maximum extracted deployment package size (250 MB, mirroring the AWS @@ -1027,9 +1274,7 @@ const MAX_EXTRACTED_SIZE: u64 = 250 * 1024 * 1024; /// /// Rejects entries whose normalized path escapes `target` (path traversal) /// and archives whose entries would expand beyond [`MAX_EXTRACTED_SIZE`] -/// (zip bombs). Returns a non-`InvalidZipFile` error to signal the bytes -/// weren't a valid archive — callers may choose to ignore that case (e.g. -/// test stubs). +/// (zip bombs). ZIP and filesystem failures are always returned to the caller. /// /// Synchronous std::fs is intentional: this runs inside `spawn_blocking` and /// the `zip` crate's reader API is itself blocking, so wrapping each I/O in @@ -1046,15 +1291,22 @@ fn extract_zip( }; let cursor = Cursor::new(zip_bytes); - let mut archive = zip::ZipArchive::new(cursor).map_err(|e| LambdaServiceError::Internal { - message: format!("not a valid zip archive: {e}"), - })?; + let mut archive = + zip::ZipArchive::new(cursor).map_err(|e| LambdaServiceError::InvalidZipFile { + message: format!("not a valid zip archive: {e}"), + })?; + if archive.len() > 10_000 { + return Err(LambdaServiceError::InvalidZipFile { + message: "Too many ZIP entries".into(), + }); + } + let mut paths = std::collections::HashSet::new(); let mut extracted_total: u64 = 0; for i in 0..archive.len() { let mut entry = archive .by_index(i) - .map_err(|e| LambdaServiceError::Internal { + .map_err(|e| LambdaServiceError::InvalidZipFile { message: format!("zip entry {i}: {e}"), })?; let Some(rel) = entry.enclosed_name() else { @@ -1062,6 +1314,18 @@ fn extract_zip( message: format!("zip entry has invalid path: {}", entry.name()), }); }; + if entry.name().len() > 1024 + || entry.name().contains('\\') + || entry.name().split('/').any(|component| component == "..") + || entry + .unix_mode() + .is_some_and(|mode| mode & 0o170_000 == 0o120_000 || mode & 0o7000 != 0) + || !paths.insert(rel.clone()) + { + return Err(LambdaServiceError::InvalidZipFile { + message: "Unsafe or duplicate ZIP entry".into(), + }); + } let out_path = target.join(&rel); // Defense in depth: ensure the resolved path stays within target. if !out_path.starts_with(target) { @@ -1091,15 +1355,22 @@ fn extract_zip( message: format!("create parent {}: {e}", parent.display()), })?; } - let mut out = File::create(&out_path).map_err(|e| LambdaServiceError::Internal { - message: format!("create file {}: {e}", out_path.display()), - })?; - let copied = io::copy(&mut (&mut entry).take(remaining + 1), &mut out).map_err(|e| { - LambdaServiceError::Internal { - message: format!("write file {}: {e}", out_path.display()), + let mut out = File::create_new(&out_path) + .map_err(|source| LambdaServiceError::ArtifactIo { source })?; + let copied = io::copy( + &mut (&mut entry).take(remaining.saturating_add(1)), + &mut out, + ) + .map_err(|source| { + if source.kind() == ErrorKind::InvalidData { + LambdaServiceError::InvalidZipFile { + message: format!("ZIP CRC/decompression failed: {source}"), + } + } else { + LambdaServiceError::ArtifactIo { source } } })?; - extracted_total += copied; + extracted_total = extracted_total.saturating_add(copied); if extracted_total > max_extracted { return Err(LambdaServiceError::InvalidZipFile { message: format!("zip archive expands beyond {max_extracted} bytes"), @@ -1133,6 +1404,196 @@ pub fn compute_sha256(data: &[u8]) -> String { mod tests { use super::*; + #[tokio::test] + async fn test_should_reject_hostile_names_at_every_store_boundary() { + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let zip = crate::test_zip::minimal_zip(b"safe"); + for name in [ + "../escape", + "/absolute", + "a/b", + "a\\b", + "a%2fb", + "..", + "", + "é", + ] { + assert!(store.insert(sample_record(name)).is_err()); + assert!(store.get(name).is_err()); + assert!(store.contains(name).is_err()); + assert!(store.remove(name).is_err()); + assert!(store.update(name, |_| ()).is_err()); + assert!(store.store_zip_code(name, "$LATEST", &zip).await.is_err()); + assert!(store.cleanup_code(name).await.is_err()); + } + assert!(store.is_empty()); + assert!( + tokio::fs::read_dir(root.path()) + .await + .unwrap() + .next_entry() + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn test_should_stage_immutable_artifacts_and_preserve_old_on_bad_zip() { + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let (a, _, _) = store + .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .await + .unwrap(); + assert!(matches!( + store.store_zip_code("safe", "$LATEST", b"bad ZIP").await, + Err(LambdaServiceError::InvalidZipFile { .. }) + )); + let (b, _, _) = store + .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"B")) + .await + .unwrap(); + assert_ne!(a, b); + assert_eq!(tokio::fs::read(a.join("fixture.txt")).await.unwrap(), b"A"); + assert_eq!(tokio::fs::read(b.join("fixture.txt")).await.unwrap(), b"B"); + assert!( + !a.components() + .any(|component| component.as_os_str() == "safe") + ); + } + + #[tokio::test] + async fn test_should_propagate_crc_and_staging_io_errors_without_replacing_old_artifact() { + use std::io::{Cursor, Write}; + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + let (old, _, _) = store + .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"old")) + .await + .unwrap(); + let mut corrupt = crate::test_zip::minimal_zip(b"crc-marker"); + let position = corrupt + .windows(10) + .position(|bytes| bytes == b"crc-marker") + .unwrap(); + corrupt[position] ^= 1; + assert!( + store + .store_zip_code("safe", "$LATEST", &corrupt) + .await + .is_err() + ); + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("file", zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(b"file").unwrap(); + writer + .start_file("file/child", zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(b"cannot create child below file").unwrap(); + let conflict = writer.finish().unwrap().into_inner(); + assert!( + store + .store_zip_code("safe", "$LATEST", &conflict) + .await + .is_err() + ); + assert_eq!( + tokio::fs::read(old.join("fixture.txt")).await.unwrap(), + b"old" + ); + assert_eq!( + store.artifacts.len(), + 1, + "failed staging cannot publish an artifact" + ); + let mut dirs = tokio::fs::read_dir(root.path()).await.unwrap(); + let mut count = 0; + while dirs.next_entry().await.unwrap().is_some() { + count += 1; + } + assert_eq!(count, 1, "failed staging directories are removed"); + } + + #[tokio::test] + async fn test_should_preserve_store_on_invalid_snapshot_and_update_identity() { + let root = tempfile::tempdir().unwrap(); + let store = FunctionStore::new(root.path()); + store.insert(sample_record("safe")).unwrap(); + let mut snapshot = store.export_snapshot(); + snapshot.functions.first_mut().unwrap().name = "../escape".into(); + assert!(store.import_snapshot(snapshot).await.is_err()); + assert!(store.get("safe").unwrap().is_some()); + assert!( + store + .update("safe", |record| record.name = "../escape".into()) + .is_err() + ); + assert_eq!(store.get("safe").unwrap().unwrap().name, "safe"); + } + + #[cfg(unix)] + #[tokio::test] + async fn test_should_reject_symlink_root_and_replaced_artifact_without_touching_sentinel() { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + let sentinel = outside.path().join("sentinel"); + tokio::fs::write(&sentinel, b"untouched").await.unwrap(); + let link = root.path().join("linked"); + std::os::unix::fs::symlink(outside.path(), &link).unwrap(); + let store = FunctionStore::new(&link); + assert!( + store + .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .await + .is_err() + ); + assert!(store.cleanup_code("safe").await.is_err()); + assert!( + store + .import_snapshot(FunctionStoreSnapshot::default()) + .await + .is_err() + ); + let store = FunctionStore::new(root.path().join("owned")); + let (artifact, _, _) = store + .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .await + .unwrap(); + tokio::fs::remove_dir_all(&artifact).await.unwrap(); + std::os::unix::fs::symlink(outside.path(), &artifact).unwrap(); + assert!(store.validate_artifact(&artifact).await.is_err()); + drop(store); + assert_eq!(tokio::fs::read(&sentinel).await.unwrap(), b"untouched"); + let managed = root.path().join("replaceable"); + let store = FunctionStore::new(&managed); + let (artifact, _, _) = store + .store_zip_code("safe", "$LATEST", &crate::test_zip::minimal_zip(b"A")) + .await + .unwrap(); + let artifact_id = artifact.parent().unwrap().file_name().unwrap(); + let outside_artifact = outside.path().join(artifact_id); + tokio::fs::create_dir(&outside_artifact).await.unwrap(); + tokio::fs::write(outside_artifact.join("sentinel"), b"outside") + .await + .unwrap(); + tokio::fs::rename(&managed, root.path().join("original-root")) + .await + .unwrap(); + std::os::unix::fs::symlink(outside.path(), &managed).unwrap(); + assert!(store.cleanup_code("safe").await.is_err()); + drop(store); + assert_eq!( + tokio::fs::read(outside_artifact.join("sentinel")) + .await + .unwrap(), + b"outside" + ); + } + fn test_store() -> FunctionStore { FunctionStore::new("/tmp/rustack-lambda-test") } @@ -1192,7 +1653,7 @@ mod tests { let record = sample_record("my-func"); store.insert(record).unwrap(); - let retrieved = store.get("my-func").unwrap(); + let retrieved = store.get("my-func").unwrap().unwrap(); assert_eq!(retrieved.name, "my-func"); } @@ -1215,7 +1676,7 @@ mod tests { }) .unwrap(); - let retrieved = store.get("my-func").unwrap(); + let retrieved = store.get("my-func").unwrap().unwrap(); assert_eq!(retrieved.latest.timeout, 30); } @@ -1230,11 +1691,11 @@ mod tests { fn test_should_remove_function() { let store = test_store(); store.insert(sample_record("my-func")).unwrap(); - assert!(store.contains("my-func")); + assert!(store.contains("my-func").unwrap()); - let removed = store.remove("my-func"); + let removed = store.remove("my-func").unwrap(); assert!(removed.is_some()); - assert!(!store.contains("my-func")); + assert!(!store.contains("my-func").unwrap()); } #[test] @@ -1379,12 +1840,9 @@ mod tests { let tmp = tempfile::tempdir().unwrap(); let store = FunctionStore::new(tmp.path()); - // Stub bytes: not a valid zip, but storage tolerates it (extraction is - // silently skipped) so older tests that pre-date real packaging keep - // working. Returned dir is the (empty) extracted root. - let zip_data = b"PK\x03\x04fake-zip-data"; + let zip_data = crate::test_zip::minimal_zip(b"fixture"); let (dir, sha256, size) = store - .store_zip_code("test-func", "$LATEST", zip_data) + .store_zip_code("test-func", "$LATEST", &zip_data) .await .unwrap(); @@ -1400,7 +1858,12 @@ mod tests { assert!(!sha256.is_empty()); assert_eq!(size, zip_data.len() as u64); - store.cleanup_code("test-func").await; + store.cleanup_code("test-func").await.unwrap(); + assert!( + dir.exists(), + "in-flight immutable artifacts survive logical deletion" + ); + drop(store); assert!(!dir.exists()); } diff --git a/crates/rustack-lambda-core/src/work.rs b/crates/rustack-lambda-core/src/work.rs new file mode 100644 index 0000000..85e9710 --- /dev/null +++ b/crates/rustack-lambda-core/src/work.rs @@ -0,0 +1,344 @@ +//! Bounded, supervised invocation lifecycle. No detached Event tasks. +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration, +}; + +use dashmap::DashMap; +use tokio::{ + sync::{Mutex, Notify, OwnedSemaphorePermit, Semaphore, oneshot}, + task::{AbortHandle, JoinSet}, +}; + +use crate::{ + error::LambdaServiceError, + executor::{Executor, InvokeRequest, InvokeResponse}, +}; + +#[derive(Debug)] +pub(crate) struct WorkManager { + tasks: Mutex>, + accepting: AtomicBool, + capacity: Arc, + global: Arc, + functions: DashMap>, +} +#[derive(Debug, Default)] +struct FunctionCapacity { + active: AtomicUsize, + changed: Notify, +} +#[derive(Debug)] +struct FunctionPermit(Arc); +impl Drop for FunctionPermit { + fn drop(&mut self) { + self.0.active.fetch_sub(1, Ordering::AcqRel); + self.0.changed.notify_waiters(); + } +} +#[derive(Debug)] +pub(crate) struct CancelOnDrop(pub(crate) AbortHandle); +impl Drop for CancelOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +fn unavailable(message: &str) -> LambdaServiceError { + LambdaServiceError::ResourceNotReady { + message: message.into(), + } +} +fn throttled() -> LambdaServiceError { + LambdaServiceError::TooManyRequests +} + +impl FunctionCapacity { + fn try_acquire(self: &Arc, limit: usize) -> Result { + self.active + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| { + if count < limit { + count.checked_add(1) + } else { + None + } + }) + .map_err(|_| throttled())?; + Ok(FunctionPermit(Arc::clone(self))) + } + async fn acquire(self: &Arc, limit: usize) -> FunctionPermit { + loop { + let changed = self.changed.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + if let Ok(permit) = self.try_acquire(limit) { + return permit; + } + changed.await; + } + } +} + +impl WorkManager { + pub(crate) fn new() -> Self { + Self { + tasks: Mutex::new(JoinSet::new()), + accepting: AtomicBool::new(true), + capacity: Arc::new(Semaphore::new(128)), + global: Arc::new(Semaphore::new(32)), + functions: DashMap::new(), + } + } + + pub(crate) async fn submit( + &self, + executor: Arc, + req: InvokeRequest, + reserved: Option, + asynchronous: bool, + ) -> Result< + ( + oneshot::Receiver>, + AbortHandle, + ), + LambdaServiceError, + > { + executor.available()?; + let limit = usize::try_from(reserved.unwrap_or(8)) + .map_err(|_| unavailable("Invalid reserved concurrency"))?; + if limit == 0 { + return Err(throttled()); + } + let slot = Arc::clone(&self.capacity) + .try_acquire_owned() + .map_err(|_| throttled())?; + self.functions.retain(|_, capacity| { + Arc::strong_count(capacity) > 1 || capacity.active.load(Ordering::Acquire) != 0 + }); + let function = Arc::clone( + self.functions + .entry(req.function_name.clone()) + .or_insert_with(|| Arc::new(FunctionCapacity::default())) + .value(), + ); + let permits = if asynchronous { + None + } else { + Some(( + function.try_acquire(limit)?, + Arc::clone(&self.global) + .try_acquire_owned() + .map_err(|_| throttled())?, + )) + }; + let mut tasks = self.tasks.lock().await; + while let Some(result) = tasks.try_join_next() { + if let Err(error) = result { + tracing::error!(%error, "Lambda worker failed"); + } + } + if !self.accepting.load(Ordering::Acquire) { + return Err(unavailable("Lambda is quiescing")); + } + let (tx, rx) = oneshot::channel(); + let global = Arc::clone(&self.global); + let handle = tasks.spawn(async move { + let _slot = slot; + let result = execute(executor, req, function, global, limit, permits).await; + match &result { + Ok(_) => tracing::debug!("Lambda invocation completed"), + Err(error) => tracing::warn!(%error, "Lambda invocation failed"), + } + if tx.send(result).is_err() { + tracing::debug!("Lambda invocation completed without a waiting caller"); + } + }); + Ok((rx, handle)) + } + + pub(crate) async fn quiesce(&self, timeout: Duration) -> Result<(), LambdaServiceError> { + self.accepting.store(false, Ordering::Release); + // Own the JoinSet across awaits so cancellation of quiesce drops/aborts every task. + let mut tasks = std::mem::take(&mut *self.tasks.lock().await); + let drained = tokio::time::timeout(timeout, async { + while let Some(result) = tasks.join_next().await { + if let Err(error) = result { + return Err(unavailable(&format!("Lambda task failed: {error}"))); + } + } + // Includes workers aborted by a previously cancelled quiesce future. + let _all_capacity = Arc::clone(&self.capacity) + .acquire_many_owned(128) + .await + .map_err(|_| unavailable("Lambda accepted-work capacity closed"))?; + Ok(()) + }) + .await; + match drained { + Ok(Ok(())) => Ok(()), + result => { + tasks.abort_all(); + while let Some(result) = tasks.join_next().await { + if let Err(error) = result { + tracing::warn!(%error, "Lambda task cancelled during quiesce"); + } + } + match result { + Ok(Err(error)) => Err(error), + _ => Err(unavailable( + "Lambda quiesce deadline exceeded; remaining invocations cancelled", + )), + } + } + } + } +} + +async fn execute( + executor: Arc, + req: InvokeRequest, + function: Arc, + global: Arc, + limit: usize, + permits: Option<(FunctionPermit, OwnedSemaphorePermit)>, +) -> Result { + let (_function, _global) = match permits { + Some(permits) => permits, + None => ( + function.acquire(limit).await, + global + .acquire_owned() + .await + .map_err(|_| unavailable("Lambda execution closed"))?, + ), + }; + let deadline = req.timeout.saturating_add(Duration::from_secs(10)); + match tokio::time::timeout(deadline, executor.invoke(req)).await { + Ok(result) => result.map_err(LambdaServiceError::from), + Err(_) => Err(unavailable("Lambda invocation deadline exceeded")), + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use async_trait::async_trait; + use bytes::Bytes; + + use super::*; + use crate::executor::{ExecutorError, PackageType}; + + #[derive(Debug)] + struct WaitingExecutor; + #[async_trait] + impl Executor for WaitingExecutor { + async fn invoke(&self, _request: InvokeRequest) -> Result { + std::future::pending().await + } + async fn shutdown(&self) {} + } + fn request(name: &str) -> InvokeRequest { + InvokeRequest { + function_name: name.into(), + function_arn: "arn".into(), + qualifier: "$LATEST".into(), + runtime: None, + handler: None, + architectures: vec![], + package_type: PackageType::Zip, + code_root: None, + code_zip: None, + image_uri: None, + environment: HashMap::new(), + timeout: Duration::from_mins(1), + memory_mb: 128, + payload: Bytes::new(), + capture_logs: false, + } + } + #[tokio::test] + async fn test_should_bound_event_set_before_accepting_and_release_on_cancel() { + let work = WorkManager::new(); + for _ in 0..128 { + work.submit(Arc::new(WaitingExecutor), request("bounded"), None, true) + .await + .unwrap(); + } + assert!(matches!( + work.submit(Arc::new(WaitingExecutor), request("bounded"), None, true) + .await, + Err(LambdaServiceError::TooManyRequests) + )); + tokio::task::yield_now().await; + assert!( + work.functions + .get("bounded") + .unwrap() + .active + .load(Ordering::Acquire) + <= 8 + ); + assert!(work.global.available_permits() >= 24); + assert!(work.quiesce(Duration::ZERO).await.is_err()); + assert_eq!(work.capacity.available_permits(), 128); + assert_eq!(work.global.available_permits(), 32); + assert_eq!( + work.functions + .get("bounded") + .unwrap() + .active + .load(Ordering::Acquire), + 0 + ); + assert!( + work.submit(Arc::new(WaitingExecutor), request("bounded"), None, true) + .await + .is_err() + ); + } + #[tokio::test] + async fn test_should_enforce_function_global_and_zero_reserved_execution_limits() { + let work = WorkManager::new(); + assert!(matches!( + work.submit(Arc::new(WaitingExecutor), request("zero"), Some(0), true) + .await, + Err(LambdaServiceError::TooManyRequests) + )); + for name in ["a", "b", "c", "d"] { + for _ in 0..8 { + work.submit(Arc::new(WaitingExecutor), request(name), None, false) + .await + .unwrap(); + } + assert!(matches!( + work.submit(Arc::new(WaitingExecutor), request(name), None, false) + .await, + Err(LambdaServiceError::TooManyRequests) + )); + } + assert!(matches!( + work.submit(Arc::new(WaitingExecutor), request("e"), Some(32), false) + .await, + Err(LambdaServiceError::TooManyRequests) + )); + assert_eq!(work.global.available_permits(), 0); + assert!(work.quiesce(Duration::ZERO).await.is_err()); + assert_eq!(work.global.available_permits(), 32); + } + #[tokio::test] + async fn test_should_release_capacity_when_sync_caller_cancels() { + let work = WorkManager::new(); + let (_response, handle) = work + .submit(Arc::new(WaitingExecutor), request("cancel"), None, false) + .await + .unwrap(); + drop(CancelOnDrop(handle)); + assert!(work.quiesce(Duration::from_secs(1)).await.is_err()); + assert_eq!(work.capacity.available_permits(), 128); + assert_eq!(work.global.available_permits(), 32); + } +} diff --git a/crates/rustack-lambda-http/Cargo.toml b/crates/rustack-lambda-http/Cargo.toml index 6528bcd..5639859 100644 --- a/crates/rustack-lambda-http/Cargo.toml +++ b/crates/rustack-lambda-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-lambda-http" description = "Lambda HTTP service layer for Rustack (restJson1 protocol)" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-lambda-model = { workspace = true } rustack-auth = { workspace = true } hyper = { workspace = true } diff --git a/crates/rustack-lambda-http/src/service.rs b/crates/rustack-lambda-http/src/service.rs index 09b1650..06a0283 100644 --- a/crates/rustack-lambda-http/src/service.rs +++ b/crates/rustack-lambda-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_lambda_model::error::LambdaError; @@ -116,25 +115,60 @@ async fn process_request( let query = parts.uri.query().unwrap_or("").to_owned(); // 3. Collect body. - let body = match collect_body(incoming).await { + let budget = match op { + rustack_lambda_model::operations::LambdaOperation::CreateFunction + | rustack_lambda_model::operations::LambdaOperation::UpdateFunctionCode => { + rustack_core::http::BodyBudget::lambda_code() + } + rustack_lambda_model::operations::LambdaOperation::Invoke => { + let limit = if parts + .headers + .get("x-amz-invocation-type") + .is_some_and(|value| value == "Event") + { + 1024 * 1024 + } else { + 6 * 1024 * 1024 + }; + match std::num::NonZeroU64::new(limit) { + Some(limit) => rustack_core::http::BodyBudget::control().capped(limit), + None => { + return wrap_error_response( + &LambdaError::service_error("invalid invocation byte budget"), + request_id, + ); + } + } + } + _ => rustack_core::http::BodyBudget::control(), + }; + let body = match collect_body(incoming, budget).await { Ok(body) => body, - Err(err) => return wrap_error_response(&err, request_id), + Err(err) => { + let code = if matches!(err, rustack_core::http::BodyReadError::TooLarge) { + rustack_lambda_model::error::LambdaErrorCode::RequestTooLargeException + } else { + rustack_lambda_model::error::LambdaErrorCode::InvalidRequestContentException + }; + let error = LambdaError::new(code, err.to_string()); + let mut response = wrap_error_response(&error, request_id); + *response.status_mut() = err.status_code(); + return response; + } }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = LambdaError::new( - rustack_lambda_model::error::LambdaErrorCode::InvalidRequestContentException, - auth_err.to_string(), - ); - return wrap_error_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let err = LambdaError::new( + rustack_lambda_model::error::LambdaErrorCode::InvalidRequestContentException, + auth_err.to_string(), + ); + return wrap_error_response(&err, request_id); } // 5. Dispatch to handler. @@ -176,12 +210,11 @@ fn wrap_error_response( } /// Collect the incoming body into a single `Bytes` buffer. -async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() - .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| LambdaError::service_error(format!("Failed to read request body: {e}"))) +async fn collect_body( + incoming: Incoming, + budget: rustack_core::http::BodyBudget, +) -> Result { + rustack_core::http::collect_body(incoming, budget).await } /// Add common response headers to every Lambda response. diff --git a/crates/rustack-logs-core/Cargo.toml b/crates/rustack-logs-core/Cargo.toml index 0e14908..e2c0a2e 100644 --- a/crates/rustack-logs-core/Cargo.toml +++ b/crates/rustack-logs-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-logs-core" description = "CloudWatch Logs business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-logs-core/src/config.rs b/crates/rustack-logs-core/src/config.rs index 9be8109..65e1e8c 100644 --- a/crates/rustack-logs-core/src/config.rs +++ b/crates/rustack-logs-core/src/config.rs @@ -1,6 +1,6 @@ //! CloudWatch Logs service configuration. -use std::env; +use rustack_core::settings as env; /// CloudWatch Logs service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-logs-http/Cargo.toml b/crates/rustack-logs-http/Cargo.toml index ad4a08f..7f43cb5 100644 --- a/crates/rustack-logs-http/Cargo.toml +++ b/crates/rustack-logs-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-logs-http" description = "CloudWatch Logs HTTP service layer for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-logs-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-logs-http/src/service.rs b/crates/rustack-logs-http/src/service.rs index 61271e6..f2c22b8 100644 --- a/crates/rustack-logs-http/src/service.rs +++ b/crates/rustack-logs-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_logs_model::error::LogsError; @@ -126,19 +125,18 @@ async fn process_request( }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = LogsError::with_message( - rustack_logs_model::error::LogsErrorCode::ValidationException, - auth_err.to_string(), - ); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let mut err = LogsError::with_message( + rustack_logs_model::error::LogsErrorCode::ValidationException, + auth_err.to_string(), + ); + err.status_code = http::StatusCode::FORBIDDEN; + return error_to_response(&err, request_id); } // 5. Dispatch to handler. @@ -150,11 +148,13 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| LogsError::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = LogsError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every CloudWatch Logs response. diff --git a/crates/rustack-s3-core/Cargo.toml b/crates/rustack-s3-core/Cargo.toml index 442b69c..bb5f19a 100644 --- a/crates/rustack-s3-core/Cargo.toml +++ b/crates/rustack-s3-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-s3-core" description = "S3 service implementation for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-s3-core/src/config.rs b/crates/rustack-s3-core/src/config.rs index 6e929b5..97c4389 100644 --- a/crates/rustack-s3-core/src/config.rs +++ b/crates/rustack-s3-core/src/config.rs @@ -18,14 +18,14 @@ use typed_builder::TypedBuilder; /// use rustack_s3_core::config::S3Config; /// /// let config = S3Config::default(); -/// assert_eq!(config.gateway_listen, "0.0.0.0:4566"); +/// assert_eq!(config.gateway_listen, "127.0.0.1:4566"); /// assert!(config.s3_virtual_hosting); /// ``` #[derive(Debug, Clone, Serialize, Deserialize, TypedBuilder)] #[serde(rename_all = "camelCase")] pub struct S3Config { - /// Bind address for the gateway (e.g. `"0.0.0.0:4566"`). - #[builder(default = String::from("0.0.0.0:4566"))] + /// Bind address for the gateway (e.g. `"127.0.0.1:4566"`). + #[builder(default = String::from("127.0.0.1:4566"))] pub gateway_listen: String, /// Whether S3 virtual-hosted-style addressing is enabled. @@ -64,7 +64,7 @@ pub struct S3Config { impl Default for S3Config { fn default() -> Self { Self { - gateway_listen: String::from("0.0.0.0:4566"), + gateway_listen: String::from("127.0.0.1:4566"), s3_virtual_hosting: true, s3_domain: String::from("s3.localhost.localstack.cloud"), s3_skip_signature_validation: true, @@ -84,7 +84,7 @@ impl S3Config { /// /// | Variable | Default | /// |----------|---------| - /// | `GATEWAY_LISTEN` | `0.0.0.0:4566` | + /// | `GATEWAY_LISTEN` | `127.0.0.1:4566` | /// | `S3_VIRTUAL_HOSTING` | `true` | /// | `S3_DOMAIN` | `s3.localhost.localstack.cloud` | /// | `S3_SKIP_SIGNATURE_VALIDATION` | `true` | @@ -106,33 +106,33 @@ impl S3Config { pub fn from_env() -> Self { let mut config = Self::default(); - if let Ok(v) = std::env::var("GATEWAY_LISTEN") { + if let Ok(v) = rustack_core::settings::var("GATEWAY_LISTEN") { config.gateway_listen = v; } - if let Ok(v) = std::env::var("S3_VIRTUAL_HOSTING") { + if let Ok(v) = rustack_core::settings::var("S3_VIRTUAL_HOSTING") { config.s3_virtual_hosting = parse_bool(&v); } - if let Ok(v) = std::env::var("S3_DOMAIN") { + if let Ok(v) = rustack_core::settings::var("S3_DOMAIN") { config.s3_domain = v; } - if let Ok(v) = std::env::var("S3_SKIP_SIGNATURE_VALIDATION") { + if let Ok(v) = rustack_core::settings::var("S3_SKIP_SIGNATURE_VALIDATION") { config.s3_skip_signature_validation = parse_bool(&v); } - if let Ok(v) = std::env::var("S3_MAX_MEMORY_OBJECT_SIZE") { + if let Ok(v) = rustack_core::settings::var("S3_MAX_MEMORY_OBJECT_SIZE") { if let Ok(n) = v.parse::() { config.s3_max_memory_object_size = n; } } - if let Ok(v) = std::env::var("DEFAULT_REGION") { + if let Ok(v) = rustack_core::settings::var("DEFAULT_REGION") { config.default_region = v; } - if let Ok(v) = std::env::var("LOG_LEVEL") { + if let Ok(v) = rustack_core::settings::var("LOG_LEVEL") { config.log_level = v; } - if let Ok(v) = std::env::var("PERSISTENCE") { + if let Ok(v) = rustack_core::settings::var("PERSISTENCE") { config.persistence = parse_bool(&v); } - if let Ok(v) = std::env::var("DATA_DIR") { + if let Ok(v) = rustack_core::settings::var("DATA_DIR") { config.data_dir = v; } @@ -152,7 +152,7 @@ mod tests { #[test] fn test_should_create_default_config() { let config = S3Config::default(); - assert_eq!(config.gateway_listen, "0.0.0.0:4566"); + assert_eq!(config.gateway_listen, "127.0.0.1:4566"); assert!(config.s3_virtual_hosting); assert_eq!(config.s3_domain, "s3.localhost.localstack.cloud"); assert!(config.s3_skip_signature_validation); diff --git a/crates/rustack-s3-core/src/ops/bucket.rs b/crates/rustack-s3-core/src/ops/bucket.rs index a6374f3..8be2702 100644 --- a/crates/rustack-s3-core/src/ops/bucket.rs +++ b/crates/rustack-s3-core/src/ops/bucket.rs @@ -27,10 +27,9 @@ pub(crate) fn to_model_owner(owner: &InternalOwner) -> Owner { } } -// These handler methods must remain async because some operations involve -// storage I/O. Methods that are fully synchronous are allowed to be async -// for consistency. -#[allow(clippy::unused_async)] +// Keep handlers lazy and uniformly awaitable at the dispatch boundary, including +// in-memory operations that currently complete without yielding. +#[allow(clippy::unused_async, clippy::unused_async_trait_impl)] impl RustackS3 { /// Create a new S3 bucket. pub async fn handle_create_bucket( diff --git a/crates/rustack-s3-core/src/ops/bucket_config.rs b/crates/rustack-s3-core/src/ops/bucket_config.rs index 6f4a9cc..b0f2085 100644 --- a/crates/rustack-s3-core/src/ops/bucket_config.rs +++ b/crates/rustack-s3-core/src/ops/bucket_config.rs @@ -60,8 +60,9 @@ use crate::{ }, }; -// These handler methods must remain async for consistency. -#[allow(clippy::unused_async)] +// Keep handlers lazy and uniformly awaitable at the dispatch boundary, including +// in-memory operations that currently complete without yielding. +#[allow(clippy::unused_async, clippy::unused_async_trait_impl)] impl RustackS3 { // ----------------------------------------------------------------------- // Versioning diff --git a/crates/rustack-s3-core/src/ops/list.rs b/crates/rustack-s3-core/src/ops/list.rs index 32841ec..4394d3b 100644 --- a/crates/rustack-s3-core/src/ops/list.rs +++ b/crates/rustack-s3-core/src/ops/list.rs @@ -28,7 +28,6 @@ const DEFAULT_MAX_KEYS: i32 = 1000; /// # Errors /// /// Returns [`S3Error`] with [`S3ErrorCode::InvalidArgument`] if `max_keys` is negative. -#[allow(clippy::result_large_err)] fn validate_max_keys(max_keys: Option) -> Result { let value = max_keys.unwrap_or(DEFAULT_MAX_KEYS); if value < 0 { @@ -82,11 +81,14 @@ fn to_common_prefixes(prefixes: &[String]) -> Vec { // AWS S3 DTOs use signed integers (i32/i64) for inherently non-negative values. // These handler methods must remain async for consistency with other handlers. +// Keep handlers lazy and uniformly awaitable at the dispatch boundary, including +// in-memory operations that currently complete without yielding. #[allow( clippy::cast_possible_wrap, clippy::cast_possible_truncation, clippy::cast_sign_loss, - clippy::unused_async + clippy::unused_async, + clippy::unused_async_trait_impl )] impl RustackS3 { /// List objects (v1 API). diff --git a/crates/rustack-s3-core/src/ops/multipart.rs b/crates/rustack-s3-core/src/ops/multipart.rs index e17af6c..26e0854 100644 --- a/crates/rustack-s3-core/src/ops/multipart.rs +++ b/crates/rustack-s3-core/src/ops/multipart.rs @@ -46,11 +46,14 @@ use super::bucket::to_model_owner; // AWS S3 DTOs use signed integers (i32/i64) for inherently non-negative values. // These handler methods must remain async for consistency. +// Keep handlers lazy and uniformly awaitable at the dispatch boundary, including +// in-memory operations that currently complete without yielding. #[allow( clippy::cast_possible_wrap, clippy::cast_possible_truncation, clippy::cast_sign_loss, - clippy::unused_async + clippy::unused_async, + clippy::unused_async_trait_impl )] impl RustackS3 { /// Create a new multipart upload. @@ -180,10 +183,28 @@ impl RustackS3 { } /// Upload a single part of a multipart upload. - #[allow(clippy::too_many_lines)] pub async fn handle_upload_part( + &self, + input: UploadPartInput, + ) -> Result { + self.upload_part_body(input, None).await + } + + /// Publish an incrementally staged multipart part after checksum validation. + pub async fn handle_upload_part_staged( + &self, + input: UploadPartInput, + upload: std::sync::Arc, + ) -> Result { + self.upload_part_body(input, Some(upload)).await + } + + // Preserve the existing multipart validation/publication operation as one sequence. + #[allow(clippy::too_many_lines)] + async fn upload_part_body( &self, mut input: UploadPartInput, + staged: Option>, ) -> Result { // Extract checksum before moving fields out of input. let part_checksum = extract_checksum_from_part(&input)?; @@ -219,14 +240,33 @@ impl RustackS3 { let body_data = input.body.take().map(|b| b.data).unwrap_or_default(); // Validate Content-MD5 if provided. - validate_content_md5(input.content_md5.as_deref(), &body_data) - .map_err(S3ServiceError::into_s3_error)?; + if let Some(ref staged) = staged { + staged + .validate_md5(input.content_md5.as_deref()) + .map_err(S3ServiceError::into_s3_error)?; + } else { + validate_content_md5(input.content_md5.as_deref(), &body_data) + .map_err(S3ServiceError::into_s3_error)?; + } + let compute = |algorithm| -> Result { + match &staged { + Some(upload) => upload.checksum(algorithm).map(str::to_owned), + None => Ok(compute_checksum(algorithm, &body_data)), + } + }; + if let Some(ref checksum) = part_checksum { + let algorithm = CoreChecksumAlgorithm::from_str(&checksum.algorithm) + .map_err(|_| S3ServiceError::BadDigest.into_s3_error())?; + if checksum.value != compute(algorithm).map_err(S3ServiceError::into_s3_error)? { + return Err(S3ServiceError::BadDigest.into_s3_error()); + } + } // If the multipart upload has a checksum algorithm, validate the part // checksum and compute server-side if not provided. let checksum = if let Some(ref algo_str) = upload_checksum_algorithm { if let Ok(algo) = CoreChecksumAlgorithm::from_str(algo_str) { - let computed = compute_checksum(algo, &body_data); + let computed = compute(algo).map_err(S3ServiceError::into_s3_error)?; if let Some(ref client_cksum) = part_checksum { // Validate algorithm matches. if !client_cksum.algorithm.eq_ignore_ascii_case(algo_str) { @@ -256,11 +296,15 @@ impl RustackS3 { }; // Write part to storage. - let write_result = self - .storage - .write_part(&bucket_name, &upload_id, part_number as u32, body_data) - .await - .map_err(|e| S3ServiceError::Internal(anyhow::anyhow!("{e}")).into_s3_error())?; + let write_result = if let Some(staged) = staged { + self.storage + .write_staged_part(&bucket_name, &upload_id, part_number as u32, staged) + } else { + self.storage + .write_part(&bucket_name, &upload_id, part_number as u32, body_data) + .await + .map_err(S3ServiceError::into_s3_error)? + }; // Build checksum output fields. let (out_crc32, out_crc32c, out_crc64nvme, out_sha1, out_sha256) = @@ -317,18 +361,21 @@ impl RustackS3 { // Read source object data. let src_vid = src_version_id.as_deref().unwrap_or("null"); - let data = self - .storage - .read_object(&src_bucket, &src_key, src_vid, None) - .await - .map_err(|e| S3ServiceError::Internal(anyhow::anyhow!("{e}")).into_s3_error())?; - - // Write as part. - let write_result = self - .storage - .write_part(&bucket_name, &upload_id, part_number as u32, data) - .await - .map_err(|e| S3ServiceError::Internal(anyhow::anyhow!("{e}")).into_s3_error())?; + let write_result = + if let Some(staged) = self.storage.staged_object(&src_bucket, &src_key, src_vid) { + self.storage + .write_staged_part(&bucket_name, &upload_id, part_number as u32, staged) + } else { + let data = self + .storage + .read_object(&src_bucket, &src_key, src_vid, None) + .await + .map_err(S3ServiceError::into_s3_error)?; + self.storage + .write_part(&bucket_name, &upload_id, part_number as u32, data) + .await + .map_err(S3ServiceError::into_s3_error)? + }; // Record the part metadata. let bucket = self @@ -478,12 +525,14 @@ impl RustackS3 { let value = if checksum_type_str == "FULL_OBJECT" { let assembled = self .storage - .read_object(&bucket_name, &key, &version_id, None) - .await - .map_err(|e| { - S3ServiceError::Internal(anyhow::anyhow!("{e}")).into_s3_error() + .staged_object(&bucket_name, &key, &version_id) + .ok_or_else(|| { + S3Error::internal_error("missing assembled multipart artifact") })?; - compute_checksum(algo, &assembled) + assembled + .checksum(algo) + .map_err(S3ServiceError::into_s3_error)? + .to_owned() } else { // Collect part checksums in order for composite calculation. let part_checksums: Vec = part_numbers @@ -793,7 +842,6 @@ type ChecksumOutputFields = ( /// /// Returns at most one checksum. If multiple checksum fields are set, returns /// an error. -#[allow(clippy::result_large_err)] fn extract_checksum_from_part(input: &UploadPartInput) -> Result, S3Error> { let candidates: [(&str, &Option); 5] = [ ("CRC32", &input.checksum_crc32), diff --git a/crates/rustack-s3-core/src/ops/object.rs b/crates/rustack-s3-core/src/ops/object.rs index aa30246..152a97b 100644 --- a/crates/rustack-s3-core/src/ops/object.rs +++ b/crates/rustack-s3-core/src/ops/object.rs @@ -37,6 +37,51 @@ use crate::{ validation::{validate_content_md5, validate_metadata, validate_object_key}, }; +/// Look up an object while preserving delete-marker metadata on GET and HEAD errors. +fn lookup_object<'a>( + store: &'a ObjectStore, + key: &str, + version_id: Option<&str>, +) -> Result<&'a S3Object, S3Error> { + let object = match version_id { + Some(id) => store.get_version(key, id), + None => store.get(key), + }; + object.ok_or_else(|| { + if let Some(marker) = store.delete_marker(key, version_id) { + let code = if version_id.is_some() { + S3ErrorCode::MethodNotAllowed + } else { + S3ErrorCode::NoSuchKey + }; + let mut error = S3Error::new(code) + .with_header("x-amz-delete-marker", "true") + .with_header("x-amz-version-id", marker.version_id.clone()); + if version_id.is_some() { + error = error.with_header( + "Last-Modified", + marker + .last_modified + .format("%a, %d %b %Y %H:%M:%S GMT") + .to_string(), + ); + } + error + } else if let Some(id) = version_id { + S3ServiceError::NoSuchVersion { + key: key.to_owned(), + version_id: id.to_owned(), + } + .into_s3_error() + } else { + S3ServiceError::NoSuchKey { + key: key.to_owned(), + } + .into_s3_error() + } + }) +} + /// Check whether Object Lock (legal hold or retention) prevents deletion of a /// specific object version. /// @@ -53,7 +98,6 @@ use crate::{ /// COMPLIANCE-mode or legal holds. /// /// Returns `Ok(())` when the deletion is allowed. -#[allow(clippy::result_large_err)] fn check_object_lock_for_delete( store: &ObjectStore, key: &str, @@ -93,17 +137,37 @@ fn check_object_lock_for_delete( // (sizes, part counts). Casting from u64/u32/usize is safe in practice. // These handler methods must remain async because some operations involve // storage I/O. +// Keep handlers lazy and uniformly awaitable at the dispatch boundary, including +// in-memory operations that currently complete without yielding. #[allow( clippy::cast_possible_wrap, clippy::cast_possible_truncation, clippy::cast_sign_loss, - clippy::unused_async + clippy::unused_async, + clippy::unused_async_trait_impl )] impl RustackS3 { /// Put (upload) a new object. pub async fn handle_put_object( + &self, + input: PutObjectInput, + ) -> Result { + self.put_object_body(input, None).await + } + + /// Publish an upload whose bytes were incrementally staged and authenticated by HTTP. + pub async fn handle_put_object_staged( + &self, + input: PutObjectInput, + upload: std::sync::Arc, + ) -> Result { + self.put_object_body(input, Some(upload)).await + } + + async fn put_object_body( &self, mut input: PutObjectInput, + upload: Option>, ) -> Result { let bucket_name = input.bucket.clone(); let key = input.key.clone(); @@ -120,8 +184,14 @@ impl RustackS3 { let body_data = input.body.take().map_or_else(Bytes::new, |b| b.data); // Validate Content-MD5 if provided. - validate_content_md5(input.content_md5.as_deref(), &body_data) - .map_err(S3ServiceError::into_s3_error)?; + if let Some(ref upload) = upload { + upload + .validate_md5(input.content_md5.as_deref()) + .map_err(S3ServiceError::into_s3_error)?; + } else { + validate_content_md5(input.content_md5.as_deref(), &body_data) + .map_err(S3ServiceError::into_s3_error)?; + } // Extract metadata from the request. let metadata = build_metadata(&input); @@ -134,34 +204,44 @@ impl RustackS3 { "null".to_owned() }; - // Write to storage. - let write_result = self - .storage - .write_object(&bucket_name, &key, &version_id, body_data.clone()) - .await - .map_err(|e| S3ServiceError::Internal(anyhow::anyhow!("{e}")).into_s3_error())?; - // Extract checksum from the request, or compute CRC32 by default. let client_checksum = extract_checksum_from_put(&input).map_err(S3ServiceError::into_s3_error)?; let is_client_provided = client_checksum.is_some(); - let checksum = client_checksum.unwrap_or_else(|| ChecksumData { - algorithm: "CRC32".to_owned(), - value: compute_checksum(ChecksumAlgorithm::Crc32, &body_data), - checksum_type: "FULL_OBJECT".to_owned(), - }); - - // Validate client-provided checksum against server-computed value. + let compute = |algorithm| -> Result { + match &upload { + Some(upload) => upload.checksum(algorithm).map(str::to_owned), + None => Ok(compute_checksum(algorithm, &body_data)), + } + }; + let checksum = match client_checksum { + Some(checksum) => checksum, + None => ChecksumData { + algorithm: "CRC32".to_owned(), + value: compute(ChecksumAlgorithm::Crc32).map_err(S3ServiceError::into_s3_error)?, + checksum_type: "FULL_OBJECT".to_owned(), + }, + }; if is_client_provided { - if let Ok(algo) = ChecksumAlgorithm::from_str(&checksum.algorithm) { - let computed = compute_checksum(algo, &body_data); - if checksum.value != computed { - return Err(S3ServiceError::BadDigest.into_s3_error()); - } + let algo = ChecksumAlgorithm::from_str(&checksum.algorithm) + .map_err(|_| S3ServiceError::BadDigest.into_s3_error())?; + if checksum.value != compute(algo).map_err(S3ServiceError::into_s3_error)? { + return Err(S3ServiceError::BadDigest.into_s3_error()); } } + // Publish only after every checksum has passed. Failed uploads preserve old data. + let write_result = if let Some(upload) = upload { + self.storage + .write_staged_object(&bucket_name, &key, &version_id, upload) + } else { + self.storage + .write_object(&bucket_name, &key, &version_id, body_data) + .await + .map_err(S3ServiceError::into_s3_error)? + }; + // Build the S3Object. let owner = InternalOwner::default(); let obj = S3Object { @@ -210,11 +290,30 @@ impl RustackS3 { } /// Get (download) an object. - #[allow(clippy::too_many_lines)] pub async fn handle_get_object( &self, input: GetObjectInput, ) -> Result { + self.get_object_body(input, false) + .await + .map(|(output, _)| output) + } + + /// Get validated response metadata plus a bounded streaming file when available. + pub async fn handle_get_object_streaming( + &self, + input: GetObjectInput, + ) -> Result<(GetObjectOutput, Option), S3Error> { + self.get_object_body(input, true).await + } + + // Metadata projection retains the existing GetObject operation's field-by-field mapping. + #[allow(clippy::too_many_lines)] + async fn get_object_body( + &self, + input: GetObjectInput, + streaming: bool, + ) -> Result<(GetObjectOutput, Option), S3Error> { let bucket_name = input.bucket; let key = input.key; let version_id_param = input.version_id; @@ -251,27 +350,7 @@ impl RustackS3 { .map_err(S3ServiceError::into_s3_error)?; let store = bucket.objects.read(); - let obj = if let Some(ref version_id) = version_id_param { - store.get_version(&key, version_id).ok_or_else(|| { - // Check if the version is a delete marker. - if store.is_delete_marker(&key, version_id) { - S3ServiceError::MethodNotAllowed - .into_s3_error() - .with_header("x-amz-delete-marker", "true") - .with_header("x-amz-version-id", version_id.clone()) - } else { - S3ServiceError::NoSuchVersion { - key: key.clone(), - version_id: version_id.clone(), - } - .into_s3_error() - } - })? - } else { - store - .get(&key) - .ok_or_else(|| S3ServiceError::NoSuchKey { key: key.clone() }.into_s3_error())? - }; + let obj = lookup_object(&store, &key, version_id_param.as_deref())?; // Conditional request checks. if let Some(ref if_match) = if_match_param { @@ -313,17 +392,37 @@ impl RustackS3 { None }; - // Read data from storage. - let data = self - .storage - .read_object(&bucket_name, &key, &version_for_storage, range) - .await - .map_err(|e| S3ServiceError::Internal(anyhow::anyhow!("{e}")).into_s3_error())?; - - let content_length = data.len() as i64; - - // Build the streaming body from the data bytes. - let body = StreamingBlob::new(data); + let staged = if streaming { + self.storage + .staged_object(&bucket_name, &key, &version_for_storage) + .map(|upload| { + let (offset, length) = range.map_or((0, obj_size), |(start, end)| { + (start, end.saturating_sub(start).saturating_add(1)) + }); + crate::storage::StagedRead { + upload, + offset, + length, + } + }) + } else { + None + }; + let (body, content_length) = if let Some(ref staged) = staged { + ( + None, + i64::try_from(staged.length) + .map_err(|_| S3ServiceError::InvalidRange.into_s3_error())?, + ) + } else { + let data = self + .storage + .read_object(&bucket_name, &key, &version_for_storage, range) + .await + .map_err(S3ServiceError::into_s3_error)?; + let length = data.len() as i64; + (Some(StreamingBlob::new(data)), length) + }; let content_range = range.map(|(start, end)| format!("bytes {start}-{end}/{obj_size}")); @@ -354,7 +453,7 @@ impl RustackS3 { }; let output = GetObjectOutput { accept_ranges: Some("bytes".to_owned()), - body: Some(body), + body, cache_control: override_cache_control.or(obj_meta.cache_control), checksum_crc32: cksum.as_ref().and_then(|c| c.crc32.clone()), checksum_crc32c: cksum.as_ref().and_then(|c| c.crc32c.clone()), @@ -393,7 +492,7 @@ impl RustackS3 { version_id: obj_version_id, ..GetObjectOutput::default() }; - Ok(output) + Ok((output, staged)) } /// Head object (get metadata without body). @@ -421,26 +520,7 @@ impl RustackS3 { .map_err(S3ServiceError::into_s3_error)?; let store = bucket.objects.read(); - let obj = if let Some(ref version_id) = version_id_param { - store.get_version(&key, version_id).ok_or_else(|| { - if store.is_delete_marker(&key, version_id) { - S3ServiceError::MethodNotAllowed - .into_s3_error() - .with_header("x-amz-delete-marker", "true") - .with_header("x-amz-version-id", version_id.clone()) - } else { - S3ServiceError::NoSuchVersion { - key: key.clone(), - version_id: version_id.clone(), - } - .into_s3_error() - } - })? - } else { - store - .get(&key) - .ok_or_else(|| S3ServiceError::NoSuchKey { key: key.clone() }.into_s3_error())? - }; + let obj = lookup_object(&store, &key, version_id_param.as_deref())?; let obj_version_id = if obj.version_id == "null" { None @@ -992,6 +1072,52 @@ fn extract_checksum_from_put( mod tests { use super::*; + #[tokio::test] + async fn test_should_preserve_existing_object_after_bad_staged_checksum() { + use std::sync::Arc; + + use crate::{config::S3Config, storage::UploadWriter}; + let provider = RustackS3::new(S3Config::default()); + provider + .handle_create_bucket(rustack_s3_model::input::CreateBucketInput { + bucket: "stream-test".to_owned(), + ..Default::default() + }) + .await + .unwrap(); + let input = PutObjectInput { + bucket: "stream-test".to_owned(), + key: "key".to_owned(), + body: Some(StreamingBlob::new(Bytes::from_static(b"original"))), + ..Default::default() + }; + provider.handle_put_object(input).await.unwrap(); + let mut writer = UploadWriter::new().await.unwrap(); + writer.write(b"modified").await.unwrap(); + let upload = Arc::new(writer.finish().await.unwrap()); + let input = PutObjectInput { + bucket: "stream-test".to_owned(), + key: "key".to_owned(), + checksum_crc32: Some("AAAAAA==".to_owned()), + ..Default::default() + }; + assert!( + provider + .handle_put_object_staged(input, upload) + .await + .is_err() + ); + let output = provider + .handle_get_object(GetObjectInput { + bucket: "stream-test".to_owned(), + key: "key".to_owned(), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(output.body.unwrap().data.as_ref(), b"original"); + } + #[test] fn test_should_parse_copy_source_simple() { let (bucket, key, vid) = parse_copy_source("my-bucket/my-key").unwrap(); diff --git a/crates/rustack-s3-core/src/ops/object_config.rs b/crates/rustack-s3-core/src/ops/object_config.rs index f88e0cc..cff9107 100644 --- a/crates/rustack-s3-core/src/ops/object_config.rs +++ b/crates/rustack-s3-core/src/ops/object_config.rs @@ -31,11 +31,14 @@ use crate::{error::S3ServiceError, provider::RustackS3, state::object::CannedAcl // AWS S3 DTOs use signed integers (i32/i64) for inherently non-negative values. // These handler methods must remain async for consistency. +// Keep handlers lazy and uniformly awaitable at the dispatch boundary, including +// in-memory operations that currently complete without yielding. #[allow( clippy::cast_possible_wrap, clippy::cast_possible_truncation, clippy::cast_sign_loss, - clippy::unused_async + clippy::unused_async, + clippy::unused_async_trait_impl )] impl RustackS3 { // ----------------------------------------------------------------------- diff --git a/crates/rustack-s3-core/src/provider.rs b/crates/rustack-s3-core/src/provider.rs index 555bf70..4be6559 100644 --- a/crates/rustack-s3-core/src/provider.rs +++ b/crates/rustack-s3-core/src/provider.rs @@ -96,7 +96,7 @@ mod tests { #[test] fn test_should_create_provider_with_defaults() { let provider = RustackS3::new(S3Config::default()); - assert_eq!(provider.config().gateway_listen, "0.0.0.0:4566"); + assert_eq!(provider.config().gateway_listen, "127.0.0.1:4566"); assert!(provider.state().list_buckets().is_empty()); } diff --git a/crates/rustack-s3-core/src/state/keystore.rs b/crates/rustack-s3-core/src/state/keystore.rs index e9997a3..ec5e2d6 100644 --- a/crates/rustack-s3-core/src/state/keystore.rs +++ b/crates/rustack-s3-core/src/state/keystore.rs @@ -75,6 +75,26 @@ impl Default for ObjectStore { } impl ObjectStore { + /// Resolve a delete marker for the current or explicitly requested version. + pub(crate) fn delete_marker( + &self, + key: &str, + version_id: Option<&str>, + ) -> Option<&S3DeleteMarker> { + let Self::Versioned(store) = self else { + return None; + }; + let versions = store.objects.get(key)?; + let version = match version_id { + Some(id) => versions.iter().find(|version| version.version_id() == id), + None => versions.first(), + }?; + match version { + ObjectVersion::DeleteMarker(marker) => Some(marker), + ObjectVersion::Object(_) => None, + } + } + /// Store an object. Returns the previous object for un-versioned stores. pub fn put(&mut self, object: S3Object) -> Option { match self { diff --git a/crates/rustack-s3-core/src/storage.rs b/crates/rustack-s3-core/src/storage.rs index a8d999c..ebae268 100644 --- a/crates/rustack-s3-core/src/storage.rs +++ b/crates/rustack-s3-core/src/storage.rs @@ -13,15 +13,200 @@ //! is automatically cleaned up when the entry is removed from the map (via //! the internal stored data `Drop` implementation). -use std::path::PathBuf; +use std::{ + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; -use bytes::{Bytes, BytesMut}; +use bytes::Bytes; use dashmap::DashMap; -use tokio::io::AsyncReadExt as _; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tracing::{debug, trace, warn}; use crate::{checksums, error::S3ServiceError}; +/// Immutable, unpublished upload data; the temporary file is removed on last drop. +#[derive(Debug)] +pub struct StagedUpload { + path: tempfile::TempPath, + size: u64, + hashes: checksums::HasherResult, +} + +impl StagedUpload { + /// Open the immutable upload for bounded reads. + pub async fn open(&self) -> Result { + tokio::time::timeout(Duration::from_secs(5), tokio::fs::File::open(&self.path)).await? + } + + /// Size of decoded object bytes. + #[must_use] + pub fn size(&self) -> u64 { + self.size + } + + /// Incrementally computed checksums over decoded object bytes. + #[must_use] + pub fn hashes(&self) -> &checksums::HasherResult { + &self.hashes + } + + /// Return a checksum computed while staging the object. + pub fn checksum( + &self, + algorithm: checksums::ChecksumAlgorithm, + ) -> Result<&str, S3ServiceError> { + self.hashes + .checksums + .iter() + .find(|value| value.algorithm == algorithm) + .map(|value| value.value.as_str()) + .ok_or_else(|| S3ServiceError::Internal(anyhow::anyhow!("missing staged checksum"))) + } + + /// Validate an optional Content-MD5 declaration against staged bytes. + pub fn validate_md5(&self, declared: Option<&str>) -> Result<(), S3ServiceError> { + use base64::Engine as _; + if let Some(declared) = declared { + let bytes = base64::engine::general_purpose::STANDARD + .decode(declared) + .map_err(|_| S3ServiceError::BadDigest)?; + if hex::encode(bytes) != self.hashes.md5_hex { + return Err(S3ServiceError::BadDigest); + } + } + Ok(()) + } + + /// Private generated file location, never derived from an S3 key. + #[must_use] + pub fn path(&self) -> &Path { + &self.path + } +} + +/// Immutable file and validated byte range for a streaming download. +#[derive(Debug, Clone)] +pub struct StagedRead { + /// Keep the immutable file alive until the response finishes. + pub upload: Arc, + /// First byte of the selected range. + pub offset: u64, + /// Number of bytes to deliver. + pub length: u64, +} + +impl StagedRead { + /// Read a proxy-sized object range with a pre-allocation cap and IO deadline. + /// # Errors + /// Rejects oversized ranges before opening the file and propagates read errors. + pub async fn collect_bounded(&self, limit: u64) -> Result { + if self.length > limit + || self + .offset + .checked_add(self.length) + .is_none_or(|end| end > self.upload.size()) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "object range exceeds proxy budget", + )); + } + let read = async { + use tokio::io::AsyncSeekExt as _; + let mut file = self.upload.open().await?; + file.seek(std::io::SeekFrom::Start(self.offset)).await?; + let mut remaining = self.length; + let mut data = Vec::new(); + let mut buffer = vec![0; 64 * 1024]; + while remaining != 0 { + let max = usize::try_from(remaining.min(buffer.len() as u64)) + .map_err(std::io::Error::other)?; + let target = buffer + .get_mut(..max) + .ok_or_else(|| std::io::Error::other("invalid read length"))?; + let count = file.read(target).await?; + if count == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "truncated stored object", + )); + } + rustack_core::http::append_bounded( + &mut data, + target + .get(..count) + .ok_or_else(|| std::io::Error::other("invalid read count"))?, + limit, + ) + .map_err(std::io::Error::other)?; + remaining = remaining.saturating_sub(count as u64); + } + Ok(Bytes::from(data)) + }; + tokio::time::timeout( + Duration::from_secs(rustack_core::settings::budgets().body_total_seconds), + read, + ) + .await? + } +} + +/// Bounded-memory staging writer. Dropping before finish removes unpublished data. +#[derive(Debug)] +pub struct UploadWriter { + file: tokio::fs::File, + path: tempfile::TempPath, + size: u64, + hasher: checksums::StreamingHasher, +} + +impl UploadWriter { + /// Create a private staging file outside the object namespace. + pub async fn new() -> Result { + let temporary = tokio::task::spawn_blocking(tempfile::NamedTempFile::new).await??; + let (file, path) = temporary.into_parts(); + Ok(Self { + file: tokio::fs::File::from_std(file), + path, + size: 0, + hasher: checksums::StreamingHasher::new(&[ + checksums::ChecksumAlgorithm::Crc32, + checksums::ChecksumAlgorithm::Crc32c, + checksums::ChecksumAlgorithm::Crc64Nvme, + checksums::ChecksumAlgorithm::Sha1, + checksums::ChecksumAlgorithm::Sha256, + ]), + }) + } + + /// Write one decoded frame with independent storage deadline and size enforcement. + pub async fn write(&mut self, data: &[u8]) -> Result<(), std::io::Error> { + let size = self + .size + .checked_add(data.len() as u64) + .filter(|size| *size <= rustack_core::settings::budgets().s3_object_body_bytes) + .ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "S3 object exceeds 5 GiB") + })?; + tokio::time::timeout(Duration::from_secs(5), self.file.write_all(data)).await??; + self.hasher.update(data); + self.size = size; + Ok(()) + } + + /// Flush and freeze the upload; this does not publish it into object storage. + pub async fn finish(mut self) -> Result { + tokio::time::timeout(Duration::from_secs(5), self.file.flush()).await??; + Ok(StagedUpload { + path: self.path, + size: self.size, + hashes: self.hasher.finish(), + }) + } +} + /// Composite key identifying a stored object: `(bucket, key, version_id)`. type StorageKey = (String, String, String); @@ -62,6 +247,8 @@ pub struct WriteResult { /// to a temporary file on disk. When a [`StoredData::OnDisk`] value is /// dropped, the temporary file is removed. enum StoredData { + /// Validated staged data shared with active readers. + Staged(Arc), /// Small objects kept entirely in memory. InMemory { /// The raw object bytes. @@ -79,6 +266,7 @@ enum StoredData { impl std::fmt::Debug for StoredData { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Self::Staged(upload) => f.debug_tuple("Staged").field(&upload.size()).finish(), Self::InMemory { data } => f .debug_struct("InMemory") .field("size", &data.len()) @@ -115,9 +303,71 @@ impl Drop for StoredData { } impl StoredData { + async fn append_to(&self, writer: &mut UploadWriter) -> Result { + let mut hasher = checksums::StreamingHasher::new(&[]); + if let Self::InMemory { data } = self { + writer + .write(data) + .await + .map_err(|error| S3ServiceError::Internal(error.into()))?; + hasher.update(data); + } else { + let mut file = match self { + Self::Staged(upload) => upload.open().await, + Self::OnDisk { path, .. } => { + tokio::time::timeout(Duration::from_secs(5), tokio::fs::File::open(path)) + .await + .map_err(|error| S3ServiceError::Internal(error.into()))? + } + Self::InMemory { .. } => { + return Err(S3ServiceError::Internal(anyhow::anyhow!( + "unexpected storage variant" + ))); + } + } + .map_err(|error| S3ServiceError::Internal(error.into()))?; + let mut buffer = vec![0; 64 * 1024]; + loop { + let length = tokio::time::timeout(Duration::from_secs(5), file.read(&mut buffer)) + .await + .map_err(|error| S3ServiceError::Internal(error.into()))? + .map_err(|error| S3ServiceError::Internal(error.into()))?; + if length == 0 { + break; + } + let data = buffer.get(..length).ok_or_else(|| { + S3ServiceError::Internal(anyhow::anyhow!("invalid file read length")) + })?; + hasher.update(data); + writer + .write(data) + .await + .map_err(|error| S3ServiceError::Internal(error.into()))?; + } + } + Ok(hasher.finish().md5_hex) + } + /// Read the full data from this stored entry. async fn read_all(&self) -> Result { match self { + Self::Staged(upload) => { + if upload.size() > rustack_core::http::UPSTREAM_BODY_LIMIT { + return Err(S3ServiceError::Internal(anyhow::anyhow!( + "large object requires streaming read" + ))); + } + let mut file = upload + .open() + .await + .map_err(|e| S3ServiceError::Internal(e.into()))?; + let mut data = Vec::new(); + tokio::time::timeout(Duration::from_secs(30), file.read_to_end(&mut data)) + .await + .map_err(|e| S3ServiceError::Internal(e.into()))? + .map_err(|e| S3ServiceError::Internal(e.into()))?; + Ok(Bytes::from(data)) + } Self::InMemory { data } => Ok(data.clone()), Self::OnDisk { path, size } => { let mut file = tokio::fs::File::open(path).await.map_err(|e| { @@ -252,6 +502,67 @@ impl InMemoryStorage { }) } + /// Publish a fully validated immutable staged object without copying its bytes. + #[must_use] + pub fn write_staged_object( + &self, + bucket: &str, + key: &str, + version_id: &str, + upload: Arc, + ) -> WriteResult { + let result = WriteResult { + etag: format!("\"{}\"", upload.hashes.md5_hex), + size: upload.size, + md5_hex: upload.hashes.md5_hex.clone(), + }; + self.objects.insert( + (bucket.to_owned(), key.to_owned(), version_id.to_owned()), + StoredData::Staged(upload), + ); + result + } + + /// Publish a fully validated immutable staged multipart part. + #[must_use] + pub fn write_staged_part( + &self, + bucket: &str, + upload_id: &str, + part_number: u32, + upload: Arc, + ) -> WriteResult { + let result = WriteResult { + etag: format!("\"{}\"", upload.hashes.md5_hex), + size: upload.size, + md5_hex: upload.hashes.md5_hex.clone(), + }; + self.parts.insert( + (bucket.to_owned(), upload_id.to_owned(), part_number), + StoredData::Staged(upload), + ); + result + } + + /// Return a shared immutable staged object for streaming HTTP responses. + #[must_use] + pub fn staged_object( + &self, + bucket: &str, + key: &str, + version_id: &str, + ) -> Option> { + self.objects + .get(&(bucket.to_owned(), key.to_owned(), version_id.to_owned())) + .and_then(|entry| { + if let StoredData::Staged(upload) = entry.value() { + Some(Arc::clone(upload)) + } else { + None + } + }) + } + /// Read object data. Returns the full [`Bytes`] for the object. /// /// If `range` is specified as `(start, end)` (inclusive on both ends), @@ -311,6 +622,9 @@ impl InMemoryStorage { dst_key: &str, dst_version_id: &str, ) -> Result { + if let Some(upload) = self.staged_object(src_bucket, src_key, src_version_id) { + return Ok(self.write_staged_object(dst_bucket, dst_key, dst_version_id, upload)); + } let data = self .read_object(src_bucket, src_key, src_version_id, None) .await?; @@ -417,24 +731,30 @@ impl InMemoryStorage { version_id: &str, part_numbers: &[u32], ) -> Result<(WriteResult, Vec), S3ServiceError> { - let mut combined = BytesMut::new(); + let mut writer = UploadWriter::new() + .await + .map_err(|error| S3ServiceError::Internal(error.into()))?; let mut part_md5_hexes = Vec::with_capacity(part_numbers.len()); - for &part_number in part_numbers { - let part_data = self.read_part(bucket, upload_id, part_number).await?; - let md5_hex = checksums::compute_md5(&part_data); - part_md5_hexes.push(md5_hex); - combined.extend_from_slice(&part_data); + let entry = self + .parts + .get(&(bucket.to_owned(), upload_id.to_owned(), part_number)) + .ok_or(S3ServiceError::InvalidPart)?; + part_md5_hexes.push(entry.value().append_to(&mut writer).await?); } - - let combined_bytes = combined.freeze(); - let size = combined_bytes.len() as u64; + let upload = Arc::new( + writer + .finish() + .await + .map_err(|error| S3ServiceError::Internal(error.into()))?, + ); + let size = upload.size(); // Compute composite ETag: MD5-of-concatenated-MD5s with part count suffix. let etag = checksums::compute_multipart_etag(&part_md5_hexes, part_numbers.len()); // Store the assembled object. - let stored = self.store_data(combined_bytes).await?; + let stored = StoredData::Staged(upload); self.objects.insert( (bucket.to_owned(), key.to_owned(), version_id.to_owned()), stored, diff --git a/crates/rustack-s3-http/Cargo.toml b/crates/rustack-s3-http/Cargo.toml index b27414d..4dfbfd3 100644 --- a/crates/rustack-s3-http/Cargo.toml +++ b/crates/rustack-s3-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-s3-http" description = "S3 HTTP routing, request parsing, response serialization, and hyper service for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,8 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } +rustack-s3-core = { workspace = true } rustack-s3-model = { workspace = true } rustack-auth = { workspace = true } rustack-s3-xml = { workspace = true } diff --git a/crates/rustack-s3-http/src/body.rs b/crates/rustack-s3-http/src/body.rs index 991ca6d..3b7bf82 100644 --- a/crates/rustack-s3-http/src/body.rs +++ b/crates/rustack-s3-http/src/body.rs @@ -22,6 +22,8 @@ use http_body_util::Full; /// Implements [`http_body::Body`] so it can be used directly with hyper responses. #[derive(Debug, Default)] pub enum S3ResponseBody { + /// Bounded file-backed object response with byte and time budgets. + Streaming(rustack_core::http::BudgetedBody), /// Buffered body for small responses: XML payloads, error bodies, raw bytes. Buffered(Full), /// Empty body for 204 responses, DELETE confirmations, HEAD responses, etc. @@ -30,6 +32,40 @@ pub enum S3ResponseBody { } impl S3ResponseBody { + /// Open a validated immutable object range without reading it into memory. + /// # Errors + /// Returns file-open/seek or budget validation failures. + pub async fn from_staged( + read: rustack_s3_core::storage::StagedRead, + ) -> Result { + use tokio::io::AsyncSeekExt as _; + let mut file = read.upload.open().await?; + tokio::time::timeout( + std::time::Duration::from_secs(5), + file.seek(std::io::SeekFrom::Start(read.offset)), + ) + .await??; + if read + .offset + .checked_add(read.length) + .is_none_or(|end| end > read.upload.size()) + { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "invalid stored object range", + )); + } + let budget = rustack_core::http::BodyBudget::s3_object(); + let body = FileBody { + file, + remaining: read.length, + upload: read.upload, + }; + Ok(Self::Streaming(rustack_core::http::BudgetedBody::new( + body, budget, + ))) + } + /// Create a buffered body from bytes. #[must_use] pub fn from_bytes(data: impl Into) -> Self { @@ -64,6 +100,7 @@ impl http_body::Body for S3ResponseBody { cx: &mut Context<'_>, ) -> Poll, Self::Error>>> { match self.get_mut() { + Self::Streaming(body) => Pin::new(body).poll_frame(cx).map_err(std::io::Error::other), Self::Buffered(full) => Pin::new(full) .poll_frame(cx) .map_err(|never| match never {}), @@ -73,6 +110,7 @@ impl http_body::Body for S3ResponseBody { fn is_end_stream(&self) -> bool { match self { + Self::Streaming(body) => body.is_end_stream(), Self::Buffered(full) => full.is_end_stream(), Self::Empty => true, } @@ -80,18 +118,101 @@ impl http_body::Body for S3ResponseBody { fn size_hint(&self) -> http_body::SizeHint { match self { + Self::Streaming(body) => body.size_hint(), Self::Buffered(full) => full.size_hint(), Self::Empty => http_body::SizeHint::with_exact(0), } } } +/// Bounded file reader retaining the immutable artifact for the response lifetime. +#[derive(Debug)] +pub struct FileBody { + file: tokio::fs::File, + remaining: u64, + upload: std::sync::Arc, +} + +impl http_body::Body for FileBody { + type Data = Bytes; + type Error = std::io::Error; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, std::io::Error>>> { + use tokio::io::AsyncRead as _; + let this = self.get_mut(); + if this.remaining == 0 { + return Poll::Ready(None); + } + let length = usize::try_from(this.remaining.min(64 * 1024)).map_err(std::io::Error::other); + let length = match length { + Ok(length) => length, + Err(err) => return Poll::Ready(Some(Err(err))), + }; + let mut buffer = vec![0; length]; + let mut read = tokio::io::ReadBuf::new(&mut buffer); + match Pin::new(&mut this.file).poll_read(cx, &mut read) { + Poll::Pending => Poll::Pending, + Poll::Ready(Err(error)) => Poll::Ready(Some(Err(error))), + Poll::Ready(Ok(())) => { + let size = read.filled().len(); + if size == 0 { + return Poll::Ready(Some(Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "truncated stored object", + )))); + } + this.remaining = this.remaining.saturating_sub(size as u64); + buffer.truncate(size); + Poll::Ready(Some(Ok(http_body::Frame::data(Bytes::from(buffer))))) + } + } + } + + fn size_hint(&self) -> http_body::SizeHint { + // Reading the owner also documents why it must remain alive through the last frame. + http_body::SizeHint::with_exact(self.remaining.min(self.upload.size())) + } +} + #[cfg(test)] mod tests { use http_body::Body; use super::*; + #[tokio::test] + async fn test_should_stream_validated_range_in_bounded_frames() { + use std::sync::Arc; + + use http_body_util::BodyExt; + use rustack_s3_core::storage::{StagedRead, UploadWriter}; + let data = vec![42; 200_000]; + let mut writer = UploadWriter::new().await.unwrap(); + writer.write(&data).await.unwrap(); + let upload = Arc::new(writer.finish().await.unwrap()); + let path = upload.path().to_owned(); + let mut body = S3ResponseBody::from_staged(StagedRead { + upload, + offset: 10, + length: 150_000, + }) + .await + .unwrap(); + let mut count = 0; + while let Some(frame) = body.frame().await { + let data = frame.unwrap().into_data().unwrap(); + assert!(data.len() <= 64 * 1024); + assert!(data.iter().all(|byte| *byte == 42)); + count += data.len(); + } + assert_eq!(count, 150_000); + drop(body); + assert!(!path.exists()); + } + #[test] fn test_should_report_empty_body_as_end_of_stream() { let body = S3ResponseBody::empty(); diff --git a/crates/rustack-s3-http/src/dispatch.rs b/crates/rustack-s3-http/src/dispatch.rs index b8a98a3..77c1d00 100644 --- a/crates/rustack-s3-http/src/dispatch.rs +++ b/crates/rustack-s3-http/src/dispatch.rs @@ -30,6 +30,19 @@ use crate::{body::S3ResponseBody, router::RoutingContext}; /// This trait uses `async-trait`-style boxing because it needs to be used /// with `Arc` for dynamic dispatch in the service layer. pub trait S3Handler: Send + Sync + 'static { + /// Handle an authenticated immutable upload without buffering its bytes. + /// + /// Custom handlers must implement this method for PutObject/UploadPart. The default + /// rejects the operation rather than handing an empty body to a buffered handler. + fn handle_staged_upload( + &self, + _parts: http::request::Parts, + _upload: std::sync::Arc, + ctx: RoutingContext, + ) -> Pin, S3Error>> + Send>> { + Box::pin(async move { Err(S3Error::not_implemented(ctx.operation.as_str())) }) + } + /// Handle an S3 operation and produce an HTTP response. /// /// The implementor receives the identified operation, raw HTTP parts, and body, diff --git a/crates/rustack-s3-http/src/lib.rs b/crates/rustack-s3-http/src/lib.rs index 2d7d81f..cddef06 100644 --- a/crates/rustack-s3-http/src/lib.rs +++ b/crates/rustack-s3-http/src/lib.rs @@ -45,12 +45,6 @@ //! // Use `service` with hyper server. //! ``` -// S3Error is a fundamental domain error type used pervasively as Result. -// Its size (128 bytes) is inherent to its fields (code, message, resource, request_id, -// status_code, source, headers). Boxing S3Error in every Result would add indirection -// on the hot path for negligible benefit. -#![allow(clippy::result_large_err)] - pub mod body; pub mod codec; pub mod dispatch; @@ -59,6 +53,7 @@ pub mod request; pub mod response; pub mod router; pub mod service; +mod upload; // Re-export key types for convenience. pub use body::S3ResponseBody; diff --git a/crates/rustack-s3-http/src/response.rs b/crates/rustack-s3-http/src/response.rs index ec4f4b5..f87fc91 100644 --- a/crates/rustack-s3-http/src/response.rs +++ b/crates/rustack-s3-http/src/response.rs @@ -1047,6 +1047,9 @@ pub fn error_to_response(err: &S3Error, request_id: &str) -> http::Response() < 128); + let err = S3Error::no_such_key("folder/a&b") + .with_request_id("request-123") + .with_source(io::Error::other("storage failure")) + .with_header("x-amz-delete-marker", "true"); + assert_eq!(err.resource.as_deref(), Some("folder/a&b")); + assert_eq!(err.request_id.as_deref(), Some("request-123")); + assert_eq!( + err.source().map(ToString::to_string).as_deref(), + Some("storage failure") + ); + + let response = error_to_response(&err, "request-123"); + assert_eq!(response.status(), http::StatusCode::NOT_FOUND); + assert_eq!(response.headers()["x-amz-delete-marker"], "true"); + let body = response + .into_body() + .collect() + .await + .expect("error body") + .to_bytes(); + let xml = String::from_utf8(body.to_vec()).expect("UTF-8 error XML"); + assert!(xml.contains("NoSuchKey")); + assert!(xml.contains("folder/a&b")); + assert!(xml.contains("request-123")); + } + #[test] fn test_should_create_head_bucket_response() { let output = HeadBucketOutput { diff --git a/crates/rustack-s3-http/src/service.rs b/crates/rustack-s3-http/src/service.rs index a07cfb2..8f7649f 100644 --- a/crates/rustack-s3-http/src/service.rs +++ b/crates/rustack-s3-http/src/service.rs @@ -15,7 +15,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::{body::Incoming, service::Service}; use rustack_auth::CredentialProvider; use rustack_s3_model::error::{S3Error, S3ErrorCode}; @@ -157,7 +156,7 @@ async fn process_request( ) -> http::Response { let method = req.method().clone(); let uri = req.uri().clone(); - debug!(%method, %uri, request_id, "processing S3 request"); + debug!(%method, request_id, "processing S3 request"); // 1. Health check interception. if is_health_check(&method, uri.path()) { @@ -180,7 +179,7 @@ async fn process_request( Ok(ctx) => ctx, Err(err) => { warn!( - %method, %uri, error = %err, request_id, + %method, error = %err, request_id, "failed to route S3 request" ); return error_to_response(&err, request_id); @@ -197,12 +196,46 @@ async fn process_request( // 4. Collect body. let (mut parts, incoming) = req.into_parts(); - let mut body = match collect_body(incoming).await { + let auth_mode = match rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) { + Ok(mode) => mode, + Err(err) => { + return error_to_response( + &S3Error::with_message(S3ErrorCode::AccessDenied, err.to_string()), + request_id, + ); + } + }; + if matches!( + ctx.operation, + rustack_s3_model::S3Operation::PutObject | rustack_s3_model::S3Operation::UploadPart + ) { + let upload = match crate::upload::receive(&mut parts, incoming, auth_mode).await { + Ok(upload) => upload, + Err(err) => return error_to_response(&err, request_id), + }; + return match handler.handle_staged_upload(parts, upload, ctx).await { + Ok(response) => response, + Err(err) => error_to_response(&err, request_id), + }; + } + if crate::codec::is_aws_chunked(&parts) { + return error_to_response( + &S3Error::with_message( + S3ErrorCode::InvalidRequest, + "aws-chunked is only supported for object uploads", + ), + request_id, + ); + } + let body = match collect_body(incoming).await { Ok(body) => body, Err(err) => { error!(error = %err, request_id, "failed to collect request body"); - let s3_err = - rustack_s3_model::error::S3Error::internal_error("Failed to read request body"); + let mut s3_err = S3Error::with_message(S3ErrorCode::InvalidRequest, err.to_string()); + s3_err.status_code = err.status_code(); return error_to_response(&s3_err, request_id); } }; @@ -213,74 +246,34 @@ async fn process_request( return error_to_response(&s3_err, request_id); } - // 4c. Decode AWS chunked transfer encoding. - if crate::codec::is_aws_chunked(&parts) { - match crate::codec::decode_aws_chunked(&body) { - Ok(result) => { - debug!( - raw_len = body.len(), - decoded_len = result.body.len(), - trailing_header_count = result.trailing_headers.len(), - request_id, - "decoded aws-chunked body" - ); - body = result.body; - crate::codec::strip_aws_chunked_encoding(&mut parts.headers); - - // Inject trailing headers (e.g. checksum values) into request - // headers so downstream request parsing picks them up. - for (key, value) in &result.trailing_headers { - if let Ok(hv) = http::header::HeaderValue::from_str(value) { - if let Ok(hn) = http::header::HeaderName::from_bytes(key.as_bytes()) { - // Only insert if not already present in the request headers. - parts.headers.entry(hn).or_insert(hv); - } - } - } - } - Err(s3_err) => { - warn!(error = %s3_err.message, request_id, "failed to decode aws-chunked body"); - return error_to_response(&s3_err, request_id); - } - } - } - // 5. Authentication. - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let has_presigned = parts - .uri - .query() - .is_some_and(|q| q.contains("X-Amz-Signature")); - - let auth_result = if has_presigned { - rustack_auth::verify_presigned(&parts, cred_provider.as_ref()) - } else if let Some(auth_header) = parts - .headers - .get("authorization") - .and_then(|v| v.to_str().ok()) - { - if rustack_auth::is_sigv2(auth_header) { - rustack_auth::verify_sigv2(&parts, cred_provider.as_ref()) - } else { - let body_hash = rustack_auth::hash_payload(&body); - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - } + if let rustack_auth::AuthMode::Required(cred_provider) = auth_mode { + let has_presigned = parts + .uri + .query() + .is_some_and(|q| q.contains("X-Amz-Signature")); + + let auth_result = if has_presigned { + rustack_auth::verify_presigned(&parts, cred_provider) + } else if let Some(auth_header) = parts + .headers + .get("authorization") + .and_then(|v| v.to_str().ok()) + { + if rustack_auth::is_sigv2(auth_header) { + rustack_auth::verify_sigv2(&parts, cred_provider) } else { - // Anonymous request — allow through. - Ok(rustack_auth::AuthResult { - access_key_id: String::new(), - region: String::new(), - service: String::new(), - signed_headers: Vec::new(), - }) - }; - - if let Err(auth_err) = auth_result { - warn!(error = %auth_err, request_id, "authentication failed"); - let s3_err = S3Error::with_message(S3ErrorCode::AccessDenied, auth_err.to_string()); - return error_to_response(&s3_err, request_id); + let body_hash = rustack_auth::hash_payload(&body); + rustack_auth::sigv4::verify_s3_sigv4(&parts, &body_hash, cred_provider) } + } else { + Err(rustack_auth::AuthError::MissingAuthHeader) + }; + + if let Err(auth_err) = auth_result { + warn!(error = %auth_err, request_id, "authentication failed"); + let s3_err = S3Error::with_message(S3ErrorCode::AccessDenied, auth_err.to_string()); + return error_to_response(&s3_err, request_id); } } @@ -299,9 +292,8 @@ async fn process_request( } /// Collect the full body from a hyper `Incoming` stream into `Bytes`. -async fn collect_body(incoming: Incoming) -> Result { - let collected = incoming.collect().await?; - Ok(collected.to_bytes()) +async fn collect_body(incoming: Incoming) -> Result { + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()).await } /// Validate the `X-Amz-Content-Sha256` header against the request body. @@ -311,6 +303,12 @@ async fn collect_body(incoming: Incoming) -> Result { /// placeholder), we verify it matches the actual body content. An invalid or /// mismatching value returns `XAmzContentSHA256Mismatch`. fn validate_content_sha256(parts: &http::request::Parts, body: &[u8]) -> Result<(), S3Error> { + if parts.headers.get_all("x-amz-content-sha256").iter().count() > 1 { + return Err(S3Error::with_message( + S3ErrorCode::XAmzContentSHA256Mismatch, + "Duplicate payload hash header", + )); + } let Some(header_value) = parts.headers.get("x-amz-content-sha256") else { return Ok(()); }; @@ -322,14 +320,8 @@ fn validate_content_sha256(parts: &http::request::Parts, body: &[u8]) -> Result< ) })?; - // Skip validation for streaming and unsigned payload placeholders. - // AWS SDKs use various STREAMING-* prefixes (SigV4, SigV4a, CRT-based, - // etc.) and UNSIGNED-PAYLOAD variants. Rather than maintaining an - // exhaustive allowlist, accept any recognised placeholder pattern. - if hash_str == "UNSIGNED-PAYLOAD" - || hash_str.starts_with("STREAMING-") - || hash_str.starts_with("UNSIGNED-PAYLOAD-") - { + // Streaming protocols have a separate object-upload boundary and verifier. + if hash_str == "UNSIGNED-PAYLOAD" { return Ok(()); } @@ -596,33 +588,17 @@ mod tests { } #[test] - fn test_should_accept_streaming_payload() { - let parts = parts_with_sha256("STREAMING-AWS4-HMAC-SHA256-PAYLOAD"); - assert!(validate_content_sha256(&parts, b"hello").is_ok()); - } - - #[test] - fn test_should_accept_streaming_payload_trailer() { - let parts = parts_with_sha256("STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER"); - assert!(validate_content_sha256(&parts, b"hello").is_ok()); - } - - #[test] - fn test_should_accept_streaming_sigv4a_payload() { - let parts = parts_with_sha256("STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD"); - assert!(validate_content_sha256(&parts, b"hello").is_ok()); - } - - #[test] - fn test_should_accept_streaming_unsigned_payload_trailer() { - let parts = parts_with_sha256("STREAMING-UNSIGNED-PAYLOAD-TRAILER"); - assert!(validate_content_sha256(&parts, b"hello").is_ok()); - } - - #[test] - fn test_should_accept_unsigned_payload_trailer() { - let parts = parts_with_sha256("UNSIGNED-PAYLOAD-TRAILER"); - assert!(validate_content_sha256(&parts, b"hello").is_ok()); + fn test_should_reject_streaming_markers_outside_upload_protocol() { + for marker in [ + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER", + "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD", + "STREAMING-UNSIGNED-PAYLOAD-TRAILER", + "UNSIGNED-PAYLOAD-TRAILER", + ] { + let parts = parts_with_sha256(marker); + assert!(validate_content_sha256(&parts, b"hello").is_err()); + } } #[test] diff --git a/crates/rustack-s3-http/src/upload.rs b/crates/rustack-s3-http/src/upload.rs new file mode 100644 index 0000000..29bed56 --- /dev/null +++ b/crates/rustack-s3-http/src/upload.rs @@ -0,0 +1,689 @@ +//! Bounded, authenticated S3 upload staging; publication is performed by the core handler. + +use std::{collections::BTreeMap, sync::Arc}; + +use bytes::{Buf, Bytes}; +use http_body_util::BodyExt; +use rustack_auth::{AuthMode, sigv4::StreamingVerifier}; +use rustack_core::http::{BodyBudget, BudgetedBody, S3_OBJECT_BODY_LIMIT}; +use rustack_s3_core::{ + checksums::ChecksumAlgorithm, + storage::{StagedUpload, UploadWriter}, +}; +use rustack_s3_model::error::{S3Error, S3ErrorCode}; +use sha2::{Digest, Sha256}; + +fn invalid(message: impl std::fmt::Display) -> S3Error { + S3Error::with_message(S3ErrorCode::InvalidRequest, message.to_string()) +} +fn read_error(error: &rustack_core::http::BodyReadError) -> S3Error { + let mut result = invalid(error.to_string()); + result.status_code = error.status_code(); + result +} + +fn denied(error: impl std::fmt::Display) -> S3Error { + S3Error::with_message(S3ErrorCode::AccessDenied, error.to_string()) +} + +/// Stage and authenticate decoded bytes without allocating a complete object. +pub(crate) async fn receive( + parts: &mut http::request::Parts, + incoming: B, + mode: AuthMode<'_>, +) -> Result, S3Error> +where + B: http_body::Body, + B::Error: std::error::Error + Send + Sync + 'static, +{ + for name in [ + "x-amz-content-sha256", + "x-amz-trailer", + "x-amz-decoded-content-length", + "authorization", + ] { + if parts.headers.get_all(name).iter().count() > 1 { + return Err(invalid("duplicate upload integrity header")); + } + } + let marker = parts + .headers + .get("x-amz-content-sha256") + .map(|value| value.to_str()) + .transpose() + .map_err(|_| invalid("invalid payload hash header"))? + .unwrap_or("") + .to_owned(); + let chunked = crate::codec::is_aws_chunked(parts); + let signed_chunks = matches!( + marker.as_str(), + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" | "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" + ); + if chunked && !signed_chunks && marker != "STREAMING-UNSIGNED-PAYLOAD-TRAILER" { + return Err(invalid("unsupported aws-chunked payload protocol")); + } + let mut verifier = match mode { + AuthMode::Required(provider) if signed_chunks => { + Some(StreamingVerifier::new(parts, provider).map_err(denied)?) + } + _ => None, + }; + // Non-chunked strict uploads fail identity checks before any body is staged. + if !chunked { + if let AuthMode::Required(provider) = mode { + fast_fail_strict_auth(parts, provider)?; + } + } + // Encoded framing overhead is bounded independently; decoded object bytes have a 5 GiB cap. + let budget = if chunked { + BodyBudget::s3_encoded() + } else { + BodyBudget::s3_object() + }; + let mut source = Source { + body: BudgetedBody::new(incoming, budget), + pending: Bytes::new(), + }; + let mut writer = UploadWriter::new().await.map_err(invalid)?; + let mut digest = Sha256::new(); + let mut trailers = BTreeMap::new(); + if chunked { + trailers = source + .copy_chunked(&mut writer, &mut digest, verifier.as_mut()) + .await?; + } else { + while let Some(data) = source.next().await? { + digest.update(&data); + writer.write(&data).await.map_err(invalid)?; + } + } + let upload = Arc::new(writer.finish().await.map_err(invalid)?); + let actual_hash = hex::encode(digest.finalize()); + if let Some(value) = parts.headers.get("x-amz-decoded-content-length") { + let declared = value + .to_str() + .map_err(invalid)? + .parse::() + .map_err(invalid)?; + if declared != upload.size() { + return Err(invalid("decoded content length mismatch")); + } + } + if !marker.is_empty() + && !marker.starts_with("STREAMING-") + && marker != "UNSIGNED-PAYLOAD" + && marker != actual_hash + { + return Err(S3Error::with_message( + S3ErrorCode::XAmzContentSHA256Mismatch, + "payload digest mismatch", + )); + } + if marker.starts_with("STREAMING-") && !chunked { + return Err(invalid("streaming marker requires chunk framing")); + } + validate_trailers(parts, &upload, &trailers, verifier.as_mut(), &marker)?; + authenticate(parts, mode, &actual_hash)?; + for (name, value) in trailers { + if name == "x-amz-trailer-signature" { + continue; + } + let name = http::HeaderName::from_bytes(name.as_bytes()).map_err(invalid)?; + let value = http::HeaderValue::from_str(&value).map_err(invalid)?; + parts.headers.insert(name, value); + } + crate::codec::strip_aws_chunked_encoding(&mut parts.headers); + Ok(upload) +} + +/// Reject clearly invalid credentials before a strict-mode upload can spool any +/// bytes to disk: presigned URLs verify fully without a body, and a missing or +/// unknown access key cannot become valid by reading the body. +fn fast_fail_strict_auth( + parts: &http::request::Parts, + provider: &dyn rustack_auth::CredentialProvider, +) -> Result<(), S3Error> { + if parts + .uri + .query() + .is_some_and(|query| query.contains("X-Amz-Signature")) + { + return rustack_auth::verify_presigned(parts, provider) + .map(|_| ()) + .map_err(denied); + } + let authorization = parts + .headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| denied("missing signature"))?; + let access_key = if let Some(rest) = authorization.strip_prefix("AWS4-HMAC-SHA256 Credential=") + { + rest.split_once('/').map(|(key, _)| key) + } else if let Some(rest) = authorization.strip_prefix("AWS ") { + rest.split_once(':').map(|(key, _)| key) + } else { + None + }; + if let Some(key) = access_key { + provider + .get_secret_key(key) + .map_err(|_| denied("invalid access key"))?; + } else if !authorization.starts_with("AWS4-HMAC-SHA256") { + return Err(denied("malformed signature")); + } + Ok(()) +} + +fn authenticate( + parts: &http::request::Parts, + mode: AuthMode<'_>, + actual_hash: &str, +) -> Result<(), S3Error> { + if let AuthMode::Required(provider) = mode { + if parts + .uri + .query() + .is_some_and(|query| query.contains("X-Amz-Signature")) + { + rustack_auth::verify_presigned(parts, provider).map_err(denied)?; + } else if parts + .headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + .is_some_and(rustack_auth::is_sigv2) + { + rustack_auth::verify_sigv2(parts, provider).map_err(denied)?; + } else { + rustack_auth::sigv4::verify_s3_sigv4(parts, actual_hash, provider).map_err(denied)?; + } + } + Ok(()) +} + +fn validate_trailers( + parts: &http::request::Parts, + upload: &StagedUpload, + trailers: &BTreeMap, + verifier: Option<&mut StreamingVerifier>, + marker: &str, +) -> Result<(), S3Error> { + let declared = parts + .headers + .get("x-amz-trailer") + .map(|value| value.to_str()) + .transpose() + .map_err(invalid)? + .unwrap_or(""); + let mut names: Vec<&str> = declared + .split(',') + .filter(|name| !name.is_empty()) + .collect(); + names.sort_unstable(); + if names.windows(2).any(|pair| pair.first() == pair.get(1)) { + return Err(invalid("duplicate declared trailer")); + } + if marker.ends_with("-TRAILER") && names.is_empty() { + return Err(invalid("missing declared checksum trailer")); + } + let mut canonical = String::new(); + for name in &names { + let algorithm = match *name { + "x-amz-checksum-crc32" => ChecksumAlgorithm::Crc32, + "x-amz-checksum-crc32c" => ChecksumAlgorithm::Crc32c, + "x-amz-checksum-crc64nvme" => ChecksumAlgorithm::Crc64Nvme, + "x-amz-checksum-sha1" => ChecksumAlgorithm::Sha1, + "x-amz-checksum-sha256" => ChecksumAlgorithm::Sha256, + _ => return Err(invalid("invalid declared checksum trailer")), + }; + let value = trailers + .get(*name) + .ok_or_else(|| invalid("missing checksum trailer"))?; + if upload.checksum(algorithm).map_err(invalid)? != value { + return Err(S3Error::with_message( + S3ErrorCode::BadDigest, + "trailer checksum mismatch", + )); + } + if parts.headers.contains_key(*name) { + return Err(invalid("checksum duplicated in header and trailer")); + } + canonical.push_str(name); + canonical.push(':'); + canonical.push_str(value); + canonical.push('\n'); + } + if trailers + .keys() + .any(|name| name != "x-amz-trailer-signature" && !names.contains(&name.as_str())) + { + return Err(invalid("undeclared trailer")); + } + if marker == "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" { + let signature = trailers + .get("x-amz-trailer-signature") + .ok_or_else(|| denied("missing trailer signature"))?; + if let Some(verifier) = verifier { + verifier + .verify_trailer(&canonical, signature) + .map_err(denied)?; + } + } else if trailers.contains_key("x-amz-trailer-signature") { + return Err(invalid("unexpected trailer signature")); + } + Ok(()) +} + +struct Source { + body: BudgetedBody, + pending: Bytes, +} + +impl Source +where + B: http_body::Body, + B::Error: std::error::Error + Send + Sync + 'static, +{ + async fn copy_chunked( + &mut self, + writer: &mut UploadWriter, + digest: &mut Sha256, + mut verifier: Option<&mut StreamingVerifier>, + ) -> Result, S3Error> { + loop { + let line = self.line().await?; + let (size_text, signature) = match line.split_once(';') { + Some((size, extension)) => ( + size, + Some( + extension + .strip_prefix("chunk-signature=") + .ok_or_else(|| invalid("invalid chunk extension"))?, + ), + ), + None => (line.as_str(), None), + }; + if size_text.is_empty() + || size_text.len() > 16 + || !size_text.bytes().all(|b| b.is_ascii_hexdigit()) + { + return Err(invalid("invalid chunk size")); + } + let size = u64::from_str_radix(size_text, 16).map_err(invalid)?; + if size > S3_OBJECT_BODY_LIMIT { + return Err(invalid("chunk exceeds object budget")); + } + let mut remaining = size; + let mut chunk_digest = Sha256::new(); + while remaining != 0 { + let data = self.take(remaining).await?; + remaining = remaining + .checked_sub(data.len() as u64) + .ok_or_else(|| invalid("chunk length overflow"))?; + chunk_digest.update(&data); + digest.update(&data); + writer.write(&data).await.map_err(invalid)?; + } + if let Some(verifier) = verifier.as_mut() { + verifier + .verify_chunk( + &hex::encode(chunk_digest.finalize()), + signature.ok_or_else(|| denied("missing chunk signature"))?, + ) + .map_err(denied)?; + } + if size == 0 { + return self.trailers().await; + } + if !self.line().await?.is_empty() { + return Err(invalid("missing chunk data terminator")); + } + } + } + + async fn trailers(&mut self) -> Result, S3Error> { + let mut trailers = BTreeMap::new(); + loop { + let line = self.line().await?; + if line.is_empty() { + break; + } + if trailers.len() >= 8 { + return Err(invalid("too many trailing headers")); + } + let (name, value) = line + .split_once(':') + .ok_or_else(|| invalid("invalid trailer"))?; + let name = name.to_ascii_lowercase(); + if !matches!( + name.as_str(), + "x-amz-checksum-crc32" + | "x-amz-checksum-crc32c" + | "x-amz-checksum-crc64nvme" + | "x-amz-checksum-sha1" + | "x-amz-checksum-sha256" + | "x-amz-trailer-signature" + ) { + return Err(invalid("unsupported trailer")); + } + if trailers.insert(name, value.trim().to_owned()).is_some() { + return Err(invalid("duplicate trailer")); + } + } + self.finish().await?; + Ok(trailers) + } + + async fn next(&mut self) -> Result, S3Error> { + if !self.pending.is_empty() { + return Ok(Some(std::mem::take(&mut self.pending))); + } + while let Some(frame) = self.body.frame().await { + let frame = frame.map_err(|error| read_error(&error))?; + match frame.into_data() { + Ok(data) if !data.is_empty() => return Ok(Some(data)), + Ok(_) => {} + Err(_) => return Err(invalid("HTTP trailers are not aws-chunked trailers")), + } + } + Ok(None) + } + + async fn take(&mut self, max: u64) -> Result { + let mut data = self + .next() + .await? + .ok_or_else(|| invalid("truncated upload"))?; + let length = usize::try_from(max).unwrap_or(usize::MAX).min(data.len()); + let result = data.split_to(length); + self.pending = data; + Ok(result) + } + + async fn line(&mut self) -> Result { + let mut line = Vec::new(); + loop { + let mut data = self + .next() + .await? + .ok_or_else(|| invalid("truncated chunk framing"))?; + while let Some(byte) = data.first().copied() { + data.advance(1); + line.push(byte); + if line.len() > 8192 { + return Err(invalid("chunk header exceeds 8 KiB")); + } + if line.ends_with(b"\r\n") { + line.truncate(line.len().saturating_sub(2)); + self.pending = data; + return String::from_utf8(line).map_err(invalid); + } + } + } + } + + async fn finish(&mut self) -> Result<(), S3Error> { + if self.next().await?.is_some() { + return Err(invalid("unexpected data after terminal chunk")); + } + Ok(()) + } +} +#[cfg(test)] +mod tests { + use std::convert::Infallible; + + use http_body_util::{Full, StreamBody}; + use rustack_auth::{ + StaticCredentialProvider, + sigv4::{build_string_to_sign, compute_signature, derive_signing_key}, + }; + + use super::*; + + fn parts(marker: &str) -> http::request::Parts { + http::Request::builder() + .method("PUT") + .uri("/bucket/key") + .header("x-amz-content-sha256", marker) + .body(()) + .unwrap() + .into_parts() + .0 + } + + #[tokio::test] + async fn test_should_stage_object_larger_than_control_limit_in_bounded_frames() { + let chunk = Bytes::from(vec![7; 64 * 1024]); + let frames = futures::stream::iter( + (0..272).map(move |_| Ok::<_, Infallible>(http_body::Frame::data(chunk.clone()))), + ); + let mut parts = parts("UNSIGNED-PAYLOAD"); + let upload = receive(&mut parts, StreamBody::new(frames), AuthMode::Development) + .await + .unwrap(); + assert_eq!(upload.size(), 17 * 1024 * 1024); + let path = upload.path().to_owned(); + assert!(path.exists()); + drop(upload); + assert!( + !path.exists(), + "unpublished staged data must be removed on cancellation/drop" + ); + } + + #[tokio::test] + async fn test_should_validate_unsigned_streaming_trailer_and_declared_length() { + let wire = b"5\r\nhello\r\n0\r\nx-amz-checksum-crc32:NhCmhg==\r\n\r\n"; + let mut request = parts("STREAMING-UNSIGNED-PAYLOAD-TRAILER"); + request + .headers + .insert("x-amz-trailer", "x-amz-checksum-crc32".parse().unwrap()); + request + .headers + .insert("x-amz-decoded-content-length", "5".parse().unwrap()); + let upload = receive( + &mut request, + Full::new(Bytes::from_static(wire)), + AuthMode::Development, + ) + .await + .unwrap(); + assert_eq!(upload.size(), 5); + assert_eq!( + upload.checksum(ChecksumAlgorithm::Crc32).unwrap(), + "NhCmhg==" + ); + for bad in [b"5\r\njello\r\n0\r\nx-amz-checksum-crc32:NhCmhg==\r\n\r\n".as_slice(), b"5\r\nhello\r\n0\r\nx-amz-checksum-crc32:NhCmhg==\r\nx-amz-checksum-crc32:NhCmhg==\r\n\r\n", b"5\r\nhello\r\n0\r\nx-amz-checksum-crc32:NhCmhg==\r\n\r\nextra"] { + let mut request = parts("STREAMING-UNSIGNED-PAYLOAD-TRAILER"); + request.headers.insert("x-amz-trailer", "x-amz-checksum-crc32".parse().unwrap()); + assert!(receive(&mut request, Full::new(Bytes::copy_from_slice(bad)), AuthMode::Development).await.is_err()); + } + } + + #[tokio::test] + async fn test_should_reject_invalid_auth_before_staging_any_bytes() { + use std::{ + pin::Pin, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, + task::{Context, Poll}, + }; + + use http_body::Frame; + + struct Counting { + polled: Arc, + data: Bytes, + } + impl http_body::Body for Counting { + type Data = Bytes; + type Error = Infallible; + fn poll_frame( + mut self: Pin<&mut Self>, + _cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + self.polled.fetch_add(1, Ordering::Relaxed); + if self.data.is_empty() { + return Poll::Ready(None); + } + let data = std::mem::take(&mut self.data); + Poll::Ready(Some(Ok(Frame::data(data)))) + } + } + + let provider = StaticCredentialProvider::new(vec![]); + let polled = Arc::new(AtomicUsize::new(0)); + for authorization in [ + None, + Some( + "AWS4-HMAC-SHA256 \ + Credential=UNKNOWN/20260101/us-east-1/s3/aws4_request,SignedHeaders=host,\ + Signature=00", + ), + ] { + let mut request = parts("UNSIGNED-PAYLOAD"); + request.headers.insert("host", "localhost".parse().unwrap()); + if let Some(authorization) = authorization { + request + .headers + .insert("authorization", authorization.parse().unwrap()); + } + let error = receive( + &mut request, + Counting { + polled: Arc::clone(&polled), + data: Bytes::from(vec![7; 64 * 1024]), + }, + AuthMode::Required(&provider), + ) + .await + .unwrap_err(); + assert!( + error.to_string().contains("AccessDenied"), + "unexpected error: {error}" + ); + } + assert_eq!( + polled.load(Ordering::Relaxed), + 0, + "strict-mode rejection must not read any body bytes" + ); + } + + // The signing matrix is long on purpose; fixtures stay readable in one test. + #[allow(clippy::too_many_lines)] + #[tokio::test] + async fn test_should_verify_signed_stream_and_reject_corrupt_chunk_before_publication() { + for marker in [ + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER", + ] { + let provider = + StaticCredentialProvider::new(vec![("key".to_owned(), "secret".to_owned())]); + let key = derive_signing_key("secret", "20260101", "us-east-1", "s3"); + let canonical = rustack_auth::canonical::build_canonical_request( + "PUT", + "/bucket/key", + "", + &[("host", "localhost"), ("x-amz-date", "20260101T000000Z")], + &["host", "x-amz-date"], + marker, + ); + let seed = compute_signature( + &key, + &build_string_to_sign( + "20260101T000000Z", + "20260101/us-east-1/s3/aws4_request", + &rustack_auth::hash_payload(canonical.as_bytes()), + ), + ); + let make_parts = || { + let mut request = parts(marker); + if marker.ends_with("-TRAILER") { + request + .headers + .insert("x-amz-trailer", "x-amz-checksum-crc32".parse().unwrap()); + } + request.headers.insert("host", "localhost".parse().unwrap()); + request + .headers + .insert("x-amz-date", "20260101T000000Z".parse().unwrap()); + request.headers.insert( + "authorization", + format!( + "AWS4-HMAC-SHA256 \ + Credential=key/20260101/us-east-1/s3/aws4_request,SignedHeaders=host;\ + x-amz-date,Signature={seed}" + ) + .parse() + .unwrap(), + ); + request + }; + let sign = |previous: &str, data: &[u8]| { + compute_signature( + &key, + &format!( + "AWS4-HMAC-SHA256-PAYLOAD\n20260101T000000Z\n20260101/us-east-1/s3/\ + aws4_request\n{previous}\n{}\n{}", + rustack_auth::hash_payload(b""), + rustack_auth::hash_payload(data) + ), + ) + }; + let first = sign(&seed, b"hello"); + let terminal = sign(&first, b""); + let trailers = if marker.ends_with("-TRAILER") { + let signature = compute_signature( + &key, + &format!( + "AWS4-HMAC-SHA256-TRAILER\n20260101T000000Z\n20260101/us-east-1/s3/\ + aws4_request\n{terminal}\n{}", + rustack_auth::hash_payload(b"x-amz-checksum-crc32:NhCmhg==\n") + ), + ); + format!("x-amz-checksum-crc32:NhCmhg==\r\nx-amz-trailer-signature:{signature}\r\n") + } else { + String::new() + }; + let mut wire = + format!("5;chunk-signature={first}\r\nhello\r\n0;chunk-signature={terminal}\r\n"); + wire.push_str(&trailers); + wire.push_str("\r\n"); + let upload = receive( + &mut make_parts(), + Full::new(Bytes::from(wire.clone())), + AuthMode::Required(&provider), + ) + .await + .unwrap(); + assert_eq!(upload.size(), 5); + if marker.ends_with("-TRAILER") { + let bad_signature = + wire.replace("x-amz-trailer-signature:", "x-amz-trailer-signature:0"); + assert!( + receive( + &mut make_parts(), + Full::new(Bytes::from(bad_signature)), + AuthMode::Required(&provider) + ) + .await + .is_err() + ); + } + let tampered = wire.replace("hello", "jello"); + assert!( + receive( + &mut make_parts(), + Full::new(Bytes::from(tampered)), + AuthMode::Required(&provider) + ) + .await + .is_err() + ); + } + } +} diff --git a/crates/rustack-s3-model/Cargo.toml b/crates/rustack-s3-model/Cargo.toml index f379084..8b3cc92 100644 --- a/crates/rustack-s3-model/Cargo.toml +++ b/crates/rustack-s3-model/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-s3-model" description = "Rustack S3 data model types and structures" -version.workspace = true +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-s3-model/src/error.rs b/crates/rustack-s3-model/src/error.rs index e89650c..8a349aa 100644 --- a/crates/rustack-s3-model/src/error.rs +++ b/crates/rustack-s3-model/src/error.rs @@ -312,6 +312,9 @@ impl fmt::Display for S3ErrorCode { } /// An S3 error response. +/// +/// Immutable resource and request-ID strings omit spare capacity to keep errors +/// compact when returned by value. #[derive(Debug)] pub struct S3Error { /// The error code. @@ -319,9 +322,9 @@ pub struct S3Error { /// A human-readable error message. pub message: String, /// The resource that caused the error. - pub resource: Option, + pub resource: Option>, /// The request ID. - pub request_id: Option, + pub request_id: Option>, /// The HTTP status code. pub status_code: http::StatusCode, /// The underlying source error, if any. @@ -380,14 +383,14 @@ impl S3Error { /// Set the resource that caused this error. #[must_use] pub fn with_resource(mut self, resource: impl Into) -> Self { - self.resource = Some(resource.into()); + self.resource = Some(resource.into().into_boxed_str()); self } /// Set the request ID. #[must_use] pub fn with_request_id(mut self, request_id: impl Into) -> Self { - self.request_id = Some(request_id.into()); + self.request_id = Some(request_id.into().into_boxed_str()); self } diff --git a/crates/rustack-s3-xml/Cargo.toml b/crates/rustack-s3-xml/Cargo.toml index 1826e69..b96bfad 100644 --- a/crates/rustack-s3-xml/Cargo.toml +++ b/crates/rustack-s3-xml/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-s3-xml" description = "XML serialization/deserialization for Rustack S3" -version.workspace = true +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -12,7 +12,7 @@ workspace = true [dependencies] rustack-s3-model = { workspace = true } -quick-xml = "0.39" +quick-xml = { workspace = true } bytes = "1.11" chrono = { version = "0.4", features = ["serde"] } thiserror = "2.0" diff --git a/crates/rustack-secretsmanager-core/Cargo.toml b/crates/rustack-secretsmanager-core/Cargo.toml index a1c01c8..c1dc887 100644 --- a/crates/rustack-secretsmanager-core/Cargo.toml +++ b/crates/rustack-secretsmanager-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-secretsmanager-core" description = "Secrets Manager business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-secretsmanager-core/src/config.rs b/crates/rustack-secretsmanager-core/src/config.rs index 6638a29..c06109e 100644 --- a/crates/rustack-secretsmanager-core/src/config.rs +++ b/crates/rustack-secretsmanager-core/src/config.rs @@ -1,6 +1,6 @@ //! Secrets Manager service configuration. -use std::env; +use rustack_core::settings as env; /// Secrets Manager service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-secretsmanager-http/Cargo.toml b/crates/rustack-secretsmanager-http/Cargo.toml index 6dedf72..82b0b0f 100644 --- a/crates/rustack-secretsmanager-http/Cargo.toml +++ b/crates/rustack-secretsmanager-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-secretsmanager-http" description = "Secrets Manager HTTP service layer for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-secretsmanager-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-secretsmanager-http/src/service.rs b/crates/rustack-secretsmanager-http/src/service.rs index 5c650a2..207aa32 100644 --- a/crates/rustack-secretsmanager-http/src/service.rs +++ b/crates/rustack-secretsmanager-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_secretsmanager_model::error::{SecretsManagerError, SecretsManagerErrorCode}; @@ -128,19 +127,18 @@ async fn process_request( }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = SecretsManagerError::with_message( - SecretsManagerErrorCode::InvalidRequestException, - auth_err.to_string(), - ); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let mut err = SecretsManagerError::with_message( + SecretsManagerErrorCode::InvalidRequestException, + auth_err.to_string(), + ); + err.status_code = http::StatusCode::FORBIDDEN; + return error_to_response(&err, request_id); } // 5. Dispatch to handler. @@ -152,12 +150,12 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) .map_err(|e| { - SecretsManagerError::internal_error(format!("Failed to read request body: {e}")) + let mut err = SecretsManagerError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err }) } diff --git a/crates/rustack-ses-core/Cargo.toml b/crates/rustack-ses-core/Cargo.toml index f04208a..5320059 100644 --- a/crates/rustack-ses-core/Cargo.toml +++ b/crates/rustack-ses-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-ses-core" description = "SES business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-ses-core/src/config.rs b/crates/rustack-ses-core/src/config.rs index 2953cb2..62072f6 100644 --- a/crates/rustack-ses-core/src/config.rs +++ b/crates/rustack-ses-core/src/config.rs @@ -1,6 +1,6 @@ //! SES service configuration. -use std::env; +use rustack_core::settings as env; /// SES service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-ses-http/Cargo.toml b/crates/rustack-ses-http/Cargo.toml index 44fd9d7..a1aa1c1 100644 --- a/crates/rustack-ses-http/Cargo.toml +++ b/crates/rustack-ses-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-ses-http" description = "SES HTTP service layer for Rustack (awsQuery + restJson1 protocol)" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-ses-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-ses-http/src/service.rs b/crates/rustack-ses-http/src/service.rs index d26b952..7ee0f5b 100644 --- a/crates/rustack-ses-http/src/service.rs +++ b/crates/rustack-ses-http/src/service.rs @@ -6,7 +6,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_ses_model::error::SesError; @@ -138,16 +137,15 @@ async fn process_request( }; // 5. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = SesError::internal_error(auth_err.to_string()); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let mut err = SesError::internal_error(auth_err.to_string()); + err.status_code = http::StatusCode::FORBIDDEN; + return error_to_response(&err, request_id); } // 6. Dispatch to handler (pass raw body so handler can re-parse as needed). @@ -159,11 +157,13 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| SesError::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = SesError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every SES response. diff --git a/crates/rustack-sns-core/Cargo.toml b/crates/rustack-sns-core/Cargo.toml index 8597c4e..b737ae2 100644 --- a/crates/rustack-sns-core/Cargo.toml +++ b/crates/rustack-sns-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-sns-core" description = "SNS business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-sns-core/src/config.rs b/crates/rustack-sns-core/src/config.rs index f7ecc1a..90a5c61 100644 --- a/crates/rustack-sns-core/src/config.rs +++ b/crates/rustack-sns-core/src/config.rs @@ -1,6 +1,6 @@ //! SNS service configuration. -use std::env; +use rustack_core::settings as env; /// SNS service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-sns-core/src/provider.rs b/crates/rustack-sns-core/src/provider.rs index adbc58f..3c28219 100644 --- a/crates/rustack-sns-core/src/provider.rs +++ b/crates/rustack-sns-core/src/provider.rs @@ -11,6 +11,7 @@ use std::{ collections::{HashMap, HashSet}, fmt, sync::{Arc, atomic::AtomicU64}, + time::Duration, }; use dashmap::DashMap; @@ -64,7 +65,7 @@ use crate::{ config::SnsConfig, delivery::{EnvelopeParams, build_sns_envelope}, filter::{evaluate_filter_policy, resolve_protocol_message}, - publisher::SqsPublisher, + publisher::{DeliveryStats, PublishLifecycle, SqsPublisher, send_guarded}, state::TopicStore, subscription::{ FilterPolicyScope, SubscriptionAttributes, SubscriptionProtocol, SubscriptionRecord, @@ -132,6 +133,7 @@ pub struct RustackSns { topics: TopicStore, /// SQS publisher for fan-out delivery. sqs_publisher: Arc, + publish_lifecycle: PublishLifecycle, /// Service configuration. config: Arc, /// Platform applications keyed by ARN. @@ -151,6 +153,7 @@ impl fmt::Debug for RustackSns { f.debug_struct("RustackSns") .field("topics", &self.topics) .field("sqs_publisher", &"") + .field("publish_lifecycle", &self.publish_lifecycle) .field("config", &self.config) .field("platform_apps", &self.platform_apps.len()) .field("platform_endpoints", &self.platform_endpoints.len()) @@ -168,6 +171,7 @@ impl RustackSns { Self { topics: TopicStore::new(), sqs_publisher, + publish_lifecycle: PublishLifecycle::default(), config: Arc::new(config), platform_apps: DashMap::new(), platform_endpoints: DashMap::new(), @@ -177,6 +181,28 @@ impl RustackSns { } } + /// Whether request-bound publication remains open; target failures are diagnostic. + #[must_use] + pub fn is_ready(&self) -> bool { + self.publish_lifecycle.is_ready() + } + + /// Return cumulative target-attempt outcomes, including cancellation failures. + #[must_use] + pub fn delivery_stats(&self) -> DeliveryStats { + self.publish_lifecycle.stats() + } + + /// Stop new Publish operations and wait for admitted fanout without deleting metadata. + pub async fn quiesce(&self) { + self.publish_lifecycle.quiesce().await; + } + + /// Idempotently stop publishing; the runtime supplies its total deadline. + pub async fn shutdown(&self) { + self.quiesce().await; + } + // ---- Topic Management ---- /// Handle `CreateTopic`. @@ -388,7 +414,21 @@ impl RustackSns { .ok_or_else(|| SnsError::not_found("Topic does not exist"))?; let protocol = SubscriptionProtocol::parse(&input.protocol)?; - + if protocol != SubscriptionProtocol::Sqs { + return Err(SnsError::invalid_parameter( + "Unsupported delivery protocol: only SQS subscriptions are executable", + )); + } + if input.attributes.keys().any(|key| { + matches!( + key.as_str(), + "RedrivePolicy" | "DeliveryPolicy" | "SubscriptionRoleArn" + ) + }) { + return Err(SnsError::invalid_parameter( + "Unsupported subscription delivery parameter", + )); + } let endpoint = input.endpoint.unwrap_or_default(); if endpoint.is_empty() { return Err(SnsError::invalid_parameter( @@ -396,6 +436,18 @@ impl RustackSns { )); } + if endpoint + .rsplit_once('.') + .is_some_and(|(_, suffix)| suffix == "fifo") + && !topic.is_fifo + { + return Err(SnsError::invalid_parameter( + "Unsupported delivery: FIFO queues require a FIFO SNS topic", + )); + } + self.sqs_publisher + .validate(&endpoint) + .map_err(|error| SnsError::invalid_parameter(error.to_string()))?; // Idempotent: check for duplicate (same protocol + endpoint). for existing in &topic.subscriptions { if existing.protocol == protocol && existing.endpoint == endpoint { @@ -410,6 +462,11 @@ impl RustackSns { } } + if topic.subscriptions.len() >= 128 { + return Err(SnsError::invalid_parameter( + "At most 128 subscriptions per topic are supported", + )); + } // Parse subscription attributes. let sub_attrs = SubscriptionAttributes::from_input(&input.attributes)?; @@ -740,16 +797,22 @@ impl RustackSns { /// Handle `Publish`. pub async fn publish(&self, input: PublishInput) -> Result { + let _permit = self + .publish_lifecycle + .admit() + .map_err(|error| SnsError::internal_error(error.to_string()))?; let target = resolve_publish_target(&input)?.to_owned(); validate_publish_message(&input)?; - // Direct-to-phone-number publish (SMS stub): no topic lookup needed. - if input.phone_number.is_some() && input.topic_arn.is_none() && input.target_arn.is_none() { - debug!(phone_number = %target, "SMS publish (stub, not delivered)"); - return Ok(PublishOutput { - message_id: Some(uuid::Uuid::new_v4().to_string()), - sequence_number: None, - }); + if input.phone_number.is_some() + || input + .target_arn + .as_ref() + .is_some_and(|arn| arn.contains(":endpoint/")) + { + return Err(SnsError::invalid_parameter( + "Unsupported delivery: SMS and platform push are metadata-only", + )); } let topic_arn = target; @@ -761,6 +824,17 @@ impl RustackSns { .ok_or_else(|| SnsError::not_found("Topic does not exist"))?; validate_fifo_publish(&input, &topic)?; + if !input.message_attributes.is_empty() + && topic + .subscriptions + .iter() + .any(|sub| sub.attributes.raw_message_delivery) + { + return Err(SnsError::invalid_parameter( + "Unsupported delivery: raw SNS message attributes are not forwarded; use envelope \ + delivery", + )); + } let is_fifo = topic.is_fifo; @@ -1521,20 +1595,14 @@ impl RustackSns { is_fifo: bool, ) { for sub in subs { - match sub.protocol { - SubscriptionProtocol::Sqs => { - self.deliver_to_sqs( - sub, input, topic_arn, message_id, region, host, port, is_fifo, - ) - .await; - } - _ => { - debug!( - protocol = %sub.protocol, - endpoint = %sub.endpoint, - "skipping delivery for unsupported protocol" - ); - } + if sub.protocol == SubscriptionProtocol::Sqs { + self.deliver_to_sqs( + sub, input, topic_arn, message_id, region, host, port, is_fifo, + ) + .await; + } else { + let _attempt = self.publish_lifecycle.attempt(); + warn!(protocol = %sub.protocol, "Unsupported SNS delivery protocol"); } } } @@ -1552,6 +1620,7 @@ impl RustackSns { port: u16, is_fifo: bool, ) { + let mut attempt = self.publish_lifecycle.attempt(); // Resolve the effective message for this subscriber's protocol. let effective_message = resolve_effective_message(&input.message, input.message_structure.as_deref(), sub); @@ -1582,28 +1651,39 @@ impl RustackSns { } }; - let group_id = if is_fifo { + // FIFO SNS topics may also fan out to standard SQS queues; only FIFO + // endpoints receive FIFO-only SendMessage parameters. + let fifo_endpoint = is_fifo + && sub + .endpoint + .rsplit_once('.') + .is_some_and(|(_, suffix)| suffix == "fifo"); + let group_id = if fifo_endpoint { input.message_group_id.as_deref() } else { None }; - let dedup_id = if is_fifo { + let dedup_id = if fifo_endpoint { input.message_deduplication_id.as_deref() } else { None }; - if let Err(e) = self - .sqs_publisher - .send_message(&sub.endpoint, &body, group_id, dedup_id) - .await + match tokio::time::timeout( + Duration::from_secs(5), + send_guarded( + self.sqs_publisher.as_ref(), + &sub.endpoint, + &body, + group_id, + dedup_id, + ), + ) + .await { - warn!( - subscription_arn = %sub.arn, - endpoint = %sub.endpoint, - error = %e, - "failed to deliver message to SQS" - ); + Ok(Ok(())) => attempt.delivered(), + Ok(Err(error)) => warn!(error = %error, "SNS target delivery failed"), + Err(_) => warn!("SNS target delivery timed out"), } } } @@ -1707,15 +1787,10 @@ fn apply_subscription_attribute( "RawMessageDelivery" => { sub.attributes.raw_message_delivery = value.eq_ignore_ascii_case("true"); } - "RedrivePolicy" => { - sub.attributes.redrive_policy = if value.is_empty() { None } else { Some(value) }; - } - "DeliveryPolicy" => { - sub.attributes.delivery_policy = if value.is_empty() { None } else { Some(value) }; - } - "SubscriptionRoleArn" => { - sub.attributes.subscription_role_arn = - if value.is_empty() { None } else { Some(value) }; + "RedrivePolicy" | "DeliveryPolicy" | "SubscriptionRoleArn" => { + return Err(SnsError::invalid_parameter( + "Unsupported subscription delivery parameter", + )); } other => { return Err(SnsError::invalid_parameter(format!( @@ -1895,10 +1970,36 @@ fn build_permission_statement( #[cfg(test)] mod tests { use super::*; - use crate::publisher::NoopSqsPublisher; + use crate::publisher::DeliveryError; + + #[derive(Default)] + struct RecordingPublisher(dashmap::DashMap); + #[async_trait::async_trait] + impl SqsPublisher for RecordingPublisher { + fn validate(&self, arn: &str) -> Result<(), DeliveryError> { + if arn.starts_with("arn:aws:sqs:") { + Ok(()) + } else { + Err(DeliveryError::Unsupported("Expected SQS ARN".into())) + } + } + async fn send_message( + &self, + arn: &str, + body: &str, + _: Option<&str>, + _: Option<&str>, + ) -> Result<(), DeliveryError> { + self.0.insert(arn.to_owned(), body.to_owned()); + Ok(()) + } + } fn make_provider() -> RustackSns { - RustackSns::new(SnsConfig::default(), Arc::new(NoopSqsPublisher)) + RustackSns::new( + SnsConfig::default(), + Arc::new(RecordingPublisher::default()), + ) } #[test] @@ -2283,7 +2384,7 @@ mod tests { } #[test] - fn test_should_confirm_subscription() { + fn test_should_reject_unsupported_subscription_protocol_before_creating_metadata() { let provider = make_provider(); let topic = provider .create_topic(CreateTopicInput { @@ -2291,50 +2392,33 @@ mod tests { ..Default::default() }) .unwrap(); - - // HTTP requires confirmation. - let sub = provider - .subscribe(SubscribeInput { - topic_arn: topic.topic_arn.clone(), - protocol: "http".to_owned(), - endpoint: Some("http://example.com/webhook".to_owned()), - return_subscription_arn: true, - ..Default::default() - }) - .unwrap(); - - let sub_arn = sub.subscription_arn.unwrap(); - - // Get subscription and verify it's pending. - let attrs = provider - .get_subscription_attributes(&GetSubscriptionAttributesInput { - subscription_arn: sub_arn.clone(), - }) - .unwrap(); - assert_eq!(attrs.attributes.get("PendingConfirmation").unwrap(), "true"); - - // Confirm with a token. - let confirm_output = provider - .confirm_subscription(&ConfirmSubscriptionInput { - topic_arn: topic.topic_arn.clone(), - token: "any-valid-token".to_owned(), - authenticate_on_unsubscribe: None, - }) - .unwrap(); - assert_eq!( - confirm_output.subscription_arn.as_deref(), - Some(sub_arn.as_str()) - ); - - // Verify it's now confirmed. - let attrs = provider - .get_subscription_attributes(&GetSubscriptionAttributesInput { - subscription_arn: sub_arn, - }) - .unwrap(); - assert_eq!( - attrs.attributes.get("PendingConfirmation").unwrap(), - "false" + for protocol in [ + "http", + "https", + "email", + "email-json", + "sms", + "lambda", + "application", + "firehose", + ] { + let error = provider + .subscribe(SubscribeInput { + topic_arn: topic.topic_arn.clone(), + protocol: protocol.to_owned(), + endpoint: Some("http://example.com/webhook".to_owned()), + ..Default::default() + }) + .unwrap_err(); + assert!(error.message.contains("Unsupported delivery protocol")); + } + assert!( + provider + .topics + .get_topic(&topic.topic_arn) + .unwrap() + .subscriptions + .is_empty() ); } diff --git a/crates/rustack-sns-core/src/publisher.rs b/crates/rustack-sns-core/src/publisher.rs index a4c8995..c6c410a 100644 --- a/crates/rustack-sns-core/src/publisher.rs +++ b/crates/rustack-sns-core/src/publisher.rs @@ -1,41 +1,46 @@ -//! Publisher traits for cross-service message delivery. -//! -//! `SqsPublisher` is defined here (in sns-core) and implemented -//! in the server binary wrapping the actual SQS provider. -//! This keeps sns-core decoupled from sqs-core. -//! -//! Uses `async-trait` because `SqsPublisher` requires object safety -//! (used as `Arc` for dynamic dispatch). +//! Object-safe app-owned SQS publisher and bounded synchronous publish lifecycle. +//! `async-trait` is required because providers hold `Arc`. +use std::{ + future::{Future, poll_fn}, + panic::{AssertUnwindSafe, catch_unwind}, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + task::Poll, +}; use async_trait::async_trait; +use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore}; -/// Error type for message delivery failures. +/// Explicit cross-service delivery failure. #[derive(Debug, thiserror::Error)] pub enum DeliveryError { - /// SQS delivery failed. + /// A target implementation panicked; subsequent subscribers are still attempted. + #[error("Target publisher panicked")] + TargetPanicked, + /// SQS target rejected a delivery. #[error("SQS delivery failed to {queue_arn}: {reason}")] SqsDeliveryFailed { - /// The ARN of the target queue. + /// Target ARN. queue_arn: String, - /// The underlying error message. - reason: String, - }, - /// HTTP delivery failed. - #[error("HTTP delivery failed to {endpoint}: {reason}")] - HttpDeliveryFailed { - /// The HTTP endpoint. - endpoint: String, - /// The underlying error message. + /// Failure reason. reason: String, }, + /// Required service is unavailable. + #[error("Delivery service unavailable: {0}")] + Unavailable(String), + /// Execution capability is not implemented. + #[error("Unsupported delivery: {0}")] + Unsupported(String), } -/// Trait for delivering messages to SQS queues. -/// -/// Implemented by the server binary to bridge SNS fan-out to actual SQS queues. +/// Application bridge to a real SQS provider. #[async_trait] pub trait SqsPublisher: Send + Sync + 'static { - /// Send a message to an SQS queue identified by its ARN. + /// Validate that the ARN can be delivered by this runtime. + fn validate(&self, queue_arn: &str) -> Result<(), DeliveryError>; + /// Deliver one message, preserving FIFO identity. async fn send_message( &self, queue_arn: &str, @@ -45,19 +50,199 @@ pub trait SqsPublisher: Send + Sync + 'static { ) -> Result<(), DeliveryError>; } -/// No-op publisher for testing or when SQS is not available. -#[derive(Debug)] -pub struct NoopSqsPublisher; +/// Isolate each target's future without spawning detached work. After a panic +/// the future is never polled again; its error is counted and fanout continues. +pub(crate) async fn send_guarded( + publisher: &dyn SqsPublisher, + arn: &str, + body: &str, + group: Option<&str>, + dedup: Option<&str>, +) -> Result<(), DeliveryError> { + let delivery = async { publisher.send_message(arn, body, group, dedup).await }; + tokio::pin!(delivery); + poll_fn(|context| { + catch_unwind(AssertUnwindSafe(|| delivery.as_mut().poll(context))) + .unwrap_or(Poll::Ready(Err(DeliveryError::TargetPanicked))) + }) + .await +} +/// Explicit absent SQS dependency; never claims delivery success. +#[derive(Debug)] +pub struct UnavailableSqsPublisher; #[async_trait] -impl SqsPublisher for NoopSqsPublisher { +impl SqsPublisher for UnavailableSqsPublisher { + fn validate(&self, _: &str) -> Result<(), DeliveryError> { + Err(DeliveryError::Unavailable("SQS is not enabled".into())) + } async fn send_message( &self, - _queue_arn: &str, - _message_body: &str, - _message_group_id: Option<&str>, - _message_deduplication_id: Option<&str>, + arn: &str, + _: &str, + _: Option<&str>, + _: Option<&str>, ) -> Result<(), DeliveryError> { - Ok(()) + self.validate(arn) + } +} + +/// Cumulative target-attempt outcomes (not topic Publish counts). +#[derive(Debug, Clone, Copy)] +pub struct DeliveryStats { + /// Accepted target attempts. + pub accepted: u64, + /// Delivered target attempts. + pub delivered: u64, + /// Failed/cancelled target attempts. + pub failed: u64, + /// Publication admission rejections (capacity or shutdown). + pub rejected: u64, +} + +#[derive(Debug)] +pub(crate) struct PublishLifecycle { + permits: Arc, + wake: Arc, + accepted: AtomicU64, + delivered: AtomicU64, + failed: AtomicU64, + rejected: AtomicU64, +} +impl Default for PublishLifecycle { + fn default() -> Self { + Self { + permits: Arc::new(Semaphore::new(128)), + wake: Arc::new(Notify::new()), + accepted: AtomicU64::new(0), + delivered: AtomicU64::new(0), + failed: AtomicU64::new(0), + rejected: AtomicU64::new(0), + } + } +} +impl PublishLifecycle { + pub fn is_ready(&self) -> bool { + !self.permits.is_closed() + } + pub fn admit(&self) -> Result { + let permit = Arc::clone(&self.permits).try_acquire_owned().map_err(|_| { + self.rejected.fetch_add(1, Ordering::Relaxed); + DeliveryError::Unavailable("SNS publish is full or quiescing".into()) + })?; + Ok(PublishPermit { + permit: Some(permit), + wake: Arc::clone(&self.wake), + }) + } + pub fn attempt(&self) -> Attempt<'_> { + self.accepted.fetch_add(1, Ordering::Relaxed); + Attempt { + lifecycle: self, + delivered: false, + } + } + pub fn stats(&self) -> DeliveryStats { + DeliveryStats { + accepted: self.accepted.load(Ordering::Relaxed), + delivered: self.delivered.load(Ordering::Relaxed), + failed: self.failed.load(Ordering::Relaxed), + rejected: self.rejected.load(Ordering::Relaxed), + } + } + pub async fn quiesce(&self) { + self.permits.close(); + loop { + let notified = self.wake.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if self.permits.available_permits() == 128 { + return; + } + notified.await; + } + } +} + +pub(crate) struct PublishPermit { + permit: Option, + wake: Arc, +} +impl Drop for PublishPermit { + fn drop(&mut self) { + drop(self.permit.take()); + self.wake.notify_waiters(); + } +} +pub(crate) struct Attempt<'a> { + lifecycle: &'a PublishLifecycle, + delivered: bool, +} +impl Attempt<'_> { + pub fn delivered(&mut self) { + self.delivered = true; + } +} +impl Drop for Attempt<'_> { + fn drop(&mut self) { + if self.delivered { + self.lifecycle.delivered.fetch_add(1, Ordering::Relaxed); + } else { + self.lifecycle.failed.fetch_add(1, Ordering::Relaxed); + } + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use super::*; + + #[tokio::test] + async fn test_should_bound_publish_and_drain_cancelled_permits() { + let lifecycle = PublishLifecycle::default(); + let permits: Vec<_> = (0..128).map(|_| lifecycle.admit().unwrap()).collect(); + assert!(lifecycle.admit().is_err()); + assert!( + tokio::time::timeout(Duration::from_millis(1), lifecycle.quiesce()) + .await + .is_err() + ); + assert!(lifecycle.admit().is_err()); + drop(permits); + tokio::time::timeout(Duration::from_secs(1), lifecycle.quiesce()) + .await + .unwrap(); + } + + #[test] + fn test_should_record_cancelled_attempt_as_failure() { + let lifecycle = PublishLifecycle::default(); + { + let _attempt = lifecycle.attempt(); + } + { + let mut attempt = lifecycle.attempt(); + attempt.delivered(); + } + assert_eq!(lifecycle.stats().accepted, 2); + assert_eq!(lifecycle.stats().failed, 1); + assert_eq!(lifecycle.stats().delivered, 1); + } + + #[tokio::test] + async fn test_should_never_report_unavailable_publisher_as_success() { + assert!( + UnavailableSqsPublisher + .validate("arn:aws:sqs:us-east-1:000000000000:q") + .is_err() + ); + assert!( + UnavailableSqsPublisher + .send_message("arn", "message", None, None) + .await + .is_err() + ); } } diff --git a/crates/rustack-sns-http/Cargo.toml b/crates/rustack-sns-http/Cargo.toml index 64303bd..5add048 100644 --- a/crates/rustack-sns-http/Cargo.toml +++ b/crates/rustack-sns-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-sns-http" description = "SNS HTTP service layer for Rustack (awsQuery protocol)" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-sns-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-sns-http/src/service.rs b/crates/rustack-sns-http/src/service.rs index 7ccf529..db0df02 100644 --- a/crates/rustack-sns-http/src/service.rs +++ b/crates/rustack-sns-http/src/service.rs @@ -8,7 +8,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_sns_model::error::SnsError; @@ -128,7 +127,12 @@ async fn process_request( // 2. Collect body. let body = match collect_body(incoming).await { Ok(body) => body, - Err(err) => return error_to_response(&err, request_id), + Err(err) => { + let mut response = + error_to_response(&SnsError::invalid_parameter(err.to_string()), request_id); + *response.status_mut() = err.status_code(); + return response; + } }; // 3. Parse form params to extract Action for routing. @@ -141,16 +145,14 @@ async fn process_request( }; // 5. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = SnsError::invalid_security(auth_err.to_string()); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let err = SnsError::invalid_security(auth_err.to_string()); + return error_to_response(&err, request_id); } // 6. Dispatch to handler (pass raw body so handler can re-parse as needed). @@ -161,12 +163,8 @@ async fn process_request( } /// Collect the incoming body into a single `Bytes` buffer. -async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() - .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| SnsError::internal_error(format!("Failed to read request body: {e}"))) +async fn collect_body(incoming: Incoming) -> Result { + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()).await } /// Add common response headers to every SNS response. diff --git a/crates/rustack-sqs-core/Cargo.toml b/crates/rustack-sqs-core/Cargo.toml index a84ba1d..49e65fe 100644 --- a/crates/rustack-sqs-core/Cargo.toml +++ b/crates/rustack-sqs-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-sqs-core" description = "SQS business logic and queue engine for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-sqs-core/src/config.rs b/crates/rustack-sqs-core/src/config.rs index 60c85eb..5677c90 100644 --- a/crates/rustack-sqs-core/src/config.rs +++ b/crates/rustack-sqs-core/src/config.rs @@ -1,6 +1,6 @@ //! SQS service configuration. -use std::env; +use rustack_core::settings as env; /// SQS service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-sqs-core/src/message.rs b/crates/rustack-sqs-core/src/message.rs index 5929cfe..248c0b9 100644 --- a/crates/rustack-sqs-core/src/message.rs +++ b/crates/rustack-sqs-core/src/message.rs @@ -32,6 +32,8 @@ pub struct QueueMessage { pub message_group_id: Option, /// FIFO-only: message deduplication ID. pub message_deduplication_id: Option, + /// ARN of the queue from which this message was dead-lettered. + pub dead_letter_queue_source_arn: Option, /// When this message becomes available (for delayed messages). pub available_at: tokio::time::Instant, /// Per-message delay in seconds (0 = no delay). diff --git a/crates/rustack-sqs-core/src/provider.rs b/crates/rustack-sqs-core/src/provider.rs index 877c21e..7a8b37a 100644 --- a/crates/rustack-sqs-core/src/provider.rs +++ b/crates/rustack-sqs-core/src/provider.rs @@ -5,7 +5,10 @@ use std::{ collections::HashMap, - sync::{Arc, atomic::AtomicBool}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, }; use dashmap::DashMap; @@ -31,7 +34,7 @@ use rustack_sqs_model::{ }, }; use serde::{Deserialize, Serialize}; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, watch}; use crate::{ config::SqsConfig, @@ -43,13 +46,18 @@ use crate::{ }, }; +#[cfg(test)] +#[path = "redrive_tests.rs"] +mod redrive_tests; + /// Main SQS provider. Acts as the queue manager that owns all queue actors. #[derive(Debug)] pub struct RustackSqs { /// Queue registry: queue_name -> QueueHandle. - queues: DashMap>, + queues: Arc>>, /// Configuration. config: Arc, + quiescing: AtomicBool, } /// Serializable SQS provider snapshot. @@ -77,9 +85,47 @@ impl RustackSqs { #[must_use] pub fn new(config: SqsConfig) -> Self { Self { - queues: DashMap::new(), + queues: Arc::new(DashMap::new()), config: Arc::new(config), + quiescing: AtomicBool::new(false), + } + } + + /// Whether the provider and every currently registered queue actor are healthy. + #[must_use] + pub fn is_ready(&self) -> bool { + !self.quiescing.load(Ordering::Acquire) + && self.queues.iter().all(|queue| { + !queue.shutdown.load(Ordering::Acquire) + && !queue.sender.is_closed() + && queue.completion.borrow().is_none() + && queue.completion.has_changed().is_ok() + }) + } + + /// Resolve an exact local-account/region SQS ARN to its configured queue URL. + /// + /// # Errors + /// Rejects malformed, foreign-account, foreign-region and unsupported ARNs. + pub fn queue_url_for_arn(&self, arn: &str) -> Result { + let parts: Vec<_> = arn.split(':').collect(); + let ["arn", "aws", "sqs", region, account, name] = parts.as_slice() else { + return Err(SqsError::invalid_parameter_value( + "Expected an SQS queue ARN", + )); + }; + if *region != self.config.default_region || *account != self.config.account_id { + return Err(SqsError::invalid_parameter_value( + "SQS queue ARN must use the local account and region", + )); } + validate_queue_name(name)?; + let host = if self.config.host.contains(':') && !self.config.host.starts_with('[') { + format!("[{}]", self.config.host) + } else { + self.config.host.clone() + }; + Ok(queue_url(&host, self.config.port, account, name)) } /// Resolve a queue name from a queue URL. @@ -161,11 +207,30 @@ impl RustackSqs { }) .await?; } + self.quiescing.store(false, Ordering::Release); + Ok(()) + } + + /// Quiesce DLQ handoffs without deleting queues or messages. + /// + /// # Errors + /// Returns an error if an actor cannot acknowledge its consistency barrier. + pub async fn quiesce(&self) -> Result<(), SqsError> { + self.quiescing.store(true, Ordering::Release); + let handles: Vec<_> = self + .queues + .iter() + .map(|entry| Arc::clone(entry.value())) + .collect(); + for handle in handles { + handle.quiesce().await?; + } Ok(()) } /// Shut down all queue actors and clear the registry. pub async fn shutdown_all(&self) { + self.quiescing.store(true, Ordering::Release); let handles: Vec> = self .queues .iter() @@ -267,8 +332,17 @@ impl RustackSqs { input.tags, self.config.account_id.clone(), now, - ); - let task = tokio::spawn(actor.run()); + ) + .with_routes(Arc::downgrade(&self.queues)); + let (finished, completion) = watch::channel(None); + // One supervisor per queue observes actor panics and makes shutdown joinable. + tokio::spawn(async move { + let result = tokio::spawn(actor.run()).await; + if let Err(error) = &result { + tracing::error!(error = %error, "SQS queue actor failed"); + } + let _ = finished.send(Some(result.is_ok())); + }); let handle = Arc::new(QueueHandle { sender, @@ -279,7 +353,7 @@ impl RustackSqs { is_fifo, created_at: now, }, - task, + completion, shutdown: Arc::new(AtomicBool::new(false)), }); @@ -306,7 +380,11 @@ impl RustackSqs { } /// Handle `GetQueueUrl`. - #[allow(clippy::unused_async)] // Must be async to match the handler trait interface. + #[allow( + clippy::unused_async, + clippy::unused_async_trait_impl, + reason = "Keep the provider API lazy and awaitable like the async handler interface" + )] pub async fn get_queue_url( &self, input: GetQueueUrlInput, @@ -322,7 +400,11 @@ impl RustackSqs { } /// Handle `ListQueues`. - #[allow(clippy::unused_async)] // Must be async to match the handler trait interface. + #[allow( + clippy::unused_async, + clippy::unused_async_trait_impl, + reason = "Keep the provider API lazy and awaitable like the async handler interface" + )] #[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)] // max_results clamped to 1..=1000, always positive and fits in usize. pub async fn list_queues(&self, input: ListQueuesInput) -> Result { @@ -648,7 +730,11 @@ impl RustackSqs { } /// Handle `RemovePermission`. - #[allow(clippy::unused_async)] // Must be async to match the handler trait interface. + #[allow( + clippy::unused_async, + clippy::unused_async_trait_impl, + reason = "Keep the provider API lazy and awaitable like the async handler interface" + )] pub async fn remove_permission( &self, input: RemovePermissionInput, @@ -660,7 +746,11 @@ impl RustackSqs { // ---- Message Move Task Operations (stubs) ---- /// Handle `StartMessageMoveTask`. - #[allow(clippy::unused_async)] // Must be async to match the handler trait interface. + #[allow( + clippy::unused_async, + clippy::unused_async_trait_impl, + reason = "Keep the provider API lazy and awaitable like the async handler interface" + )] pub async fn start_message_move_task( &self, _input: StartMessageMoveTaskInput, @@ -672,7 +762,11 @@ impl RustackSqs { } /// Handle `CancelMessageMoveTask`. - #[allow(clippy::unused_async)] // Must be async to match the handler trait interface. + #[allow( + clippy::unused_async, + clippy::unused_async_trait_impl, + reason = "Keep the provider API lazy and awaitable like the async handler interface" + )] pub async fn cancel_message_move_task( &self, _input: CancelMessageMoveTaskInput, @@ -684,7 +778,11 @@ impl RustackSqs { } /// Handle `ListMessageMoveTasks`. - #[allow(clippy::unused_async)] // Must be async to match the handler trait interface. + #[allow( + clippy::unused_async, + clippy::unused_async_trait_impl, + reason = "Keep the provider API lazy and awaitable like the async handler interface" + )] pub async fn list_message_move_tasks( &self, _input: ListMessageMoveTasksInput, diff --git a/crates/rustack-sqs-core/src/queue/actor.rs b/crates/rustack-sqs-core/src/queue/actor.rs index 9703bae..4b1de8a 100644 --- a/crates/rustack-sqs-core/src/queue/actor.rs +++ b/crates/rustack-sqs-core/src/queue/actor.rs @@ -6,13 +6,17 @@ use std::{ collections::HashMap, + future::{Future, poll_fn}, + pin::Pin, sync::{ - Arc, + Arc, Weak, atomic::{AtomicBool, Ordering}, }, + task::Poll, time::Duration, }; +use dashmap::DashMap; use rustack_sqs_model::{ error::SqsError, input::{ReceiveMessageInput, SendMessageInput}, @@ -20,21 +24,117 @@ use rustack_sqs_model::{ types::Message, }; use tokio::{ - sync::{mpsc, oneshot}, + sync::{mpsc, oneshot, watch}, time::Instant, }; use super::{ attributes::QueueAttributes, - storage::{EnqueueResult, FifoQueueStorage, StandardQueueStorage}, + storage::{DedupKey, EnqueueResult, FifoQueueStorage, StandardQueueStorage}, }; use crate::message::{ InFlightMessage, QueueMessage, generate_receipt_handle, md5_of_body, md5_of_message_attributes, now_epoch_millis, }; +/// Registry routing shared with the queue manager, never held across await. +type QueueRoutes = Weak>>; + +#[derive(Debug, Default)] +struct RedriveManager { + routes: QueueRoutes, + pending: Vec, + quiescing: bool, +} + +#[derive(Debug)] +struct PendingTransfer { + message: QueueMessage, + reply: oneshot::Receiver>, + completion: Option>, +} + +impl RedriveManager { + fn handoff(&mut self, message: &QueueMessage, target_arn: &str, source_arn: &str) -> bool { + if self.quiescing || self.pending.len() >= 128 || source_arn == target_arn { + return false; + } + let Some(routes) = self.routes.upgrade() else { + return false; + }; + let Some(name) = target_arn.rsplit(':').next() else { + return false; + }; + let Some(target) = routes.get(name).map(|entry| Arc::clone(entry.value())) else { + tracing::warn!("DLQ target unavailable; retaining source message"); + return false; + }; + if target.metadata.arn != target_arn || target.shutdown.load(Ordering::Acquire) { + return false; + } + let (reply, receiver) = oneshot::channel(); + if target + .sender + .try_send(QueueCommand::Transfer { + message: message.clone(), + source_arn: source_arn.to_owned(), + reply, + }) + .is_err() + { + tracing::warn!("DLQ target channel unavailable/full; retaining source message"); + return false; + } + self.pending.push(PendingTransfer { + message: message.clone(), + reply: receiver, + completion: None, + }); + true + } +} + +/// Register each acknowledgment with the actor's waker; no timer polling or +/// actor-to-actor await blocks ordinary queue commands. +async fn wait_for_transfer(pending: &mut [PendingTransfer]) { + poll_fn(|context| { + for transfer in &mut *pending { + if transfer.completion.is_some() { + return Poll::Ready(()); + } + if let Poll::Ready(result) = Pin::new(&mut transfer.reply).poll(context) { + transfer.completion = Some(result.unwrap_or_else(|_| { + Err(SqsError::internal_error( + "DLQ actor closed before acknowledgment", + )) + })); + return Poll::Ready(()); + } + } + Poll::Pending + }) + .await; +} + /// Commands sent to a queue actor via its channel. pub enum QueueCommand { + /// Test-only actor supervision fault injection. + #[cfg(test)] + CrashActor, + /// Transfer a dead-letter message without blocking the source actor. + Transfer { + /// Original message retained by the source until acknowledgment. + message: QueueMessage, + /// Exact source queue ARN. + source_arn: String, + /// Confirms successful enqueue or rejection before mutation. + reply: oneshot::Sender>, + }, + /// Stop initiating redrives and acknowledge after pending transfers settle. + Quiesce { + /// Completion acknowledgment. + reply: oneshot::Sender<()>, + }, /// Send a message to the queue. SendMessage { /// The send message input. @@ -110,6 +210,8 @@ pub enum QueueCommand { impl std::fmt::Debug for QueueCommand { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Self::Transfer { .. } => write!(f, "Transfer"), + Self::Quiesce { .. } => write!(f, "Quiesce"), Self::SendMessage { .. } => write!(f, "SendMessage"), Self::ReceiveMessage { .. } => write!(f, "ReceiveMessage"), Self::DeleteMessage { .. } => write!(f, "DeleteMessage"), @@ -120,6 +222,8 @@ impl std::fmt::Debug for QueueCommand { Self::GetTags { .. } => write!(f, "GetTags"), Self::SetTags { .. } => write!(f, "SetTags"), Self::RemoveTags { .. } => write!(f, "RemoveTags"), + #[cfg(test)] + Self::CrashActor => write!(f, "CrashActor"), Self::Shutdown => write!(f, "Shutdown"), } } @@ -168,6 +272,9 @@ pub struct QueueActor { account_id: String, /// Pending long-poll receivers. pending_long_polls: Vec, + redrive: RedriveManager, + quiesce_reply: Option>, + stopping: bool, } /// A pending long-poll request waiting for messages. @@ -228,13 +335,32 @@ impl QueueActor { last_purge_at: None, account_id, pending_long_polls: Vec::new(), + redrive: RedriveManager::default(), + quiesce_reply: None, + stopping: false, } } + /// Connect this actor to the manager's queue routing registry. + pub(crate) fn with_routes(mut self, routes: QueueRoutes) -> Self { + self.redrive.routes = routes; + self + } + /// Run the actor event loop. pub async fn run(mut self) { let mut cleanup_interval = tokio::time::interval(Duration::from_secs(1)); + let mut commands_open = true; loop { + self.settle_transfers(); + if self.redrive.pending.is_empty() { + if let Some(reply) = self.quiesce_reply.take() { + let _ = reply.send(()); + } + if self.stopping { + break; + } + } // Compute the earliest long-poll deadline so we can expire it precisely // instead of waiting for the next 1-second cleanup tick. let next_poll_deadline = self @@ -245,15 +371,26 @@ impl QueueActor { .unwrap_or_else(|| Instant::now() + Duration::from_hours(24)); tokio::select! { - Some(cmd) = self.commands.recv() => { + command = self.commands.recv(), if commands_open => { + let Some(cmd) = command else { + commands_open = false; + self.stopping = true; + self.redrive.quiescing = true; + continue; + }; let should_fulfill_long_polls = match cmd { - QueueCommand::Shutdown => break, + QueueCommand::Shutdown => { + self.stopping = true; + self.redrive.quiescing = true; + false + }, cmd => self.handle_command(cmd), }; if should_fulfill_long_polls && !self.pending_long_polls.is_empty() { self.fulfill_pending_long_polls(); } } + () = wait_for_transfer(&mut self.redrive.pending), if !self.redrive.pending.is_empty() => {} _ = cleanup_interval.tick() => { self.periodic_cleanup(); } @@ -269,7 +406,36 @@ impl QueueActor { #[allow(clippy::too_many_lines)] fn handle_command(&mut self, cmd: QueueCommand) -> bool { match cmd { + #[cfg(test)] + QueueCommand::CrashActor => panic!("injected queue actor crash"), + QueueCommand::Transfer { + message, + source_arn, + reply, + } => { + if reply.is_closed() { + return false; + } + let result = self.accept_transfer(message, &source_arn); + let accepted = result.is_ok(); + let _ = reply.send(result); + accepted + } + QueueCommand::Quiesce { reply } => { + self.redrive.quiescing = true; + if self + .quiesce_reply + .as_ref() + .is_none_or(oneshot::Sender::is_closed) + { + self.quiesce_reply = Some(reply); + } + false + } QueueCommand::SendMessage { input, reply } => { + if reply.is_closed() { + return false; + } let result = self.handle_send_message(input); let should_fulfill_long_polls = matches!(&result, Ok((_, true))); let _ = reply.send(result.map(|(output, _)| output)); @@ -301,10 +467,13 @@ impl QueueActor { attribute_names, reply, } => { - let counts = match &self.storage { + let mut counts = match &self.storage { QueueStorage::Standard(s) => s.counts(), QueueStorage::Fifo(s) => s.counts(), }; + counts.1 = counts + .1 + .saturating_add(u32::try_from(self.redrive.pending.len()).unwrap_or(u32::MAX)); let attrs = self.attributes.to_map( &attribute_names, self.is_fifo, @@ -352,6 +521,107 @@ impl QueueActor { } } + fn accept_transfer( + &mut self, + mut message: QueueMessage, + source_arn: &str, + ) -> Result<(), SqsError> { + if self.stopping || self.is_fifo != message.message_group_id.is_some() { + return Err(SqsError::invalid_parameter_value( + "DLQ unavailable or queue type mismatch", + )); + } + if message.body.len() > usize::try_from(self.attributes.maximum_message_size).unwrap_or(0) { + return Err(SqsError::invalid_parameter_value( + "Message exceeds DLQ maximum size", + )); + } + let source_scope = source_arn.rsplit_once(':').map(|(scope, _)| scope); + let target_scope = self.arn.rsplit_once(':').map(|(scope, _)| scope); + if source_scope != target_scope { + return Err(SqsError::invalid_parameter_value( + "DLQ must be in the same account and region", + )); + } + if let Some(policy) = &self.attributes.redrive_allow_policy { + let policy: serde_json::Value = serde_json::from_str(policy) + .map_err(|_| SqsError::invalid_parameter_value("Invalid RedriveAllowPolicy"))?; + let allowed = match policy + .get("redrivePermission") + .and_then(serde_json::Value::as_str) + { + Some("allowAll") => true, + Some("byQueue") => policy + .get("sourceQueueArns") + .and_then(serde_json::Value::as_array) + .is_some_and(|arns| arns.iter().any(|arn| arn.as_str() == Some(source_arn))), + _ => false, + }; + if !allowed { + return Err(SqsError::invalid_parameter_value( + "DLQ redrive permission denied", + )); + } + } + message.dead_letter_queue_source_arn = Some(source_arn.to_owned()); + message.approximate_receive_count = 0; + message.approximate_first_receive_timestamp = None; + message.available_at = Instant::now(); + message.delay_seconds = 0; + match &mut self.storage { + QueueStorage::Standard(storage) => storage.available.push_back(message), + QueueStorage::Fifo(storage) => { + message.sent_timestamp = now_epoch_millis(); + message.message_deduplication_id = Some(message.message_id.clone()); + let key = DedupKey::Redrive { + source: source_arn.to_owned(), + message: message.message_id.clone(), + }; + storage.enqueue(message, &key); + } + } + Ok(()) + } + + fn settle_transfers(&mut self) { + let pending = std::mem::take(&mut self.redrive.pending); + for mut transfer in pending { + let result = if let Some(completed) = transfer.completion.take() { + completed + } else { + match transfer.reply.try_recv() { + Ok(result) => result, + Err(oneshot::error::TryRecvError::Empty) => { + self.redrive.pending.push(transfer); + continue; + } + Err(oneshot::error::TryRecvError::Closed) => Err(SqsError::internal_error( + "DLQ actor closed before acknowledgment", + )), + } + }; + let group = transfer + .message + .message_group_id + .clone() + .unwrap_or_default(); + let restore = if result.is_err() { + tracing::warn!("DLQ delivery failed; restoring source message"); + Some(transfer.message) + } else { + None + }; + match &mut self.storage { + QueueStorage::Standard(storage) => { + if let Some(message) = restore { + storage.available.push_front(message); + } + } + QueueStorage::Fifo(storage) => storage.finish_redrive(&group, restore), + } + } + } + /// Handle `SendMessage`. #[allow(clippy::cast_sign_loss)] fn handle_send_message( @@ -423,6 +693,7 @@ impl QueueActor { }; let msg = QueueMessage { + dead_letter_queue_source_arn: None, message_id: message_id.clone(), body: input.message_body, md5_of_body: body_md5.clone(), @@ -499,10 +770,14 @@ impl QueueActor { // Build the effective dedup key based on DeduplicationScope. // "queue" scope: global dedup across all groups (default). // "messageGroup" scope: dedup only within the same group. + let message_deduplication_id = Some(dedup_id.clone()); let effective_dedup_key = if self.attributes.deduplication_scope == "messageGroup" { - format!("{group_id}:{dedup_id}") + DedupKey::Group { + group: group_id.clone(), + id: dedup_id, + } } else { - dedup_id + DedupKey::Queue(dedup_id) }; let message_id = uuid::Uuid::new_v4().to_string(); @@ -510,6 +785,7 @@ impl QueueActor { let attr_md5 = md5_of_message_attributes(&input.message_attributes); let msg = QueueMessage { + dead_letter_queue_source_arn: None, message_id: message_id.clone(), body: input.message_body, md5_of_body: body_md5.clone(), @@ -521,7 +797,7 @@ impl QueueActor { approximate_first_receive_timestamp: None, sequence_number: None, message_group_id: Some(group_id), - message_deduplication_id: input.message_deduplication_id, + message_deduplication_id, available_at: Instant::now(), delay_seconds: 0, }; @@ -611,9 +887,23 @@ impl QueueActor { system_attribute_names: &[String], message_attribute_names: &[String], ) -> Vec { + // Visibility is evaluated at receive time, not only on the periodic timer. + match &mut self.storage { + QueueStorage::Standard(storage) => { + storage.return_expired_inflight(); + storage.promote_delayed(); + } + QueueStorage::Fifo(storage) => { + storage.return_expired_inflight(); + } + } let merged_sys_attrs = merge_attribute_names(attribute_names, system_attribute_names); let vis_timeout = Duration::from_secs(visibility_timeout as u64); - + let mut redrive = RedriveRequest { + manager: &mut self.redrive, + attributes: &self.attributes, + source_arn: &self.arn, + }; match &mut self.storage { QueueStorage::Standard(storage) => try_receive_standard( storage, @@ -621,7 +911,7 @@ impl QueueActor { vis_timeout, &merged_sys_attrs, message_attribute_names, - &self.attributes, + &mut redrive, ), QueueStorage::Fifo(storage) => try_receive_fifo( storage, @@ -629,6 +919,7 @@ impl QueueActor { vis_timeout, &merged_sys_attrs, message_attribute_names, + &mut redrive, ), } } @@ -700,6 +991,7 @@ impl QueueActor { return Err(SqsError::purge_queue_in_progress()); } } + self.redrive.pending.clear(); match &mut self.storage { QueueStorage::Standard(s) => s.purge(), QueueStorage::Fifo(s) => s.purge(), @@ -785,6 +1077,25 @@ impl QueueActor { // Standard queue receive helper // --------------------------------------------------------------------------- +struct RedriveRequest<'a> { + manager: &'a mut RedriveManager, + attributes: &'a QueueAttributes, + source_arn: &'a str, +} + +impl RedriveRequest<'_> { + fn target(&self, message: &QueueMessage) -> Option { + self.attributes + .redrive_policy + .as_ref() + .filter(|policy| { + message.approximate_receive_count + >= u32::try_from(policy.max_receive_count).unwrap_or(u32::MAX) + }) + .map(|policy| policy.dead_letter_target_arn.clone()) + } +} + /// Receive messages from a standard queue. #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] fn try_receive_standard( @@ -793,34 +1104,30 @@ fn try_receive_standard( vis_timeout: Duration, sys_attrs: &[String], msg_attrs: &[String], - queue_attrs: &QueueAttributes, + redrive: &mut RedriveRequest<'_>, ) -> Vec { let mut result = Vec::new(); - while result.len() < max { + let scan_budget = storage.available.len().min(128); + for _ in 0..scan_budget { + if result.len() >= max { + break; + } match storage.available.pop_front() { Some(mut msg) => { - msg.approximate_receive_count += 1; + if let Some(target) = redrive.target(&msg) { + if !redrive.manager.handoff(&msg, &target, redrive.source_arn) { + // Standard queues have no group ordering constraint: retain + // the poison message without starving healthy messages. + storage.available.push_back(msg); + } + continue; + } + msg.approximate_receive_count = msg.approximate_receive_count.saturating_add(1); if msg.approximate_first_receive_timestamp.is_none() { msg.approximate_first_receive_timestamp = Some(now_epoch_millis()); } - // Check DLQ redrive threshold. - if let Some(ref policy) = queue_attrs.redrive_policy { - #[allow(clippy::cast_sign_loss)] - if msg.approximate_receive_count > policy.max_receive_count as u32 { - // Move to dead_letters storage instead of silently dropping. - // Actual DLQ routing (cross-queue send) can be added later. - tracing::debug!( - message_id = %msg.message_id, - receive_count = msg.approximate_receive_count, - "message exceeded maxReceiveCount, moved to dead letters" - ); - storage.dead_letters.push(msg); - continue; - } - } - let receipt_handle = generate_receipt_handle(&msg.message_id); let message = build_message(&msg, &receipt_handle, sys_attrs, msg_attrs); @@ -852,12 +1159,19 @@ fn try_receive_fifo( vis_timeout: Duration, sys_attrs: &[String], msg_attrs: &[String], + redrive: &mut RedriveRequest<'_>, ) -> Vec { let received = storage.receive(max); let mut result = Vec::new(); for (mut msg, group_id) in received { - msg.approximate_receive_count += 1; + if let Some(target) = redrive.target(&msg) { + if !redrive.manager.handoff(&msg, &target, redrive.source_arn) { + storage.finish_redrive(&group_id, Some(msg)); + } + continue; + } + msg.approximate_receive_count = msg.approximate_receive_count.saturating_add(1); if msg.approximate_first_receive_timestamp.is_none() { msg.approximate_first_receive_timestamp = Some(now_epoch_millis()); } @@ -883,8 +1197,8 @@ pub struct QueueHandle { pub sender: mpsc::Sender, /// Queue metadata (read-only after creation). pub metadata: QueueMetadata, - /// Actor task join handle. - pub task: tokio::task::JoinHandle<()>, + /// Supervisor-observed actor completion, including panic status. + pub completion: watch::Receiver>, /// Shutdown flag. pub shutdown: Arc, } @@ -1037,10 +1351,37 @@ impl QueueHandle { Ok(()) } + /// Stop initiating redrive and wait for all accepted handoffs without clearing messages. + /// + /// # Errors + /// Returns an error if the actor is closed or a simultaneous quiesce is in progress. + pub async fn quiesce(&self) -> Result<(), SqsError> { + let (reply, receiver) = oneshot::channel(); + self.sender + .send(QueueCommand::Quiesce { reply }) + .await + .map_err(|_| SqsError::internal_error("Queue actor is not running"))?; + receiver + .await + .map_err(|_| SqsError::internal_error("Queue quiesce did not complete")) + } + /// Shutdown the queue actor. pub async fn shutdown(&self) { self.shutdown.store(true, Ordering::SeqCst); let _ = self.sender.send(QueueCommand::Shutdown).await; + let mut completion = self.completion.clone(); + loop { + if let Some(success) = *completion.borrow_and_update() { + if !success { + tracing::error!("SQS actor shutdown observed a panic"); + } + return; + } + if completion.changed().await.is_err() { + return; + } + } } } @@ -1074,6 +1415,11 @@ fn build_message( let want_sys = |name: &str| want_all_sys || system_attr_names.iter().any(|n| n == name); let mut attributes = HashMap::new(); + if want_sys("DeadLetterQueueSourceArn") { + if let Some(arn) = &msg.dead_letter_queue_source_arn { + attributes.insert("DeadLetterQueueSourceArn".to_owned(), arn.clone()); + } + } if want_sys("SenderId") { attributes.insert("SenderId".to_owned(), msg.sender_id.clone()); } diff --git a/crates/rustack-sqs-core/src/queue/storage.rs b/crates/rustack-sqs-core/src/queue/storage.rs index d5b483a..e435d17 100644 --- a/crates/rustack-sqs-core/src/queue/storage.rs +++ b/crates/rustack-sqs-core/src/queue/storage.rs @@ -18,9 +18,6 @@ pub struct StandardQueueStorage { pub delayed: Vec, /// Messages currently being processed by consumers. pub in_flight: HashMap, - /// Messages that exceeded the DLQ `maxReceiveCount` threshold. - /// Stored here until actual DLQ routing is implemented. - pub dead_letters: Vec, } impl StandardQueueStorage { @@ -77,10 +74,30 @@ impl StandardQueueStorage { self.available.clear(); self.delayed.clear(); self.in_flight.clear(); - self.dead_letters.clear(); } } +/// Unambiguous FIFO deduplication identity, including the configured scope. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum DedupKey { + /// Queue-wide identity. + Queue(String), + /// Internal handoff identity, isolated from caller-supplied deduplication IDs. + Redrive { + /// Source queue ARN. + source: String, + /// Original message ID. + message: String, + }, + /// Group-local identity. + Group { + /// Message group identifier. + group: String, + /// Deduplication identifier. + id: String, + }, +} + /// Cached information for a deduplicated message. #[derive(Debug, Clone)] pub struct DedupCacheEntry { @@ -102,7 +119,7 @@ pub struct FifoQueueStorage { /// Messages currently being processed by consumers. in_flight: HashMap, /// Deduplication cache: effective_dedup_key -> original message info. - dedup_cache: HashMap, + dedup_cache: HashMap, /// Monotonically increasing sequence number. next_sequence: AtomicU64, } @@ -155,9 +172,12 @@ pub enum EnqueueResult { impl FifoQueueStorage { /// Attempt to enqueue a message with deduplication. /// - /// The `effective_dedup_key` should already incorporate the dedup scope - /// (e.g., prefixed with group ID when scope is `messageGroup`). - pub fn enqueue(&mut self, mut msg: QueueMessage, effective_dedup_key: &str) -> EnqueueResult { + /// The structured key includes the scope without delimiter collisions. + pub fn enqueue( + &mut self, + mut msg: QueueMessage, + effective_dedup_key: &DedupKey, + ) -> EnqueueResult { // Check dedup cache. if let Some(entry) = self.dedup_cache.get(effective_dedup_key) { if Instant::now() < entry.expiry { @@ -225,6 +245,17 @@ impl FifoQueueStorage { result } + /// Complete a reserved redrive, restoring to the group head on failure. + pub(crate) fn finish_redrive(&mut self, group_id: &str, restore: Option) { + self.blocked_groups.remove(group_id); + if let Some(message) = restore { + self.groups + .entry(group_id.to_owned()) + .or_default() + .push_front(message); + } + } + /// Record a message as in-flight. pub fn mark_in_flight( &mut self, diff --git a/crates/rustack-sqs-core/src/queue/url.rs b/crates/rustack-sqs-core/src/queue/url.rs index eba8a59..4cff068 100644 --- a/crates/rustack-sqs-core/src/queue/url.rs +++ b/crates/rustack-sqs-core/src/queue/url.rs @@ -1,10 +1,31 @@ //! Queue URL and ARN generation and parsing. +use rustack_core::settings::advertised_endpoint; + /// Build a queue URL. /// -/// Format: `http://://` +/// Uses the validated runtime advertised endpoint when installed; otherwise +/// falls back to `http://://`. #[must_use] pub fn queue_url(host: &str, port: u16, account_id: &str, queue_name: &str) -> String { + queue_url_with_endpoint(advertised_endpoint(), host, port, account_id, queue_name) +} + +fn queue_url_with_endpoint( + endpoint: Option<&str>, + host: &str, + port: u16, + account_id: &str, + queue_name: &str, +) -> String { + if let Some(endpoint) = endpoint { + return format!("{endpoint}/{account_id}/{queue_name}"); + } + let host = if host.contains(':') && !host.starts_with('[') { + format!("[{host}]") + } else { + host.to_owned() + }; format!("http://{host}:{port}/{account_id}/{queue_name}") } @@ -54,6 +75,18 @@ mod tests { ); } + #[test] + fn test_should_preserve_advertised_scheme_authority_and_ipv6() { + assert_eq!( + queue_url_with_endpoint(Some("https://[::1]:8443"), "ignored", 1, "123", "queue"), + "https://[::1]:8443/123/queue" + ); + assert_eq!( + queue_url_with_endpoint(None, "::1", 4567, "123", "queue"), + "http://[::1]:4567/123/queue" + ); + } + #[test] fn test_should_build_queue_arn() { assert_eq!( diff --git a/crates/rustack-sqs-core/src/redrive_tests.rs b/crates/rustack-sqs-core/src/redrive_tests.rs new file mode 100644 index 0000000..93649f6 --- /dev/null +++ b/crates/rustack-sqs-core/src/redrive_tests.rs @@ -0,0 +1,234 @@ +//! End-to-end DLQ and tuple-key regressions using real actors. +use rustack_sqs_model::types::Message; + +use super::*; + +async fn create(provider: &RustackSqs, name: &str, target: Option<&str>) -> String { + let mut attributes = HashMap::new(); + if name + .rsplit_once('.') + .is_some_and(|(_, suffix)| suffix == "fifo") + { + attributes.insert("FifoQueue".into(), "true".into()); + attributes.insert("ContentBasedDeduplication".into(), "true".into()); + attributes.insert("DeduplicationScope".into(), "messageGroup".into()); + } + if let Some(target) = target { + attributes.insert("RedrivePolicy".into(), serde_json::json!({ + "deadLetterTargetArn":format!("arn:aws:sqs:us-east-1:000000000000:{target}"), "maxReceiveCount":1 + }).to_string()); + } + provider + .create_queue(CreateQueueInput { + queue_name: name.into(), + attributes, + tags: HashMap::new(), + }) + .await + .unwrap() + .queue_url + .unwrap() +} + +async fn receive(provider: &RustackSqs, url: &str) -> Vec { + provider + .receive_message(ReceiveMessageInput { + queue_url: url.into(), + max_number_of_messages: Some(10), + visibility_timeout: Some(0), + message_system_attribute_names: vec!["All".into()], + ..Default::default() + }) + .await + .unwrap() + .messages +} + +#[tokio::test] +async fn test_should_deliver_standard_and_fifo_dead_letters_to_real_target() { + for suffix in ["", ".fifo"] { + let provider = RustackSqs::new(SqsConfig::default()); + let target_name = format!("dead{suffix}"); + let target = create(&provider, &target_name, None).await; + let source = create(&provider, &format!("source{suffix}"), Some(&target_name)).await; + provider + .send_message(SendMessageInput { + queue_url: source.clone(), + message_body: "body".into(), + message_group_id: if suffix.is_empty() { + None + } else { + Some("g".into()) + }, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(receive(&provider, &source).await.len(), 1); + assert!(receive(&provider, &source).await.is_empty()); + provider.quiesce().await.unwrap(); + let messages = receive(&provider, &target).await; + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].body.as_deref(), Some("body")); + assert_eq!( + messages[0].attributes.get("DeadLetterQueueSourceArn"), + Some(&format!( + "arn:aws:sqs:us-east-1:000000000000:source{suffix}" + )) + ); + assert!(receive(&provider, &source).await.is_empty()); + provider.shutdown_all().await; + } +} + +#[tokio::test] +async fn test_should_retain_dead_letters_until_missing_target_becomes_available() { + for suffix in ["", ".fifo"] { + let provider = RustackSqs::new(SqsConfig::default()); + let target_name = format!("missing{suffix}"); + let source = create(&provider, &format!("source{suffix}"), Some(&target_name)).await; + provider + .send_message(SendMessageInput { + queue_url: source.clone(), + message_body: "retained".into(), + message_group_id: if suffix.is_empty() { + None + } else { + Some("g".into()) + }, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(receive(&provider, &source).await.len(), 1); + assert!(receive(&provider, &source).await.is_empty()); + let target = create(&provider, &target_name, None).await; + assert!(receive(&provider, &source).await.is_empty()); + provider.quiesce().await.unwrap(); + assert_eq!(receive(&provider, &target).await.len(), 1); + provider.shutdown_all().await; + } +} + +#[tokio::test] +async fn test_should_restore_source_after_target_rejects_transfer() { + for suffix in ["", ".fifo"] { + let provider = RustackSqs::new(SqsConfig::default()); + let target_name = format!("denied{suffix}"); + let target = create(&provider, &target_name, None).await; + provider + .set_queue_attributes(SetQueueAttributesInput { + queue_url: target.clone(), + attributes: HashMap::from([( + "RedriveAllowPolicy".into(), + "{\"redrivePermission\":\"denyAll\"}".into(), + )]), + }) + .await + .unwrap(); + let source = create(&provider, &format!("source{suffix}"), Some(&target_name)).await; + provider + .send_message(SendMessageInput { + queue_url: source.clone(), + message_body: "retain".into(), + message_group_id: if suffix.is_empty() { + None + } else { + Some("g".into()) + }, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(receive(&provider, &source).await.len(), 1); + assert!(receive(&provider, &source).await.is_empty()); + provider.quiesce().await.unwrap(); + let attributes = provider + .get_queue_attributes(GetQueueAttributesInput { + queue_url: source, + attribute_names: vec!["All".into()], + }) + .await + .unwrap() + .attributes; + assert_eq!( + attributes + .get("ApproximateNumberOfMessages") + .map(String::as_str), + Some("1") + ); + assert!(receive(&provider, &target).await.is_empty()); + provider.shutdown_all().await; + } +} + +#[tokio::test] +async fn test_should_retain_source_when_target_command_channel_is_full_or_closed() { + use crate::queue::actor::QueueCommand; + for closed in [false, true] { + let provider = RustackSqs::new(SqsConfig::default()); + let target_url = create(&provider, "target", None).await; + let source = create(&provider, "source", Some("target")).await; + let real_target = provider.get_queue(&target_url).unwrap(); + let (sender, receiver) = mpsc::channel(1); + let (_finished, completion) = tokio::sync::watch::channel(Some(true)); + let _receiver = if closed { + drop(receiver); + None + } else { + sender.try_send(QueueCommand::Shutdown).unwrap(); + Some(receiver) + }; + provider.queues.insert( + "target".into(), + Arc::new(QueueHandle { + sender, + metadata: real_target.metadata.clone(), + completion, + shutdown: Arc::new(AtomicBool::new(false)), + }), + ); + provider + .send_message(SendMessageInput { + queue_url: source.clone(), + message_body: "retain".into(), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(receive(&provider, &source).await.len(), 1); + assert!(receive(&provider, &source).await.is_empty()); + provider.queues.insert("target".into(), real_target); + assert!(receive(&provider, &source).await.is_empty()); + provider.quiesce().await.unwrap(); + assert_eq!(receive(&provider, &target_url).await.len(), 1); + provider.shutdown_all().await; + } +} + +#[tokio::test] +async fn test_should_not_collide_fifo_group_deduplication_tuples() { + let provider = RustackSqs::new(SqsConfig::default()); + let url = create(&provider, "tuples.fifo", None).await; + let mut ids = Vec::new(); + for (group, dedup) in [("a:b", "c"), ("a", "b:c"), ("a:b", "c"), ("other", "c")] { + ids.push( + provider + .send_message(SendMessageInput { + queue_url: url.clone(), + message_body: format!("{group}/{dedup}"), + message_group_id: Some(group.into()), + message_deduplication_id: Some(dedup.into()), + ..Default::default() + }) + .await + .unwrap() + .message_id, + ); + } + assert_eq!(ids[0], ids[2]); + assert_ne!(ids[0], ids[1]); + assert_ne!(ids[0], ids[3]); + assert_eq!(receive(&provider, &url).await.len(), 3); + provider.shutdown_all().await; +} diff --git a/crates/rustack-sqs-http/Cargo.toml b/crates/rustack-sqs-http/Cargo.toml index 8e9d534..2acc70b 100644 --- a/crates/rustack-sqs-http/Cargo.toml +++ b/crates/rustack-sqs-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-sqs-http" description = "SQS HTTP service layer for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-sqs-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-sqs-http/src/service.rs b/crates/rustack-sqs-http/src/service.rs index bbf9b88..fc77b30 100644 --- a/crates/rustack-sqs-http/src/service.rs +++ b/crates/rustack-sqs-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_sqs_model::error::SqsError; @@ -96,12 +95,16 @@ impl hyper::service::Service> for SqsHttp } /// Process a single SQS HTTP request through the full pipeline. -async fn process_request( - req: http::Request, +async fn process_request( + req: http::Request, handler: &H, config: &SqsHttpConfig, request_id: &str, -) -> http::Response { +) -> http::Response +where + B: http_body::Body, + B::Error: std::error::Error + Send + Sync + 'static, +{ let (parts, incoming) = req.into_parts(); // 1. Verify POST method (SQS only accepts POST). @@ -122,23 +125,29 @@ async fn process_request( // 3. Collect body. let body = match collect_body(incoming).await { Ok(body) => body, - Err(err) => return error_to_response(&err, request_id), + Err(err) => { + let error = SqsError::new( + rustack_sqs_model::error::SqsErrorCode::InvalidParameterValue, + err.to_string(), + ); + let mut response = error_to_response(&error, request_id); + *response.status_mut() = err.status_code(); + return response; + } }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = SqsError::new( - rustack_sqs_model::error::SqsErrorCode::InvalidSecurity, - auth_err.to_string(), - ); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let err = SqsError::new( + rustack_sqs_model::error::SqsErrorCode::InvalidSecurity, + auth_err.to_string(), + ); + return error_to_response(&err, request_id); } // 5. Dispatch to handler. @@ -149,12 +158,12 @@ async fn process_request( } /// Collect the incoming body into a single `Bytes` buffer. -async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() - .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| SqsError::internal_error(format!("Failed to read request body: {e}"))) +async fn collect_body(incoming: B) -> Result +where + B: http_body::Body, + B::Error: std::error::Error + Send + Sync + 'static, +{ + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()).await } /// Add common response headers to every SQS response. @@ -182,3 +191,134 @@ fn add_common_headers( response } + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use http_body_util::Full; + use rustack_auth::{ + StaticCredentialProvider, + canonical::build_canonical_request, + sigv4::{build_string_to_sign, compute_signature, derive_signing_key}, + }; + + use super::*; + + #[derive(Default)] + struct Counter(AtomicUsize); + impl SqsHandler for Counter { + fn handle_operation( + &self, + _: rustack_sqs_model::operations::SqsOperation, + _: Bytes, + ) -> Pin, SqsError>> + Send>> + { + self.0.fetch_add(1, Ordering::SeqCst); + Box::pin(async { + Ok(http::Response::new(SqsResponseBody::from_bytes( + Bytes::from_static(b"{}"), + ))) + }) + } + } + + fn request(body: &[u8], original: Option<&[u8]>) -> http::Request> { + let mut request = http::Request::builder() + .method("POST") + .uri("/") + .header("host", "localhost") + .header("x-amz-date", "20260101T000000Z") + .header("x-amz-target", "AmazonSQS.SendMessage") + .body(Full::new(Bytes::copy_from_slice(body))) + .unwrap(); + if let Some(original) = original { + let hash = rustack_auth::hash_payload(original); + let canonical = build_canonical_request( + "POST", + "/", + "", + &[("host", "localhost"), ("x-amz-date", "20260101T000000Z")], + &["host", "x-amz-date"], + &hash, + ); + let text = build_string_to_sign( + "20260101T000000Z", + "20260101/us-east-1/sqs/aws4_request", + &rustack_auth::hash_payload(canonical.as_bytes()), + ); + let signature = compute_signature( + &derive_signing_key("secret", "20260101", "us-east-1", "sqs"), + &text, + ); + request.headers_mut().insert( + "authorization", + format!( + "AWS4-HMAC-SHA256 \ + Credential=key/20260101/us-east-1/sqs/aws4_request,SignedHeaders=host;\ + x-amz-date,Signature={signature}" + ) + .parse() + .unwrap(), + ); + request + .headers_mut() + .insert("x-amz-content-sha256", hash.parse().unwrap()); + } + request + } + + #[tokio::test] + async fn test_should_never_dispatch_missing_credentials_or_tampered_payload() { + let handler = Counter::default(); + let mut config = SqsHttpConfig { + skip_signature_validation: false, + ..SqsHttpConfig::default() + }; + let response = process_request(request(b"original", None), &handler, &config, "test").await; + assert!(!response.status().is_success()); + config.credential_provider = Some(Arc::new(StaticCredentialProvider::new(vec![( + "key".to_owned(), + "secret".to_owned(), + )]))); + for request in [ + request(b"original", None), + request(b"modified", Some(b"original")), + ] { + assert!( + !process_request(request, &handler, &config, "test") + .await + .status() + .is_success() + ); + } + assert_eq!(handler.0.load(Ordering::SeqCst), 0); + assert!( + process_request( + request(b"original", Some(b"original")), + &handler, + &config, + "test" + ) + .await + .status() + .is_success() + ); + assert_eq!(handler.0.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn test_should_reject_oversized_body_without_dispatch() { + let handler = Counter::default(); + let body = vec![0; 16 * 1024 * 1024 + 1]; + let response = process_request( + request(&body, None), + &handler, + &SqsHttpConfig::default(), + "test", + ) + .await; + assert!(!response.status().is_success()); + assert_eq!(handler.0.load(Ordering::SeqCst), 0); + } +} diff --git a/crates/rustack-ssm-core/Cargo.toml b/crates/rustack-ssm-core/Cargo.toml index 03c72d2..127f047 100644 --- a/crates/rustack-ssm-core/Cargo.toml +++ b/crates/rustack-ssm-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-ssm-core" description = "SSM Parameter Store business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-ssm-core/src/config.rs b/crates/rustack-ssm-core/src/config.rs index e77d2b4..b78903f 100644 --- a/crates/rustack-ssm-core/src/config.rs +++ b/crates/rustack-ssm-core/src/config.rs @@ -1,6 +1,6 @@ //! SSM service configuration. -use std::env; +use rustack_core::settings as env; /// SSM service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-ssm-http/Cargo.toml b/crates/rustack-ssm-http/Cargo.toml index 87dfd36..1da276e 100644 --- a/crates/rustack-ssm-http/Cargo.toml +++ b/crates/rustack-ssm-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-ssm-http" description = "SSM Parameter Store HTTP service layer for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-ssm-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-ssm-http/src/service.rs b/crates/rustack-ssm-http/src/service.rs index a623981..f3fe8ef 100644 --- a/crates/rustack-ssm-http/src/service.rs +++ b/crates/rustack-ssm-http/src/service.rs @@ -3,7 +3,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_ssm_model::error::SsmError; @@ -123,19 +122,18 @@ async fn process_request( }; // 4. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = SsmError::with_message( - rustack_ssm_model::error::SsmErrorCode::ValidationException, - auth_err.to_string(), - ); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let mut err = SsmError::with_message( + rustack_ssm_model::error::SsmErrorCode::ValidationException, + auth_err.to_string(), + ); + err.status_code = http::StatusCode::FORBIDDEN; + return error_to_response(&err, request_id); } // 5. Dispatch to handler. @@ -147,11 +145,13 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| SsmError::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = SsmError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every SSM response. diff --git a/crates/rustack-sts-core/Cargo.toml b/crates/rustack-sts-core/Cargo.toml index 1cc4a60..735c769 100644 --- a/crates/rustack-sts-core/Cargo.toml +++ b/crates/rustack-sts-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-sts-core" description = "STS business logic for Rustack" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true diff --git a/crates/rustack-sts-core/src/config.rs b/crates/rustack-sts-core/src/config.rs index effc93c..34da021 100644 --- a/crates/rustack-sts-core/src/config.rs +++ b/crates/rustack-sts-core/src/config.rs @@ -1,6 +1,6 @@ //! STS service configuration. -use std::env; +use rustack_core::settings as env; /// STS service configuration. #[derive(Debug, Clone)] diff --git a/crates/rustack-sts-http/Cargo.toml b/crates/rustack-sts-http/Cargo.toml index 88ab768..372f977 100644 --- a/crates/rustack-sts-http/Cargo.toml +++ b/crates/rustack-sts-http/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rustack-sts-http" description = "STS HTTP service layer for Rustack (awsQuery protocol)" -version = "0.9.0" +version = "0.10.0" edition.workspace = true license.workspace = true authors.workspace = true @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +rustack-core = { workspace = true } rustack-sts-model = { workspace = true } rustack-auth = { workspace = true } diff --git a/crates/rustack-sts-http/src/service.rs b/crates/rustack-sts-http/src/service.rs index 338a0d4..bc1e94f 100644 --- a/crates/rustack-sts-http/src/service.rs +++ b/crates/rustack-sts-http/src/service.rs @@ -7,7 +7,6 @@ use std::{convert::Infallible, future::Future, pin::Pin, sync::Arc}; use bytes::Bytes; -use http_body_util::BodyExt; use hyper::body::Incoming; use rustack_sts_model::error::StsError; @@ -148,16 +147,14 @@ async fn process_request( .and_then(extract_access_key_from_auth); // 6. Authenticate (if enabled). - if !config.skip_signature_validation { - if let Some(ref cred_provider) = config.credential_provider { - let body_hash = rustack_auth::hash_payload(&body); - if let Err(auth_err) = - rustack_auth::verify_sigv4(&parts, &body_hash, cred_provider.as_ref()) - { - let err = StsError::invalid_client_token_id(auth_err.to_string()); - return error_to_response(&err, request_id); - } - } + if let Err(auth_err) = rustack_auth::AuthMode::resolve( + config.skip_signature_validation, + config.credential_provider.as_deref(), + ) + .and_then(|mode| mode.verify(&parts, &rustack_auth::hash_payload(&body))) + { + let err = StsError::invalid_client_token_id(auth_err.to_string()); + return error_to_response(&err, request_id); } // 7. Dispatch to handler. @@ -169,11 +166,13 @@ async fn process_request( /// Collect the incoming body into a single `Bytes` buffer. async fn collect_body(incoming: Incoming) -> Result { - incoming - .collect() + rustack_core::http::collect_body(incoming, rustack_core::http::BodyBudget::control()) .await - .map(http_body_util::Collected::to_bytes) - .map_err(|e| StsError::internal_error(format!("Failed to read request body: {e}"))) + .map_err(|e| { + let mut err = StsError::internal_error(e.to_string()); + err.status_code = e.status_code(); + err + }) } /// Add common response headers to every STS response. diff --git a/docs/index.md b/docs/index.md index ddb2059..7d86de2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,7 @@ # Documentation - [Pulumi](./pulumi.md) - Use Rustack as a Pulumi AWS provider deployment target. +- [运行时硬化操作与能力指南](./runtime-hardening.md) - 配置/信任边界/执行与持久化范围/健康与能力端点。 ## Research diff --git a/docs/runtime-hardening.md b/docs/runtime-hardening.md new file mode 100644 index 0000000..8005aac --- /dev/null +++ b/docs/runtime-hardening.md @@ -0,0 +1,135 @@ +# 运行时硬化操作与能力指南 + +本文面向把 Rustack 当作本地 AWS 替代品的开发与 CI 操作者,记录 R01–R18 修复后**可观察的安全边界、执行与持久化范围、配置格式**,以及从旧默认值迁移的影响。协议层面的详细理由见各硬化设计 spec。 + +## 1. 配置来源与验证 + +### 1.1 三层来源 + +启动时按以下优先级合成一份**不可变**设置(`rustack-core::settings`): + +1. YAML 文件(`RUSTACK_CONFIG` 指向的路径,≤1 MiB,5 秒读取截止); +2. 进程环境变量(覆盖 YAML 中同名 `environment` 键); +3. 编译期默认值(最后应用)。 + +YAML 顶层结构: + +```yaml +# 配置示例:显示所有可用键;按需保留 +environment: + GATEWAY_LISTEN: "127.0.0.1:4566" + SERVICES: "s3,dynamodb,sqs,lambda" + LOG_LEVEL: "info" + LAMBDA_EXECUTOR: "disabled" # 执行器必须显式选择 + S3_SKIP_SIGNATURE_VALIDATION: true # 本地开发免签名;false 需要凭据 + ACCESS_KEY: "local-test" + SECRET_KEY: "local-secret" +budgets: + connections: 256 # 最大 HTTP 连接(含少量诊断保留位) + requests: 128 # 同时在飞业务请求 + headerSeconds: 5 + requestSeconds: 30 # 控制面请求总截止 + lambdaInvokeSeconds: 930 # 同步 Invoke 总截止(可为 900 s 执行 + init) + shutdownSeconds: 30 + controlBodyBytes: 16777216 + lambdaCodeBodyBytes: 100663296 + upstreamBodyBytes: 67108864 + bodyIdleSeconds: 5 + bodyTotalSeconds: 30 + s3BodyTotalSeconds: 3600 + s3ObjectBodyBytes: 5368709120 +advertisedEndpoint: "http://localhost:4566" +``` + +### 1.2 验证规则(失败即拒绝启动) + +- YAML 未知键、非标量、坏数值、越界预算一律拒绝,错误不打印配置值; +- 布尔键只接受 `true/false/1/0/yes/no/on/off`,执行器枚举只接受 `disabled/native/auto/docker/squib`; +- `DEFAULT_REGION` 与 `AWS_DEFAULT_REGION`、`ACCESS_KEY` 与 `AWS_ACCESS_KEY_ID`、账号别名之间冲突拒绝; +- `GATEWAY_LISTEN` 只接受带非零端口的 IP(含 IPv6),裸进程默认 `127.0.0.1:4566`; +- 已启用且声明 `_SKIP_SIGNATURE_VALIDATION=false` 的服务,若缺少完整 access/secret 凭据则拒绝启动(**凭据缺失不再放开认证**); +- `SERVICES` 里出现未编译或未知服务名直接报错退出,不再静默跳过; +- 首个参数不是已知旗标时按未知参数报错;`--health-check` 与 `--snapshot` 互斥且各自只能出现一次。 + +`--help` / `--version` 不需要读取或验证配置,保证廉价无副作用。 + +### 1.3 从旧默认值的迁移影响 + +| 项目 | 旧行为 | 新行为 | +|---|---|---| +| 裸进程监听 | `0.0.0.0:4566`(所有接口) | `127.0.0.1:4566`(回环) | +| 容器镜像监听 | 无声明 | `0.0.0.0:4566`(Dockerfile 显式设置) | +| Lambda 执行器 | 未设置回退 docker/native | 显式 `disabled`,Invoke 明确报错 | +| 无凭据 + 严格签名 | 放行 | 启动失败(fail closed) | +| 未知/未编译服务 | 警告并跳过 | 启动失败 | +| 冗余 CLI 参数 | 静默忽略 | 启动失败 | + +## 2. 信任边界与请求处理 + +- 每条业务请求在网关获得**在飞许可**(`budgets.requests`),许可持有到**响应体被消费完**而不是只到响应头;健康/诊断请求走保留连接且不占业务许可。 +- 每个 DATA 帧在实际复制前接受字节/总时长/空闲时长三重检查(16 MiB 控制面、96 MiB Lambda ZIP JSON、64 MiB 上游、S3 对象 5 GiB 流式、响应按 S3 流式预算)。 +- 签名校验使用**实际收到的字节摘要**,不接受请求声明的 `x-amz-content-sha256` 替身;`UNSIGNED-PAYLOAD` 及流式签名标记只对 S3 有收窄豁免;未知流式格式拒绝。日志与错误不输出签名材料。 +- 上游/集成客户端:不跟随任何重定向、不继承 `HTTP_PROXY`/`HTTPS_PROXY`/`ALL_PROXY`;请求同时受 idle 与**总** wall-clock 截止约束(APIGW 遗留记录无 timeout 时默认 30 s 兜底);`advertisedEndpoint`/集成 URL 之外的本地回环目标只有在显式配置的模拟器拓扑(本机 SDK/CLI 或本机 fixture 源)中允许。 +- **边界**:请求侧的读 idle/total deadline 在响应**发送**期间不重启;若客户端停止读取响应,hyper 停 poll 后相关 permit/连接会保留到停机 drain(默认 30 s)才释放。面向停滞客户端的独立写 deadline 未实现,属已知边界而非不受限保证。 +- Lambda FunctionName/Qualifier/ARN 在 resolver、store、快照导入各边界独立校验(1–64 ASCII 标识符),逻辑名永不拼接进文件系统路径。 + +## 3. 执行、投递与持久化范围 + +### 3.1 Lambda + +默认 `disabled`;Docker **不支持并显式报错**。`native`/`auto`(squib 需要可用虚拟化)是**可信主机执行**,不是沙箱。执行准入:全局 32、每函数默认 8,保留并发为 0 时拒绝;异步事件预算 128(采用当前 AWS 异步载荷上限 1 MiB,同步 6 MiB)。ZIP 上传不可变、按内部 UUID 落地,`latest` 与已发布版本不再共享同一可覆盖路径;坏 ZIP/CRC/路径/大小/IO 在发布前被拒绝且保留先前版本与 warm 指纹。 + +### 3.2 快照(`--snapshot `) + +- 目录 `RUSTACK_SNAPSHOT_DIR`(默认 `.rustack/snapshots`)下按名字排他锁(`.{name}.lock`),同一快照名同时只能一个运行时拥有; +- 保存:staged 目录写入与 fsync → 校验 manifest → 发布为目录级原子替换,替换前保留 `.{name}.previous`,崩溃或发布间隙通过恢复逻辑回到上一个完整代际;发布失败/恢复失败会**报错退出**,不会静默以空状态继续; +- 加载:先恢复再读取;目标缺失且无可恢复代际才视为空起始(首次运行建快照的合法场景); +- **覆盖范围不等于全量持久化**:S3/DynamoDB 持久资源+数据,DynamoDB Streams 记录,Lambda 元数据+代码(不可变 artifact),其余服务为资源或缓存;SQS **消息**、Events/SNS 投递历史、无快照声明的服务不随快照保留。日志明确打印本次 coverage。 + +### 3.3 停机(SIGINT/SIGTERM) + +收到信号后:停止接受连接(保留诊断连接)→ 在共享总 deadline(`budgets.shutdownSeconds`,默认 30 s)内排空 HTTP → 按顺序 quiesce:Events/SNS 停新投递并等待已接受投递 → DynamoDB 排空已准入 blocking 工作 → Lambda 停新执行并收割子进程 → SQS 停新移交并排空 DLQ 移交 → 快照保存(有快照名且前序成功)→ 并行停止各 provider。quiesce/保存超时不发布一致性快照;总截止前留出清理余量,被卡住的 blocking IO 不会让进程无限挂起。 + +## 4. 健康与能力端点 + +GET/HEAD(不占业务许可、无签名要求,仅供运维探针): + +| 路径 | 语义 | +|---|---| +| `/_localstack/health`、`/health`、`/_health` | 兼容别名,与就绪相同响应体 | +| `/_health/ready`、`/minio/health/ready` | **就绪**:网关未排空、worker 存活、服务非空且每个状态 `running` | +| `/_health/live`、`/minio/health/live` | 存活(进程在);排空时仍 200 | +| `/_rustack/capabilities` | 结构化能力:逐服务 `status/snapshot/execution`、`ready`、worker 投递计数、本地模拟端点说明 | + +`services` 映射中受管 worker(Events/SNS/SQS/DDB)失败终态标 `failed`;未声明执行后端时(如 Lambda 默认 `disabled`)标 `disabled`(是明确能力而非故障);排空阶段标 `draining`。`rustack --health-check` 与 readiness 探针接受 `running` 与 `disabled`,拒绝 `failed`/`draining`。`workers` 字段含 `delivery.{accepted,delivered,failed,rejected}` 计数(投递口径,API 接受不等于投递成功)。 + +`rustack --health-check` 现在走真实 HTTP 状态码 + 有界响应解析(≤64 KiB、3 秒总截止、解析 JSON `ready`/服务状态),不再字符串匹配 `200 OK`/`"running"` 子串。 + +## 5. 客户端建议与示例 + +```bash +# 仅本机使用:默认即安全,无需额外设置 +rustack +export AWS_ENDPOINT_URL=http://127.0.0.1:4566 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test +aws s3 mb s3://my-bucket + +# CI 健康检查 +rustack --health-check + +# 严格签名(可选,需要凭据) +RUSTACK_CONFIG=/path/to/rustack.yaml SERVICES=s3,dynamodb \ +AWS_ACCESS_KEY_ID=my-access AWS_SECRET_ACCESS_KEY=my-secret rustack +``` + +建议显式 `SERVICES=` 列出 CI 实际使用的服务:启动更快、能力面更小、误配置在启动即报错。 + +## 6. 常见失败与诊断 + +| 症状 | 原因 | +|---|---| +| 启动即退出,提示 invalid configuration | YAML/环境值或别名冲突;见 1.2 | +| Invoke 返回 Lambda 不可用 | 默认 Disabled;需显式 `LAMBDA_EXECUTOR=native`(可信执行) | +| 跨容器/远程 SDK 连不上 | 容器显式 `GATEWAY_LISTEN=0.0.0.0`;裸进程默认回环 | +| SQS/Events 消息在重启后消失 | 资源型快照不保留消息;quiesce 只排空 DLQ 移交不承诺持久化 | +| `--health-check` 报告 unhealthy | 用 `curl /_health/ready` 与 `/_rustack/capabilities` 查 worker/投递计数 | +| 快照目录出现 `.name.previous` | 正常发布保留最近一次提交用于崩溃恢复;无歧义时由下次启动清理 | diff --git a/rust-toolchain.toml b/rust-toolchain.toml deleted file mode 100644 index 6360c18..0000000 --- a/rust-toolchain.toml +++ /dev/null @@ -1,3 +0,0 @@ -[toolchain] -channel = "1.95.0" -components = ["rustfmt", "clippy"] diff --git a/specs/README.md b/specs/README.md index de079b9..282420f 100644 --- a/specs/README.md +++ b/specs/README.md @@ -2,6 +2,8 @@ All specs that for AI to generate code. +> Canonical navigation: [specs/index.md](./index.md). Start with the [system review](./rustack-system-review.md) for the current security, correctness, architecture, and developer-experience findings. The catalogue below is historical; use the canonical index for current file links. + ## Index | Spec | Type | Status | Description | diff --git a/specs/index.md b/specs/index.md new file mode 100644 index 0000000..99434ca --- /dev/null +++ b/specs/index.md @@ -0,0 +1,65 @@ +# Rustack Specs 索引 + +本文件是 specs 的导航入口;文件中的 Draft / Implemented 状态以各原文为准,不代表经过本次运行验收。 + +## 推荐阅读顺序 + +1. [系统审查与改进 spec](./rustack-system-review.md):安全、正确性、架构及开发者体验;按 R01–R18 定位问题,按 M0–M4 安排修复。 +2. 按问题查阅下表中的既有服务设计。Review 不整体替代服务设计,也不将设计中的未交付能力视为已经实现。 +3. 快照相关工作按 PRD → 二进制归档设计 → 实施计划 → 验证计划阅读;早期 JSON 设计仅作历史背景。 + +```text +系统审查 + ├─ M0 安全边界 ── M1 有界执行、可靠停机 ── M2 数据语义 + │ │ + └─ 既有服务设计提供协议与领域约束 ────────────────────┤ + ▼ + M3 跨服务、配置契约 ── M4 体验与验收 +``` + +## 跨系统与架构 + +| Spec | 类型 / 用途 | +|---|---| +| [rustack-system-review](./rustack-system-review.md) | Review:本次代码审查、风险排序、修复契约与验收场景 | +| [rustack-system-hardening-impl-plan](./rustack-system-hardening-impl-plan.md) | Impl plan:R01–R18全量修复、依赖顺序及验收门禁 | +| [rustack-runtime-hardening-design](./rustack-runtime-hardening-design.md) | Design:有界runtime、YAML、停机恢复、健康与能力 | +| [rustack-http-hardening-design](./rustack-http-hardening-design.md) | Design:认证、真实payload、HTTP预算与代理边界 | +| [rustack-lambda-hardening-design](./rustack-lambda-hardening-design.md) | Design:安全artifact、版本、执行准入及子进程生命周期 | +| [rustack-data-correctness-design](./rustack-data-correctness-design.md) | Design:R07/R08 DDB原子性与幂等、R09/R10 SQS、R15投递契约 | +| [rust-rewrite-feasibility](./rust-rewrite-feasibility.md) | Feasibility:LocalStack Rust 重写分析 | +| [smithy-codegen-all-services-design](./smithy-codegen-all-services-design.md) | Design:多服务 Smithy 代码生成 | +| [service-operations-gap-impl-plan](./service-operations-gap-impl-plan.md) | Impl plan:操作覆盖差距与分期 | +| [ruststack-pulumi-target-design](./ruststack-pulumi-target-design.md) | Design:Pulumi provider endpoint 集成 | +| [ruststack-pulumi-hackathon-app](./ruststack-pulumi-hackathon-app.md) | Validation:serverless 应用拓扑验证 | + +## 服务设计 + +| 服务 | Specs | +|---|---| +| S3 | [早期实现](./ruststack-s3-implementation.md)、[Smithy 重设计](./smithy-s3-redesign-design.md)、[Checksum parity](./s3-checksum-parity-design.md) | +| DynamoDB | [设计](./ruststack-dynamodb-design.md)、[Streams](./ruststack-dynamodbstreams-design.md) | +| SQS | [设计](./ruststack-sqs-design.md)、[Long-poll / DashMap safety fix](./fixes/sqs-longpoll-dashmap-safety.md) | +| SSM | [Parameter Store](./ruststack-ssm-design.md) | +| SNS | [设计](./ruststack-sns-design.md) | +| Lambda | [服务设计](./ruststack-lambda-design.md)、[Executor](./ruststack-lambda-executor-design.md)、[Squib runtime](./ruststack-lambda-squib-runtime-design.md)、[S3 code packages](./ruststack-lambda-s3-code-design.md) | +| EventBridge | [设计](./ruststack-events-design.md) | +| CloudWatch | [Metrics](./ruststack-cloudwatch-design.md)、[Logs](./ruststack-logs-design.md) | +| KMS | [设计](./ruststack-kms-design.md) | +| Kinesis | [设计](./ruststack-kinesis-design.md) | +| Secrets Manager | [设计](./ruststack-secretsmanager-design.md) | +| SES | [设计](./ruststack-ses-design.md) | +| API Gateway V2 | [设计](./ruststack-apigatewayv2-design.md) | +| IAM | [设计](./ruststack-iam-design.md) | +| STS | [设计](./ruststack-sts-design.md) | +| CloudFront | [Management plane](./rustack-cloudfront-design.md)、[Data plane](./rustack-cloudfront-dataplane-design.md) | + +## Runtime Snapshot + +| Spec | 类型 / 用途 | +|---|---| +| [ruststack-snapshot-prd](./ruststack-snapshot-prd.md) | PRD:用户可见的命名快照契约 | +| [ruststack-snapshot-design](./ruststack-snapshot-design.md) | Design:早期 JSON 布局、服务边界与生命周期背景 | +| [ruststack-snapshot-binary-archive-design](./ruststack-snapshot-binary-archive-design.md) | Design:当前二进制归档目标 | +| [ruststack-snapshot-impl-plan](./ruststack-snapshot-impl-plan.md) | Impl plan:依赖顺序与实施阶段 | +| [ruststack-snapshot-verification-plan](./ruststack-snapshot-verification-plan.md) | Verification plan:快照测试与验收 | diff --git a/specs/rustack-data-correctness-design.md b/specs/rustack-data-correctness-design.md new file mode 100644 index 0000000..c19e73c --- /dev/null +++ b/specs/rustack-data-correctness-design.md @@ -0,0 +1,99 @@ +# 数据正确性与跨服务投递设计 + +状态:实施契约。范围:系统审查 R07/R08/R09/R10/R15;实施任务 M2.1、M2.2、M3.1。 + +## 1. 权威与边界 + +本设计细化 [系统审查](./rustack-system-review.md) 和 [实施计划](./rustack-system-hardening-impl-plan.md)。协议依据 [DDB](./ruststack-dynamodb-design.md#116-transaction-operations)、[SQS](./ruststack-sqs-design.md)、[Events](./ruststack-events-design.md)、[SNS](./ruststack-sns-design.md)。研究依据 [DDB API](../docs/research/dynamodb-api-research.md#35-transaction-operations-2)、[SQS API](../docs/research/sqs-api-research.md#74-dead-letter-queues-dlq)。旧设计中的 Noop 成功、unsupported 只日志策略由本设计明确替代。没有新增依赖,不实现网络 SNS、WAL、跨进程 exactly-once 或消息快照持久化。 + +## 2. DDB:协调、prepare、commit + +### 2.1 所有操作的同一隔离域 + +现有 provider 为同步 API,HTTP handler 同步调用,且 snapshot 也是同步入口。保留 API,采用 provider 唯一 `parking_lot::Mutex<()>` 操作门(已有依赖),只协调临界区,不将数据、map、非 Send 状态包入锁。状态继续由已有 service/table storage 持有。所有公开 handle 操作(普通读写、batch、Query/Scan、元数据 CRUD 和 transaction)先取得操作门;内部组合调用使用私有 `*_inner`,禁止递归获取门。reset/import/export 同样协调。不存在 transaction-only 锁。 + +选择理由:将这些同步 API 强行转换为 Tokio actor 会使同步调用在当前线程 runtime 阻塞/死锁,或要求全 HTTP/snapshot API 异步迁移,超出本 slice。单位锁是同步临界区协议,不是 `Mutex`,没有锁跨 await;串行化全部本地 DDB 操作换取可审查的串行化隔离。未来 actor 迁移可沿相同命令边界完成,不改变本不变量。HTTP handler 在 spawn_blocking **之前**从 provider 共享的128槽 semaphore取得 RAII permit,permit移入blocking closure(HTTP future取消不释放它),操作结束才释放并notify。`RustackDynamoDB::quiesce().await -> Result<(), DynamoDBError>`先close admission、排空permit,再通过blocking barrier等待已执行的同步provider操作。普通handle在操作门内检查admission关闭,排队未开始的操作显式失败;snapshot/reset/import是父控制的管理面,仍允许在quiesce后持门执行。父在HTTP关闭后Events/SNS之后、snapshot之前调用DDB quiesce,外包总deadline;quiesce取消不重新开放准入,可重试。新provider构造即restart,不复用已停机实例;不修改父拥有的gateway。 + +```text +普通 CRUD / batch / Query / Scan / transaction / snapshot + │ + ▼ + ┌─ 唯一 provider 操作门 ─────────────────────┐ + │ token lookup/admission │ + │ validate keys / all expressions/conditions │ + │ prepare Vec │ + │ failure ──► discard (no data/stream) │ + │ success ──► infallible storage commit │ + │ ──► publish stream │ + │ ──► token completed(now) │ + └────────────────────────────────────────────┘ +``` + +### 2.2 Prepare 产物和不可失败提交 + +先检查1..100 action、每项恰好一种 action、表存在、重复(table,key),合法 key 类型/非空/长度及 key-only 请求无多余属性。逐项解析并计算条件和 UpdateExpression,验证名称/值占位符、路径冲突、不可修改主键、数值/空集合/400KiB item/4MiB transaction 限额。prepare 只生成已计算最终 item、已解析主键、旧 image 与 table Arc,不修改 storage、不发 stream。ConditionCheck 同样在协调域里求值。 + +提交 storage API 接收已验证主键,不重新解析、不返回可恢复错误。所有 mutation 完成后才发布 stream。BatchWriteItem 同样先构建已验证 mutation 集合,共用不可失败提交与提交后 stream 路径,避免 malformed 后项让整个请求报错却已写入前项;这不承诺 AWS 批操作本身具有跨请求事务属性。删除不存在 item 不发布 Remove;更新/put 发布一条 Insert/Modify。失败 prepare 无部分数据、计数或 stream;普通条件写也在整个检查与写入期间持门。TransactGetItems 整体持门,故不可能看到两次提交混合。snapshot 导出在门内获得一致切面;导入应先构建合法替换状态再发布。 + +### 2.3 Token 状态 + +scope 为 provider 的 account/region。token 1..36 bytes,指纹是去掉 token 后请求的稳定 JSON(递归排序对象键)的 SHA-256,复用已有 rustack-auth::hash_payload,不新增依赖;覆盖所有事务内容与返回选项,避免随机 HashMap 顺序或弱哈希碰撞。最多1024记录,以 DashMap 存放,操作门下读写;每条仅保存64字节十六进制摘要、完成时间和响应,不保留请求正文。指纹序列化上限8MiB,另有每事务4MiB/100操作预算。可测试配置小容量/受控时间。 + +门的排他性使进入 prepare 的事务唯一:同 token 并发请求等待门,第一条失败不记录;成功后第二条重放已存结果而不求值条件/不写/不发 stream。容量准入在任何 mutation 前;只删除完成时间距今不少于600秒的成功记录;不能驱逐未过期成功记录。当前实现无独立在途条目:门内的新 token 已保留一个容量槽,门外请求不能进入或淘汰它,因此等价在途保护。窗口从 commit+stream 完成计时,重试不延长窗口;冲突返回 IdempotentParameterMismatchException;满时新 token 返回明确限流错误,已有 token 仍重放;没有 token 的事务不占token容量。reset/import 清除token,token不承诺跨重启幂等。 + +## 3. SQS DLQ:路由经理与actor异步移交 + +保留每队列actor唯一消息状态所有权,队列 registry 持有可克隆 routing handle。源actor不等待目标actor:通过manager路由的有界 `try_send` 将 Transfer 命令送目标,并保留消息及 oneshot acknowledgment 到 pending handoff;正常tick轮询ack。目标actor同步校验/入队/回复,其后source删去保留副本。禁止 actor A await B 或为每条消息spawn任务。 + +```text +source available ── receive_count >= max ──► reserve pending (FIFO保持group blocked) + ▲ │ manager resolve exact ARN + │ target missing/closed/full ▼ + └──── retryable restore ◄── negative ack / try_send failure + │ bounded Transfer command + ▼ + target actor validate/enqueue + │ success acknowledgment + ▼ + source release reservation/group +``` + +源仅在目标明确成功后忘记消息。关闭/找不到/队列类型不同/消息超过目标限制/目标命令容量满时恢复源可见并保留receive_count,下一次receive重试,不能把失败消息交给普通consumer绕过redrive。不用不可消费dead_letters Vec。pending有固定上限128,每次receive最多10;满时不移走消息。pending纳入不可见计数;FIFO组在成功/失败恢复前保持阻塞,失败恢复到组头。目标保留body、attributes、group及source ARN;重置receive_count/first_receive_timestamp,FIFO dedup用源消息ID防正常dedup误吞,standard保留sent timestamp,FIFO迁移重设sent timestamp。移交的enqueue不再应用源阈值。 + +目标 Transfer 不执行网络IO;ack无超时重放歧义:通道关闭说明命令未确认,只有目标未提交才允许恢复。实现必须保证目标提交与成功ack的相邻同步步骤,源actor在pending存在时不提前停止。runtime quiesce停止新hand-off并排空已有pending,保存后才shutdown队列;不能用shutdown_all作为quiesce(它清空消息)。显式Delete/Purge仍是用户要求删除数据,不承诺保留被删除队列的消息。消息仍不随资源型snapshot持久化。 + +## 4. FIFO去重键 + +`DedupKey::Queue(String)` 与 `DedupKey::Group { group: String, id: String }` derive Eq/Hash。禁止字符串拼接。两个scope属于不同键空间,切换scope不会命中旧scope的entry。合法标点不收紧;(a:b,c)、(a,b:c)都入队,同pair仅一次,同id不同group不互相去重。 + +## 5. Events/SNS:完整参数和能力 + +### 5.1 配置与wire + +Events直接存完整 `Target`,ListTargetsByRule clone原模型,不重建丢字段。SqsParameters为显式结构,AWS目前字段 `MessageGroupId`(1..128 ASCII可打印非空字节,保留标点),未知字段拒绝,不把JSON Value无校验传下游。TargetDelivery接收完整Target及已转换body,且提供同步validate方法。仅SQS target可执行;unsupported ARN/参数明确失败PutTargets的该entry。暂不执行RoleArn、retry_policy、dead_letter_config及非SQS参数,拒绝而不是暗示受支持。Input/InputPath/InputTransformer仍按现有实现,但互斥且限制长度/数量。FIFO target必须有MessageGroupId;bridge真实SendMessage携带该值,使用SQS content-based dedup,若目标未启用则得到明确delivery failure(不得凭空生成AWS不支持的SqsParameters字段)。 + +UnavailableTargetDelivery/UnavailableSqsPublisher替代生产Noop,validate与deliver都返回Unavailable。应用registry声明Events/SNS仅有SQS投递能力;未启用SQS时资源CRUD可工作,但配置SQS target/subscription返回显式错误,readiness显示degraded。core不依赖其他service core,bridge仍在app中。 + +SNS仅支持SQS订阅执行(每topic最多128订阅);不支持的protocol在Subscribe拒绝;SMS/platform publish明确unsupported(对应CRUD metadata可保留)。SQS publisher同步validate精确ARN/scope;FIFO topic→FIFO queue保留group/dedup,FIFO topic→standard queue不传FIFO-only参数,standard topic→FIFO queue明确unsupported。SNS envelope保留属性;raw delivery当前仅支持无message attributes,带属性的raw publish明确拒绝(不能默默丢属性)。RedrivePolicy/DeliveryPolicy/SubscriptionRoleArn在Subscribe和SetSubscriptionAttributes均明确拒绝。 + +### 5.2 有界接受、终态、停机 + +Events保持PutEvents异步接受语义,每event匹配形成有界delivery batch,最多128目标,每事件body上限256KiB;入有界128槽actor channel成功才返回event_id,满/停机/无runtime该entry返回明确错误且不发生任何该事件投递。单worker按入队顺序处理,target每次投递5秒deadline,失败不重试(最多一次尝试、明确终态;配置retry被拒绝)。单event目标顺序确定,FIFO同目标事件不会因无界spawn乱序。worker用受控子任务执行一次调用并await,panic/timeout/失败分别记录为failed,不杀死worker。 + +公开delivery_stats显示accepted/delivered/failed/rejected,pending=accepted-delivered-failed(delivery attempt口径另明确);不记录body/secret。quiesce关闭新接受、让已接受工作完成并等待worker确认,一致切面后没有后台变更。shutdown幂等、同样等待结束;父runtime外包总剩余timeout,超时必须非成功且不保存一致快照。构造器保持同步,可在无runtime做metadata操作;worker首次提交lazy启动,无runtime拒绝而不是panic。 + +SNS当前publish在请求内await fanout,保留同步执行而非人为改异步承诺;在开始任何delivery前获取有界128个publish permit,每次下游调用5秒timeout。quiesce关闭permit准入并等待全部已接受publish结束,shutdown同义,统计成功/失败明确可查询。无每请求detached spawn。失败delivery不能count delivered;unsupported不得返回伪装的SMS message_id。 + +## 6. 回归与退出 + +- DDB:后序坏表达式/类型运算/修改主键/过大item使前序Put不可见且stream零;两个barrier竞争条件事务至多一个;普通写与事务竞争、transaction read无撕裂;顺序/并发相同token仅一次增量和stream;内容冲突、TTL边界、容量满首token仍可重放。 +- SQS:真实provider创建source+target,首次receive、visibility=0、下次receive触发;目标收到body/source ARN,源不可见;standard/FIFO都测;目标缺失/删除/命令容量满不丢;FIFO碰撞与scope切换。 +- Events:PutTargets/List完整SqsParameters roundtrip,真实app bridge到FIFO接收到group=g;无SQS/unsupported明确失败;容量满拒绝、慢目标timeout、panic计failed、quiesce后无接受/变更。 +- SNS:真实SQS fanout保留FIFO参数;unsupported protocol/SMS错误;unavailable配置错误;quiesce和失败统计。 +- targeted cargo build/test/clippy,nightly rustfmt仅改动文件,不动根Cargo.lock/Toml/Makefile。父负责全workspace门禁和独立review;本worker不委派、不commit。 + +## 7. 父runtime接线 + +1. main禁用SQS分支用Unavailable*而非Noop;保留Events/SNS provider Arc于runtime registry。 +2. 停止HTTP准入后先Events/SNS quiesce,再SQS handoff quiesce,最后snapshot;保存后SQS shutdown_all。每步消费同一总deadline,失败不得宣称保存一致切面。 +3. readiness/capabilities披露SQS依赖与仅SQS目标支持、消息不持久化;stats接diagnostics。此设计不要求app snapshot扩大范围。 diff --git a/specs/rustack-http-hardening-design.md b/specs/rustack-http-hardening-design.md new file mode 100644 index 0000000..8b9aa73 --- /dev/null +++ b/specs/rustack-http-hardening-design.md @@ -0,0 +1,82 @@ +# HTTP trust boundaries and IO hardening design + +Implements [system review](rustack-system-review.md) R02/R03/R04 and service-side R05; gateway connection/admission and operator YAML wiring belong to the root integrator. Budgets follow [implementation plan](rustack-system-hardening-impl-plan.md) §3. Prior art: [S3 streaming research](../docs/research/s3s-crate-research.md), [checksum protocol](s3-checksum-parity-design.md). + +## Request pipeline and ownership + +```text +Gateway: validated config → connection/request admission (root owner) + │ +HTTP protocol: route → resolve AuthMode → bounded frame reader + │ ├─ byte overflow → protocol error, drop source + │ └─ idle/total deadline → error, drop source + ▼ + actual payload SHA256 → signature verification + ▼ + decode → handler → response +Proxy: configured URL → no redirects/no environment proxy → bounded upstream frames + └─ failure cancels response, never inserts partial cache +``` + +No business handler executes after auth/body validation failure. Early source enforcement is mandatory: Content-Length is only an optimization, never the authority. Every DATA frame is checked before copying into an aggregate. Transport failures are propagated, not interpreted as empty EOF. Cancellation drops the owned source; no detached reader tasks. + +## Authentication API and compatibility + +`rustack-auth::AuthMode` expresses `Development` or `Required(&dyn CredentialProvider)`; resolving legacy `(skip_signature_validation, Option)` rejects strict+None. Keep existing public service config fields for source compatibility; root startup must reject the same invalid combination before listening. All 19 HTTP adapters, including formerly ignored CloudFront config, use this rule. Strict S3 anonymous requests are rejected; presigned and SigV2 remain explicit S3-only paths. + +Ordinary `verify_sigv4` accepts an actual body digest, never a client-selected digest. At most one x-amz-content-sha256 header is allowed; concrete values are exactly 64 lowercase hexadecimal bytes and equal the digest of the received bytes. Missing hash headers use actual digest. Duplicate Authorization headers are rejected. No Authorization, expected signature or supplied signature is emitted to tracing/errors. The separately named `verify_s3_sigv4` entry point allows exact `UNSIGNED-PAYLOAD` and authenticates seeds for `STREAMING-AWS4-HMAC-SHA256-PAYLOAD`, its `-TRAILER` variant, and `STREAMING-UNSIGNED-PAYLOAD-TRAILER`. `StreamingVerifier` verifies each SHA256/HMAC chain element (including zero-size terminal chunk) and the canonical declared trailer block before publication. Unknown markers, including previously accepted but never cryptographically implemented SigV4a/ECDSA markers, fail closed rather than claiming verification. This is not a blanket removal of streaming support. S3 unsigned trailer encoding must decode and validate declared checksums before publication; generic JSON/Query adapters reject all placeholders. + +## IO API and budget matrix + +Shared `rustack-core::http` owns a validated `BodyBudget`, a `BudgetedBody` (actual DATA byte count, absolute total deadline, reset-on-progress idle deadline), and `collect_body`. Validation rejects zero limits/durations. Default control-plane budget is 16 MiB / 30 s total / 5 s idle. Lambda code JSON envelope is 96 MiB; invoke requests use the existing synchronous/asynchronous service limits and cannot inherit the code-upload allowance. S3 XML/control-plane remains 16 MiB. Object upload paths require 5 GiB streaming with bounded frames/storage, not a 5 GiB `Bytes` allocation. S3 integration now stages PutObject and UploadPart with `UploadWriter`/`StagedUpload`: private RAII tempfile, incremental MD5/CRC32/CRC32C/CRC64NVME/SHA1/SHA256, and 64 KiB disk read buffers. The raw encoded stream has a separate ceiling of twice decoded capacity plus 16 MiB for framing; decoded writes enforce the exact configured object ceiling. Chunk lines are capped at 8 KiB; at most eight allowlisted trailers, declared exactly once, no duplicate header/trailer checksum. The S3 total deadline defaults to one hour with 5 s idle; control total remains 30 s. The HTTP reader enforces actual bytes before the staging writer, not after receiving a full body. + +Upstream response default is 64 MiB, 30 s total and 5 s idle. Reqwest clients disable redirects and environment proxy inheritance and set connect/request deadlines. Readers use `Response::chunk`, check remaining capacity before append and abort immediately on overflow; advertised lengths do not replace accounting. CloudFront's configured smaller response limit remains authoritative. APIGW integration timeout may tighten but never remove the default deadline. Ordinary control responses use protocol-native error envelopes. Budget errors must not be emitted as successful responses. + +## Proxy URL/protocol policy + +Only operator-configured integration/origin determines initial authority. Parse URLs, allow HTTP/HTTPS only, disallow credentials/fragments; request path cannot replace authority. Local HTTP and private/loopback destinations are intentional local-emulator operator configuration exceptions, not arbitrary viewer-selected egress. Do not follow any 301/302/303/307/308 or relative/protocol-relative Location. Forward original status and Location unchanged. Never inherit HTTP_PROXY/HTTPS_PROXY/ALL_PROXY. This closes redirect destination expansion without claiming a general DNS-pinning/remote-host isolation feature. + +## Verification and exit evidence + +- R02: every adapter strict+None rejects before handler; required credentials accept correctly signed request and reject missing/malformed signatures. Development remains explicit. +- R03: known signing vectors; changed payload; injected unsigned hash header; malformed/duplicate hash headers; absent header; ordinary placeholders rejected; S3 exceptions are narrow; tracing has no signature material. +- R04: loopback A returns each 30x pointing to B (absolute, relative, protocol-relative, IPv6); B receives zero requests and response preserves Location; no second response cached. Client construction statically requires no-proxy/no-redirect. +- R05: B bytes pass, B+1 bytes fail during frame read even without Content-Length; oversized frame is not copied; pending source reaches idle/total deadline; progress cannot reset total deadline; dropping read releases source. Slow/oversized upstream rejected before aggregate growth exceeds budget. S3 streaming tests cover staging cleanup, range download chunk sizes, bad-checksum overwrite preservation and incremental multipart assembly. +- Targeted cargo checks/tests use managed background jobs; root owner runs workspace gates and maintains Cargo.lock/Makefile/apps. Independent review is performed by root owner (this worker must not delegate). + +## Integration changes and evidence log + +Shared helper adds `pub mod http;` to rustack-core. Dependency additions only reuse workspace crates and were coordinated with the root owner. No gateway/main changes are owned here. Root settings supplies validated body/upstream byte, idle, total and S3-specific budgets; `BodyBudget` constructors consume them. `append_bounded` caps buffer capacity growth, not only used length. Native protocol error envelopes preserve 413 for overflow, 408 for idle/total timeouts, and 400 for broken transport. + +### S3 publication and public integration contract + +```text +Incoming DATA ─► encoded byte/deadline budget ─► AWS chunk parser / HMAC chain + │ bounded pieces, no full object + ▼ + UploadWriter (private tempfile) + │ finish + all digest/trailer checks + ▼ + Arc ─► S3Handler::handle_staged_upload + │ core validates MD5/checksum/metadata + ▼ + storage publication + │ immutable shared file + metadata + GetObject ─► StagedRead ─► bounded FileBody ─► viewer +``` + +`S3Handler` adds `handle_staged_upload(parts, Arc, ctx)` with a rejecting default, so an old custom buffered handler never receives an empty upload disguised as a valid request. The application bridge implements the method; generated model blobs stay unchanged. Core adds staged PutObject/UploadPart and streaming GetObject entry points. S3 HTTP adds `S3ResponseBody::Streaming` and `from_staged`; immutable file ownership remains alive through response completion. CopyObject and whole-object UploadPartCopy reuse immutable staged data; multipart assembly copies bounded pieces to a new staged artifact rather than concatenating all parts in RAM. HTTP->S3-core dependency is intentionally limited to this existing storage/checksum domain contract, as CloudFront's HTTP layering already depends on its core. There is no core->HTTP edge or cycle; moving the S3-specific checksum/tempfile contract into shared rustack-core would unnecessarily add all S3 crypto/storage dependencies to every service. An external custom S3Handler now needs the new method for uploads; this explicit compatibility change was sent to the root integrator. + +APIGW `RustackApiGatewayV2::new` returns `Result` because constructing the isolated client is fallible and must not panic/fallback. CloudFront `dispatch_s3_origin` accepts a final `max_body` argument and rejects a staged object exceeding the proxy cap before reading the file. Root main/API constructor calls and gateway ingress use these contracts. + +### Verification log + +- First auth/core unit run: 42/43 auth passed; missing Authorization incorrectly mapped to InvalidAuthHeader. Corrected duplicate detection to reject count >1, preserving MissingAuthHeader. +- Early S3/auth/core targeted `cargo check`: passed. +- Two test/clippy attempts observed root-owned settings.rs mid-edit (missing module/delimiter and aliases); no worker changes to root-owned code were made. +- One cargo attempt was sandbox-denied during concurrent rustup self-update cleanup. Root owner completed the external cleanup and reran the exact command; no alternate cargo path or permission workaround was used. +- Root-assisted auth46/core16 passed; S3-core missing-part variant regression corrected from Internal to InvalidPart. +- S3-core loopback-default provider test updated consistently with root-approved safe default. +- Managed `bash-64`: `cargo test --lib` for all 18 *-http crates plus auth/core/S3-core/CloudFront-dataplane/APIGW-core passed. New regressions include SQS handler-not-called, tampered/injected hashes, signed S3 chunk corruption, unsigned trailer checksums, 17 MiB staging in 64 KiB frames and temp-file deletion, all 30x classes with absolute/relative/protocol-relative/IPv6 targets, chunked proxy overflow and never-EOF upstream deadline. +- Targeted nightly rustfmt completed successfully (explicit owned paths, not workspace formatting). +- Later targeted clippy exposed moved legacy operation allowances and a new long upload parser; the legacy annotations were moved with their existing code, and the new parser was factored into bounded protocol stages rather than suppressed. Final gate results are appended at handoff. diff --git a/specs/rustack-lambda-hardening-design.md b/specs/rustack-lambda-hardening-design.md new file mode 100644 index 0000000..0916d82 --- /dev/null +++ b/specs/rustack-lambda-hardening-design.md @@ -0,0 +1,39 @@ +# Lambda hardening design + +Status: implementation contract for review R01/R06/R11/R12 and Lambda R16. Supersedes unsafe defaults and mutable layout in [executor design](ruststack-lambda-executor-design.md); retains [Cargo Lambda Runtime API contract](../docs/research/spike-cargo-lambda-runtime-execution.md). + +## Boundaries and API + +FunctionName is a private validated newtype: 1–64 ASCII alphanumeric, underscore or hyphen bytes. Qualifier is `$LATEST`, positive decimal version or 1–128 ASCII alphanumeric/underscore/hyphen alias. Full ARN must have exactly seven/eight components, supported AWS partition, Lambda service, nonempty bounded ASCII region, twelve-digit account and `function` resource. Partial account ARN and simple qualified names remain supported. No percent-decoding occurs in core: encoded separators are invalid. Store insertion/update and snapshot import validate records independently of HTTP. Explicit qualifiers are checked even when a reference contains a qualifier. + +`LambdaConfig::from_env() -> Result` fails on malformed booleans, enums, numbers and conflicts; no environment setting and Default both choose Disabled. Disabled returns explicit unavailable error, never echo. Docker is unsupported and fails explicitly, never native/noop fallback. Native and Auto require explicit operator selection and are trusted-host execution, not a sandbox. + +## Immutable artifacts and failures + +```text +external name/ARN + ZIP + │ validate identifiers and size + ▼ +store-owned random artifact ID (never function name) + │ private staging / code.zip / extracted + │ validate ZIP, paths, entry modes, CRC, actual expanded bytes + ├── failure ──► remove only new staging, return ZIP/IO error + ▼ +complete immutable artifact ──► atomic metadata reference replacement + │ │ +published version keeps reference latest switches to another artifact +``` + +Each upload receives an unguessable internal UUID directory. Paths do not join logical function names or qualifiers. Reject symlinks in root ancestry and artifact traversal; ZIP symlinks, absolute/parent/backslash entries and duplicate paths are rejected. Root is operator-owned; native execution is explicitly trusted and cannot be treated as protection against a malicious same-user process racing filesystem operations. Extraction runs on bounded blocking workers, maximum ZIP 50 MiB, expansion 250 MiB, maximum 10,000 entries. No validation or IO failure is swallowed. Staging lifetime owns cleanup on error/cancellation. Completed artifacts are retained while versions/in-flight requests can reference them; cleanup never recursively deletes a name-derived path. Snapshot restore stages all records first, validates names/ARNs/versions/aliases and package metadata, and only then replaces records. It never clears the code root before validating input. + +## Revisions, admission and lifecycle + +Warm identity includes unique artifact path plus execution configuration fingerprint and resolved version. New uploads/recreation receive distinct artifact IDs; configuration changes alter the fingerprint. Published versions share only immutable references. Existing invocations finish on their captured revision; no new invocation reuses old revision processes. Warm pool is bounded globally to 32 and per function to 1, independent of execution permits. + +Provider admits at most 32 execution requests globally and default 8 per logical function across versions; reserved concurrency overrides default, 0 rejects. Event work is held in a tracked bounded task set (128 slots): capacity is acquired before 202; queueing/execution owns its capacity until terminal outcome. Execution permits are acquired immediately before execution, retained through timeout/cancellation and released by RAII. Synchronous overload is an explicit throttling error. Event workers may wait for execution capacity within the 128 accepted-work budget. Supervisor records completion/error/panic/cancellation without logging payloads. + +`RustackLambda::quiesce(timeout: Duration) -> Result<(), LambdaServiceError>` stops admission, waits for accepted synchronous/background work, and cancels remaining work on deadline with an explicit failure. It does not delete resource metadata. `shutdown()` cancels/joins accepted work before stopping executor resources. Parent runtime calls quiesce before snapshot and shutdown after snapshot; timeout means do not publish a successful consistency snapshot. + +## Verification + +Real minimal ZIPs replace historical fake PK blobs. Test illegal name/reference/qualifier at resolver, store, provider and snapshot boundaries; temporary outside sentinels and symlink roots stay unchanged. Bad ZIP/CRC/path/size/IO update preserves prior hash/revision/files. A/B artifacts preserve published A and latest B; warm tests include configuration changes and delete/recreate. Admission tests cover 32/8, reserved=0, 128 Event saturation, cancellation and quiesce deadline. Native bootstrap fixture tests use host-built binaries; Linux Cargo Lambda ELF is not executable natively on macOS. All targeted verification explicitly uses workspace CARGO_TARGET_DIR and managed jobs. Parent owns root manifest/lock, HTTP and application runtime integration. diff --git a/specs/rustack-runtime-hardening-design.md b/specs/rustack-runtime-hardening-design.md new file mode 100644 index 0000000..c6384f2 --- /dev/null +++ b/specs/rustack-runtime-hardening-design.md @@ -0,0 +1,55 @@ +# Runtime hardening:配置、准入、停机与快照 + +状态:实施契约。依赖:[review](./rustack-system-review.md) R05/R13/R14/R16/R17/R18、[实施计划](./rustack-system-hardening-impl-plan.md)。不替代领域状态的原子性,也不扩展 snapshot 数据范围。 + +## 1. 一次解析配置 + +`RUSTACK_CONFIG` 指定 YAML 文件;未指定时不要求文件存在。配置 schema:`environment` 为已有环境配置名称到标量值的映射(兼容现有服务变量命名),`budgets` 为 camelCase 的强类型资源预算,`advertisedEndpoint` 为可选本地公共入口。顶层未知字段拒绝、environment未知键拒绝,数组/嵌套对象不允许伪装 scalar;值先转换为字符串后执行同一范围/枚举/布尔/字节校验。真实进程环境覆盖 YAML,默认值最后。不能通过 `set_var` 注入:使用一次安装的已验证 settings facade,各 service config 的 `env::var` 改经该 facade,测试未初始化时仍能独立构造配置。 + +读配置使用 config crate 的 YAML source(显式feature),错误含 key 和来源、不得含 secret 值。`--help`/`--version` 不触发文件/网络;运行和 `--health-check` 使用同一有效配置。位置参数、未知服务名、重复/冲突 flag 拒绝。任何 strict signature=false skip 配置若缺配对credentials,在监听前失败;协议层仍独立 fail closed。 + +bind 默认 `127.0.0.1:4566`;advertised endpoint 默认由bind推导(wildcard变loopback),不能从请求Host信任推导,IPv6必须括号包围。Lambda等URL由有效endpoint生成;APIGW可保留AWS-shaped ApiEndpoint,但capabilities明确提供 `/_aws/execute-api/{api}/{stage}` 本地映射。 + +## 2. Gateway / supervisor + +`RuntimeStatus` 共享原子 serving/draining/failed 状态,capability records列出service、compiled/enabled、snapshot kind、执行与target能力;liveness证明进程能响应,readiness证明请求服务已安装且未draining/failed。配置了不可用的必要backend应fail startup或明确degraded,不把未执行runtime probe描述成已验证。 + +每连接获取permit(默认256);每业务请求获取独立permit(128),permit保留到响应body完成/丢弃,不在拿到response headers后提前释放。health无需业务permit,但连接也有总限额;HTTP/2业务streams仍受请求permit覆盖。HTTP读头5秒,控制面总请求30秒;S3大对象由协议body idle/实际字节预算约束。错误在原body消费前也不能触发无界drain。超额503/协议错误附request id。 + +supervisor用JoinSet跟踪connection任务(panic/IO错误有日志和诊断)。SIGINT/SIGTERM共用停机路径;先标draining、停止accept,已有connection graceful drain在剩余总预算内完成。runtime worker registry独立于SnapshotService,包括Lambda/Events/SNS/SQS,按Events/SNS→Lambda→SQS顺序quiesce,禁止新的cross-service投递越过切面。失败/超时中止保存并保留旧快照;最终shutdown销毁worker,不重启后回报假成功。正常save必须发生在quiesce后;unsupported persistence只有摘要,不能承诺全部消息恢复。 + +```text +┌─ immutable validated config ──┐ +│ YAML < env; budgets; endpoint │ +└─────────────┬────────────────┘ + ▼ +┌─ Gateway ───────────────────────────────────────────────────────┐ +│ connection permits → request permits → protocol IO budgets │ +│ health/status ◄─ RuntimeStatus │ +└─────────────┬────────────────────────────────────────────────────┘ + ▼ +┌─ Supervisor (independent of snapshots) ──────────────────────────┐ +│ JoinSet connections + typed worker handles │ +│ SIGINT/TERM → drain → Events/SNS → Lambda → SQS quiesce │ +│ failure ──► no save, report nonzero exit │ +│ success ──► snapshot.save → workers.stop │ +└──────────────────────────────────────────┬───────────────────────┘ + ▼ + previous / next snapshot +``` + +## 3. 快照恢复协议 + +保留当前 `/manifest.ss.zst` 用户目录及旧文件格式。覆盖写使用稳定 `.previous` recovery目录(内部隐藏名),受同名snapshot排他锁保护;写temp全部shards/manifest并sync,再target→previous,temp→target,再sync父目录,最后清理previous。发布过程中任何失败保留/恢复完整旧目录,不忽略restore失败。load前在锁内执行恢复:target缺失且previous存在则验证previous manifest与完整性并恢复;target存在且previous存在则验证target(失败回退previous),绝不静默空启动。没有target/previous才是初次空状态。并发两个Rustack写同名snapshot必须拒绝,不last-writer-wins,锁必须跨进程且异常退出能释放。输入快照路径symlink/components继续按archive既有边界验证。 + +测试注入target→previous之后失败、temp→target前/后失败、恢复失败、同时load/save、损坏目标/完好previous;恢复只能完整旧/新。该协议不是WAL,不保证SIGKILL前的最新请求被持久化,但必须保住旧已完成快照。 + +## 4. 探针和用户诊断 + +`/_health/live`是liveness,`/_health/ready`是readiness;兼容 `/_localstack/health`、`/_health`、`/health`与MinIO路径,支持GET/HEAD。capabilities用独立JSON端点,提供version、services状态、snapshot coverage、execution restrictions/local URL patterns。HEAD无body,content type/status与GET一致。probe解析真实HTTP status和JSON readiness,3秒总deadline、64KiB上限、不用正文substring;stderr包含connect/status/parse/timeout类别且不含secret。 + +新用户文档以package rustack-cli、binary rustack区分;5分钟walkthrough不含首次编译。能力表不使用routed数量代替实现,部分metadata-only明确列出。配置示例YAML+env优先级、loopback容器publish、native显式可信选择、快照资源/数据覆盖都可复制。 + +## 5. 规范与验收 + +Errors、Async、Safety、Serde、Testing、Tracing、Performance与Documentation按AGENTS.md对应章节。新边界禁止unwrap/expect/index panic/unsafe。public类型文档与脱敏Debug齐全,使用有界channel/JoinSet和非零预算。回归覆盖配置坏值、CLI冲突、IPv6、伪HTTP health、慢/超大响应、SIGINT/TERM子进程、deadline、连接/请求permit释放与snapshot故障恢复。后续执行记录附在实施计划,不在实现尚未验证时标implemented。 diff --git a/specs/rustack-system-hardening-impl-plan.md b/specs/rustack-system-hardening-impl-plan.md new file mode 100644 index 0000000..5806f8c --- /dev/null +++ b/specs/rustack-system-hardening-impl-plan.md @@ -0,0 +1,107 @@ +# Rustack 系统修复实施计划 + +状态:实施中。范围:[系统审查 R01–R18](./rustack-system-review.md),所有里程碑均为本次交付范围,不以 P0 修复代替全部完成。 + +## 1. 完成定义 + +每个 R 项同时交付代码、针对原失败链的回归、必要的配置/能力说明。保留 AWS wire 的大小写/协议;禁止以禁用测试、吞错、只改 README 为方式关闭已有领域行为缺陷。安全默认对 CLI 的兼容变化需明确迁移说明。代码提交不包含用户已有的工具链删除或无关文件。 + +工程规范以 [AGENTS.md](../AGENTS.md) 为准。实施前各组件将 review 细化为设计契约并更新本索引;设计可补充选择,但不能降低原不变量。用独立 reviewer 对最终 diff 逐项核查,不以分工作者自检替代。 + +## 2. 共同决定 + +- **D-H1**:默认 CLI bind 改为 `127.0.0.1:4566`;Lambda 默认 Disabled(不能默默 echo 成功),native/auto 需 operator 显式选择。Docker 如监听全网是容器内部 networking 需要,宿主示例端口仅发布到 loopback,并明示风险。 +- **D-H2**:严格认证必须有 provider。保留各协议现有配置入口以减少无关 public API 变更,但归一化为内部 AuthMode,非法组合在启动和每个 HTTP 服务边界均拒绝;不能只靠 main 启动检查。 +- **D-H3**:普通 SigV4 校验实际正文摘要;S3 特殊签名协议有独立受限策略,不能扩散到 JSON 服务。日志不记录 Authorization 或预期签名。 +- **D-H4**:HTTP proxy 不跟随重定向、不继承环境 proxy;原 30x/Location 转发给 viewer。仅按 operator 配置的 origin/integration 连接,允许本地模拟器明确配置的 HTTP fixture(记录为相对 AGENTS.md 默认 https-only 的本地集成例外),不增加任意 request 指定上游的能力。 +- **D-H5**:新代码目录不可变且与逻辑名称解耦。验证完 ZIP 再发布 artifact 引用;发布版本复用不可变 artifact,而非 latest 目录。warm key 包括 artifact/config revision。 +- **D-H6**:runtime supervisor 的成员不由 snapshot 支持与否决定。接受工作前获取有界容量,quiesce 先于 snapshot,最终终止在保存之后。不能清空队列再保存。 +- **D-H7**:快照发布采用可恢复的提交协议;正常新目录布局与现有 snapshot 可读性保留,不引入 WAL。恢复必须显式,不能把存在完整旧 backup 的情况解释为初次空启动。 +- **D-H8**:Data APIs 的原子性及 token 窗口在状态所有者实现;不能仅为 transaction 请求加锁、让普通写入绕过。DLQ handoff 通过 manager/actor 协议,不跨 actor 相互等待。 + +## 3. 公共预算与兼容规则 + +预算均为有效配置的一部分,使用非零数值和上限校验;环境覆盖 YAML,显式坏值不得 fallback。初始默认值:256 个已接入 HTTP connection、128 个业务 in-flight、5 秒读头、30 秒控制面请求、30 秒上游、30 秒总停机;SQS 长轮询可正常完成。小控制面默认最多 16 MiB,Lambda ZIP JSON envelope 最多 96 MiB,Lambda Invoke 仍按模式/服务限制;S3 对象输入按 5 GiB 上限有界 streaming,XML/控制面不得借此变成 5 GiB collect。上游响应默认最多 64 MiB,必须逐帧计数而非读后检查。body idle/总 deadline 与 HTTP/2 streams 同时受准入约束。健康/诊断小响应最多 64 KiB,探针总期限 3 秒。 + +Lambda 默认全局并发 32、每函数并发 8、全局空闲实例最多 32、每函数空闲最多 1;显式 reserved concurrency 覆盖函数默认且 0 拒绝调用。异步 Event 队列/工作集合容量 128;返回 202 前必须占用容量,任务成功/错误/取消均记录,退出必须等待或在预算内取消。最终数值属于保守安全默认,不声称代表经性能调优的吞吐上限。 + +快照只保留已声明持久化的服务/状态。所有已接受后台工作都受监督,但 SQS 消息和不支持持久化的服务并不因此获得跨重启保证;输出包含/遗漏摘要。能力状态为 implemented/partial/metadata-only/unsupported,健康的 liveness 与 readiness 分开。 + +## 4. 任务及里程碑 + +| 阶段 / 任务 | R 项 | 交付 | 退出测试 | +|---|---|---|---| +| M0.1 认证边界 | R02/R03 | 内部 AuthMode、所有验签入口 fail closed、实际摘要校验、脱敏 | 缺凭证/坏签名/篡改/重复哈希头 handler 不执行 | +| M0.2 artifact 安全 | R01 | 私有有效名称、ARN/qualifier 解析、受控物理路径和 symlink 防护 | 临时哨兵在恶意 CRUD/导入后不变 | +| M0.3 proxy 边界 | R04 | 禁止 redirect follow/environment proxy、规范 URL | 各类 30x 的第二上游请求数为零 | +| M1.1 IO 预算 | R05 | collect 前限制、流式计数、连接/请求限额、deadline | 超额和慢 body/upstream 在预算内退出且资源释放 | +| M1.2 Lambda 工作管理 | R06 | 有界 Event、执行 permit、reserved concurrency、warm 总量 | N+K 并发不超过 N、reserved=0、退出无遗留任务 | +| M1.3 supervisor / snapshot | R13/R14 | SIGINT/TERM、quiesce/drain/save/stop、有期限、恢复协议 | 真实子进程信号与发布步骤失败注入 | +| M2.1 DDB 事务与 token | R07/R08 | prepare/commit 隔离、提交后 stream、有界10分钟幂等状态 | 坏后项全回滚、竞争事务、读一致性、满缓存重放 | +| M2.2 SQS | R09/R10 | 可恢复 DLQ manager、结构化 dedup key | standard/FIFO 阈值、目标异常不丢、跨组碰撞回归 | +| M2.3 Lambda 发布 | R11/R12 | staging ZIP、传播错误、不可变版本、warm revision | A/B 冷热版本不变量、所有失败更新保旧 | +| M3.1 bridge | R15 | 完整 SqsParameters、明确 unavailable/unsupported、受监督投递 | FIFO group roundtrip/消费、缺依赖不假成功 | +| M3.2 config | R16 | 统一 YAML+env 校验、有效配置、bind/advertised endpoint | 拼写/范围/冲突拒绝,非默认端口/IPv6 URL | +| M4.1 health | R17 | 状态/能力 registry、GET/HEAD readiness、有界结构化探针 | 假响应/永不EOF/失败依赖/限额压力 | +| M4.2 developer UX | R18 | 修正 package/bin 示例、能力/快照声明、唯一目录索引 | 文档 walkthrough、链接、示例操作回归 | + +独立领域可并行实施,但合并时以上共享契约优先。每个组件设计记录实际结构/API,不能用本表的组件名替代实现设计。 + +## 5. 状态/生命周期 + +```text +┌─ Validated runtime ──────────────────────────────────────────────┐ +│ cfg/auth/budgets ──► Gateway admission ──► protocol adapters │ +│ │ │ │ +│ └────► Runtime supervisor ◄─────────┐ │ +│ │ │ │ +│ bounded jobs ──► Lambda / Events / SNS / SQS handoff workers ──┘ │ +│ │ │ +│ signal ──► stop ingress ──► quiesce ──► snapshot ──► stop │ +└─────────────────────────────────────────────────────────────────┘ +``` + +拒绝/错误不能触发后续业务变更;accepted 与 delivered 分开;先校验后入队,先持有 capacity 后返回接受。snapshot 出错返回非成功退出并保持上一份可恢复状态。 + +## 6. 质量门禁与交付 + +1. 每任务 targeted unit/integration 回归;仅使用临时目录、loopback fixture、无真实账号请求。 +2. 完成后运行现有 Rust build、test、nightly fmt、strict clippy,含 workspace/all-targets;运行 rustdoc broken-link 检查与必要 feature 组合、边界 lint。 +3. 依赖变化运行 cargo audit/deny,不能忽略失败或无依据消除诊断。 +4. 独立代码审查按 R-ID 给证据;有效缺陷全部修复再复验。 +5. 命名路径逐项 stage,保留用户其他改动;向现有 origin 推送独立分支并创建 PR,PR body 给每项实现/测试索引、兼容变化和任何真实未通过的外部条件。只有完成全部 R 项才使用 completed 声明。 + +## 7. 追踪入口 + +原始证据:[rustack-system-review.md](./rustack-system-review.md)。设计与验证证据通过 [index.md](./index.md) 导航;实现过程产生的新发现继续追加到原 review 的独立后续段落,不丢在聊天中。 + +## 8. 实施与验证记录(分支 fix/system-review-r01-r18) + +状态:R01–R18 全部交付并通过验证。本段如实记录已执行证据与剩余外部条件,不改写原审查段落。 + +### 8.1 交付范围 + +- M0:`rustack-auth` 内部 `AuthMode`,19 个 HTTP adapter fail closed;普通 SigV4 校验实际正文摘要;S3 流式/UNSIGNED 收窄策略与 trailer 校验;CF/APIGW proxy 无 redirect/无环境 proxy;Lambda 标识/路径/ARN 校验与不可变 artifact、发布版本引用、warm 指纹。 +- M1:`rustack-core::http` BodyBudget/BudgetedBody/collect_body;网关连接+业务信号量与 HTTP/2 流预算、响应体结束才释放;DDB permit 移入 spawn_blocking;Lambda 32/8/128/reserved=0/Event 1 MiB;显式 `Runtime` + `shutdown_timeout(0)`。 +- M1.3:`RuntimeWorkers` supervisor 顺序 quiesce→snapshot→并行 stop;SIGINT/TERM;快照排他锁、fsync staged、`.name.previous` 恢复、失败不空启动。 +- M2:DDB operation gate prepare/commit、提交后 stream、10 分钟 token SHA-256(满额不淘汰有效项);SQS manager/actor DLQ 移交、FIFO 结构化 dedup;Events/SNS unavailable/unsupported、监督投递与统计。 +- M3/M4:`settings` YAML+env 校验与 `RUSTACK_CONFIG`/`RUSTACK_ADVERTISED_ENDPOINT`;GET/HEAD health + `/_health/{ready,live}` + `/_rustack/capabilities`;有界结构化探针;README/Makefile/Dockerfile/docs 能力与安全指南。 +- 依赖修复:quick-xml 0.39→0.41(RUSTSEC-2026-0194/0195)、h2 0.4.13→0.4.19(RUSTSEC-2026-0258)、quinn-proto→0.11.17、anyhow→1.0.104(RUSTSEC-2026-0190)、chacha20/spin 去 yank。 +- 独立审查发现的 P2 全部修复:SQS 毒消息不阻塞健康消息(skip 失败项继续扫描)、SNS 下游 panic 隔离与 Subscribe 幂等容量顺序、Events unsupported 参数显式建模并拒绝、CF `unsafe map_unchecked_mut` 消除并 forbid unsafe、测试 wire 构造缺失尾部 CRLF 修复。 + +### 8.2 验证命令与结果(全部在显式 `+1.98.1` 工具链、共享 workspace target 下运行) + +- `cargo +1.98.1 check --workspace --all-targets --all-features`:通过。 +- `cargo +1.98.1 test --workspace --all-features --exclude rustack-integration`:119 组 1683 项全部通过;DDB/SQS/Events/SNS 域 393+,S3/HTTP 105,Lambda core 143 + 原生 A/B warm/Event 收割 fixture。 +- `cargo +nightly fmt --all -- --check`:通过。 +- `cargo +1.98.1 clippy --workspace --all-targets --all-features --no-deps -- -D warnings -A renamed-and-removed-lints -A unused_async -A clippy::unused_async -A clippy::unused-async-trait-impl -A clippy::result_large_err`:通过。允许的两类存量 lint(大量既有 async 无 await 的 handler、模型 Err 大类型)在基线中已存在且与本次领域行为无关,另两个 allow 仅为 lint 改名兼容;不掩盖本次新增代码问题(新增文件/改动已逐条修复至零告警)。 +- `cargo audit`:exit 0(仅 aws-sdk-s3 dev/test 传递的 lru 0.16.4 unsound 警告,修复需 sdk≥1.145,超出仓库保留工具链;生产路径不包含);`cargo deny check`:ok。 +- 真实进程冒烟(debug 二进制 + `--snapshot devsmoke`):health 200、capabilities 结构化、DynamoDB CreateTable/PutItem → SIGTERM → 同快照重启 GetItem/ListTables 数据恢复;退出码 0。 +- rustack-cli bin 单测新增:CLI 严格解析冲突、settings 解析失败矩阵、health probe 拒假状态/无 EOF/超限、快照恢复/排他锁/publish 保留旧代际、S3 17 MiB 流式 spool 与签名 trailer 篡改、events 注入 worker 崩溃 readiness 等。 + +### 8.3 诚实边界(非代码缺陷但应公开) + +- 未修改/不恢复用户删除的 `rust-toolchain.toml`;仓库提交的固定工具链沿用基线值(本次全部验证用显式 `+1.98.1`)。 +- 全 workspace pedantic 存量 lint(上表 allow)与 dev-only lru 警告不随本 PR 修复,原因如上。 +- Dockerfile 重写为 BuildKit 缓存/`--locked`/scratch 非 root,本机无 docker 未执行镜像构建;rust:1.94-slim 基镜像仅提供构建工具链,最终产物无工具链依赖。 +- `rustack-integration`(需运行中 server 的 aws-sdk 套件)未纳入默认 workspace 运行;真实进程冒烟已覆盖代表性 s3+dynamodb 路径。 diff --git a/specs/rustack-system-review.md b/specs/rustack-system-review.md new file mode 100644 index 0000000..b092e80 --- /dev/null +++ b/specs/rustack-system-review.md @@ -0,0 +1,427 @@ +# Rustack 系统审查与改进 Spec + +- **类型**:Review / 修复建议与验收契约 +- **日期**:2026-09-07 +- **状态**:静态审查完成;修复方案待实施,运行验收未执行 +- **基线**:`03599661f0a4c7cd2937e90691851fc65a5496df`,以审查时工作区源码为准 +- **范围**:安全、数据正确性、生命周期、跨服务架构、CLI / SDK / IaC 开发者体验 +- **入口**:[specs/index.md](./index.md) + +> **修复与验收后续**(2026-09-07):R01–R18 已在分支 `fix/system-review-r01-r18` 实现并通过验证。原审查发现与建议段落保持为历史结论不变;实现决策见四个 hardening 设计,逐项实现/测试/已知边界见 [实施计划 §8 实施与验证记录](./rustack-system-hardening-impl-plan.md#8-实施与验证记录分支-fixsystem-review-r01-r18)。 + +## 1. 目的与结论 + +Rustack 的核心价值不是复刻全部 AWS,而是让开发者在本地及 CI 中快速、可信地验证 AWS 应用。本次审查关注两类损失:恶意或错误输入影响运行 Rustack 的机器,以及本地“成功”掩盖真实的数据、投递或部署错误。 + +代码已有清晰的 `model / http / core` 分层、跨服务 trait bridge、SQS actor、部分资源校验和流式解压上限。优先问题不是拆更多 crate 或换更多依赖,而是补齐**边界验证、原子提交、有界执行、生命周期和能力声明**这些跨模块契约。 + +审查收敛为 **18 项**。最高优先级是 Lambda 路径越界、严格验签失效和 SigV4 正文完整性;其次是事务部分提交、消息不可达、Lambda 版本失真与可靠停机。不能把当前实例当作生产 AWS、IAM 授权测试平台或安全的多租户执行环境。 + +### 1.1 方法与可信度 + +- 沿 HTTP → handler → provider → storage / executor / bridge 调用链核查,而不是按 `unwrap`、`spawn` 搜索数量推断漏洞。 +- 深查 gateway、auth、Lambda、DynamoDB transactions、SQS、EventBridge/SNS bridge、CloudFront/APIGW HTTP proxy、snapshot;其他服务的协议层和配置仅抽样。**不是全部 18 个服务、所有生成模型和算法的穷尽审计。** +- 查阅根目录 [AGENTS.md](../AGENTS.md)、现有 specs、`docs/research` 的 11 篇 memo 的相关章节;`vendors/localstack` 当前为空,未初始化 submodule 或把旧研究当作本次实测。 +- 问题中的代码行为均有静态依据;“触发 / 验收”是待实施的回归场景,**没有实际发送攻击请求、破坏文件、执行不可信 Lambda 或进行压测**。并发时序、进程退出与重定向影响仍需隔离环境运行确认。 +- 本次只修改 specs 文档,不修改执行逻辑、Rust 构建 / 测试 / 工具链契约。Rust build/test/fmt/clippy 验证不适用;不宣称 cargo audit 全量通过。 +- 开始时已有 `rust-toolchain.toml` 删除和 `.DS_Store` 未跟踪状态;均保留,不归因于代码缺陷,也不恢复用户改动。 + +### 1.2 优先级 + +| 级别 | 含义 | +|---|---| +| P0 | 外部输入可越过文件或认证完整性边界;共享 / 非可信网络使用前必须处理 | +| P1 | 高影响资源耗尽、数据错误、消息丢失或关键部署语义失真;稳定性里程碑必须处理 | +| P2 | 有限条件下的兼容性、可诊断性、体验问题;不能长期用“本地工具”解释 | + +这里的 P0–P2 是工程优先级,不是 CVSS。风险随网络可达性、进程权限、执行器和部署方式变化。 + +## 2. 信任边界与非目标 + +```text +外部 / 非可信输入 +┌─────────────────────────────────────────────────────────────────────────┐ +│ AWS SDK / CLI / IaC HTTP body / headers Lambda ZIP 上游 HTTP 30x │ +└──────────────┬───────────────────────────────────────────────┬──────────┘ + │ │ +┌──────────────▼──────────── Rustack 进程 ──────────────────────┼──────────┐ +│ Gateway: 路由 / 限流 / 健康 / 生命周期 │ │ +│ │ │ │ +│ ▼ ▼ │ +│ 各服务 HTTP + Auth ──► 已验证领域命令 CF / APIGW HTTP client │ +│ │ │ │ │ +│ ▼ ▼ │ │ +│ DDB 数据状态 SQS queue actor ◄─ SNS / Events bridge │ +│ 事务 / 索引 / Streams 队列 / 去重 / DLQ │ │ +│ │ │ │ │ +│ └──────────► SnapshotService registry │ │ +│ │ │ │ +│ Lambda provider ──► 代码包存储 ──► Executor / warm pool │ │ +└────────────────────────────┼─────────────────┼────────────────┼──────────┘ + ▼ ▼ ▼ + 宿主文件系统 / 快照 native 子进程 指定上游及内网 + 或 Squib VM 必须限制目的地 +``` + +**部署提醒**:`main.rs:815-816` 默认监听 `0.0.0.0:4566`;默认跳过验签;`LambdaConfig::from_env` 默认 `auto`(`config.rs:54-65`),非 macOS 的 auto 走 native(`executor/auto.rs:40-51`)。native 直接执行 bootstrap(`executor/native.rs:100-119`),清空继承环境并不隔离文件系统、网络和进程权限。这个组合意味着:可访问 API 的调用者可提交以 Rustack 权限运行的代码。 + +这不是 shell 注入或已证明的 VM 逃逸,native 无沙箱也是[既有执行器设计](./ruststack-lambda-executor-design.md)的明确非目标。但默认暴露面应收紧:安全默认监听 loopback、执行用户代码需显式选择可信 native 模式;容器对外发布端口需 operator 主动确认。过渡期至少禁用不需要的 Lambda、使用隔离测试账号/主机和最小文件权限,勿上传真实秘密。**`LAMBDA_EXECUTOR=disabled` 只关闭执行,不能阻止 R01 的代码存储越界。** + +其他非目标:不要求本轮实现生产 IAM、全部 AWS 服务、跨账户隔离、WAL 或完整服务持久化。不把“支持 metadata”当成“支持执行”;可以只模拟配置,但必须显式披露。 + +## 3. 问题总表 + +| ID | 优先级 | 问题 | 主要影响 | +|---|---|---|---| +| R01 | P0 | Lambda FunctionName 未限制路径语义 | 代码根外写入、递归删除 | +| R02 | P0 | 严格验签缺凭证时 fail-open | 未签名请求仍执行 | +| R03 | P0 | SigV4 声明哈希覆盖实际正文哈希 | 有效签名可搭配被篡改正文 | +| R04 | P1 | HTTP proxy 默认跟随重定向 | 有前提 SSRF、代理语义变化 | +| R05 | P1 | 请求 / 上游正文及连接缺少有效全程预算 | 内存、连接、任务资源耗尽 | +| R06 | P1 | Lambda warm 限额不限制 in-flight / Event | 进程、FD、后台任务无界增长 | +| R07 | P1 | DynamoDB 事务非原子、缺隔离 | 失败仍提交、条件竞争、撕裂读取 | +| R08 | P1 | DynamoDB 忽略 ClientRequestToken | 重试重复业务更新 | +| R09 | P1 | SQS DLQ 只移动到内部不可读 Vec | 消息从用户视角消失 | +| R10 | P2 | FIFO messageGroup 去重键有歧义 | 跨组错误去重 | +| R11 | P1 | Lambda ZIP 更新非原子且吞解压错误 | 错误成功、旧代码被破坏 | +| R12 | P1 | Lambda 发布版本共享可变代码目录 / warm key | 版本执行错误、更新后仍执行旧代码 | +| R13 | P1 | 停机只处理 Ctrl+C,drain 无期限 | 容器退出不保存、挂起停机 | +| R14 | P2 | Snapshot 双 rename 中断无恢复 | 已完成旧快照被当成不存在 | +| R15 | P1 | 跨服务投递契约缺参数、缺能力失败语义 | FIFO 投递失败、Noop 假成功 | +| R16 | P2 | 配置静默回退、URL 与监听不一致 | 错配置难发现、生成错误入口 | +| R17 | P2 | 健康是注册状态,探针无界且用子串判断 | 假健康、探针卡住 | +| R18 | P2 | 开发入口和能力说明不可靠 | 启动失败、本地验证误导 | + +## 4. 安全与资源问题 + +### R01 — Lambda 函数名可越过代码目录边界 + +**证据**:[provider.rs](../crates/rustack-lambda-core/src/provider.rs) `311-316` 只检查 FunctionName 长度;`440-442` 将其传给 `process_code`。[storage.rs](../crates/rustack-lambda-core/src/storage.rs) `954-977` 用 `code_dir.join(function_name).join(version)` 写 `code.zip` 并删除既有 `extracted`;`1014-1017` 清理时递归删除 `code_dir.join(function_name)`。[router.rs](../crates/rustack-lambda-http/src/router.rs) `70-73` 会解码路径参数,provider `767-791` 删除记录后调用清理。 + +**触发 / 影响**:可调用 Lambda CRUD 且进程对目标可写时,创建名为绝对路径或 `../...` 的函数,会写到根外 `<目标>/$LATEST/`;再以编码后的名称删除,可删除已登记函数对应的整个目标目录。无需 Invoke,disabled executor 也不防护。这里不是说任意单次 DELETE 都能删除不存在的函数:删除要求记录存在。 + +**已有防护**:ZIP entry 的 `enclosed_name` 约束的是内层文件,不能证明外层 extraction root 安全。 + +**修复契约**:名称/ARN/qualifier 分别解析成私有字段 newtype;逻辑名称不得直接充当磁盘路径,使用内部不可控 ID / 内容摘要布局。所有 CRUD、版本、快照导入共用边界校验;拒绝路径分隔符、绝对路径、父目录组件,并防预存 symlink。按 AGENTS.md § Input Validation / Path traversal 执行,不能只做字符串替换。 + +**验收**:仅在测试临时目录设置根外哨兵;绝对路径、`..`、编码斜杠、symlink root 的创建/更新/删除均失败,哨兵与原文件树不变;合法函数和合法 ARN 引用仍正常。 + +### R02 — 配置要求验签,却因缺凭证继续放行 + +**证据**:[main.rs](../apps/rustack/src/main.rs) `608-616` 缺任意一项凭证返回 `None`;`395-401` 仍构造 SQS HTTP config。[SQS service.rs](../crates/rustack-sqs-http/src/service.rs) `128-145` 使用 `!skip_signature_validation` 与 `Some(provider)` 两层 if,缺 provider 时直接 dispatch。 + +**触发**:设置 `SQS_SKIP_SIGNATURE_VALIDATION=false`,但 ACCESS_KEY / SECRET_KEY 及 AWS fallback 缺两者或缺一者;无签名合法 CreateQueue 仍能到业务层。与“默认开发模式跳过验签”的有意设计不同,这是显式安全配置未被履行。此条有完整 SQS 证据,不据此假定每个服务行为完全相同。 + +**修复契约**:严格模式凭证缺失在监听前报错;HTTP 层也 fail closed。把非法组合改为 `AuthMode::Required(provider)` / 显式开发模式,并记录启动时认证模式而非秘密。 + +**验收**:false + 缺任意凭据无法启动 / 业务 handler 调用数为 0;完整凭据下有效签名通过、无签名/坏签名拒绝。扫描其他协议实现同一模式,不能仅修 SQS。 + +### R03 — SigV4 正文可以与签名脱钩 + +**证据**:[SQS service.rs](../crates/rustack-sqs-http/src/service.rs) `122-145` 计算真实 body hash,但 [sigv4.rs](../crates/rustack-auth/src/sigv4.rs) `249-256` 无条件优先使用请求声明的 `x-amz-content-sha256`;`258-286` 用旧声明值重算签名而不核对实际正文。 + +**触发 / 前提**:拥有一份合法已签 SQS SendMessage 请求,不需要 secret;保留签名、路径和签名头,将 MessageBody 改为同长度内容并保持旧哈希头,仍会按旧哈希验证。即使原请求没带哈希头,也可新增未列入 SignedHeaders 的该头,值取原正文哈希,造成同样脱钩。严格验签且凭据完整也受影响。 + +**已有防护 / 限定**:HMAC 使用 constant-time 比较是正确的;普通非流式 S3 HTTP 层有额外正文校验,不能直接推断该 SQS 调用链同样攻破全部 S3 路径。 + +**修复契约**:普通 SHA-256 声明必须格式合法且等于实际字节摘要;不能把“客户端签了某个哈希”当成“正文就是该哈希”。`UNSIGNED-PAYLOAD` / `STREAMING-*` 等占位符由明确协议策略处理,流式场景须校验真实 chunk / trailer 完整性,禁止为兼容 S3 而放宽所有 JSON 服务。 + +**验收**:有效请求通过;保持签名并改 1 字节正文拒绝且 handler 未调用;覆盖新增未签名哈希头、无哈希头、格式错误、重复头与允许的 S3 流式用例。 + +### R04 — CloudFront / APIGW 自动跟随上游重定向 + +**证据**:[CF plane.rs](../crates/rustack-cloudfront-dataplane/src/plane.rs) `84-91` 仅设置 timeout,未设 redirect policy;[dispatch.rs](../crates/rustack-cloudfront-dataplane/src/dispatch.rs) `248-270,284-305` 请求上游并回传最终响应。[APIGW provider.rs](../crates/rustack-apigatewayv2-core/src/provider.rs) `49-54` 使用默认 Client;[http_proxy.rs](../crates/rustack-apigatewayv2-core/src/execution/http_proxy.rs) `22-64` 同样请求并返回最终正文。reqwest 默认会跟随重定向。 + +**触发 / 前提**:攻击者控制已经配置的上游响应,或上游有路径型 open redirect;合法 viewer 请求匹配路由 / CF cache miss,上游返回 Location 指向 Rustack 可访问的其他 loopback、私网或 link-local 服务。代理可能回传该服务内容;CF 还可能缓存结果。不假定攻击者可拿到 IMDS 凭据,也不假定未转发的 query 能触发 redirect。 + +**设计区分**:主动配置 localhost origin 是本地模拟器的功能,不独立判漏洞;问题是**上游响应可把选定目的地扩展到另一个服务**。已有 CF timeout、method 检查、hop-header 过滤不限制重定向目的地。 + +**修复契约**:默认不跟随,转发原始 30x / Location。若存在需要跟随的显式模式,operator egress policy 对初始 URL、更新后的 URL 和每跳 URL 校验 scheme、host/IP、port,检查所有 A/AAAA 并绑定获准解析结果,处理代理设置与 DNS rebinding。生产式硬化模式遵循 AGENTS.md § URL / SSRF;本地 HTTP fixture 例外必须显式、精确且被记录,不能泛放所有内网地址。 + +**验收**:A 上游返回 301/302/303/307/308 指向未授权 B;B 请求数为 0,viewer 收到原 30x 或明确错误,B 正文不得进入 CF cache。覆盖相对/绝对/协议相对 Location、不同端口、IPv6/mapped address、DNS 公转私。 + +### R05 — 资源上限未贯穿整个 IO 生命周期 + +**证据**:[main.rs](../apps/rustack/src/main.rs) `640-659` 每个 accept 都 spawn,没有应用级连接准入预算;[SQS service.rs](../crates/rustack-sqs-http/src/service.rs) `122-133,151-157` 在验签前直接 `.collect()`;[CF dispatch.rs](../crates/rustack-cloudfront-dataplane/src/dispatch.rs) `287-297` 先 `.bytes().await`,完整分配后才检查 `max_body`;[APIGW http_proxy.rs](../crates/rustack-apigatewayv2-core/src/execution/http_proxy.rs) `45-55` 完整收集上游响应。 + +**触发**:合法操作头 + 超大/持续 chunked body,或上游持续输出、大量慢连接。单个业务字段上限、Content-Length 校验、CF 读后大小检查,都不能限制实际收集阶段的总占用。此结论是应用预算缺失,不声称 Hyper 自身没有任何协议默认限额。 + +**修复契约**:统一定义连接、in-flight、正文实际字节、读头/读体/上游 deadline;JSON 控制面可在 collect 前使用 `http_body_util::Limited`,S3 数据面保留有界 streaming,不能把一个过小全局 body limit 粗暴套到对象上传。上游逐帧计数并超限中止,限制聚合内存;超额返回协议对应错误。 + +**验收**:预算 B 下,B+1 实际字节即失败;无 Content-Length/chunked 不能绕过;慢请求按 deadline 结束;并发 N+1 不产生无界任务;健康探针在压力下仍有独立资源可用。RSS / FD / 任务峰值记录预算内断言,不能只断言最后返回 413。 + +### R06 — Lambda 并发限制只管空闲池 + +**证据**:[provider.rs](../crates/rustack-lambda-core/src/provider.rs) `872-887` 每次 Event detached spawn;[instance.rs](../crates/rustack-lambda-core/src/executor/instance.rs) `117-122` 无空闲实例就新建,`186-195` 仅在归还时检查 `max_warm`,`199-211` 新建 Runtime API 和 backend。PutFunctionConcurrency 保存的 metadata 未参与这条执行准入路径。 + +**影响**:大量慢 Invoke / Event 可创建大量 native 进程、socket、任务。外层 HTTP 并发限额无法覆盖已返回 202 的 Event。已有 invoke timeout、kill_on_drop、4 KiB 日志环并不是全局并发预算。 + +**修复契约**:入队前获取全局与每函数执行配额,reserved concurrency=0 明确拒绝;Event 使用有界队列并由 supervisor 管理。permit 覆盖排队/执行的明确阶段,异常、超时和取消必须释放。区分 `maxIdleInstances` 与 `maxConcurrentInvocations`,限制全局 warm 总量;native 的 OS 资源控制不能用请求内存 metadata 冒充。 + +**验收**:设置并发 N,发起 N+K 同步和异步慢调用,运行实例至多 N,其余明确拒绝或有界排队;返回 202 必须已获队列容量;异常/取消/停机后无 permit 泄漏、后台工作可收拢。 + +## 5. 数据与执行正确性 + +### R07 — DynamoDB 事务失败后仍留下前序写入 + +**证据**:[provider.rs](../crates/rustack-dynamodb-core/src/provider.rs) `2455-2479` 预检 condition,`2481-2509` 立即写入并发出 stream;`2549-2558` 才解析/求值 UpdateExpression,错误直接退出。条件检查和提交之间也无整体隔离;事务读是逐项取值。 + +**最小场景**:空表 T,主键 pk:S;同一 TransactWriteItems 先 Put(pk=a),后 Update(pk=b, UpdateExpression="SET"),均无条件。第二项解析失败,但 a 已写入;开启 Streams 时还有前序记录。并发客户端做 `attribute_not_exists` 条件事务,也可能均通过预检。 + +**修复契约**:prepare 阶段完成全部表达式、类型、大小、主键和条件计算,形成暂存变更;单一事务协调状态所有者负责隔离、原子提交与提交后 stream 发布。普通写入、读和事务必须参与同一协调协议,不能只串行化 TransactWriteItems;禁止用“多张表各自 DashMap”冒充事务。 + +**验收**:上述坏 Update、运算失败、后序非法主键均不留下前序数据 / stream;用 barrier 控制两个竞争条件事务,至多一个成功;TransactGetItems 不读到新旧混合。现有失败 ConditionCheck 测试不能替代提交阶段失败测试。与[既有操作差距计划](./service-operations-gap-impl-plan.md)中的事务原子性要求一致。 + +### R08 — DynamoDB 事务重试没有幂等语义 + +**证据**:[model/input.rs](../crates/rustack-dynamodb-model/src/input.rs) `585-587` 接受 `client_request_token`,但 [provider.rs](../crates/rustack-dynamodb-core/src/provider.rs) `2384-2594` 的事务实现不使用该字段;core 源码搜索无该字段命中。 + +**场景**:初始 n=0;使用相同 token、相同 Update `ADD n :one` 连续提交两次,每次都会重新执行,结果 n=2。修复 R07 并不会自动修复重试。 + +**修复契约**:按已有 spec 的 10 分钟窗口实现有界 token 状态,窗口从原成功请求完成时计起;token 绑定请求指纹、scope 和结果,并与提交协调。并发重复请求只执行一次;相同 token 不同内容返回对应 `IdempotentParameterMismatch`。避免只在成功后往 HashMap 写记录造成并发双执行。在途 token 和窗口内已提交 token 不得因容量压力被提前淘汰;容量不足时,应在新 token 对应事务发生变更前明确拒绝,已有 token 的重放仍可处理,或采用不损害完整窗口的存储策略。 + +**验收**:顺序重试、并发重试、客户端超时后重试均仅增量一次且 stream 一次;内容冲突拒绝,窗口过期可重用;小容量缓存写满后,首 token 在 TTL 内重放仍只执行一次,在途记录也不可淘汰。缓存容量与过期清理不得削弱幂等窗口。 + +### R09 — SQS DLQ 中的消息实际不可消费 + +**证据**:[queue/actor.rs](../crates/rustack-sqs-core/src/queue/actor.rs) `808-820` 超过 maxReceiveCount 后仅 `storage.dead_letters.push(msg)`;[storage.rs](../crates/rustack-sqs-core/src/queue/storage.rs) `21-23,64-80` 该 Vec 不计入可用/在途/延迟计数,没有目标队列转交路径,仅 purge 清空。FIFO 接收 `actor.rs:849-868` 也没应用 redrive policy。[README](../README.md) `156` 宣称支持 DLQ redrive。 + +**场景**:src 配置目标 dlq,maxReceiveCount=1;发送后第一次接收,visibility 设 0,再接收触发阈值。消息从 src 消失但 dlq 没收到,既未删除也未过期。 + +**修复契约**:queue manager / 路由 actor 负责可恢复移交,成功加入目标前不得不可逆丢弃源消息;避免两个 queue actor 相互等待造成环路。目标不存在、关闭、容量不足必须有清楚失败/重试语义;standard/FIFO 分别落实契约。 + +**验收**:阈值边界、移交后源不可见且目标可消费、目标关闭/删除/满时消息不丢;覆盖 FIFO,不能拿内部 Vec 长度等于 1 当 DLQ 验收。 + +### R10 — FIFO 复合去重键碰撞 + +**证据**:[actor.rs](../crates/rustack-sqs-core/src/queue/actor.rs) `499-505` 在 messageGroup scope 拼 `"{group_id}:{dedup_id}"`;[storage.rs](../crates/rustack-sqs-core/src/queue/storage.rs) `160-168` 命中便返回旧消息成功结果而不入队。 + +**场景**:同一 FIFO queue 开启 messageGroup 去重,发送 `(group="a:b", dedup="c")` 和 `(group="a", dedup="b:c")`;合法不同组合产生同一字符串,第二条被吞。 + +**修复契约**:用 `DedupKey::Queue(id)` / `DedupKey::Group { group, id }` 的结构化键,不靠换分隔符或拒绝 AWS 允许的标点。 + +**验收**:以上两条都可收到;同 pair 重复只一次;同 id 不同组不去重;scope 切换不和旧键空间误撞。 + +### R11 — Lambda 部署包错误被吞,代码更新先破坏旧文件 + +**证据**:[storage.rs](../crates/rustack-lambda-core/src/storage.rs) `967-987` 先覆盖 zip、删除旧 extracted;`998-1003` 只上报 `InvalidZipFile`,其他错误被忽略。`1049-1051` 无效 archive 被归为 Internal,文件创建/写入/CRC 读取错误也可走 Internal。[provider.rs](../crates/rustack-lambda-core/src/provider.rs) `611-627` 接着更新函数 metadata。 + +**场景**:对已有函数更新为可解码 Base64、但不是有效 ZIP 的内容,API 可能成功并登记新 hash,旧可用目录已删除。即使路径检查/大小上限返回错误,先删除旧代码仍破坏“失败更新保持原状”。现有逻辑明确为测试 stub 容忍无效 ZIP,这种便利不能进入生产路径。 + +**修复契约**:全部 ZIP/IO 错误传播为明确错误;先在新 staging 目录完成验证、解压、可读性检查,再原子提交代码引用。失败清理新目录而非旧目录,正常单测使用真实最小 ZIP,不保留 stub 容错分支。 + +**验收**:坏 ZIP、CRC 错误、写盘失败、entry 越界、解压超限后,旧 code hash / revision / 文件和可执行行为全部不变;有效更新成功切换。 + +### R12 — Lambda 发布版本并不拥有不可变的可执行代码 + +**证据**:[provider.rs](../crates/rustack-lambda-core/src/provider.rs) `949-962` 将 latest clone 成发布版本,只改版本字段,保留 code_path;更新 `611-612` 总写 `$LATEST` 目录;调用 `918-928` 从记录取 code_root,native [native.rs](../crates/rustack-lambda-core/src/executor/native.rs) `100-106` 执行其 bootstrap。另 [instance.rs](../crates/rustack-lambda-core/src/executor/instance.rs) `112-122` 的 warm key 只有 function/qualifier,不含代码 revision,更新路径没有失效旧 warm pool。 + +**场景**:上传 A、发布 v1、更新 latest 为 B;native 下 v1 冷启动可能执行 B,虽然它的 metadata / ZIP bytes 仍描述 A。另已预热的 latest 更新到 B 后可能仍复用 A 进程。不同 executor 取 code_root / code_zip 的方式不同,不能宣称所有 backend 都同样受影响。 + +**修复契约**:代码目录按不可变 digest / version artifact 管理,版本引用不能指向可变 latest 目录;warm key 纳入代码与执行配置 revision,更新/删除后旧实例不再接新工作,已在途调用按明确 drain 策略完成。 + +**验收**:A/B 两种明确响应的 bootstrap:v1 无论冷/热均为 A,更新后 latest 为 B;删除重建同名函数不得复用旧实例;覆盖 native,其他 backend 做版本契约回归。 + +## 6. 生命周期、架构与用户体验 + +### R13 — 停机路径不覆盖容器常用信号,也没有总期限 + +**证据**:[main.rs](../apps/rustack/src/main.rs) `633-635` 只等待 `ctrl_c()`;`669-670` 无限等 graceful drain;`1322-1328` serve 返回后才 save、再 providers.shutdown。[Dockerfile](../Dockerfile) `167-181` 直接以 Rustack 为入口且没有 STOPSIGNAL 改为 SIGINT。 + +**影响 / 限定**:普通 Unix SIGTERM 或 Docker stop 不进入应用的 Ctrl+C 保存路径;PID 1 对默认信号的表现与环境有关,可能被终止或等待至强杀,但都没有显式保存保证。Ctrl+C 下一个不完成的请求也能一直阻塞保存。后台 Event invoke / 投递不随 HTTP drain 自动结束,save-before-worker-shutdown 不等于一致切面。 + +**修复契约**:运行时 supervisor 独立于 snapshot registry,统一处理 SIGINT/SIGTERM,跟踪任务结果/panic、预算和取消。先停止接入与后台新工作,所有已接收工作在 quiesce 前完成或产生明确失败/取消结果,再保存、最后销毁执行资源;quiesce 不能简单实现为先清空 SQS 存储。快照只覆盖 capability 声明支持持久化的状态和副作用;未持久化消息/服务必须有摘要,不承诺重启保留。排空超时或快照失败必须清楚非成功退出,保留旧快照;不宣称 SIGKILL 能保存最新状态。 + +**验收**:隔离子进程中分别发送 SIGINT/SIGTERM、模拟不完成 body / 慢上游 / Event invoke;进程在配置总期限内退出。分别断言无后台变更越过一致切面、声明支持的状态恢复正确、未持久化范围有明确说明;不要求所有 202 / SendMessage 成功请求跨重启恢复。无法满足时有明确错误,旧快照仍可用。Docker stop 单独做 PID 1 回归。 + +### R14 — Snapshot 发布存在“目录消失但旧快照仍在”的窗口 + +**证据**:[snapshot.rs](../apps/rustack/src/snapshot.rs) `848-859` 先 target→backup,再 temp→target;`866-869` 仅正常错误尝试回滚;`448-451` load 发现 target 不存在直接空启动,不查可恢复 backup。 + +**场景**:已有成功快照 S,覆盖保存时在两次 rename 中间中断;磁盘有完整旧 `.bak`,但再次 `--snapshot S` 静默启动为空。这不是要求中断时保存最新写入,而是要求**此前成功保存的状态仍可识别/恢复**。旧文件尚可能人工恢复,不表述为一定物理销毁。 + +**修复契约**:选择不可变 generation + 原子 current 指针发布;保留上一有效 generation,加载验证完整性后选择 committed generation。也可采用明确可验证的 backup 恢复协议,但不能遇到可恢复旧状态仍按新名字空启动。文件/目录同步与平台替换语义必须在后续设计中明确,不能把两次 rename 称为单步原子操作。 + +**验收**:对每个发布步骤注入失败/终止,重启只能见完整旧或完整新状态,不能空/混合;“从未有过这个名字”和“发布中断待恢复”有不同诊断。与[Snapshot PRD](./ruststack-snapshot-prd.md)及[二进制设计](./ruststack-snapshot-binary-archive-design.md)衔接,不引入 WAL 非目标。 + +### R15 — 跨服务 bridge 没有完整的投递能力契约 + +**证据 A(实际 FIFO 错误)**:[Events provider.rs](../crates/rustack-events-core/src/provider.rs) `687-701` 保存 target 时丢弃 SqsParameters;[events_bridge.rs](../apps/rustack/src/events_bridge.rs) `55-66` 仅发 URL/body;SQS `actor.rs:479-481` 必须有 MessageGroupId。Events `849-857` 失败仅 warn。 + +**场景 A**:FIFO queue 开 content-based dedup,PutTargets 带 MessageGroupId=g,配置成功;PutEvents 被接受,但目标因缺 group 拒绝、队列为空。这里 PutEvents 成功只表示接受事件,本身不应被误解为同步送达承诺;真正错误是已接受的 target 参数被丢弃,且无恢复投递路径。 + +**证据 B(假成功)**:[main.rs](../apps/rustack/src/main.rs) `1009-1017,1033-1043` 当 runtime 禁用 SQS 时接入 Noop publisher;[SNS publisher.rs](../crates/rustack-sns-core/src/publisher.rs) `48-62`、[Events delivery.rs](../crates/rustack-events-core/src/delivery.rs) `30-38` 返回 Ok。bridge `68-74` 对未支持 target 同样 Ok。 + +**修复契约**:保留 app 层 bridge 隔离 core-to-core 依赖这一优点,扩展为带完整目标参数的类型化命令。runtime registry 声明依赖与支持 target;配置时拒绝不支持参数 / 目标或显式 metadata-only 状态,生产 wiring 不使用代表成功的 Noop。异步接受、投递成功、重试耗尽分别计数和可查询;不强行把 PutEvents 改成同步送达 API。 + +**验收**:SqsParameters roundtrip + FIFO 端到端收到 group=g;SERVICES=events 或 sns 缺 SQS 的配置能明确诊断;unsupported target 不计 delivered;队列关闭/投递失败有可观测终态。 + +### R16 — 配置没有统一校验和入口地址契约 + +**证据**:[Lambda config.rs](../crates/rustack-lambda-core/src/config.rs) `56-63` executor 解析失败静默退回 Auto/Docker,`97-107` Default 又为 Disabled;[main.rs](../apps/rustack/src/main.rs) `1280-1287` 未知或未编译服务仅 warn,若还有其他服务则启动;`256-309` CLI 静默忽略位置参数。[SQS config.rs](../crates/rustack-sqs-core/src/config.rs) `24-38` 会由 GATEWAY_LISTEN 推导端口,但 Lambda `config.rs:69-73` 仍默认 4566,其 Function URL `provider.rs:1518-1521` 使用该值。APIGW `provider.rs:86-90` 返回 AWS 域名而非本地可调用 URL。 + +**场景**:`LAMBDA_EXECUTOR=disable` 拼错反而选择 Auto;`SERVICES=s3,dynamdb` 启动后健康却少服务;监听 4567 时 Lambda URL 仍指 4566;直接复制 APIGW ApiEndpoint 无法指向当前本地执行入口。AWS-shaped 管理 metadata 可以保留,但必须给出明确 local endpoint 映射。 + +**修复契约**:一次解析为 `ValidatedRuntimeConfig`,值有来源、优先级、范围,非法安全开关/枚举/端口/服务名 fail fast;Default 与环境无值行为一致。区分 bind address 与 advertised endpoint,不信任任意请求 Host 生成绝对 URL;输出有效配置时脱敏。按 AGENTS.md § Async & Concurrency 使用统一 YAML 配置,现有 env override 的兼容与优先级显式定义,不要求本轮重写 AWS wire 格式。 + +**验收**:表驱动覆盖拼写错误、缺值、范围错误、重复与冲突参数;非默认端口、容器 DNS、IPv6 都生成正确且可解释的本地入口;启动前列出 compiled/enabled/disabled/degraded 及原因。 + +### R17 — 健康端点和探针不能证明就绪 + +**证据**:[gateway.rs](../apps/rustack/src/gateway.rs) `94-106` 所有注册服务固定 `running`;`84-91` 只拦 GET。[main.rs](../apps/rustack/src/main.rs) `788-804` 探针没有内部 timeout / 字节预算,读至 EOF,仅以 `200 OK` 和 `"running"` 子串判断;`1264-1268` 丢弃详细错误。[main.rs](../apps/rustack/src/main.rs) `1213-1222` CF data plane build 失败只 warn,control plane 仍注册。 + +**影响**:依赖/数据平面失败仍展示健康;仅运行部分服务也可被误认为全部预期服务 ready。服务不关闭连接时 CLI probe 卡住,Docker 外层 timeout 只能覆盖 Docker 用法。文本含两段关键字也不能证明 HTTP status/JSON 真健康。 + +**修复契约**:拆 liveness/readiness/capabilities,readiness 包括 requested services 和必要 runtime 依赖;兼容旧 health JSON,增加稳定降级原因和版本。探针用结构化 HTTP/JSON 解析、deadline、响应大小上限、stderr 诊断;HEAD 按健康接口契约处理。 + +**验收**:延迟恢复、缺依赖、backend unavailable、HEAD、空 services、非 200 正文夹关键字、永不 EOF、超大 body 都有确定结果;普通 probe 在预算内返回。轻量健康检查不必每次拉起 Lambda,但必须披露 backend 尚未验证/不可用,而非声称已可执行。 + +### R18 — 开发入口和能力说明会误导用户 + +**证据**:[apps/rustack/Cargo.toml](../apps/rustack/Cargo.toml) `1-14` package=`rustack-cli`、binary=`rustack`;[Makefile](../Makefile) `22-23` 却用 `cargo run -p rustack`,[README](../README.md) `311-313` 同样以旧 package 名说明 selective build。README `3,70,75-96` 混用 routed operation 数和 operation 总数,并在 `156` 宣称 DLQ redrive。快照 runtime registry 仅注册部分服务([snapshot.rs](../apps/rustack/src/snapshot.rs) `62-111`),SQS 当前是资源型 snapshot,不代表消息持久化。旧 specs README 的多处 `rustack-*` 链接与实际 `ruststack-*` 文件名不符。 + +**改进契约**:区分 package/bin 名;提供可复制的首次启动、endpoint/region/credentials、一次读写、停机恢复示例。按操作列出 `implemented / partial / metadata-only / unsupported`,并明确 `routed` 不等于语义通过。snapshot 启动/保存摘要列出包含服务和资源/数据边界,不支持者必须可见;不因文档缺失强制实现全部持久化。维护单一 specs index,旧入口明确指向它。 + +**验收**:干净环境的文档 walkthrough 可在 5 分钟内完成启动和一次读写(不把首次编译时间计入指标);所有示例 package 名正确;核心操作状态有对应测试或明确限制;存取 snapshot 前后用户知道哪些数据会保留。此处是待修复体验需求,不在本次修改 Makefile、CI 或现有工具链行为。 + +## 7. 建议的架构收敛 + +### 7.1 保留分层,补共享契约 + +不建议“大一统 service framework”或仅按行数拆 `main.rs`。应先把重复实现中已经出现的差异变成单一权威契约: + +| 组件边界 | 应拥有的契约 | 关联问题 | +|---|---|---| +| ValidatedRuntimeConfig | auth mode、服务依赖、bind/advertised URL、资源预算 | R02/R05/R16 | +| Shared HTTP policy + protocol adapter | body budget、deadline、错误映射、完整性验证;协议占位符仍由各协议定义 | R03/R04/R05 | +| Runtime supervisor | 接入、后台队列、任务错误/panic、quiesce、save、终止 | R06/R13/R17 | +| Service capability registry | compiled/enabled、target 能力、snapshot 范围、readiness | R15/R17/R18 | +| Domain state owners | transaction prepare/commit、token、DLQ handoff、typed dedup key | R07–R10 | +| Immutable artifact store | ZIP staging、version artifact、warm revision、GC | R01/R11/R12 | + +`SnapshotService` 不应兼任所有运行时服务的生命周期清单:没有 snapshot 能力的 Events/SNS worker 同样需要监督。内部 actor 消息以有界 channel 传递,不让队列互等;不在 DashMap guard 上跨 await。已有[SQS guard 修复规格](./fixes/sqs-longpoll-dashmap-safety.md)继续保留其不变量。 + +### 7.2 停机时序契约 + +```text +Signal / operator Runtime supervisor Workers / stores Snapshot store + │ │ │ │ + │ 1 SIGINT / SIGTERM ──────────►│ │ │ + │ │ 2 readiness=false │ │ + │ │ stop ingress │ │ + │ │ 3 stop new jobs ────────►│ │ + │ │ │ finish / bounded │ + │ │ │ cancel accepted work │ + │ │ 4 quiesced ◄────────────│ │ + │ │ (failure: report, keep previous snapshot) │ + │ │ 5 export consistent state ──────────────────►│ + │ │ │ write generation │ + │ │ │ validate + sync │ + │ │ │ publish current │ + │ │ 6 committed / failed ◄───────────────────────│ + │ │ 7 stop resources ──────►│ │ + │ 8 bounded exit + status ◄─────│ │ │ +``` + +整条时序共享总停机预算,各阶段只消费剩余预算。部分任务无法收拢时不把不一致快照发布成成功;保留旧版本并给出错误。具体预算值由后续运行实验校准,不把 README 的亚秒启动/保存声明当作测量结果。 + +### 7.3 Rust 工程约束 + +修复遵循 [AGENTS.md](../AGENTS.md),不通过大范围 allow lint、忽略异常、Noop 成功实现来绕过: + +- **错误**:§ Error Handling;core 使用带 source 的 `thiserror` 领域错误,app 使用 anyhow context;输入错误不能 panic,ZIP/IO 失败不可吞掉。 +- **并发**:§ Async & Concurrency;actor 拥有状态、有界 channel、任务受监督且结果可观察;阻塞压缩/解压在 `spawn_blocking` 并受并发预算约束。object-safe bridge 可保留有文档理由的 async-trait。 +- **类型 / 安全**:§ Type Design & API / Safety & Security;validated name、AuthMode、typed dedup key、不可变 artifact;`forbid(unsafe_code)`、checked arithmetic、secret redaction。不会为性能引入 unsafe。 +- **序列化**:§ Serialization & Data;新内部配置/诊断 JSON 用 camelCase,反序列化必须走校验。**已有 AWS wire 的 PascalCase / XML 是协议要求,不应机械改成 camelCase**;这是相对通用规范的明确协议例外,保持既有服务 schema。 +- **测试 / 日志 / 性能**:§ Testing / Logging & Observability / Performance;问题场景用 `test_should_*`、真实最小 ZIP、可控时钟/barrier、失败注入;tracing 仅记录 request ID、错误类别和脱敏状态,不记录 Authorization/secret/正文;优化先测量,关注峰值而非平均空闲内存。 +- **文档 / API**:§ Documentation;公开错误、取消语义、边界与示例可被测试。后续涉及 Rust 的实现阶段遵循现有 Toolchain & Build 质量门禁;本 review 不新增或修改其执行规则。 + +## 8. 分期与验收 + +这是单篇 review,不另生成重复 PRD / design / roadmap 文档。下列 milestone 面向用户结果,工程阶段与其 1:1 配对;具体服务设计仍为协议权威。 + +### 8.1 用户可见里程碑 + +| 里程碑 | 用户获得什么 | 退出条件 | +|---|---|---| +| M0 安全边界可信 | 错误凭证不静默放行,文件/正文/上游目的地不可越界 | R01–R04 负向场景通过,危险默认部署组合有明确阻断/选择 | +| M1 有界运行与可靠退出 | 慢请求、并发函数、容器停机不会不可控 | R05/R06/R13/R14 场景通过,旧快照可恢复 | +| M2 数据与版本可信 | 事务重试、DLQ、FIFO 和 Lambda 版本语义可依赖 | R07–R12 全部达到各自不变量 | +| M3 集成可预测 | FIFO target 参数保留、依赖缺失/错误配置能早发现 | R15/R16 端到端与错误配置场景通过 | +| M4 首次使用与诊断可信 | 开发入口能复制运行,健康/能力/持久化说明诚实 | R17/R18 walkthrough 与故障诊断通过 | + +### 8.2 工程依赖顺序与粗估 + +| 阶段 | 先做的契约,再做的消费者 | 交付与验证 | 粗估人日 | +|---|---|---|---| +| Phase 0 → M0 | 先补失败回归与威胁模型;validated name / AuthMode / payload policy / egress policy,再接全部入口 | R01–R04;补服务差异清单,不把仅一个 HTTP crate 修复当全系统完成 | 6–10 | +| Phase 1 → M1 | 先运行时预算和 supervisor,再 Event queue / body adapter / stop / snapshot publish | R05/R06/R13/R14;必须包含真实子进程与发布步骤故障注入 | 8–13 | +| Phase 2 → M2 | 先 DDB commit coordinator 再 token;先 DLQ handoff 再用户可见计数;先 immutable artifact 再 warm revision | R07–R12;数据、stream、代码行为联合断言 | 12–20 | +| Phase 3 → M3 | 先 target/capability/config schema,再 bridge 和配置输出 | R15/R16;standard/FIFO、不同 SERVICES/端口组合 | 5–8 | +| Phase 4 → M4 | 用真实 registry 生成 readiness/能力说明,再修使用示例与导航 | R17/R18;干净环境 walkthrough、文档链接检查 | 3–5 | + +合计 **34–56 人日**,按单人全职约 **7–12 工作周**,不含新服务功能。此为基于源码改动面、并发与故障验证成本的规划区间,**不是 benchmark 或研究 memo 验证过的工期**;Phase 0 完成后重估。SQS、DDB、Lambda 的领域实现可并行,但共享 HTTP/supervisor 契约先落地,避免消费者反复返工。 + +优先级与阶段不同:M0 后尚不能把服务当成多租户平台;M1 的监督与预算先于重投递等新后台逻辑,否则越修功能越放大资源问题。每阶段必须通过其关联 R 项测试,不能只以“接口存在 / 返回 200 / 文档已写”退出。 + +### 8.3 验证边界与待覆盖面 + +- 所有负向安全实验限定 loopback、临时目录和自建上游,不访问真实 metadata 服务或真实账号。 +- 事务/去重使用固定输入、可控时间和并发 barrier;DLQ/版本/bridge 使用实际状态所有者,而非返回成功的 mock。 +- 资源测试同时量化 admission、实际字节、进程/任务/FD 峰值和取消释放;保留吞吐/延迟基线,防过度限流破坏正常长轮询和大对象上传。 +- 延续研究中 SDK + compatibility suite 分层策略:DynamoDB Alternator、S3 Mint、SQS 有针对性的多 SDK 测试;不能用 happy-path CRUD 覆盖数证明完整兼容。 +- 全量 RustSec/许可证审计、TLS 配置、IAM/KMS 密码学、所有分页/表达式边界、S3 对象生命周期、CloudFront cache correctness、各 feature 组合仍需后续专项审查;本次不对未深查区域出具“无漏洞”结论。 + +## 9. 关键决策与已排除误报 + +### D1 — 按模拟器边界评估,而非要求生产 AWS + +保留 metadata-only、localhost fixture、可信 native 的合理用途;选择显式能力和 operator policy,而不是默认宣称全功能/安全多租户,也不一刀切禁止所有本地 origin。关联 R04/R15/R18。 + +### D2 — 边界和原子性修在状态所有者,不补表面 if + +R01 用受控 artifact 路径,R02 用可表达合法状态的 AuthMode,R07 用原子提交,R10 用结构化键。仅增加一条 regex、换分隔符、加一次预检或换成 DashMap 不能替代这些契约。 + +### D3 — runtime lifecycle 不依附持久化支持 + +选择独立 supervisor + capability registry;SnapshotService 只负责可持久化状态。未持久化的后台任务也必须被停机管理。关联 R06/R13/R17。 + +### D4 — 先完整生成新状态,再提交可见引用 + +ZIP artifact 和 snapshot generation 都采取 stage/validate/commit;反对就地覆盖后尝试回滚。此决定不要求为普通业务写入增加 WAL。关联 R11/R12/R14。 + +### 已排除 / 降级 + +- Lambda ZIP entry 已有 `enclosed_name`、250 MiB 声明/实际解压字节限制,不能报成“完全没有 Zip Slip / zip-bomb 防护”;R01 针对外层 FunctionName,R11 针对错误与提交顺序。 +- snapshot zstd 解压使用 `take(MAX_ARCHIVE_BYTES + 1)`([archive.rs](../apps/rustack/src/snapshot/archive.rs) `518-530`),不能报无界解压;多任务总预算可继续改进。 +- SNS 当前非 SQS 协议分支不执行 HTTP 请求;endpoint 存储宽松不是已可达 SSRF。CloudFront 的 in-process S3 origin 同样不是任意网络 fetch。 +- native 的直接执行是有意能力,不称作 shell 注入或 VM 逃逸;其默认部署组合仍须明确告警/安全模式。 +- SQS 资源型 snapshot 不保消息、未纳入 registry 的服务不持久化,是阶段性能力边界;本次归 R18 的披露要求,不要求无条件实现全服务持久化。 +- 不把静态常量构造 response 的 `expect` 与外部输入可达 panic 混为一谈,也不把每个 Mutex 一律判为死锁。 +- 当前 [Cargo.lock](../Cargo.lock) `5855-5856` 是 zip 8.5.1;查阅 [RUSTSEC-2025-0168](https://rustsec.org/advisories/RUSTSEC-2025-0168.html) 的受影响范围是 `>=1.3.0, <2.3.0`,**不适用于该锁定版本**。R01 是应用路径逻辑缺陷,不借该公告冒称依赖 CVE。没有运行全量 audit,不能推出其他依赖无风险。 + +## 10. 交叉引用与外部校对 + +### 既有设计 + +- [Lambda executor](./ruststack-lambda-executor-design.md)、[Squib runtime](./ruststack-lambda-squib-runtime-design.md)、[S3 code packages](./ruststack-lambda-s3-code-design.md):代码包与执行约束。 +- [DynamoDB](./ruststack-dynamodb-design.md)、[操作差距计划](./service-operations-gap-impl-plan.md):事务、token 语义。 +- [SQS](./ruststack-sqs-design.md)、[EventBridge](./ruststack-events-design.md)、[SNS](./ruststack-sns-design.md):队列和投递协议。 +- [Snapshot PRD](./ruststack-snapshot-prd.md)、[二进制归档](./ruststack-snapshot-binary-archive-design.md)、[验证计划](./ruststack-snapshot-verification-plan.md):保存边界与恢复。 +- [CloudFront data plane](./rustack-cloudfront-dataplane-design.md)、[APIGW](./ruststack-apigatewayv2-design.md):代理集成。 + +### 研究背景(历史结论,不作为当前版本实测) + +- [Cargo Lambda 执行 spike](../docs/research/spike-cargo-lambda-runtime-execution.md):bootstrap/Runtime API 合约与 macOS/Linux artifact 区分;本次不重新否定已验证的打包路径。 +- [DynamoDB API](../docs/research/dynamodb-api-research.md)、[测试分层](../docs/research/dynamodb-integration-test-suites-research.md):原子性/隔离需求与 SDK/Alternator 分层。 +- [SQS API](../docs/research/sqs-api-research.md)、[SQS suites](../docs/research/sqs-test-suites.md):FIFO/时序及协议覆盖要求。 +- [LocalStack 容器/CI](../docs/research/localstack-container-ci-research.md):统一入口、健康、信号/容器集成背景。 +- [LocalStack S3](../docs/research/localstack-s3-research.md)、[S3 suites](../docs/research/s3-integration-test-suites-research.md)、[s3s](../docs/research/s3s-crate-research.md)、[Smithy server codegen](../docs/research/smithy-rs-server-codegen-research.md)、[SSM](../docs/research/ssm-parameter-store-research.md):分层与能力范围背景;不据其历史版本表建议盲目迁移依赖。 + +### 外部资料(2026-09-07 查询) + +- [http-body-util Limited API](https://docs.rs/http-body-util/latest/http_body_util/struct.Limited.html):实际 poll 字节上限,可用于 R05;查询时 docs.rs 最新为 0.1.5,方案不要求本次升级依赖。 +- [reqwest redirect policy](https://docs.rs/reqwest/latest/reqwest/redirect/index.html):默认自动跟随最多 10 跳,支持 R04 的默认客户端行为判断。 +- [RustSec zip advisory](https://rustsec.org/advisories/RUSTSEC-2025-0168.html):仅用来排除错误 CVE 归因,不代替全量依赖审计。 + +后续修复应按 R-ID 关联实现、回归测试和能力说明;关闭条目时记录实测证据与范围,而不抹去这份审查基线。 diff --git a/tests/integration/src/test_lambda.rs b/tests/integration/src/test_lambda.rs index 9b23a7e..f6f087c 100644 --- a/tests/integration/src/test_lambda.rs +++ b/tests/integration/src/test_lambda.rs @@ -12,16 +12,20 @@ mod tests { use crate::lambda_client; + /// Valid empty ZIP archive (22-byte EOCD); uploads are validated as real ZIPs. + const EMPTY_ZIP: &[u8] = &[ + 80, 75, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + /// Helper: generate a unique function name. fn func_name(prefix: &str) -> String { let id = uuid::Uuid::new_v4().to_string()[..8].to_owned(); format!("test-{prefix}-{id}") } - /// Create a minimal zip file (PK header + dummy data) as base64 for code uploads. + /// A valid, real ZIP archive for code uploads. fn dummy_zip_blob() -> Blob { - // Minimal zip content (not a real zip, but enough for the server to store). - Blob::new(b"PK\x03\x04fake-lambda-code".to_vec()) + Blob::new(EMPTY_ZIP.to_vec()) } /// Helper: create a function and return its name. @@ -140,7 +144,7 @@ mod tests { let updated = client .update_function_code() .function_name(&name) - .zip_file(Blob::new(b"PK\x03\x04new-code-data".to_vec())) + .zip_file(Blob::new(EMPTY_ZIP.to_vec())) .send() .await .expect("update code should succeed"); @@ -168,9 +172,7 @@ mod tests { s3.put_object() .bucket(&bucket) .key(&key) - .body(aws_sdk_s3::primitives::ByteStream::from_static( - b"PK\x03\x04fake-s3-lambda-code", - )) + .body(aws_sdk_s3::primitives::ByteStream::from_static(EMPTY_ZIP)) .send() .await .expect("upload code object"); diff --git a/tests/integration/src/test_lambda_invoke.rs b/tests/integration/src/test_lambda_invoke.rs index 764dd65..9d11676 100644 --- a/tests/integration/src/test_lambda_invoke.rs +++ b/tests/integration/src/test_lambda_invoke.rs @@ -199,7 +199,7 @@ mod tests { config.init_timeout = Duration::from_secs(10); config.idle_timeout = Duration::from_mins(1); config.max_warm_instances = 1; - config.squib = SquibExecutorConfig::from_env(); + config.squib = SquibExecutorConfig::from_env().expect("Squib configuration must be valid"); config.squib.connect_timeout = Duration::from_secs(30); let tmp = tempfile::Builder::new() diff --git a/tests/integration/src/test_versioning.rs b/tests/integration/src/test_versioning.rs index 8ceb99e..2535fc2 100644 --- a/tests/integration/src/test_versioning.rs +++ b/tests/integration/src/test_versioning.rs @@ -2,9 +2,12 @@ #[cfg(test)] mod tests { + use std::time::Duration; + use aws_sdk_s3::{primitives::ByteStream, types::BucketVersioningStatus}; + use reqwest::{Client, Method, Url}; - use crate::{cleanup_bucket, create_test_bucket, s3_client}; + use crate::{cleanup_bucket, create_test_bucket, endpoint_url, s3_client}; async fn enable_versioning(client: &aws_sdk_s3::Client, bucket: &str) { client @@ -169,6 +172,40 @@ mod tests { .await; assert!(result.is_err(), "get after delete marker should fail"); + // GET and HEAD must distinguish a current delete marker from a missing key. + // An explicitly selected marker is 405 and also includes Last-Modified. + let http = Client::new(); + let marker_id = del.version_id().expect("delete marker version"); + let url = format!("{}/{bucket}/to-delete.txt", endpoint_url()); + for method in [Method::GET, Method::HEAD] { + for explicit_version in [false, true] { + let mut request_url = Url::parse(&url).expect("object URL"); + if explicit_version { + request_url + .query_pairs_mut() + .append_pair("versionId", marker_id); + } + let response = http + .request(method.clone(), request_url) + .timeout(Duration::from_secs(5)) + .send() + .await + .expect("delete marker response"); + assert_eq!( + response.status().as_u16(), + if explicit_version { 405 } else { 404 } + ); + assert_eq!(response.headers()["x-amz-delete-marker"], "true"); + assert_eq!(response.headers()["x-amz-version-id"], marker_id); + if explicit_version { + assert!(response.headers().contains_key("last-modified")); + } + if method == Method::HEAD { + assert!(response.bytes().await.expect("HEAD body").is_empty()); + } + } + } + // But versions should show both the object and the delete marker. let versions = client .list_object_versions() diff --git a/tests/lambda-fixtures/echo-bootstrap/Cargo.toml b/tests/lambda-fixtures/echo-bootstrap/Cargo.toml index c71551b..e3e882d 100644 --- a/tests/lambda-fixtures/echo-bootstrap/Cargo.toml +++ b/tests/lambda-fixtures/echo-bootstrap/Cargo.toml @@ -18,3 +18,11 @@ path = "src/main.rs" [dependencies] ureq = { version = "2", default-features = false } serde_json = { workspace = true } + +[dev-dependencies] +rustack-lambda-core = { workspace = true } +rustack-lambda-model = { workspace = true } +base64 = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true } +zip = { workspace = true } diff --git a/tests/lambda-fixtures/echo-bootstrap/src/main.rs b/tests/lambda-fixtures/echo-bootstrap/src/main.rs index 806b641..3ea7871 100644 --- a/tests/lambda-fixtures/echo-bootstrap/src/main.rs +++ b/tests/lambda-fixtures/echo-bootstrap/src/main.rs @@ -13,6 +13,7 @@ use std::{io::Read as _, time::Duration}; +#[allow(clippy::disallowed_methods)] // This standalone bootstrap deliberately uses synchronous IO, not an async runtime. fn main() { let api = std::env::var("AWS_LAMBDA_RUNTIME_API") .expect("AWS_LAMBDA_RUNTIME_API must be set by the runtime"); @@ -25,6 +26,21 @@ fn main() { .and_then(|v| v.parse().ok()) .unwrap_or(0); + let code_marker = match std::fs::read_to_string("revision.txt") { + Ok(marker) => Some(marker), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => { + eprintln!("Read revision marker: {error}"); + return; + } + }; + let config_marker = std::env::var("CONFIG_MARKER").ok(); + if let Ok(path) = std::env::var("STARTED_FILE") { + if let Err(error) = std::fs::write(path, std::process::id().to_string()) { + eprintln!("Write start marker: {error}"); + return; + } + } loop { let next = agent .get(&format!("http://{api}/2018-06-01/runtime/invocation/next")) @@ -65,6 +81,9 @@ fn main() { let echo = serde_json::json!({ "echo": parsed, "request_id": request_id, + "codeMarker": code_marker, + "configMarker": config_marker, + "pid": std::process::id(), }); let _ = agent .post(&format!( diff --git a/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs b/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs new file mode 100644 index 0000000..58fa5fb --- /dev/null +++ b/tests/lambda-fixtures/echo-bootstrap/tests/hardening.rs @@ -0,0 +1,215 @@ +//! Real host-native bootstrap regressions for immutable artifacts and warm revisions. +use std::{ + collections::HashMap, + io::{Cursor, Write}, + time::Duration, +}; + +use base64::{Engine, engine::general_purpose::STANDARD}; +use rustack_lambda_core::{ + config::LambdaConfig, + executor::ExecutorBackend, + provider::{InvokeKind, InvokeOutcome, RustackLambda}, + storage::FunctionStore, +}; +use rustack_lambda_model::{ + input::{ + CreateFunctionInput, PublishVersionInput, UpdateFunctionCodeInput, + UpdateFunctionConfigurationInput, + }, + types::{Environment, FunctionCode}, +}; + +#[allow(clippy::disallowed_methods)] // Standalone fixture without async runtime defers the CARGO_BIN_EXE read to runtime. +fn package(marker: &str) -> String { + // CARGO_BIN_EXE points at the fixture bootstrap when cargo builds test + // executables; `cargo check --all-targets` only compiles this test without + // materializing that binary, so defer the read to runtime. + let Ok(binary) = std::fs::read(env!("CARGO_BIN_EXE_bootstrap")) else { + return String::new(); + }; + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "bootstrap", + zip::write::SimpleFileOptions::default().unix_permissions(0o755), + ) + .unwrap(); + writer.write_all(&binary).unwrap(); + writer + .start_file("revision.txt", zip::write::SimpleFileOptions::default()) + .unwrap(); + writer.write_all(marker.as_bytes()).unwrap(); + STANDARD.encode(writer.finish().unwrap().into_inner()) +} +fn create(marker: &str) -> CreateFunctionInput { + CreateFunctionInput { + function_name: "revision-test".into(), + runtime: Some("provided.al2023".into()), + handler: Some("bootstrap".into()), + role: "arn:aws:iam::000000000000:role/test".into(), + timeout: Some(5), + architectures: Some(vec![ + if cfg!(target_arch = "aarch64") { + "arm64" + } else { + "x86_64" + } + .into(), + ]), + code: FunctionCode { + zip_file: Some(package(marker)), + ..Default::default() + }, + ..Default::default() + } +} +async fn invoke(provider: &RustackLambda, version: Option<&str>) -> serde_json::Value { + let InvokeOutcome::Sync(response) = provider + .invoke("revision-test", version, b"{}", InvokeKind::RequestResponse) + .await + .unwrap() + else { + panic!("Expected synchronous result") + }; + serde_json::from_slice(&response.payload).unwrap() +} + +#[tokio::test] +async fn test_should_preserve_published_code_and_refresh_warm_revisions() { + let root = tempfile::tempdir().unwrap(); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path()), + LambdaConfig { + executor: ExecutorBackend::Native, + ..Default::default() + }, + ); + provider.create_function(create("A")).await.unwrap(); + let a = invoke(&provider, None).await; + assert_eq!(a["codeMarker"], "A"); + assert_eq!(invoke(&provider, None).await["pid"], a["pid"]); + provider + .publish_version("revision-test", &PublishVersionInput::default()) + .unwrap(); + provider + .update_function_code( + "revision-test", + UpdateFunctionCodeInput { + zip_file: Some(package("B")), + ..Default::default() + }, + ) + .await + .unwrap(); + let b = invoke(&provider, None).await; + assert_eq!(b["codeMarker"], "B"); + assert_ne!(b["pid"], a["pid"]); + assert_eq!(invoke(&provider, Some("1")).await["codeMarker"], "A"); + assert_eq!(invoke(&provider, Some("1")).await["codeMarker"], "A"); + provider + .update_function_configuration( + "revision-test", + &UpdateFunctionConfigurationInput { + environment: Some(Environment { + variables: Some(HashMap::from([("CONFIG_MARKER".into(), "updated".into())])), + }), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(invoke(&provider, None).await["configMarker"], "updated"); + let before = provider + .get_function("revision-test", None) + .unwrap() + .configuration + .unwrap(); + assert!( + provider + .update_function_code( + "revision-test", + UpdateFunctionCodeInput { + zip_file: Some(STANDARD.encode(b"not ZIP")), + ..Default::default() + } + ) + .await + .is_err() + ); + let after = provider + .get_function("revision-test", None) + .unwrap() + .configuration + .unwrap(); + assert_eq!(before.code_sha256, after.code_sha256); + assert_eq!(before.revision_id, after.revision_id); + assert_eq!(invoke(&provider, None).await["codeMarker"], "B"); + provider + .delete_function("revision-test", None) + .await + .unwrap(); + provider.create_function(create("C")).await.unwrap(); + assert_eq!(invoke(&provider, None).await["codeMarker"], "C"); + provider.quiesce(Duration::from_secs(2)).await.unwrap(); + provider.shutdown().await; +} + +#[cfg(unix)] +#[tokio::test] +async fn test_should_cancel_and_reap_native_event_on_quiesce_deadline() { + let root = tempfile::tempdir().unwrap(); + let started = root.path().join("started"); + let provider = RustackLambda::with_store( + FunctionStore::new(root.path().join("code")), + LambdaConfig { + executor: ExecutorBackend::Native, + ..Default::default() + }, + ); + let mut input = create("slow"); + input.timeout = Some(120); + input.environment = Some(Environment { + variables: Some(HashMap::from([ + ("SLEEP_SECS".into(), "60".into()), + ( + "STARTED_FILE".into(), + started.to_string_lossy().into_owned(), + ), + ])), + }); + provider.create_function(input).await.unwrap(); + assert!(matches!( + provider + .invoke("revision-test", None, b"{}", InvokeKind::Event) + .await + .unwrap(), + InvokeOutcome::Async { .. } + )); + let pid = tokio::time::timeout(Duration::from_secs(5), async { + let mut tick = tokio::time::interval(Duration::from_millis(10)); + loop { + tick.tick().await; + match tokio::fs::read_to_string(&started).await { + Ok(pid) if !pid.is_empty() => break pid, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => {} + Err(error) => panic!("Read start marker: {error}"), + } + } + }) + .await + .unwrap(); + assert!(provider.quiesce(Duration::from_millis(500)).await.is_err()); + let status = tokio::process::Command::new("kill") + .arg("-0") + .arg(pid.trim()) + .stderr(std::process::Stdio::null()) + .status() + .await + .unwrap(); + assert!( + !status.success(), + "native child must be reaped before failed quiesce returns" + ); + provider.shutdown().await; +} diff --git a/tests/lambda-fixtures/minimal_zip.rs b/tests/lambda-fixtures/minimal_zip.rs new file mode 100644 index 0000000..a2bf97e --- /dev/null +++ b/tests/lambda-fixtures/minimal_zip.rs @@ -0,0 +1,16 @@ +//! Real deterministic ZIP package helper for storage/control-plane tests. +use std::io::{Cursor, Write}; + +/// Build a ZIP containing a single ordinary file with the supplied contents. +pub fn minimal_zip(contents: &[u8]) -> Vec { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "fixture.txt", + zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Stored), + ) + .unwrap(); + writer.write_all(contents).unwrap(); + writer.finish().unwrap().into_inner() +}