diff --git a/crates/s3s-fs/src/fs.rs b/crates/s3s-fs/src/fs.rs index 543ffcef2..2e1ff6216 100644 --- a/crates/s3s-fs/src/fs.rs +++ b/crates/s3s-fs/src/fs.rs @@ -45,6 +45,19 @@ async fn read_sidecar(path: &Path) -> Result>> { } } +/// Read a sidecar like [`read_sidecar`], also treating a name the filesystem cannot represent as absent. +/// +/// A sidecar name holds the whole encoded key in one path component, so an object imported under a +/// long nested key can have a sidecar name over the component limit. No sidecar can exist under such +/// a name, and a listing reports the object as it would one whose sidecar is missing. +async fn read_nameable_sidecar(path: &Path) -> Result>> { + match fs::read(path).await { + Ok(content) => Ok(Some(content)), + Err(err) if matches!(err.kind(), std::io::ErrorKind::NotFound | std::io::ErrorKind::InvalidFilename) => Ok(None), + Err(err) => Err(err.into()), + } +} + /// Stores standard object attributes alongside user metadata #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] pub(crate) struct ObjectAttributes { @@ -239,6 +252,16 @@ impl FileSystem { Ok(Some(map)) } + /// Load internal info for a listed object, where a sidecar name too long to exist reads as absent. + pub(crate) async fn load_listed_internal_info(&self, bucket: &str, key: &str) -> Result> { + let path = self.get_internal_info_path(bucket, key)?; + let Some(content) = read_nameable_sidecar(&path).await? else { + return Ok(None); + }; + let map = serde_json::from_slice(&content)?; + Ok(Some(map)) + } + pub(crate) async fn save_internal_info(&self, bucket: &str, key: &str, info: &InternalInfo) -> Result<()> { let path = self.get_internal_info_path(bucket, key)?; let content = serde_json::to_vec(info)?; diff --git a/crates/s3s-fs/src/s3.rs b/crates/s3s-fs/src/s3.rs index 976d1db6c..e9355dca9 100644 --- a/crates/s3s-fs/src/s3.rs +++ b/crates/s3s-fs/src/s3.rs @@ -782,17 +782,25 @@ impl S3 for FileSystem { async fn list_objects(&self, req: S3Request) -> S3Result> { let v2_resp = self.list_objects_v2(req.map_input(Into::into)).await?; - Ok(v2_resp.map_output(|v2| ListObjectsOutput { - contents: v2.contents, - common_prefixes: v2.common_prefixes, - delimiter: v2.delimiter, - encoding_type: v2.encoding_type, - name: v2.name, - prefix: v2.prefix, - max_keys: v2.max_keys, - is_truncated: v2.is_truncated, - next_marker: v2.next_continuation_token, - ..Default::default() + // The V1 marker travels as the V2 start-after, so the V2 output already echoes it encoded. + // `NextMarker` is a key, unlike the opaque V2 continuation token, and is encoded like one. + Ok(v2_resp.map_output(|v2| { + let url_encoded = is_url_encoding(v2.encoding_type.as_ref()); + ListObjectsOutput { + contents: v2.contents, + common_prefixes: v2.common_prefixes, + delimiter: v2.delimiter, + name: v2.name, + prefix: v2.prefix, + marker: v2.start_after, + max_keys: v2.max_keys, + is_truncated: v2.is_truncated, + next_marker: v2 + .next_continuation_token + .map(|marker| if url_encoded { url_encode_listed(&marker) } else { marker }), + encoding_type: v2.encoding_type, + ..Default::default() + } })) } @@ -825,11 +833,47 @@ impl S3 for FileSystem { // The walk collects one page in key order, so a page no longer has to be cut out of the whole // bucket. The page arithmetic itself is shared with the full scan the walk replaced. - let page = self.list_page(&path, &query).await?; + let mut page = self.list_page(&path, &query).await?; + + // Listed objects carry the ETag the PUT stored, as S3 does; a client that + // reads ETags from a listing (conditional writes, consistency checks) needs it. + for object in &mut page.objects { + if let Some(key) = object.key.as_deref() { + let info = self.load_listed_internal_info(&input.bucket, key).await?; + object.e_tag = info.as_ref().and_then(crate::checksum::load_e_tag).map(ETag::Strong); + } + } + + // Under encoding-type=url, S3 returns every key and common prefix + // URL-encoded and clients decode `+` as a space, so a raw `+` would + // read back as a different key. + let url_encoded = is_url_encoding(input.encoding_type.as_ref()); + if url_encoded { + for object in &mut page.objects { + if let Some(key) = object.key.as_mut() { + *key = url_encode_listed(key); + } + } + for prefix in &mut page.common_prefixes { + if let Some(p) = prefix.prefix.as_mut() { + *p = url_encode_listed(p); + } + } + } let contents = page.objects.is_empty().not().then_some(page.objects); let common_prefixes = page.common_prefixes.is_empty().not().then_some(page.common_prefixes); + // The echoed prefix, delimiter and start-after are encoded only now, after the walk filtered + // with the raw values. The continuation token is opaque and returned as the client sent it. + let encode_echo = |value: Option| { + if url_encoded { + value.map(|v| url_encode_listed(&v)) + } else { + value + } + }; + let output = ListObjectsV2Output { key_count: Some(page.key_count), max_keys: Some(max_keys), @@ -838,11 +882,11 @@ impl S3 for FileSystem { common_prefixes, continuation_token: input.continuation_token, next_continuation_token: page.next_continuation_token, - delimiter: input.delimiter, + delimiter: encode_echo(input.delimiter), encoding_type: input.encoding_type, name: Some(input.bucket), - prefix: input.prefix, - start_after: input.start_after, + prefix: encode_echo(input.prefix), + start_after: encode_echo(input.start_after), ..Default::default() }; Ok(S3Response::new(output)) @@ -1726,6 +1770,26 @@ impl ListingQuery<'_> { } } +/// Whether a listing request asked for `encoding-type=url`. +fn is_url_encoding(encoding_type: Option<&EncodingType>) -> bool { + encoding_type.is_some_and(|e| e.as_str() == EncodingType::URL) +} + +/// Percent-encodes every byte outside the unreserved set and `/`. +fn url_encode_listed(s: &str) -> String { + use std::fmt::Write as _; + + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~' | b'/') { + out.push(char::from(b)); + } else { + let _ = write!(out, "%{b:02X}"); + } + } + out +} + /// One page of a listing, plus what follows it. #[derive(Debug, Default)] struct ListingPage { diff --git a/crates/s3s-fs/tests/aws/list.rs b/crates/s3s-fs/tests/aws/list.rs index fddc621fc..b4e6fd26d 100644 --- a/crates/s3s-fs/tests/aws/list.rs +++ b/crates/s3s-fs/tests/aws/list.rs @@ -2,8 +2,9 @@ // SPDX-FileCopyrightText: 2023-2026 The s3s Authors use crate::case; -use crate::suite::{List, create_bucket, delete_bucket, delete_object}; +use crate::suite::{FS_ROOT, List, create_bucket, delete_bucket, delete_object}; +use std::path::Path; use std::sync::Arc; use aws_sdk_s3::primitives::ByteStream; @@ -29,6 +30,11 @@ pub fn register(tcx: &mut TestContext) { case!(tcx, FsServer, List, test_list_objects_v2_continuation_token_pagination); case!(tcx, FsServer, List, test_list_objects_v2_continuation_token_and_start_after_uses_max); case!(tcx, FsServer, List, test_list_objects_v2_max_keys_zero); + case!(tcx, FsServer, List, test_list_objects_v2_lists_etags); + case!(tcx, FsServer, List, test_list_objects_v2_url_encodes_keys); + case!(tcx, FsServer, List, test_list_objects_lists_imported_key_with_unnameable_sidecar); + case!(tcx, FsServer, List, test_list_objects_v2_url_encodes_echoed_fields); + case!(tcx, FsServer, List, test_list_objects_v1_url_encodes_echoed_fields); } impl List { @@ -751,4 +757,271 @@ impl List { Ok(()) } + + async fn test_list_objects_v2_lists_etags(self: Arc) -> Result<()> { + let c = &self.s3; + let bucket = format!("test-list-etags-{}", Uuid::new_v4()); + let bucket = bucket.as_str(); + create_bucket(c, bucket).await?; + + let put = c + .put_object() + .bucket(bucket) + .key("a.txt") + .body(ByteStream::from_static(b"hello")) + .send() + .await?; + + let result = c.list_objects_v2().bucket(bucket).send().await?; + let listed: Vec<_> = result.contents().iter().map(|obj| obj.e_tag()).collect(); + assert_eq!(listed, [put.e_tag()], "a listed object carries the ETag its PUT returned"); + + delete_object(c, bucket, "a.txt").await?; + delete_bucket(c, bucket).await?; + + Ok(()) + } + + async fn test_list_objects_v2_url_encodes_keys(self: Arc) -> Result<()> { + let c = &self.s3; + let bucket = format!("test-list-url-encoding-{}", Uuid::new_v4()); + let bucket = bucket.as_str(); + create_bucket(c, bucket).await?; + + let keys = ["d/a b", "d/a+b", "e f/x"]; + for key in &keys { + c.put_object() + .bucket(bucket) + .key(*key) + .body(ByteStream::from_static(b"x")) + .send() + .await?; + } + + let encoded = c + .list_objects_v2() + .bucket(bucket) + .encoding_type(aws_sdk_s3::types::EncodingType::Url) + .send() + .await?; + let listed: Vec<_> = encoded.contents().iter().filter_map(|obj| obj.key()).collect(); + assert_eq!(listed, ["d/a%20b", "d/a%2Bb", "e%20f/x"], "keys are url-encoded, `+` included"); + + let grouped = c + .list_objects_v2() + .bucket(bucket) + .delimiter("/") + .encoding_type(aws_sdk_s3::types::EncodingType::Url) + .send() + .await?; + let prefixes: Vec<_> = grouped.common_prefixes().iter().filter_map(|p| p.prefix()).collect(); + assert_eq!(prefixes, ["d/", "e%20f/"], "common prefixes are url-encoded too"); + + let raw = c.list_objects_v2().bucket(bucket).send().await?; + let listed: Vec<_> = raw.contents().iter().filter_map(|obj| obj.key()).collect(); + assert_eq!(listed, keys, "without encoding-type the keys are returned as stored"); + + for key in &keys { + delete_object(c, bucket, key).await?; + } + delete_bucket(c, bucket).await?; + + Ok(()) + } + + async fn test_list_objects_lists_imported_key_with_unnameable_sidecar(self: Arc) -> Result<()> { + let c = &self.s3; + let bucket = format!("test-list-long-key-{}", Uuid::new_v4()); + let bucket = bucket.as_str(); + create_bucket(c, bucket).await?; + + let put = c + .put_object() + .bucket(bucket) + .key("a.txt") + .body(ByteStream::from_static(b"hello")) + .send() + .await?; + + // Every path component of the key is short, so the object itself can be imported, but the + // sidecar name embeds the whole encoded key in one component. + let long_key = format!("{}object", "segment/".repeat(25)); + let encode = |s: &str| base64_simd::URL_SAFE_NO_PAD.encode_to_string(s); + let sidecar_name = format!(".bucket-{}.object-{}.internal.json", encode(bucket), encode(&long_key)); + assert!(sidecar_name.len() > 255, "the sidecar name must exceed the 255-byte component limit"); + + let object_path = Path::new(FS_ROOT).join(bucket).join(&long_key); + if let Some(parent) = object_path.parent() { + std::fs::create_dir_all(parent)?; + } + std::fs::write(&object_path, b"imported")?; + + for v1 in [false, true] { + let listed: Vec<(Option, Option)> = if v1 { + let result = c.list_objects().bucket(bucket).send().await?; + result + .contents() + .iter() + .map(|obj| (obj.key().map(str::to_owned), obj.e_tag().map(str::to_owned))) + .collect() + } else { + let result = c.list_objects_v2().bucket(bucket).send().await?; + result + .contents() + .iter() + .map(|obj| (obj.key().map(str::to_owned), obj.e_tag().map(str::to_owned))) + .collect() + }; + assert_eq!( + listed, + [ + (Some("a.txt".to_owned()), put.e_tag().map(str::to_owned)), + (Some(long_key.clone()), None), + ], + "the imported key lists without an ETag and the stored object keeps its ETag (v1: {v1})" + ); + } + + delete_bucket(c, bucket).await?; + + Ok(()) + } + + async fn test_list_objects_v2_url_encodes_echoed_fields(self: Arc) -> Result<()> { + use aws_sdk_s3::types::EncodingType; + + let c = &self.s3; + let bucket = format!("test-list-url-echo-v2-{}", Uuid::new_v4()); + let bucket = bucket.as_str(); + create_bucket(c, bucket).await?; + + // The prefix holds a literal `%2F`: a client that decodes the echoed prefix must get it back. + let keys = ["a%2Fb/x+1", "a%2Fb/y"]; + for key in &keys { + c.put_object() + .bucket(bucket) + .key(*key) + .body(ByteStream::from_static(b"x")) + .send() + .await?; + } + + let grouped = c + .list_objects_v2() + .bucket(bucket) + .prefix("a%2Fb/") + .delimiter("+") + .start_after("a%2Fb/") + .encoding_type(EncodingType::Url) + .send() + .await?; + assert_eq!(grouped.prefix(), Some("a%252Fb/"), "the echoed prefix is url-encoded"); + assert_eq!(grouped.delimiter(), Some("%2B"), "the echoed delimiter is url-encoded"); + assert_eq!(grouped.start_after(), Some("a%252Fb/"), "the echoed start-after is url-encoded"); + let listed: Vec<_> = grouped.contents().iter().filter_map(|obj| obj.key()).collect(); + assert_eq!(listed, ["a%252Fb/y"], "filtering uses the raw prefix, delimiter and start-after"); + let prefixes: Vec<_> = grouped.common_prefixes().iter().filter_map(|p| p.prefix()).collect(); + assert_eq!(prefixes, ["a%252Fb/x%2B"]); + + // The continuation token is opaque: it is passed back as returned and echoed as sent. + let first = c + .list_objects_v2() + .bucket(bucket) + .prefix("a%2Fb/") + .max_keys(1) + .encoding_type(EncodingType::Url) + .send() + .await?; + let listed: Vec<_> = first.contents().iter().filter_map(|obj| obj.key()).collect(); + assert_eq!(listed, ["a%252Fb/x%2B1"]); + let token = first + .next_continuation_token() + .expect("a truncated page carries a continuation token"); + + let second = c + .list_objects_v2() + .bucket(bucket) + .prefix("a%2Fb/") + .max_keys(1) + .continuation_token(token) + .encoding_type(EncodingType::Url) + .send() + .await?; + assert_eq!(second.continuation_token(), Some(token), "the continuation token is echoed as sent"); + let listed: Vec<_> = second.contents().iter().filter_map(|obj| obj.key()).collect(); + assert_eq!(listed, ["a%252Fb/y"], "the returned continuation token resumes after the first page"); + + for key in &keys { + delete_object(c, bucket, key).await?; + } + delete_bucket(c, bucket).await?; + + Ok(()) + } + + async fn test_list_objects_v1_url_encodes_echoed_fields(self: Arc) -> Result<()> { + use aws_sdk_s3::types::EncodingType; + + let c = &self.s3; + let bucket = format!("test-list-url-echo-v1-{}", Uuid::new_v4()); + let bucket = bucket.as_str(); + create_bucket(c, bucket).await?; + + let keys = ["a%2Fb/x+1", "a%2Fb/y"]; + for key in &keys { + c.put_object() + .bucket(bucket) + .key(*key) + .body(ByteStream::from_static(b"x")) + .send() + .await?; + } + + let grouped = c + .list_objects() + .bucket(bucket) + .prefix("a%2Fb/") + .delimiter("+") + .marker("a%2Fb/") + .encoding_type(EncodingType::Url) + .send() + .await?; + assert_eq!(grouped.prefix(), Some("a%252Fb/"), "the echoed prefix is url-encoded"); + assert_eq!(grouped.delimiter(), Some("%2B"), "the echoed delimiter is url-encoded"); + assert_eq!(grouped.marker(), Some("a%252Fb/"), "the echoed marker is url-encoded"); + let listed: Vec<_> = grouped.contents().iter().filter_map(|obj| obj.key()).collect(); + assert_eq!(listed, ["a%252Fb/y"], "filtering uses the raw prefix, delimiter and marker"); + let prefixes: Vec<_> = grouped.common_prefixes().iter().filter_map(|p| p.prefix()).collect(); + assert_eq!(prefixes, ["a%252Fb/x%2B"]); + + // `NextMarker` is a key, so it is encoded like one; the client decodes it before sending it back. + let first = c + .list_objects() + .bucket(bucket) + .prefix("a%2Fb/") + .max_keys(1) + .encoding_type(EncodingType::Url) + .send() + .await?; + assert_eq!(first.next_marker(), Some("a%252Fb/x%2B1"), "the next marker is url-encoded"); + + let second = c + .list_objects() + .bucket(bucket) + .prefix("a%2Fb/") + .max_keys(1) + .marker("a%2Fb/x+1") + .encoding_type(EncodingType::Url) + .send() + .await?; + let listed: Vec<_> = second.contents().iter().filter_map(|obj| obj.key()).collect(); + assert_eq!(listed, ["a%252Fb/y"], "the decoded next marker resumes after the first page"); + + for key in &keys { + delete_object(c, bucket, key).await?; + } + delete_bucket(c, bucket).await?; + + Ok(()) + } }