fix(s3s-fs): handle long sidecar names and encode listing echoes

crates/s3s-fs/src/fs.rs

@@ -45,6 +45,19 @@ async fn read_sidecar(path: &Path) -> Result<Option<Vec<u8>>> {
     }
 }
 
+/// Read a sidecar like [`read_sidecar`], also treating a name the filesystem cannot represent as absent.
+///
+/// A sidecar name holds the whole encoded key in one path component, so an object imported under a
+/// long nested key can have a sidecar name over the component limit. No sidecar can exist under such
+/// a name, and a listing reports the object as it would one whose sidecar is missing.
+async fn read_nameable_sidecar(path: &Path) -> Result<Option<Vec<u8>>> {
+    match fs::read(path).await {
+        Ok(content) => Ok(Some(content)),
+        Err(err) if matches!(err.kind(), std::io::ErrorKind::NotFound | std::io::ErrorKind::InvalidFilename) => Ok(None),
+        Err(err) => Err(err.into()),
+    }
+}
+
 /// Stores standard object attributes alongside user metadata
 #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)]
 pub(crate) struct ObjectAttributes {
@@ -239,6 +252,16 @@ impl FileSystem {
         Ok(Some(map))
     }
 
+    /// Load internal info for a listed object, where a sidecar name too long to exist reads as absent.
+    pub(crate) async fn load_listed_internal_info(&self, bucket: &str, key: &str) -> Result<Option<InternalInfo>> {
+        let path = self.get_internal_info_path(bucket, key)?;
+        let Some(content) = read_nameable_sidecar(&path).await? else {
+            return Ok(None);
+        };
+        let map = serde_json::from_slice(&content)?;
+        Ok(Some(map))
+    }
+
     pub(crate) async fn save_internal_info(&self, bucket: &str, key: &str, info: &InternalInfo) -> Result<()> {
         let path = self.get_internal_info_path(bucket, key)?;
         let content = serde_json::to_vec(info)?;

crates/s3s-fs/src/s3.rs

@@ -782,17 +782,25 @@ impl S3 for FileSystem {
     async fn list_objects(&self, req: S3Request<ListObjectsInput>) -> S3Result<S3Response<ListObjectsOutput>> {
         let v2_resp = self.list_objects_v2(req.map_input(Into::into)).await?;
 
-        Ok(v2_resp.map_output(|v2| ListObjectsOutput {
-            contents: v2.contents,
-            common_prefixes: v2.common_prefixes,
-            delimiter: v2.delimiter,
-            encoding_type: v2.encoding_type,
-            name: v2.name,
-            prefix: v2.prefix,
-            max_keys: v2.max_keys,
-            is_truncated: v2.is_truncated,
-            next_marker: v2.next_continuation_token,
-            ..Default::default()
+        // The V1 marker travels as the V2 start-after, so the V2 output already echoes it encoded.
+        // `NextMarker` is a key, unlike the opaque V2 continuation token, and is encoded like one.
+        Ok(v2_resp.map_output(|v2| {
+            let url_encoded = is_url_encoding(v2.encoding_type.as_ref());
+            ListObjectsOutput {
+                contents: v2.contents,
+                common_prefixes: v2.common_prefixes,
+                delimiter: v2.delimiter,
+                name: v2.name,
+                prefix: v2.prefix,
+                marker: v2.start_after,
+                max_keys: v2.max_keys,
+                is_truncated: v2.is_truncated,
+                next_marker: v2
+                    .next_continuation_token
+                    .map(|marker| if url_encoded { url_encode_listed(&marker) } else { marker }),
+                encoding_type: v2.encoding_type,
+                ..Default::default()
+            }
         }))
     }
 
@@ -825,11 +833,47 @@ impl S3 for FileSystem {
 
         // The walk collects one page in key order, so a page no longer has to be cut out of the whole
         // bucket. The page arithmetic itself is shared with the full scan the walk replaced.
-        let page = self.list_page(&path, &query).await?;
+        let mut page = self.list_page(&path, &query).await?;
+
+        // Listed objects carry the ETag the PUT stored, as S3 does; a client that
+        // reads ETags from a listing (conditional writes, consistency checks) needs it.
+        for object in &mut page.objects {
+            if let Some(key) = object.key.as_deref() {
+                let info = self.load_listed_internal_info(&input.bucket, key).await?;
+                object.e_tag = info.as_ref().and_then(crate::checksum::load_e_tag).map(ETag::Strong);
+            }
+        }
+
+        // Under encoding-type=url, S3 returns every key and common prefix
+        // URL-encoded and clients decode `+` as a space, so a raw `+` would
+        // read back as a different key.
+        let url_encoded = is_url_encoding(input.encoding_type.as_ref());
+        if url_encoded {
+            for object in &mut page.objects {
+                if let Some(key) = object.key.as_mut() {
+                    *key = url_encode_listed(key);
+                }
+            }
+            for prefix in &mut page.common_prefixes {
+                if let Some(p) = prefix.prefix.as_mut() {
+                    *p = url_encode_listed(p);
+                }
+            }
+        }
 
         let contents = page.objects.is_empty().not().then_some(page.objects);
         let common_prefixes = page.common_prefixes.is_empty().not().then_some(page.common_prefixes);
 
+        // The echoed prefix, delimiter and start-after are encoded only now, after the walk filtered
+        // with the raw values. The continuation token is opaque and returned as the client sent it.
+        let encode_echo = |value: Option<String>| {
+            if url_encoded {
+                value.map(|v| url_encode_listed(&v))
+            } else {
+                value
+            }
+        };
+
         let output = ListObjectsV2Output {
             key_count: Some(page.key_count),
             max_keys: Some(max_keys),
@@ -838,11 +882,11 @@ impl S3 for FileSystem {
             common_prefixes,
             continuation_token: input.continuation_token,
             next_continuation_token: page.next_continuation_token,
-            delimiter: input.delimiter,
+            delimiter: encode_echo(input.delimiter),
             encoding_type: input.encoding_type,
             name: Some(input.bucket),
-            prefix: input.prefix,
-            start_after: input.start_after,
+            prefix: encode_echo(input.prefix),
+            start_after: encode_echo(input.start_after),
             ..Default::default()
         };
         Ok(S3Response::new(output))
@@ -1726,6 +1770,26 @@ impl ListingQuery<'_> {
     }
 }
 
+/// Whether a listing request asked for `encoding-type=url`.
+fn is_url_encoding(encoding_type: Option<&EncodingType>) -> bool {
+    encoding_type.is_some_and(|e| e.as_str() == EncodingType::URL)
+}
+
+/// Percent-encodes every byte outside the unreserved set and `/`.
+fn url_encode_listed(s: &str) -> String {
+    use std::fmt::Write as _;
+
+    let mut out = String::with_capacity(s.len());
+    for b in s.bytes() {
+        if b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.' | b'~' | b'/') {
+            out.push(char::from(b));
+        } else {
+            let _ = write!(out, "%{b:02X}");
+        }
+    }
+    out
+}
+
 /// One page of a listing, plus what follows it.
 #[derive(Debug, Default)]
 struct ListingPage {