feat(services/azblob): implement list_with_deleted operation in azblob

.github/workflows/test_fuzz.yml

@@ -62,14 +62,10 @@ jobs:
       - name: Set Rust Fuzz
         shell: bash
         run: |
-          sudo apt-get update
-          sudo apt-get install -y libfuzzer-14-dev
           rustup install ${{ env.RUST_TOOLCHAIN }}
           cargo +${{ env.RUST_TOOLCHAIN }} install cargo-fuzz
       - name: Fuzz
         uses: ./.github/actions/fuzz_test
-        env:
-          CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
         with:
           setup: ${{ matrix.cases.setup }}
           service: ${{ matrix.cases.service }}

bindings/dotnet/OpenDAL/ServiceConfig/AzblobServiceConfig.cs

@@ -46,6 +46,10 @@ public sealed class AzblobServiceConfig : IServiceConfig
         /// </summary>
         public string? Container { get; init; }
         /// <summary>
+        /// Enable soft deletes for this storage account.
+        /// </summary>
+        public bool? EnableSoftDeletes { get; init; }
+        /// <summary>
         /// The encryption algorithm of Azblob service backend.
         /// </summary>
         public string? EncryptionAlgorithm { get; init; }
@@ -97,6 +101,10 @@ public IReadOnlyDictionary<string, string> ToOptions()
             {
                 map["container"] = Utilities.ToOptionString(Container);
             }
+            if (EnableSoftDeletes is not null)
+            {
+                map["enable_soft_deletes"] = Utilities.ToOptionString(EnableSoftDeletes);
+            }
             if (EncryptionAlgorithm is not null)
             {
                 map["encryption_algorithm"] = Utilities.ToOptionString(EncryptionAlgorithm);

bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java

@@ -182,6 +182,10 @@ class Azblob implements ServiceConfig {
          * <p>The container name of Azblob service backend.</p>
          */
         public final @NonNull String container;
+        /**
+         * <p>Enable soft deletes for this storage account.</p>
+         */
+        public final Boolean enableSoftDeletes;
         /**
          * <p>The encryption algorithm of Azblob service backend.</p>
          */
@@ -235,6 +239,9 @@ public Map<String, String> configMap() {
                 map.put("batch_max_operations", String.valueOf(batchMaxOperations));
             }
             map.put("container", container);
+            if (enableSoftDeletes != null) {
+                map.put("enable_soft_deletes", String.valueOf(enableSoftDeletes));
+            }
             if (encryptionAlgorithm != null) {
                 map.put("encryption_algorithm", encryptionAlgorithm);
             }

bindings/python/python/opendal/config.py

@@ -93,6 +93,8 @@ class AzblobConfig(TypedDict):
     """Deprecated: Azblob delete batch capability is enabled by default with Azure Blob's 256-operation batch limit. [Deprecated since 0.57.0] Azblob delete batch capability is enabled by default with Azure Blob's 256-operation batch limit. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints."""
     container: Required[str]
     """The container name of Azblob service backend."""
+    enable_soft_deletes: NotRequired[bool]
+    """Enable soft deletes for this storage account."""
     encryption_algorithm: NotRequired[str]
     """The encryption algorithm of Azblob service backend."""
     encryption_key: NotRequired[str]

core/services/azblob/src/backend.rs

@@ -264,6 +264,15 @@ impl AzblobBuilder {
         self
     }
 
+    /// Set the soft delete feature for this backend.
+    ///
+    /// If enabled, deleted blobs will be retained for the configured retention period
+    /// and can be listed using list_with_deleted.
+    pub fn enable_soft_deletes(mut self, enabled: bool) -> Self {
+        self.config.enable_soft_deletes = enabled;
+        self
+    }
+
     /// from_connection_string will make a builder from connection string
     ///
     /// connection string looks like:
@@ -442,6 +451,7 @@ impl Builder for AzblobBuilder {
 
             list: true,
             list_with_recursive: true,
+            list_with_deleted: self.config.enable_soft_deletes,
 
             presign: self.config.sas_token.is_some(),
             presign_stat: self.config.sas_token.is_some(),
@@ -588,6 +598,7 @@ impl Service for AzblobBackend {
                 path.to_string(),
                 args.recursive(),
                 args.limit(),
+                args.deleted(),
             );
 
             Ok(oio::PageLister::new(l))

core/services/azblob/src/config.rs

@@ -84,6 +84,10 @@ pub struct AzblobConfig {
     /// Skip signature will skip loading credentials and signing requests.
     #[serde(default)]
     pub skip_signature: bool,
+
+    /// Enable soft deletes for this storage account.
+    #[serde(default)]
+    pub enable_soft_deletes: bool,
 }
 
 impl Debug for AzblobConfig {
@@ -92,6 +96,7 @@ impl Debug for AzblobConfig {
             .field("root", &self.root)
             .field("container", &self.container)
             .field("endpoint", &self.endpoint)
+            .field("enable_soft_deletes", &self.enable_soft_deletes)
             .finish_non_exhaustive()
     }
 }

core/services/azblob/src/core.rs

@@ -824,6 +824,7 @@ impl AzblobCore {
         next_marker: &str,
         delimiter: &str,
         limit: Option<usize>,
+        include_deleted: bool,
     ) -> Result<Response<Buffer>> {
         let p = build_abs_path(&self.root, path);
         let mut url = QueryPairsWriter::new(&format!("{}/{}", self.endpoint, self.container))
@@ -843,6 +844,10 @@ impl AzblobCore {
             url = url.push("marker", &percent_encode_path(next_marker));
         }
 
+        if include_deleted {
+            url = url.push("include", "deleted");
+        }
+
         let req = Request::get(url.finish())
             .extension(Operation::List)
             .extension(ServiceOperation("ListBlobs"))
@@ -916,6 +921,8 @@ pub struct BlobPrefix {
 pub struct Blob {
     pub properties: Properties,
     pub name: String,
+    #[serde(rename = "Deleted")]
+    pub deleted: Option<bool>,
 }
 
 #[derive(Default, Debug, Deserialize)]

core/services/azblob/src/lister.rs

@@ -34,6 +34,7 @@ pub struct AzblobLister {
     path: String,
     delimiter: &'static str,
     limit: Option<usize>,
+    deleted: bool,
 }
 
 impl AzblobLister {
@@ -43,6 +44,7 @@ impl AzblobLister {
         path: String,
         recursive: bool,
         limit: Option<usize>,
+        deleted: bool,
     ) -> Self {
         let delimiter = if recursive { "" } else { "/" };
 
@@ -52,6 +54,7 @@ impl AzblobLister {
             path,
             delimiter,
             limit,
+            deleted,
         }
     }
 }
@@ -66,6 +69,7 @@ impl oio::PageList for AzblobLister {
                 &ctx.token,
                 self.delimiter,
                 self.limit,
+                self.deleted,
             )
             .await?;
 
@@ -118,6 +122,10 @@ impl oio::PageList for AzblobLister {
                     object.properties.last_modified.as_str(),
                 )?);
 
+            if object.deleted.unwrap_or(false) {
+                meta.is_deleted(true);
+            }
+
             let de = oio::Entry::with(path, meta.build());
             ctx.entries.push_back(de);
         }