From 1646da78485744a2e08c5eb10898e988949c100a Mon Sep 17 00:00:00 2001 From: Fatorin Date: Wed, 28 Oct 2026 13:27:20 +0000 Subject: [PATCH] feat(bindings/dotnet)!: generate operation options from core Render the .NET operation options from core/core/src/types/options.rs so new core options reach the binding through `just generate dotnet` instead of hand-written mirrors. For each options struct the generator emits the public `*Options` class, a `#[repr(C)]` mirror with its C# interop struct, and the conversion into core options. Each operation receives its options as one pointer to that struct, null for the defaults, so no signature grows past clippy's argument limit and each operation crosses the FFI once instead of building and releasing a native options handle around the call. Optional values follow the metadata mirror's `*_has_value` convention, strings are null-terminated UTF-8, and string maps are parallel key and value arrays. C# writes the struct and everything it points to into one buffer per call, on the stack when it fits and in native memory otherwise. Native code only borrows it and copies everything into core options before the call returns, so async operations keep no reference to it. This fills the options the binding was missing: - `IfVersionMatch`/`IfVersionNotMatch` on read, stat, write, delete, and copy. - `IfNotChanged` on write, delete, and copy. The metadata crosses the FFI in full as the same `OpendalMetadata` struct the binding returns, written and rebuilt by code generated from core `Metadata`, so the binding never tracks which members a condition reads. - `ContentLengthHint` on read and `SuffixLength` for suffix reads. - `IfNoneMatch` on delete. - New `CopyOptions` and `RenameOptions`, accepted by `Copy`/`CopyAsync` and `Rename`/`RenameAsync`. Constructor options cross as key and value arrays. A null option value is reported as ConfigInvalid naming its key. The .NET guides document suffix reads, conditional copy and rename, and `IfNotChanged`. BREAKING CHANGE: `CopyAsync` and `RenameAsync` take an optional options argument before the cancellation token, matching the other operations. Pass the token by name, e.g. `CopyAsync(from, to, cancellationToken: ct)`. `IOptions` is now internal and `NativeOptionsHandle` is removed; both exposed interop details that callers had no use for. --- .../Behavior/CopyBehaviorTest.cs | 65 +- .../Behavior/DeleteBehaviorTest.cs | 24 + .../Behavior/ReadBehaviorTest.cs | 47 ++ .../Behavior/RenameBehaviorTest.cs | 47 +- .../Behavior/StreamBehaviorTest.cs | 19 + .../Behavior/WriteBehaviorTest.cs | 74 ++ .../dotnet/OpenDAL.Tests/ConstructorTest.cs | 44 +- .../Interop/NativeObject/OpenDALMetadata.cs | 32 + bindings/dotnet/OpenDAL/NativeMethods.cs | 171 +--- bindings/dotnet/OpenDAL/Operator.cs | 250 +++--- .../OpenDAL/Options/Abstractions/IOptions.cs | 24 +- .../dotnet/OpenDAL/Options/CopyOptions.cs | 142 ++++ .../dotnet/OpenDAL/Options/DeleteOptions.cs | 84 +- .../dotnet/OpenDAL/Options/ListOptions.cs | 72 +- .../OpenDAL/Options/NativeOptionsBuilder.cs | 175 ---- .../OpenDAL/Options/NativeOptionsHandle.cs | 48 -- .../OpenDAL/Options/NativeOptionsWriter.cs | 279 +++++++ .../OpenDAL/Options/OptionValidators.cs | 13 + .../dotnet/OpenDAL/Options/ReadOptions.cs | 187 ++++- .../RenameOptions.cs} | 44 +- .../dotnet/OpenDAL/Options/StatOptions.cs | 111 ++- .../dotnet/OpenDAL/Options/WriteOptions.cs | 145 +++- bindings/dotnet/OpenDAL/Utilities.cs | 11 + bindings/dotnet/src/metadata.rs | 80 ++ bindings/dotnet/src/operator.rs | 465 +++-------- bindings/dotnet/src/options.rs | 775 ++++++++++++++---- bindings/dotnet/src/result.rs | 13 - bindings/dotnet/src/utils.rs | 152 ++-- bindings/dotnet/src/validators.rs | 5 +- bindings/dotnet/src/validators/options.rs | 26 +- dev/src/generate/core_options.rs | 206 +++++ dev/src/generate/dotnet.rs | 627 ++++++++++++++ .../generate/dotnet/metadata_interop_cs.j2 | 14 + dev/src/generate/dotnet/metadata_rs.j2 | 42 + dev/src/generate/dotnet/options_cs.j2 | 69 ++ dev/src/generate/dotnet/options_rs.j2 | 268 ++++++ dev/src/generate/metadata.rs | 55 ++ dev/src/generate/mod.rs | 1 + website/docs/20-bindings/dotnet/04-tasks.md | 30 + 39 files changed, 3712 insertions(+), 1224 deletions(-) create mode 100644 bindings/dotnet/OpenDAL/Options/CopyOptions.cs delete mode 100644 bindings/dotnet/OpenDAL/Options/NativeOptionsBuilder.cs delete mode 100644 bindings/dotnet/OpenDAL/Options/NativeOptionsHandle.cs create mode 100644 bindings/dotnet/OpenDAL/Options/NativeOptionsWriter.cs rename bindings/dotnet/OpenDAL/{Interop/Result/OpenDALOptionsResult.cs => Options/RenameOptions.cs} (50%) create mode 100644 dev/src/generate/core_options.rs create mode 100644 dev/src/generate/dotnet/options_cs.j2 create mode 100644 dev/src/generate/dotnet/options_rs.j2 diff --git a/bindings/dotnet/OpenDAL.Tests/Behavior/CopyBehaviorTest.cs b/bindings/dotnet/OpenDAL.Tests/Behavior/CopyBehaviorTest.cs index c11f7b978673..5904a08b3efe 100644 --- a/bindings/dotnet/OpenDAL.Tests/Behavior/CopyBehaviorTest.cs +++ b/bindings/dotnet/OpenDAL.Tests/Behavior/CopyBehaviorTest.cs @@ -17,6 +17,8 @@ * under the License. */ +using OpenDAL.Options; + namespace OpenDAL.Tests; [Collection("BehaviorOperator")] @@ -65,7 +67,7 @@ public async Task CopyBehavior_CreatesTargetWithSameContentAsync() var content = RandomBytes(256); await Op.WriteAsync(sourcePath, content, cancellationToken: CT); - var meta = await Op.CopyAsync(sourcePath, targetPath, CT); + var meta = await Op.CopyAsync(sourcePath, targetPath, cancellationToken: CT); if (meta.IsFile) { @@ -74,4 +76,65 @@ public async Task CopyBehavior_CreatesTargetWithSameContentAsync() Assert.Equal(content, await Op.ReadAsync(targetPath, cancellationToken: CT)); } + + [Fact] + public void CopyBehavior_IfNotExists_CopiesToNewPath() + { + if (!Supports(c => c.Copy && c.Read && c.Write && c.CopyWithIfNotExists)) + { + return; + } + + var sourcePath = NewPath("copy-if-not-exists-source"); + var targetPath = NewPath("copy-if-not-exists-target"); + var content = RandomBytes(256); + + Op.Write(sourcePath, content); + Op.Copy(sourcePath, targetPath, new CopyOptions { IfNotExists = true }); + + Assert.Equal(content, Op.Read(targetPath)); + } + + [Fact] + public async Task CopyBehavior_IfNotExists_RejectsExistingTargetAsync() + { + if (!Supports(c => c.Copy && c.Read && c.Write && c.CopyWithIfNotExists)) + { + return; + } + + var sourcePath = NewPath("copy-if-not-exists-existing-source"); + var targetPath = NewPath("copy-if-not-exists-existing-target"); + var targetContent = RandomBytes(128); + + await Op.WriteAsync(sourcePath, RandomBytes(256), cancellationToken: CT); + await Op.WriteAsync(targetPath, targetContent, cancellationToken: CT); + + var ex = await Assert.ThrowsAsync( + () => Op.CopyAsync(sourcePath, targetPath, new CopyOptions { IfNotExists = true }, CT)); + Assert.Equal(ErrorCode.ConditionNotMatch, ex.Code); + Assert.Equal(targetContent, await Op.ReadAsync(targetPath, cancellationToken: CT)); + } + + [Fact] + public async Task CopyBehavior_IfNotChanged_RejectsStaleTargetMetadataAsync() + { + if (!Supports(c => c.Copy && c.Write && c.Stat && (c.CopyWithIfMatch || c.CopyWithIfVersionMatch))) + { + return; + } + + var sourcePath = NewPath("copy-if-not-changed-source"); + var targetPath = NewPath("copy-if-not-changed-target"); + + await Op.WriteAsync(sourcePath, RandomBytes(256), cancellationToken: CT); + await Op.WriteAsync(targetPath, RandomBytes(128), cancellationToken: CT); + var expected = await Op.StatAsync(targetPath, cancellationToken: CT); + + await Op.CopyAsync(sourcePath, targetPath, new CopyOptions { IfNotChanged = expected }, CT); + + var ex = await Assert.ThrowsAsync( + () => Op.CopyAsync(sourcePath, targetPath, new CopyOptions { IfNotChanged = expected }, CT)); + Assert.Equal(ErrorCode.ConditionNotMatch, ex.Code); + } } diff --git a/bindings/dotnet/OpenDAL.Tests/Behavior/DeleteBehaviorTest.cs b/bindings/dotnet/OpenDAL.Tests/Behavior/DeleteBehaviorTest.cs index a8b13c7e6bba..536a75231332 100644 --- a/bindings/dotnet/OpenDAL.Tests/Behavior/DeleteBehaviorTest.cs +++ b/bindings/dotnet/OpenDAL.Tests/Behavior/DeleteBehaviorTest.cs @@ -192,4 +192,28 @@ public async Task DeleteBehavior_WithIfMatch_DoesNotRemoveObjectIfNotMatchedAsyn var stat = await Op.StatAsync(path, cancellationToken: CT); Assert.Equal(etag, stat.ETag); } + + [Fact] + public async Task DeleteBehavior_IfNotChanged_RejectsStaleMetadataAsync() + { + if (!Supports(c => c.Write && c.Stat && (c.DeleteWithIfMatch || c.DeleteWithIfVersionMatch))) + { + return; + } + + var path = NewPath("delete-if-not-changed"); + await Op.WriteAsync(path, RandomBytes(16), cancellationToken: CT); + var stale = await Op.StatAsync(path, cancellationToken: CT); + await Op.WriteAsync(path, RandomBytes(32), cancellationToken: CT); + + var ex = await Assert.ThrowsAsync( + () => Op.DeleteAsync(path, new DeleteOptions { IfNotChanged = stale }, CT)); + Assert.Equal(ErrorCode.ConditionNotMatch, ex.Code); + + var current = await Op.StatAsync(path, cancellationToken: CT); + await Op.DeleteAsync(path, new DeleteOptions { IfNotChanged = current }, CT); + + var missing = await Assert.ThrowsAsync(() => Op.StatAsync(path, cancellationToken: CT)); + Assert.True(IsMissingError(missing)); + } } diff --git a/bindings/dotnet/OpenDAL.Tests/Behavior/ReadBehaviorTest.cs b/bindings/dotnet/OpenDAL.Tests/Behavior/ReadBehaviorTest.cs index c20a8f3895e6..806cd3280c3d 100644 --- a/bindings/dotnet/OpenDAL.Tests/Behavior/ReadBehaviorTest.cs +++ b/bindings/dotnet/OpenDAL.Tests/Behavior/ReadBehaviorTest.cs @@ -297,4 +297,51 @@ public async Task ReadBehavior_ConsumeCallback_RejectsNullConsumer() await Assert.ThrowsAsync(() => Op.ReadAsync( path, (Func, byte[]>)null!, cancellationToken: CT)); } + + [Fact] + public async Task ReadBehavior_WithSuffixLength_ReturnsTailAsync() + { + if (!Supports(c => c.Read && c.Write && c.ReadWithSuffix)) + { + return; + } + + var path = NewPath("read-suffix"); + var content = RandomBytes(1024); + + await Op.WriteAsync(path, content, cancellationToken: CT); + var tail = await Op.ReadAsync(path, new ReadOptions { SuffixLength = 257 }, CT); + + Assert.Equal(content.AsSpan(content.Length - 257).ToArray(), tail); + } + + [Fact] + public void ReadBehavior_WithSuffixLengthLargerThanFile_ReturnsWholeFile() + { + if (!Supports(c => c.Read && c.Write && c.ReadWithSuffix)) + { + return; + } + + var path = NewPath("read-suffix-larger"); + var content = RandomBytes(1024); + + Op.Write(path, content); + var tail = Op.Read(path, new ReadOptions { SuffixLength = content.Length * 2 }); + + Assert.Equal(content, tail); + } + + [Fact] + public void ReadBehavior_WithSuffixLengthAndOffset_RejectsOptions() + { + if (!Supports(c => c.Read)) + { + return; + } + + var ex = Assert.Throws( + () => Op.Read(NewPath("read-suffix-offset"), new ReadOptions { Offset = 1, SuffixLength = 1 })); + Assert.Equal("SuffixLength", ex.ParamName); + } } diff --git a/bindings/dotnet/OpenDAL.Tests/Behavior/RenameBehaviorTest.cs b/bindings/dotnet/OpenDAL.Tests/Behavior/RenameBehaviorTest.cs index a0824fd190af..0cced44dbfdd 100644 --- a/bindings/dotnet/OpenDAL.Tests/Behavior/RenameBehaviorTest.cs +++ b/bindings/dotnet/OpenDAL.Tests/Behavior/RenameBehaviorTest.cs @@ -17,6 +17,8 @@ * under the License. */ +using OpenDAL.Options; + namespace OpenDAL.Tests; [Collection("BehaviorOperator")] @@ -62,10 +64,53 @@ public async Task RenameBehavior_MovesContentToTargetPathAsync() var content = RandomBytes(111); await Op.WriteAsync(sourcePath, content, cancellationToken: CT); - await Op.RenameAsync(sourcePath, targetPath, CT); + await Op.RenameAsync(sourcePath, targetPath, cancellationToken: CT); Assert.Equal(content, await Op.ReadAsync(targetPath, cancellationToken: CT)); var ex = await Assert.ThrowsAsync(() => Op.ReadAsync(sourcePath, cancellationToken: CT)); Assert.True(IsMissingError(ex)); } + + [Fact] + public void RenameBehavior_IfNotExists_MovesToNewPath() + { + if (!Supports(c => c.Rename && c.Read && c.Write && c.RenameWithIfNotExists)) + { + return; + } + + var sourcePath = NewPath("rename-if-not-exists-source"); + var targetPath = NewPath("rename-if-not-exists-target"); + var content = RandomBytes(111); + + Op.Write(sourcePath, content); + Op.Rename(sourcePath, targetPath, new RenameOptions { IfNotExists = true }); + + Assert.Equal(content, Op.Read(targetPath)); + var ex = Assert.Throws(() => Op.Read(sourcePath)); + Assert.True(IsMissingError(ex)); + } + + [Fact] + public async Task RenameBehavior_IfNotExists_RejectsExistingTargetAsync() + { + if (!Supports(c => c.Rename && c.Read && c.Write && c.RenameWithIfNotExists)) + { + return; + } + + var sourcePath = NewPath("rename-if-not-exists-existing-source"); + var targetPath = NewPath("rename-if-not-exists-existing-target"); + var sourceContent = RandomBytes(111); + var targetContent = RandomBytes(222); + + await Op.WriteAsync(sourcePath, sourceContent, cancellationToken: CT); + await Op.WriteAsync(targetPath, targetContent, cancellationToken: CT); + + var ex = await Assert.ThrowsAsync( + () => Op.RenameAsync(sourcePath, targetPath, new RenameOptions { IfNotExists = true }, CT)); + Assert.Equal(ErrorCode.ConditionNotMatch, ex.Code); + Assert.Equal(sourceContent, await Op.ReadAsync(sourcePath, cancellationToken: CT)); + Assert.Equal(targetContent, await Op.ReadAsync(targetPath, cancellationToken: CT)); + } } diff --git a/bindings/dotnet/OpenDAL.Tests/Behavior/StreamBehaviorTest.cs b/bindings/dotnet/OpenDAL.Tests/Behavior/StreamBehaviorTest.cs index 44e9ca8372c8..4ca5d597be61 100644 --- a/bindings/dotnet/OpenDAL.Tests/Behavior/StreamBehaviorTest.cs +++ b/bindings/dotnet/OpenDAL.Tests/Behavior/StreamBehaviorTest.cs @@ -298,4 +298,23 @@ public void StreamBehavior_OpenReadStream_VersionUnsupported_SurfacesUnsupported Assert.Equal(ErrorCode.Unsupported, ex.Code); } + + [Fact] + public void StreamBehavior_OpenReadStream_WithSuffixLength_ReturnsTail() + { + if (!Supports(c => c.Read && c.Write && c.ReadWithSuffix)) + { + return; + } + + var path = NewPath("stream-suffix"); + var content = RandomBytes(1024); + Op.Write(path, content); + + using var input = Op.OpenReadStream(path, new ReadOptions { SuffixLength = 100 }); + using var output = new MemoryStream(); + input.CopyTo(output); + + Assert.Equal(content.AsSpan(content.Length - 100).ToArray(), output.ToArray()); + } } diff --git a/bindings/dotnet/OpenDAL.Tests/Behavior/WriteBehaviorTest.cs b/bindings/dotnet/OpenDAL.Tests/Behavior/WriteBehaviorTest.cs index 5abb751dc9d0..9c1804538f92 100644 --- a/bindings/dotnet/OpenDAL.Tests/Behavior/WriteBehaviorTest.cs +++ b/bindings/dotnet/OpenDAL.Tests/Behavior/WriteBehaviorTest.cs @@ -419,4 +419,78 @@ public void WriteBehavior_WithUserMetadata_RoundtripsThroughStat() Assert.NotNull(meta.UserMetadata); Assert.Equal("everywhere", meta.UserMetadata["location"]); } + + [Fact] + public async Task WriteBehavior_IfNotChanged_RejectsStaleMetadataAsync() + { + if (!Supports(c => c.Write && c.Stat && (c.WriteWithIfMatch || c.WriteWithIfVersionMatch))) + { + return; + } + + var path = NewPath("write-if-not-changed"); + await Op.WriteAsync(path, RandomBytes(64), cancellationToken: CT); + var expected = await Op.StatAsync(path, cancellationToken: CT); + + var replacement = RandomBytes(96); + await Op.WriteAsync(path, replacement, new WriteOptions { IfNotChanged = expected }, CT); + + var ex = await Assert.ThrowsAsync( + () => Op.WriteAsync(path, RandomBytes(32), new WriteOptions { IfNotChanged = expected }, CT)); + Assert.Equal(ErrorCode.ConditionNotMatch, ex.Code); + Assert.Equal(replacement, await Op.ReadAsync(path, cancellationToken: CT)); + } + + [Fact] + public async Task WriteBehavior_OptionsLargerThanStackBuffer_RoundTripAsync() + { + if (!Supports(c => c.Write && c.Stat && c.WriteWithContentType)) + { + return; + } + + // Well past the stack buffer, so the options go through native memory + // instead. ASCII only, since it travels as an HTTP header. + var contentType = $"text/x-{new string('a', 1024)}"; + var syncPath = NewPath("write-large-options"); + var asyncPath = NewPath("write-large-options-async"); + + Op.Write(syncPath, new byte[] { 1 }, new WriteOptions { ContentType = contentType }); + await Op.WriteAsync(asyncPath, new byte[] { 1 }, new WriteOptions { ContentType = contentType }, CT); + + Assert.Equal(contentType, (await Op.StatAsync(syncPath, cancellationToken: CT)).ContentType); + Assert.Equal(contentType, (await Op.StatAsync(asyncPath, cancellationToken: CT)).ContentType); + } + + [Fact] + public async Task WriteBehavior_ConcurrentWrites_KeepEachCallsOptionsAsync() + { + // The race this guards against is in the binding, not the service, and + // it needs many concurrent calls to show, so only memory runs it. + if (!IsService("memory") || !Supports(c => c.Write && c.Stat && c.WriteWithContentType)) + { + return; + } + + // Each call writes its options into a buffer of its own, so + // concurrent calls must never see each other's options. + var writes = Enumerable.Range(0, 1000).Select(i => Task.Run(async () => + { + var path = NewPath($"write-concurrent-{i}"); + var options = new WriteOptions { ContentType = $"text/x-{i}-{new string('a', i % 64)}" }; + if (i % 2 == 0) + { + Op.Write(path, new byte[] { 1 }, options); + } + else + { + await Op.WriteAsync(path, new byte[] { 1 }, options, CT); + } + + var meta = await Op.StatAsync(path, cancellationToken: CT); + Assert.Equal(options.ContentType, meta.ContentType); + }, CT)); + + await Task.WhenAll(writes); + } } diff --git a/bindings/dotnet/OpenDAL.Tests/ConstructorTest.cs b/bindings/dotnet/OpenDAL.Tests/ConstructorTest.cs index 499d98999f52..b6bc7e8c9c43 100644 --- a/bindings/dotnet/OpenDAL.Tests/ConstructorTest.cs +++ b/bindings/dotnet/OpenDAL.Tests/ConstructorTest.cs @@ -63,7 +63,7 @@ public void Constructor_OptionValueIsNull_ThrowsConfigInvalid() var ex = Assert.Throws(() => new Operator("memory", options)); Assert.Equal(ErrorCode.ConfigInvalid, ex.Code); - Assert.Contains("value at index 0", ex.Message); + Assert.Contains("value of option root is null", ex.Message); } [Fact] @@ -99,4 +99,46 @@ public void Constructor_ServiceConfigProvided_ConstructsOperator() Assert.NotEqual(IntPtr.Zero, op.DangerousGetHandle()); } + + [Fact] + public void Constructor_OptionsDictionaryCallsBackIntoOpenDAL_KeepsOptionsIntact() + { + using var other = new Operator("memory"); + other.Write("file", new byte[] { 1, 2, 3 }); + var options = new ReentrantDictionary(other, new Dictionary { ["root"] = "/reentrant/" }); + + using var op = new Operator("memory", options); + + Assert.Equal("/reentrant/", op.Info.Root); + } + + /// + /// Serializes other options on the same thread while it is being enumerated. + /// + private sealed class ReentrantDictionary(Operator other, Dictionary inner) + : IReadOnlyDictionary + { + public string this[string key] => inner[key]; + + public IEnumerable Keys => inner.Keys; + + public IEnumerable Values => inner.Values; + + public int Count => inner.Count; + + public bool ContainsKey(string key) => inner.ContainsKey(key); + + public bool TryGetValue(string key, out string value) => inner.TryGetValue(key, out value!); + + public IEnumerator> GetEnumerator() + { + foreach (var entry in inner) + { + other.Read("file", new OpenDAL.Options.ReadOptions { Offset = 1, Length = 1 }); + yield return entry; + } + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator(); + } } diff --git a/bindings/dotnet/OpenDAL/Interop/NativeObject/OpenDALMetadata.cs b/bindings/dotnet/OpenDAL/Interop/NativeObject/OpenDALMetadata.cs index 5e0b69944237..921f1c20c28e 100644 --- a/bindings/dotnet/OpenDAL/Interop/NativeObject/OpenDALMetadata.cs +++ b/bindings/dotnet/OpenDAL/Interop/NativeObject/OpenDALMetadata.cs @@ -20,6 +20,7 @@ // DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. using System.Runtime.InteropServices; +using OpenDAL.Options; namespace OpenDAL.Interop.NativeObject; @@ -72,4 +73,35 @@ public readonly Metadata ToMetadata() UserMetadataHasValue != 0 ? Utilities.ReadStringPairs(UserMetadataKeys, UserMetadataValues, UserMetadataLen, StringComparer.Ordinal) : null ); } + + /// + /// Fills a payload from managed metadata for native code to borrow, for + /// options such as IfNotChanged. + /// + public static OpenDALMetadata From(Metadata metadata, ref NativeOptionsWriter writer) + { + var native = new OpenDALMetadata(); + native.Mode = (int)metadata.Mode; + if (metadata.IsCurrent is { } isCurrent) + { + native.IsCurrentHasValue = 1; + native.IsCurrent = isCurrent ? (byte)1 : (byte)0; + } + native.IsDeleted = metadata.IsDeleted ? (byte)1 : (byte)0; + native.CacheControl = writer.Utf8(metadata.CacheControl); + native.ContentLength = metadata.ContentLength; + native.ContentMd5 = writer.Utf8(metadata.ContentMd5); + native.ContentType = writer.Utf8(metadata.ContentType); + native.ContentEncoding = writer.Utf8(metadata.ContentEncoding); + if (metadata.LastModified is { } lastModified) + { + native.LastModifiedHasValue = 1; + (native.LastModifiedSecond, native.LastModifiedNanosecond) = Utilities.ToUnixSecondAndNanosecond(lastModified); + } + native.ETag = writer.Utf8(metadata.ETag); + native.ContentDisposition = writer.Utf8(metadata.ContentDisposition); + native.Version = writer.Utf8(metadata.Version); + (native.UserMetadataHasValue, native.UserMetadataKeys, native.UserMetadataValues, native.UserMetadataLen) = writer.Pairs(metadata.UserMetadata); + return native; + } } diff --git a/bindings/dotnet/OpenDAL/NativeMethods.cs b/bindings/dotnet/OpenDAL/NativeMethods.cs index 212619200309..c6c40b8679a6 100644 --- a/bindings/dotnet/OpenDAL/NativeMethods.cs +++ b/bindings/dotnet/OpenDAL/NativeMethods.cs @@ -21,6 +21,7 @@ using System.Runtime.CompilerServices; using OpenDAL.Interop.Buffers; using OpenDAL.Interop.Result; +using OpenDAL.Options; namespace OpenDAL; @@ -32,24 +33,14 @@ internal partial class NativeMethods [LibraryImport(__DllName, EntryPoint = "operator_construct", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALOperatorResult operator_construct( + internal static unsafe partial OpenDALOperatorResult operator_construct( string scheme, - IntPtr options, + IntPtr* optionKeys, + IntPtr* optionValues, + nuint optionLen, IntPtr executor ); - [LibraryImport(__DllName, EntryPoint = "constructor_option_build", StringMarshalling = StringMarshalling.Utf8)] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALOptionsResult constructor_option_build( - [In] string[] keys, - [In] string[] values, - nuint len - ); - - [LibraryImport(__DllName, EntryPoint = "constructor_option_free")] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial void constructor_option_free(IntPtr options); - [LibraryImport(__DllName, EntryPoint = "operator_free")] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial void operator_free(IntPtr op); @@ -64,90 +55,6 @@ nuint len #endregion - #region Option Builders - - #region ReadOption - - [LibraryImport(__DllName, EntryPoint = "read_option_build", StringMarshalling = StringMarshalling.Utf8)] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALOptionsResult read_option_build( - [In] string[] keys, - [In] string[] values, - nuint len - ); - - [LibraryImport(__DllName, EntryPoint = "read_option_free")] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial void read_option_free(IntPtr options); - - #endregion - - #region WriteOption - - [LibraryImport(__DllName, EntryPoint = "write_option_build", StringMarshalling = StringMarshalling.Utf8)] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALOptionsResult write_option_build( - [In] string[] keys, - [In] string[] values, - nuint len - ); - - [LibraryImport(__DllName, EntryPoint = "write_option_free")] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial void write_option_free(IntPtr options); - - #endregion - - #region StatOption - - [LibraryImport(__DllName, EntryPoint = "stat_option_build", StringMarshalling = StringMarshalling.Utf8)] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALOptionsResult stat_option_build( - [In] string[] keys, - [In] string[] values, - nuint len - ); - - [LibraryImport(__DllName, EntryPoint = "stat_option_free")] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial void stat_option_free(IntPtr options); - - #endregion - - #region ListOption - - [LibraryImport(__DllName, EntryPoint = "list_option_build", StringMarshalling = StringMarshalling.Utf8)] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALOptionsResult list_option_build( - [In] string[] keys, - [In] string[] values, - nuint len - ); - - [LibraryImport(__DllName, EntryPoint = "list_option_free")] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial void list_option_free(IntPtr options); - - #endregion - - #region DeleteOption - - [LibraryImport(__DllName, EntryPoint = "delete_option_build", StringMarshalling = StringMarshalling.Utf8)] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALOptionsResult delete_option_build( - [In] string[] keys, - [In] string[] values, - nuint len - ); - - [LibraryImport(__DllName, EntryPoint = "delete_option_free")] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial void delete_option_free(IntPtr options); - - #endregion - - #endregion - #region Layer [LibraryImport(__DllName, EntryPoint = "operator_layer_retry")] @@ -244,12 +151,12 @@ nuint minCapacity [LibraryImport(__DllName, EntryPoint = "operator_write_bytes_with_options", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALMetadataResult operator_write_bytes_with_options( + internal static unsafe partial OpenDALMetadataResult operator_write_bytes_with_options( Operator op, string path, [In] byte[] data, nuint len, - IntPtr options + OpenDALWriteOptions* options ); [LibraryImport(__DllName, EntryPoint = "operator_write_bytes_with_options_async", StringMarshalling = StringMarshalling.Utf8)] @@ -259,19 +166,19 @@ internal static unsafe partial OpenDALResult operator_write_bytes_with_options_a string path, [In] byte[] data, nuint len, - IntPtr options, + OpenDALWriteOptions* options, delegate* unmanaged[Cdecl] callback, long context ); [LibraryImport(__DllName, EntryPoint = "operator_write_with_options", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALMetadataResult operator_write_with_options( + internal static unsafe partial OpenDALMetadataResult operator_write_with_options( Operator op, string path, IntPtr buffer, nuint committedInCurrent, - IntPtr options + OpenDALWriteOptions* options ); [LibraryImport(__DllName, EntryPoint = "operator_write_with_options_async", StringMarshalling = StringMarshalling.Utf8)] @@ -281,7 +188,7 @@ internal static unsafe partial OpenDALResult operator_write_with_options_async( string path, IntPtr buffer, nuint committedInCurrent, - IntPtr options, + OpenDALWriteOptions* options, delegate* unmanaged[Cdecl] callback, long context ); @@ -292,10 +199,10 @@ long context [LibraryImport(__DllName, EntryPoint = "operator_read_with_options", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALReadResult operator_read_with_options( + internal static unsafe partial OpenDALReadResult operator_read_with_options( Operator op, string path, - IntPtr options + OpenDALReadOptions* options ); [LibraryImport(__DllName, EntryPoint = "operator_read_with_options_async", StringMarshalling = StringMarshalling.Utf8)] @@ -303,7 +210,7 @@ IntPtr options internal static unsafe partial OpenDALResult operator_read_with_options_async( Operator op, string path, - IntPtr options, + OpenDALReadOptions* options, delegate* unmanaged[Cdecl] callback, long context ); @@ -314,10 +221,10 @@ long context [LibraryImport(__DllName, EntryPoint = "operator_stat_with_options", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALMetadataResult operator_stat_with_options( + internal static unsafe partial OpenDALMetadataResult operator_stat_with_options( Operator op, string path, - IntPtr options + OpenDALStatOptions* options ); [LibraryImport(__DllName, EntryPoint = "operator_stat_with_options_async", StringMarshalling = StringMarshalling.Utf8)] @@ -325,7 +232,7 @@ IntPtr options internal static unsafe partial OpenDALResult operator_stat_with_options_async( Operator op, string path, - IntPtr options, + OpenDALStatOptions* options, delegate* unmanaged[Cdecl] callback, long context ); @@ -372,10 +279,10 @@ long context [LibraryImport(__DllName, EntryPoint = "operator_list_with_options", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALEntryListResult operator_list_with_options( + internal static unsafe partial OpenDALEntryListResult operator_list_with_options( Operator op, string path, - IntPtr options + OpenDALListOptions* options ); [LibraryImport(__DllName, EntryPoint = "operator_list_with_options_async", StringMarshalling = StringMarshalling.Utf8)] @@ -383,7 +290,7 @@ IntPtr options internal static unsafe partial OpenDALResult operator_list_with_options_async( Operator op, string path, - IntPtr options, + OpenDALListOptions* options, delegate* unmanaged[Cdecl] callback, long context ); @@ -394,10 +301,10 @@ long context [LibraryImport(__DllName, EntryPoint = "operator_delete_with_options", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALResult operator_delete_with_options( + internal static unsafe partial OpenDALResult operator_delete_with_options( Operator op, string path, - IntPtr options + OpenDALDeleteOptions* options ); [LibraryImport(__DllName, EntryPoint = "operator_delete_with_options_async", StringMarshalling = StringMarshalling.Utf8)] @@ -405,7 +312,7 @@ IntPtr options internal static unsafe partial OpenDALResult operator_delete_with_options_async( Operator op, string path, - IntPtr options, + OpenDALDeleteOptions* options, delegate* unmanaged[Cdecl] callback, long context ); @@ -434,20 +341,22 @@ long context #region Copy - [LibraryImport(__DllName, EntryPoint = "operator_copy", StringMarshalling = StringMarshalling.Utf8)] + [LibraryImport(__DllName, EntryPoint = "operator_copy_with_options", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALMetadataResult operator_copy( + internal static unsafe partial OpenDALMetadataResult operator_copy_with_options( Operator op, string sourcePath, - string targetPath + string targetPath, + OpenDALCopyOptions* options ); - [LibraryImport(__DllName, EntryPoint = "operator_copy_async", StringMarshalling = StringMarshalling.Utf8)] + [LibraryImport(__DllName, EntryPoint = "operator_copy_with_options_async", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static unsafe partial OpenDALResult operator_copy_async( + internal static unsafe partial OpenDALResult operator_copy_with_options_async( Operator op, string sourcePath, string targetPath, + OpenDALCopyOptions* options, delegate* unmanaged[Cdecl] callback, long context ); @@ -456,20 +365,22 @@ long context #region Rename - [LibraryImport(__DllName, EntryPoint = "operator_rename", StringMarshalling = StringMarshalling.Utf8)] + [LibraryImport(__DllName, EntryPoint = "operator_rename_with_options", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALResult operator_rename( + internal static unsafe partial OpenDALResult operator_rename_with_options( Operator op, string sourcePath, - string targetPath + string targetPath, + OpenDALRenameOptions* options ); - [LibraryImport(__DllName, EntryPoint = "operator_rename_async", StringMarshalling = StringMarshalling.Utf8)] + [LibraryImport(__DllName, EntryPoint = "operator_rename_with_options_async", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static unsafe partial OpenDALResult operator_rename_async( + internal static unsafe partial OpenDALResult operator_rename_with_options_async( Operator op, string sourcePath, string targetPath, + OpenDALRenameOptions* options, delegate* unmanaged[Cdecl] callback, long context ); @@ -524,10 +435,10 @@ long context [LibraryImport(__DllName, EntryPoint = "operator_input_stream_create", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALOperatorResult operator_input_stream_create( + internal static unsafe partial OpenDALOperatorResult operator_input_stream_create( Operator op, string path, - IntPtr options + OpenDALReadOptions* options ); [LibraryImport(__DllName, EntryPoint = "operator_input_stream_read_next")] @@ -548,10 +459,10 @@ long context [LibraryImport(__DllName, EntryPoint = "operator_output_stream_create", StringMarshalling = StringMarshalling.Utf8)] [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] - internal static partial OpenDALOperatorResult operator_output_stream_create( + internal static unsafe partial OpenDALOperatorResult operator_output_stream_create( Operator op, string path, - IntPtr options + OpenDALWriteOptions* options ); [LibraryImport(__DllName, EntryPoint = "operator_output_stream_write")] diff --git a/bindings/dotnet/OpenDAL/Operator.cs b/bindings/dotnet/OpenDAL/Operator.cs index 79c2a11cde04..d0ee0d5f8e4a 100644 --- a/bindings/dotnet/OpenDAL/Operator.cs +++ b/bindings/dotnet/OpenDAL/Operator.cs @@ -34,6 +34,9 @@ namespace OpenDAL; /// /// Managed wrapper over an OpenDAL native operator handle. /// +// Options are written to a stack buffer per call and native code reads only +// what was written, so the buffer does not need zeroing. +[SkipLocalsInit] public partial class Operator : SafeHandle { private readonly OperatorInfo info; @@ -108,24 +111,38 @@ public Operator(IServiceConfig config, Executor? executor = null) : base(IntPtr. info = CreateOperatorInfo(); } - private static IntPtr Construct( + private static unsafe IntPtr Construct( string scheme, IReadOnlyDictionary? options, Executor? executor) { - using var nativeOptionsHandle = CreateConstructorOptionsHandle(options); + // Construction is rare, so the pairs always go to native memory, + // sized by a writer that only counts. + var counter = new NativeOptionsWriter(null, 0); + counter.Pairs(options); + var size = counter.RequiredCapacity; + var buffer = NativeMemory.Alloc((nuint)size); var executorAddRefed = false; try { + var writer = new NativeOptionsWriter((byte*)buffer, size); + var (_, keys, values, len) = writer.Pairs(options); + if (writer.Overflowed) + { + throw NativeOptionsWriter.Changed(); + } executor?.DangerousAddRef(ref executorAddRefed); var result = NativeMethods.operator_construct( scheme, - GetOptionsHandle(nativeOptionsHandle), + (IntPtr*)keys, + (IntPtr*)values, + len, executor?.DangerousGetHandle() ?? IntPtr.Zero); return ToValueOrThrowAndRelease(result); } finally { + NativeMemory.Free(buffer); if (executorAddRefed) { executor!.DangerousRelease(); @@ -155,18 +172,18 @@ public Operator WithLayer(ILayer layer) /// Bytes to write. /// Additional write options. /// Metadata of the written object. - public Metadata Write(string path, byte[] content, WriteOptions? options = null) + public unsafe Metadata Write(string path, byte[] content, WriteOptions? options = null) { ArgumentNullException.ThrowIfNull(content); ObjectDisposedException.ThrowIf(IsInvalid, this); - using var nativeOptionsHandle = options?.BuildNativeOptionsHandle(); + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); var result = NativeMethods.operator_write_bytes_with_options( this, path, content, (nuint)content.Length, - GetOptionsHandle(nativeOptionsHandle) + nativeOptions.Pointer ); return ToValueOrThrowAndRelease(result); @@ -233,7 +250,7 @@ public Metadata Write( /// The operator or buffer has been disposed. /// The buffer contents were already consumed by a write. /// Native write fails. - internal Metadata Write(string path, WriteBuffer content, WriteOptions? options = null) + internal unsafe Metadata Write(string path, WriteBuffer content, WriteOptions? options = null) { ArgumentNullException.ThrowIfNull(content); ObjectDisposedException.ThrowIf(IsInvalid, this); @@ -241,13 +258,13 @@ internal Metadata Write(string path, WriteBuffer content, WriteOptions? options var bufferHandle = content.Handle; var committedInTail = content.TailWritten; - using var nativeOptionsHandle = options?.BuildNativeOptionsHandle(); + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); var result = NativeMethods.operator_write_with_options( this, path, bufferHandle, (nuint)committedInTail, - GetOptionsHandle(nativeOptionsHandle) + nativeOptions.Pointer ); if (!result.Error.IsError) @@ -275,18 +292,19 @@ public Task WriteAsync( ArgumentNullException.ThrowIfNull(content); ObjectDisposedException.ThrowIf(IsInvalid, this); - return SubmitAsyncOperation(options, DispatchWriteBytesAsync, cancellationToken); + return SubmitAsyncOperation(DispatchWriteBytesAsync, cancellationToken); - OpenDALResult DispatchWriteBytesAsync(long context, IntPtr optionsHandle) + OpenDALResult DispatchWriteBytesAsync(long context) { unsafe { + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); return NativeMethods.operator_write_bytes_with_options_async( this, path, content, (nuint)content.Length, - optionsHandle, + nativeOptions.Pointer, &OnWriteCompleted, context ); @@ -358,18 +376,19 @@ internal Task WriteAsync( var bufferHandle = content.Handle; var committedInTail = content.TailWritten; - return SubmitAsyncOperation(options, DispatchWriteBufferAsync, cancellationToken); + return SubmitAsyncOperation(DispatchWriteBufferAsync, cancellationToken); - OpenDALResult DispatchWriteBufferAsync(long context, IntPtr optionsHandle) + OpenDALResult DispatchWriteBufferAsync(long context) { unsafe { + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); var result = NativeMethods.operator_write_with_options_async( this, path, bufferHandle, (nuint)committedInTail, - optionsHandle, + nativeOptions.Pointer, &OnWriteCompleted, context ); @@ -468,13 +487,13 @@ public Task ReadAsync( /// The content as a disposable native buffer. /// The operator or executor has been disposed. /// Native read fails. - internal ReadBuffer ReadBuffer(string path, ReadOptions? options = null) + internal unsafe ReadBuffer ReadBuffer(string path, ReadOptions? options = null) { ObjectDisposedException.ThrowIf(IsInvalid, this); - using var nativeOptionsHandle = options?.BuildNativeOptionsHandle(); + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); var result = NativeMethods.operator_read_with_options( - this, path, GetOptionsHandle(nativeOptionsHandle)); + this, path, nativeOptions.Pointer); return Interop.Buffers.ReadBuffer.FromResult(result); } @@ -504,24 +523,9 @@ internal async Task ReadBufferAsync( cancellationToken.ThrowIfCancellationRequested(); var context = AsyncStateRegistry.Register(out var state); - OpenDALResult submit; - using (var nativeOptionsHandle = options?.BuildNativeOptionsHandle()) - { - unsafe - { - submit = NativeMethods.operator_read_with_options_async( - this, - path, - GetOptionsHandle(nativeOptionsHandle), - &OnReadResultRetained, - context - ); - } - } - try { - ThrowIfErrorAndRelease(submit); + ThrowIfErrorAndRelease(SubmitReadBufferAsync(path, options, context)); } catch { @@ -534,6 +538,19 @@ internal async Task ReadBufferAsync( return Interop.Buffers.ReadBuffer.FromResult(result); } + // Async methods cannot hold the stack buffer the options are written to. + private unsafe OpenDALResult SubmitReadBufferAsync(string path, ReadOptions? options, long context) + { + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); + return NativeMethods.operator_read_with_options_async( + this, + path, + nativeOptions.Pointer, + &OnReadResultRetained, + context + ); + } + /// /// Reads a path and consumes the payload in place as a /// over native memory, without copying @@ -613,13 +630,13 @@ internal static void OnReadResultRetained(long context, OpenDALReadResult result /// Target path in the configured backend. /// Additional stat options. /// Metadata of the target path. - public Metadata Stat(string path, StatOptions? options = null) + public unsafe Metadata Stat(string path, StatOptions? options = null) { ObjectDisposedException.ThrowIf(IsInvalid, this); OpenDALMetadataResult result; - using var nativeOptionsHandle = options?.BuildNativeOptionsHandle(); - result = NativeMethods.operator_stat_with_options(this, path, GetOptionsHandle(nativeOptionsHandle)); + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); + result = NativeMethods.operator_stat_with_options(this, path, nativeOptions.Pointer); return ToValueOrThrowAndRelease(result); } @@ -638,16 +655,17 @@ public Task StatAsync( { ObjectDisposedException.ThrowIf(IsInvalid, this); - return SubmitAsyncOperation(options, SubmitStatAsync, cancellationToken); + return SubmitAsyncOperation(SubmitStatAsync, cancellationToken); - OpenDALResult SubmitStatAsync(long context, IntPtr optionsHandle) + OpenDALResult SubmitStatAsync(long context) { unsafe { + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); return NativeMethods.operator_stat_with_options_async( this, path, - optionsHandle, + nativeOptions.Pointer, &OnStatCompleted, context ); @@ -749,13 +767,13 @@ OpenDALResult SubmitCheckAsync(long context) /// Target path in the configured backend. /// Additional list options. /// Listed entries. - public IReadOnlyList List(string path, ListOptions? options = null) + public unsafe IReadOnlyList List(string path, ListOptions? options = null) { ObjectDisposedException.ThrowIf(IsInvalid, this); OpenDALEntryListResult result; - using var nativeOptionsHandle = options?.BuildNativeOptionsHandle(); - result = NativeMethods.operator_list_with_options(this, path, GetOptionsHandle(nativeOptionsHandle)); + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); + result = NativeMethods.operator_list_with_options(this, path, nativeOptions.Pointer); return ToValueOrThrowAndRelease, OpenDALEntryListResult>(result); } @@ -774,16 +792,17 @@ public Task> ListAsync( { ObjectDisposedException.ThrowIf(IsInvalid, this); - return SubmitAsyncOperation, ListOptions>(options, SubmitListAsync, cancellationToken); + return SubmitAsyncOperation>(SubmitListAsync, cancellationToken); - OpenDALResult SubmitListAsync(long context, IntPtr optionsHandle) + OpenDALResult SubmitListAsync(long context) { unsafe { + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); return NativeMethods.operator_list_with_options_async( this, path, - optionsHandle, + nativeOptions.Pointer, &OnListCompleted, context ); @@ -816,11 +835,11 @@ public Operator Duplicate() /// /// Target path in the configured backend. /// Additional delete options. - public void Delete(string path, DeleteOptions? options = null) + public unsafe void Delete(string path, DeleteOptions? options = null) { ObjectDisposedException.ThrowIf(IsInvalid, this); - using var nativeOptionsHandle = options?.BuildNativeOptionsHandle(); - var result = NativeMethods.operator_delete_with_options(this, path, GetOptionsHandle(nativeOptionsHandle)); + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); + var result = NativeMethods.operator_delete_with_options(this, path, nativeOptions.Pointer); ThrowIfErrorAndRelease(result); } @@ -835,16 +854,17 @@ public Task DeleteAsync(string path, DeleteOptions? options = null, Cancellation { ObjectDisposedException.ThrowIf(IsInvalid, this); - return SubmitAsyncOperation(options, SubmitDeleteAsync, cancellationToken); + return SubmitAsyncOperation(SubmitDeleteAsync, cancellationToken); - OpenDALResult SubmitDeleteAsync(long context, IntPtr optionsHandle) + OpenDALResult SubmitDeleteAsync(long context) { unsafe { + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); return NativeMethods.operator_delete_with_options_async( this, path, - optionsHandle, + nativeOptions.Pointer, &OnDeleteCompleted, context ); @@ -894,11 +914,17 @@ OpenDALResult SubmitCreateDirAsync(long context) /// /// Source path in the configured backend. /// Target path in the configured backend. + /// Additional copy options. /// Metadata of the copied object at . - public Metadata Copy(string sourcePath, string targetPath) + public unsafe Metadata Copy(string sourcePath, string targetPath, CopyOptions? options = null) { ObjectDisposedException.ThrowIf(IsInvalid, this); - var result = NativeMethods.operator_copy(this, sourcePath, targetPath); + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); + var result = NativeMethods.operator_copy_with_options( + this, + sourcePath, + targetPath, + nativeOptions.Pointer); return ToValueOrThrowAndRelease(result); } @@ -907,11 +933,13 @@ public Metadata Copy(string sourcePath, string targetPath) /// /// Source path in the configured backend. /// Target path in the configured backend. + /// Additional copy options. /// Cancellation token for the managed task. /// A task that resolves with the metadata of the copied object. public Task CopyAsync( string sourcePath, string targetPath, + CopyOptions? options = null, CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(IsInvalid, this); @@ -922,10 +950,12 @@ OpenDALResult SubmitCopyAsync(long context) { unsafe { - return NativeMethods.operator_copy_async( + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); + return NativeMethods.operator_copy_with_options_async( this, sourcePath, targetPath, + nativeOptions.Pointer, &OnCopyCompleted, context ); @@ -938,10 +968,16 @@ OpenDALResult SubmitCopyAsync(long context) /// /// Source path in the configured backend. /// Target path in the configured backend. - public void Rename(string sourcePath, string targetPath) + /// Additional rename options. + public unsafe void Rename(string sourcePath, string targetPath, RenameOptions? options = null) { ObjectDisposedException.ThrowIf(IsInvalid, this); - var result = NativeMethods.operator_rename(this, sourcePath, targetPath); + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); + var result = NativeMethods.operator_rename_with_options( + this, + sourcePath, + targetPath, + nativeOptions.Pointer); ThrowIfErrorAndRelease(result); } @@ -950,11 +986,13 @@ public void Rename(string sourcePath, string targetPath) /// /// Source path in the configured backend. /// Target path in the configured backend. + /// Additional rename options. /// Cancellation token for the managed task. /// A task that completes when the native callback reports completion. public Task RenameAsync( string sourcePath, string targetPath, + RenameOptions? options = null, CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(IsInvalid, this); @@ -965,10 +1003,12 @@ OpenDALResult SubmitRenameAsync(long context) { unsafe { - return NativeMethods.operator_rename_async( + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); + return NativeMethods.operator_rename_with_options_async( this, sourcePath, targetPath, + nativeOptions.Pointer, &OnRenameCompleted, context ); @@ -1116,17 +1156,17 @@ OpenDALResult SubmitPresignDeleteAsync(long context) /// Target path in the configured backend. /// Optional read options. /// A readable stream over the given path. - public OperatorInputStream OpenReadStream( + public unsafe OperatorInputStream OpenReadStream( string path, ReadOptions? options = null) { ObjectDisposedException.ThrowIf(IsInvalid, this); - using var nativeOptions = options?.BuildNativeOptionsHandle(); + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); var result = NativeMethods.operator_input_stream_create( this, path, - GetOptionsHandle(nativeOptions) + nativeOptions.Pointer ); var streamHandle = ToValueOrThrowAndRelease(result); @@ -1145,18 +1185,18 @@ public OperatorInputStream OpenReadStream( /// Optional write options. /// Buffer size used by the managed write stream. /// A writable stream over the given path. - public OperatorOutputStream OpenWriteStream( + public unsafe OperatorOutputStream OpenWriteStream( string path, WriteOptions? options = null, int bufferSize = OperatorOutputStream.DefaultBufferSize) { ObjectDisposedException.ThrowIf(IsInvalid, this); - using var nativeOptions = options?.BuildNativeOptionsHandle(); + using var nativeOptions = NativeOptionsScope.Create(options, stackalloc byte[NativeOptionsScope.StackSize]); var result = NativeMethods.operator_output_stream_create( this, path, - GetOptionsHandle(nativeOptions) + nativeOptions.Pointer ); var streamHandle = ToValueOrThrowAndRelease(result); @@ -1196,16 +1236,6 @@ internal Operator ApplyLayerResult(OpenDALOperatorResult result) return new Operator(newHandle); } - /// - /// Gets the native options pointer from an optional native options handle. - /// - /// Native options handle or . - /// Native options pointer, or when options are not provided. - private static IntPtr GetOptionsHandle(NativeOptionsHandle? options) - { - return options is null ? IntPtr.Zero : options.DangerousGetHandle(); - } - /// /// Retrieves operator info from the native layer. /// @@ -1218,25 +1248,6 @@ private OperatorInfo CreateOperatorInfo() return ToValueOrThrowAndRelease(result); } - /// - /// Builds constructor options for operator creation when key/value options are provided. - /// - /// Backend options dictionary. - /// Native options handle, or when options are empty. - private static NativeOptionsHandle? CreateConstructorOptionsHandle(IReadOnlyDictionary? options) - { - if (options is null || options.Count == 0) - { - return null; - } - - return NativeOptionsBuilder.BuildNativeOptionsHandle( - options, - NativeMethods.constructor_option_build, - NativeMethods.constructor_option_free - ); - } - /// /// Converts a native result into a managed value, throwing on native error and always releasing native resources. /// @@ -1287,59 +1298,6 @@ internal static void ThrowIfErrorAndRelease(TResult result) } } - /// - /// Submits a native async operation and binds it to a managed task completion source. - /// - /// Managed task result type. - /// Managed options type. - /// Optional managed options for this operation. - /// Submission delegate that invokes the native async API. - /// Cancellation token for managed task observation. - /// A task completed by the corresponding native callback. - /// is already canceled. - /// Native submission returns an immediate error. - internal static Task SubmitAsyncOperation( - TOptions? options, - Func submit, - CancellationToken cancellationToken) - where TOptions : class, IOptions - { - cancellationToken.ThrowIfCancellationRequested(); - var context = AsyncStateRegistry.Register(out var asyncState); - try - { - using var nativeOptionsHandle = options?.BuildNativeOptionsHandle(); - var submitResult = submit(context, GetOptionsHandle(nativeOptionsHandle)); - ThrowIfErrorAndRelease(submitResult); - asyncState.BindCancellation(cancellationToken); - return asyncState.Completion.Task; - } - catch - { - AsyncStateRegistry.Unregister(context); - throw; - } - } - - /// - /// Submits a native async operation and binds it to a managed task completion source. - /// - /// Managed options type. - /// Optional managed options for this operation. - /// Submission delegate that invokes the native async API. - /// Cancellation token for managed task observation. - /// A task completed by the corresponding native callback. - /// is already canceled. - /// Native submission returns an immediate error. - internal static Task SubmitAsyncOperation( - TOptions? options, - Func submit, - CancellationToken cancellationToken) - where TOptions : class, IOptions - { - return SubmitAsyncOperation(options, submit, cancellationToken); - } - /// /// Submits a native async operation without options and binds it to a managed task completion source. /// diff --git a/bindings/dotnet/OpenDAL/Options/Abstractions/IOptions.cs b/bindings/dotnet/OpenDAL/Options/Abstractions/IOptions.cs index 5fb29ca21702..ee320d1609e1 100644 --- a/bindings/dotnet/OpenDAL/Options/Abstractions/IOptions.cs +++ b/bindings/dotnet/OpenDAL/Options/Abstractions/IOptions.cs @@ -20,9 +20,25 @@ namespace OpenDAL.Options.Abstractions; /// -/// Represents operation options that can build native option payload handles. +/// Operation options that fill the native struct . /// -public interface IOptions +internal interface IOptions where TNative : unmanaged { - NativeOptionsHandle BuildNativeOptionsHandle(); -} \ No newline at end of file + /// + /// Validates the options before anything crosses the FFI. Options with + /// nothing to validate keep this empty default. + /// + void Validate() + { + } + + /// + /// Fills the native struct, writing everything it points to through + /// . + /// + /// + /// May run twice for one call, when the first buffer is too small, so it + /// must have no side effects beyond the writer. + /// + TNative ToNative(ref NativeOptionsWriter writer); +} diff --git a/bindings/dotnet/OpenDAL/Options/CopyOptions.cs b/bindings/dotnet/OpenDAL/Options/CopyOptions.cs new file mode 100644 index 000000000000..667f83af79d7 --- /dev/null +++ b/bindings/dotnet/OpenDAL/Options/CopyOptions.cs @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. + +using System.Runtime.InteropServices; +using OpenDAL.Options.Abstractions; + +namespace OpenDAL.Options; + +/// +/// Options for copy operations. +/// +public sealed class CopyOptions : IOptions +{ + /// + /// Copy only when no file exists at the destination path. + /// + /// + public bool IfNotExists { get; init; } + + /// + /// Copy only when the destination file has this exact ETag. + /// + /// + public string? IfMatch { get; init; } + + /// + /// Copy only when the destination file does not have this ETag. + /// + /// + public string? IfNoneMatch { get; init; } + + /// + /// Copy only when the destination file has this exact version. + /// + /// + public string? IfVersionMatch { get; init; } + + /// + /// Copy only when the destination file does not have this version. + /// + /// + public string? IfVersionNotMatch { get; init; } + + /// + /// Copy only when the destination file still has the identity recorded in this metadata. + /// + public Metadata? IfNotChanged { get; init; } + + /// + /// Copy from a specific source file version. + /// + /// + public string? SourceVersion { get; init; } + + /// + /// Asserted complete content length of the source object. + /// + public long? SourceContentLengthHint { get; init; } + + /// + /// Sets concurrent copy operations for this copier. + /// + public int Concurrent { get; init; } = 1; + + /// + /// Sets chunk size for segmented copy operations. + /// + /// + /// + /// + public long? Chunk { get; init; } + + void IOptions.Validate() + { + OptionValidators.RequireNullableGreaterThanOrEqualZero(SourceContentLengthHint, nameof(SourceContentLengthHint)); + OptionValidators.RequireGreaterThanZero(Concurrent, nameof(Concurrent)); + OptionValidators.RequireNullableGreaterThanZero(Chunk, nameof(Chunk)); + } + + OpenDALCopyOptions IOptions.ToNative(ref NativeOptionsWriter writer) + { + var native = new OpenDALCopyOptions(); + native.IfNotExists = IfNotExists ? (byte)1 : (byte)0; + native.IfMatch = writer.String(IfMatch); + native.IfNoneMatch = writer.String(IfNoneMatch); + native.IfVersionMatch = writer.String(IfVersionMatch); + native.IfVersionNotMatch = writer.String(IfVersionNotMatch); + native.IfNotChanged = writer.MetadataPointer(IfNotChanged); + native.SourceVersion = writer.String(SourceVersion); + if (SourceContentLengthHint is { } sourceContentLengthHint) + { + native.SourceContentLengthHintHasValue = 1; + native.SourceContentLengthHint = (ulong)sourceContentLengthHint; + } + native.Concurrent = (nuint)Concurrent; + if (Chunk is { } chunk) + { + native.ChunkHasValue = 1; + native.Chunk = (nuint)chunk; + } + return native; + } +} + +/// +/// Mirror of the native OpendalCopyOptions struct. Pointers refer +/// into the options buffer of a single call. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct OpenDALCopyOptions +{ + public byte IfNotExists; + public IntPtr IfMatch; + public IntPtr IfNoneMatch; + public IntPtr IfVersionMatch; + public IntPtr IfVersionNotMatch; + public IntPtr IfNotChanged; + public IntPtr SourceVersion; + public byte SourceContentLengthHintHasValue; + public ulong SourceContentLengthHint; + public nuint Concurrent; + public byte ChunkHasValue; + public nuint Chunk; +} diff --git a/bindings/dotnet/OpenDAL/Options/DeleteOptions.cs b/bindings/dotnet/OpenDAL/Options/DeleteOptions.cs index 73f3b43b7c0d..4b9d406299c5 100644 --- a/bindings/dotnet/OpenDAL/Options/DeleteOptions.cs +++ b/bindings/dotnet/OpenDAL/Options/DeleteOptions.cs @@ -17,6 +17,9 @@ * under the License. */ +// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. + +using System.Runtime.InteropServices; using OpenDAL.Options.Abstractions; namespace OpenDAL.Options; @@ -24,45 +27,74 @@ namespace OpenDAL.Options; /// /// Options for delete operations. /// -public sealed class DeleteOptions : IOptions +public sealed class DeleteOptions : IOptions { /// - /// The version of the file to delete. + /// Delete the given version of the file instead of the current one. /// + /// public string? Version { get; init; } /// /// Whether to delete the target recursively. /// - /// - /// If `false`, behaves like the traditional single-object delete. - /// If `true`, all entries under the path (or sharing the prefix for file-like paths) will be removed. - /// public bool Recursive { get; init; } /// - /// Sets the condition that the delete operation will succeed only if the - /// existing object's ETag matches the given value. + /// Delete only when the file at the delete path has this exact ETag. /// - /// - /// Check before using this feature. - /// If supported, the delete operation will only succeed when the existing - /// object's ETag matches the given value. - /// + /// public string? IfMatch { get; init; } - public NativeOptionsHandle BuildNativeOptionsHandle() - { - var nativeOptions = new NativeOptionsBuilder() - .AddString("version", Version) - .AddBoolTrue("recursive", Recursive) - .AddString("if_match", IfMatch) - .Build(); + /// + /// Delete only when the file at the delete path does not have this ETag. + /// + /// + public string? IfNoneMatch { get; init; } + + /// + /// Delete only when the file at the delete path has this exact version. + /// + /// + public string? IfVersionMatch { get; init; } + + /// + /// Delete only when the file at the delete path does not have this version. + /// + /// + public string? IfVersionNotMatch { get; init; } - return NativeOptionsBuilder.BuildNativeOptionsHandle( - nativeOptions, - NativeMethods.delete_option_build, - NativeMethods.delete_option_free - ); + /// + /// Delete only when the file at the delete path still has the identity recorded in this metadata. + /// + public Metadata? IfNotChanged { get; init; } + + OpenDALDeleteOptions IOptions.ToNative(ref NativeOptionsWriter writer) + { + var native = new OpenDALDeleteOptions(); + native.Version = writer.String(Version); + native.Recursive = Recursive ? (byte)1 : (byte)0; + native.IfMatch = writer.String(IfMatch); + native.IfNoneMatch = writer.String(IfNoneMatch); + native.IfVersionMatch = writer.String(IfVersionMatch); + native.IfVersionNotMatch = writer.String(IfVersionNotMatch); + native.IfNotChanged = writer.MetadataPointer(IfNotChanged); + return native; } -} \ No newline at end of file +} + +/// +/// Mirror of the native OpendalDeleteOptions struct. Pointers refer +/// into the options buffer of a single call. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct OpenDALDeleteOptions +{ + public IntPtr Version; + public byte Recursive; + public IntPtr IfMatch; + public IntPtr IfNoneMatch; + public IntPtr IfVersionMatch; + public IntPtr IfVersionNotMatch; + public IntPtr IfNotChanged; +} diff --git a/bindings/dotnet/OpenDAL/Options/ListOptions.cs b/bindings/dotnet/OpenDAL/Options/ListOptions.cs index 888c4572a708..1b7a9bc330df 100644 --- a/bindings/dotnet/OpenDAL/Options/ListOptions.cs +++ b/bindings/dotnet/OpenDAL/Options/ListOptions.cs @@ -17,41 +17,75 @@ * under the License. */ +// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. + +using System.Runtime.InteropServices; using OpenDAL.Options.Abstractions; namespace OpenDAL.Options; /// -/// Additional options for list operations. +/// Options for list operations. /// -public sealed class ListOptions : IOptions +public sealed class ListOptions : IOptions { - public bool Recursive { get; init; } - + /// + /// Maximum results per request (backend hint) to control memory and throttling. + /// public long? Limit { get; init; } + /// + /// The start_after passes to underlying service to specify the specified key to start listing from. + /// public string? StartAfter { get; init; } + /// + /// Whether to list recursively under the prefix; default . + /// + public bool Recursive { get; init; } + + /// + /// Include object versions when supported by the backend; default . + /// public bool Versions { get; init; } + /// + /// Include delete markers when supported by version-aware backends; default . + /// public bool Deleted { get; init; } - public NativeOptionsHandle BuildNativeOptionsHandle() + void IOptions.Validate() { OptionValidators.RequireNullableGreaterThanZero(Limit, nameof(Limit)); + } - var nativeOptions = new NativeOptionsBuilder() - .AddBoolTrue("recursive", Recursive) - .AddNullableInt64("limit", Limit) - .AddString("start_after", StartAfter) - .AddBoolTrue("versions", Versions) - .AddBoolTrue("deleted", Deleted) - .Build(); - - return NativeOptionsBuilder.BuildNativeOptionsHandle( - nativeOptions, - NativeMethods.list_option_build, - NativeMethods.list_option_free - ); + OpenDALListOptions IOptions.ToNative(ref NativeOptionsWriter writer) + { + var native = new OpenDALListOptions(); + if (Limit is { } limit) + { + native.LimitHasValue = 1; + native.Limit = (nuint)limit; + } + native.StartAfter = writer.String(StartAfter); + native.Recursive = Recursive ? (byte)1 : (byte)0; + native.Versions = Versions ? (byte)1 : (byte)0; + native.Deleted = Deleted ? (byte)1 : (byte)0; + return native; } -} \ No newline at end of file +} + +/// +/// Mirror of the native OpendalListOptions struct. Pointers refer +/// into the options buffer of a single call. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct OpenDALListOptions +{ + public byte LimitHasValue; + public nuint Limit; + public IntPtr StartAfter; + public byte Recursive; + public byte Versions; + public byte Deleted; +} diff --git a/bindings/dotnet/OpenDAL/Options/NativeOptionsBuilder.cs b/bindings/dotnet/OpenDAL/Options/NativeOptionsBuilder.cs deleted file mode 100644 index 15f98443f54e..000000000000 --- a/bindings/dotnet/OpenDAL/Options/NativeOptionsBuilder.cs +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -using OpenDAL.Interop.Result; - -namespace OpenDAL.Options; - -/// -/// Delegate that builds a native options payload from managed key/value arrays. -/// -/// Option keys. -/// Option values aligned by index with . -/// Number of key/value pairs. -/// Native options build result. -internal delegate OpenDALOptionsResult NativeBuildOptionsDelegate(string[] keys, string[] values, nuint len); - -/// -/// Builder that incrementally collects native key/value options. -/// -internal sealed class NativeOptionsBuilder -{ - private readonly Dictionary options = new(); - - /// - /// Adds the option with value "true" when is true. - /// - public NativeOptionsBuilder AddBoolTrue(string key, bool value) - { - if (value) - { - options[key] = "true"; - } - - return this; - } - - /// - /// Adds the option when the string value is not null or empty. - /// - public NativeOptionsBuilder AddString(string key, string? value) - { - if (!string.IsNullOrEmpty(value)) - { - options[key] = value; - } - - return this; - } - - /// - /// Adds the option when the int value differs from the specified default. - /// - public NativeOptionsBuilder AddInt32IfNotDefault(string key, int value, int defaultValue) - { - if (value != defaultValue) - { - options[key] = value.ToString(System.Globalization.CultureInfo.InvariantCulture); - } - - return this; - } - - /// - /// Adds the option when the long value differs from the specified default. - /// - public NativeOptionsBuilder AddInt64IfNotDefault(string key, long value, long defaultValue) - { - if (value != defaultValue) - { - options[key] = value.ToString(System.Globalization.CultureInfo.InvariantCulture); - } - - return this; - } - - /// - /// Adds the option when the nullable long value is provided. - /// - public NativeOptionsBuilder AddNullableInt64(string key, long? value) - { - if (value is not null) - { - options[key] = value.Value.ToString(System.Globalization.CultureInfo.InvariantCulture); - } - - return this; - } - - /// - /// Adds the option as Unix time milliseconds when the timestamp is provided. - /// - public NativeOptionsBuilder AddUnixTimeMilliseconds(string key, DateTimeOffset? value) - { - if (value is not null) - { - options[key] = value.Value.ToUnixTimeMilliseconds().ToString(System.Globalization.CultureInfo.InvariantCulture); - } - - return this; - } - - /// - /// Adds prefixed entries from the provided dictionary. - /// - public NativeOptionsBuilder AddPrefixedEntries(string prefix, IReadOnlyDictionary? values) - { - if (values is null) - { - return this; - } - - foreach (var entry in values) - { - options[$"{prefix}{entry.Key}"] = entry.Value; - } - - return this; - } - - /// - /// Builds the final native options dictionary. - /// - public IReadOnlyDictionary Build() - { - return options; - } - - /// - /// Builds a native options handle from managed key/value options. - /// - /// Managed options dictionary. - /// Native build function used to allocate and populate options payload. - /// Native release function used by the resulting handle. - /// A safe handle that owns the native options payload. - /// is null. - /// Native options build fails. - public static NativeOptionsHandle BuildNativeOptionsHandle( - IReadOnlyDictionary options, - NativeBuildOptionsDelegate build, - Action release) - { - ArgumentNullException.ThrowIfNull(options); - - var keys = new string[options.Count]; - var values = new string[options.Count]; - var index = 0; - - foreach (var option in options) - { - keys[index] = option.Key; - values[index] = option.Value; - index++; - } - - var result = build(keys, values, (nuint)options.Count); - var handle = Operator.ToValueOrThrowAndRelease(result); - return new NativeOptionsHandle(handle, release); - } -} \ No newline at end of file diff --git a/bindings/dotnet/OpenDAL/Options/NativeOptionsHandle.cs b/bindings/dotnet/OpenDAL/Options/NativeOptionsHandle.cs deleted file mode 100644 index afe364e7d4f3..000000000000 --- a/bindings/dotnet/OpenDAL/Options/NativeOptionsHandle.cs +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -using System.Runtime.InteropServices; - -namespace OpenDAL.Options; - -public sealed class NativeOptionsHandle : SafeHandle -{ - private readonly Action release; - - public NativeOptionsHandle(IntPtr ptr, Action release) - : base(IntPtr.Zero, true) - { - this.release = release; - SetHandle(ptr); - } - - public override bool IsInvalid => handle == IntPtr.Zero; - - protected override bool ReleaseHandle() - { - if (IsInvalid) - { - return true; - } - - release(handle); - handle = IntPtr.Zero; - return true; - } -} diff --git a/bindings/dotnet/OpenDAL/Options/NativeOptionsWriter.cs b/bindings/dotnet/OpenDAL/Options/NativeOptionsWriter.cs new file mode 100644 index 000000000000..bc6d97832027 --- /dev/null +++ b/bindings/dotnet/OpenDAL/Options/NativeOptionsWriter.cs @@ -0,0 +1,279 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; +using OpenDAL.Interop.NativeObject; +using OpenDAL.Options.Abstractions; + +namespace OpenDAL.Options; + +/// +/// Writes options into one buffer that native code borrows for a single +/// call: the native struct, then every string, string-pair array, and +/// nested metadata it points to. +/// +/// +/// Pointers into the buffer are written as they go, so the buffer cannot +/// grow. Once a write does not fit, the writer stops writing but keeps +/// counting, so tells the caller how large a +/// buffer to write the same options into again. +/// +internal unsafe ref struct NativeOptionsWriter +{ + private const int Alignment = 8; + + private readonly byte* start; + private readonly int capacity; + private int required; + + /// + /// Creates a writer over bytes at + /// , which may be null to only count. + /// + internal NativeOptionsWriter(byte* buffer, int length) + { + // Align the start, so aligning offsets also aligns addresses. + start = (byte*)(((nuint)buffer + Alignment - 1) & ~(nuint)(Alignment - 1)); + capacity = buffer is null ? 0 : Math.Max(0, length - (int)(start - buffer)); + required = 0; + Overflowed = false; + } + + /// + /// Whether a write did not fit, which leaves the struct incomplete. + /// + public bool Overflowed { get; private set; } + + /// + /// Bytes of a buffer that fits everything written so far, including the + /// padding needed to align its start. + /// + public int RequiredCapacity => required + Alignment - 1; + + /// + /// Reserves an aligned, uninitialized , or returns + /// null when it does not fit. + /// + public T* Reserve() where T : unmanaged => (T*)Allocate(sizeof(T), Alignment); + + /// + /// Writes an option string, where null and empty both mean unset. + /// + public IntPtr String(string? value) => string.IsNullOrEmpty(value) ? IntPtr.Zero : Utf8(value); + + /// + /// Writes a null-terminated UTF-8 string, keeping an empty one. + /// + public IntPtr Utf8(string? value) + { + if (value is null) + { + return IntPtr.Zero; + } + + var length = Encoding.UTF8.GetByteCount(value); + var target = Allocate(checked(length + 1), 1); + if (target is null) + { + return IntPtr.Zero; + } + + Encoding.UTF8.GetBytes(value, new Span(target, length)); + target[length] = 0; + return (IntPtr)target; + } + + /// + /// Writes string pairs as two parallel arrays of string pointers. A null + /// value stays a null pointer, which native code reports by key. + /// + public (byte HasValue, IntPtr Keys, IntPtr Values, nuint Len) Pairs(IReadOnlyDictionary? pairs) + { + if (pairs is null) + { + return default; + } + + var count = pairs.Count; + var keys = (IntPtr*)Allocate(checked(count * IntPtr.Size), Alignment); + var values = (IntPtr*)Allocate(checked(count * IntPtr.Size), Alignment); + var index = 0; + foreach (var entry in pairs) + { + if (index == count) + { + throw Changed(); + } + + // Count the strings even when the arrays did not fit. + var key = Utf8(entry.Key); + var value = Utf8(entry.Value); + if (keys is not null && values is not null) + { + keys[index] = key; + values[index] = value; + } + + index++; + } + + if (index != count) + { + throw Changed(); + } + + return (1, (IntPtr)keys, (IntPtr)values, (nuint)count); + } + + /// + /// Writes metadata for native code to rebuild, or returns null for none. + /// + public IntPtr MetadataPointer(Metadata? metadata) + { + if (metadata is null) + { + return IntPtr.Zero; + } + + var target = Reserve(); + var native = OpenDALMetadata.From(metadata, ref this); + if (target is null) + { + return IntPtr.Zero; + } + + *target = native; + return (IntPtr)target; + } + + /// + /// The exception thrown when options change between counting and + /// writing, for example a dictionary modified by another thread. + /// + internal static InvalidOperationException Changed() => + new("Options changed while they were being passed to native code."); + + private byte* Allocate(int size, int alignment) + { + var offset = (required + alignment - 1) & ~(alignment - 1); + required = checked(offset + size); + if (Overflowed || required > capacity) + { + Overflowed = true; + return null; + } + + return start + offset; + } +} + +/// +/// The options buffer of one native call. Disposing it frees a buffer that +/// did not fit on the caller's stack. +/// +internal readonly unsafe struct NativeOptionsScope : IDisposable where TNative : unmanaged +{ + private readonly void* heap; + + internal NativeOptionsScope(TNative* pointer, void* heap) + { + Pointer = pointer; + this.heap = heap; + } + + /// + /// The options for native code, or null for the defaults. + /// + public TNative* Pointer { get; } + + public void Dispose() + { + if (heap is not null) + { + NativeMemory.Free(heap); + } + } +} + +/// +/// Creates values. +/// +internal static unsafe class NativeOptionsScope +{ + /// + /// Size of the stack buffer callers pass to . Options + /// that need more go to native memory instead. + /// + public const int StackSize = 512; + + /// + /// Validates and writes them for one native + /// call: into in a single pass when they fit, + /// otherwise once more into native memory of the size the first pass + /// counted. + /// + /// + /// must be stack memory of the caller, which + /// must dispose the scope before returning and must not use + /// after that. + /// + public static NativeOptionsScope Create(IOptions? options, Span stack) + where TNative : unmanaged + { + if (options is null) + { + return default; + } + + options.Validate(); + var writer = new NativeOptionsWriter( + (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(stack)), + stack.Length); + var native = writer.Reserve(); + var value = options.ToNative(ref writer); + if (!writer.Overflowed) + { + *native = value; + return new NativeOptionsScope(native, null); + } + + var size = writer.RequiredCapacity; + var heap = NativeMemory.Alloc((nuint)size); + try + { + writer = new NativeOptionsWriter((byte*)heap, size); + native = writer.Reserve(); + value = options.ToNative(ref writer); + if (writer.Overflowed) + { + throw NativeOptionsWriter.Changed(); + } + + *native = value; + return new NativeOptionsScope(native, heap); + } + catch + { + NativeMemory.Free(heap); + throw; + } + } +} diff --git a/bindings/dotnet/OpenDAL/Options/OptionValidators.cs b/bindings/dotnet/OpenDAL/Options/OptionValidators.cs index c1bfe7404419..90b22b0cab80 100644 --- a/bindings/dotnet/OpenDAL/Options/OptionValidators.cs +++ b/bindings/dotnet/OpenDAL/Options/OptionValidators.cs @@ -67,4 +67,17 @@ public static void RequireNullableGreaterThanZero(long? value, string paramName) throw new ArgumentOutOfRangeException(paramName, $"{paramName} must be > 0 when provided."); } } + + /// + /// Ensures a suffix range is not combined with an offset or a length. + /// + public static void RequireSuffixExclusive(long offset, long? length, long? suffixLength) + { + if (suffixLength is not null && (offset != 0 || length is not null)) + { + throw new ArgumentException( + "SuffixLength cannot be combined with Offset or Length.", + "SuffixLength"); + } + } } \ No newline at end of file diff --git a/bindings/dotnet/OpenDAL/Options/ReadOptions.cs b/bindings/dotnet/OpenDAL/Options/ReadOptions.cs index f90195c7f268..b4d25d458a63 100644 --- a/bindings/dotnet/OpenDAL/Options/ReadOptions.cs +++ b/bindings/dotnet/OpenDAL/Options/ReadOptions.cs @@ -17,69 +17,206 @@ * under the License. */ +// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. + +using System.Runtime.InteropServices; using OpenDAL.Options.Abstractions; namespace OpenDAL.Options; /// -/// Additional options for read operations. +/// Options for read operations. /// -public sealed class ReadOptions : IOptions +public sealed class ReadOptions : IOptions { + /// + /// The byte offset to start reading from. + /// public long Offset { get; init; } + /// + /// The number of bytes to read from , or to read to the end. + /// public long? Length { get; init; } + /// + /// Read the last bytes instead of a range from . It cannot be combined with or . + /// + /// + public long? SuffixLength { get; init; } + + /// + /// Read the given version of the file instead of the current one. + /// + /// public string? Version { get; init; } + /// + /// Read only when the file has this exact ETag. + /// + /// public string? IfMatch { get; init; } + /// + /// Read only when the file does not have this ETag. + /// + /// public string? IfNoneMatch { get; init; } + /// + /// Read only when the file has this exact version. + /// + /// + public string? IfVersionMatch { get; init; } + + /// + /// Read only when the file does not have this version. + /// + /// + public string? IfVersionNotMatch { get; init; } + + /// + /// Read only when the file was modified after this timestamp. + /// + /// public DateTimeOffset? IfModifiedSince { get; init; } + /// + /// Read only when the file was not modified after this timestamp. + /// + /// public DateTimeOffset? IfUnmodifiedSince { get; init; } + /// + /// Known content length of the object. + /// + public long? ContentLengthHint { get; init; } + + /// + /// Set for the operation. + /// public int Concurrent { get; init; } = 1; + /// + /// Set for the operation. + /// public long? Chunk { get; init; } + /// + /// Controls the optimization strategy for range reads in Reader::fetch. + /// public long? Gap { get; init; } + /// + /// Specify the content-type header that should be sent back by the operation. + /// public string? OverrideContentType { get; init; } + /// + /// Specify the cache-control header that should be sent back by the operation. + /// public string? OverrideCacheControl { get; init; } + /// + /// Specify the content-disposition header that should be sent back by the operation. + /// public string? OverrideContentDisposition { get; init; } - public NativeOptionsHandle BuildNativeOptionsHandle() + void IOptions.Validate() { OptionValidators.RequireGreaterThanOrEqualZero(Offset, nameof(Offset)); OptionValidators.RequireNullableGreaterThanOrEqualZero(Length, nameof(Length)); + OptionValidators.RequireNullableGreaterThanOrEqualZero(SuffixLength, nameof(SuffixLength)); + OptionValidators.RequireSuffixExclusive(Offset, Length, SuffixLength); + OptionValidators.RequireNullableGreaterThanOrEqualZero(ContentLengthHint, nameof(ContentLengthHint)); OptionValidators.RequireGreaterThanZero(Concurrent, nameof(Concurrent)); OptionValidators.RequireNullableGreaterThanZero(Chunk, nameof(Chunk)); OptionValidators.RequireNullableGreaterThanZero(Gap, nameof(Gap)); + } - var nativeOptions = new NativeOptionsBuilder() - .AddInt64IfNotDefault("offset", Offset, 0) - .AddNullableInt64("length", Length) - .AddString("version", Version) - .AddString("if_match", IfMatch) - .AddString("if_none_match", IfNoneMatch) - .AddUnixTimeMilliseconds("if_modified_since", IfModifiedSince) - .AddUnixTimeMilliseconds("if_unmodified_since", IfUnmodifiedSince) - .AddInt32IfNotDefault("concurrent", Concurrent, 1) - .AddNullableInt64("chunk", Chunk) - .AddNullableInt64("gap", Gap) - .AddString("override_content_type", OverrideContentType) - .AddString("override_cache_control", OverrideCacheControl) - .AddString("override_content_disposition", OverrideContentDisposition) - .Build(); - - return NativeOptionsBuilder.BuildNativeOptionsHandle( - nativeOptions, - NativeMethods.read_option_build, - NativeMethods.read_option_free - ); + OpenDALReadOptions IOptions.ToNative(ref NativeOptionsWriter writer) + { + var native = new OpenDALReadOptions(); + native.Offset = (ulong)Offset; + if (Length is { } length) + { + native.LengthHasValue = 1; + native.Length = (ulong)length; + } + if (SuffixLength is { } suffixLength) + { + native.SuffixLengthHasValue = 1; + native.SuffixLength = (ulong)suffixLength; + } + native.Version = writer.String(Version); + native.IfMatch = writer.String(IfMatch); + native.IfNoneMatch = writer.String(IfNoneMatch); + native.IfVersionMatch = writer.String(IfVersionMatch); + native.IfVersionNotMatch = writer.String(IfVersionNotMatch); + if (IfModifiedSince is { } ifModifiedSince) + { + native.IfModifiedSinceHasValue = 1; + (native.IfModifiedSinceSecond, native.IfModifiedSinceNanosecond) = Utilities.ToUnixSecondAndNanosecond(ifModifiedSince); + } + if (IfUnmodifiedSince is { } ifUnmodifiedSince) + { + native.IfUnmodifiedSinceHasValue = 1; + (native.IfUnmodifiedSinceSecond, native.IfUnmodifiedSinceNanosecond) = Utilities.ToUnixSecondAndNanosecond(ifUnmodifiedSince); + } + if (ContentLengthHint is { } contentLengthHint) + { + native.ContentLengthHintHasValue = 1; + native.ContentLengthHint = (ulong)contentLengthHint; + } + native.Concurrent = (nuint)Concurrent; + if (Chunk is { } chunk) + { + native.ChunkHasValue = 1; + native.Chunk = (nuint)chunk; + } + if (Gap is { } gap) + { + native.GapHasValue = 1; + native.Gap = (nuint)gap; + } + native.OverrideContentType = writer.String(OverrideContentType); + native.OverrideCacheControl = writer.String(OverrideCacheControl); + native.OverrideContentDisposition = writer.String(OverrideContentDisposition); + return native; } -} \ No newline at end of file +} + +/// +/// Mirror of the native OpendalReadOptions struct. Pointers refer +/// into the options buffer of a single call. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct OpenDALReadOptions +{ + public ulong Offset; + public byte LengthHasValue; + public ulong Length; + public byte SuffixLengthHasValue; + public ulong SuffixLength; + public IntPtr Version; + public IntPtr IfMatch; + public IntPtr IfNoneMatch; + public IntPtr IfVersionMatch; + public IntPtr IfVersionNotMatch; + public byte IfModifiedSinceHasValue; + public long IfModifiedSinceSecond; + public int IfModifiedSinceNanosecond; + public byte IfUnmodifiedSinceHasValue; + public long IfUnmodifiedSinceSecond; + public int IfUnmodifiedSinceNanosecond; + public byte ContentLengthHintHasValue; + public ulong ContentLengthHint; + public nuint Concurrent; + public byte ChunkHasValue; + public nuint Chunk; + public byte GapHasValue; + public nuint Gap; + public IntPtr OverrideContentType; + public IntPtr OverrideCacheControl; + public IntPtr OverrideContentDisposition; +} diff --git a/bindings/dotnet/OpenDAL/Interop/Result/OpenDALOptionsResult.cs b/bindings/dotnet/OpenDAL/Options/RenameOptions.cs similarity index 50% rename from bindings/dotnet/OpenDAL/Interop/Result/OpenDALOptionsResult.cs rename to bindings/dotnet/OpenDAL/Options/RenameOptions.cs index 03d790c2d64f..d40f01097c36 100644 --- a/bindings/dotnet/OpenDAL/Interop/Result/OpenDALOptionsResult.cs +++ b/bindings/dotnet/OpenDAL/Options/RenameOptions.cs @@ -17,34 +17,38 @@ * under the License. */ +// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. + using System.Runtime.InteropServices; -using OpenDAL.Interop.Result.Abstractions; +using OpenDAL.Options.Abstractions; -namespace OpenDAL.Interop.Result; +namespace OpenDAL.Options; -[StructLayout(LayoutKind.Sequential)] /// -/// Result wrapper for operations that return an options pointer payload. +/// Options for rename operations. /// -internal struct OpenDALOptionsResult - : INativeValueResult +public sealed class RenameOptions : IOptions { - public IntPtr Ptr; + /// + /// Rename only when no file exists at the destination path. + /// + /// + public bool IfNotExists { get; init; } - public OpenDALError Error; - - public readonly void Release() + OpenDALRenameOptions IOptions.ToNative(ref NativeOptionsWriter writer) { - NativeMethods.opendal_error_release(Error); - } - - public readonly OpenDALError GetError() - { - return Error; + var native = new OpenDALRenameOptions(); + native.IfNotExists = IfNotExists ? (byte)1 : (byte)0; + return native; } +} - public readonly IntPtr ToValue() - { - return Ptr; - } +/// +/// Mirror of the native OpendalRenameOptions struct. Pointers refer +/// into the options buffer of a single call. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct OpenDALRenameOptions +{ + public byte IfNotExists; } diff --git a/bindings/dotnet/OpenDAL/Options/StatOptions.cs b/bindings/dotnet/OpenDAL/Options/StatOptions.cs index e678e8ab61ea..90b255af480b 100644 --- a/bindings/dotnet/OpenDAL/Options/StatOptions.cs +++ b/bindings/dotnet/OpenDAL/Options/StatOptions.cs @@ -17,48 +17,119 @@ * under the License. */ +// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. + +using System.Runtime.InteropServices; using OpenDAL.Options.Abstractions; namespace OpenDAL.Options; /// -/// Additional options for stat operations. +/// Options for stat operations. /// -public sealed class StatOptions : IOptions +public sealed class StatOptions : IOptions { + /// + /// Stat the given version of the file instead of the current one. + /// + /// public string? Version { get; init; } + /// + /// Stat only when the file has this exact ETag. + /// + /// public string? IfMatch { get; init; } + /// + /// Stat only when the file does not have this ETag. + /// + /// public string? IfNoneMatch { get; init; } + /// + /// Stat only when the file has this exact version. + /// + /// + public string? IfVersionMatch { get; init; } + + /// + /// Stat only when the file does not have this version. + /// + /// + public string? IfVersionNotMatch { get; init; } + + /// + /// Stat only when the file was modified after this timestamp. + /// + /// public DateTimeOffset? IfModifiedSince { get; init; } + /// + /// Stat only when the file was not modified after this timestamp. + /// + /// public DateTimeOffset? IfUnmodifiedSince { get; init; } + /// + /// Specify the content-type header that should be sent back by the operation. + /// public string? OverrideContentType { get; init; } + /// + /// Specify the cache-control header that should be sent back by the operation. + /// public string? OverrideCacheControl { get; init; } + /// + /// Specify the content-disposition header that should be sent back by the operation. + /// public string? OverrideContentDisposition { get; init; } - public NativeOptionsHandle BuildNativeOptionsHandle() + OpenDALStatOptions IOptions.ToNative(ref NativeOptionsWriter writer) { - var nativeOptions = new NativeOptionsBuilder() - .AddString("version", Version) - .AddString("if_match", IfMatch) - .AddString("if_none_match", IfNoneMatch) - .AddUnixTimeMilliseconds("if_modified_since", IfModifiedSince) - .AddUnixTimeMilliseconds("if_unmodified_since", IfUnmodifiedSince) - .AddString("override_content_type", OverrideContentType) - .AddString("override_cache_control", OverrideCacheControl) - .AddString("override_content_disposition", OverrideContentDisposition) - .Build(); - - return NativeOptionsBuilder.BuildNativeOptionsHandle( - nativeOptions, - NativeMethods.stat_option_build, - NativeMethods.stat_option_free - ); + var native = new OpenDALStatOptions(); + native.Version = writer.String(Version); + native.IfMatch = writer.String(IfMatch); + native.IfNoneMatch = writer.String(IfNoneMatch); + native.IfVersionMatch = writer.String(IfVersionMatch); + native.IfVersionNotMatch = writer.String(IfVersionNotMatch); + if (IfModifiedSince is { } ifModifiedSince) + { + native.IfModifiedSinceHasValue = 1; + (native.IfModifiedSinceSecond, native.IfModifiedSinceNanosecond) = Utilities.ToUnixSecondAndNanosecond(ifModifiedSince); + } + if (IfUnmodifiedSince is { } ifUnmodifiedSince) + { + native.IfUnmodifiedSinceHasValue = 1; + (native.IfUnmodifiedSinceSecond, native.IfUnmodifiedSinceNanosecond) = Utilities.ToUnixSecondAndNanosecond(ifUnmodifiedSince); + } + native.OverrideContentType = writer.String(OverrideContentType); + native.OverrideCacheControl = writer.String(OverrideCacheControl); + native.OverrideContentDisposition = writer.String(OverrideContentDisposition); + return native; } -} \ No newline at end of file +} + +/// +/// Mirror of the native OpendalStatOptions struct. Pointers refer +/// into the options buffer of a single call. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct OpenDALStatOptions +{ + public IntPtr Version; + public IntPtr IfMatch; + public IntPtr IfNoneMatch; + public IntPtr IfVersionMatch; + public IntPtr IfVersionNotMatch; + public byte IfModifiedSinceHasValue; + public long IfModifiedSinceSecond; + public int IfModifiedSinceNanosecond; + public byte IfUnmodifiedSinceHasValue; + public long IfUnmodifiedSinceSecond; + public int IfUnmodifiedSinceNanosecond; + public IntPtr OverrideContentType; + public IntPtr OverrideCacheControl; + public IntPtr OverrideContentDisposition; +} diff --git a/bindings/dotnet/OpenDAL/Options/WriteOptions.cs b/bindings/dotnet/OpenDAL/Options/WriteOptions.cs index 3f3523b3cd5b..dfa7790f103e 100644 --- a/bindings/dotnet/OpenDAL/Options/WriteOptions.cs +++ b/bindings/dotnet/OpenDAL/Options/WriteOptions.cs @@ -17,60 +17,155 @@ * under the License. */ +// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. + +using System.Runtime.InteropServices; using OpenDAL.Options.Abstractions; namespace OpenDAL.Options; /// -/// Additional options for write operations. +/// Options for write operations. /// -public sealed class WriteOptions : IOptions +public sealed class WriteOptions : IOptions { + /// + /// Sets append mode for this operation. + /// + /// public bool Append { get; init; } + /// + /// Sets Cache-Control header for this write operation. + /// + /// public string? CacheControl { get; init; } + /// + /// Sets Content-Type header for this write operation. + /// + /// public string? ContentType { get; init; } + /// + /// Sets Content-Disposition header for this write request. + /// + /// public string? ContentDisposition { get; init; } + /// + /// Sets Content-Encoding header for this write request. + /// + /// public string? ContentEncoding { get; init; } + /// + /// Sets user metadata for this write request. + /// + /// + public IReadOnlyDictionary? UserMetadata { get; init; } + + /// + /// Write only when the file at the write path has this exact ETag. + /// + /// public string? IfMatch { get; init; } + /// + /// Write only when the file at the write path does not have this ETag. + /// + /// public string? IfNoneMatch { get; init; } + /// + /// Write only when the file at the write path has this exact version. + /// + /// + public string? IfVersionMatch { get; init; } + + /// + /// Write only when the file at the write path does not have this version. + /// + /// + public string? IfVersionNotMatch { get; init; } + + /// + /// Write only when no file exists at the write path. + /// + /// public bool IfNotExists { get; init; } + /// + /// Write only when the file at the write path still has the identity recorded in this metadata. + /// + public Metadata? IfNotChanged { get; init; } + + /// + /// Sets concurrent write operations for this writer. + /// public int Concurrent { get; init; } = 1; + /// + /// Sets chunk size for buffered writes. + /// + /// + /// public long? Chunk { get; init; } - public IReadOnlyDictionary? UserMetadata { get; init; } - - public NativeOptionsHandle BuildNativeOptionsHandle() + void IOptions.Validate() { OptionValidators.RequireGreaterThanZero(Concurrent, nameof(Concurrent)); OptionValidators.RequireNullableGreaterThanZero(Chunk, nameof(Chunk)); + } - var nativeOptions = new NativeOptionsBuilder() - .AddBoolTrue("append", Append) - .AddString("cache_control", CacheControl) - .AddString("content_type", ContentType) - .AddString("content_disposition", ContentDisposition) - .AddString("content_encoding", ContentEncoding) - .AddString("if_match", IfMatch) - .AddString("if_none_match", IfNoneMatch) - .AddBoolTrue("if_not_exists", IfNotExists) - .AddInt32IfNotDefault("concurrent", Concurrent, 1) - .AddNullableInt64("chunk", Chunk) - .AddPrefixedEntries("user_metadata.", UserMetadata) - .Build(); - - return NativeOptionsBuilder.BuildNativeOptionsHandle( - nativeOptions, - NativeMethods.write_option_build, - NativeMethods.write_option_free - ); + OpenDALWriteOptions IOptions.ToNative(ref NativeOptionsWriter writer) + { + var native = new OpenDALWriteOptions(); + native.Append = Append ? (byte)1 : (byte)0; + native.CacheControl = writer.String(CacheControl); + native.ContentType = writer.String(ContentType); + native.ContentDisposition = writer.String(ContentDisposition); + native.ContentEncoding = writer.String(ContentEncoding); + (native.UserMetadataHasValue, native.UserMetadataKeys, native.UserMetadataValues, native.UserMetadataLen) = writer.Pairs(UserMetadata); + native.IfMatch = writer.String(IfMatch); + native.IfNoneMatch = writer.String(IfNoneMatch); + native.IfVersionMatch = writer.String(IfVersionMatch); + native.IfVersionNotMatch = writer.String(IfVersionNotMatch); + native.IfNotExists = IfNotExists ? (byte)1 : (byte)0; + native.IfNotChanged = writer.MetadataPointer(IfNotChanged); + native.Concurrent = (nuint)Concurrent; + if (Chunk is { } chunk) + { + native.ChunkHasValue = 1; + native.Chunk = (nuint)chunk; + } + return native; } -} \ No newline at end of file +} + +/// +/// Mirror of the native OpendalWriteOptions struct. Pointers refer +/// into the options buffer of a single call. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct OpenDALWriteOptions +{ + public byte Append; + public IntPtr CacheControl; + public IntPtr ContentType; + public IntPtr ContentDisposition; + public IntPtr ContentEncoding; + public byte UserMetadataHasValue; + public IntPtr UserMetadataKeys; + public IntPtr UserMetadataValues; + public nuint UserMetadataLen; + public IntPtr IfMatch; + public IntPtr IfNoneMatch; + public IntPtr IfVersionMatch; + public IntPtr IfVersionNotMatch; + public byte IfNotExists; + public IntPtr IfNotChanged; + public nuint Concurrent; + public byte ChunkHasValue; + public nuint Chunk; +} diff --git a/bindings/dotnet/OpenDAL/Utilities.cs b/bindings/dotnet/OpenDAL/Utilities.cs index fc0dbb623ff9..a3c7058a4d75 100644 --- a/bindings/dotnet/OpenDAL/Utilities.cs +++ b/bindings/dotnet/OpenDAL/Utilities.cs @@ -80,6 +80,17 @@ internal static DateTimeOffset ToDateTimeOffset(long second, int nanosecond) return DateTimeOffset.FromUnixTimeSeconds(second).AddTicks(nanosecond / NanosecondsPerTick); } + /// + /// Splits a timestamp into whole Unix seconds and the nanoseconds within + /// that second, the inverse of . + /// + internal static (long Second, int Nanosecond) ToUnixSecondAndNanosecond(DateTimeOffset value) + { + var second = value.ToUnixTimeSeconds(); + var nanosecond = (int)(value.UtcTicks % TimeSpan.TicksPerSecond * NanosecondsPerTick); + return (second, nanosecond); + } + /// /// Reads two parallel native arrays of UTF-8 C strings into a dictionary. /// diff --git a/bindings/dotnet/src/metadata.rs b/bindings/dotnet/src/metadata.rs index 97f23ee0ed25..bb04b8c98d46 100644 --- a/bindings/dotnet/src/metadata.rs +++ b/bindings/dotnet/src/metadata.rs @@ -19,6 +19,11 @@ use std::ffi::c_char; +use opendal::{EntryMode, MetadataBuilder}; + +use crate::error::OpenDALError; +use crate::utils::entry_mode_code; + /// FFI representation of `opendal::Metadata`. /// /// Fields follow the accessor order of `opendal::Metadata`. Optional values @@ -127,3 +132,78 @@ pub(crate) unsafe fn metadata_release_fields(metadata: &mut OpendalMetadata) { ); } } + +impl OpendalMetadata { + /// Rebuild the metadata through `MetadataBuilder` from fields the caller + /// owns, as options such as `if_not_changed` pass them. + /// + /// Every pointer is only borrowed: strings are copied, and nothing is + /// released. + pub(crate) fn to_metadata(&self) -> Result { + // `MetadataBuilder` takes the content length only through `file`. + let mut builder = match self.mode { + code if code == entry_mode_code(EntryMode::FILE) => { + MetadataBuilder::file(self.content_length) + } + code if code == entry_mode_code(EntryMode::DIR) => MetadataBuilder::dir(), + _ => MetadataBuilder::unknown(), + }; + builder.is_current((self.is_current_has_value != 0).then_some(self.is_current != 0)); + builder.is_deleted(self.is_deleted != 0); + if let Some(value) = crate::utils::borrowed_string(self.cache_control, "cache_control")? { + builder.cache_control(value); + } + if let Some(value) = crate::utils::borrowed_string(self.content_md5, "content_md5")? { + builder.content_md5(value); + } + if let Some(value) = crate::utils::borrowed_string(self.content_type, "content_type")? { + builder.content_type(value); + } + if let Some(value) = + crate::utils::borrowed_string(self.content_encoding, "content_encoding")? + { + builder.content_encoding(value); + } + if let Some(value) = crate::utils::borrowed_timestamp( + self.last_modified_has_value, + self.last_modified_second, + self.last_modified_nanosecond, + )? { + builder.last_modified(value); + } + if let Some(value) = crate::utils::borrowed_string(self.etag, "etag")? { + builder.etag(value); + } + if let Some(value) = + crate::utils::borrowed_string(self.content_disposition, "content_disposition")? + { + builder.content_disposition(value); + } + if let Some(value) = crate::utils::borrowed_string(self.version, "version")? { + builder.version(value); + } + if let Some(values) = crate::utils::borrowed_string_pairs( + self.user_metadata_has_value, + self.user_metadata_keys as *const *const c_char, + self.user_metadata_values as *const *const c_char, + self.user_metadata_len, + "user_metadata", + )? { + builder.user_metadata(values); + } + + Ok(builder.build()) + } +} + +/// Rebuild metadata passed by pointer, where null is `None`. +/// +/// `metadata` must be null or point to a valid mirror whose pointers stay +/// valid for the duration of the call. +pub(crate) fn borrowed_metadata( + metadata: *const OpendalMetadata, +) -> Result, OpenDALError> { + unsafe { metadata.as_ref() } + .map(OpendalMetadata::to_metadata) + .transpose() +} diff --git a/bindings/dotnet/src/operator.rs b/bindings/dotnet/src/operator.rs index 875cdef64e2b..ed02dc9815dd 100644 --- a/bindings/dotnet/src/operator.rs +++ b/bindings/dotnet/src/operator.rs @@ -23,17 +23,17 @@ use crate::{ metadata::OpendalMetadata, operator_info::{OpendalOperatorInfo, into_operator_info}, options::{ - parse_delete_options, parse_list_options, parse_read_options, parse_stat_options, - parse_write_options, + OpendalCopyOptions, OpendalDeleteOptions, OpendalListOptions, OpendalReadOptions, + OpendalRenameOptions, OpendalStatOptions, OpendalWriteOptions, read_options, }, presign::into_presigned_request_ptr, result::{ OpendalBoolResult, OpendalEntryListResult, OpendalMetadataResult, - OpendalOperatorInfoResult, OpendalOperatorResult, OpendalOptionsResult, - OpendalPresignedRequestResult, OpendalReadResult, OpendalResult, + OpendalOperatorInfoResult, OpendalOperatorResult, OpendalPresignedRequestResult, + OpendalReadResult, OpendalResult, }, utils::{ - collect_options, release_c_string, require_callback, require_cstr, require_data_ptr, + borrowed_string_map, release_c_string, require_callback, require_cstr, require_data_ptr, require_op_handle, }, validators::prelude::{ @@ -42,7 +42,6 @@ use crate::{ }, }; -use std::collections::HashMap; use std::ffi::c_void; use std::os::raw::c_char; use std::sync::Arc; @@ -103,216 +102,26 @@ type MetadataCallback = extern "C" fn(context: i64, result: OpendalMetadataResul type ListCallback = extern "C" fn(context: i64, result: OpendalEntryListResult); type PresignCallback = extern "C" fn(context: i64, result: OpendalPresignedRequestResult); -/// Collect key/value arrays, parse them with `parse`, and box the result into -/// an options payload released by the matching `*_option_free`. -/// -/// # Safety -/// -/// Same contract as the exported `*_option_build` functions. -unsafe fn build_options( - keys: *const *const c_char, - values: *const *const c_char, - len: usize, - parse: impl FnOnce(HashMap) -> Result, -) -> OpendalOptionsResult { - match unsafe { collect_options(keys, values, len) }.and_then(parse) { - Ok(options) => OpendalOptionsResult::ok(Box::into_raw(Box::new(options)) as *mut c_void), - Err(error) => OpendalOptionsResult::from_error(error), - } -} - -/// Release an options payload produced by `build_options`. -/// -/// # Safety -/// -/// - `options` must be null or a pointer returned by `build_options` for the -/// same `T`. -/// - This function must be called at most once for the same pointer. -unsafe fn free_options(options: *mut T) { - if options.is_null() { - return; - } - - drop(unsafe { Box::from_raw(options) }); -} - -/// Build constructor options from raw C string key/value arrays. -/// -/// On success, the returned pointer must be released by -/// `constructor_option_free`. -/// # Safety -/// -/// - When `len > 0`, `keys` and `values` must be non-null pointers to arrays -/// containing at least `len` C-string pointers. -/// - Each key/value entry must be a valid null-terminated UTF-8 string. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn constructor_option_build( - keys: *const *const c_char, - values: *const *const c_char, - len: usize, -) -> OpendalOptionsResult { - unsafe { build_options(keys, values, len, Ok) } -} - -/// # Safety -/// -/// - `options` must be null or a pointer returned by -/// `constructor_option_build`. -/// - This function must be called at most once for the same pointer. -#[unsafe(no_mangle)] -pub extern "C" fn constructor_option_free(options: *mut HashMap) { - unsafe { free_options(options) } -} - -/// Build read options from raw C string key/value arrays. -/// -/// On success, the returned pointer must be released by `read_option_free`. -/// # Safety -/// -/// - When `len > 0`, `keys` and `values` must be non-null pointers to arrays -/// containing at least `len` C-string pointers. -/// - Each key/value entry must be a valid null-terminated UTF-8 string. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn read_option_build( - keys: *const *const c_char, - values: *const *const c_char, - len: usize, -) -> OpendalOptionsResult { - unsafe { build_options(keys, values, len, |values| parse_read_options(&values)) } -} - -/// # Safety -/// -/// - `options` must be null or a pointer returned by `read_option_build`. -/// - This function must be called at most once for the same pointer. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn read_option_free(options: *mut opendal::options::ReadOptions) { - unsafe { free_options(options) } -} - -/// Build write options from raw C string key/value arrays. -/// -/// On success, the returned pointer must be released by `write_option_free`. -/// # Safety -/// -/// - When `len > 0`, `keys` and `values` must be non-null pointers to arrays -/// containing at least `len` C-string pointers. -/// - Each key/value entry must be a valid null-terminated UTF-8 string. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn write_option_build( - keys: *const *const c_char, - values: *const *const c_char, - len: usize, -) -> OpendalOptionsResult { - unsafe { build_options(keys, values, len, |values| parse_write_options(&values)) } -} - -/// # Safety -/// -/// - `options` must be null or a pointer returned by `write_option_build`. -/// - This function must be called at most once for the same pointer. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn write_option_free(options: *mut opendal::options::WriteOptions) { - unsafe { free_options(options) } -} - -/// Build stat options from raw C string key/value arrays. -/// -/// On success, the returned pointer must be released by `stat_option_free`. -/// # Safety -/// -/// - When `len > 0`, `keys` and `values` must be non-null pointers to arrays -/// containing at least `len` C-string pointers. -/// - Each key/value entry must be a valid null-terminated UTF-8 string. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn stat_option_build( - keys: *const *const c_char, - values: *const *const c_char, - len: usize, -) -> OpendalOptionsResult { - unsafe { build_options(keys, values, len, |values| parse_stat_options(&values)) } -} - -/// # Safety -/// -/// - `options` must be null or a pointer returned by `stat_option_build`. -/// - This function must be called at most once for the same pointer. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn stat_option_free(options: *mut opendal::options::StatOptions) { - unsafe { free_options(options) } -} - -/// Build list options from raw C string key/value arrays. -/// -/// On success, the returned pointer must be released by `list_option_free`. -/// # Safety -/// -/// - When `len > 0`, `keys` and `values` must be non-null pointers to arrays -/// containing at least `len` C-string pointers. -/// - Each key/value entry must be a valid null-terminated UTF-8 string. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn list_option_build( - keys: *const *const c_char, - values: *const *const c_char, - len: usize, -) -> OpendalOptionsResult { - unsafe { build_options(keys, values, len, |values| parse_list_options(&values)) } -} - -/// # Safety -/// -/// - `options` must be null or a pointer returned by `list_option_build`. -/// - This function must be called at most once for the same pointer. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn list_option_free(options: *mut opendal::options::ListOptions) { - unsafe { free_options(options) } -} - -/// Build delete options from raw C string key/value arrays. -/// -/// On success, the returned pointer must be released by `delete_option_free`. -/// # Safety -/// -/// - When `len > 0`, `keys` and `values` must be non-null pointers to arrays -/// containing at least `len` C-string pointers. -/// - Each key/value entry must be a valid null-terminated UTF-8 string. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn delete_option_build( - keys: *const *const c_char, - values: *const *const c_char, - len: usize, -) -> OpendalOptionsResult { - unsafe { build_options(keys, values, len, |values| parse_delete_options(&values)) } -} - -/// # Safety -/// -/// - `options` must be null or a pointer returned by `delete_option_build`. -/// - This function must be called at most once for the same pointer. -#[unsafe(no_mangle)] -pub unsafe extern "C" fn delete_option_free(options: *mut opendal::options::DeleteOptions) { - unsafe { free_options(options) } -} - /// Construct an OpenDAL operator instance from a scheme and key/value options. /// /// Returns a pointer that must be released with `operator_free`. /// # Safety /// /// - `scheme` must be a valid null-terminated UTF-8 string. -/// - When `len > 0`, `keys` and `values` must be non-null and point to arrays -/// of at least `len` entries. -/// - Every key/value entry in those arrays must be a valid null-terminated -/// UTF-8 string. +/// - When `option_len > 0`, `option_keys` and `option_values` must be valid +/// arrays of `option_len` null-terminated UTF-8 strings; a null value is +/// reported by key. /// - `executor` must be either null or a live pointer returned by /// `executor_create`. The operator binds it for its whole lifetime. #[unsafe(no_mangle)] pub unsafe extern "C" fn operator_construct( scheme: *const c_char, - options: *const HashMap, + option_keys: *const *const c_char, + option_values: *const *const c_char, + option_len: usize, executor: *const c_void, ) -> OpendalOperatorResult { - match operator_construct_inner(scheme, options, executor) { + match operator_construct_inner(scheme, option_keys, option_values, option_len, executor) { Ok(op) => OpendalOperatorResult::ok(op), Err(error) => OpendalOperatorResult::from_error(error), } @@ -320,15 +129,13 @@ pub unsafe extern "C" fn operator_construct( fn operator_construct_inner( scheme: *const c_char, - options: *const HashMap, + option_keys: *const *const c_char, + option_values: *const *const c_char, + option_len: usize, executor: *const c_void, ) -> Result<*mut c_void, OpenDALError> { let scheme = require_cstr(scheme, "scheme")?; - let options = if options.is_null() { - HashMap::default() - } else { - unsafe { (&*options).clone() } - }; + let options = borrowed_string_map(option_keys, option_values, option_len, "option")?; // SAFETY: the caller keeps the executor handle alive across this call per // `operator_construct`'s contract; the clone owns the runtime afterwards. let executor = unsafe { executor_or_default(executor)? }; @@ -649,12 +456,13 @@ fn operator_duplicate_inner(op_handle: *const OperatorHandle) -> Result<*mut c_v /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `path` must be a valid null-terminated UTF-8 string. -/// - `options` must be null or a pointer returned by `delete_option_build`. +/// - `options` must be null or point to a valid `OpendalDeleteOptions` whose +/// pointers stay valid for the duration of the call. #[unsafe(no_mangle)] pub extern "C" fn operator_delete_with_options( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::DeleteOptions, + options: *const OpendalDeleteOptions, ) -> OpendalResult { match operator_delete_with_options_inner(op_handle, path, options) { Ok(()) => OpendalResult::ok(), @@ -665,17 +473,13 @@ pub extern "C" fn operator_delete_with_options( fn operator_delete_with_options_inner( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::DeleteOptions, + options: *const OpendalDeleteOptions, ) -> Result<(), OpenDALError> { let handle = require_op_handle(op_handle)?; let executor = handle.executor.clone(); let path = require_cstr(path, "path")?; - let options = if options.is_null() { - opendal::options::DeleteOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; executor .block_on(handle.delete_options(path, options)) @@ -689,13 +493,14 @@ fn operator_delete_with_options_inner( /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `path` must be a valid null-terminated UTF-8 string. -/// - `options` must be null or a pointer returned by `delete_option_build`. +/// - `options` must be null or point to a valid `OpendalDeleteOptions` whose +/// pointers stay valid for the duration of the call. /// - `callback` must be a valid function pointer and remain callable until invoked. #[unsafe(no_mangle)] pub extern "C" fn operator_delete_with_options_async( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::DeleteOptions, + options: *const OpendalDeleteOptions, callback: Option, context: i64, ) -> OpendalResult { @@ -708,7 +513,7 @@ pub extern "C" fn operator_delete_with_options_async( fn operator_delete_with_options_async_inner( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::DeleteOptions, + options: *const OpendalDeleteOptions, callback: Option, context: i64, ) -> Result<(), OpenDALError> { @@ -717,11 +522,7 @@ fn operator_delete_with_options_async_inner( let path = require_cstr(path, "path")?.to_string(); let callback = require_callback(callback)?; - let options = if options.is_null() { - opendal::options::DeleteOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; let op = handle.operator(); executor.spawn(async move { @@ -822,7 +623,7 @@ fn operator_create_dir_async_inner( Ok(()) } -/// Copy from `source_path` to `target_path` synchronously. +/// Copy from `source_path` to `target_path` synchronously with options. /// /// On success, the returned payload must be released with /// `opendal_metadata_result_release`. @@ -830,35 +631,40 @@ fn operator_create_dir_async_inner( /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `source_path` and `target_path` must be valid null-terminated UTF-8 strings. +/// - `options` must be null or point to a valid `OpendalCopyOptions` whose +/// pointers stay valid for the duration of the call. #[unsafe(no_mangle)] -pub extern "C" fn operator_copy( +pub extern "C" fn operator_copy_with_options( op_handle: *const OperatorHandle, source_path: *const c_char, target_path: *const c_char, + options: *const OpendalCopyOptions, ) -> OpendalMetadataResult { - match operator_copy_inner(op_handle, source_path, target_path) { + match operator_copy_with_options_inner(op_handle, source_path, target_path, options) { Ok(value) => OpendalMetadataResult::ok(value as *mut c_void), Err(error) => OpendalMetadataResult::from_error(error), } } -fn operator_copy_inner( +fn operator_copy_with_options_inner( op_handle: *const OperatorHandle, source_path: *const c_char, target_path: *const c_char, + options: *const OpendalCopyOptions, ) -> Result<*mut OpendalMetadata, OpenDALError> { let handle = require_op_handle(op_handle)?; let executor = handle.executor.clone(); let source_path = require_cstr(source_path, "source_path")?; let target_path = require_cstr(target_path, "target_path")?; + let options = read_options(options)?; let metadata = executor - .block_on(handle.copy(source_path, target_path)) + .block_on(handle.copy_options(source_path, target_path, options)) .map_err(OpenDALError::from_opendal_error)?; Ok(into_metadata_ptr(metadata)) } -/// Copy from `source_path` to `target_path` asynchronously. +/// Copy from `source_path` to `target_path` asynchronously with options. /// /// The callback is invoked exactly once. On success, the callback result must /// be released with `opendal_metadata_result_release`. @@ -866,25 +672,36 @@ fn operator_copy_inner( /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `source_path` and `target_path` must be valid null-terminated UTF-8 strings. +/// - `options` must be null or point to a valid `OpendalCopyOptions` whose +/// pointers stay valid for the duration of the call. /// - `callback` must be a valid function pointer and remain callable until invoked. #[unsafe(no_mangle)] -pub extern "C" fn operator_copy_async( +pub extern "C" fn operator_copy_with_options_async( op_handle: *const OperatorHandle, source_path: *const c_char, target_path: *const c_char, + options: *const OpendalCopyOptions, callback: Option, context: i64, ) -> OpendalResult { - match operator_copy_async_inner(op_handle, source_path, target_path, callback, context) { + match operator_copy_with_options_async_inner( + op_handle, + source_path, + target_path, + options, + callback, + context, + ) { Ok(()) => OpendalResult::ok(), Err(error) => OpendalResult::from_error(error), } } -fn operator_copy_async_inner( +fn operator_copy_with_options_async_inner( op_handle: *const OperatorHandle, source_path: *const c_char, target_path: *const c_char, + options: *const OpendalCopyOptions, callback: Option, context: i64, ) -> Result<(), OpenDALError> { @@ -893,11 +710,12 @@ fn operator_copy_async_inner( let source_path = require_cstr(source_path, "source_path")?.to_string(); let target_path = require_cstr(target_path, "target_path")?.to_string(); let callback = require_callback(callback)?; + let options = read_options(options)?; let op = handle.operator(); executor.spawn(async move { let result = op - .copy(&source_path, &target_path) + .copy_options(&source_path, &target_path, options) .await .map(into_metadata_ptr) .map_err(OpenDALError::from_opendal_error); @@ -914,64 +732,80 @@ fn operator_copy_async_inner( Ok(()) } -/// Rename from `source_path` to `target_path` synchronously. +/// Rename from `source_path` to `target_path` synchronously with options. /// # Safety /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `source_path` and `target_path` must be valid null-terminated UTF-8 strings. +/// - `options` must be null or point to a valid `OpendalRenameOptions` whose +/// pointers stay valid for the duration of the call. #[unsafe(no_mangle)] -pub extern "C" fn operator_rename( +pub extern "C" fn operator_rename_with_options( op_handle: *const OperatorHandle, source_path: *const c_char, target_path: *const c_char, + options: *const OpendalRenameOptions, ) -> OpendalResult { - match operator_rename_inner(op_handle, source_path, target_path) { + match operator_rename_with_options_inner(op_handle, source_path, target_path, options) { Ok(()) => OpendalResult::ok(), Err(error) => OpendalResult::from_error(error), } } -fn operator_rename_inner( +fn operator_rename_with_options_inner( op_handle: *const OperatorHandle, source_path: *const c_char, target_path: *const c_char, + options: *const OpendalRenameOptions, ) -> Result<(), OpenDALError> { let handle = require_op_handle(op_handle)?; let executor = handle.executor.clone(); let source_path = require_cstr(source_path, "source_path")?; let target_path = require_cstr(target_path, "target_path")?; + let options = read_options(options)?; executor - .block_on(handle.rename(source_path, target_path)) + .block_on(handle.rename_options(source_path, target_path, options)) .map_err(OpenDALError::from_opendal_error) } -/// Rename from `source_path` to `target_path` asynchronously. +/// Rename from `source_path` to `target_path` asynchronously with options. /// /// The callback is invoked exactly once with the final result. /// # Safety /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `source_path` and `target_path` must be valid null-terminated UTF-8 strings. +/// - `options` must be null or point to a valid `OpendalRenameOptions` whose +/// pointers stay valid for the duration of the call. /// - `callback` must be a valid function pointer and remain callable until invoked. #[unsafe(no_mangle)] -pub extern "C" fn operator_rename_async( +pub extern "C" fn operator_rename_with_options_async( op_handle: *const OperatorHandle, source_path: *const c_char, target_path: *const c_char, + options: *const OpendalRenameOptions, callback: Option, context: i64, ) -> OpendalResult { - match operator_rename_async_inner(op_handle, source_path, target_path, callback, context) { + match operator_rename_with_options_async_inner( + op_handle, + source_path, + target_path, + options, + callback, + context, + ) { Ok(()) => OpendalResult::ok(), Err(error) => OpendalResult::from_error(error), } } -fn operator_rename_async_inner( +fn operator_rename_with_options_async_inner( op_handle: *const OperatorHandle, source_path: *const c_char, target_path: *const c_char, + options: *const OpendalRenameOptions, callback: Option, context: i64, ) -> Result<(), OpenDALError> { @@ -980,11 +814,12 @@ fn operator_rename_async_inner( let source_path = require_cstr(source_path, "source_path")?.to_string(); let target_path = require_cstr(target_path, "target_path")?.to_string(); let callback = require_callback(callback)?; + let options = read_options(options)?; let op = handle.operator(); executor.spawn(async move { let result = op - .rename(&source_path, &target_path) + .rename_options(&source_path, &target_path, options) .await .map_err(OpenDALError::from_opendal_error); @@ -1231,7 +1066,7 @@ fn operator_presign_delete_async_inner( pub extern "C" fn operator_input_stream_create( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::ReadOptions, + options: *const OpendalReadOptions, ) -> OpendalOperatorResult { match operator_input_stream_create_inner(op_handle, path, options) { Ok(value) => OpendalOperatorResult::ok(value), @@ -1285,22 +1120,20 @@ fn next_chunk_to_buffer( fn operator_input_stream_create_inner( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::ReadOptions, + options: *const OpendalReadOptions, ) -> Result<*mut c_void, OpenDALError> { let handle = require_op_handle(op_handle)?; let executor = handle.executor.clone(); let path = require_cstr(path, "path")?.to_string(); - let options = if options.is_null() { - opendal::options::ReadOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; let range = options.range; let reader_options = opendal::options::ReaderOptions { version: options.version, if_match: options.if_match, if_none_match: options.if_none_match, + if_version_match: options.if_version_match, + if_version_not_match: options.if_version_not_match, if_modified_since: options.if_modified_since, if_unmodified_since: options.if_unmodified_since, content_length_hint: options.content_length_hint, @@ -1422,7 +1255,7 @@ pub unsafe extern "C" fn operator_input_stream_free(stream: *mut c_void) { pub extern "C" fn operator_output_stream_create( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::WriteOptions, + options: *const OpendalWriteOptions, ) -> OpendalOperatorResult { match operator_output_stream_create_inner(op_handle, path, options) { Ok(value) => OpendalOperatorResult::ok(value), @@ -1454,16 +1287,12 @@ fn require_output_stream<'a>(stream: *mut c_void) -> Result<&'a OutputStream, Op fn operator_output_stream_create_inner( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::WriteOptions, + options: *const OpendalWriteOptions, ) -> Result<*mut c_void, OpenDALError> { let handle = require_op_handle(op_handle)?; let executor = handle.executor.clone(); let path = require_cstr(path, "path")?.to_string(); - let options = if options.is_null() { - opendal::options::WriteOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; let writer_op = handle.operator(); let writer = executor @@ -1709,7 +1538,7 @@ pub unsafe extern "C" fn operator_write_with_options( path: *const c_char, buffer: *mut c_void, committed_in_current: usize, - options: *const opendal::options::WriteOptions, + options: *const OpendalWriteOptions, ) -> OpendalMetadataResult { match operator_write_with_options_inner(op_handle, path, buffer, committed_in_current, options) { @@ -1723,7 +1552,7 @@ fn operator_write_with_options_inner( path: *const c_char, buffer: *mut c_void, committed_in_current: usize, - options: *const opendal::options::WriteOptions, + options: *const OpendalWriteOptions, ) -> Result<*mut OpendalMetadata, OpenDALError> { let handle = require_op_handle(op_handle)?; let executor = handle.executor.clone(); @@ -1733,11 +1562,7 @@ fn operator_write_with_options_inner( "write buffer handle is null", )); } - let options = if options.is_null() { - opendal::options::WriteOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; // Taken only after every fallible check above, so an error result means // ownership never transferred and the buffer stays usable. @@ -1770,7 +1595,7 @@ pub unsafe extern "C" fn operator_write_with_options_async( path: *const c_char, buffer: *mut c_void, committed_in_current: usize, - options: *const opendal::options::WriteOptions, + options: *const OpendalWriteOptions, callback: Option, context: i64, ) -> OpendalResult { @@ -1797,7 +1622,7 @@ fn operator_write_with_options_async_inner( path: *const c_char, buffer: *mut c_void, committed_in_current: usize, - options: *const opendal::options::WriteOptions, + options: *const OpendalWriteOptions, callback: Option, context: i64, ) -> Result<(), OpenDALError> { @@ -1810,11 +1635,7 @@ fn operator_write_with_options_async_inner( "write buffer handle is null", )); } - let options = if options.is_null() { - opendal::options::WriteOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; // Taken only after every fallible check above, so an error result means // ownership never transferred and the buffer stays usable. @@ -1857,7 +1678,7 @@ pub unsafe extern "C" fn operator_write_bytes_with_options( path: *const c_char, data: *const u8, len: usize, - options: *const opendal::options::WriteOptions, + options: *const OpendalWriteOptions, ) -> OpendalMetadataResult { match operator_write_bytes_with_options_inner(op_handle, path, data, len, options) { Ok(value) => OpendalMetadataResult::ok(value as *mut c_void), @@ -1870,17 +1691,13 @@ fn operator_write_bytes_with_options_inner( path: *const c_char, data: *const u8, len: usize, - options: *const opendal::options::WriteOptions, + options: *const OpendalWriteOptions, ) -> Result<*mut OpendalMetadata, OpenDALError> { let handle = require_op_handle(op_handle)?; let executor = handle.executor.clone(); let path = require_cstr(path, "path")?; require_data_ptr(data, len)?; - let options = if options.is_null() { - opendal::options::WriteOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; // Copied explicitly: the only `Into` impl for slices is // `From<&'static [u8]>`, which is zero-copy, so passing the raw slice @@ -1916,7 +1733,7 @@ pub unsafe extern "C" fn operator_write_bytes_with_options_async( path: *const c_char, data: *const u8, len: usize, - options: *const opendal::options::WriteOptions, + options: *const OpendalWriteOptions, callback: Option, context: i64, ) -> OpendalResult { @@ -1937,7 +1754,7 @@ fn operator_write_bytes_with_options_async_inner( path: *const c_char, data: *const u8, len: usize, - options: *const opendal::options::WriteOptions, + options: *const OpendalWriteOptions, callback: Option, context: i64, ) -> Result<(), OpenDALError> { @@ -1946,11 +1763,7 @@ fn operator_write_bytes_with_options_async_inner( let path = require_cstr(path, "path")?.to_string(); require_data_ptr(data, len)?; let callback = require_callback(callback)?; - let options = if options.is_null() { - opendal::options::WriteOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; // Copied before the task is spawned, so the caller only has to keep its // array alive for the duration of this call. @@ -1987,12 +1800,13 @@ fn operator_write_bytes_with_options_async_inner( /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `path` must be a valid null-terminated UTF-8 string. -/// - When `option_len > 0`, `option_keys` and `option_values` must be valid arrays. +/// - `options` must be null or point to a valid `OpendalReadOptions` whose +/// pointers stay valid for the duration of the call. #[unsafe(no_mangle)] pub unsafe extern "C" fn operator_read_with_options( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::ReadOptions, + options: *const OpendalReadOptions, ) -> OpendalReadResult { match operator_read_with_options_inner(op_handle, path, options) { Ok(value) => OpendalReadResult::ok(value), @@ -2003,16 +1817,12 @@ pub unsafe extern "C" fn operator_read_with_options( fn operator_read_with_options_inner( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::ReadOptions, + options: *const OpendalReadOptions, ) -> Result { let handle = require_op_handle(op_handle)?; let executor = handle.executor.clone(); let path = require_cstr(path, "path")?; - let options = if options.is_null() { - opendal::options::ReadOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; let value = executor .block_on(handle.read_options(path, options)) @@ -2029,13 +1839,14 @@ fn operator_read_with_options_inner( /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `path` must be a valid null-terminated UTF-8 string. -/// - When `option_len > 0`, `option_keys` and `option_values` must be valid arrays. +/// - `options` must be null or point to a valid `OpendalReadOptions` whose +/// pointers stay valid for the duration of the call. /// - `callback` must be a valid function pointer and remain callable until invoked. #[unsafe(no_mangle)] pub extern "C" fn operator_read_with_options_async( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::ReadOptions, + options: *const OpendalReadOptions, callback: Option, context: i64, ) -> OpendalResult { @@ -2048,7 +1859,7 @@ pub extern "C" fn operator_read_with_options_async( fn operator_read_with_options_async_inner( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::ReadOptions, + options: *const OpendalReadOptions, callback: Option, context: i64, ) -> Result<(), OpenDALError> { @@ -2056,11 +1867,7 @@ fn operator_read_with_options_async_inner( let executor = handle.executor.clone(); let path = require_cstr(path, "path")?.to_string(); let callback = require_callback(callback)?; - let options = if options.is_null() { - opendal::options::ReadOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; let op = handle.operator(); executor.spawn(async move { @@ -2089,12 +1896,13 @@ fn operator_read_with_options_async_inner( /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `path` must be a valid null-terminated UTF-8 string. -/// - When `option_len > 0`, `option_keys` and `option_values` must be valid arrays. +/// - `options` must be null or point to a valid `OpendalStatOptions` whose +/// pointers stay valid for the duration of the call. #[unsafe(no_mangle)] pub extern "C" fn operator_stat_with_options( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::StatOptions, + options: *const OpendalStatOptions, ) -> OpendalMetadataResult { match operator_stat_with_options_inner(op_handle, path, options) { Ok(value) => OpendalMetadataResult::ok(value as *mut c_void), @@ -2105,16 +1913,12 @@ pub extern "C" fn operator_stat_with_options( fn operator_stat_with_options_inner( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::StatOptions, + options: *const OpendalStatOptions, ) -> Result<*mut OpendalMetadata, OpenDALError> { let handle = require_op_handle(op_handle)?; let executor = handle.executor.clone(); let path = require_cstr(path, "path")?; - let options = if options.is_null() { - opendal::options::StatOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; let metadata = executor .block_on(handle.stat_options(path, options)) @@ -2130,13 +1934,14 @@ fn operator_stat_with_options_inner( /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `path` must be a valid null-terminated UTF-8 string. -/// - When `option_len > 0`, `option_keys` and `option_values` must be valid arrays. +/// - `options` must be null or point to a valid `OpendalStatOptions` whose +/// pointers stay valid for the duration of the call. /// - `callback` must be a valid function pointer and remain callable until invoked. #[unsafe(no_mangle)] pub extern "C" fn operator_stat_with_options_async( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::StatOptions, + options: *const OpendalStatOptions, callback: Option, context: i64, ) -> OpendalResult { @@ -2149,7 +1954,7 @@ pub extern "C" fn operator_stat_with_options_async( fn operator_stat_with_options_async_inner( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::StatOptions, + options: *const OpendalStatOptions, callback: Option, context: i64, ) -> Result<(), OpenDALError> { @@ -2157,11 +1962,7 @@ fn operator_stat_with_options_async_inner( let executor = handle.executor.clone(); let path = require_cstr(path, "path")?.to_string(); let callback = require_callback(callback)?; - let options = if options.is_null() { - opendal::options::StatOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; let op = handle.operator(); executor.spawn(async move { @@ -2335,12 +2136,13 @@ fn operator_check_async_inner( /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `path` must be a valid null-terminated UTF-8 string. -/// - When `option_len > 0`, `option_keys` and `option_values` must be valid arrays. +/// - `options` must be null or point to a valid `OpendalListOptions` whose +/// pointers stay valid for the duration of the call. #[unsafe(no_mangle)] pub extern "C" fn operator_list_with_options( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::ListOptions, + options: *const OpendalListOptions, ) -> OpendalEntryListResult { match operator_list_with_options_inner(op_handle, path, options) { Ok(value) => OpendalEntryListResult::ok(value), @@ -2351,16 +2153,12 @@ pub extern "C" fn operator_list_with_options( fn operator_list_with_options_inner( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::ListOptions, + options: *const OpendalListOptions, ) -> Result<*mut c_void, OpenDALError> { let handle = require_op_handle(op_handle)?; let executor = handle.executor.clone(); let path = require_cstr(path, "path")?; - let options = if options.is_null() { - opendal::options::ListOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; let entries = executor .block_on(handle.list_options(path, options)) @@ -2377,13 +2175,14 @@ fn operator_list_with_options_inner( /// /// - `op_handle` must be a valid operator pointer from `operator_construct`. /// - `path` must be a valid null-terminated UTF-8 string. -/// - When `option_len > 0`, `option_keys` and `option_values` must be valid arrays. +/// - `options` must be null or point to a valid `OpendalListOptions` whose +/// pointers stay valid for the duration of the call. /// - `callback` must be a valid function pointer and remain callable until invoked. #[unsafe(no_mangle)] pub extern "C" fn operator_list_with_options_async( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::ListOptions, + options: *const OpendalListOptions, callback: Option, context: i64, ) -> OpendalResult { @@ -2396,7 +2195,7 @@ pub extern "C" fn operator_list_with_options_async( fn operator_list_with_options_async_inner( op_handle: *const OperatorHandle, path: *const c_char, - options: *const opendal::options::ListOptions, + options: *const OpendalListOptions, callback: Option, context: i64, ) -> Result<(), OpenDALError> { @@ -2404,11 +2203,7 @@ fn operator_list_with_options_async_inner( let executor = handle.executor.clone(); let path = require_cstr(path, "path")?.to_string(); let callback = require_callback(callback)?; - let options = if options.is_null() { - opendal::options::ListOptions::default() - } else { - unsafe { (&*options).clone() } - }; + let options = read_options(options)?; let op = handle.operator(); executor.spawn(async move { diff --git a/bindings/dotnet/src/options.rs b/bindings/dotnet/src/options.rs index cde52d31daa4..164aca8833bd 100644 --- a/bindings/dotnet/src/options.rs +++ b/bindings/dotnet/src/options.rs @@ -15,209 +15,680 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; -use std::fmt::Display; -use std::str::FromStr; +// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. -use opendal::raw::Timestamp; +// Each field converts in its own statement, so a new core option only adds one. +#![allow(clippy::field_reassign_with_default)] + +use std::os::raw::c_char; + +use opendal::BytesRange; use crate::error::OpenDALError; -use crate::validators::prelude::{ - validate_list_limit, validate_read_chunk, validate_read_concurrent, validate_read_gap, - validate_read_range_end, validate_write_chunk, validate_write_concurrent, +use crate::metadata::{OpendalMetadata, borrowed_metadata}; +use crate::utils::{ + borrowed_string, borrowed_string_pairs, borrowed_timestamp, config_invalid_error, }; +use crate::validators::prelude::validate_read_range_end; +use crate::validators::validate_positive_integer; + +/// A `#[repr(C)]` mirror of one core options struct. +/// +/// Pointers are borrowed from the caller for the duration of the call and +/// copied into owned values by `to_options`, so native code never keeps them. +/// A null string pointer is `None`; optional values flatten into a +/// `*_has_value` byte, `1` when present, followed by their parts. +pub(crate) trait NativeOptions { + /// The core options struct this mirror converts into. + type Options: Default; + + /// Validate the mirror and convert it into core options. + fn to_options(&self) -> Result; +} + +/// Convert options passed by pointer, where null selects the defaults. +/// +/// `options` must be null or point to a valid mirror whose pointers stay +/// valid for the duration of the call. +pub(crate) fn read_options( + options: *const T, +) -> Result { + match unsafe { options.as_ref() } { + Some(options) => options.to_options(), + None => Ok(T::Options::default()), + } +} + +/// Build a byte range from `offset` and `length`, or from `suffix_length` +/// for a suffix range. +fn range( + offset: u64, + length: Option, + suffix_length: Option, +) -> Result { + if let Some(size) = suffix_length { + if offset > 0 || length.is_some() { + return Err(config_invalid_error( + "suffix_length cannot be combined with offset or length", + )); + } + return Ok(BytesRange::suffix(size)); + } -fn parse_bool(values: &HashMap, key: &str) -> Result, OpenDALError> { - let Some(raw) = values.get(key) else { - return Ok(None); - }; - - match raw.to_ascii_lowercase().as_str() { - "true" | "1" => Ok(Some(true)), - "false" | "0" => Ok(Some(false)), - _ => Err(crate::utils::config_invalid_error(format!( - "invalid boolean value for {key}: {raw}" - ))), - } -} - -fn parse_number(values: &HashMap, key: &str) -> Result, OpenDALError> -where - T: FromStr, - T::Err: Display, -{ - let Some(raw) = values.get(key) else { - return Ok(None); - }; - - raw.parse::().map(Some).map_err(|err| { - crate::utils::config_invalid_error(format!( - "invalid {} value for {key}: {raw}, {err}", - std::any::type_name::() - )) + Ok(match validate_read_range_end(offset, length)? { + Some(end) => (offset..end).into(), + None => (offset..).into(), }) } -fn parse_timestamp( - values: &HashMap, - key: &str, -) -> Result, OpenDALError> { - let Some(raw) = values.get(key) else { - return Ok(None); - }; +/// FFI representation of `opendal::options::ReadOptions`. +#[repr(C)] +pub struct OpendalReadOptions { + pub offset: u64, + pub length_has_value: u8, + pub length: u64, + pub suffix_length_has_value: u8, + pub suffix_length: u64, + pub version: *const c_char, + pub if_match: *const c_char, + pub if_none_match: *const c_char, + pub if_version_match: *const c_char, + pub if_version_not_match: *const c_char, + pub if_modified_since_has_value: u8, + pub if_modified_since_second: i64, + pub if_modified_since_nanosecond: i32, + pub if_unmodified_since_has_value: u8, + pub if_unmodified_since_second: i64, + pub if_unmodified_since_nanosecond: i32, + pub content_length_hint_has_value: u8, + pub content_length_hint: u64, + pub concurrent: usize, + pub chunk_has_value: u8, + pub chunk: usize, + pub gap_has_value: u8, + pub gap: usize, + pub override_content_type: *const c_char, + pub override_cache_control: *const c_char, + pub override_content_disposition: *const c_char, +} - let millis = raw.parse::().map_err(|err| { - crate::utils::config_invalid_error(format!( - "invalid timestamp milliseconds for {key}: {raw}, {err}" - )) - })?; +impl NativeOptions for OpendalReadOptions { + type Options = opendal::options::ReadOptions; + + fn to_options(&self) -> Result { + let mut options = Self::Options::default(); + options.range = range( + self.offset, + (self.length_has_value != 0).then_some(self.length), + (self.suffix_length_has_value != 0).then_some(self.suffix_length), + )?; + options.version = borrowed_string(self.version, "version")?; + options.if_match = borrowed_string(self.if_match, "if_match")?; + options.if_none_match = borrowed_string(self.if_none_match, "if_none_match")?; + options.if_version_match = borrowed_string(self.if_version_match, "if_version_match")?; + options.if_version_not_match = + borrowed_string(self.if_version_not_match, "if_version_not_match")?; + options.if_modified_since = borrowed_timestamp( + self.if_modified_since_has_value, + self.if_modified_since_second, + self.if_modified_since_nanosecond, + )?; + options.if_unmodified_since = borrowed_timestamp( + self.if_unmodified_since_has_value, + self.if_unmodified_since_second, + self.if_unmodified_since_nanosecond, + )?; + options.content_length_hint = + (self.content_length_hint_has_value != 0).then_some(self.content_length_hint); + validate_positive_integer(self.concurrent, "read concurrent")?; + options.concurrent = self.concurrent; + if self.chunk_has_value != 0 { + validate_positive_integer(self.chunk, "read chunk")?; + options.chunk = Some(self.chunk); + } + if self.gap_has_value != 0 { + validate_positive_integer(self.gap, "read gap")?; + options.gap = Some(self.gap); + } + options.override_content_type = + borrowed_string(self.override_content_type, "override_content_type")?; + options.override_cache_control = + borrowed_string(self.override_cache_control, "override_cache_control")?; + options.override_content_disposition = borrowed_string( + self.override_content_disposition, + "override_content_disposition", + )?; + + Ok(options) + } +} - Timestamp::from_millisecond(millis) - .map(Some) - .map_err(OpenDALError::from_opendal_error) +/// The mirror of default options, as the C# class sends it unchanged. +#[cfg(test)] +impl Default for OpendalReadOptions { + fn default() -> Self { + Self { + offset: 0, + length_has_value: 0, + length: 0, + suffix_length_has_value: 0, + suffix_length: 0, + version: std::ptr::null(), + if_match: std::ptr::null(), + if_none_match: std::ptr::null(), + if_version_match: std::ptr::null(), + if_version_not_match: std::ptr::null(), + if_modified_since_has_value: 0, + if_modified_since_second: 0, + if_modified_since_nanosecond: 0, + if_unmodified_since_has_value: 0, + if_unmodified_since_second: 0, + if_unmodified_since_nanosecond: 0, + content_length_hint_has_value: 0, + content_length_hint: 0, + concurrent: 1, + chunk_has_value: 0, + chunk: 0, + gap_has_value: 0, + gap: 0, + override_content_type: std::ptr::null(), + override_cache_control: std::ptr::null(), + override_content_disposition: std::ptr::null(), + } + } } -fn parse_string(values: &HashMap, key: &str) -> Option { - values.get(key).cloned() +/// FFI representation of `opendal::options::WriteOptions`. +#[repr(C)] +pub struct OpendalWriteOptions { + pub append: u8, + pub cache_control: *const c_char, + pub content_type: *const c_char, + pub content_disposition: *const c_char, + pub content_encoding: *const c_char, + pub user_metadata_has_value: u8, + pub user_metadata_keys: *const *const c_char, + pub user_metadata_values: *const *const c_char, + pub user_metadata_len: usize, + pub if_match: *const c_char, + pub if_none_match: *const c_char, + pub if_version_match: *const c_char, + pub if_version_not_match: *const c_char, + pub if_not_exists: u8, + pub if_not_changed: *const OpendalMetadata, + pub concurrent: usize, + pub chunk_has_value: u8, + pub chunk: usize, } -pub fn parse_read_options( - values: &HashMap, -) -> Result { - let mut options = opendal::options::ReadOptions::default(); +impl NativeOptions for OpendalWriteOptions { + type Options = opendal::options::WriteOptions; + + fn to_options(&self) -> Result { + let mut options = Self::Options::default(); + options.append = self.append != 0; + options.cache_control = borrowed_string(self.cache_control, "cache_control")?; + options.content_type = borrowed_string(self.content_type, "content_type")?; + options.content_disposition = + borrowed_string(self.content_disposition, "content_disposition")?; + options.content_encoding = borrowed_string(self.content_encoding, "content_encoding")?; + options.user_metadata = borrowed_string_pairs( + self.user_metadata_has_value, + self.user_metadata_keys, + self.user_metadata_values, + self.user_metadata_len, + "user_metadata", + )?; + options.if_match = borrowed_string(self.if_match, "if_match")?; + options.if_none_match = borrowed_string(self.if_none_match, "if_none_match")?; + options.if_version_match = borrowed_string(self.if_version_match, "if_version_match")?; + options.if_version_not_match = + borrowed_string(self.if_version_not_match, "if_version_not_match")?; + options.if_not_exists = self.if_not_exists != 0; + options.if_not_changed = borrowed_metadata(self.if_not_changed)?; + validate_positive_integer(self.concurrent, "write concurrent")?; + options.concurrent = self.concurrent; + if self.chunk_has_value != 0 { + validate_positive_integer(self.chunk, "write chunk")?; + options.chunk = Some(self.chunk); + } + + Ok(options) + } +} - let offset = parse_number::(values, "offset")?.unwrap_or_default(); - let length = parse_number::(values, "length")?; - if offset > 0 || length.is_some() { - options.range = match validate_read_range_end(offset, length)? { - Some(end) => (offset..end).into(), - None => (offset..).into(), - }; +/// The mirror of default options, as the C# class sends it unchanged. +#[cfg(test)] +impl Default for OpendalWriteOptions { + fn default() -> Self { + Self { + append: 0, + cache_control: std::ptr::null(), + content_type: std::ptr::null(), + content_disposition: std::ptr::null(), + content_encoding: std::ptr::null(), + user_metadata_has_value: 0, + user_metadata_keys: std::ptr::null(), + user_metadata_values: std::ptr::null(), + user_metadata_len: 0, + if_match: std::ptr::null(), + if_none_match: std::ptr::null(), + if_version_match: std::ptr::null(), + if_version_not_match: std::ptr::null(), + if_not_exists: 0, + if_not_changed: std::ptr::null(), + concurrent: 1, + chunk_has_value: 0, + chunk: 0, + } } +} - options.version = parse_string(values, "version"); - options.if_match = parse_string(values, "if_match"); - options.if_none_match = parse_string(values, "if_none_match"); - options.if_modified_since = parse_timestamp(values, "if_modified_since")?; - options.if_unmodified_since = parse_timestamp(values, "if_unmodified_since")?; +/// FFI representation of `opendal::options::StatOptions`. +#[repr(C)] +pub struct OpendalStatOptions { + pub version: *const c_char, + pub if_match: *const c_char, + pub if_none_match: *const c_char, + pub if_version_match: *const c_char, + pub if_version_not_match: *const c_char, + pub if_modified_since_has_value: u8, + pub if_modified_since_second: i64, + pub if_modified_since_nanosecond: i32, + pub if_unmodified_since_has_value: u8, + pub if_unmodified_since_second: i64, + pub if_unmodified_since_nanosecond: i32, + pub override_content_type: *const c_char, + pub override_cache_control: *const c_char, + pub override_content_disposition: *const c_char, +} - if let Some(concurrent) = parse_number::(values, "concurrent")? { - validate_read_concurrent(concurrent)?; - options.concurrent = concurrent; +impl NativeOptions for OpendalStatOptions { + type Options = opendal::options::StatOptions; + + fn to_options(&self) -> Result { + let mut options = Self::Options::default(); + options.version = borrowed_string(self.version, "version")?; + options.if_match = borrowed_string(self.if_match, "if_match")?; + options.if_none_match = borrowed_string(self.if_none_match, "if_none_match")?; + options.if_version_match = borrowed_string(self.if_version_match, "if_version_match")?; + options.if_version_not_match = + borrowed_string(self.if_version_not_match, "if_version_not_match")?; + options.if_modified_since = borrowed_timestamp( + self.if_modified_since_has_value, + self.if_modified_since_second, + self.if_modified_since_nanosecond, + )?; + options.if_unmodified_since = borrowed_timestamp( + self.if_unmodified_since_has_value, + self.if_unmodified_since_second, + self.if_unmodified_since_nanosecond, + )?; + options.override_content_type = + borrowed_string(self.override_content_type, "override_content_type")?; + options.override_cache_control = + borrowed_string(self.override_cache_control, "override_cache_control")?; + options.override_content_disposition = borrowed_string( + self.override_content_disposition, + "override_content_disposition", + )?; + + Ok(options) } +} - if let Some(chunk) = parse_number::(values, "chunk")? { - validate_read_chunk(chunk)?; - options.chunk = Some(chunk); +/// The mirror of default options, as the C# class sends it unchanged. +#[cfg(test)] +impl Default for OpendalStatOptions { + fn default() -> Self { + Self { + version: std::ptr::null(), + if_match: std::ptr::null(), + if_none_match: std::ptr::null(), + if_version_match: std::ptr::null(), + if_version_not_match: std::ptr::null(), + if_modified_since_has_value: 0, + if_modified_since_second: 0, + if_modified_since_nanosecond: 0, + if_unmodified_since_has_value: 0, + if_unmodified_since_second: 0, + if_unmodified_since_nanosecond: 0, + override_content_type: std::ptr::null(), + override_cache_control: std::ptr::null(), + override_content_disposition: std::ptr::null(), + } } +} - if let Some(gap) = parse_number::(values, "gap")? { - validate_read_gap(gap)?; - options.gap = Some(gap); +/// FFI representation of `opendal::options::ListOptions`. +#[repr(C)] +pub struct OpendalListOptions { + pub limit_has_value: u8, + pub limit: usize, + pub start_after: *const c_char, + pub recursive: u8, + pub versions: u8, + pub deleted: u8, +} + +impl NativeOptions for OpendalListOptions { + type Options = opendal::options::ListOptions; + + fn to_options(&self) -> Result { + let mut options = Self::Options::default(); + if self.limit_has_value != 0 { + validate_positive_integer(self.limit, "list limit")?; + options.limit = Some(self.limit); + } + options.start_after = borrowed_string(self.start_after, "start_after")?; + options.recursive = self.recursive != 0; + options.versions = self.versions != 0; + options.deleted = self.deleted != 0; + + Ok(options) } +} - options.override_content_type = parse_string(values, "override_content_type"); - options.override_cache_control = parse_string(values, "override_cache_control"); - options.override_content_disposition = parse_string(values, "override_content_disposition"); +/// The mirror of default options, as the C# class sends it unchanged. +#[cfg(test)] +impl Default for OpendalListOptions { + fn default() -> Self { + Self { + limit_has_value: 0, + limit: 0, + start_after: std::ptr::null(), + recursive: 0, + versions: 0, + deleted: 0, + } + } +} - Ok(options) +/// FFI representation of `opendal::options::DeleteOptions`. +#[repr(C)] +pub struct OpendalDeleteOptions { + pub version: *const c_char, + pub recursive: u8, + pub if_match: *const c_char, + pub if_none_match: *const c_char, + pub if_version_match: *const c_char, + pub if_version_not_match: *const c_char, + pub if_not_changed: *const OpendalMetadata, } -pub fn parse_write_options( - values: &HashMap, -) -> Result { - let mut options = opendal::options::WriteOptions::default(); +impl NativeOptions for OpendalDeleteOptions { + type Options = opendal::options::DeleteOptions; + + fn to_options(&self) -> Result { + let mut options = Self::Options::default(); + options.version = borrowed_string(self.version, "version")?; + options.recursive = self.recursive != 0; + options.if_match = borrowed_string(self.if_match, "if_match")?; + options.if_none_match = borrowed_string(self.if_none_match, "if_none_match")?; + options.if_version_match = borrowed_string(self.if_version_match, "if_version_match")?; + options.if_version_not_match = + borrowed_string(self.if_version_not_match, "if_version_not_match")?; + options.if_not_changed = borrowed_metadata(self.if_not_changed)?; + + Ok(options) + } +} - if let Some(append) = parse_bool(values, "append")? { - options.append = append; +/// The mirror of default options, as the C# class sends it unchanged. +#[cfg(test)] +impl Default for OpendalDeleteOptions { + fn default() -> Self { + Self { + version: std::ptr::null(), + recursive: 0, + if_match: std::ptr::null(), + if_none_match: std::ptr::null(), + if_version_match: std::ptr::null(), + if_version_not_match: std::ptr::null(), + if_not_changed: std::ptr::null(), + } } +} - options.cache_control = parse_string(values, "cache_control"); - options.content_type = parse_string(values, "content_type"); - options.content_disposition = parse_string(values, "content_disposition"); - options.content_encoding = parse_string(values, "content_encoding"); - options.if_match = parse_string(values, "if_match"); - options.if_none_match = parse_string(values, "if_none_match"); +/// FFI representation of `opendal::options::CopyOptions`. +#[repr(C)] +pub struct OpendalCopyOptions { + pub if_not_exists: u8, + pub if_match: *const c_char, + pub if_none_match: *const c_char, + pub if_version_match: *const c_char, + pub if_version_not_match: *const c_char, + pub if_not_changed: *const OpendalMetadata, + pub source_version: *const c_char, + pub source_content_length_hint_has_value: u8, + pub source_content_length_hint: u64, + pub concurrent: usize, + pub chunk_has_value: u8, + pub chunk: usize, +} - if let Some(if_not_exists) = parse_bool(values, "if_not_exists")? { - options.if_not_exists = if_not_exists; +impl NativeOptions for OpendalCopyOptions { + type Options = opendal::options::CopyOptions; + + fn to_options(&self) -> Result { + let mut options = Self::Options::default(); + options.if_not_exists = self.if_not_exists != 0; + options.if_match = borrowed_string(self.if_match, "if_match")?; + options.if_none_match = borrowed_string(self.if_none_match, "if_none_match")?; + options.if_version_match = borrowed_string(self.if_version_match, "if_version_match")?; + options.if_version_not_match = + borrowed_string(self.if_version_not_match, "if_version_not_match")?; + options.if_not_changed = borrowed_metadata(self.if_not_changed)?; + options.source_version = borrowed_string(self.source_version, "source_version")?; + options.source_content_length_hint = (self.source_content_length_hint_has_value != 0) + .then_some(self.source_content_length_hint); + validate_positive_integer(self.concurrent, "copy concurrent")?; + options.concurrent = self.concurrent; + if self.chunk_has_value != 0 { + validate_positive_integer(self.chunk, "copy chunk")?; + options.chunk = Some(self.chunk); + } + + Ok(options) } +} - if let Some(concurrent) = parse_number::(values, "concurrent")? { - validate_write_concurrent(concurrent)?; - options.concurrent = concurrent; +/// The mirror of default options, as the C# class sends it unchanged. +#[cfg(test)] +impl Default for OpendalCopyOptions { + fn default() -> Self { + Self { + if_not_exists: 0, + if_match: std::ptr::null(), + if_none_match: std::ptr::null(), + if_version_match: std::ptr::null(), + if_version_not_match: std::ptr::null(), + if_not_changed: std::ptr::null(), + source_version: std::ptr::null(), + source_content_length_hint_has_value: 0, + source_content_length_hint: 0, + concurrent: 1, + chunk_has_value: 0, + chunk: 0, + } } +} - if let Some(chunk) = parse_number::(values, "chunk")? { - validate_write_chunk(chunk)?; - options.chunk = Some(chunk); - } +/// FFI representation of `opendal::options::RenameOptions`. +#[repr(C)] +pub struct OpendalRenameOptions { + pub if_not_exists: u8, +} - let user_metadata = values - .iter() - .filter_map(|(key, value)| { - key.strip_prefix("user_metadata.") - .map(|k| (k.to_string(), value.to_string())) - }) - .collect::>(); +impl NativeOptions for OpendalRenameOptions { + type Options = opendal::options::RenameOptions; - if !user_metadata.is_empty() { - options.user_metadata = Some(user_metadata); - } + fn to_options(&self) -> Result { + let mut options = Self::Options::default(); + options.if_not_exists = self.if_not_exists != 0; - Ok(options) + Ok(options) + } } -pub fn parse_stat_options( - values: &HashMap, -) -> Result { - Ok(opendal::options::StatOptions { - version: parse_string(values, "version"), - if_match: parse_string(values, "if_match"), - if_none_match: parse_string(values, "if_none_match"), - if_modified_since: parse_timestamp(values, "if_modified_since")?, - if_unmodified_since: parse_timestamp(values, "if_unmodified_since")?, - override_content_type: parse_string(values, "override_content_type"), - override_cache_control: parse_string(values, "override_cache_control"), - override_content_disposition: parse_string(values, "override_content_disposition"), - ..Default::default() - }) +/// The mirror of default options, as the C# class sends it unchanged. +#[cfg(test)] +impl Default for OpendalRenameOptions { + fn default() -> Self { + Self { if_not_exists: 0 } + } } -pub fn parse_list_options( - values: &HashMap, -) -> Result { - let mut options = opendal::options::ListOptions::default(); +#[cfg(test)] +mod tests { + use std::ffi::{CStr, CString}; + use std::ptr; - if let Some(recursive) = parse_bool(values, "recursive")? { - options.recursive = recursive; + use opendal::raw::Timestamp; + + use super::*; + + fn message(error: &OpenDALError) -> String { + unsafe { CStr::from_ptr(error.message) } + .to_string_lossy() + .into_owned() } - if let Some(limit) = parse_number::(values, "limit")? { - validate_list_limit(limit)?; - options.limit = Some(limit); + #[test] + fn read_options_defaults_on_null() { + let options = read_options::(ptr::null()).unwrap(); + assert_eq!(options, opendal::options::ReadOptions::default()); } - options.start_after = parse_string(values, "start_after"); + #[test] + fn read_options_matches_core_defaults() { + // C# sends a count of 1 where core defaults to 0; core reads both as + // no concurrency. + let options = read_options(&OpendalReadOptions::default()).unwrap(); + assert_eq!( + options, + opendal::options::ReadOptions { + concurrent: 1, + ..Default::default() + } + ); + } - if let Some(versions) = parse_bool(values, "versions")? { - options.versions = versions; + #[test] + fn read_options_copies_borrowed_values() { + let etag = CString::new("\"e\"").unwrap(); + let native = OpendalReadOptions { + offset: 4, + length_has_value: 1, + length: 6, + if_match: etag.as_ptr(), + if_modified_since_has_value: 1, + if_modified_since_second: 1_700_000_000, + if_modified_since_nanosecond: 123, + chunk_has_value: 1, + chunk: 1024, + ..Default::default() + }; + let options = read_options(&native).unwrap(); + drop(etag); + + assert_eq!(options.range, BytesRange::from(4..10)); + assert_eq!(options.if_match.as_deref(), Some("\"e\"")); + assert_eq!( + options.if_modified_since, + Some(Timestamp::new(1_700_000_000, 123).unwrap()) + ); + assert_eq!(options.chunk, Some(1024)); } - if let Some(deleted) = parse_bool(values, "deleted")? { - options.deleted = deleted; + #[test] + fn read_options_rejects_suffix_with_offset() { + let native = OpendalReadOptions { + offset: 1, + suffix_length_has_value: 1, + suffix_length: 1, + ..Default::default() + }; + let err = read_options(&native).expect_err("a suffix range cannot have an offset"); + assert!(message(&err).contains("suffix_length")); } - Ok(options) -} + #[test] + fn read_options_rejects_zero_concurrent() { + let native = OpendalReadOptions { + concurrent: 0, + ..Default::default() + }; + let err = read_options(&native).expect_err("concurrent must be positive"); + assert!(message(&err).contains("read concurrent")); + } -pub fn parse_delete_options( - values: &HashMap, -) -> Result { - Ok(opendal::options::DeleteOptions { - version: parse_string(values, "version"), - recursive: parse_bool(values, "recursive")?.unwrap_or(false), - if_match: parse_string(values, "if_match"), - ..Default::default() - }) + #[test] + fn read_options_names_invalid_utf8() { + let invalid = [0xffu8, 0]; + let native = OpendalReadOptions { + version: invalid.as_ptr().cast(), + ..Default::default() + }; + let err = read_options(&native).expect_err("invalid UTF-8 must fail"); + assert!(message(&err).contains("version")); + } + + #[test] + fn write_options_rebuild_user_metadata_and_metadata() { + let key = CString::new("k").unwrap(); + let value = CString::new("v").unwrap(); + let keys = [key.as_ptr()]; + let values = [value.as_ptr()]; + let etag = CString::new("e").unwrap(); + + // Metadata without values has only null strings, so replacing one + // with a borrowed pointer neither overwrites nor leaks owned memory. + let mut metadata = + OpendalMetadata::from_metadata(opendal::MetadataBuilder::file(0).build()); + metadata.content_length = 5; + metadata.etag = etag.as_ptr().cast_mut(); + + let native = OpendalWriteOptions { + user_metadata_has_value: 1, + user_metadata_keys: keys.as_ptr(), + user_metadata_values: values.as_ptr(), + user_metadata_len: 1, + if_not_changed: &metadata, + ..Default::default() + }; + let options = read_options(&native).unwrap(); + + assert_eq!(options.user_metadata.unwrap()["k"], "v"); + let rebuilt = options.if_not_changed.unwrap(); + assert!(rebuilt.is_file()); + assert_eq!(rebuilt.content_length(), 5); + assert_eq!(rebuilt.etag(), Some("e")); + } + + #[test] + fn write_options_keep_empty_user_metadata() { + let native = OpendalWriteOptions { + user_metadata_has_value: 1, + ..Default::default() + }; + let options = read_options(&native).unwrap(); + assert_eq!(options.user_metadata, Some(Default::default())); + } + + #[test] + fn write_options_name_null_user_metadata_value() { + let key = CString::new("k").unwrap(); + let keys = [key.as_ptr()]; + let values = [ptr::null::()]; + let native = OpendalWriteOptions { + user_metadata_has_value: 1, + user_metadata_keys: keys.as_ptr(), + user_metadata_values: values.as_ptr(), + user_metadata_len: 1, + ..Default::default() + }; + let err = read_options(&native).expect_err("a null value must fail"); + assert!(message(&err).contains("value of user_metadata k is null")); + } } diff --git a/bindings/dotnet/src/result.rs b/bindings/dotnet/src/result.rs index 6bde8b8a883c..3cbf682ecd6a 100644 --- a/bindings/dotnet/src/result.rs +++ b/bindings/dotnet/src/result.rs @@ -64,13 +64,6 @@ pub struct OpendalOperatorResult { pub error: OpenDALError, } -#[repr(C)] -/// Result for operations returning an options handle pointer. -pub struct OpendalOptionsResult { - pub ptr: *mut c_void, - pub error: OpenDALError, -} - #[repr(C)] /// Result for operations returning an executor handle pointer. pub struct OpendalExecutorResult { @@ -176,12 +169,6 @@ define_result!( error_value = std::ptr::null_mut() ); -define_result!( - OpendalOptionsResult, - field = ptr: *mut c_void, - error_value = std::ptr::null_mut() -); - define_result!( OpendalExecutorResult, field = ptr: *mut c_void, diff --git a/bindings/dotnet/src/utils.rs b/bindings/dotnet/src/utils.rs index 5bd80eb8344e..a9a63a86f765 100644 --- a/bindings/dotnet/src/utils.rs +++ b/bindings/dotnet/src/utils.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::os::raw::c_char; -use std::{collections::HashMap, ffi::CStr}; use crate::error::{ErrorCode, OpenDALError}; @@ -37,10 +37,6 @@ pub fn invalid_utf8_message(field: &str) -> String { format!("{field} is null or invalid UTF-8") } -pub fn invalid_utf8_message_at(field: &str, index: usize) -> String { - format!("{field} at index {index} is null or invalid UTF-8") -} - pub fn require_cstr<'a>(value: *const c_char, field: &str) -> Result<&'a str, OpenDALError> { cstr_to_str(value).ok_or_else(|| { OpenDALError::from_error(ErrorCode::ConfigInvalid, invalid_utf8_message(field)) @@ -69,55 +65,6 @@ pub fn require_data_ptr(data: *const u8, len: usize) -> Result<(), OpenDALError> Ok(()) } -/// # Safety -/// -/// - When `len > 0`, `keys` and `values` must be non-null pointers to arrays -/// containing at least `len` C-string pointers. -/// - Each entry pointer must be non-null and valid UTF-8. -pub unsafe fn collect_options( - keys: *const *const c_char, - values: *const *const c_char, - len: usize, -) -> Result, OpenDALError> { - if len == 0 { - return Ok(HashMap::new()); - } - - if keys.is_null() { - return Err(config_invalid_error("keys pointer is null while len > 0")); - } - - if values.is_null() { - return Err(config_invalid_error("values pointer is null while len > 0")); - } - - let mut map = HashMap::with_capacity(len); - for index in 0..len { - let key_ptr = unsafe { *keys.add(index) }; - let value_ptr = unsafe { *values.add(index) }; - - if key_ptr.is_null() { - return Err(config_invalid_error(invalid_utf8_message_at("key", index))); - } - if value_ptr.is_null() { - return Err(config_invalid_error(invalid_utf8_message_at( - "value", index, - ))); - } - - let key = unsafe { CStr::from_ptr(key_ptr) } - .to_str() - .map_err(|_| config_invalid_error(invalid_utf8_message_at("key", index)))?; - let value = unsafe { CStr::from_ptr(value_ptr) } - .to_str() - .map_err(|_| config_invalid_error(invalid_utf8_message_at("value", index)))?; - - map.insert(key.to_string(), value.to_string()); - } - - Ok(map) -} - pub fn into_string_ptr(message: impl Into) -> *mut c_char { match std::ffi::CString::new(message.into()) { Ok(msg) => msg.into_raw(), @@ -236,8 +183,90 @@ pub unsafe fn release_string_pairs( *len = 0; } +/// Copy a borrowed C string into an owned `String`, where null is `None`. +/// +/// `value` must be null or a valid null-terminated string. +pub fn borrowed_string(value: *const c_char, field: &str) -> Result, OpenDALError> { + if value.is_null() { + return Ok(None); + } + + cstr_to_str(value) + .map(|value| Some(value.to_owned())) + .ok_or_else(|| config_invalid_error(invalid_utf8_message(field))) +} + +/// Rebuild a timestamp flattened by `optional_timestamp`. +pub fn borrowed_timestamp( + has_value: u8, + second: i64, + nanosecond: i32, +) -> Result, OpenDALError> { + if has_value == 0 { + return Ok(None); + } + + opendal::raw::Timestamp::new(second, nanosecond) + .map(Some) + .map_err(OpenDALError::from_opendal_error) +} + +/// Copy borrowed parallel arrays of `len` key and value C strings into a map. +/// +/// A null value is reported by key, so callers can tell which entry was +/// unset. When `len > 0`, `keys` and `values` must be valid arrays of `len` +/// pointers, each null or a valid null-terminated string. +pub fn borrowed_string_map( + keys: *const *const c_char, + values: *const *const c_char, + len: usize, + field: &str, +) -> Result, OpenDALError> { + if len == 0 { + return Ok(HashMap::new()); + } + if keys.is_null() || values.is_null() { + return Err(config_invalid_error(format!( + "{field} arrays are null while len > 0" + ))); + } + + let keys = unsafe { std::slice::from_raw_parts(keys, len) }; + let values = unsafe { std::slice::from_raw_parts(values, len) }; + let mut map = HashMap::with_capacity(len); + for (&key, &value) in keys.iter().zip(values) { + let key = require_cstr(key, &format!("{field} key"))?; + if value.is_null() { + return Err(config_invalid_error(format!( + "value of {field} {key} is null" + ))); + } + let value = require_cstr(value, &format!("value of {field} {key}"))?; + map.insert(key.to_owned(), value.to_owned()); + } + Ok(map) +} + +/// Rebuild optional string pairs flattened by `string_pairs`, borrowing the +/// arrays as `borrowed_string_map` does. +pub fn borrowed_string_pairs( + has_value: u8, + keys: *const *const c_char, + values: *const *const c_char, + len: usize, + field: &str, +) -> Result>, OpenDALError> { + if has_value == 0 { + return Ok(None); + } + + borrowed_string_map(keys, values, len, field).map(Some) +} + #[cfg(test)] mod tests { + use std::ffi::CString; + use super::*; #[test] @@ -282,4 +311,19 @@ mod tests { assert_eq!((has_value, len), (1, 0)); assert!(keys.is_null()); } + + #[test] + fn borrowed_string_map_copies_and_names_null_values() { + let key = CString::new("root").unwrap(); + let value = CString::new("/tmp").unwrap(); + let keys = [key.as_ptr()]; + let values = [value.as_ptr()]; + let map = borrowed_string_map(keys.as_ptr(), values.as_ptr(), 1, "option").unwrap(); + assert_eq!(map["root"], "/tmp"); + + let nulls = [std::ptr::null()]; + let err = borrowed_string_map(keys.as_ptr(), nulls.as_ptr(), 1, "option").unwrap_err(); + let message = cstr_to_str(err.message).unwrap(); + assert!(message.contains("value of option root is null")); + } } diff --git a/bindings/dotnet/src/validators.rs b/bindings/dotnet/src/validators.rs index 9c9912b3c086..63a4932cbecf 100644 --- a/bindings/dotnet/src/validators.rs +++ b/bindings/dotnet/src/validators.rs @@ -58,8 +58,5 @@ pub(crate) mod prelude { validate_concurrent_limit_options, validate_retry_options, validate_throttle_options, validate_timeout_options, }; - pub(crate) use super::options::{ - validate_list_limit, validate_read_chunk, validate_read_concurrent, validate_read_gap, - validate_read_range_end, validate_write_chunk, validate_write_concurrent, - }; + pub(crate) use super::options::validate_read_range_end; } diff --git a/bindings/dotnet/src/validators/options.rs b/bindings/dotnet/src/validators/options.rs index 325fde62b85b..699d5d92745c 100644 --- a/bindings/dotnet/src/validators/options.rs +++ b/bindings/dotnet/src/validators/options.rs @@ -18,7 +18,7 @@ //! Validators for operation option payloads. use crate::error::OpenDALError; -use crate::validators::{validate_checked_add_u64, validate_positive_integer}; +use crate::validators::validate_checked_add_u64; pub fn validate_read_range_end( offset: u64, @@ -32,27 +32,3 @@ pub fn validate_read_range_end( Ok(Some(end)) } - -pub fn validate_read_concurrent(concurrent: usize) -> Result<(), OpenDALError> { - validate_positive_integer(concurrent, "read concurrent") -} - -pub fn validate_read_chunk(chunk: usize) -> Result<(), OpenDALError> { - validate_positive_integer(chunk, "read chunk") -} - -pub fn validate_read_gap(gap: usize) -> Result<(), OpenDALError> { - validate_positive_integer(gap, "read gap") -} - -pub fn validate_write_concurrent(concurrent: usize) -> Result<(), OpenDALError> { - validate_positive_integer(concurrent, "write concurrent") -} - -pub fn validate_write_chunk(chunk: usize) -> Result<(), OpenDALError> { - validate_positive_integer(chunk, "write chunk") -} - -pub fn validate_list_limit(limit: usize) -> Result<(), OpenDALError> { - validate_positive_integer(limit, "list limit") -} diff --git a/dev/src/generate/core_options.rs b/dev/src/generate/core_options.rs new file mode 100644 index 000000000000..f16af4d650bd --- /dev/null +++ b/dev/src/generate/core_options.rs @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Parser for `core/core/src/types/options.rs`. +//! +//! Reads the public `*Options` structs from opendal core and classifies each +//! field type into the handful of shapes binding generators know how to +//! carry, so bindings can mirror the operation options from the single source +//! of truth instead of hand-maintaining copies. + +use anyhow::{Context, Result, bail}; +use serde::Serialize; +use std::fs; +use std::path::Path; +use syn::{GenericArgument, Item, ItemStruct, PathArguments, Type, TypePath, Visibility}; + +use super::options::doc_lines; + +/// The shape of one option field. +/// +/// Only the shapes that actually occur in core are supported. Anything else +/// fails parsing so that a new field shape upstream breaks the generator +/// loudly instead of producing a silently wrong mirror. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum OptionKind { + /// `bool` + Bool, + /// `usize` + Usize, + /// `Option` + OptionUsize, + /// `Option` + OptionU64, + /// `Option` + OptionString, + /// `Option` + OptionTimestamp, + /// `Option>` + OptionStringMap, + /// `Option` + OptionMetadata, + /// `BytesRange` + BytesRange, +} + +/// One field of a core options struct. +#[derive(Debug, Clone)] +pub struct OptionField { + /// Field name in snake_case, exactly as written in core. + pub name: String, + /// The shape of the field type. + pub kind: OptionKind, + /// Doc comment of the field, one entry per line. + pub doc: Vec, +} + +/// One public `*Options` struct of core. +#[derive(Debug, Clone)] +pub struct OptionsStruct { + /// The struct name, e.g. `ReadOptions`. + pub name: String, + /// Doc comment of the struct, one entry per line. + pub doc: Vec, + /// The fields in declaration order. + pub fields: Vec, +} + +/// Parse every public `*Options` struct out of the given core source file. +pub fn parse(path: &Path) -> Result> { + let content = + fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; + let ast = syn::parse_file(&content)?; + + let mut structs = Vec::new(); + for item in ast.items { + if let Item::Struct(struct_) = item + && matches!(struct_.vis, Visibility::Public(_)) + && struct_.ident.to_string().ends_with("Options") + { + structs.push(parse_struct(&struct_)?); + } + } + Ok(structs) +} + +fn parse_struct(struct_: &ItemStruct) -> Result { + let name = struct_.ident.to_string(); + let syn::Fields::Named(named) = &struct_.fields else { + bail!("options struct {name} must have named fields"); + }; + + let mut fields = Vec::with_capacity(named.named.len()); + for field in &named.named { + let field_name = field + .ident + .as_ref() + .expect("named field always has an ident") + .to_string(); + let Some(kind) = classify(&field.ty) else { + bail!( + "option field `{name}::{field_name}` has an unsupported type; \ + teach dev/src/generate/core_options.rs about it before regenerating" + ); + }; + fields.push(OptionField { + name: field_name, + kind, + doc: doc_lines(&field.attrs), + }); + } + + Ok(OptionsStruct { + doc: doc_lines(&struct_.attrs), + name, + fields, + }) +} + +fn classify(ty: &Type) -> Option { + let segment = last_segment(ty)?; + match segment.ident.to_string().as_str() { + "bool" => Some(OptionKind::Bool), + "usize" => Some(OptionKind::Usize), + "BytesRange" => Some(OptionKind::BytesRange), + "Option" => { + let PathArguments::AngleBracketed(args) = &segment.arguments else { + return None; + }; + let Some(GenericArgument::Type(inner)) = args.args.first() else { + return None; + }; + match last_segment(inner)?.ident.to_string().as_str() { + "usize" => Some(OptionKind::OptionUsize), + "u64" => Some(OptionKind::OptionU64), + "String" => Some(OptionKind::OptionString), + "Timestamp" => Some(OptionKind::OptionTimestamp), + "HashMap" => Some(OptionKind::OptionStringMap), + "Metadata" => Some(OptionKind::OptionMetadata), + _ => None, + } + } + _ => None, + } +} + +fn last_segment(ty: &Type) -> Option<&syn::PathSegment> { + let Type::Path(TypePath { path, .. }) = ty else { + return None; + }; + path.segments.last() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_core_options() { + let path = crate::workspace_dir().join("core/core/src/types/options.rs"); + let structs = parse(&path).expect("core options must parse"); + + let names: Vec<&str> = structs.iter().map(|s| s.name.as_str()).collect(); + for expected in [ + "ReadOptions", + "WriteOptions", + "StatOptions", + "ListOptions", + "DeleteOptions", + "CopyOptions", + "RenameOptions", + ] { + assert!(names.contains(&expected), "missing {expected}"); + } + + let read = structs.iter().find(|s| s.name == "ReadOptions").unwrap(); + let kind = |name: &str| read.fields.iter().find(|f| f.name == name).unwrap().kind; + assert_eq!(kind("range"), OptionKind::BytesRange); + assert_eq!(kind("version"), OptionKind::OptionString); + assert_eq!(kind("if_modified_since"), OptionKind::OptionTimestamp); + assert_eq!(kind("content_length_hint"), OptionKind::OptionU64); + assert_eq!(kind("concurrent"), OptionKind::Usize); + assert_eq!(kind("chunk"), OptionKind::OptionUsize); + + let write = structs.iter().find(|s| s.name == "WriteOptions").unwrap(); + let kind = |name: &str| write.fields.iter().find(|f| f.name == name).unwrap().kind; + assert_eq!(kind("append"), OptionKind::Bool); + assert_eq!(kind("user_metadata"), OptionKind::OptionStringMap); + assert_eq!(kind("if_not_changed"), OptionKind::OptionMetadata); + assert!(!write.fields[0].doc.is_empty()); + } +} diff --git a/dev/src/generate/dotnet.rs b/dev/src/generate/dotnet.rs index 5cb57af7b80d..a24adc07aee0 100644 --- a/dev/src/generate/dotnet.rs +++ b/dev/src/generate/dotnet.rs @@ -31,6 +31,13 @@ //! The typed `*ServiceConfig` classes mirror each service's config struct and //! are rendered from the same parsed service definitions the Java and Python //! generators consume. +//! +//! The operation options cross the FFI boundary as one `#[repr(C)]` struct +//! per call, which native code only borrows and converts into the core +//! options struct before the call returns. The public C# classes, their +//! interop structs, and the Rust mirrors are all rendered from +//! `core/core/src/types/options.rs`, so a new core option only needs a +//! regeneration. use anyhow::{Result, bail}; use minijinja::{Environment, context}; @@ -39,6 +46,7 @@ use std::fs; use std::path::{Path, PathBuf}; use super::capability; +use super::core_options::{self, OptionField, OptionKind, OptionsStruct}; use super::metadata::{self, MetadataKind}; use super::parser::{Config, ConfigType, Services, sorted_services}; @@ -87,6 +95,13 @@ struct MetadataField { cs_type: String, /// Expression that reads the public value from the interop struct's fields. cs_read: String, + /// Statement that applies the borrowed mirror fields to a + /// `MetadataBuilder`, empty for the members the builder is constructed + /// from. + rs_rebuild: String, + /// C# statement that fills the interop fields from the public value when + /// metadata travels back to native code. + cs_write: String, } /// One field of the mirrors. An accessor with an optional or composite @@ -123,9 +138,75 @@ struct ConfigField { doc: String, } +/// Render model for one public `*Options` class and its native parser. +#[derive(Serialize)] +struct OptionsClass { + /// The class name, identical to the core struct name, e.g. `ReadOptions`. + name: String, + /// snake_case operation used in native validation messages, e.g. `read`. + operation: String, + /// Class summary, already rendered as C# XML doc text. + summary: String, + /// The public properties in core field order. + properties: Vec, + /// C# statements that validate the properties before crossing the FFI. + validations: Vec, + /// Fields of the `#[repr(C)]` mirror and its C# interop struct, in order. + parts: Vec, + /// C# statements that fill the interop struct from the properties. + cs_writes: Vec, + /// Rust statements that convert the mirror into core options. + rs_convert: Vec, +} + +/// One field of an options mirror. An optional or composite value flattens +/// into several, following the metadata mirror's `*_has_value` convention. +#[derive(Serialize)] +struct OptionPart { + /// snake_case field name in the Rust mirror. + name: String, + /// PascalCase field name in the C# interop struct. + pascal: String, + /// Field type in the Rust `#[repr(C)]` mirror. + rs_type: String, + /// Field type in the C# interop struct. + cs_type: String, + /// The field's value in the default mirror, matching the C# defaults. + rs_default: String, +} + +/// One public property of an `*Options` class. A core field usually maps to +/// one property; a `BytesRange` flattens into several. +#[derive(Serialize)] +struct OptionProperty { + /// PascalCase property name. + pascal: String, + /// The C# property type. + cs_type: String, + /// Property initializer including the leading ` = `, empty when none. + initializer: String, + /// Property summary, already rendered as C# XML doc text. + summary: String, + /// `cref` targets of the capabilities the core doc refers to. + see_also: Vec, +} + +/// The core option structs the binding exposes, with the snake_case +/// operation their native validation messages name. +const EXPOSED_OPTIONS: &[(&str, &str)] = &[ + ("ReadOptions", "read"), + ("WriteOptions", "write"), + ("StatOptions", "stat"), + ("ListOptions", "list"), + ("DeleteOptions", "delete"), + ("CopyOptions", "copy"), + ("RenameOptions", "rename"), +]; + pub fn generate(workspace_dir: PathBuf, services: Services) -> Result<()> { generate_capability(&workspace_dir)?; generate_metadata(&workspace_dir)?; + generate_options(&workspace_dir)?; generate_service_configs(&workspace_dir, services) } @@ -173,6 +254,7 @@ fn generate_metadata(workspace_dir: &Path) -> Result<()> { .into_iter() .map(metadata_field) .collect(); + validate_metadata_rebuild(&fields, &metadata::parse_builder_setters(&source)?)?; let mut env = Environment::new(); env.add_template("metadata_rs", include_str!("dotnet/metadata_rs.j2"))?; @@ -202,6 +284,43 @@ fn generate_metadata(workspace_dir: &Path) -> Result<()> { Ok(()) } +/// Check that native code can rebuild every metadata accessor. +/// +/// Each accessor needs a `MetadataBuilder` setter of the same name, except +/// `mode` and `content_length`, which the template passes to +/// `MetadataBuilder::file`. Anything else fails loudly, so a new accessor +/// upstream is never silently dropped on the way back to native code. +fn validate_metadata_rebuild(fields: &[MetadataField], setters: &[String]) -> Result<()> { + for required in ["mode", "content_length"] { + if !fields.iter().any(|f| f.name == required) { + bail!( + "metadata accessor `{required}` is gone; \ + update the builder construction in dotnet/metadata_rs.j2" + ); + } + } + + for f in fields { + if f.rs_rebuild.is_empty() { + if !matches!(f.name.as_str(), "mode" | "content_length") { + bail!( + "metadata accessor `{}` is not rebuilt; \ + teach dev/src/generate/dotnet.rs how to apply it", + f.name + ); + } + } else if !setters.contains(&f.name) { + bail!( + "metadata accessor `{}` has no `MetadataBuilder` setter of the same name; \ + teach dev/src/generate/dotnet.rs how to rebuild it", + f.name + ); + } + } + + Ok(()) +} + /// The C# name for a metadata accessor, allowing the established casing of /// names that predate codegen. fn metadata_pascal(name: &str) -> String { @@ -297,6 +416,53 @@ fn metadata_field(f: metadata::MetadataField) -> MetadataField { ), }; + // Metadata travels back to native code in the same struct, for options + // such as `if_not_changed`, with every pointer borrowed from the caller. + // `mode` and `content_length` have no setter of their own: the template + // rebuilds them through `MetadataBuilder::file`. + let (rs_rebuild, cs_write) = match f.kind { + MetadataKind::Mode => ( + String::new(), + format!("native.{pascal} = (int)metadata.{pascal};"), + ), + MetadataKind::U64 => ( + String::new(), + format!("native.{pascal} = metadata.{pascal};"), + ), + MetadataKind::Bool => ( + format!("builder.{name}(self.{name} != 0);"), + format!("native.{pascal} = metadata.{pascal} ? (byte)1 : (byte)0;"), + ), + MetadataKind::OptionBool => ( + format!("builder.{name}((self.{name}_has_value != 0).then_some(self.{name} != 0));"), + format!( + "if (metadata.{pascal} is {{ }} {camel})\n {{\n native.{pascal}HasValue = 1;\n native.{pascal} = {camel} ? (byte)1 : (byte)0;\n }}" + ), + ), + MetadataKind::OptionStr => ( + format!( + "if let Some(value) = crate::utils::borrowed_string(self.{name}, \"{name}\")? {{\n builder.{name}(value);\n }}" + ), + format!("native.{pascal} = writer.Utf8(metadata.{pascal});"), + ), + MetadataKind::OptionTimestamp => ( + format!( + "if let Some(value) = crate::utils::borrowed_timestamp(\n self.{name}_has_value,\n self.{name}_second,\n self.{name}_nanosecond,\n )? {{\n builder.{name}(value);\n }}" + ), + format!( + "if (metadata.{pascal} is {{ }} {camel})\n {{\n native.{pascal}HasValue = 1;\n (native.{pascal}Second, native.{pascal}Nanosecond) = Utilities.ToUnixSecondAndNanosecond({camel});\n }}" + ), + ), + MetadataKind::OptionUserMetadata => ( + format!( + "if let Some(values) = crate::utils::borrowed_string_pairs(\n self.{name}_has_value,\n self.{name}_keys as *const *const c_char,\n self.{name}_values as *const *const c_char,\n self.{name}_len,\n \"{name}\",\n )? {{\n builder.{name}(values);\n }}" + ), + format!( + "(native.{pascal}HasValue, native.{pascal}Keys, native.{pascal}Values, native.{pascal}Len) = writer.Pairs(metadata.{pascal});" + ), + ), + }; + let rs_bind = match parts.as_slice() { [only] => only.name.clone(), many => { @@ -314,6 +480,8 @@ fn metadata_field(f: metadata::MetadataField) -> MetadataField { rs_release, cs_type: cs_type.to_string(), cs_read, + rs_rebuild, + cs_write, pascal, camel, doc: f.doc, @@ -321,6 +489,389 @@ fn metadata_field(f: metadata::MetadataField) -> MetadataField { } } +fn generate_options(workspace_dir: &Path) -> Result<()> { + let source = workspace_dir.join("core/core/src/types/options.rs"); + let structs = core_options::parse(&source)?; + + let mut classes = Vec::with_capacity(EXPOSED_OPTIONS.len()); + for (name, operation) in EXPOSED_OPTIONS { + let Some(options) = structs.iter().find(|s| s.name == *name) else { + bail!("options struct {name} not found in {}", source.display()); + }; + classes.push(options_class(options, operation)); + } + + let mut env = Environment::new(); + env.add_template("options_rs", include_str!("dotnet/options_rs.j2"))?; + env.add_template("options_cs", include_str!("dotnet/options_cs.j2"))?; + + let rendered = env + .get_template("options_rs")? + .render(context! { classes => classes })?; + write_rendered( + &workspace_dir.join("bindings/dotnet/src/options.rs"), + rendered, + )?; + + let tmpl = env.get_template("options_cs")?; + for class in &classes { + let rendered = tmpl.render(context! { class => class })?; + let output = + workspace_dir.join(format!("bindings/dotnet/OpenDAL/Options/{}.cs", class.name)); + write_rendered(&output, rendered)?; + } + + Ok(()) +} + +/// Precompute every per-kind decision for one options struct. +/// +/// This table is the only place that knows how an option shape crosses the +/// FFI boundary: the C# properties it surfaces as, how C# validates them and +/// fills the interop struct, and how the Rust mirror declares and converts +/// them. A new shape needs one arm here; fields that reuse a known shape need +/// nothing. +/// +/// Counts (`usize` and `Option`) must be positive, matching every +/// count core exposes today. +fn options_class(options: &OptionsStruct, operation: &str) -> OptionsClass { + let ctx = DocContext { + fields: &options.fields, + }; + + let mut properties = Vec::new(); + let mut validations = Vec::new(); + let mut parts = Vec::new(); + let mut cs_writes = Vec::new(); + let mut rs_convert = Vec::new(); + + for f in &options.fields { + let key = f.name.as_str(); + let pascal = heck::AsUpperCamelCase(key).to_string(); + let camel = heck::AsLowerCamelCase(key).to_string(); + let property = |cs_type: &str, initializer: &str| OptionProperty { + pascal: pascal.clone(), + cs_type: cs_type.to_string(), + initializer: initializer.to_string(), + summary: ctx.summary(&f.doc), + see_also: capability_refs(&f.doc), + }; + let part = |name: &str, rs_type: &str, cs_type: &str| OptionPart { + name: name.to_string(), + pascal: heck::AsUpperCamelCase(name).to_string(), + rs_type: rs_type.to_string(), + cs_type: cs_type.to_string(), + rs_default: if rs_type.starts_with('*') { + "std::ptr::null()".to_string() + } else { + "0".to_string() + }, + }; + let has_value = format!("{key}_has_value"); + + match f.kind { + OptionKind::Bool => { + properties.push(property("bool", "")); + parts.push(part(key, "u8", "byte")); + cs_writes.push(format!("native.{pascal} = {pascal} ? (byte)1 : (byte)0;")); + rs_convert.push(format!("options.{key} = self.{key} != 0;")); + } + OptionKind::Usize => { + properties.push(property("int", " = 1;")); + validations.push(format!( + "OptionValidators.RequireGreaterThanZero({pascal}, nameof({pascal}));" + )); + parts.push(OptionPart { + rs_default: "1".to_string(), + ..part(key, "usize", "nuint") + }); + cs_writes.push(format!("native.{pascal} = (nuint){pascal};")); + rs_convert.push(format!( + "validate_positive_integer(self.{key}, \"{operation} {key}\")?;\n options.{key} = self.{key};" + )); + } + OptionKind::OptionUsize => { + properties.push(property("long?", "")); + validations.push(format!( + "OptionValidators.RequireNullableGreaterThanZero({pascal}, nameof({pascal}));" + )); + parts.push(part(&has_value, "u8", "byte")); + parts.push(part(key, "usize", "nuint")); + cs_writes.push(format!( + "if ({pascal} is {{ }} {camel})\n {{\n native.{pascal}HasValue = 1;\n native.{pascal} = (nuint){camel};\n }}" + )); + rs_convert.push(format!( + "if self.{has_value} != 0 {{\n validate_positive_integer(self.{key}, \"{operation} {key}\")?;\n options.{key} = Some(self.{key});\n }}" + )); + } + OptionKind::OptionU64 => { + properties.push(property("long?", "")); + validations.push(format!( + "OptionValidators.RequireNullableGreaterThanOrEqualZero({pascal}, nameof({pascal}));" + )); + parts.push(part(&has_value, "u8", "byte")); + parts.push(part(key, "u64", "ulong")); + cs_writes.push(format!( + "if ({pascal} is {{ }} {camel})\n {{\n native.{pascal}HasValue = 1;\n native.{pascal} = (ulong){camel};\n }}" + )); + rs_convert.push(format!( + "options.{key} = (self.{has_value} != 0).then_some(self.{key});" + )); + } + OptionKind::OptionString => { + properties.push(property("string?", "")); + parts.push(part(key, "*const c_char", "IntPtr")); + cs_writes.push(format!("native.{pascal} = writer.String({pascal});")); + rs_convert.push(format!( + "options.{key} = borrowed_string(self.{key}, \"{key}\")?;" + )); + } + OptionKind::OptionTimestamp => { + properties.push(property("DateTimeOffset?", "")); + parts.push(part(&has_value, "u8", "byte")); + parts.push(part(&format!("{key}_second"), "i64", "long")); + parts.push(part(&format!("{key}_nanosecond"), "i32", "int")); + cs_writes.push(format!( + "if ({pascal} is {{ }} {camel})\n {{\n native.{pascal}HasValue = 1;\n (native.{pascal}Second, native.{pascal}Nanosecond) = Utilities.ToUnixSecondAndNanosecond({camel});\n }}" + )); + rs_convert.push(format!( + "options.{key} =\n borrowed_timestamp(self.{has_value}, self.{key}_second, self.{key}_nanosecond)?;" + )); + } + OptionKind::OptionStringMap => { + properties.push(property("IReadOnlyDictionary?", "")); + parts.push(part(&has_value, "u8", "byte")); + parts.push(part(&format!("{key}_keys"), "*const *const c_char", "IntPtr")); + parts.push(part(&format!("{key}_values"), "*const *const c_char", "IntPtr")); + parts.push(part(&format!("{key}_len"), "usize", "nuint")); + cs_writes.push(format!( + "(native.{pascal}HasValue, native.{pascal}Keys, native.{pascal}Values, native.{pascal}Len) = writer.Pairs({pascal});" + )); + rs_convert.push(format!( + "options.{key} = borrowed_string_pairs(\n self.{has_value},\n self.{key}_keys,\n self.{key}_values,\n self.{key}_len,\n \"{key}\",\n )?;" + )); + } + OptionKind::OptionMetadata => { + properties.push(property("Metadata?", "")); + parts.push(part(key, "*const OpendalMetadata", "IntPtr")); + cs_writes.push(format!("native.{pascal} = writer.MetadataPointer({pascal});")); + rs_convert.push(format!( + "options.{key} = borrowed_metadata(self.{key})?;" + )); + } + OptionKind::BytesRange => { + // The established surface spells a range as an offset and an + // optional length; a suffix range is a third, exclusive form. + let range = |pascal: &str, cs_type: &str, summary: &str, see_also: &[&str]| { + OptionProperty { + pascal: pascal.to_string(), + cs_type: cs_type.to_string(), + initializer: String::new(), + summary: summary.to_string(), + see_also: see_also.iter().map(|s| s.to_string()).collect(), + } + }; + properties.push(range( + "Offset", + "long", + "The byte offset to start reading from.", + &[], + )); + properties.push(range( + "Length", + "long?", + "The number of bytes to read from , or to read to the end.", + &[], + )); + properties.push(range( + "SuffixLength", + "long?", + "Read the last bytes instead of a range from . It cannot be combined with or .", + &["Capability.ReadWithSuffix"], + )); + validations.extend([ + "OptionValidators.RequireGreaterThanOrEqualZero(Offset, nameof(Offset));" + .to_string(), + "OptionValidators.RequireNullableGreaterThanOrEqualZero(Length, nameof(Length));" + .to_string(), + "OptionValidators.RequireNullableGreaterThanOrEqualZero(SuffixLength, nameof(SuffixLength));" + .to_string(), + "OptionValidators.RequireSuffixExclusive(Offset, Length, SuffixLength);" + .to_string(), + ]); + parts.extend([ + part("offset", "u64", "ulong"), + part("length_has_value", "u8", "byte"), + part("length", "u64", "ulong"), + part("suffix_length_has_value", "u8", "byte"), + part("suffix_length", "u64", "ulong"), + ]); + cs_writes.extend([ + "native.Offset = (ulong)Offset;".to_string(), + "if (Length is { } length)\n {\n native.LengthHasValue = 1;\n native.Length = (ulong)length;\n }" + .to_string(), + "if (SuffixLength is { } suffixLength)\n {\n native.SuffixLengthHasValue = 1;\n native.SuffixLength = (ulong)suffixLength;\n }" + .to_string(), + ]); + rs_convert.push(format!( + "options.{key} = range(\n self.offset,\n (self.length_has_value != 0).then_some(self.length),\n (self.suffix_length_has_value != 0).then_some(self.suffix_length),\n )?;" + )); + } + } + } + + let summary = match ctx.summary(&options.doc) { + s if s.is_empty() => format!("Options for {operation} operations."), + s => s, + }; + + OptionsClass { + name: options.name.clone(), + operation: operation.to_string(), + summary, + properties, + validations, + parts, + cs_writes, + rs_convert, + } +} + +/// Renders rustdoc of one options struct as C# XML doc text. +struct DocContext<'a> { + /// Fields of the struct, so code spans naming them become `cref`s. + fields: &'a [OptionField], +} + +impl DocContext<'_> { + /// The first paragraph of a rustdoc comment as a single line of C# XML + /// doc text. + fn summary(&self, lines: &[String]) -> String { + let paragraph = lines + .iter() + .map(|line| line.trim()) + .take_while(|line| !line.is_empty()) + .collect::>() + .join(" "); + self.inline(¶graph) + } + + /// Translate inline rustdoc markup: intra-doc links, Markdown links, and + /// code spans. Everything else is XML-escaped text. + fn inline(&self, text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(pos) = rest.find(['[', '`']) { + out.push_str(&xml_escape(&rest[..pos])); + rest = &rest[pos..]; + + if let Some(code) = rest.strip_prefix('`') { + match code.find('`') { + Some(end) => { + out.push_str(&self.code(&code[..end])); + rest = &code[end + 1..]; + } + None => { + out.push('`'); + rest = code; + } + } + continue; + } + + let Some(close) = rest.find(']') else { + out.push('['); + rest = &rest[1..]; + continue; + }; + let label = &rest[1..close]; + let after = &rest[close + 1..]; + if let Some(target) = after.strip_prefix('(') + && let Some(end) = target.find(')') + { + out.push_str(&format!( + "{}", + xml_escape(&target[..end]), + self.inline(label) + )); + rest = &target[end + 1..]; + } else if let Some(target) = after.strip_prefix('[') + && let Some(end) = target.find(']') + { + // A reference link points into the Rust docs; keep its text. + out.push_str(&self.inline(label)); + rest = &target[end + 1..]; + } else if let Some(path) = label.strip_prefix('`').and_then(|l| l.strip_suffix('`')) { + out.push_str(&self.path(path)); + rest = after; + } else { + out.push_str(&self.inline(label)); + rest = after; + } + } + out.push_str(&xml_escape(rest)); + out + } + + /// An intra-doc link target, e.g. `crate::Capability::read_with_version`. + fn path(&self, path: &str) -> String { + let path = path.strip_prefix("crate::").unwrap_or(path); + let segments: Vec<&str> = path.split("::").collect(); + match segments.as_slice() { + ["Capability", field] => format!( + "", + heck::AsUpperCamelCase(field) + ), + ["ErrorKind", variant] => format!(""), + ["Self", field] => self.code(field), + [ty] if EXPOSED_OPTIONS.iter().any(|(name, _)| name == ty) => { + format!("") + } + [ty, field] if EXPOSED_OPTIONS.iter().any(|(name, _)| name == ty) => { + format!("", heck::AsUpperCamelCase(field)) + } + _ => format!("{}", xml_escape(path)), + } + } + + /// A code span: a sibling field becomes a `cref` to its property. + fn code(&self, code: &str) -> String { + if self + .fields + .iter() + .any(|f| f.name == code && f.kind != OptionKind::BytesRange) + { + format!("", heck::AsUpperCamelCase(code)) + } else if matches!(code, "true" | "false") { + format!("") + } else { + format!("{}", xml_escape(code)) + } + } +} + +/// The `cref` targets of every `Capability` field a rustdoc comment links to, +/// in order of first mention. +fn capability_refs(lines: &[String]) -> Vec { + const MARKER: &str = "crate::Capability::"; + + let mut refs = Vec::new(); + for line in lines { + let mut rest = line.as_str(); + while let Some(pos) = rest.find(MARKER) { + rest = &rest[pos + MARKER.len()..]; + let end = rest + .find(|c: char| !(c.is_ascii_alphanumeric() || c == '_')) + .unwrap_or(rest.len()); + let cref = format!("Capability.{}", heck::AsUpperCamelCase(&rest[..end])); + if !refs.contains(&cref) { + refs.push(cref); + } + } + } + refs +} + fn enabled_service(srv: &str) -> bool { match srv { // not enabled in bindings/dotnet/Cargo.toml @@ -480,3 +1031,79 @@ fn write_rendered(output: &std::path::Path, mut rendered: String) -> Result<()> fs::write(output, rendered)?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_options_doc_inline() { + let fields = [OptionField { + name: "if_match".to_string(), + kind: OptionKind::OptionString, + doc: vec![], + }]; + let ctx = DocContext { fields: &fields }; + + assert_eq!( + ctx.inline("Check [`crate::Capability::read_with_if_match`] first."), + "Check first." + ); + assert_eq!( + ctx.inline("Fails with [`crate::ErrorKind::ConditionNotMatch`]."), + "Fails with ." + ); + assert_eq!( + ctx.inline("Conflicts with `if_match`, default `false`, see `a, default , see a<b." + ); + assert_eq!( + ctx.inline("See [the spec][crate::docs::specs::x] and [`crate::Reader::fetch`]."), + "See the spec and Reader::fetch." + ); + assert_eq!( + ctx.inline("Read [MDN](https://example.com/a?b&c)."), + "Read MDN." + ); + } + + #[test] + fn test_metadata_rebuild_requires_a_setter() { + let field = |name: &str, kind| { + metadata_field(metadata::MetadataField { + name: name.to_string(), + kind, + doc: String::new(), + }) + }; + let fields = vec![ + field("mode", MetadataKind::Mode), + field("content_length", MetadataKind::U64), + field("etag", MetadataKind::OptionStr), + ]; + let setters = vec!["etag".to_string()]; + assert!(validate_metadata_rebuild(&fields, &setters).is_ok()); + + let mut with_new_accessor = fields; + with_new_accessor.push(field("storage_class", MetadataKind::OptionStr)); + let err = validate_metadata_rebuild(&with_new_accessor, &setters) + .expect_err("an accessor without a setter must fail"); + assert!(err.to_string().contains("storage_class")); + + let without_mode = vec![field("etag", MetadataKind::OptionStr)]; + assert!(validate_metadata_rebuild(&without_mode, &setters).is_err()); + } + + #[test] + fn test_capability_refs() { + let lines = vec![ + "Check [`crate::Capability::copy_can_multi`] and".to_string(), + "[`crate::Capability::copy_multi_min_size`]; see `copy_can_multi`".to_string(), + "again: [`crate::Capability::copy_can_multi`].".to_string(), + ]; + assert_eq!( + capability_refs(&lines), + ["Capability.CopyCanMulti", "Capability.CopyMultiMinSize"] + ); + } +} diff --git a/dev/src/generate/dotnet/metadata_interop_cs.j2 b/dev/src/generate/dotnet/metadata_interop_cs.j2 index d6cb061ab5c1..ed4bf07c4d36 100644 --- a/dev/src/generate/dotnet/metadata_interop_cs.j2 +++ b/dev/src/generate/dotnet/metadata_interop_cs.j2 @@ -20,6 +20,7 @@ // DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. using System.Runtime.InteropServices; +using OpenDAL.Options; namespace OpenDAL.Interop.NativeObject; @@ -48,4 +49,17 @@ internal struct OpenDALMetadata {%- endfor %} ); } + + /// + /// Fills a payload from managed metadata for native code to borrow, for + /// options such as IfNotChanged. + /// + public static OpenDALMetadata From(Metadata metadata, ref NativeOptionsWriter writer) + { + var native = new OpenDALMetadata(); +{%- for f in fields %} + {{ f.cs_write }} +{%- endfor %} + return native; + } } diff --git a/dev/src/generate/dotnet/metadata_rs.j2 b/dev/src/generate/dotnet/metadata_rs.j2 index a2504023b68e..d703a9c81512 100644 --- a/dev/src/generate/dotnet/metadata_rs.j2 +++ b/dev/src/generate/dotnet/metadata_rs.j2 @@ -19,6 +19,11 @@ use std::ffi::c_char; +use opendal::{EntryMode, MetadataBuilder}; + +use crate::error::OpenDALError; +use crate::utils::entry_mode_code; + /// FFI representation of `opendal::Metadata`. /// /// Fields follow the accessor order of `opendal::Metadata`. Optional values @@ -70,3 +75,40 @@ pub(crate) unsafe fn metadata_release_fields(metadata: &mut OpendalMetadata) { {%- endfor %} } } + +impl OpendalMetadata { + /// Rebuild the metadata through `MetadataBuilder` from fields the caller + /// owns, as options such as `if_not_changed` pass them. + /// + /// Every pointer is only borrowed: strings are copied, and nothing is + /// released. + pub(crate) fn to_metadata(&self) -> Result { + // `MetadataBuilder` takes the content length only through `file`. + let mut builder = match self.mode { + code if code == entry_mode_code(EntryMode::FILE) => { + MetadataBuilder::file(self.content_length) + } + code if code == entry_mode_code(EntryMode::DIR) => MetadataBuilder::dir(), + _ => MetadataBuilder::unknown(), + }; +{%- for f in fields %} +{%- if f.rs_rebuild %} + {{ f.rs_rebuild }} +{%- endif %} +{%- endfor %} + + Ok(builder.build()) + } +} + +/// Rebuild metadata passed by pointer, where null is `None`. +/// +/// `metadata` must be null or point to a valid mirror whose pointers stay +/// valid for the duration of the call. +pub(crate) fn borrowed_metadata( + metadata: *const OpendalMetadata, +) -> Result, OpenDALError> { + unsafe { metadata.as_ref() } + .map(OpendalMetadata::to_metadata) + .transpose() +} diff --git a/dev/src/generate/dotnet/options_cs.j2 b/dev/src/generate/dotnet/options_cs.j2 new file mode 100644 index 000000000000..375fd10e6b09 --- /dev/null +++ b/dev/src/generate/dotnet/options_cs.j2 @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. + +using System.Runtime.InteropServices; +using OpenDAL.Options.Abstractions; + +namespace OpenDAL.Options; + +/// +/// {{ class.summary }} +/// +public sealed class {{ class.name }} : IOptions +{ +{%- for p in class.properties %} + /// + /// {{ p.summary }} + /// +{%- for cref in p.see_also %} + /// +{%- endfor %} + public {{ p.cs_type }} {{ p.pascal }} { get; init; }{{ p.initializer }} +{% endfor %} +{%- if class.validations %} + void IOptions.Validate() + { +{%- for v in class.validations %} + {{ v }} +{%- endfor %} + } +{% endif %} + OpenDAL{{ class.name }} IOptions.ToNative(ref NativeOptionsWriter writer) + { + var native = new OpenDAL{{ class.name }}(); +{%- for w in class.cs_writes %} + {{ w }} +{%- endfor %} + return native; + } +} + +/// +/// Mirror of the native Opendal{{ class.name }} struct. Pointers refer +/// into the options buffer of a single call. +/// +[StructLayout(LayoutKind.Sequential)] +internal struct OpenDAL{{ class.name }} +{ +{%- for p in class.parts %} + public {{ p.cs_type }} {{ p.pascal }}; +{%- endfor %} +} diff --git a/dev/src/generate/dotnet/options_rs.j2 b/dev/src/generate/dotnet/options_rs.j2 new file mode 100644 index 000000000000..82283e6e6ae4 --- /dev/null +++ b/dev/src/generate/dotnet/options_rs.j2 @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// DO NOT EDIT IT MANUALLY. This file is generated by opendal/dev/src/generate/dotnet.rs. + +// Each field converts in its own statement, so a new core option only adds one. +#![allow(clippy::field_reassign_with_default)] + +use std::os::raw::c_char; + +use opendal::BytesRange; + +use crate::error::OpenDALError; +use crate::metadata::{OpendalMetadata, borrowed_metadata}; +use crate::utils::{ + borrowed_string, borrowed_string_pairs, borrowed_timestamp, config_invalid_error, +}; +use crate::validators::prelude::validate_read_range_end; +use crate::validators::validate_positive_integer; + +/// A `#[repr(C)]` mirror of one core options struct. +/// +/// Pointers are borrowed from the caller for the duration of the call and +/// copied into owned values by `to_options`, so native code never keeps them. +/// A null string pointer is `None`; optional values flatten into a +/// `*_has_value` byte, `1` when present, followed by their parts. +pub(crate) trait NativeOptions { + /// The core options struct this mirror converts into. + type Options: Default; + + /// Validate the mirror and convert it into core options. + fn to_options(&self) -> Result; +} + +/// Convert options passed by pointer, where null selects the defaults. +/// +/// `options` must be null or point to a valid mirror whose pointers stay +/// valid for the duration of the call. +pub(crate) fn read_options(options: *const T) -> Result { + match unsafe { options.as_ref() } { + Some(options) => options.to_options(), + None => Ok(T::Options::default()), + } +} + +/// Build a byte range from `offset` and `length`, or from `suffix_length` +/// for a suffix range. +fn range( + offset: u64, + length: Option, + suffix_length: Option, +) -> Result { + if let Some(size) = suffix_length { + if offset > 0 || length.is_some() { + return Err(config_invalid_error( + "suffix_length cannot be combined with offset or length", + )); + } + return Ok(BytesRange::suffix(size)); + } + + Ok(match validate_read_range_end(offset, length)? { + Some(end) => (offset..end).into(), + None => (offset..).into(), + }) +} +{% for c in classes %} +/// FFI representation of `opendal::options::{{ c.name }}`. +#[repr(C)] +pub struct Opendal{{ c.name }} { +{%- for p in c.parts %} + pub {{ p.name }}: {{ p.rs_type }}, +{%- endfor %} +} + +impl NativeOptions for Opendal{{ c.name }} { + type Options = opendal::options::{{ c.name }}; + + fn to_options(&self) -> Result { + let mut options = Self::Options::default(); +{%- for stmt in c.rs_convert %} + {{ stmt }} +{%- endfor %} + + Ok(options) + } +} + +/// The mirror of default options, as the C# class sends it unchanged. +#[cfg(test)] +impl Default for Opendal{{ c.name }} { + fn default() -> Self { + Self { +{%- for p in c.parts %} + {{ p.name }}: {{ p.rs_default }}, +{%- endfor %} + } + } +} +{% endfor %} + +#[cfg(test)] +mod tests { + use std::ffi::{CStr, CString}; + use std::ptr; + + use opendal::raw::Timestamp; + + use super::*; + + fn message(error: &OpenDALError) -> String { + unsafe { CStr::from_ptr(error.message) } + .to_string_lossy() + .into_owned() + } + + #[test] + fn read_options_defaults_on_null() { + let options = read_options::(ptr::null()).unwrap(); + assert_eq!(options, opendal::options::ReadOptions::default()); + } + + #[test] + fn read_options_matches_core_defaults() { + // C# sends a count of 1 where core defaults to 0; core reads both as + // no concurrency. + let options = read_options(&OpendalReadOptions::default()).unwrap(); + assert_eq!( + options, + opendal::options::ReadOptions { + concurrent: 1, + ..Default::default() + } + ); + } + + #[test] + fn read_options_copies_borrowed_values() { + let etag = CString::new("\"e\"").unwrap(); + let native = OpendalReadOptions { + offset: 4, + length_has_value: 1, + length: 6, + if_match: etag.as_ptr(), + if_modified_since_has_value: 1, + if_modified_since_second: 1_700_000_000, + if_modified_since_nanosecond: 123, + chunk_has_value: 1, + chunk: 1024, + ..Default::default() + }; + let options = read_options(&native).unwrap(); + drop(etag); + + assert_eq!(options.range, BytesRange::from(4..10)); + assert_eq!(options.if_match.as_deref(), Some("\"e\"")); + assert_eq!( + options.if_modified_since, + Some(Timestamp::new(1_700_000_000, 123).unwrap()) + ); + assert_eq!(options.chunk, Some(1024)); + } + + #[test] + fn read_options_rejects_suffix_with_offset() { + let native = OpendalReadOptions { + offset: 1, + suffix_length_has_value: 1, + suffix_length: 1, + ..Default::default() + }; + let err = read_options(&native).expect_err("a suffix range cannot have an offset"); + assert!(message(&err).contains("suffix_length")); + } + + #[test] + fn read_options_rejects_zero_concurrent() { + let native = OpendalReadOptions { + concurrent: 0, + ..Default::default() + }; + let err = read_options(&native).expect_err("concurrent must be positive"); + assert!(message(&err).contains("read concurrent")); + } + + #[test] + fn read_options_names_invalid_utf8() { + let invalid = [0xffu8, 0]; + let native = OpendalReadOptions { + version: invalid.as_ptr().cast(), + ..Default::default() + }; + let err = read_options(&native).expect_err("invalid UTF-8 must fail"); + assert!(message(&err).contains("version")); + } + + #[test] + fn write_options_rebuild_user_metadata_and_metadata() { + let key = CString::new("k").unwrap(); + let value = CString::new("v").unwrap(); + let keys = [key.as_ptr()]; + let values = [value.as_ptr()]; + let etag = CString::new("e").unwrap(); + + // Metadata without values has only null strings, so replacing one + // with a borrowed pointer neither overwrites nor leaks owned memory. + let mut metadata = + OpendalMetadata::from_metadata(opendal::MetadataBuilder::file(0).build()); + metadata.content_length = 5; + metadata.etag = etag.as_ptr().cast_mut(); + + let native = OpendalWriteOptions { + user_metadata_has_value: 1, + user_metadata_keys: keys.as_ptr(), + user_metadata_values: values.as_ptr(), + user_metadata_len: 1, + if_not_changed: &metadata, + ..Default::default() + }; + let options = read_options(&native).unwrap(); + + assert_eq!(options.user_metadata.unwrap()["k"], "v"); + let rebuilt = options.if_not_changed.unwrap(); + assert!(rebuilt.is_file()); + assert_eq!(rebuilt.content_length(), 5); + assert_eq!(rebuilt.etag(), Some("e")); + } + + #[test] + fn write_options_keep_empty_user_metadata() { + let native = OpendalWriteOptions { + user_metadata_has_value: 1, + ..Default::default() + }; + let options = read_options(&native).unwrap(); + assert_eq!(options.user_metadata, Some(Default::default())); + } + + #[test] + fn write_options_name_null_user_metadata_value() { + let key = CString::new("k").unwrap(); + let keys = [key.as_ptr()]; + let values = [ptr::null::()]; + let native = OpendalWriteOptions { + user_metadata_has_value: 1, + user_metadata_keys: keys.as_ptr(), + user_metadata_values: values.as_ptr(), + user_metadata_len: 1, + ..Default::default() + }; + let err = read_options(&native).expect_err("a null value must fail"); + assert!(message(&err).contains("value of user_metadata k is null")); + } +} diff --git a/dev/src/generate/metadata.rs b/dev/src/generate/metadata.rs index dd2684fc36f6..6e68332aaf12 100644 --- a/dev/src/generate/metadata.rs +++ b/dev/src/generate/metadata.rs @@ -117,6 +117,44 @@ pub fn parse(path: &Path) -> Result> { Ok(fields) } +/// Parse the names of the public `impl MetadataBuilder` methods that take +/// `&mut self`, which are the setters a metadata mirror can rebuild through. +pub fn parse_builder_setters(path: &Path) -> Result> { + let content = + fs::read_to_string(path).with_context(|| format!("failed to read {}", path.display()))?; + let ast = syn::parse_file(&content)?; + + let mut setters = Vec::new(); + for item in ast.items { + let Item::Impl(impl_) = item else { + continue; + }; + let is_builder = + matches!(&*impl_.self_ty, Type::Path(path) if path.path.is_ident("MetadataBuilder")); + if impl_.trait_.is_some() || !is_builder { + continue; + } + + for item in impl_.items { + if let ImplItem::Fn(func) = item + && matches!(func.vis, syn::Visibility::Public(_)) + && matches!( + func.sig.inputs.first(), + Some(FnArg::Receiver(receiver)) if receiver.reference.is_some() && receiver.mutability.is_some() + ) + { + setters.push(func.sig.ident.to_string()); + } + } + } + + if setters.is_empty() { + bail!("impl MetadataBuilder not found in {}", path.display()); + } + + Ok(setters) +} + fn is_metadata(ty: &Type) -> bool { matches!(ty, Type::Path(path) if path.path.is_ident("Metadata")) } @@ -217,4 +255,21 @@ mod tests { let etag = fields.iter().find(|f| f.name == "etag").unwrap(); assert_eq!(etag.kind, MetadataKind::OptionStr); } + + #[test] + fn test_parse_core_metadata_builder_setters() { + let path = crate::workspace_dir().join("core/core/src/types/metadata.rs"); + let setters = parse_builder_setters(&path).expect("core metadata builder must parse"); + + assert!(setters.contains(&"etag".to_string())); + assert!(setters.contains(&"last_modified".to_string())); + assert!( + !setters.contains(&"build".to_string()), + "build consumes self" + ); + assert!( + !setters.contains(&"file".to_string()), + "file is a constructor" + ); + } } diff --git a/dev/src/generate/mod.rs b/dev/src/generate/mod.rs index 6d4a5c59bb60..f82058ae94f3 100644 --- a/dev/src/generate/mod.rs +++ b/dev/src/generate/mod.rs @@ -16,6 +16,7 @@ // under the License. mod capability; +mod core_options; mod docs; mod dotnet; mod gcs_grpc; diff --git a/website/docs/20-bindings/dotnet/04-tasks.md b/website/docs/20-bindings/dotnet/04-tasks.md index a52a380b3124..3a3a75716401 100644 --- a/website/docs/20-bindings/dotnet/04-tasks.md +++ b/website/docs/20-bindings/dotnet/04-tasks.md @@ -53,6 +53,13 @@ using OpenDAL.Options; byte[] head = op.Read("path/to/file", new ReadOptions { Offset = 0, Length = 1024 }); ``` +`SuffixLength` reads the end of a file instead, on services that report +`Capability.ReadWithSuffix`: + +```csharp +byte[] tail = op.Read("path/to/file", new ReadOptions { SuffixLength = 1024 }); +``` + ## Write a whole file ```csharp @@ -194,6 +201,29 @@ op.Rename("old.txt", "new.txt"); Both operate within a single operator and require a service that supports them; see [capability checks](./05-production.md#capability-checks). +`CopyOptions` and `RenameOptions` add conditions, such as refusing to replace +an existing target. A failed condition throws `OpenDALException` with +`ErrorCode.ConditionNotMatch`: + +```csharp +op.Rename("old.txt", "new.txt", new RenameOptions { IfNotExists = true }); +``` + +## Write only if a file is unchanged + +Pass metadata from an earlier `Stat` as `IfNotChanged` to make a write, +delete, or copy fail with `ErrorCode.ConditionNotMatch` when someone else +changed the file in between: + +```csharp +var meta = op.Stat("config.json"); +// ... compute the new content ... +op.Write("config.json", content, new WriteOptions { IfNotChanged = meta }); +``` + +The condition compares the version or the ETag, whichever the service +supports. + ## Generate a presigned URL Hand a time-limited URL to a third party so they can access an object without