From 38ca92c6e0b106eb5cd99fd96b37efa797600ad5 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Thu, 23 Jul 2026 12:53:49 +0530 Subject: [PATCH 01/30] feat: implement CachedGetRequestClient with hybrid caching and user-specific request keys --- .../TestData/v2-config/appsettings.json | 9 +- .../packages.lock.json | 54 ++- ...AAS.TwinEngine.DataEngine.UnitTests.csproj | 6 + .../Caching/CachedGetRequestClientTests.cs | 401 ++++++++++++++++ .../SubmodelDescriptorProviderTests.cs | 321 +++++-------- .../Services/TemplateProviderTests.cs | 427 +++++++----------- .../packages.lock.json | 72 ++- .../AAS.TwinEngine.DataEngine.csproj | 1 + .../Clients/Caching/CachedGetRequestClient.cs | 104 +++++ .../Caching/ICachedGetRequestClient.cs | 6 + .../Services/SubmodelDescriptorProvider.cs | 56 +-- .../Services/TemplateProvider.cs | 84 +--- source/AAS.TwinEngine.DataEngine/Program.cs | 1 + .../Config/TemplateManagementConfig.cs | 1 + ...astructureDependencyInjectionExtensions.cs | 4 +- .../appsettings.development.json | 7 +- .../appsettings.json | 15 +- .../packages.lock.json | 6 + 18 files changed, 944 insertions(+), 631 deletions(-) create mode 100644 source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs create mode 100644 source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs create mode 100644 source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/ICachedGetRequestClient.cs diff --git a/source/AAS.TwinEngine.DataEngine.ModuleTests/TestData/v2-config/appsettings.json b/source/AAS.TwinEngine.DataEngine.ModuleTests/TestData/v2-config/appsettings.json index 07de2a22..a215d0c3 100644 --- a/source/AAS.TwinEngine.DataEngine.ModuleTests/TestData/v2-config/appsettings.json +++ b/source/AAS.TwinEngine.DataEngine.ModuleTests/TestData/v2-config/appsettings.json @@ -126,6 +126,7 @@ "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { "source": "Authorization", @@ -144,20 +145,23 @@ "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", "headerMappings": [], - "ConcurrentOperationsLimit": 10 + "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5 }, "ConceptDescriptionTemplateRepository": { "Name": "ConceptDescriptionTemplateRepository", "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", "headerMappings": [], - "ConcurrentOperationsLimit": 10 + "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5 }, "AasTemplateRegistry": { "Name": "AasTemplateRegistry", "baseUrl": "http://localhost:8082", "healthEndpoint": "/actuator/health", "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { "source": "Authorization", @@ -171,6 +175,7 @@ "baseUrl": "http://localhost:8083", "healthEndpoint": "/actuator/health", "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { "source": "Authorization", diff --git a/source/AAS.TwinEngine.DataEngine.ModuleTests/packages.lock.json b/source/AAS.TwinEngine.DataEngine.ModuleTests/packages.lock.json index 21c7687b..3cfa78d5 100644 --- a/source/AAS.TwinEngine.DataEngine.ModuleTests/packages.lock.json +++ b/source/AAS.TwinEngine.DataEngine.ModuleTests/packages.lock.json @@ -157,6 +157,37 @@ "resolved": "10.0.0", "contentHash": "NCWCGiwRwje8773yzPQhvucYnnfeR+ZoB1VRIrIMp4uaeUNw7jvEPHij3HIbwCDuNCrNcphA00KSAR9yD9qmbg==" }, + "Microsoft.Extensions.Caching.Abstractions": { + "type": "Transitive", + "resolved": "10.0.10", + "contentHash": "4ZFBNE+jzR+CrWWlhOesnmywCW7pYKT0dxyAQRdL11yJwxe4jvcAu31eorFtEkoFeCDcUTeNssgPv2yaRRptaQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.10" + } + }, + "Microsoft.Extensions.Caching.Hybrid": { + "type": "Transitive", + "resolved": "10.8.0", + "contentHash": "RCtCTK3eqKXx8LXqd7B8GjbTkIp4k3EgKeunaCmBJ+WA55Nva83CI2I+wVyp0S9jTG+aT6AYWQbkKOfvFOlmLA==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.10", + "Microsoft.Extensions.Caching.Memory": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10" + } + }, + "Microsoft.Extensions.Caching.Memory": { + "type": "Transitive", + "resolved": "10.0.10", + "contentHash": "N1w5H7uK6gCTnCBZAWzE0/EQYSPysij/uYwDqntqBVvBa6bjMmBKitsnEFd6yh/SX3wLm67nO6+OnZ84K+gZWg==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.10", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10", + "Microsoft.Extensions.Primitives": "10.0.10" + } + }, "Microsoft.Extensions.Compliance.Abstractions": { "type": "Transitive", "resolved": "10.6.0", @@ -254,8 +285,8 @@ }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "10.0.8", - "contentHash": "21nbDV60SRPWGIivsyl6lqBeEJNG1sginhhfWgRrr3Ais7aQ12To25OAHQxgoiJkjqy1aQ6RxpZBGYuTi7Ge6A==" + "resolved": "10.0.10", + "contentHash": "z/2xXlFw2aLGjHyEm6E0tQ+In6VfzQzTrtArbQ2c0TQE16ZbyDCMGPvaUT9I0s8rgy9sRWlU2P9waW37qV04qA==" }, "Microsoft.Extensions.DependencyInjection.AutoActivation": { "type": "Transitive", @@ -405,10 +436,10 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "10.0.8", - "contentHash": "fdVadZmsC8jRP0KvKy8mO8f6GV/HyBvElfcSxEhd+5FM5boAw/01iSaCto5G3G37ApJira4A3pNaVvBv8cUiLQ==", + "resolved": "10.0.10", + "contentHash": "zkFxGYUvdxAvIKTyXHrmW+Sux53D4SezD9dMyZ6hrwwzPQJNuwCRy1f5W7AvYTqacEGhWF2XderRQG1OvbV8og==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.8" + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" } }, "Microsoft.Extensions.Logging.Configuration": { @@ -479,11 +510,11 @@ }, "Microsoft.Extensions.Options": { "type": "Transitive", - "resolved": "10.0.8", - "contentHash": "VBD+131DpTNCNDfA4kIyKTiCySvJGNhwibdWBSdFRu7GMfXLXcXODkgA+KStKbbhzraLglZWUN4nXyHgW4JIRA==", + "resolved": "10.0.10", + "contentHash": "srnhnk7nE8krBiIXp71LvBmKBtraBONWSRzdjJgRv1Ko9Mp8IVNqv4vIS9hGeVteBig8aQkva9ZG+sC+o5sVcA==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.8", - "Microsoft.Extensions.Primitives": "10.0.8" + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10", + "Microsoft.Extensions.Primitives": "10.0.10" } }, "Microsoft.Extensions.Options.ConfigurationExtensions": { @@ -500,8 +531,8 @@ }, "Microsoft.Extensions.Primitives": { "type": "Transitive", - "resolved": "10.0.8", - "contentHash": "OBPo4nYhMyIbtueoC10CBm6AGAbo/A9IV8QQ/6ryZS7VvmqpGT7hunazeHLxFawRzn3oLOq4jhqhpBX4tfswWQ==" + "resolved": "10.0.10", + "contentHash": "5wu/GrYVd8mG2DVUw3vFJzF+O336TyTGg/Kmcgw9bfwYhCoFiV5lR5QeEmKecJyrW4W54nMfD3p3589E8a7czQ==" }, "Microsoft.Extensions.Resilience": { "type": "Transitive", @@ -937,6 +968,7 @@ "Asp.Versioning.Mvc": "[10.0.0, 10.0.0]", "Cronos": "[0.13.0, 0.13.0]", "JsonSchema.Net": "[7.3.4, 7.3.4]", + "Microsoft.Extensions.Caching.Hybrid": "[10.8.0, )", "Microsoft.Extensions.DependencyInjection": "[10.0.8, 10.0.8]", "Microsoft.Extensions.Http": "[10.0.8, 10.0.8]", "Microsoft.Extensions.Http.Resilience": "[10.6.0, 10.6.0]", diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/AAS.TwinEngine.DataEngine.UnitTests.csproj b/source/AAS.TwinEngine.DataEngine.UnitTests/AAS.TwinEngine.DataEngine.UnitTests.csproj index 048108bc..1c50ce82 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/AAS.TwinEngine.DataEngine.UnitTests.csproj +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/AAS.TwinEngine.DataEngine.UnitTests.csproj @@ -17,6 +17,8 @@ + + @@ -35,5 +37,9 @@ + + + + diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs new file mode 100644 index 00000000..bb808204 --- /dev/null +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs @@ -0,0 +1,401 @@ +using System.Net; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text; + +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; +using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients; +using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients.Caching; + +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Caching.Hybrid; +using Microsoft.Extensions.Logging; + +using NSubstitute; + +using UnauthorizedAccessException = AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException; + +namespace AAS.TwinEngine.DataEngine.UnitTests.Infrastructure.Http.Clients.Caching; + +public class CachedGetRequestClientTests +{ + private readonly ICreateClient _clientFactory; + private readonly HybridCache _cache; + private readonly IHttpContextAccessor _httpContextAccessor; + private readonly ILogger _logger; + private readonly CachedGetRequestClient _sut; + + public CachedGetRequestClientTests() + { + _clientFactory = Substitute.For(); + _cache = Substitute.For(); + _httpContextAccessor = Substitute.For(); + _logger = Substitute.For>(); + + _sut = new CachedGetRequestClient( + _clientFactory, + _cache, + _httpContextAccessor, + _logger); + } + + [Fact] + public async Task GetStringAsync_CacheHit_ReturnsCachedValueWithoutCallingHttp() + { + // Arrange + const string RelativeUrl = "api/test"; + const string HttpClientName = "TestClient"; + const string CachedResponse = "Cached data"; + + _httpContextAccessor.HttpContext.Returns((HttpContext)null!); + SetupCacheHit(CachedResponse); + + // Act + var result = await _sut.GetStringAsync(RelativeUrl, HttpClientName, 5, CancellationToken.None); + + // Assert + Assert.Equal(CachedResponse, result); + _clientFactory.DidNotReceive().CreateClient(Arg.Any()); + } + + [Fact] + public async Task GetStringAsync_CacheMiss_CallsHttpClientAndReturnsResponse() + { + // Arrange + const string RelativeUrl = "api/test"; + const string HttpClientName = "TestClient"; + const string ExpectedResponse = "Direct HTTP data"; + + _httpContextAccessor.HttpContext.Returns((HttpContext)null!); + SetupCacheMiss(); + + using var httpResponse = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ExpectedResponse) + }; + SetupHttpClient(HttpClientName, httpResponse); + + // Act + var result = await _sut.GetStringAsync(RelativeUrl, HttpClientName, 5, CancellationToken.None); + + // Assert + Assert.Equal(ExpectedResponse, result); + _clientFactory.Received(1).CreateClient(HttpClientName); + } + + [Theory] + [InlineData(HttpStatusCode.NotFound, typeof(ResourceNotFoundException))] + [InlineData(HttpStatusCode.Unauthorized, typeof(UnauthorizedAccessException))] + [InlineData(HttpStatusCode.Forbidden, typeof(UnauthorizedAccessException))] + [InlineData(HttpStatusCode.RequestTimeout, typeof(RequestTimeoutException))] + [InlineData(HttpStatusCode.InternalServerError, typeof(ValidationFailedException))] + [InlineData(HttpStatusCode.BadRequest, typeof(ValidationFailedException))] + public async Task GetStringAsync_CacheMiss_MapsHttpStatusCodesToExpectedExceptions(HttpStatusCode statusCode, Type expectedExceptionType) + { + // Arrange + const string RelativeUrl = "api/test"; + const string HttpClientName = "TestClient"; + + _httpContextAccessor.HttpContext.Returns((HttpContext)null!); + SetupCacheMiss(); + + using var httpResponse = new HttpResponseMessage(statusCode) + { + Content = new StringContent("Error response body") + }; + SetupHttpClient(HttpClientName, httpResponse); + + // Act & Assert + await Assert.ThrowsAsync(expectedExceptionType, () => + _sut.GetStringAsync(RelativeUrl, HttpClientName, 5, CancellationToken.None)); + } + + [Fact] + public async Task GetStringAsync_BuildsExpectedCacheKey_ForNullHttpContext() + { + // Arrange + const string RelativeUrl = "api/test"; + _httpContextAccessor.HttpContext.Returns((HttpContext)null!); + + string? capturedKey = null; + SetupCacheCapture(key => capturedKey = key); + + // Act + await _sut.GetStringAsync(RelativeUrl, "client", 5, CancellationToken.None); + + // Assert + var expectedHash = ComputeHash(RelativeUrl); + Assert.Equal($"anonymous:req:{expectedHash}", capturedKey); + } + + [Fact] + public async Task GetStringAsync_BuildsExpectedCacheKey_ForNullUser() + { + // Arrange + const string RelativeUrl = "api/test"; + var httpContext = new DefaultHttpContext { User = null! }; + _httpContextAccessor.HttpContext.Returns(httpContext); + + string? capturedKey = null; + SetupCacheCapture(key => capturedKey = key); + + // Act + await _sut.GetStringAsync(RelativeUrl, "client", 5, CancellationToken.None); + + // Assert + var expectedHash = ComputeHash(RelativeUrl); + Assert.Equal($"anonymous:req:{expectedHash}", capturedKey); + } + + [Fact] + public async Task GetStringAsync_BuildsExpectedCacheKey_ForUnauthenticatedUser() + { + // Arrange + const string RelativeUrl = "api/test"; + var httpContext = new DefaultHttpContext(); + var identity = new ClaimsIdentity(); // IsAuthenticated = false + httpContext.User = new ClaimsPrincipal(identity); + _httpContextAccessor.HttpContext.Returns(httpContext); + + string? capturedKey = null; + SetupCacheCapture(key => capturedKey = key); + + // Act + await _sut.GetStringAsync(RelativeUrl, "client", 5, CancellationToken.None); + + // Assert + var expectedHash = ComputeHash(RelativeUrl); + Assert.Equal($"anonymous:req:{expectedHash}", capturedKey); + } + + [Fact] + public async Task GetStringAsync_BuildsExpectedCacheKey_ForAuthenticatedUserWithNameIdentifier() + { + // Arrange + const string RelativeUrl = "api/test"; + const string UserId = "user123"; + var claims = new List + { + new(ClaimTypes.NameIdentifier, UserId), + new("role", "admin") + }; + + var httpContext = new DefaultHttpContext(); + var identity = new ClaimsIdentity(claims, "TestAuthType"); + httpContext.User = new ClaimsPrincipal(identity); + _httpContextAccessor.HttpContext.Returns(httpContext); + + string? capturedKey = null; + SetupCacheCapture(key => capturedKey = key); + + // Act + await _sut.GetStringAsync(RelativeUrl, "client", 5, CancellationToken.None); + + // Assert + var requestHash = ComputeHash(RelativeUrl); + var permissionHash = ComputePermissionHash(claims); + Assert.Equal($"user:{UserId}:ph:{permissionHash}:req:{requestHash}", capturedKey); + } + + [Fact] + public async Task GetStringAsync_BuildsExpectedCacheKey_ForAuthenticatedUserWithSub() + { + // Arrange + const string RelativeUrl = "api/test"; + const string SubId = "sub456"; + var claims = new List + { + new("sub", SubId), + new("role", "user") + }; + + var httpContext = new DefaultHttpContext(); + var identity = new ClaimsIdentity(claims, "TestAuthType"); + httpContext.User = new ClaimsPrincipal(identity); + _httpContextAccessor.HttpContext.Returns(httpContext); + + string? capturedKey = null; + SetupCacheCapture(key => capturedKey = key); + + // Act + await _sut.GetStringAsync(RelativeUrl, "client", 5, CancellationToken.None); + + // Assert + var requestHash = ComputeHash(RelativeUrl); + var permissionHash = ComputePermissionHash(claims); + Assert.Equal($"user:{SubId}:ph:{permissionHash}:req:{requestHash}", capturedKey); + } + + [Fact] + public async Task GetStringAsync_BuildsExpectedCacheKey_ForAuthenticatedUserFallsBackToUnknown() + { + // Arrange + const string RelativeUrl = "api/test"; + var claims = new List + { + new("role", "user") + }; + + var httpContext = new DefaultHttpContext(); + var identity = new ClaimsIdentity(claims, "TestAuthType"); + httpContext.User = new ClaimsPrincipal(identity); + _httpContextAccessor.HttpContext.Returns(httpContext); + + string? capturedKey = null; + SetupCacheCapture(key => capturedKey = key); + + // Act + await _sut.GetStringAsync(RelativeUrl, "client", 5, CancellationToken.None); + + // Assert + var requestHash = ComputeHash(RelativeUrl); + var permissionHash = ComputePermissionHash(claims); + Assert.Equal($"user:unknown:ph:{permissionHash}:req:{requestHash}", capturedKey); + } + + [Fact] + public async Task GetStringAsync_BuildsExpectedCacheKey_ClaimsOrderingDoesNotAffectPermissionHash() + { + // Arrange + const string RelativeUrl = "api/test"; + var claimsA = new List + { + new("role", "admin"), + new("permission", "read"), + new(ClaimTypes.NameIdentifier, "user123") + }; + var claimsB = new List + { + new(ClaimTypes.NameIdentifier, "user123"), + new("permission", "read"), + new("role", "admin") + }; + + var httpContextA = new DefaultHttpContext { User = new ClaimsPrincipal(new ClaimsIdentity(claimsA, "Test")) }; + var httpContextB = new DefaultHttpContext { User = new ClaimsPrincipal(new ClaimsIdentity(claimsB, "Test")) }; + + string? capturedKeyA = null; + string? capturedKeyB = null; + + SetupCacheCapture(key => capturedKeyA = key); + _httpContextAccessor.HttpContext.Returns(httpContextA); + await _sut.GetStringAsync(RelativeUrl, "client", 5, CancellationToken.None); + + SetupCacheCapture(key => capturedKeyB = key); + _httpContextAccessor.HttpContext.Returns(httpContextB); + await _sut.GetStringAsync(RelativeUrl, "client", 5, CancellationToken.None); + + // Assert + Assert.Equal(capturedKeyA, capturedKeyB); + } + + [Fact] + public async Task GetStringAsync_ConfiguresHybridCacheEntryOptionsCorrectly() + { + // Arrange + const string RelativeUrl = "api/test"; + const int ExpirationTimeMinutes = 15; + _httpContextAccessor.HttpContext.Returns((HttpContext)null!); + + HybridCacheEntryOptions? capturedOptions = null; + SetupCacheCapture(key => { }, options => capturedOptions = options); + + // Act + await _sut.GetStringAsync(RelativeUrl, "client", ExpirationTimeMinutes, CancellationToken.None); + + // Assert + Assert.NotNull(capturedOptions); + Assert.Equal(TimeSpan.FromMinutes(ExpirationTimeMinutes), capturedOptions.Expiration); + Assert.Equal(TimeSpan.FromMinutes(ExpirationTimeMinutes), capturedOptions.LocalCacheExpiration); + } + + [Fact] + public async Task GetStringAsync_PropagatesCancellationToken() + { + // Arrange + const string RelativeUrl = "api/test"; + using var cts = new CancellationTokenSource(); + var cancellationToken = cts.Token; + _httpContextAccessor.HttpContext.Returns((HttpContext)null!); + + var capturedToken = CancellationToken.None; + SetupCacheCapture(key => { }, null, token => capturedToken = token); + + // Act + await _sut.GetStringAsync(RelativeUrl, "client", 5, cancellationToken); + + // Assert + Assert.Equal(cancellationToken, capturedToken); + } + + private static string ComputeHash(string input) + { + var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(input)); + return Convert.ToHexStringLower(bytes); + } + + private static string ComputePermissionHash(IEnumerable claims) + { + var claimsString = string.Join("|", claims + .OrderBy(c => c.Type, StringComparer.Ordinal) + .ThenBy(c => c.Value, StringComparer.Ordinal) + .Select(c => $"{c.Type}={c.Value}")); + + return ComputeHash(claimsString); + } + + private void SetupCacheHit(string response) + { + _cache.GetOrCreateAsync( + Arg.Any(), + Arg.Any>>(), + Arg.Any(), + Arg.Any>(), + Arg.Any() + ).Returns(new ValueTask(response)); + } + + private void SetupCacheMiss() + { + _cache.GetOrCreateAsync( + Arg.Any(), + Arg.Any>>(), + Arg.Any(), + Arg.Any>(), + Arg.Any() + ).Returns(info => + { + var factory = info.ArgAt>>(1); + return factory(CancellationToken.None); + }); + } + + private void SetupCacheCapture(Action onKeyCaptured, Action? onOptionsCaptured = null, Action? onTokenCaptured = null) + { + _cache.GetOrCreateAsync( + Arg.Do(key => onKeyCaptured(key)), + Arg.Any>>(), + Arg.Do(options => onOptionsCaptured?.Invoke(options)), + Arg.Any>(), + Arg.Do(token => onTokenCaptured?.Invoke(token)) + ).Returns(new ValueTask("")); + } + + private void SetupHttpClient(string httpClientName, HttpResponseMessage response) + { + var messageHandler = new FakeHttpMessageHandler((req, token) => Task.FromResult(response)); + var httpClient = new HttpClient(messageHandler) + { + BaseAddress = new Uri("http://localhost/") + }; + + _clientFactory.CreateClient(httpClientName).Returns(httpClient); + } +} + +internal sealed class FakeHttpMessageHandler(Func> handler) : HttpMessageHandler +{ + private readonly Func> _handler = handler; + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => _handler(request, cancellationToken); +} diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/SubmodelRegistryProvider/Services/SubmodelDescriptorProviderTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/SubmodelRegistryProvider/Services/SubmodelDescriptorProviderTests.cs index 9c72dc00..e6c181d7 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/SubmodelRegistryProvider/Services/SubmodelDescriptorProviderTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/SubmodelRegistryProvider/Services/SubmodelDescriptorProviderTests.cs @@ -4,14 +4,16 @@ using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRegistry; using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients; +using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients.Caching; using AAS.TwinEngine.DataEngine.Infrastructure.Providers.SubmodelRegistryProvider.Services; -using AAS.TwinEngine.DataEngine.UnitTests.Infrastructure.Providers.PluginDataProvider.Services; using AasCore.Aas3_1; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using NSubstitute; +using NSubstitute.ExceptionExtensions; using UnauthorizedAccessException = AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException; @@ -20,106 +22,102 @@ namespace AAS.TwinEngine.DataEngine.UnitTests.Infrastructure.Providers.SubmodelR public class SubmodelDescriptorProviderTests { private readonly ILogger _logger = Substitute.For>(); - private readonly ICreateClient _clientFactory = Substitute.For(); + private readonly ICachedGetRequestClient _cachedHttp = Substitute.For(); private readonly SubmodelDescriptorProvider _sut; - public SubmodelDescriptorProviderTests() => _sut = new SubmodelDescriptorProvider(_logger, _clientFactory); + public SubmodelDescriptorProviderTests() + { + var options = Substitute.For>(); + var config = new TemplateManagementConfig + { + SubmodelTemplateRegistry = new ServiceInstance { LocalCacheExpirationInMinutes = 5 } + }; + options.Value.Returns(config); + + _sut = new SubmodelDescriptorProvider(_logger, options, _cachedHttp); + } [Fact] public async Task GetDataForSubmodelDescriptorByIdAsync_ReturnsSubmodelDesciptor_WhenResponseIsSuccessful() { const string Id = "ContactInformation"; - var expectedContent = new StringContent("{ \"id\": \"ContactInformation\" }"); var expectedDescriptor = new SubmodelDescriptor { Id = "ContactInformation" }; - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = expectedContent - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry) - .Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .Returns("{ \"id\": \"ContactInformation\" }"); var result = await _sut.GetDataForSubmodelDescriptorByIdAsync(Id, CancellationToken.None); Assert.Equal(expectedDescriptor.Id, result.Id); } - [Fact] - public async Task GetDataForSubmodelDescriptorByIdAsync_DeserializesAasNestedTypes_WhenResponseContainsAasPayload() - { - const string id = "https://mm-software.com/submodel/nameplate"; - const string jsonResponse = """ - { - "description": [ - { - "language": "en", - "text": "Nameplate Submodel Template" - } - ], - "displayName": [ - { - "language": "en", - "text": "Nameplate" - } - ], - "extensions": [ - { - "name": "templateSource", - "valueType": "xs:string", - "value": "Nameplate" - } - ], - "administration": { - "version": "1", - "revision": "0" - }, - "idShort": "Nameplate", - "id": "https://mm-software.com/submodel/nameplate", - "semanticId": { - "type": "ExternalReference", - "keys": [ - { - "type": "GlobalReference", - "value": "https://admin-shell.io/zvei/nameplate/2/0/Nameplate" - } - ] - } - } - """; - - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent(jsonResponse) - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry).Returns(httpClient); - - var result = await _sut.GetDataForSubmodelDescriptorByIdAsync(id, CancellationToken.None); - - Assert.NotNull(result.Description); - Assert.Equal("en", result.Description![0].Language); - Assert.Equal("Nameplate Submodel Template", result.Description[0].Text); - - Assert.NotNull(result.DisplayName); - Assert.Equal("Nameplate", result.DisplayName![0].Text); - - Assert.NotNull(result.Extensions); - Assert.Equal("templateSource", result.Extensions![0].Name); - Assert.Equal(DataTypeDefXsd.String, result.Extensions[0].ValueType); - Assert.Equal("Nameplate", result.Extensions[0].Value); - - Assert.NotNull(result.Administration); - Assert.Equal("1", result.Administration!.Version); - Assert.Equal("0", result.Administration.Revision); - - Assert.NotNull(result.SemanticId); - Assert.Equal(ReferenceTypes.ExternalReference, result.SemanticId!.Type); - Assert.Equal("https://admin-shell.io/zvei/nameplate/2/0/Nameplate", result.SemanticId.Keys[0].Value); - } + [Fact] + public async Task GetDataForSubmodelDescriptorByIdAsync_DeserializesAasNestedTypes_WhenResponseContainsAasPayload() + { + const string id = "https://mm-software.com/submodel/nameplate"; + const string jsonResponse = """ + { + "description": [ + { + "language": "en", + "text": "Nameplate Submodel Template" + } + ], + "displayName": [ + { + "language": "en", + "text": "Nameplate" + } + ], + "extensions": [ + { + "name": "templateSource", + "valueType": "xs:string", + "value": "Nameplate" + } + ], + "administration": { + "version": "1", + "revision": "0" + }, + "idShort": "Nameplate", + "id": "https://mm-software.com/submodel/nameplate", + "semanticId": { + "type": "ExternalReference", + "keys": [ + { + "type": "GlobalReference", + "value": "https://admin-shell.io/zvei/nameplate/2/0/Nameplate" + } + ] + } + } + """; + + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .Returns(jsonResponse); + + var result = await _sut.GetDataForSubmodelDescriptorByIdAsync(id, CancellationToken.None); + + Assert.NotNull(result.Description); + Assert.Equal("en", result.Description![0].Language); + Assert.Equal("Nameplate Submodel Template", result.Description[0].Text); + + Assert.NotNull(result.DisplayName); + Assert.Equal("Nameplate", result.DisplayName![0].Text); + + Assert.NotNull(result.Extensions); + Assert.Equal("templateSource", result.Extensions![0].Name); + Assert.Equal(DataTypeDefXsd.String, result.Extensions[0].ValueType); + Assert.Equal("Nameplate", result.Extensions[0].Value); + + Assert.NotNull(result.Administration); + Assert.Equal("1", result.Administration!.Version); + Assert.Equal("0", result.Administration.Revision); + + Assert.NotNull(result.SemanticId); + Assert.Equal(ReferenceTypes.ExternalReference, result.SemanticId!.Type); + Assert.Equal("https://admin-shell.io/zvei/nameplate/2/0/Nameplate", result.SemanticId.Keys[0].Value); + } [Fact] public async Task GetDataForSubmodelDescriptorByIdAsync_ReturnsSubmodelDescriptor_WhenResponseIsWrappedInResultProperty() @@ -140,14 +138,8 @@ public async Task GetDataForSubmodelDescriptorByIdAsync_ReturnsSubmodelDescripto } """; - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent(jsonResponse) - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .Returns(jsonResponse); var result = await _sut.GetDataForSubmodelDescriptorByIdAsync(id, CancellationToken.None); @@ -166,14 +158,8 @@ public async Task GetDataForSubmodelDescriptorByIdAsync_ReturnsDescriptor_WhenOn const string id = "simple-id"; const string jsonResponse = """{ "id": "simple-id", "idShort": "SimpleSubmodel" }"""; - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent(jsonResponse) - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .Returns(jsonResponse); var result = await _sut.GetDataForSubmodelDescriptorByIdAsync(id, CancellationToken.None); @@ -209,14 +195,8 @@ public async Task GetDataForSubmodelDescriptorByIdAsync_DeserializesSupplemental } """; - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent(jsonResponse) - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .Returns(jsonResponse); var result = await _sut.GetDataForSubmodelDescriptorByIdAsync(id, CancellationToken.None); @@ -233,18 +213,12 @@ public async Task GetDataForSubmodelDescriptorByIdAsync_UsesBase64UrlEncodedIdIn var requestPath = string.Empty; const string jsonResponse = """{ "id": "https://mm-software.com/submodel/nameplate", "idShort": "Nameplate" }"""; - using var messageHandler = new FakeHttpMessageHandler((request, _) => - { - requestPath = request.RequestUri?.ToString() ?? string.Empty; - return Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = new StringContent(jsonResponse) - }); - }); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .Returns(info => + { + requestPath = info.ArgAt(0); + return jsonResponse; + }); _ = await _sut.GetDataForSubmodelDescriptorByIdAsync(id, CancellationToken.None); @@ -257,16 +231,8 @@ public async Task GetDataForSubmodelDescriptorByIdAsync_UsesBase64UrlEncodedIdIn public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsResponseParsingException_WhenDeserializationFails() { const string Id = "ContactInformation"; - var invalidJson = new StringContent("This is not valid JSON"); - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = invalidJson - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry) - .Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .Returns("This is not valid JSON"); await Assert.ThrowsAsync(() => _sut.GetDataForSubmodelDescriptorByIdAsync(Id, CancellationToken.None)); } @@ -275,110 +241,51 @@ public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsResponseParsingExc public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsResponseParsingException_WhenDeserializedObjectIsNull() { const string Id = "null-object-id"; - var emptyJson = new StringContent("null"); - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.OK, - Content = emptyJson - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry) - .Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .Returns("null"); await Assert.ThrowsAsync(() => _sut.GetDataForSubmodelDescriptorByIdAsync(Id, CancellationToken.None)); } [Fact] - public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsResourceNotFoundException_WhenResponseIsNotSuccessful() + public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsResourceNotFoundException_WhenResourceNotFoundIsThrown() { const string Id = "test-id"; - var errorContent = new StringContent("Not found"); - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.NotFound, - Content = errorContent - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry) - .Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .ThrowsAsync(new ResourceNotFoundException()); await Assert.ThrowsAsync(() => - _sut.GetDataForSubmodelDescriptorByIdAsync(Id, CancellationToken.None)); - } - - [Fact] - public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsServiceAuthorizationException_WhenUnauthorized() - { - const string Id = "auth-fail-id"; - var errorContent = new StringContent("Unauthorized"); - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.Unauthorized, - Content = errorContent - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry) - .Returns(httpClient); - - await Assert.ThrowsAsync(() => _sut.GetDataForSubmodelDescriptorByIdAsync(Id, CancellationToken.None)); } [Fact] - public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsServiceAuthorizationException_WhenForbidden() + public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsServiceAuthorizationException_WhenUnauthorizedIsThrown() { - const string Id = "forbidden-id"; - var errorContent = new StringContent("Forbidden"); - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.Forbidden, - Content = errorContent - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry) - .Returns(httpClient); + const string Id = "auth-fail-id"; + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .ThrowsAsync(new UnauthorizedAccessException()); await Assert.ThrowsAsync(() => _sut.GetDataForSubmodelDescriptorByIdAsync(Id, CancellationToken.None)); } [Fact] - public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsRequestTimeoutException_WhenTimeout() + public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsRequestTimeoutException_WhenTimeoutIsThrown() { const string Id = "timeout-id"; - var errorContent = new StringContent("Request timed out"); - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.RequestTimeout, - Content = errorContent - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry) - .Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .ThrowsAsync(new RequestTimeoutException()); await Assert.ThrowsAsync(() => _sut.GetDataForSubmodelDescriptorByIdAsync(Id, CancellationToken.None)); } [Fact] - public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsValidationFailedException_WhenOtherError() + public async Task GetDataForSubmodelDescriptorByIdAsync_ThrowsValidationFailedException_WhenValidationFailedIsThrown() { const string Id = "badrequest-id"; - var errorContent = new StringContent("Bad request"); - using var messageHandler = new FakeHttpMessageHandler((_, _) => Task.FromResult(new HttpResponseMessage - { - StatusCode = HttpStatusCode.BadRequest, - Content = errorContent - })); - using var httpClient = new HttpClient(messageHandler); - httpClient.BaseAddress = new Uri("https://mm-software/fakeUrl"); - _clientFactory.CreateClient(HttpClientNames.SubmodelRegistry) - .Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelRegistry, Arg.Any(), Arg.Any()) + .ThrowsAsync(new ValidationFailedException()); await Assert.ThrowsAsync(() => _sut.GetDataForSubmodelDescriptorByIdAsync(Id, CancellationToken.None)); diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/TemplateProvider/Services/TemplateProviderTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/TemplateProvider/Services/TemplateProviderTests.cs index 0937b23b..5c0291ce 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/TemplateProvider/Services/TemplateProviderTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/TemplateProvider/Services/TemplateProviderTests.cs @@ -1,18 +1,22 @@ -using System.Net; +using System.Net; using System.Linq; using System.Text; using System.Text.Json; +using System.Text.Json.Nodes; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Observability; using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients; +using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients.Caching; using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; using AasCore.Aas3_1; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using NSubstitute; +using NSubstitute.ExceptionExtensions; using Template = AAS.TwinEngine.DataEngine.Infrastructure.Providers.TemplateProvider.Services.TemplateProvider; using UnauthorizedAccessException = AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException; @@ -22,7 +26,7 @@ namespace AAS.TwinEngine.DataEngine.UnitTests.Infrastructure.Providers.TemplateP public class TemplateProviderTests { - private readonly ICreateClient _httpClientFactory; + private readonly ICachedGetRequestClient _cachedHttp; private readonly Template _sut; private const string TemplateId = "Nameplate"; @@ -31,8 +35,20 @@ public class TemplateProviderTests public TemplateProviderTests() { var logger = Substitute.For>(); - _httpClientFactory = Substitute.For(); - _sut = new Template(logger, _httpClientFactory); + _cachedHttp = Substitute.For(); + + var options = Substitute.For>(); + var config = new TemplateManagementConfig + { + SubmodelTemplateRepository = new ServiceInstance { LocalCacheExpirationInMinutes = 5 }, + AasTemplateRegistry = new ServiceInstance { LocalCacheExpirationInMinutes = 5 }, + AasTemplateRepository = new ServiceInstance { LocalCacheExpirationInMinutes = 5 }, + ConceptDescriptionTemplateRepository = new ServiceInstance { LocalCacheExpirationInMinutes = 5 }, + SubmodelTemplateRegistry = new ServiceInstance { LocalCacheExpirationInMinutes = 5 } + }; + options.Value.Returns(config); + + _sut = new Template(logger, options, _cachedHttp); } [Fact] @@ -57,12 +73,8 @@ public async Task GetShellDescriptorTemplateAsync_ReturnsShellDescriptor_WhenVal } """; - using var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK); - mockHttpResponse.Content = new StringContent(JsonResponse); - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasRegistry, Arg.Any(), Arg.Any()) + .Returns(JsonResponse); var result = await _sut.GetShellDescriptorTemplateAsync(TemplateId, CancellationToken.None); @@ -72,94 +84,86 @@ public async Task GetShellDescriptorTemplateAsync_ReturnsShellDescriptor_WhenVal Assert.Equal("https://admin-shell.io/idta/asset/ContactInformation/1/0", result.GlobalAssetId); } - [Fact] - public async Task GetShellDescriptorTemplateAsync_DeserializesAasNestedTypes_WhenResponseContainsAasPayload() - { - const string JsonResponse = """ - { - "description": [ - { - "language": "en", - "text": "Template Asset Administration Shell for example environments." - } - ], - "displayName": [ - { - "language": "en", - "text": "AAS Template" - } - ], - "extensions": [ - { - "name": "templateSource", - "valueType": "xs:string", - "value": "ShellTemplate" - } - ], - "administration": { - "version": "1", - "revision": "0" - }, - "assetKind": "Instance", - "id": "https://mm-software.com/aas/aasTemplate", - "specificAssetIds": [ - { - "name": "SerialNumber", - "value": "SN-9429", - "externalSubjectId": { - "type": "ExternalReference", - "keys": [ - { - "type": "GlobalReference", - "value": "https://example.com/subjects/serial-number" - } - ] - } - } - ] - } - """; - - using var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK); - mockHttpResponse.Content = new StringContent(JsonResponse); - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasRegistry).Returns(httpClient); - - var result = await _sut.GetShellDescriptorTemplateAsync(TemplateId, CancellationToken.None); - - Assert.NotNull(result.Description); - Assert.Equal("en", result.Description![0].Language); - Assert.Equal("Template Asset Administration Shell for example environments.", result.Description[0].Text); - - Assert.NotNull(result.DisplayName); - Assert.Equal("AAS Template", result.DisplayName![0].Text); - - Assert.NotNull(result.Extensions); - Assert.Equal("templateSource", result.Extensions![0].Name); - Assert.Equal(DataTypeDefXsd.String, result.Extensions[0].ValueType); - Assert.Equal("ShellTemplate", result.Extensions[0].Value); - - Assert.NotNull(result.Administration); - Assert.Equal("1", result.Administration!.Version); - Assert.Equal("0", result.Administration.Revision); - - Assert.NotNull(result.SpecificAssetIds); - Assert.Equal("SerialNumber", result.SpecificAssetIds![0].Name); - Assert.NotNull(result.SpecificAssetIds[0].ExternalSubjectId); - Assert.Equal("https://example.com/subjects/serial-number", result.SpecificAssetIds[0].ExternalSubjectId!.Keys[0].Value); - } + [Fact] + public async Task GetShellDescriptorTemplateAsync_DeserializesAasNestedTypes_WhenResponseContainsAasPayload() + { + const string JsonResponse = """ + { + "description": [ + { + "language": "en", + "text": "Template Asset Administration Shell for example environments." + } + ], + "displayName": [ + { + "language": "en", + "text": "AAS Template" + } + ], + "extensions": [ + { + "name": "templateSource", + "valueType": "xs:string", + "value": "ShellTemplate" + } + ], + "administration": { + "version": "1", + "revision": "0" + }, + "assetKind": "Instance", + "id": "https://mm-software.com/aas/aasTemplate", + "specificAssetIds": [ + { + "name": "SerialNumber", + "value": "SN-9429", + "externalSubjectId": { + "type": "ExternalReference", + "keys": [ + { + "type": "GlobalReference", + "value": "https://example.com/subjects/serial-number" + } + ] + } + } + ] + } + """; + + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasRegistry, Arg.Any(), Arg.Any()) + .Returns(JsonResponse); + + var result = await _sut.GetShellDescriptorTemplateAsync(TemplateId, CancellationToken.None); + + Assert.NotNull(result.Description); + Assert.Equal("en", result.Description![0].Language); + Assert.Equal("Template Asset Administration Shell for example environments.", result.Description[0].Text); + + Assert.NotNull(result.DisplayName); + Assert.Equal("AAS Template", result.DisplayName![0].Text); + + Assert.NotNull(result.Extensions); + Assert.Equal("templateSource", result.Extensions![0].Name); + Assert.Equal(DataTypeDefXsd.String, result.Extensions[0].ValueType); + Assert.Equal("ShellTemplate", result.Extensions[0].Value); + + Assert.NotNull(result.Administration); + Assert.Equal("1", result.Administration!.Version); + Assert.Equal("0", result.Administration.Revision); + + Assert.NotNull(result.SpecificAssetIds); + Assert.Equal("SerialNumber", result.SpecificAssetIds![0].Name); + Assert.NotNull(result.SpecificAssetIds[0].ExternalSubjectId); + Assert.Equal("https://example.com/subjects/serial-number", result.SpecificAssetIds[0].ExternalSubjectId!.Keys[0].Value); + } [Fact] public async Task GetShellDescriptorTemplateAsync_ThrowsResponseParsingException_WhenInvalidJsonResponse() { - using var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK); - mockHttpResponse.Content = new StringContent("{ invalid json }"); - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasRegistry, Arg.Any(), Arg.Any()) + .Returns("{ invalid json }"); await Assert.ThrowsAsync(() => _sut.GetShellDescriptorTemplateAsync(TemplateId, CancellationToken.None)); } @@ -167,12 +171,8 @@ public async Task GetShellDescriptorTemplateAsync_ThrowsResponseParsingException [Fact] public async Task GetShellDescriptorTemplateAsync_ThrowsResponseParsingException_WhenDeserializationReturnsNull() { - using var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK); - mockHttpResponse.Content = new StringContent("null"); - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasRegistry, Arg.Any(), Arg.Any()) + .Returns("null"); await Assert.ThrowsAsync(() => _sut.GetShellDescriptorTemplateAsync(TemplateId, CancellationToken.None)); } @@ -180,12 +180,8 @@ public async Task GetShellDescriptorTemplateAsync_ThrowsResponseParsingException [Fact] public async Task GetShellDescriptorTemplateAsync_ThrowsResourceNotFoundException_WhenNotFoundResponse() { - using var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.NotFound); - mockHttpResponse.Content = new StringContent("Not found"); - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasRegistry, Arg.Any(), Arg.Any()) + .ThrowsAsync(new ResourceNotFoundException()); await Assert.ThrowsAsync(() => _sut.GetShellDescriptorTemplateAsync(TemplateId, CancellationToken.None)); } @@ -193,10 +189,8 @@ public async Task GetShellDescriptorTemplateAsync_ThrowsResourceNotFoundExceptio [Fact] public async Task GetShellDescriptorTemplateAsync_ThrowsException_WhenHttpClientFails() { - using var mockHttpMessageHandler = new FakeHttpMessageHandler(new HttpRequestException("Network error")); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasRegistry, Arg.Any(), Arg.Any()) + .ThrowsAsync(new HttpRequestException("Network error")); var exception = await Assert.ThrowsAsync(() => _sut.GetShellDescriptorTemplateAsync(TemplateId, CancellationToken.None)); Assert.Equal("Network error", exception.Message); @@ -216,17 +210,17 @@ public async Task GetShellDescriptorTemplateAsync_UsesBase64UrlEncodedTemplateId } """; - using var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK); - mockHttpResponse.Content = new StringContent(JsonResponse); - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse, request => requestPath = request.RequestUri?.ToString() ?? string.Empty); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasRegistry, Arg.Any(), Arg.Any()) + .Returns(info => + { + requestPath = info.ArgAt(0); + return JsonResponse; + }); _ = await _sut.GetShellDescriptorTemplateAsync(templateId, CancellationToken.None); var expectedEncodedTemplateId = Convert.ToBase64String(Encoding.UTF8.GetBytes(templateId)).TrimEnd('=').Replace('+', '-').Replace('/', '_'); - Assert.EndsWith($"/shell-descriptors/{expectedEncodedTemplateId}", requestPath, StringComparison.Ordinal); + Assert.EndsWith($"shell-descriptors/{expectedEncodedTemplateId}", requestPath, StringComparison.Ordinal); } [Fact] @@ -243,12 +237,8 @@ public async Task GetShellDescriptorTemplateAsync_ReturnsShellDescriptor_WhenRes } """; - using var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK); - mockHttpResponse.Content = new StringContent(JsonResponse); - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasRegistry, Arg.Any(), Arg.Any()) + .Returns(JsonResponse); var result = await _sut.GetShellDescriptorTemplateAsync(TemplateId, CancellationToken.None); @@ -271,12 +261,8 @@ public async Task GetShellDescriptorTemplateAsync_DeserializesAssetKindAndAssetT } """; - using var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK); - mockHttpResponse.Content = new StringContent(JsonResponse); - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasRegistry, Arg.Any(), Arg.Any()) + .Returns(JsonResponse); var result = await _sut.GetShellDescriptorTemplateAsync(TemplateId, CancellationToken.None); @@ -295,12 +281,8 @@ public async Task GetShellDescriptorTemplateAsync_ReturnsNullAssetKindAndAssetTy } """; - using var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK); - mockHttpResponse.Content = new StringContent(JsonResponse); - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasRegistry).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasRegistry, Arg.Any(), Arg.Any()) + .Returns(JsonResponse); var result = await _sut.GetShellDescriptorTemplateAsync(TemplateId, CancellationToken.None); @@ -315,15 +297,11 @@ public async Task GetShellDescriptorTemplateAsync_ReturnsNullAssetKindAndAssetTy Assert.Null(result.SubmodelDescriptors); } - [Fact] public async Task GetShellTemplateAsync_ReturnsShell_WhenValidResponse() { - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(ProviderTestData.ValidateShellResponse) }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .Returns(ProviderTestData.ValidateShellResponse); var result = await _sut.GetShellTemplateAsync(TemplateId, CancellationToken.None); @@ -335,14 +313,8 @@ public async Task GetShellTemplateAsync_ReturnsShell_WhenValidResponse() [Fact] public async Task GetShellTemplateAsync_ThrowsResponseParsingException_WhenInvalidJsonResponse() { - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("{ invalid json }") - }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .Returns("{ invalid json }"); await Assert.ThrowsAsync(() => _sut.GetShellTemplateAsync(TemplateId, CancellationToken.None)); } @@ -350,40 +322,27 @@ public async Task GetShellTemplateAsync_ThrowsResponseParsingException_WhenInval [Fact] public async Task GetShellTemplateAsync_ThrowsResourceNotFoundException_WhenNotFoundResponse() { - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.NotFound) - { - Content = new StringContent("Not found") - }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .ThrowsAsync(new ResourceNotFoundException()); - await Assert.ThrowsAsync(() => - _sut.GetShellTemplateAsync(TemplateId, CancellationToken.None)); + await Assert.ThrowsAsync(() => _sut.GetShellTemplateAsync(TemplateId, CancellationToken.None)); } [Fact] public async Task GetShellTemplateAsync_ThrowsException_WhenHttpClientFails() { - using var mockHttpMessageHandler = new FakeHttpMessageHandler(new HttpRequestException("Network error")); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .ThrowsAsync(new HttpRequestException("Network error")); - var exception = await Assert.ThrowsAsync(() => - _sut.GetShellTemplateAsync(TemplateId, CancellationToken.None)); + var exception = await Assert.ThrowsAsync(() => _sut.GetShellTemplateAsync(TemplateId, CancellationToken.None)); Assert.Equal("Network error", exception.Message); } [Fact] public async Task GetAssetInformationTemplateAsync_ReturnsAssetInformation_WhenValidResponse() { - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(ProviderTestData.ValidateAssetInformationResponse) }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .Returns(ProviderTestData.ValidateAssetInformationResponse); var result = await _sut.GetAssetInformationTemplateAsync(TemplateId, CancellationToken.None); @@ -394,14 +353,8 @@ public async Task GetAssetInformationTemplateAsync_ReturnsAssetInformation_WhenV [Fact] public async Task GetAssetInformationTemplateAsync_ThrowsResponseParsingException_WhenInvalidJsonResponse() { - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("{ invalid json }") - }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .Returns("{ invalid json }"); await Assert.ThrowsAsync(() => _sut.GetAssetInformationTemplateAsync(TemplateId, CancellationToken.None)); } @@ -409,43 +362,27 @@ public async Task GetAssetInformationTemplateAsync_ThrowsResponseParsingExceptio [Fact] public async Task GetAssetInformationTemplateAsync_ThrowsResourceNotFoundException_WhenNotFoundResponse() { - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.NotFound) - { - Content = new StringContent("Not found") - }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .ThrowsAsync(new ResourceNotFoundException()); - await Assert.ThrowsAsync(() => - _sut.GetAssetInformationTemplateAsync(TemplateId, CancellationToken.None)); + await Assert.ThrowsAsync(() => _sut.GetAssetInformationTemplateAsync(TemplateId, CancellationToken.None)); } [Fact] public async Task GetAssetInformationTemplateAsync_ThrowsException_WhenHttpClientFails() { - using var mockHttpMessageHandler = new FakeHttpMessageHandler(new HttpRequestException("Network error")); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .ThrowsAsync(new HttpRequestException("Network error")); - var exception = await Assert.ThrowsAsync(() => - _sut.GetAssetInformationTemplateAsync(TemplateId, CancellationToken.None)); + var exception = await Assert.ThrowsAsync(() => _sut.GetAssetInformationTemplateAsync(TemplateId, CancellationToken.None)); Assert.Equal("Network error", exception.Message); } [Fact] public async Task GetSubmodelRefByIdAsync_ReturnsSubmodelRefs_WhenValidResponse() { - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(ProviderTestData.ValidateSubmodelRefResponse) - }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .Returns(ProviderTestData.ValidateSubmodelRefResponse); var result = await _sut.GetSubmodelRefByIdAsync(TemplateId, CancellationToken.None); @@ -457,14 +394,8 @@ public async Task GetSubmodelRefByIdAsync_ReturnsSubmodelRefs_WhenValidResponse( [Fact] public async Task GetSubmodelRefByIdAsync_ThrowsResourceNotFoundException_WhenResultArrayMissing() { - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("{ \"unexpected\": [] }") - }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .Returns("{ \"unexpected\": [] }"); await Assert.ThrowsAsync(() => _sut.GetSubmodelRefByIdAsync(TemplateId, CancellationToken.None)); } @@ -472,14 +403,8 @@ public async Task GetSubmodelRefByIdAsync_ThrowsResourceNotFoundException_WhenRe [Fact] public async Task GetSubmodelRefByIdAsync_ThrowsResourceNotFoundException_WhenResultArrayIsEmpty() { - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("{ \"result\": [] }") - }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .Returns("{ \"result\": [] }"); await Assert.ThrowsAsync(() => _sut.GetSubmodelRefByIdAsync(TemplateId, CancellationToken.None)); } @@ -487,14 +412,8 @@ public async Task GetSubmodelRefByIdAsync_ThrowsResourceNotFoundException_WhenRe [Fact] public async Task GetSubmodelRefByIdAsync_ThrowsResponseParsingException_WhenInvalidJson() { - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("{ invalid json }") - }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .Returns("{ invalid json }"); await Assert.ThrowsAsync(() => _sut.GetSubmodelRefByIdAsync(TemplateId, CancellationToken.None)); } @@ -502,13 +421,10 @@ public async Task GetSubmodelRefByIdAsync_ThrowsResponseParsingException_WhenInv [Fact] public async Task GetSubmodelRefByIdAsync_ThrowsHttpRequestException_WhenHttpFails() { - using var mockHttpMessageHandler = new FakeHttpMessageHandler(new HttpRequestException("Network error")); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.AasTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.AasTemplateRepository, Arg.Any(), Arg.Any()) + .ThrowsAsync(new HttpRequestException("Network error")); - var exception = await Assert.ThrowsAsync(() => - _sut.GetSubmodelRefByIdAsync(TemplateId, CancellationToken.None)); + var exception = await Assert.ThrowsAsync(() => _sut.GetSubmodelRefByIdAsync(TemplateId, CancellationToken.None)); Assert.Equal("Network error", exception.Message); } @@ -516,15 +432,8 @@ public async Task GetSubmodelRefByIdAsync_ThrowsHttpRequestException_WhenHttpFai public async Task GetConceptDescriptionByIdAsync_ReturnsConceptDescription_WhenResponseIsValid() { const string CdIdentifier = "test-id"; - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(ProviderTestData.ValidConceptDescription) - }; - using var mockHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.ConceptDescriptorTemplateRepository) - .Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.ConceptDescriptorTemplateRepository, Arg.Any(), Arg.Any()) + .Returns(ProviderTestData.ValidConceptDescription); var result = await _sut.GetConceptDescriptionByIdAsync(CdIdentifier, CancellationToken.None); @@ -541,11 +450,8 @@ public async Task GetConceptDescriptionByIdAsync_ReturnsNull_OnHandledExceptions { const string CdIdentifier = "test-id"; var exception = (Exception)Activator.CreateInstance(exceptionType)!; - using var mockHandler = new FakeHttpMessageHandler(exception); - using var httpClient = new HttpClient(mockHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.ConceptDescriptorTemplateRepository) - .Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.ConceptDescriptorTemplateRepository, Arg.Any(), Arg.Any()) + .ThrowsAsync(exception); var result = await _sut.GetConceptDescriptionByIdAsync(CdIdentifier, CancellationToken.None); @@ -558,11 +464,8 @@ public async Task GetSubmodelTemplateAsync_StartsFetchTemplateSpan_WithTemplateI const string TemplateIdForSpan = "Nameplate"; using var fixture = CreateFixture(); - var mockHttpResponse = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(ProviderTestData.ValidateSubmodelResponse) }; - using var mockHttpMessageHandler = new FakeHttpMessageHandler(mockHttpResponse); - using var httpClient = new HttpClient(mockHttpMessageHandler); - httpClient.BaseAddress = new Uri("https://www.mm-software.com/fakeurl"); - _httpClientFactory.CreateClient(HttpClientNames.SubmodelTemplateRepository).Returns(httpClient); + _cachedHttp.GetStringAsync(Arg.Any(), HttpClientNames.SubmodelTemplateRepository, Arg.Any(), Arg.Any()) + .Returns(ProviderTestData.ValidateSubmodelResponse); _ = await _sut.GetFilteredSubmodelTemplateAsync(TemplateIdForSpan, null, CancellationToken.None); @@ -572,25 +475,3 @@ public async Task GetSubmodelTemplateAsync_StartsFetchTemplateSpan_WithTemplateI Assert.Equal(TemplateIdForSpan, span.GetTagItem(DataEngineTracing.Attributes.TemplateId)); } } - -public class FakeHttpMessageHandler : HttpMessageHandler -{ - private readonly HttpResponseMessage _response = null!; - private readonly Exception? _exceptionToThrow; - private readonly Action? _onRequest; - - public FakeHttpMessageHandler(HttpResponseMessage response, Action? onRequest = null) - { - _response = response; - _onRequest = onRequest; - } - - public FakeHttpMessageHandler(Exception exception) => _exceptionToThrow = exception; - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) - { - _onRequest?.Invoke(request); - return _exceptionToThrow != null ? throw _exceptionToThrow : Task.FromResult(_response); - } - -} diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/packages.lock.json b/source/AAS.TwinEngine.DataEngine.UnitTests/packages.lock.json index 7b04f209..46b6d900 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/packages.lock.json +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/packages.lock.json @@ -8,6 +8,27 @@ "resolved": "10.0.1", "contentHash": "27jXSV/0DbVqF5jDrAxuQFZ9oaz6gmG03p8ttxAFk+X0M4woFYj7MoWDLCna5EGLb0CE6OE7X6ZH3Wt5smTtaA==" }, + "Microsoft.Extensions.Caching.Hybrid": { + "type": "Direct", + "requested": "[10.8.0, )", + "resolved": "10.8.0", + "contentHash": "RCtCTK3eqKXx8LXqd7B8GjbTkIp4k3EgKeunaCmBJ+WA55Nva83CI2I+wVyp0S9jTG+aT6AYWQbkKOfvFOlmLA==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.10", + "Microsoft.Extensions.Caching.Memory": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Direct", + "requested": "[10.0.10, )", + "resolved": "10.0.10", + "contentHash": "ANyvsgkNBRvcJh2XLgn8veGmajf+8m0AbKK+HPWdRL1yraSNVVSmQhFntLtdz/C795jxqqup+k05cs/3jZQPOA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" + } + }, "Microsoft.NET.Test.Sdk": { "type": "Direct", "requested": "[18.6.0, 18.6.0]", @@ -161,6 +182,26 @@ "resolved": "10.0.0", "contentHash": "NCWCGiwRwje8773yzPQhvucYnnfeR+ZoB1VRIrIMp4uaeUNw7jvEPHij3HIbwCDuNCrNcphA00KSAR9yD9qmbg==" }, + "Microsoft.Extensions.Caching.Abstractions": { + "type": "Transitive", + "resolved": "10.0.10", + "contentHash": "4ZFBNE+jzR+CrWWlhOesnmywCW7pYKT0dxyAQRdL11yJwxe4jvcAu31eorFtEkoFeCDcUTeNssgPv2yaRRptaQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.10" + } + }, + "Microsoft.Extensions.Caching.Memory": { + "type": "Transitive", + "resolved": "10.0.10", + "contentHash": "N1w5H7uK6gCTnCBZAWzE0/EQYSPysij/uYwDqntqBVvBa6bjMmBKitsnEFd6yh/SX3wLm67nO6+OnZ84K+gZWg==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.10", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10", + "Microsoft.Extensions.Logging.Abstractions": "10.0.10", + "Microsoft.Extensions.Options": "10.0.10", + "Microsoft.Extensions.Primitives": "10.0.10" + } + }, "Microsoft.Extensions.Compliance.Abstractions": { "type": "Transitive", "resolved": "10.6.0", @@ -196,18 +237,10 @@ "Microsoft.Extensions.Configuration.Abstractions": "10.0.8" } }, - "Microsoft.Extensions.DependencyInjection": { - "type": "Transitive", - "resolved": "10.0.8", - "contentHash": "daf62xHIrq8pnE709hgaZZN9tSam9TGGepWe1+bE6V3GEuVwJiMs6ib+38lfMCyAJAHiX0vapxBhsuMSV7U+cg==", - "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.8" - } - }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "10.0.8", - "contentHash": "21nbDV60SRPWGIivsyl6lqBeEJNG1sginhhfWgRrr3Ais7aQ12To25OAHQxgoiJkjqy1aQ6RxpZBGYuTi7Ge6A==" + "resolved": "10.0.10", + "contentHash": "z/2xXlFw2aLGjHyEm6E0tQ+In6VfzQzTrtArbQ2c0TQE16ZbyDCMGPvaUT9I0s8rgy9sRWlU2P9waW37qV04qA==" }, "Microsoft.Extensions.DependencyInjection.AutoActivation": { "type": "Transitive", @@ -313,10 +346,10 @@ }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "10.0.8", - "contentHash": "fdVadZmsC8jRP0KvKy8mO8f6GV/HyBvElfcSxEhd+5FM5boAw/01iSaCto5G3G37ApJira4A3pNaVvBv8cUiLQ==", + "resolved": "10.0.10", + "contentHash": "zkFxGYUvdxAvIKTyXHrmW+Sux53D4SezD9dMyZ6hrwwzPQJNuwCRy1f5W7AvYTqacEGhWF2XderRQG1OvbV8og==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.8" + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10" } }, "Microsoft.Extensions.Logging.Configuration": { @@ -341,11 +374,11 @@ }, "Microsoft.Extensions.Options": { "type": "Transitive", - "resolved": "10.0.8", - "contentHash": "VBD+131DpTNCNDfA4kIyKTiCySvJGNhwibdWBSdFRu7GMfXLXcXODkgA+KStKbbhzraLglZWUN4nXyHgW4JIRA==", + "resolved": "10.0.10", + "contentHash": "srnhnk7nE8krBiIXp71LvBmKBtraBONWSRzdjJgRv1Ko9Mp8IVNqv4vIS9hGeVteBig8aQkva9ZG+sC+o5sVcA==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.8", - "Microsoft.Extensions.Primitives": "10.0.8" + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.10", + "Microsoft.Extensions.Primitives": "10.0.10" } }, "Microsoft.Extensions.Options.ConfigurationExtensions": { @@ -362,8 +395,8 @@ }, "Microsoft.Extensions.Primitives": { "type": "Transitive", - "resolved": "10.0.8", - "contentHash": "OBPo4nYhMyIbtueoC10CBm6AGAbo/A9IV8QQ/6ryZS7VvmqpGT7hunazeHLxFawRzn3oLOq4jhqhpBX4tfswWQ==" + "resolved": "10.0.10", + "contentHash": "5wu/GrYVd8mG2DVUw3vFJzF+O336TyTGg/Kmcgw9bfwYhCoFiV5lR5QeEmKecJyrW4W54nMfD3p3589E8a7czQ==" }, "Microsoft.Extensions.Resilience": { "type": "Transitive", @@ -821,6 +854,7 @@ "Asp.Versioning.Mvc": "[10.0.0, 10.0.0]", "Cronos": "[0.13.0, 0.13.0]", "JsonSchema.Net": "[7.3.4, 7.3.4]", + "Microsoft.Extensions.Caching.Hybrid": "[10.8.0, )", "Microsoft.Extensions.DependencyInjection": "[10.0.8, 10.0.8]", "Microsoft.Extensions.Http": "[10.0.8, 10.0.8]", "Microsoft.Extensions.Http.Resilience": "[10.6.0, 10.6.0]", diff --git a/source/AAS.TwinEngine.DataEngine/AAS.TwinEngine.DataEngine.csproj b/source/AAS.TwinEngine.DataEngine/AAS.TwinEngine.DataEngine.csproj index 6d6e6916..6a27750e 100644 --- a/source/AAS.TwinEngine.DataEngine/AAS.TwinEngine.DataEngine.csproj +++ b/source/AAS.TwinEngine.DataEngine/AAS.TwinEngine.DataEngine.csproj @@ -25,6 +25,7 @@ + diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs new file mode 100644 index 00000000..83407e9e --- /dev/null +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs @@ -0,0 +1,104 @@ +using System.Net; +using System.Security.Claims; +using System.Security.Cryptography; +using System.Text; + +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; +using AAS.TwinEngine.DataEngine.Infrastructure.Logging; +using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; + +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.Caching.Hybrid; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +using UnauthorizedAccessException = AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException; + +namespace AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients.Caching; + +public sealed class CachedGetRequestClient( + ICreateClient clientFactory, + HybridCache cache, + IHttpContextAccessor httpContextAccessor, + ILogger logger) : ICachedGetRequestClient +{ + public async Task GetStringAsync(string relativeUrl, string httpClientName, int expirationTime, CancellationToken cancellationToken) + { + var cacheKey = BuildCacheKey(httpContextAccessor, relativeUrl); + + var entryOptions = new HybridCacheEntryOptions + { + Expiration = TimeSpan.FromMinutes(expirationTime), + LocalCacheExpiration = TimeSpan.FromMinutes(expirationTime) + }; + + return await cache.GetOrCreateAsync( + cacheKey, + async token => await FetchAsync(relativeUrl, httpClientName, token).ConfigureAwait(false), + options: entryOptions, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private async Task FetchAsync(string relativeUrl, string httpClientName, CancellationToken cancellationToken) + { + logger.LogInformation("Sending HTTP GET request to {Url}", LogSanitizerExtension.Sanitize(relativeUrl)); + + var httpClient = clientFactory.CreateClient(httpClientName); + var relativeUri = new Uri(relativeUrl, UriKind.Relative); + + var response = await httpClient.GetAsync(relativeUri, cancellationToken).ConfigureAwait(false); + + if (response.IsSuccessStatusCode) + { + logger.LogInformation("Received successful HTTP response with status code: {StatusCode}", response.StatusCode); + return await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + } + + var responseContent = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + logger.LogError("HTTP GET failed with status {StatusCode}. Response: {ResponseMessage}", response.StatusCode, responseContent); + + throw response.StatusCode switch + { + HttpStatusCode.NotFound => new ResourceNotFoundException(), + HttpStatusCode.Unauthorized or HttpStatusCode.Forbidden => new UnauthorizedAccessException(), + HttpStatusCode.RequestTimeout => new RequestTimeoutException(), + _ => new ValidationFailedException() + }; + } + + private static string BuildCacheKey(IHttpContextAccessor httpContextAccessor, string requestParts) + { + var requestHash = ComputeHash(requestParts); + + var user = httpContextAccessor.HttpContext?.User; + + if (user?.Identity is { IsAuthenticated: true }) + { + var userId = user.FindFirst(ClaimTypes.NameIdentifier)?.Value + ?? user.FindFirst("sub")?.Value + ?? "unknown"; + + var permissionHash = ComputePermissionHash(user.Claims); + + return $"user:{userId}:ph:{permissionHash}:req:{requestHash}"; + } + + return $"anonymous:req:{requestHash}"; + } + + private static string ComputePermissionHash(IEnumerable claims) + { + var claimsString = string.Join("|", claims + .OrderBy(c => c.Type, StringComparer.Ordinal) + .ThenBy(c => c.Value, StringComparer.Ordinal) + .Select(c => $"{c.Type}={c.Value}")); + + return ComputeHash(claimsString); + } + + private static string ComputeHash(string input) + { + var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(input)); + return Convert.ToHexStringLower(bytes); + } +} diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/ICachedGetRequestClient.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/ICachedGetRequestClient.cs new file mode 100644 index 00000000..f21add42 --- /dev/null +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/ICachedGetRequestClient.cs @@ -0,0 +1,6 @@ +namespace AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients.Caching; + +public interface ICachedGetRequestClient +{ + Task GetStringAsync(string relativeUrl, string httpClientName, int expirationTime, CancellationToken cancellationToken); +} diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/SubmodelRegistryProvider/Services/SubmodelDescriptorProvider.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/SubmodelRegistryProvider/Services/SubmodelDescriptorProvider.cs index eab9c1c3..40850595 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/SubmodelRegistryProvider/Services/SubmodelDescriptorProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/SubmodelRegistryProvider/Services/SubmodelDescriptorProvider.cs @@ -1,26 +1,29 @@ -using System.Net; +using System.Net; using System.Text.Json; using System.Text.Json.Nodes; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Observability; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.SubmodelRegistry.Providers; using AAS.TwinEngine.DataEngine.DomainModel.Shared; using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRegistry; -using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients; +using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients.Caching; using AAS.TwinEngine.DataEngine.Infrastructure.Shared; -using AAS.TwinEngine.DataEngine.ApplicationLogic.Observability; using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; using AasCore.Aas3_1; +using Microsoft.Extensions.Options; + using UnauthorizedAccessException = AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException; namespace AAS.TwinEngine.DataEngine.Infrastructure.Providers.SubmodelRegistryProvider.Services; -public class SubmodelDescriptorProvider(ILogger logger, ICreateClient clientFactory) : ISubmodelDescriptorProvider +public class SubmodelDescriptorProvider(ILogger logger, IOptions options, ICachedGetRequestClient cachedHttp) : ISubmodelDescriptorProvider { private const string SubModelRegistryPath = ApiPaths.SubmodelDescriptors; + private readonly TemplateManagementConfig _config = options.Value; public async Task GetDataForSubmodelDescriptorByIdAsync(string id, CancellationToken cancellationToken) { @@ -30,15 +33,7 @@ public async Task GetDataForSubmodelDescriptorByIdAsync(stri var url = $"/{SubModelRegistryPath}/{encodedAasId}"; - var relativeUri = new Uri(url, UriKind.Relative); - - var httpClient = clientFactory.CreateClient(HttpClientNames.SubmodelRegistry); - - var response = await httpClient.GetAsync(relativeUri, cancellationToken).ConfigureAwait(false); - - var httpResponseContent = await ProcessResponseAsync(response, url, cancellationToken).ConfigureAwait(false); - - var responseContent = await httpResponseContent.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var responseContent = await cachedHttp.GetStringAsync(url, HttpClientNames.SubmodelRegistry, _config.SubmodelTemplateRegistry.LocalCacheExpirationInMinutes, cancellationToken).ConfigureAwait(false); try { @@ -81,39 +76,4 @@ public async Task GetDataForSubmodelDescriptorByIdAsync(stri Endpoints = descriptorNode["endpoints"]?.Deserialize>() }; } - - private async Task ProcessResponseAsync(HttpResponseMessage response, string url, CancellationToken cancellationToken) - { - logger.LogInformation("Sending HTTP GET request to {Url}", url); - - if (response.IsSuccessStatusCode) - { - logger.LogInformation("Received successful HTTP response from {Url} with status code: {StatusCode}", url, response.StatusCode); - return response.Content; - } - - var responseContent = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - - logger.LogError("Received HTTP response from {Url} with status code: {StatusCode}. Response message: {ResponseMessage}", url, response.StatusCode, responseContent); - - switch (response.StatusCode) - { - case HttpStatusCode.NotFound: - logger.LogError("Requested resource could not be found. Endpoint: {Url}", url); - throw new ResourceNotFoundException(); - - case HttpStatusCode.Unauthorized: - case HttpStatusCode.Forbidden: - logger.LogError("Unauthorized access. Endpoint: {Url}", url); - throw new UnauthorizedAccessException(); - - case HttpStatusCode.RequestTimeout: - logger.LogError("Request timed out. Endpoint: {Url}", url); - throw new RequestTimeoutException(); - - default: - logger.LogError("Validation error encountered. Endpoint: {Url}", url); - throw new ValidationFailedException(); - } - } } diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/TemplateProvider/Services/TemplateProvider.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/TemplateProvider/Services/TemplateProvider.cs index f4192383..8153f4e1 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/TemplateProvider/Services/TemplateProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/TemplateProvider/Services/TemplateProvider.cs @@ -1,4 +1,4 @@ -using System.Text.Json; +using System.Text.Json; using System.Text.Json.Nodes; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; @@ -7,20 +7,22 @@ using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasEnvironment.Providers; using AAS.TwinEngine.DataEngine.DomainModel.AasRegistry; using AAS.TwinEngine.DataEngine.DomainModel.Shared; -using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRegistry; -using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients; +using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; +using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients.Caching; +using AAS.TwinEngine.DataEngine.Infrastructure.Logging; using AAS.TwinEngine.DataEngine.Infrastructure.Shared; using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; using AasCore.Aas3_1; +using Microsoft.Extensions.Options; + using UnauthorizedAccessException = AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException; -using AAS.TwinEngine.DataEngine.Infrastructure.Logging; namespace AAS.TwinEngine.DataEngine.Infrastructure.Providers.TemplateProvider.Services; -public class TemplateProvider(ILogger logger, ICreateClient clientFactory) : ITemplateProvider +public class TemplateProvider(ILogger logger, IOptions options, ICachedGetRequestClient cachedHttp) : ITemplateProvider { private const string SubModelRepositoryPath = ApiPaths.Submodels; private const string AasRegistryPath = ApiPaths.ShellDescriptors; @@ -28,6 +30,8 @@ public class TemplateProvider(ILogger logger, ICreateClient cl private const string SubmodelRefPath = ApiPaths.SubmodelRefs; private const string ConceptDescriptionPath = ApiPaths.ConceptDescriptions; + private readonly TemplateManagementConfig _config = options.Value; + public async Task GetFilteredSubmodelTemplateAsync(string templateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) { using var activity = DataEngineTracing.StartSpan(DataEngineTracing.Spans.GetSubmodelTemplate, DataEngineTracing.Attributes.TemplateId, templateId); @@ -64,19 +68,14 @@ public class TemplateProvider(ILogger logger, ICreateClient cl } } - private async Task GetSubmodelFromUrlAsync( - string url, - string templateId, - string errorMessage, - CancellationToken cancellationToken) + private async Task GetSubmodelFromUrlAsync(string url, string templateId, string errorMessage, CancellationToken cancellationToken) { - var response = await SendGetRequestAsync( + var content = await SendGetRequestAsync( url, HttpClientNames.SubmodelTemplateRepository, + _config.SubmodelTemplateRepository.LocalCacheExpirationInMinutes, cancellationToken).ConfigureAwait(false); - var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - try { var jsonNode = JsonNode.Parse(content); @@ -95,12 +94,7 @@ private async Task GetSubmodelFromUrlAsync( { var url = $"{SubModelRepositoryPath}?semanticId={Uri.EscapeDataString(semanticId)}"; - var response = await SendGetRequestAsync( - url, - HttpClientNames.SubmodelTemplateRepository, - cancellationToken).ConfigureAwait(false); - - var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var content = await SendGetRequestAsync(url, HttpClientNames.SubmodelTemplateRepository, _config.SubmodelTemplateRepository.LocalCacheExpirationInMinutes, cancellationToken).ConfigureAwait(false); try { @@ -128,8 +122,7 @@ public async Task GetShellDescriptorTemplateAsync(string templa var encodedTemplateId = templateId.EncodeBase64Url(logger); var url = $"{AasRegistryPath}/{encodedTemplateId}"; - var response = await SendGetRequestAsync(url, HttpClientNames.AasRegistry, cancellationToken).ConfigureAwait(false); - var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var content = await SendGetRequestAsync(url, HttpClientNames.AasRegistry, _config.AasTemplateRegistry.LocalCacheExpirationInMinutes, cancellationToken).ConfigureAwait(false); try { @@ -159,8 +152,7 @@ public async Task GetShellTemplateAsync(string templa var encodedTemplateId = templateId.EncodeBase64Url(logger); var url = $"{AasRepositoryPath}/{encodedTemplateId}"; - var response = await SendGetRequestAsync(url, HttpClientNames.AasTemplateRepository, cancellationToken).ConfigureAwait(false); - var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var content = await SendGetRequestAsync(url, HttpClientNames.AasTemplateRepository, _config.AasTemplateRepository.LocalCacheExpirationInMinutes, cancellationToken).ConfigureAwait(false); try { @@ -188,8 +180,7 @@ public async Task GetAssetInformationTemplateAsync(string tem var encodedTemplateId = templateId.EncodeBase64Url(logger); var url = $"{AasRepositoryPath}/{encodedTemplateId}/asset-information"; - var response = await SendGetRequestAsync(url, HttpClientNames.AasTemplateRepository, cancellationToken).ConfigureAwait(false); - var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var content = await SendGetRequestAsync(url, HttpClientNames.AasTemplateRepository, _config.AasTemplateRepository.LocalCacheExpirationInMinutes, cancellationToken).ConfigureAwait(false); try { @@ -217,8 +208,7 @@ public async Task> GetSubmodelRefByIdAsync(string templateId, C var encodedTemplateId = templateId.EncodeBase64Url(logger); var url = $"{AasRepositoryPath}/{encodedTemplateId}/{SubmodelRefPath}"; - var response = await SendGetRequestAsync(url, HttpClientNames.AasTemplateRepository, cancellationToken).ConfigureAwait(false); - var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var content = await SendGetRequestAsync(url, HttpClientNames.AasTemplateRepository, _config.AasTemplateRepository.LocalCacheExpirationInMinutes, cancellationToken).ConfigureAwait(false); try { @@ -266,8 +256,7 @@ public async Task> GetSubmodelRefByIdAsync(string templateId, C try { - var response = await SendGetRequestAsync(url, HttpClientNames.ConceptDescriptorTemplateRepository, cancellationToken).ConfigureAwait(false); - var content = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var content = await SendGetRequestAsync(url, HttpClientNames.ConceptDescriptorTemplateRepository, _config.ConceptDescriptionTemplateRepository.LocalCacheExpirationInMinutes, cancellationToken).ConfigureAwait(false); var jsonNode = JsonNode.Parse(content); return Jsonization.Deserialize.ConceptDescriptionFrom(jsonNode!); } @@ -281,44 +270,11 @@ public async Task> GetSubmodelRefByIdAsync(string templateId, C } } - private async Task SendGetRequestAsync(string url, string httpClientName, CancellationToken cancellationToken) + private async Task SendGetRequestAsync(string url, string httpClientName, int expirationTime, CancellationToken cancellationToken) { logger.LogInformation("Sending HTTP GET request to {Url}", LogSanitizerExtension.Sanitize(url)); - var relativeUri = new Uri(url, UriKind.Relative); - - var httpClient = clientFactory.CreateClient(httpClientName); - - var response = await httpClient.GetAsync(relativeUri, cancellationToken).ConfigureAwait(false); - - if (response.IsSuccessStatusCode) - { - logger.LogInformation("Received successful HTTP response with status code: {StatusCode}", response.StatusCode); - return response; - } - - var responseContent = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - logger.LogError("Received HTTP GET response with status code: {StatusCode}. Response message: {ResponseMessage}", response.StatusCode, responseContent); - - switch (response.StatusCode) - { - case System.Net.HttpStatusCode.NotFound: - logger.LogError("Requested resource could not be found. Endpoint: {Url}", LogSanitizerExtension.Sanitize(url)); - throw new ResourceNotFoundException(); - - case System.Net.HttpStatusCode.Unauthorized: - case System.Net.HttpStatusCode.Forbidden: - logger.LogError("Unauthorized access. Endpoint: {Url}", LogSanitizerExtension.Sanitize(url)); - throw new UnauthorizedAccessException(); - - case System.Net.HttpStatusCode.RequestTimeout: - logger.LogError("Request timed out. Endpoint: {Url}", LogSanitizerExtension.Sanitize(url)); - throw new RequestTimeoutException(); - - default: - logger.LogError("Validation error encountered. Endpoint: {Url}", LogSanitizerExtension.Sanitize(url)); - throw new ValidationFailedException(); - } + return await cachedHttp.GetStringAsync(url, httpClientName, expirationTime, cancellationToken).ConfigureAwait(false); } private static ShellDescriptor? DeserializeShellDescriptor(JsonNode? descriptorNode) diff --git a/source/AAS.TwinEngine.DataEngine/Program.cs b/source/AAS.TwinEngine.DataEngine/Program.cs index 5183f125..ba18518b 100644 --- a/source/AAS.TwinEngine.DataEngine/Program.cs +++ b/source/AAS.TwinEngine.DataEngine/Program.cs @@ -41,6 +41,7 @@ public static async Task Main(string[] args) .ToList(); }); + _ = builder.Services.AddHybridCache(); builder.Services.ConfigureInfrastructure(builder.Configuration); builder.Services.ConfigureApplication(builder.Configuration); builder.Services.ConfigureResponseCompression(); diff --git a/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/TemplateManagementConfig.cs b/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/TemplateManagementConfig.cs index 30fb3769..8eb09ae2 100644 --- a/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/TemplateManagementConfig.cs +++ b/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/TemplateManagementConfig.cs @@ -40,4 +40,5 @@ public class ServiceInstance public IList HeaderMappings { get; init; } = []; public string HealthEndpoint { get; set; } = string.Empty; public int ConcurrentOperationsLimit { get; set; } = 10; + public int LocalCacheExpirationInMinutes { get; set; } = 5; } diff --git a/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/InfrastructureDependencyInjectionExtensions.cs b/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/InfrastructureDependencyInjectionExtensions.cs index f10ff10f..3744c9b0 100644 --- a/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/InfrastructureDependencyInjectionExtensions.cs +++ b/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/InfrastructureDependencyInjectionExtensions.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasEnvironment.Providers; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasEnvironment.Providers; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin.Helper; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin.Providers; @@ -6,6 +6,7 @@ using AAS.TwinEngine.DataEngine.Infrastructure.Configuration.LegacyV1; using AAS.TwinEngine.DataEngine.Infrastructure.Http.Authorization.Headers; using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients; +using AAS.TwinEngine.DataEngine.Infrastructure.Http.Clients.Caching; using AAS.TwinEngine.DataEngine.Infrastructure.Http.Extensions; using AAS.TwinEngine.DataEngine.Infrastructure.Monitoring; using AAS.TwinEngine.DataEngine.Infrastructure.Providers.PluginDataProvider.Helper; @@ -117,5 +118,6 @@ public static void ConfigureInfrastructure(this IServiceCollection services, ICo _ = services.AddScoped(); _ = services.AddScoped(); _ = services.AddSingleton(); + _ = services.AddScoped(); } } diff --git a/source/AAS.TwinEngine.DataEngine/appsettings.development.json b/source/AAS.TwinEngine.DataEngine/appsettings.development.json index 84979b87..b0929cb6 100644 --- a/source/AAS.TwinEngine.DataEngine/appsettings.development.json +++ b/source/AAS.TwinEngine.DataEngine/appsettings.development.json @@ -1,4 +1,4 @@ - { +{ "General": { "Logging": { "LogLevel": { @@ -181,6 +181,7 @@ "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { "source": "Authorization", @@ -199,6 +200,7 @@ "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { "source": "Authorization", @@ -217,6 +219,7 @@ "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { "source": "Authorization", @@ -235,6 +238,7 @@ "baseUrl": "http://localhost:8082", "healthEndpoint": "/actuator/health", "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { "source": "Authorization", @@ -248,6 +252,7 @@ "baseUrl": "http://localhost:8083", "healthEndpoint": "/actuator/health", "ConcurrentOperationsLimit": 12, + "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { "source": "Authorization", diff --git a/source/AAS.TwinEngine.DataEngine/appsettings.json b/source/AAS.TwinEngine.DataEngine/appsettings.json index 73e6ce40..beae2e67 100644 --- a/source/AAS.TwinEngine.DataEngine/appsettings.json +++ b/source/AAS.TwinEngine.DataEngine/appsettings.json @@ -111,35 +111,40 @@ "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", "headerMappings": [], - "ConcurrentOperationsLimit": 10 + "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5 }, "SubmodelTemplateRepository": { "Name": "SubmodelTemplateRepository", "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", "headerMappings": [], - "ConcurrentOperationsLimit": 10 + "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5 }, "ConceptDescriptionTemplateRepository": { "Name": "ConceptDescriptionTemplateRepository", "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", "headerMappings": [], - "ConcurrentOperationsLimit": 10 + "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5 }, "AasTemplateRegistry": { "Name": "AasTemplateRegistry", "baseUrl": "http://localhost:8082", "healthEndpoint": "/actuator/health", "headerMappings": [], - "ConcurrentOperationsLimit": 10 + "ConcurrentOperationsLimit": 10, + "LocalCacheExpirationInMinutes": 5 }, "SubmodelTemplateRegistry": { "Name": "SubmodelTemplateRegistry", "baseUrl": "http://localhost:8083", "healthEndpoint": "/actuator/health", "headerMappings": [], - "ConcurrentOperationsLimit": 12 + "ConcurrentOperationsLimit": 12, + "LocalCacheExpirationInMinutes": 5 } } } diff --git a/source/AAS.TwinEngine.DataEngine/packages.lock.json b/source/AAS.TwinEngine.DataEngine/packages.lock.json index 39b6e367..720b87ed 100644 --- a/source/AAS.TwinEngine.DataEngine/packages.lock.json +++ b/source/AAS.TwinEngine.DataEngine/packages.lock.json @@ -41,6 +41,12 @@ "JsonPointer.Net": "5.3.1" } }, + "Microsoft.Extensions.Caching.Hybrid": { + "type": "Direct", + "requested": "[10.8.0, )", + "resolved": "10.8.0", + "contentHash": "RCtCTK3eqKXx8LXqd7B8GjbTkIp4k3EgKeunaCmBJ+WA55Nva83CI2I+wVyp0S9jTG+aT6AYWQbkKOfvFOlmLA==" + }, "Microsoft.Extensions.DependencyInjection": { "type": "Direct", "requested": "[10.0.8, 10.0.8]", From 2a99f7a28657e4bb2a51bb2aa50328fba6830469 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Thu, 23 Jul 2026 13:03:19 +0530 Subject: [PATCH 02/30] feat: implement ShellDescriptorService to retrieve and populate AAS shell descriptors with template mapping support --- .../ShellDescriptorServiceTests.cs | 161 +++++++++++++++++- .../Services/TemplateProviderTests.cs | 4 +- .../AasRegistry/ShellDescriptorService.cs | 40 +++-- source/AAS.TwinEngine.DataEngine/Program.cs | 5 +- 4 files changed, 189 insertions(+), 21 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRegistry/ShellDescriptorServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRegistry/ShellDescriptorServiceTests.cs index 0d9e8cc7..9ac1217e 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRegistry/ShellDescriptorServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRegistry/ShellDescriptorServiceTests.cs @@ -6,10 +6,12 @@ using AAS.TwinEngine.DataEngine.DomainModel.AasRegistry; using AAS.TwinEngine.DataEngine.DomainModel.Plugin; using AAS.TwinEngine.DataEngine.DomainModel.Shared; +using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; using AasCore.Aas3_1; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using NSubstitute; using NSubstitute.ExceptionExtensions; @@ -24,9 +26,18 @@ public class ShellDescriptorServiceTests private readonly IShellDescriptorDataHandler _dataHandler = Substitute.For(); private readonly IPluginManifestConflictHandler _pluginManifestConflictHandler = Substitute.For(); private readonly ILogger _logger = Substitute.For>(); + private readonly IOptions _templateManagementConfig; private readonly ShellDescriptorService _sut; - public ShellDescriptorServiceTests() => _sut = new ShellDescriptorService(_templateProvider, _shellTemplateMappingProvider, _dataHandler, _pluginDataHandler, _pluginManifestConflictHandler, _logger); + public ShellDescriptorServiceTests() + { + var config = new TemplateManagementConfig + { + AasTemplateRegistry = new ServiceInstance { ConcurrentOperationsLimit = 10 } + }; + _templateManagementConfig = Options.Create(config); + _sut = new ShellDescriptorService(_templateProvider, _shellTemplateMappingProvider, _dataHandler, _pluginDataHandler, _pluginManifestConflictHandler, _logger, _templateManagementConfig); + } [Fact] public async Task GetAllShellDescriptorsAsync_ReturnsFilledShellDescriptors() @@ -307,8 +318,152 @@ public async Task GetShellDescriptorByIdAsync_ShouldThrowShellDescriptorNotFound await Assert.ThrowsAsync(() => _sut.GetShellDescriptorByIdAsync(id, cancellationToken)); } + [Fact] + public async Task GetAllShellDescriptorsAsync_BuildsDescriptorsInParallel() + { + var cancellationToken = CancellationToken.None; + var metadataList = Enumerable.Range(1, 5) + .Select(i => new ShellDescriptorMetaData { Id = $"id{i}" }) + .ToList(); + var metaData = new ShellDescriptorsMetaData + { + PagingMetaData = null, + ShellDescriptors = metadataList + }; + + _pluginManifestConflictHandler.Manifests.Returns(new List()); + _pluginDataHandler.GetDataForAllShellDescriptorsAsync(null, null, Arg.Any>(), cancellationToken) + .Returns(metaData); + _shellTemplateMappingProvider.GetTemplateId(Arg.Any()).Returns("template-1"); + _templateProvider.GetShellDescriptorTemplateAsync("template-1", cancellationToken).Returns(GetShellDescriptorTemplate()); + _dataHandler.FillOut(Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var value = callInfo.ArgAt(1); + return new ShellDescriptor { Id = value.Id }; + }); + + var result = await _sut.GetAllShellDescriptorsAsync(null, null, cancellationToken); + + Assert.NotNull(result); + Assert.NotNull(result.Result); + Assert.Equal(5, result.Result.Count); + Assert.All(result.Result, descriptor => Assert.False(string.IsNullOrWhiteSpace(descriptor.Id))); + } + + [Fact] + public async Task GetAllShellDescriptorsAsync_RespectsMaxConcurrency() + { + var cancellationToken = CancellationToken.None; + const int concurrencyLimit = 2; + var config = new TemplateManagementConfig + { + AasTemplateRegistry = new ServiceInstance { ConcurrentOperationsLimit = concurrencyLimit } + }; + var sut = new ShellDescriptorService( + _templateProvider, _shellTemplateMappingProvider, _dataHandler, + _pluginDataHandler, _pluginManifestConflictHandler, _logger, + Options.Create(config)); + + var currentConcurrency = 0; + var maxObservedConcurrency = 0; + var lockObj = new object(); + + var metadataList = Enumerable.Range(1, 6) + .Select(i => new ShellDescriptorMetaData { Id = $"id{i}" }) + .ToList(); + var metaData = new ShellDescriptorsMetaData + { + PagingMetaData = null, + ShellDescriptors = metadataList + }; + + _pluginManifestConflictHandler.Manifests.Returns(new List()); + _pluginDataHandler.GetDataForAllShellDescriptorsAsync(null, null, Arg.Any>(), cancellationToken) + .Returns(metaData); + _shellTemplateMappingProvider.GetTemplateId(Arg.Any()).Returns("template-1"); + + _templateProvider.GetShellDescriptorTemplateAsync("template-1", cancellationToken) + .Returns(async callInfo => + { + lock (lockObj) + { + currentConcurrency++; + if (currentConcurrency > maxObservedConcurrency) + maxObservedConcurrency = currentConcurrency; + } + + await Task.Delay(50, cancellationToken); + + lock (lockObj) + { + currentConcurrency--; + } + + return GetShellDescriptorTemplate(); + }); + + _dataHandler.FillOut(Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var value = callInfo.ArgAt(1); + return new ShellDescriptor { Id = value.Id }; + }); + + var result = await sut.GetAllShellDescriptorsAsync(null, null, cancellationToken); + + Assert.NotNull(result); + Assert.Equal(6, result.Result!.Count); + Assert.True(maxObservedConcurrency <= concurrencyLimit, + $"Expected max concurrency <= {concurrencyLimit}, but observed {maxObservedConcurrency}"); + } + + [Fact] + public async Task GetAllShellDescriptorsAsync_SkipsFailedDescriptors_InParallelExecution() + { + var cancellationToken = CancellationToken.None; + var metadataList = new List + { + new() { Id = "good-1" }, + new() { Id = "bad-1" }, + new() { Id = "good-2" }, + new() { Id = null }, + new() { Id = "good-3" } + }; + var metaData = new ShellDescriptorsMetaData + { + PagingMetaData = null, + ShellDescriptors = metadataList + }; + + _pluginManifestConflictHandler.Manifests.Returns(new List()); + _pluginDataHandler.GetDataForAllShellDescriptorsAsync(null, null, Arg.Any>(), cancellationToken) + .Returns(metaData); + + _shellTemplateMappingProvider.GetTemplateId("good-1").Returns("template-1"); + _shellTemplateMappingProvider.GetTemplateId("good-2").Returns("template-1"); + _shellTemplateMappingProvider.GetTemplateId("good-3").Returns("template-1"); + _shellTemplateMappingProvider.GetTemplateId("bad-1").Throws(new ResourceNotFoundException()); + + _templateProvider.GetShellDescriptorTemplateAsync("template-1", cancellationToken).Returns(GetShellDescriptorTemplate()); + _dataHandler.FillOut(Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + var value = callInfo.ArgAt(1); + return new ShellDescriptor { Id = value.Id }; + }); + + var result = await _sut.GetAllShellDescriptorsAsync(null, null, cancellationToken); + + Assert.NotNull(result); + Assert.NotNull(result.Result); + Assert.Equal(3, result.Result.Count); + Assert.Contains(result.Result, d => d.Id == "good-1"); + Assert.Contains(result.Result, d => d.Id == "good-2"); + Assert.Contains(result.Result, d => d.Id == "good-3"); + Assert.DoesNotContain(result.Result, d => d.Id == "bad-1"); + } - #region Test Data Helpers private static List GetShellDescriptorDataList() => [ @@ -391,6 +546,4 @@ private static List GetExpectedShellDescriptors() => [ ] } ]; - - #endregion } diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/TemplateProvider/Services/TemplateProviderTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/TemplateProvider/Services/TemplateProviderTests.cs index 5c0291ce..b1c97d11 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/TemplateProvider/Services/TemplateProviderTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/TemplateProvider/Services/TemplateProviderTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Linq; using System.Text; using System.Text.Json; @@ -50,7 +50,7 @@ public TemplateProviderTests() _sut = new Template(logger, options, _cachedHttp); } - + [Fact] public async Task GetShellDescriptorTemplateAsync_ReturnsShellDescriptor_WhenValidResponse() { diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRegistry/ShellDescriptorService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRegistry/ShellDescriptorService.cs index ac3fd0e7..04c3afa8 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRegistry/ShellDescriptorService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRegistry/ShellDescriptorService.cs @@ -3,6 +3,9 @@ using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasEnvironment.Providers; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin; using AAS.TwinEngine.DataEngine.DomainModel.AasRegistry; +using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; + +using Microsoft.Extensions.Options; using UnauthorizedAccessException = AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException; @@ -14,8 +17,10 @@ public class ShellDescriptorService( IShellDescriptorDataHandler shellDescriptorDataHandler, IPluginDataHandler pluginDataHandler, IPluginManifestConflictHandler pluginManifestConflictHandler, - ILogger logger) : IShellDescriptorService + ILogger logger, + IOptions templateManagementConfig) : IShellDescriptorService { + private readonly int _concurrentOperationsLimit = templateManagementConfig.Value.AasTemplateRegistry.ConcurrentOperationsLimit; public async Task GetAllShellDescriptorsAsync(int? limit, string? cursor, CancellationToken cancellationToken) { try @@ -26,16 +31,7 @@ public class ShellDescriptorService( .ConfigureAwait(false); var shellDescriptorMetadataList = metadata.ShellDescriptors ?? []; - var shellDescriptors = new List(shellDescriptorMetadataList.Count); - - foreach (var shellDescriptorMetadata in shellDescriptorMetadataList) - { - var shellDescriptor = await TryBuildShellDescriptorAsync(shellDescriptorMetadata, cancellationToken).ConfigureAwait(false); - if (shellDescriptor is not null) - { - shellDescriptors.Add(shellDescriptor); - } - } + var shellDescriptors = await BuildShellDescriptorsInParallelAsync(shellDescriptorMetadataList, cancellationToken).ConfigureAwait(false); return new ShellDescriptors { @@ -119,6 +115,28 @@ public class ShellDescriptorService( } } + private async Task> BuildShellDescriptorsInParallelAsync( + List metadataList, + CancellationToken cancellationToken) + { + using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); + var tasks = metadataList.Select(async metadata => + { + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return await TryBuildShellDescriptorAsync(metadata, cancellationToken).ConfigureAwait(false); + } + finally + { + _ = semaphore.Release(); + } + }); + + var results = await Task.WhenAll(tasks).ConfigureAwait(false); + return [.. results.OfType()]; + } + private async Task BuildShellDescriptorAsync( ShellDescriptorMetaData shellDescriptorMetadata, string templateId, diff --git a/source/AAS.TwinEngine.DataEngine/Program.cs b/source/AAS.TwinEngine.DataEngine/Program.cs index ba18518b..0c3a5189 100644 --- a/source/AAS.TwinEngine.DataEngine/Program.cs +++ b/source/AAS.TwinEngine.DataEngine/Program.cs @@ -48,10 +48,7 @@ public static async Task Main(string[] args) _ = builder.Services.AddAuthorization(); _ = builder.Services.AddControllers() - .AddJsonOptions(options => - { - options.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter()); - }); + .AddJsonOptions(options => options.JsonSerializerOptions.Converters.Add(new JsonStringEnumConverter())); _ = builder.Services.AddEndpointsApiExplorer(); _ = builder.Services.AddOpenApiDocument(settings => From bc8e4dbdad3eaf7e1d68253bf6e37ae1dfef7853 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Thu, 23 Jul 2026 14:06:03 +0530 Subject: [PATCH 03/30] feat: implement CachedGetRequestClient for resilient HTTP requests with user-context aware caching and automated error mapping --- .../AasRepositoryServiceTests.cs | 67 +++++++++++++++++- .../Caching/CachedGetRequestClientTests.cs | 68 +++++++++++++++++++ .../AasRepository/AasRepositoryService.cs | 27 ++++++-- .../Clients/Caching/CachedGetRequestClient.cs | 31 +++++++-- 4 files changed, 181 insertions(+), 12 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs index a8467a4c..87bff847 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs @@ -7,10 +7,12 @@ using AAS.TwinEngine.DataEngine.DomainModel.AasRepository; using AAS.TwinEngine.DataEngine.DomainModel.Plugin; using AAS.TwinEngine.DataEngine.DomainModel.Shared; +using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; using AasCore.Aas3_1; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; using NSubstitute; using NSubstitute.ExceptionExtensions; @@ -23,10 +25,27 @@ public class AasRepositoryServiceTests private readonly IPluginDataHandler _pluginDataHandler = Substitute.For(); private readonly IPluginManifestConflictHandler _pluginManifestConflictHandler = Substitute.For(); private readonly ILogger _logger = Substitute.For>(); + private readonly IOptions _templateManagementConfig = Substitute.For>(); private readonly AasRepositoryService _sut; private const string AasIdentifier = "test-id"; - public AasRepositoryServiceTests() => _sut = new AasRepositoryService(_logger, _templateService, _pluginDataHandler, _pluginManifestConflictHandler); + public AasRepositoryServiceTests() + { + _templateManagementConfig.Value.Returns(new TemplateManagementConfig + { + AasTemplateRepository = new ServiceInstance + { + ConcurrentOperationsLimit = 10 + } + }); + + _sut = new AasRepositoryService( + _logger, + _templateService, + _pluginDataHandler, + _pluginManifestConflictHandler, + _templateManagementConfig); + } [Fact] public async Task GetShellByIdAsync_ShouldReturnShellWithAssetInformation() @@ -331,6 +350,52 @@ await _pluginDataHandler.Received(1) .GetDataForShellsByAssetIdsAsync(manifests, Arg.Is(f => f != null && f.IdShort == targetIdShort), cancellationToken); } + [Fact] + public async Task GetShellsByFiltersAsync_ShouldBuildShellsInParallelAndSkipFailures() + { + // Arrange + var cancellationToken = CancellationToken.None; + var manifests = new List(); + _pluginManifestConflictHandler.Manifests.Returns(manifests); + + var metadataItems = new List + { + new() { Id = "aas-1", SpecificAssetIds = [] }, + new() { Id = "aas-2", SpecificAssetIds = [] }, + new() { Id = "aas-3", SpecificAssetIds = [] } + }; + + _pluginDataHandler + .GetDataForAllShellDescriptorsAsync(null, null, manifests, cancellationToken) + .Returns(new ShellDescriptorsMetaData + { + ShellDescriptors = metadataItems, + PagingMetaData = new PagingMetaData() + }); + + _templateService.GetShellTemplateAsync("aas-1", cancellationToken) + .Returns(new AssetAdministrationShell("aas-1", new AssetInformation(AssetKind.Instance))); + + _templateService.GetShellTemplateAsync("aas-2", cancellationToken) + .Throws(new Exception("Template loading failed")); + + _templateService.GetShellTemplateAsync("aas-3", cancellationToken) + .Returns(new AssetAdministrationShell("aas-3", new AssetInformation(AssetKind.Instance))); + + // Act + var result = await _sut.GetShellsByFiltersAsync(filter: null, limit: null, cursor: null, cancellationToken); + + // Assert + Assert.NotNull(result); + Assert.Equal(2, result.Result.Count); + Assert.Contains(result.Result, s => s.Id == "aas-1"); + Assert.Contains(result.Result, s => s.Id == "aas-3"); + + await _templateService.Received(1).GetShellTemplateAsync("aas-1", cancellationToken); + await _templateService.Received(1).GetShellTemplateAsync("aas-2", cancellationToken); + await _templateService.Received(1).GetShellTemplateAsync("aas-3", cancellationToken); + } + private static AssetAdministrationShell CreateShellTemplate() => new( id: "urn:uuid:123e4567-e89b-12d3-a456-426614174000", diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs index bb808204..b4a9c838 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs @@ -328,6 +328,74 @@ public async Task GetStringAsync_PropagatesCancellationToken() Assert.Equal(cancellationToken, capturedToken); } + [Theory] + [InlineData("?isCacheEnable=false", true)] + [InlineData("?isCacheEnable=true", false)] + [InlineData("?isCacheEnable=invalid", false)] + [InlineData("", false)] + public async Task GetStringAsync_RespectsIsCacheEnabledQueryParameter(string queryString, bool expectBypass) + { + // Arrange + const string RelativeUrl = "api/test"; + const string HttpClientName = "TestClient"; + const string ExpectedResponse = "Direct HTTP data"; + + var httpContext = new DefaultHttpContext(); + if (!string.IsNullOrEmpty(queryString)) + { + httpContext.Request.QueryString = new QueryString(queryString); + } + _httpContextAccessor.HttpContext.Returns(httpContext); + + if (expectBypass) + { + // If bypassed, cache is never called. + _cache.GetOrCreateAsync( + Arg.Any(), + Arg.Any>>(), + Arg.Any(), + Arg.Any>(), + Arg.Any() + ).Returns(new ValueTask("Cached data that should be bypassed")); + } + else + { + // If cache is enabled, SetupCacheHit will return direct response. + SetupCacheHit(ExpectedResponse); + } + + using var httpResponse = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(ExpectedResponse) + }; + SetupHttpClient(HttpClientName, httpResponse); + + // Act + var result = await _sut.GetStringAsync(RelativeUrl, HttpClientName, 5, CancellationToken.None); + + // Assert + Assert.Equal(ExpectedResponse, result); + + if (expectBypass) + { + // If bypassed, HTTP client must be called directly. + _clientFactory.Received(1).CreateClient(HttpClientName); + // And cache should not be queried. + _cache.DidNotReceive().GetOrCreateAsync( + Arg.Any(), + Arg.Any>>(), + Arg.Any(), + Arg.Any>(), + Arg.Any() + ); + } + else + { + // If cache was not bypassed, the HTTP client should not be called since we mocked a cache hit. + _clientFactory.DidNotReceive().CreateClient(Arg.Any()); + } + } + private static string ComputeHash(string input) { var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(input)); diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs index 3d762c5b..129b5a34 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs @@ -5,9 +5,12 @@ using AAS.TwinEngine.DataEngine.DomainModel.AasRegistry; using AAS.TwinEngine.DataEngine.DomainModel.AasRepository; using AAS.TwinEngine.DataEngine.DomainModel.Shared; +using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; using AasCore.Aas3_1; +using Microsoft.Extensions.Options; + using UnauthorizedAccessException = AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException; namespace AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasRepository; @@ -16,8 +19,10 @@ public class AasRepositoryService( ILogger logger, IAasRepositoryTemplateService templateService, IPluginDataHandler pluginDataHandler, - IPluginManifestConflictHandler pluginManifestConflictHandler) : IAasRepositoryService + IPluginManifestConflictHandler pluginManifestConflictHandler, + IOptions templateManagementConfig) : IAasRepositoryService { + private readonly int _concurrentOperationsLimit = templateManagementConfig.Value.AasTemplateRepository.ConcurrentOperationsLimit; public async Task GetShellsByFiltersAsync(ShellSearchFilter? filter, int? limit, string? cursor, CancellationToken cancellationToken) { try @@ -239,30 +244,38 @@ private void FillShellFromMetadata(IAssetAdministrationShell shell, ShellDescrip private async Task> BuildShellsAsync(IEnumerable metadataItems, CancellationToken cancellationToken) { - var shells = new List(); + using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); - foreach (var metadata in metadataItems) + var tasks = metadataItems.Select(async metadata => { if (string.IsNullOrWhiteSpace(metadata.Id)) { - continue; + return null; } + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); try { var shell = await templateService.GetShellTemplateAsync(metadata.Id, cancellationToken).ConfigureAwait(false); FillShellFromMetadata(shell, metadata); - shells.Add(shell); + return shell; } catch (Exception ex) { logger.LogWarning(ex, "Failed to build AAS for id {AasId}. Skipping.", metadata.Id); + return null; } - } + finally + { + _ = semaphore.Release(); + } + }); + + var results = await Task.WhenAll(tasks).ConfigureAwait(false); - return shells; + return [.. results.Where(s => s is not null).Select(s => s!)]; } private static IList FilterByExternalSubjectId(IList shells, IList? filters) diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs index 83407e9e..f7aec4f6 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Security.Claims; using System.Security.Cryptography; using System.Text; @@ -24,6 +24,12 @@ public sealed class CachedGetRequestClient( { public async Task GetStringAsync(string relativeUrl, string httpClientName, int expirationTime, CancellationToken cancellationToken) { + if (!IsCacheEnabled(httpContextAccessor)) + { + logger.LogInformation("Cache bypassed because 'isCacheEnable=false' was specified."); + return await FetchAsync(relativeUrl, httpClientName, cancellationToken).ConfigureAwait(false); + } + var cacheKey = BuildCacheKey(httpContextAccessor, relativeUrl); var entryOptions = new HybridCacheEntryOptions @@ -39,12 +45,12 @@ public async Task GetStringAsync(string relativeUrl, string httpClientNa cancellationToken: cancellationToken).ConfigureAwait(false); } - private async Task FetchAsync(string relativeUrl, string httpClientName, CancellationToken cancellationToken) + private async Task FetchAsync(string url, string httpClientName, CancellationToken cancellationToken) { - logger.LogInformation("Sending HTTP GET request to {Url}", LogSanitizerExtension.Sanitize(relativeUrl)); + logger.LogInformation("Sending HTTP GET request to {Url}", LogSanitizerExtension.Sanitize(url)); var httpClient = clientFactory.CreateClient(httpClientName); - var relativeUri = new Uri(relativeUrl, UriKind.Relative); + var relativeUri = new Uri(url, UriKind.Relative); var response = await httpClient.GetAsync(relativeUri, cancellationToken).ConfigureAwait(false); @@ -101,4 +107,21 @@ private static string ComputeHash(string input) var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(input)); return Convert.ToHexStringLower(bytes); } + + private static bool IsCacheEnabled(IHttpContextAccessor httpContextAccessor) + { + var query = httpContextAccessor.HttpContext?.Request.Query; + + if (query is null) + { + return true; + } + + if (!query.TryGetValue("isCacheEnable", out var value)) + { + return true; + } + + return !bool.TryParse(value, out var enabled) || enabled; + } } From 7d096ae3e0c9628e1d698185dd5b6d403e66d80e Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Thu, 23 Jul 2026 15:23:24 +0530 Subject: [PATCH 04/30] feat: implement ShellDescriptorService to retrieve and construct shell descriptors with parallel processing --- .../Services/AasRegistry/ShellDescriptorService.cs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRegistry/ShellDescriptorService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRegistry/ShellDescriptorService.cs index 04c3afa8..fc5d15c3 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRegistry/ShellDescriptorService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRegistry/ShellDescriptorService.cs @@ -97,12 +97,6 @@ public class ShellDescriptorService( private async Task TryBuildShellDescriptorAsync(ShellDescriptorMetaData shellDescriptorMetadata, CancellationToken cancellationToken) { - if (string.IsNullOrWhiteSpace(shellDescriptorMetadata.Id)) - { - logger.LogError("Failed to process ShellDescriptor. DescriptorId is missing. Continuing with remaining descriptors."); - return null; - } - try { var templateId = shellTemplateMappingProvider.GetTemplateId(shellDescriptorMetadata.Id)!; From 1562e65f779ec51efec84b4fb23a6fc8afea63da Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Thu, 23 Jul 2026 15:29:57 +0530 Subject: [PATCH 05/30] refactor: remove redundant test for skipping failed shell descriptors in parallel execution --- .../ShellDescriptorServiceTests.cs | 46 ------------------- 1 file changed, 46 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRegistry/ShellDescriptorServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRegistry/ShellDescriptorServiceTests.cs index 9ac1217e..182e36b8 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRegistry/ShellDescriptorServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRegistry/ShellDescriptorServiceTests.cs @@ -418,52 +418,6 @@ public async Task GetAllShellDescriptorsAsync_RespectsMaxConcurrency() $"Expected max concurrency <= {concurrencyLimit}, but observed {maxObservedConcurrency}"); } - [Fact] - public async Task GetAllShellDescriptorsAsync_SkipsFailedDescriptors_InParallelExecution() - { - var cancellationToken = CancellationToken.None; - var metadataList = new List - { - new() { Id = "good-1" }, - new() { Id = "bad-1" }, - new() { Id = "good-2" }, - new() { Id = null }, - new() { Id = "good-3" } - }; - var metaData = new ShellDescriptorsMetaData - { - PagingMetaData = null, - ShellDescriptors = metadataList - }; - - _pluginManifestConflictHandler.Manifests.Returns(new List()); - _pluginDataHandler.GetDataForAllShellDescriptorsAsync(null, null, Arg.Any>(), cancellationToken) - .Returns(metaData); - - _shellTemplateMappingProvider.GetTemplateId("good-1").Returns("template-1"); - _shellTemplateMappingProvider.GetTemplateId("good-2").Returns("template-1"); - _shellTemplateMappingProvider.GetTemplateId("good-3").Returns("template-1"); - _shellTemplateMappingProvider.GetTemplateId("bad-1").Throws(new ResourceNotFoundException()); - - _templateProvider.GetShellDescriptorTemplateAsync("template-1", cancellationToken).Returns(GetShellDescriptorTemplate()); - _dataHandler.FillOut(Arg.Any(), Arg.Any()) - .Returns(callInfo => - { - var value = callInfo.ArgAt(1); - return new ShellDescriptor { Id = value.Id }; - }); - - var result = await _sut.GetAllShellDescriptorsAsync(null, null, cancellationToken); - - Assert.NotNull(result); - Assert.NotNull(result.Result); - Assert.Equal(3, result.Result.Count); - Assert.Contains(result.Result, d => d.Id == "good-1"); - Assert.Contains(result.Result, d => d.Id == "good-2"); - Assert.Contains(result.Result, d => d.Id == "good-3"); - Assert.DoesNotContain(result.Result, d => d.Id == "bad-1"); - } - private static List GetShellDescriptorDataList() => [ From 2dd641c5c66fc68cd19a7e0b100bea09a063188d Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Thu, 23 Jul 2026 15:35:30 +0530 Subject: [PATCH 06/30] refactor: simplify SendGetRequestAsync method implementation --- .../TemplateProvider/Services/TemplateProvider.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/TemplateProvider/Services/TemplateProvider.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/TemplateProvider/Services/TemplateProvider.cs index 8153f4e1..34e98c23 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/TemplateProvider/Services/TemplateProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/TemplateProvider/Services/TemplateProvider.cs @@ -1,4 +1,4 @@ -using System.Text.Json; +using System.Text.Json; using System.Text.Json.Nodes; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; @@ -270,12 +270,7 @@ public async Task> GetSubmodelRefByIdAsync(string templateId, C } } - private async Task SendGetRequestAsync(string url, string httpClientName, int expirationTime, CancellationToken cancellationToken) - { - logger.LogInformation("Sending HTTP GET request to {Url}", LogSanitizerExtension.Sanitize(url)); - - return await cachedHttp.GetStringAsync(url, httpClientName, expirationTime, cancellationToken).ConfigureAwait(false); - } + private async Task SendGetRequestAsync(string url, string httpClientName, int expirationTime, CancellationToken cancellationToken) => await cachedHttp.GetStringAsync(url, httpClientName, expirationTime, cancellationToken).ConfigureAwait(false); private static ShellDescriptor? DeserializeShellDescriptor(JsonNode? descriptorNode) { From 0b1f0e014aa02e52c4f2dd472938d202739eadc8 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Thu, 23 Jul 2026 16:09:15 +0530 Subject: [PATCH 07/30] feat: implement CachedGetRequestClient with HybridCache and user-context aware key generation --- .../Clients/Caching/CachedGetRequestClientTests.cs | 10 +++++----- .../Http/Clients/Caching/CachedGetRequestClient.cs | 13 +++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs index b4a9c838..98d66b21 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Http/Clients/Caching/CachedGetRequestClientTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Security.Claims; using System.Security.Cryptography; using System.Text; @@ -329,11 +329,11 @@ public async Task GetStringAsync_PropagatesCancellationToken() } [Theory] - [InlineData("?isCacheEnable=false", true)] - [InlineData("?isCacheEnable=true", false)] - [InlineData("?isCacheEnable=invalid", false)] + [InlineData("?noCache=true", true)] + [InlineData("?noCache=false", false)] + [InlineData("?noCache=invalid", false)] [InlineData("", false)] - public async Task GetStringAsync_RespectsIsCacheEnabledQueryParameter(string queryString, bool expectBypass) + public async Task GetStringAsync_RespectsNoCacheQueryParameter(string queryString, bool expectBypass) { // Arrange const string RelativeUrl = "api/test"; diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs index f7aec4f6..3d0e82ce 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Security.Claims; using System.Security.Cryptography; using System.Text; @@ -26,7 +26,7 @@ public async Task GetStringAsync(string relativeUrl, string httpClientNa { if (!IsCacheEnabled(httpContextAccessor)) { - logger.LogInformation("Cache bypassed because 'isCacheEnable=false' was specified."); + logger.LogInformation("Cache bypassed because 'noCache=true' was specified."); return await FetchAsync(relativeUrl, httpClientName, cancellationToken).ConfigureAwait(false); } @@ -117,11 +117,16 @@ private static bool IsCacheEnabled(IHttpContextAccessor httpContextAccessor) return true; } - if (!query.TryGetValue("isCacheEnable", out var value)) + if (!query.TryGetValue("noCache", out var value)) { return true; } - return !bool.TryParse(value, out var enabled) || enabled; + if (bool.TryParse(value, out var noCache) && noCache) + { + return false; + } + + return true; } } From 1595b621fefb2d090599efe4906b7680b3521264 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Fri, 24 Jul 2026 12:33:06 +0530 Subject: [PATCH 08/30] For testing, will be removed --- .../Clients/Caching/CachedGetRequestClient.cs | 4 +++- source/AAS.TwinEngine.DataEngine/Program.cs | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs index 3d0e82ce..8bec5e4f 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Security.Claims; using System.Security.Cryptography; using System.Text; @@ -32,6 +32,8 @@ public async Task GetStringAsync(string relativeUrl, string httpClientNa var cacheKey = BuildCacheKey(httpContextAccessor, relativeUrl); + Console.WriteLine($"Cache key: {cacheKey}"); + var entryOptions = new HybridCacheEntryOptions { Expiration = TimeSpan.FromMinutes(expirationTime), diff --git a/source/AAS.TwinEngine.DataEngine/Program.cs b/source/AAS.TwinEngine.DataEngine/Program.cs index 0c3a5189..b523c470 100644 --- a/source/AAS.TwinEngine.DataEngine/Program.cs +++ b/source/AAS.TwinEngine.DataEngine/Program.cs @@ -1,4 +1,5 @@ -using System.Text.Json.Serialization; +using System.Security.Claims; +using System.Text.Json.Serialization; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.Infrastructure.Http.Authorization.Middleware; @@ -89,6 +90,21 @@ public static async Task Main(string[] args) _ = app.UseResponseCompression(); _ = app.UseHostFiltering(); _ = app.UseMiddleware(); + _ = app.Use(async (context, next) => + { + if (context.Request.Headers.TryGetValue("X-User", out var user)) + { + context.User = new ClaimsPrincipal( + new ClaimsIdentity( + [ + new Claim(ClaimTypes.NameIdentifier, user!), + new Claim("permission", "read"), + new Claim("permission", "write") + ], "Test")); + } + + await next(); + }); _ = app.UseHttpsRedirection(); _ = app.UseAuthorization(); app.UseCorsServices(); From 9b28a135d4d5c5d460ce8225e147a2c61fc96002 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Fri, 24 Jul 2026 13:09:00 +0530 Subject: [PATCH 09/30] removed extra auth added for testing --- .../Clients/Caching/CachedGetRequestClient.cs | 2 -- source/AAS.TwinEngine.DataEngine/Program.cs | 15 --------------- 2 files changed, 17 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs index 8bec5e4f..a5f3806f 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs @@ -32,8 +32,6 @@ public async Task GetStringAsync(string relativeUrl, string httpClientNa var cacheKey = BuildCacheKey(httpContextAccessor, relativeUrl); - Console.WriteLine($"Cache key: {cacheKey}"); - var entryOptions = new HybridCacheEntryOptions { Expiration = TimeSpan.FromMinutes(expirationTime), diff --git a/source/AAS.TwinEngine.DataEngine/Program.cs b/source/AAS.TwinEngine.DataEngine/Program.cs index b523c470..d0e8471f 100644 --- a/source/AAS.TwinEngine.DataEngine/Program.cs +++ b/source/AAS.TwinEngine.DataEngine/Program.cs @@ -90,21 +90,6 @@ public static async Task Main(string[] args) _ = app.UseResponseCompression(); _ = app.UseHostFiltering(); _ = app.UseMiddleware(); - _ = app.Use(async (context, next) => - { - if (context.Request.Headers.TryGetValue("X-User", out var user)) - { - context.User = new ClaimsPrincipal( - new ClaimsIdentity( - [ - new Claim(ClaimTypes.NameIdentifier, user!), - new Claim("permission", "read"), - new Claim("permission", "write") - ], "Test")); - } - - await next(); - }); _ = app.UseHttpsRedirection(); _ = app.UseAuthorization(); app.UseCorsServices(); From 1d15a08a036ec648ed3b199c4bc775e54c0c09c4 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Fri, 31 Jul 2026 15:05:31 +0530 Subject: [PATCH 10/30] feat: implement plugin-based data retrieval and submodel repository pagination logic --- Pegination-story.md | 123 ++++++++ Submodel-Pagination-Cursor-Concept.md | 424 ++++++++++++++++++++++++++ 2 files changed, 547 insertions(+) create mode 100644 Pegination-story.md create mode 100644 Submodel-Pagination-Cursor-Concept.md diff --git a/Pegination-story.md b/Pegination-story.md new file mode 100644 index 00000000..53f71216 --- /dev/null +++ b/Pegination-story.md @@ -0,0 +1,123 @@ +### User Story1 + +**As a** DataEngine API consumer +**I want to** retrieve Submodels and Submodel Descriptors using efficient and consistent pagination +**So that** large digital twin datasets can be processed without requiring DataEngine to load and generate all Submodels before applying pagination + +--- + +### Acceptance Criteria +- [ ] DataEngine shall avoid requesting all Shell metadata from the Plugin when a cursor or limit is provided by the consumer. +- [ ] Plugin metadata/shell endpoints shall support pagination using aasId as a cursor. +- [ ] DataEngine shall be able to continue pagination correctly when the client provides a submodelId cursor. +- [ ] The pagination implementation shall preserve the logical ordering of generated Submodels within a Shell Template. +- [ ] The implementation shall ensure that no Submodels are skipped or duplicated across pages. +- [ ] Solutions for cursor translation between submodelId and aasId shall be documented and evaluated. +- [ ] Edge cases involving partially processed Shells shall be handled consistently. +- [ ] The selected approach shall be documented before implementation begins. + +--- + +### Concept + +Current implementation of both: + +GET /submodels +GET /submodel-descriptors + +does not leverage pagination during metadata retrieval. +Current flow: + +1. DataEngine requests all metadata/shells from Plugin. +2. For every Shell: + +- Resolve Shell Template using ShellTemplateMappingRule. +- Request Shell Template from Template Registry. +- Generate Submodel IDs based on Shell Template configuration. + +3. After generating all Submodel IDs: + +- Apply existing logic used by: + - Get Submodel by Id + - Get Submodel Descriptor by Id + +4. Finally return paginated response. + +This approach works for small datasets but becomes expensive when thousands of products/shells exist. + +Desired Improvement +Introduce pagination as early as possible in the processing pipeline. +**Instead of:** + +All Shells + ↓ +Generate All Submodels + ↓ +Apply Pagination + +**Use:** + +Paginated Shell Retrieval + ↓ +Generate Required Submodels + ↓ +Apply Pagination again (bcz we don't know which shell have how many submodel template ) + ↓ +Return Page + +**Key Design Challenge** + +Plugin pagination is based on: `aasId` +while DataEngine pagination is exposed using: `submodelId` +A mechanism is required to map: `submodelId -> aasId` + +to determine from which Shell Plugin pagination should continue + +--- + +### Additional Notes + +**Important Edge Case** + +Assume: + +``` +Product1 + ├─ Nameplate + ├─ TechnicalData + ├─ ContactInformation + └─ CarbonFootprint + +Product2 + ├─ Nameplate + ├─ TechnicalData + ├─ ContactInformation + └─ CarbonFootprint +``` +Consumer requests: `limit=12cursor=` + +Naive implementation: + +1. Resolve cursor to Product1 AAS. +2. Send Product1 AAS as cursor to Plugin. +3. Plugin returns Product2. + +Result: `Returned Item = Product2 Nameplate` +Expected result: `Returned Item = Product1 ContactInformation` + +because pagination should continue within the same Shell before moving to the next Shell. + +--- + +1. This PBI should initially focus on design and architecture evaluation before implementation. +2. A technical proposal should be produced describing: + +- Cursor structure. +- Ordering guarantees. +- Plugin API changes. +- Backward compatibility considerations. +- Performance impact. + +3.Final implementation should be done only after the pagination concept has been agreed upon. +4. Solution must work consistently for both: GET /submodels , GET /submodel-descriptors +5. Consider scenarios where Multiple Shell Templates exist. \ No newline at end of file diff --git a/Submodel-Pagination-Cursor-Concept.md b/Submodel-Pagination-Cursor-Concept.md new file mode 100644 index 00000000..4c3de69e --- /dev/null +++ b/Submodel-Pagination-Cursor-Concept.md @@ -0,0 +1,424 @@ +# Submodel Pagination Cursor Design - Get All Submodels + +## 1. Problem Statement + +The DataEngine's `GET /submodels` endpoint must paginate **submodels**, but the underlying Plugin API paginates **products (AAS IDs)**. A single product may contain multiple submodels, and a product can be only **partially consumed** within one response page. The cursor must encode enough state to resume mid-product without duplicating or skipping submodels. + +--- + +## 2. Data Model + +``` +Product 1 (AAS-1) Product 2 (AAS-2) Product 3 (AAS-3) +├── SM-1 ├── SM-4 ├── SM-7 +├── SM-2 ├── SM-5 ├── SM-8 +└── SM-3 ├── SM-6 └── SM-9 +``` + +- **3 products**, each with **3 submodels** → **9 submodels** total. +- Plugin returns products in a stable, deterministic order. +- Submodel IDs within a product are derived from Shell Templates and maintain a deterministic order. + +--- + +## 3. The Limit=2 Problem (Why a Simple Two-Field Cursor Fails) + +Consider `limit=2` with our example data: + +``` +Page 1: GET /submodels?limit=2 + + Expand AAS-1 → [SM-1, SM-2, SM-3] + Collect SM-1, SM-2. Limit reached. + Cursor = (AAS-1, SM-2) ← AAS-1 is PARTIALLY consumed (SM-3 still pending) + +Page 2: GET /submodels?limit=2&cursor=(AAS-1, SM-2) + + Need to resume inside AAS-1 to get SM-3. + But Plugin cursor semantics are EXCLUSIVE: passing AAS-1 returns [AAS-2, AAS-3, ...] + SM-3 is LOST! +``` + +**Root cause:** The Plugin's cursor-based pagination uses exclusive semantics ("give me items AFTER this cursor"). Once the Plugin moves past a product, there's no way to go back using the Plugin's pagination API alone. + +**The partially consumed product must be handled independently from the Plugin's forward pagination.** + +--- + +## 4. Cursor Structure (Two-Field Composite) + +### 4.1 Logical Composition + +The cursor requires **two** coordinates: + +| Field | Type | Purpose | +|-------|------|---------| +| `SubmodelId` | string | The last submodel ID **included** in the previous response. Marks the exact resume point within the product. | +| `AasId` | string | The AAS ID that serves simultaneously as the position anchor and the exclusive plugin request cursor. | + +### 4.2 Why Two Fields? + +| Scenario | What's needed | +|----------|--------------| +| Product partially consumed | `AasId` to re-expand the product on the first batch; `SubmodelId` to skip delivered items | +| Exclusive plugin request | `AasId` is passed directly as the exclusive plugin cursor (`trackingAasId`) to continue forward | + +### 4.3 Wire Format + +``` +Logical: {SubmodelId}|{AasId} +Encoded: Base64Url( UTF-8( "{SubmodelId}|{AasId}" ) ) +``` + +**Example:** + +``` +Logical: https://example.com/submodels/Nameplate|https://example.com/shells/001 +Encoded: aHR0cHM6Ly9leGFtcGxlLmNvbS9zdWJtb2RlbHMvTmFtZXBsYXRlfGh0dHBzOi8vZXhhbXBsZS5jb20vc2hlbGxzLzAwMQ== +``` + +The cursor is **opaque** to the client - they never parse or construct it. + +### 4.4 Special Cases + +| Scenario | Cursor Value | +|----------|-------------| +| First page (no cursor) | `null` / absent | +| Last page (no more data) | Response returns `cursor = null` (signals end of dataset) | +| Limit reached mid-product or at boundary | `{LastSubmodelId}|{CurrentAasId}` | +--- + +## 5. Pagination Algorithm + +### 5.1 Two-Phase Resume Strategy + +The key insight: **on resume, the partially consumed product is handled FIRST (Phase 1) using only its AAS ID, THEN forward pagination continues (Phase 2) using the Plugin's next cursor.** + +``` +┌────────────────────────────────────────────────────────────────────────┐ +│ PHASE 1: Complete the partially consumed product │ +│ │ +│ - Re-derive submodel IDs for CurrentAasId (via template expansion) │ +│ - Skip submodels up to and including LastSubmodelId │ +│ - Collect remaining submodels until limit or product exhausted │ +│ │ +│ NO Plugin pagination call needed - we already know the AAS ID. │ +├────────────────────────────────────────────────────────────────────────┤ +│ PHASE 2: Continue with next products (if limit not yet reached) │ +│ │ +│ - Call Plugin.GetProducts(cursor = PluginNextCursor) │ +│ - Expand each returned product's submodels │ +│ - Collect until limit reached or Plugin exhausted │ +└────────────────────────────────────────────────────────────────────────┘ +``` + +### 5.2 Plugin Cursor Coordination + +``` +┌───────────────────────────────────────────────────────────────────────────┐ +│ Page 1: GET /submodels?limit=2 │ +│ │ +│ Engine calls Plugin: GetProducts(limit=2, cursor=null) │ +│ Plugin returns: [AAS-1, AAS-2], pluginNextCursor = "C1" │ +│ │ +│ Expand AAS-1 → [SM-1, SM-2, SM-3] │ +│ Collect SM-1, SM-2. Limit reached. │ +│ │ +│ Cursor = Encode("C1", "AAS-1", "SM-2") │ +│ ▲ ▲ ▲ │ +│ │ │ └── Last delivered submodel │ +│ │ └──────────── Product still being consumed │ +│ └─────────────────── Plugin cursor for NEXT products │ +│ (skips AAS-1 and AAS-2) │ +├───────────────────────────────────────────────────────────────────────────┤ +│ Page 2: Resume with cursor ("C1", "AAS-1", "SM-2") │ +│ │ +│ PHASE 1: Re-expand AAS-1 → [SM-1, SM-2, SM-3] │ +│ Skip SM-1, SM-2. Collect SM-3. Count = 1. │ +│ │ +│ PHASE 2: Need 1 more. But wait - AAS-2 was in the same Plugin page! │ +│ We need AAS-2 before using "C1". │ +│ SEE SECTION 5.4 FOR HANDLING THIS │ +└───────────────────────────────────────────────────────────────────────────┘ +``` + +### 5.3 The "Same-Page Products" Problem & Solution + +When the Plugin returns `[AAS-1, AAS-2]` with `nextCursor = "C1"`: +- `"C1"` skips **both** AAS-1 and AAS-2. +- But if we only consumed AAS-1 partially, AAS-2 hasn't been processed at all! + +**Solution: Store the Plugin cursor that produced the current batch, not the "next" cursor.** + +Revised cursor structure: + +| Field | Purpose | +|-------|---------| +| `PluginPageCursor` | The cursor that was used to **fetch the current batch** (null for the first page). On resume, re-fetch the same batch from the Plugin. | +| `CurrentAasId` | The partially consumed product's AAS ID. | +| `LastSubmodelId` | Last delivered submodel within that product. | + +### 5.4 Why Store `PluginPageCursor` (Not `PluginNextCursor`) + +| Stored Value | On Resume | Problem | +|---|---|---| +| `PluginNextCursor` | Gets products AFTER current batch | Products between `currentAasId` and end of batch are LOST | +| **`PluginPageCursor`** | **Re-fetches the same batch** | **All products in the batch are available; skip already-consumed ones** | + +The trade-off: on resume, we re-fetch and re-scan some products we've already processed. But: +- The re-scan is bounded (one Plugin page, typically small). +- Correctness is guaranteed - no submodels are ever lost. +- No extra Plugin endpoints required. + +--- + +## 6. Request/Response Examples + +### 6.1 Example: `limit=2` (The Tricky Case) + +**Page 1 - `GET /submodels?limit=2`** + +``` +1. No cursor → pluginPageCursor = null, no skipping. +2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2], nextPluginCursor="C1" +3. Expand AAS-1 → [SM-1, SM-2, SM-3] +4. Collect SM-1. Count=1. +5. Collect SM-2. Count=2. Limit reached. +6. Cursor = Encode(pluginPageCursor=null, AasId="AAS-1", LastSM="SM-2") +``` + +```json +{ + "result": [ + { "id": "SM-1" }, + { "id": "SM-2" } + ], + "paging_metadata": { "cursor": "fEFBUy0xfFNNLTI=" } +} +``` + +--- + +**Page 2 - `GET /submodels?limit=2&cursor=fEFBUy0xfFNNLTI=`** + +``` +1. Decode cursor → (pluginPageCursor=null, currentAasId="AAS-1", lastSM="SM-2") +2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2], nextPluginCursor="C1" + (Same call as Page 1 - re-fetches the same batch) +3. Skip products before AAS-1: none to skip. +4. Expand AAS-1 → [SM-1, SM-2, SM-3] +5. Skip submodels up to and including SM-2. +6. Collect SM-3. Count=1. +7. Move to AAS-2 (next product in SAME batch). +8. Expand AAS-2 → [SM-4, SM-5, SM-6] +9. Collect SM-4. Count=2. Limit reached. +10. Cursor = Encode(pluginPageCursor=null, AasId="AAS-2", LastSM="SM-4") +``` + +```json +{ + "result": [ + { "id": "SM-3" }, + { "id": "SM-4" } + ], + "paging_metadata": { "cursor": "fEFBUy0yfFNNLTQ=" } +} +``` + +--- + +**Page 3 - `GET /submodels?limit=2&cursor=fEFBUy0yfFNNLTQ=`** + +``` +1. Decode cursor → (pluginPageCursor=null, currentAasId="AAS-2", lastSM="SM-4") +2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2], nextPluginCursor="C1" +3. Skip products before AAS-2: skip AAS-1. +4. Expand AAS-2 → [SM-4, SM-5, SM-6] +5. Skip submodels up to and including SM-4. +6. Collect SM-5. Count=1. +7. Collect SM-6. Count=2. Limit reached. +8. Cursor = Encode(pluginPageCursor=null, AasId="AAS-2", LastSM="SM-6") +``` + +```json +{ + "result": [ + { "id": "SM-5" }, + { "id": "SM-6" } + ], + "paging_metadata": { "cursor": "fEFBUy0yfFNNLTY=" } +} +``` + +--- + +**Page 4 - `GET /submodels?limit=2&cursor=fEFBUy0yfFNNLTY=`** + +``` +1. Decode cursor → (pluginPageCursor=null, currentAasId="AAS-2", lastSM="SM-6") +2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2], nextPluginCursor="C1" +3. Skip products before AAS-2: skip AAS-1. +4. Expand AAS-2 → [SM-4, SM-5, SM-6] +5. Skip SM-4, SM-5, SM-6 (all skipped - product fully consumed). +6. No more products in this batch. +7. Move to next Plugin page: GetProducts(cursor="C1") → [AAS-3], nextPluginCursor=null +8. Expand AAS-3 → [SM-7, SM-8, SM-9] +9. Collect SM-7. Count=1. +10. Collect SM-8. Count=2. Limit reached. +11. Cursor = Encode(pluginPageCursor="C1", AasId="AAS-3", LastSM="SM-8") +``` + +```json +{ + "result": [ + { "id": "SM-7" }, + { "id": "SM-8" } + ], + "paging_metadata": { "cursor": "QzF8QUFTLTN8U00tOA==" } +} +``` + +--- + +**Page 5 (Final) - `GET /submodels?limit=2&cursor=QzF8QUFTLTN8U00tOA==`** + +``` +1. Decode cursor → (pluginPageCursor="C1", currentAasId="AAS-3", lastSM="SM-8") +2. Call Plugin: GetProducts(cursor="C1") → [AAS-3], nextPluginCursor=null +3. Skip products before AAS-3: none. +4. Expand AAS-3 → [SM-7, SM-8, SM-9] +5. Skip SM-7, SM-8. +6. Collect SM-9. Count=1. +7. No more products, no more plugin pages. +8. End of dataset → cursor = null. +``` + +```json +{ + "result": [ + { "id": "SM-9" } + ], + "paging_metadata": { "cursor": null } +} +``` + +--- + +### 6.2 Example: `limit=4` + +**Page 1 - `GET /submodels?limit=4`** + +``` +1. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2, AAS-3], nextPluginCursor=null +2. Expand AAS-1 → [SM-1, SM-2, SM-3]. Collect all. Count=3. +3. Expand AAS-2 → [SM-4, SM-5, SM-6]. Collect SM-4. Count=4. Limit reached. +4. Cursor = Encode(pluginPageCursor=null, AasId="AAS-2", LastSM="SM-4") +``` + +```json +{ + "result": [ + { "id": "SM-1" }, + { "id": "SM-2" }, + { "id": "SM-3" }, + { "id": "SM-4" } + ], + "paging_metadata": { "cursor": "fEFBUy0yfFNNLTQ=" } +} +``` + +**Page 2 - `GET /submodels?limit=4&cursor=fEFBUy0yfFNNLTQ=`** + +``` +1. Decode → (pluginPageCursor=null, currentAasId="AAS-2", lastSM="SM-4") +2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2, AAS-3], nextPluginCursor=null +3. Skip AAS-1 (before AAS-2). +4. Expand AAS-2 → [SM-4, SM-5, SM-6]. Skip SM-4. Collect SM-5, SM-6. Count=2. +5. Expand AAS-3 → [SM-7, SM-8, SM-9]. Collect SM-7, SM-8. Count=4. Limit reached. +6. Cursor = Encode(pluginPageCursor=null, AasId="AAS-3", LastSM="SM-8") +``` + +```json +{ + "result": [ + { "id": "SM-5" }, + { "id": "SM-6" }, + { "id": "SM-7" }, + { "id": "SM-8" } + ], + "paging_metadata": { "cursor": "fEFBUy0zfFNNLTg=" } +} +``` + +**Page 3 (Final) - `GET /submodels?limit=4&cursor=fEFBUy0zfFNNLTg=`** + +``` +1. Decode → (pluginPageCursor=null, currentAasId="AAS-3", lastSM="SM-8") +2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2, AAS-3] +3. Skip AAS-1, AAS-2. Find AAS-3. +4. Expand AAS-3 → [SM-7, SM-8, SM-9]. Skip SM-7, SM-8. Collect SM-9. Count=1. +5. No more products. Plugin exhausted. +6. cursor = null. +``` + +```json +{ + "result": [ + { "id": "SM-9" } + ], + "paging_metadata": { "cursor": null } +} +``` + +--- + + +## 9. Sequence Diagram (limit=2) + +```mermaid +sequenceDiagram + autonumber + actor Client + participant API as GET /submodels + participant Engine as Pagination Engine + participant Plugin as Plugin API + + Client->>API: GET /submodels?limit=2 + API->>Engine: Paginate(limit=2, cursor=null) + + Engine->>Plugin: GetProducts(cursor=null) + Plugin-->>Engine: [AAS-1, AAS-2], nextCursor="C1" + + Note over Engine: Expand AAS-1 → [SM-1, SM-2, SM-3] + Engine->>Plugin: GetSubmodelData(SM-1) + Engine->>Plugin: GetSubmodelData(SM-2) + Note over Engine: Collected: 2. Limit reached. + Note over Engine: Cursor = Encode(null, AAS-1, SM-2) + + Engine-->>API: [SM-1, SM-2], cursor=encoded + API-->>Client: 200 OK + + Client->>API: GET /submodels?limit=2&cursor=... + API->>Engine: Paginate(limit=2, cursor=...) + + Note over Engine: Decode → (pluginPageCursor=null, AAS-1, SM-2) + Engine->>Plugin: GetProducts(cursor=null) + Note over Engine: Re-fetches SAME batch + Plugin-->>Engine: [AAS-1, AAS-2], nextCursor="C1" + + Note over Engine: Find AAS-1, expand → [SM-1, SM-2, SM-3] + Note over Engine: Skip SM-1, SM-2 (anchor found) + Engine->>Plugin: GetSubmodelData(SM-3) + Note over Engine: Collected: 1. Need 1 more. + + Note over Engine: Move to AAS-2 (same batch!) + Note over Engine: Expand AAS-2 → [SM-4, SM-5, SM-6] + Engine->>Plugin: GetSubmodelData(SM-4) + Note over Engine: Collected: 2. Limit reached. + Note over Engine: Cursor = Encode(null, AAS-2, SM-4) + + Engine-->>API: [SM-3, SM-4], cursor=encoded + API-->>Client: 200 OK +``` + +--- \ No newline at end of file From 77a54fb471d90f3a63867c9c92a4582825889d5a Mon Sep 17 00:00:00 2001 From: Hardi Shah Date: Fri, 31 Jul 2026 16:35:13 +0530 Subject: [PATCH 11/30] Refactor SubmodelRepositoryService and related components for pagination support - Updated SubmodelRepositoryService to implement pagination for submodel retrieval, including cursor handling. - Introduced SubmodelPageCursor to manage state across paginated requests. - Modified IPluginDataHandler and IPluginDataProvider interfaces to include pagination parameters. - Adjusted PluginDataHandler and PluginDataProvider implementations to support new pagination logic. - Enhanced unit tests for SubmodelRepositoryService to cover new pagination scenarios. - Added new tests for SubmodelPageCursor to ensure correct encoding and decoding behavior. - Updated existing tests in PluginDataHandler and PluginDataProvider to accommodate changes in method signatures. --- .../AasRepositoryServiceTests.cs | 6 +- .../Discovery/AssetIdSearchServiceTests.cs | 18 +- .../SubmodelPageCursorTests.cs | 48 ++++++ .../SubmodelRepositoryServiceTests.cs | 148 +++++++++++++++- .../Services/PluginDataHandlerTests.cs | 24 +-- .../Services/PluginDataProviderTests.cs | 16 +- .../AasRepository/AasRepositoryService.cs | 2 +- .../Discovery/AssetIdSearchService.cs | 4 +- .../Services/Plugin/IPluginDataHandler.cs | 4 +- .../Plugin/Providers/IPluginDataProvider.cs | 4 +- .../SubmodelRepository/SubmodelPageCursor.cs | 47 ++++++ .../SubmodelRepositoryService.cs | 158 +++++++++++++----- .../Services/PluginDataHandler.cs | 4 +- .../Services/PluginDataProvider.cs | 4 +- 14 files changed, 401 insertions(+), 86 deletions(-) create mode 100644 source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursorTests.cs create mode 100644 source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursor.cs diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs index 87bff847..c3f91968 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs @@ -328,6 +328,8 @@ public async Task GetShellsByFiltersAsync_WithIdShort_QueriesFilteredShellMetada .GetDataForShellsByAssetIdsAsync( manifests, Arg.Is(f => f != null && f.IdShort == targetIdShort), + Arg.Any(), + Arg.Any(), cancellationToken) .Returns(new ShellDescriptorsMetaData { @@ -347,7 +349,7 @@ public async Task GetShellsByFiltersAsync_WithIdShort_QueriesFilteredShellMetada Assert.Single(result.Result); Assert.Equal("aas-1", result.Result[0].Id); await _pluginDataHandler.Received(1) - .GetDataForShellsByAssetIdsAsync(manifests, Arg.Is(f => f != null && f.IdShort == targetIdShort), cancellationToken); + .GetDataForShellsByAssetIdsAsync(manifests, Arg.Is(f => f != null && f.IdShort == targetIdShort), Arg.Any(), Arg.Any(), cancellationToken); } [Fact] @@ -361,7 +363,7 @@ public async Task GetShellsByFiltersAsync_ShouldBuildShellsInParallelAndSkipFail var metadataItems = new List { new() { Id = "aas-1", SpecificAssetIds = [] }, - new() { Id = "aas-2", SpecificAssetIds = [] }, + new() { Id = "aas-2", SpecificAssetIds = [] }, new() { Id = "aas-3", SpecificAssetIds = [] } }; diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs index d3634fd1..8d34fc7d 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Base; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasRepository; @@ -51,6 +51,8 @@ public async Task SearchShellsByAssetLinkAsync_ReturnsAasIds() _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Returns(metadata); @@ -83,6 +85,8 @@ public async Task SearchShellsByAssetLinkAsync_WithPagination_ReturnsPagedResult _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Returns(metadata); @@ -102,6 +106,8 @@ public async Task SearchShellsByAssetLinkAsync_WhenPluginTimeout_ThrowsPluginNot _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Throws(new RequestTimeoutException()); @@ -119,6 +125,8 @@ public async Task SearchShellsByAssetLinkAsync_WhenUnauthorized_ThrowsServiceUnA _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Throws(new AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException()); @@ -136,6 +144,8 @@ public async Task SearchShellsByAssetLinkAsync_WhenResponseParsingError_ThrowsIn _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Throws(new ResponseParsingException()); @@ -154,6 +164,8 @@ public async Task SearchShellsByAssetLinkAsync_WhenMultiPluginConflict_ThrowsInt _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Throws(new MultiPluginConflictException()); @@ -171,6 +183,8 @@ public async Task SearchShellsByAssetLinkAsync_WhenResourceNotFound_ThrowsIntern _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Throws(new ResourceNotFoundException()); @@ -200,6 +214,8 @@ public async Task SearchShellsByAssetLinkAsync_FiltersOutEmptyIds() _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Returns(metadata); diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursorTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursorTests.cs new file mode 100644 index 00000000..7612fbbf --- /dev/null +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursorTests.cs @@ -0,0 +1,48 @@ +using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.SubmodelRepository; + +namespace AAS.TwinEngine.DataEngine.UnitTests.ApplicationLogic.Services.SubmodelRepository; + +public class SubmodelPageCursorTests +{ + [Fact] + public void EncodeThenTryDecode_RoundTripsAllFields() + { + var cursor = new SubmodelPageCursor("PLUGIN-PAGE-2", "AAS-1", "SM-2"); + + var decoded = SubmodelPageCursor.TryDecode(cursor.Encode()); + + Assert.NotNull(decoded); + Assert.Equal("PLUGIN-PAGE-2", decoded!.PluginPageCursor); + Assert.Equal("AAS-1", decoded.CurrentAasId); + Assert.Equal("SM-2", decoded.LastSubmodelId); + } + + [Fact] + public void EncodeThenTryDecode_WithNullPluginPageCursor_RoundTripsAsNull() + { + var cursor = new SubmodelPageCursor(null, "AAS-1", "SM-2"); + + var decoded = SubmodelPageCursor.TryDecode(cursor.Encode()); + + Assert.NotNull(decoded); + Assert.Null(decoded!.PluginPageCursor); + Assert.Equal("AAS-1", decoded.CurrentAasId); + Assert.Equal("SM-2", decoded.LastSubmodelId); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void TryDecode_WithEmptyOrWhitespace_ReturnsNull(string? encodedCursor) => Assert.Null(SubmodelPageCursor.TryDecode(encodedCursor)); + + [Fact] + public void TryDecode_WithWrongFieldCount_ReturnsNull() + { + // Only two fields instead of the expected three. + var malformed = "AAS-1|SM-2".EncodeBase64Url(); + + Assert.Null(SubmodelPageCursor.TryDecode(malformed)); + } +} diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs index 7b840855..70805336 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasRepository; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin; @@ -245,11 +245,33 @@ public static SemanticBranchNode CreateSubmodelTreeNode(string value) return submodel; } + private static Reference SubmodelReference(string submodelId) => + new(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, submodelId)]); + + // Arranges the template + value pipeline so every collected submodel id yields one built submodel. + private void ArrangeSubmodelBuildPipeline() + { + _templateService + .GetFilteredSubmodelTemplateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(TestData.CreateSubmodel()); + + ArrangeValuePipeline(); + } + + private void ArrangeValuePipeline() + { + _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); + _pluginDataHandler + .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); + _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(TestData.CreateFilledSubmodel()); + } + [Fact] public async Task GetAllSubmodelsAsync_ReturnsEmpty_WhenNoShellsFound() { _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [] }); var result = await _sut.GetAllSubmodelsAsync(null, null, null, null, CancellationToken.None); @@ -267,7 +289,7 @@ public async Task GetAllSubmodelsAsync_BuildsSubmodelForEachSubmodelId() var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _aasRepositoryTemplateService @@ -297,7 +319,7 @@ public async Task GetAllSubmodelsAsync_SkipsSubmodel_WhenFilteredTemplateReturns var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _aasRepositoryTemplateService @@ -319,7 +341,7 @@ public async Task GetAllSubmodelsAsync_SkipsShell_WhenSubmodelRefsFails() const string ShellId = "https://example.com/shells/001"; _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _aasRepositoryTemplateService @@ -339,7 +361,7 @@ public async Task GetAllSubmodelsAsync_DeduplicatesSubmodelIds_AcrossShells() var filledSubmodel = TestData.CreateFilledSubmodel(); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = @@ -378,6 +400,8 @@ public async Task GetAllSubmodelsAsync_FiltersShellsByIdShort_WhenIdShortProvide .GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Is(f => f != null && f.IdShort == IdShort), + Arg.Any(), + Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [] }); @@ -387,6 +411,8 @@ await _pluginDataHandler.Received(1) .GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Is(f => f != null && f.IdShort == IdShort), + Arg.Any(), + Arg.Any(), Arg.Any()); } @@ -402,7 +428,7 @@ public async Task GetAllSubmodelsAsync_WhenSemanticIdFilterProvided_LooksUpFilte var filledSubmodel = TestData.CreateFilledSubmodel(); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _templateService @@ -437,7 +463,7 @@ public async Task GetAllSubmodelsAsync_WhenSemanticIdNotFound_ThrowsSubmodelNotF var filter = new SubmodelSearchFilter { SemanticId = SemanticId }; _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = "https://example.com/shells/001" }] }); // GetFilteredSubmodelTemplateIdAsync returns null — semantic ID not found in any template @@ -458,7 +484,7 @@ public async Task GetAllSubmodelsAsync_SkipsShellsWithNullOrEmptyId() var filledSubmodel = TestData.CreateFilledSubmodel(); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = @@ -491,6 +517,110 @@ public async Task GetAllSubmodelsAsync_SkipsShellsWithNullOrEmptyId() await _aasRepositoryTemplateService.DidNotReceive().GetSubmodelRefByIdAsync(null!, Arg.Any()); } + [Fact] + public async Task GetAllSubmodelsAsync_WhenLimitReachedMidProduct_ReturnsResumeCursor() + { + const string AasId = "https://example.com/shells/001"; + const string SubmodelId1 = "https://example.com/submodels/SM-1"; + const string SubmodelId2 = "https://example.com/submodels/SM-2"; + const string SubmodelId3 = "https://example.com/submodels/SM-3"; + + _pluginDataHandler + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = AasId }] }); + + _aasRepositoryTemplateService + .GetSubmodelRefByIdAsync(AasId, Arg.Any()) + .Returns([SubmodelReference(SubmodelId1), SubmodelReference(SubmodelId2), SubmodelReference(SubmodelId3)]); + + ArrangeSubmodelBuildPipeline(); + + var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 2, cursor: null, CancellationToken.None); + + Assert.Equal(2, result.Result.Count); + + var nextCursor = SubmodelPageCursor.TryDecode(result.PagingMetaData?.Cursor); + Assert.NotNull(nextCursor); + Assert.Null(nextCursor!.PluginPageCursor); + Assert.Equal(AasId, nextCursor.CurrentAasId); + Assert.Equal(SubmodelId2, nextCursor.LastSubmodelId); + } + + [Fact] + public async Task GetAllSubmodelsAsync_WithResumeCursor_SkipsAlreadyDeliveredSubmodels() + { + const string AasId = "https://example.com/shells/001"; + const string SubmodelId1 = "https://example.com/submodels/SM-1"; + const string SubmodelId2 = "https://example.com/submodels/SM-2"; + const string SubmodelId3 = "https://example.com/submodels/SM-3"; + + var resumeCursor = new SubmodelPageCursor(null, AasId, SubmodelId2).Encode(); + + _pluginDataHandler + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = AasId }] }); + + _aasRepositoryTemplateService + .GetSubmodelRefByIdAsync(AasId, Arg.Any()) + .Returns([SubmodelReference(SubmodelId1), SubmodelReference(SubmodelId2), SubmodelReference(SubmodelId3)]); + + ArrangeSubmodelBuildPipeline(); + + var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 100, cursor: resumeCursor, CancellationToken.None); + + // Only the submodel after the resume point should remain, and the data is exhausted. + Assert.Single(result.Result); + Assert.Null(result.PagingMetaData?.Cursor); + await _templateService.Received(1).GetFilteredSubmodelTemplateAsync(SubmodelId3, (string?)null, Arg.Any(), Arg.Any()); + await _templateService.DidNotReceive().GetFilteredSubmodelTemplateAsync(SubmodelId1, (string?)null, Arg.Any(), Arg.Any()); + await _templateService.DidNotReceive().GetFilteredSubmodelTemplateAsync(SubmodelId2, (string?)null, Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task GetAllSubmodelsAsync_WhenLimitSpansPluginPages_AdvancesPluginCursor() + { + const string AasId1 = "https://example.com/shells/001"; + const string AasId2 = "https://example.com/shells/002"; + const string NextPluginCursor = "PLUGIN-PAGE-2"; + const string SubmodelId1 = "https://example.com/submodels/SM-1"; + const string SubmodelId2 = "https://example.com/submodels/SM-2"; + const string SubmodelId3 = "https://example.com/submodels/SM-3"; + const string SubmodelId4 = "https://example.com/submodels/SM-4"; + + // First plugin page (null cursor) points to the next page. + _pluginDataHandler + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Is(c => c == null), Arg.Any()) + .Returns(new ShellDescriptorsMetaData + { + ShellDescriptors = [new ShellDescriptorMetaData { Id = AasId1 }], + PagingMetaData = new PagingMetaData { Cursor = NextPluginCursor } + }); + + // Second plugin page (advanced cursor) ends the data. + _pluginDataHandler + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Is(c => c == NextPluginCursor), Arg.Any()) + .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = AasId2 }] }); + + _aasRepositoryTemplateService + .GetSubmodelRefByIdAsync(AasId1, Arg.Any()) + .Returns([SubmodelReference(SubmodelId1), SubmodelReference(SubmodelId2)]); + _aasRepositoryTemplateService + .GetSubmodelRefByIdAsync(AasId2, Arg.Any()) + .Returns([SubmodelReference(SubmodelId3), SubmodelReference(SubmodelId4)]); + + ArrangeSubmodelBuildPipeline(); + + var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 3, cursor: null, CancellationToken.None); + + Assert.Equal(3, result.Result.Count); + + var nextCursor = SubmodelPageCursor.TryDecode(result.PagingMetaData?.Cursor); + Assert.NotNull(nextCursor); + Assert.Equal(NextPluginCursor, nextCursor!.PluginPageCursor); + Assert.Equal(AasId2, nextCursor.CurrentAasId); + Assert.Equal(SubmodelId3, nextCursor.LastSubmodelId); + } + [Fact] public async Task GetAllSubmodelElementsAsync_ReturnsAllElements_WhenSubmodelExists() { diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs index 9c650eff..940339ce 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs @@ -630,14 +630,14 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_ReturnsShellDescript }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new List { httpResponse.Content }); const string Json = """[{"name":"sn","value":"123"}]"""; var assetIds = JsonSerializer.Deserialize>(Json)!; var filter = new ShellSearchFilter { SpecificAssetIds = assetIds }; - var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None); + var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); Assert.NotNull(result); Assert.Single(result.ShellDescriptors); @@ -673,12 +673,12 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenNullDeserializat }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None)); + _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None)); } [Fact] @@ -710,12 +710,12 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenInvalidJson_Thro }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None)); + _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None)); } [Fact] @@ -757,11 +757,11 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_SetsHrefOnResults() }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; - var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None); + var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); Assert.Equal(2, result.ShellDescriptors.Count); Assert.All(result.ShellDescriptors, dto => Assert.StartsWith("https://www.mm-software.com/shells/", dto.Href)); @@ -786,7 +786,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenNoAvailablePlugi var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None)); + _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None)); } [Fact] @@ -829,17 +829,17 @@ public async Task GetDataForShellsByAssetIdsAsync_WithIdShort_PassesIdShort() const string targetIdShort = "Motor001"; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), null, targetIdShort, Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Is(v => v == null), Arg.Is(v => v == targetIdShort), Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { IdShort = targetIdShort }; - var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None); + var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); Assert.NotNull(result); Assert.Single(result.ShellDescriptors); Assert.Equal("urn:aas:001", result.ShellDescriptors[0].Id); await _pluginDataProvider.Received(1) - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), null, targetIdShort, Arg.Any()); + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Is(v => v == null), Arg.Is(v => v == targetIdShort), Arg.Any(), Arg.Any(), Arg.Any()); } private const string AssetData = """ diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs index 0af87a02..89ee6b53 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Net.Http.Json; using System.Text; using System.Text.Json; @@ -391,7 +391,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_ReturnsContent() var assetIdsHeaderValue = """[{"name":"SerialNumber","value":"SN-4711"}]"""; - var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, assetIdsHeaderValue, null, CancellationToken.None); + var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, assetIdsHeaderValue, null, null, null, CancellationToken.None); Assert.NotNull(result); Assert.Single(result); @@ -431,7 +431,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WithIdShort_AddsIdSh const string idShort = "test-idshort-value"; - var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, null, idShort, CancellationToken.None); + var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, null, idShort, null, null, CancellationToken.None); Assert.NotNull(result); Assert.Single(result); @@ -461,7 +461,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenNotFound_ThrowsR var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); } [Fact] @@ -481,7 +481,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenUnauthorized_Thr var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); } [Fact] @@ -495,7 +495,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenTimeout_ThrowsRe var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); } [Fact] @@ -504,7 +504,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WithNullRequest_Skip var metadata = new List { null! }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); } [Fact] @@ -524,7 +524,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenForbidden_Throws var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); } private static List GetTestShellDescriptorDataList() diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs index 129b5a34..fbe944c0 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs @@ -230,7 +230,7 @@ private void FillShellFromMetadata(IAssetAdministrationShell shell, ShellDescrip CancellationToken cancellationToken) { var metadata = await pluginDataHandler - .GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, filter, cancellationToken) + .GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, filter, null, null, cancellationToken) .ConfigureAwait(false); var allMetadata = metadata.ShellDescriptors? diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs index e6c17cfe..df111ab3 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Base; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; @@ -73,7 +73,7 @@ private async Task GetFilteredMetadataAsync(List GetDataForAssetInformationByIdAsync(IReadOnlyList pluginManifests, string id, CancellationToken cancellationToken); - Task GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, CancellationToken cancellationToken); + Task GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, int? limit, string? cursor, CancellationToken cancellationToken); } diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs index 53b42af7..cc193f52 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.DomainModel.Plugin; +using AAS.TwinEngine.DataEngine.DomainModel.Plugin; namespace AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin.Providers; @@ -12,5 +12,5 @@ public interface IPluginDataProvider Task> GetDataForAssetInformationByIdAsync(IList pluginRequests, CancellationToken cancellationToken); - Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, CancellationToken cancellationToken); + Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, int? limit, string? cursor, CancellationToken cancellationToken); } diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursor.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursor.cs new file mode 100644 index 00000000..dd6d206b --- /dev/null +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursor.cs @@ -0,0 +1,47 @@ +using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; + +namespace AAS.TwinEngine.DataEngine.ApplicationLogic.Services.SubmodelRepository; + +/// +/// Opaque cursor for the "Get All Submodels" flow. A single product (AAS) can hold multiple +/// submodels and may be only partially delivered within one page, so resuming requires more +/// state than a single submodel id. +/// +/// +/// The plugin cursor that fetched the batch the client is currently positioned in +/// (null for the first plugin page). On resume the same batch is re-fetched so products +/// that were returned alongside a partially consumed product are not lost. +/// +/// The AAS id of the partially consumed product to resume within. +/// The last submodel id already delivered inside . +public sealed record SubmodelPageCursor(string? PluginPageCursor, string CurrentAasId, string LastSubmodelId) +{ + // Logical layout: {PluginPageCursor}|{CurrentAasId}|{LastSubmodelId} + private const char Separator = '|'; + private const int FieldCount = 3; + + /// Encodes the cursor into the opaque Base64Url token returned to the client. + public string Encode() => + string.Join(Separator, PluginPageCursor, CurrentAasId, LastSubmodelId).EncodeBase64Url(); + + /// + /// Decodes a client supplied cursor. Returns null when the cursor is absent or does not + /// have the expected shape, which callers treat as "start from the first page". + /// + public static SubmodelPageCursor? TryDecode(string? encodedCursor) + { + if (string.IsNullOrWhiteSpace(encodedCursor)) + { + return null; + } + + var parts = encodedCursor.DecodeBase64Url().Split(Separator); + if (parts.Length != FieldCount) + { + return null; + } + + var pluginPageCursor = string.IsNullOrEmpty(parts[0]) ? null : parts[0]; + return new SubmodelPageCursor(pluginPageCursor, parts[1], parts[2]); + } +} diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 50f443f2..45d323bf 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -5,6 +5,7 @@ using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin; using AAS.TwinEngine.DataEngine.DomainModel.AasRegistry; using AAS.TwinEngine.DataEngine.DomainModel.AasRepository; +using AAS.TwinEngine.DataEngine.DomainModel.Shared; using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; @@ -26,6 +27,9 @@ public class SubmodelRepositoryService( IOptions templateManagementConfig) : ISubmodelRepositoryService { private readonly int _concurrentOperationsLimit = templateManagementConfig.Value.SubmodelTemplateRepository.ConcurrentOperationsLimit; + + // Default number of submodels per page when the client does not supply a limit. + private const int DefaultPageSize = 100; public async Task GetSubmodelAsync(string submodelId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) { return await ExecuteWithExceptionHandlingAsync(async () => @@ -62,68 +66,136 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte { return await ExecuteWithExceptionHandlingAsync(async () => { - var shellSearchFilter = new ShellSearchFilter - { - IdShort = filter?.IdShort - }; - - var shellMetadata = await pluginDataHandler.GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, shellSearchFilter, cancellationToken).ConfigureAwait(false); - var shellDescriptors = shellMetadata.ShellDescriptors ?? []; - - string? filteredTemplateId = null; - if (filter?.SemanticId is not null) - { - filteredTemplateId = await submodelTemplateService.GetFilteredSubmodelTemplateIdAsync(filter.SemanticId, cancellationToken).ConfigureAwait(false); - - if (filteredTemplateId is null) - { - throw new SubmodelNotFoundException(); - } - } - - var distinctSubmodelIds = await GetDistinctSubmodelIdsAsync(shellDescriptors, cancellationToken).ConfigureAwait(false); + var filteredTemplateId = await ResolveFilteredTemplateIdAsync(filter?.SemanticId, cancellationToken).ConfigureAwait(false); - var (pagedIds, pagingMetaData) = PagingExtensions.GetPagedResult(distinctSubmodelIds, id => id, limit, cursor); + var (submodelIds, nextCursor) = await CollectSubmodelIdPageAsync(filter?.IdShort, limit, cursor, cancellationToken).ConfigureAwait(false); - var submodels = await BuildSubmodelsAsync(pagedIds, filteredTemplateId, queryOptions, cancellationToken).ConfigureAwait(false); + var submodels = await BuildSubmodelsAsync(submodelIds, filteredTemplateId, queryOptions, cancellationToken).ConfigureAwait(false); return new SubmodelList { - PagingMetaData = pagingMetaData, + PagingMetaData = new PagingMetaData { Cursor = nextCursor }, Result = submodels }; }).ConfigureAwait(false); } - private async Task> GetDistinctSubmodelIdsAsync(List shellDescriptors, CancellationToken cancellationToken) + private async Task ResolveFilteredTemplateIdAsync(string? semanticId, CancellationToken cancellationToken) { - using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); - var tasks = shellDescriptors.Where(shell => !string.IsNullOrWhiteSpace(shell.Id)).Select(async shell => + if (semanticId is null) + { + return null; + } + + return await submodelTemplateService.GetFilteredSubmodelTemplateIdAsync(semanticId, cancellationToken).ConfigureAwait(false) + ?? throw new SubmodelNotFoundException(); + } + + /// + /// Builds one page of submodel ids using the two-phase resume strategy: a partially consumed + /// product is finished first (Phase 1), then forward pagination continues through the next + /// products and plugin pages (Phase 2) until the page size is reached or the data is exhausted. + /// + private async Task<(IReadOnlyList SubmodelIds, string? NextCursor)> CollectSubmodelIdPageAsync( + string? idShort, int? limit, string? encodedCursor, CancellationToken cancellationToken) + { + var pageSize = limit ?? DefaultPageSize; + var resumePoint = SubmodelPageCursor.TryDecode(encodedCursor); + + var collected = new List(); + var seen = new HashSet(StringComparer.Ordinal); + + // Plugin cursor that produced the batch currently being scanned (null = first plugin page). + var pluginPageCursor = resumePoint?.PluginPageCursor; + // Resume coordinates; only meaningful for the first scanned batch. + var resumeAasId = resumePoint?.CurrentAasId; + var resumeAfterSubmodelId = resumePoint?.LastSubmodelId; + + while (true) + { + var batch = await FetchShellBatchAsync(idShort, pageSize, pluginPageCursor, cancellationToken).ConfigureAwait(false); + + foreach (var aasId in SkipProductsBefore(batch.ShellIds, resumeAasId)) { - await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - return await aasRepositoryTemplateService.GetSubmodelRefByIdAsync(shell.Id!, cancellationToken).ConfigureAwait(false); - } - catch (ResourceNotFoundException ex) + var submodelIds = await ExpandProductSubmodelIdsAsync(aasId, cancellationToken).ConfigureAwait(false); + + // On resume, drop the submodels that were already delivered from this product. + if (resumeAfterSubmodelId is not null) { - logger.LogWarning(ex, "Could not retrieve submodel refs for shell {ShellId}. Skipping shell.", shell.Id); - return []; + submodelIds = SkipThrough(submodelIds, resumeAfterSubmodelId); + resumeAfterSubmodelId = null; } - finally + + foreach (var submodelId in submodelIds) { - _ = semaphore.Release(); + if (!seen.Add(submodelId)) + { + continue; + } + + collected.Add(submodelId); + if (collected.Count == pageSize) + { + var nextCursor = new SubmodelPageCursor(pluginPageCursor, aasId, submodelId).Encode(); + return (collected, nextCursor); + } } - }); + } - var references = await Task.WhenAll(tasks).ConfigureAwait(false); + // The whole batch is consumed; resume state applies to the first batch only. + resumeAasId = null; - return references - .SelectMany(x => x) - .Select(reference => reference.Keys.FirstOrDefault()?.Value) + if (string.IsNullOrEmpty(batch.NextPluginCursor)) + { + return (collected, null); + } + + pluginPageCursor = batch.NextPluginCursor; + } + } + + private async Task<(List ShellIds, string? NextPluginCursor)> FetchShellBatchAsync( + string? idShort, int pageSize, string? pluginPageCursor, CancellationToken cancellationToken) + { + var shellSearchFilter = new ShellSearchFilter { IdShort = idShort }; + + var metadata = await pluginDataHandler + .GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, shellSearchFilter, pageSize, pluginPageCursor, cancellationToken) + .ConfigureAwait(false); + + var shellIds = (metadata.ShellDescriptors ?? []) + .Select(shell => shell.Id) .Where(id => !string.IsNullOrWhiteSpace(id)) - .Distinct() - .ToList()!; + .ToList(); + + return (shellIds!, metadata.PagingMetaData?.Cursor); + } + + private static IEnumerable SkipProductsBefore(IEnumerable shellIds, string? resumeAasId) => + resumeAasId is null ? shellIds : shellIds.SkipWhile(id => id != resumeAasId); + + private static List SkipThrough(List submodelIds, string lastDeliveredSubmodelId) + { + var index = submodelIds.IndexOf(lastDeliveredSubmodelId); + return index < 0 ? submodelIds : [.. submodelIds.Skip(index + 1)]; + } + + private async Task> ExpandProductSubmodelIdsAsync(string aasId, CancellationToken cancellationToken) + { + try + { + var references = await aasRepositoryTemplateService.GetSubmodelRefByIdAsync(aasId, cancellationToken).ConfigureAwait(false); + + return references + .Select(reference => reference.Keys.FirstOrDefault()?.Value) + .Where(id => !string.IsNullOrWhiteSpace(id)) + .ToList()!; + } + catch (ResourceNotFoundException ex) + { + logger.LogWarning(ex, "Could not retrieve submodel refs for shell {ShellId}. Skipping shell.", aasId); + return []; + } } private async Task> BuildSubmodelsAsync(IEnumerable submodelIds, string? filteredTemplateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs index f8aa9ca8..0b45ba05 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs @@ -207,7 +207,7 @@ public async Task GetDataForAssetInformationByIdAsync(IReadOnlyList

GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, CancellationToken cancellationToken) + public async Task GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, int? limit, string? cursor, CancellationToken cancellationToken) { using var activity = DataEngineTracing.StartSpan(DataEngineTracing.Spans.GetPluginMetadataShells); @@ -230,7 +230,7 @@ public async Task GetDataForShellsByAssetIdsAsync(IRea })) : null; - var response = await pluginDataProvider.GetDataForShellDescriptorsByAssetIdsAsync(pluginRequests, assetIdsHeaderValue, filter?.IdShort, cancellationToken).ConfigureAwait(false); + var response = await pluginDataProvider.GetDataForShellDescriptorsByAssetIdsAsync(pluginRequests, assetIdsHeaderValue, filter?.IdShort, limit, cursor, cancellationToken).ConfigureAwait(false); var result = new ShellDescriptorsMetaData(); diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs index e12c38a8..6a5b4737 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs @@ -105,14 +105,14 @@ public Task> GetDataForShellDescriptorByIdAsync(IList> GetDataForAssetInformationByIdAsync(IList pluginRequests, CancellationToken cancellationToken) => GetAndProcessAsync(pluginRequests, AssetInformationEndpoint, cancellationToken); - public async Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, CancellationToken cancellationToken) + public async Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, int? limit, string? cursor, CancellationToken cancellationToken) { var result = new List(); var exceptions = new List(); foreach (var pluginRequest in pluginRequests) { - var url = BuildUrl(ApiPaths.PluginMetadata, ShellsEndpoint); + var url = BuildShellsUrl(limit, cursor); if (pluginRequest == null) { From ef0393d97bb1c7c396b88b2087c3abba511feefa Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Fri, 31 Jul 2026 17:49:02 +0530 Subject: [PATCH 12/30] refactor: rewrite submodel pagination concept documentation to detail the two-field composite cursor design --- Submodel-Pagination-Cursor-Concept.md | 587 +++++++++++--------------- 1 file changed, 237 insertions(+), 350 deletions(-) diff --git a/Submodel-Pagination-Cursor-Concept.md b/Submodel-Pagination-Cursor-Concept.md index 4c3de69e..a6906b9f 100644 --- a/Submodel-Pagination-Cursor-Concept.md +++ b/Submodel-Pagination-Cursor-Concept.md @@ -1,424 +1,311 @@ -# Submodel Pagination Cursor Design - Get All Submodels +# Submodel Pagination Cursor Concept - Get All Submodels -## 1. Problem Statement +## Problem +The DataEngine's `GET /submodels` and `GET /submodel-descriptors` endpoints must paginate **submodels**, but the underlying Plugin APIs paginate **products (Asset Administration Shells / AAS IDs)**. In an Asset Administration Shell ecosystem, a single product (AAS) can contain multiple submodels (a `1-to-N` expansion relationship). -The DataEngine's `GET /submodels` endpoint must paginate **submodels**, but the underlying Plugin API paginates **products (AAS IDs)**. A single product may contain multiple submodels, and a product can be only **partially consumed** within one response page. The cursor must encode enough state to resume mid-product without duplicating or skipping submodels. +When a client requests a page of submodels with a specific `limit` (e.g., `limit=2` or `limit=100`), a single product's submodel list may be only **partially consumed** when the page limit is reached. If the pagination engine uses standard single-field or exclusive cursor pagination (which moves the underlying Plugin cursor past the product), any remaining uncollected submodels in that product are permanently skipped on subsequent page requests. ---- - -## 2. Data Model +## Constraints +- **O(limit) Memory Footprint:** The pagination engine must operate with constant memory proportional to the requested page `limit` and batch size, never loading the full dataset into memory. +- **No Plugin API Modifications:** Underlying Plugins implement standard IDTA / BaSyx product-level cursor pagination (`limit` and `cursor`); Plugins do not support submodel-level cursors. +- **AAS-ID-Based Exclusive Plugin Cursor:** The Plugin cursor is AAS-ID-based and uses exclusive semantics - passing an AAS ID as cursor instructs the Plugin to return all AAS strictly after that ID in its deterministic ordering. +- **Deterministic Ordering:** Plugins return products in stable, deterministic order, and submodel references within a product maintain a deterministic order. +- **Zero Data Loss / Zero Duplication:** Traversing all pages from start to finish must yield every submodel exactly once under static conditions. +- **IDTA Compliance:** The pagination solution must remain fully compliant with the IDTA AAS Part 2 API specification. The DataEngine may introduce internal pagination logic and cursor management, but the externally exposed API behavior, request/response contracts must remain compliant with the IDTA specification. -``` -Product 1 (AAS-1) Product 2 (AAS-2) Product 3 (AAS-3) -├── SM-1 ├── SM-4 ├── SM-7 -├── SM-2 ├── SM-5 ├── SM-8 -└── SM-3 ├── SM-6 └── SM-9 -``` - -- **3 products**, each with **3 submodels** → **9 submodels** total. -- Plugin returns products in a stable, deterministic order. -- Submodel IDs within a product are derived from Shell Templates and maintain a deterministic order. +## Assumptions +- Each Asset Administration Shell (`IAssetAdministrationShell`) has a unique `ID` and a deterministic list of submodel references (`Submodels`). +- The API endpoint acts as the orchestrator over one or more external Plugin instances. +- Submodel data is collected on-demand after identifying candidate submodel IDs from shell templates. --- -## 3. The Limit=2 Problem (Why a Simple Two-Field Cursor Fails) - -Consider `limit=2` with our example data: - -``` -Page 1: GET /submodels?limit=2 - - Expand AAS-1 → [SM-1, SM-2, SM-3] - Collect SM-1, SM-2. Limit reached. - Cursor = (AAS-1, SM-2) ← AAS-1 is PARTIALLY consumed (SM-3 still pending) - -Page 2: GET /submodels?limit=2&cursor=(AAS-1, SM-2) - - Need to resume inside AAS-1 to get SM-3. - But Plugin cursor semantics are EXCLUSIVE: passing AAS-1 returns [AAS-2, AAS-3, ...] - SM-3 is LOST! -``` - -**Root cause:** The Plugin's cursor-based pagination uses exclusive semantics ("give me items AFTER this cursor"). Once the Plugin moves past a product, there's no way to go back using the Plugin's pagination API alone. - -**The partially consumed product must be handled independently from the Plugin's forward pagination.** +## Considered Alternatives + +### 1. Submodel ID Cursor with Unbounded Plugin Fetching (No Plugin Limit/Cursor) +- **Description:** Store only the last delivered `SubmodelId` in the client cursor token (`cursor = LastSubmodelId`). When fetching products from the Plugin (AAS Shell request), the DataEngine does **not** provide any `limit` or `cursor` to the Plugin, thereby fetching **all products (shells)** on every page request. During submodel expansion, the DataEngine iterates through all products and submodels from the beginning, skipping submodels until it matches the requested `SubmodelId` cursor, and then collects the next page of submodels up to `limit`. +- **Pros:** + - Very simple and compact cursor token (only a single `SubmodelId`). + - Correctness is maintained (no submodels are skipped or lost across page boundaries, as all products are inspected). + - No complex multi-field cursor coordinate tracking required. +- **Cons:** + - **High Upstream Overload & Network Waste:** Fetching *all* products from the Plugin on *every single page request* destroys scalability. If there are 100,000 products, every page request fetches all 100,000 products from the Plugin just to find where `SubmodelId` is located. + +### 2. Product ID Bulk Endpoint in Plugin with DataEngine Expansion +- **Description:** Introduce a new bulk endpoint in the Plugin contract to retrieve all product IDs (or a paginated list of all product IDs). The DataEngine would retrieve these product IDs, resolve the shell template for each ID to determine the associated submodels, and then query the Plugin for the specific submodel data. Under this approach, the client cursor only needs to store the last delivered `SubmodelId`. +- **Pros:** + - **Simple Cursor Token:** The pagination cursor remains simple (e.g., just the last delivered `SubmodelId` or index). +- **Cons:** + - **Plugin Contract Breaking Change:** Requires updating the Plugin API contract to support the new bulk product ID endpoint. + - **High Architectural Complexity:** Orchestrating product ID retrieval, resolving matching shell templates, and mapping them to submodels on the fly in the DataEngine introduces significant complexity and state tracking. + - **Association / Resolution Difficulties:** It is difficult to map a product ID directly to its associated shell template and submodels without fetching the entire shell, which negates the performance benefits. + + +### 3. Two-Field Composite Cursor (`SubmodelId` | `AasId`) (Chosen) +- **Description:** The cursor stores exactly two lightweight coordinates: the last submodel ID delivered to the client (`SubmodelId`) and the AAS ID that serves simultaneously as both the position anchor and the next plugin request cursor (`AasId`). Because the plugin cursor is AAS-ID-based with exclusive semantics, the engine can derive the exact plugin request cursor directly from the AAS ID stored in the DataEngine cursor. A single tracking state - initialized from the cursor's `AasId` field and advanced each time an AAS is fully consumed - determines which AAS to start from on every plugin request. This eliminates the need for a separate opaque plugin cursor field. +- **Pros:** + - **Guaranteed Correctness:** Eliminates data loss and duplication; fully supports `1-to-N` expansion. + - **Minimal Cursor Size:** Exactly 2 pipe-delimited string fields, smaller than a 3-field cursor. + - **Constant Memory (`O(limit)`):** Streams and collects only what is needed for the current page. + - **No Plugin API Changes Required:** Works seamlessly over AAS-ID-based exclusive cursor endpoints. + - **Bounded Skip Scan:** The `SubmodelId` skip scan is confined to the first AAS of the returned batch only, never scanning the full dataset. --- -## 4. Cursor Structure (Two-Field Composite) +### Alternative Comparison Summary -### 4.1 Logical Composition +| Approach | Memory Footprint | Correctness (No Data Loss) | Cursor Token Size | Plugin API Changes Required | +|---|---|---|---|---| +| **1. Submodel ID Cursor (Unbounded Fetch)** | `O(N)` (Unbounded Fetch) | Yes | Small (`SubmodelId`) | No | +| **2. Product ID Bulk Endpoint** | `O(N)` (Unbounded Fetch of IDs) | Yes | Small (`SubmodelId`) | **Yes** | +| **3. Two-Field Composite Cursor (Chosen)** | `O(limit)` (Optimal) | **Yes** | **Smallest (2 fields)** | No | -The cursor requires **two** coordinates: +--- -| Field | Type | Purpose | -|-------|------|---------| -| `SubmodelId` | string | The last submodel ID **included** in the previous response. Marks the exact resume point within the product. | -| `AasId` | string | The AAS ID that serves simultaneously as the position anchor and the exclusive plugin request cursor. | +## Decision +We adopt **Option 3: Two-Field Composite Cursor (`SubmodelId` | `AasId`)** as the standard architectural pattern for all submodel and submodel-descriptor pagination. -### 4.2 Why Two Fields? +--- -| Scenario | What's needed | -|----------|--------------| -| Product partially consumed | `AasId` to re-expand the product on the first batch; `SubmodelId` to skip delivered items | -| Exclusive plugin request | `AasId` is passed directly as the exclusive plugin cursor (`trackingAasId`) to continue forward | +## Architectural Concept & Two-Field Composite Cursor -### 4.3 Wire Format +### 1. The `1-to-N` Expansion Problem (`limit=2` Walkthrough) +Consider an environment with 3 products (`AAS-1`, `AAS-2`, `AAS-3`), each containing 3 submodels: ``` -Logical: {SubmodelId}|{AasId} -Encoded: Base64Url( UTF-8( "{SubmodelId}|{AasId}" ) ) +Product 1 (AAS-1) Product 2 (AAS-2) Product 3 (AAS-3) +├── SM-1 ├── SM-4 ├── SM-7 +├── SM-2 ├── SM-5 ├── SM-8 +└── SM-3 ├── SM-6 └── SM-9 ``` -**Example:** - -``` -Logical: https://example.com/submodels/Nameplate|https://example.com/shells/001 -Encoded: aHR0cHM6Ly9leGFtcGxlLmNvbS9zdWJtb2RlbHMvTmFtZXBsYXRlfGh0dHBzOi8vZXhhbXBsZS5jb20vc2hlbGxzLzAwMQ== +When a client executes `GET /submodels?limit=2`: ``` +Page 1: GET /submodels?limit=2 -The cursor is **opaque** to the client - they never parse or construct it. + 1. API calls Plugin.GetShells(cursor=null) → returns [AAS-1, AAS-2] + 2. Expand AAS-1 → [SM-1, SM-2, SM-3] + 3. Collect SM-1 (Count=1), Collect SM-2 (Count=2). Page limit reached! + 4. AAS-1 is PARTIALLY consumed (SM-3 is still pending). +``` -### 4.4 Special Cases +If the API were to advance the Plugin cursor past AAS-1 for Page 2: +``` +Page 2: GET /submodels?limit=2&cursor="AAS-1" (exclusive - starts after AAS-1) -| Scenario | Cursor Value | -|----------|-------------| -| First page (no cursor) | `null` / absent | -| Last page (no more data) | Response returns `cursor = null` (signals end of dataset) | -| Limit reached mid-product or at boundary | `{LastSubmodelId}|{CurrentAasId}` | ---- + 1. API calls Plugin.GetShells(cursor="AAS-1") + 2. Plugin returns products AFTER AAS-1 → [AAS-2, AAS-3, ...] + 3. RESULT: SM-3 (from AAS-1) IS PERMANENTLY LOST! +``` -## 5. Pagination Algorithm +The cursor must not advance the Plugin cursor until the current AAS is fully consumed. -### 5.1 Two-Phase Resume Strategy +### 2. Logical Composition of the Two-Field Cursor -The key insight: **on resume, the partially consumed product is handled FIRST (Phase 1) using only its AAS ID, THEN forward pagination continues (Phase 2) using the Plugin's next cursor.** +The cursor stores two coordinates that together capture exact positioning across both the AAS layer and the Submodel layer: -``` -┌────────────────────────────────────────────────────────────────────────┐ -│ PHASE 1: Complete the partially consumed product │ -│ │ -│ - Re-derive submodel IDs for CurrentAasId (via template expansion) │ -│ - Skip submodels up to and including LastSubmodelId │ -│ - Collect remaining submodels until limit or product exhausted │ -│ │ -│ NO Plugin pagination call needed - we already know the AAS ID. │ -├────────────────────────────────────────────────────────────────────────┤ -│ PHASE 2: Continue with next products (if limit not yet reached) │ -│ │ -│ - Call Plugin.GetProducts(cursor = PluginNextCursor) │ -│ - Expand each returned product's submodels │ -│ - Collect until limit reached or Plugin exhausted │ -└────────────────────────────────────────────────────────────────────────┘ -``` - -### 5.2 Plugin Cursor Coordination +| Coordinate | Wire Field Position | Nullable | Description | +|---|---|---|---| +| `SubmodelId` | Index 0 (Prefix) | Yes (`null` when limit is reached exactly at an AAS boundary) | The unique identifier of the last submodel delivered to the client. On resume, the API scans only the **first AAS** of the returned batch to skip already-delivered submodels. | +| `AasId` | Index 1 (Suffix) | Yes (`null` on first page or when no AAS has been fully consumed) | The AAS ID that is passed directly to the Plugin as an exclusive cursor on resume. The Plugin returns all AAS strictly after this ID. Advances only when an AAS is fully consumed. | +#### Wire Encoding Strategy +The logical cursor string is formatted as UTF-8 text separated by a pipe character (`|`), then encoded using standard `Base64Url` encoding (RFC 4648) without padding: ``` -┌───────────────────────────────────────────────────────────────────────────┐ -│ Page 1: GET /submodels?limit=2 │ -│ │ -│ Engine calls Plugin: GetProducts(limit=2, cursor=null) │ -│ Plugin returns: [AAS-1, AAS-2], pluginNextCursor = "C1" │ -│ │ -│ Expand AAS-1 → [SM-1, SM-2, SM-3] │ -│ Collect SM-1, SM-2. Limit reached. │ -│ │ -│ Cursor = Encode("C1", "AAS-1", "SM-2") │ -│ ▲ ▲ ▲ │ -│ │ │ └── Last delivered submodel │ -│ │ └──────────── Product still being consumed │ -│ └─────────────────── Plugin cursor for NEXT products │ -│ (skips AAS-1 and AAS-2) │ -├───────────────────────────────────────────────────────────────────────────┤ -│ Page 2: Resume with cursor ("C1", "AAS-1", "SM-2") │ -│ │ -│ PHASE 1: Re-expand AAS-1 → [SM-1, SM-2, SM-3] │ -│ Skip SM-1, SM-2. Collect SM-3. Count = 1. │ -│ │ -│ PHASE 2: Need 1 more. But wait - AAS-2 was in the same Plugin page! │ -│ We need AAS-2 before using "C1". │ -│ SEE SECTION 5.4 FOR HANDLING THIS │ -└───────────────────────────────────────────────────────────────────────────┘ +Logical Structure: {SubmodelId}|{AasId} +Encoded Wire Token: Base64Url( UTF-8( "{SubmodelId}|{AasId}" ) ) ``` -### 5.3 The "Same-Page Products" Problem & Solution +**Examples:** +- Mid-AAS cursor, no AAS yet fully consumed (re-fetch from plugin start): + - Logical: `SM-2|` (AasId is empty/null) + - Encoded: `U00tMnw` +- Mid-AAS cursor, AAS-1 was fully consumed (plugin starts after AAS-1): + - Logical: `SM-4|AAS-1` + - Encoded: `U00tNHxBQVMtMQ` +- Limit reached at AAS boundary, AAS-2 fully consumed (plugin starts after AAS-2): + - Logical: `|AAS-2` (SubmodelId is empty/null) + - Encoded: `fEFBUy0y` -When the Plugin returns `[AAS-1, AAS-2]` with `nextCursor = "C1"`: -- `"C1"` skips **both** AAS-1 and AAS-2. -- But if we only consumed AAS-1 partially, AAS-2 hasn't been processed at all! +### 3. The AasId Tracking Logic -**Solution: Store the Plugin cursor that produced the current batch, not the "next" cursor.** +A single position-tracking state drives the cursor throughout a page execution. It is: +- **Initialized** from the incoming cursor's `AasId` field (null at the very first request). +- **Advanced** each time an AAS within the current batch is fully consumed - it is updated to that AAS's ID. +- **Captured** as the `AasId` field of the output cursor when the page limit is reached. -Revised cursor structure: +This guarantees the following invariant: the `AasId` in the output cursor always equals the ID of the **last fully consumed AAS at the moment the page limit was reached**, or null if no AAS was fully consumed since the current plugin batch started. On the next request, the API passes this `AasId` directly to the Plugin as an exclusive cursor, which returns all AAS strictly after that ID - precisely the position where collection should resume. -| Field | Purpose | -|-------|---------| -| `PluginPageCursor` | The cursor that was used to **fetch the current batch** (null for the first page). On resume, re-fetch the same batch from the Plugin. | -| `CurrentAasId` | The partially consumed product's AAS ID. | -| `LastSubmodelId` | Last delivered submodel within that product. | +When the page limit is reached **exactly at the boundary of an AAS** (i.e., the last submodel of that AAS is the last one collected): the AAS is considered fully consumed, the tracking state advances to that AAS's ID, and `SubmodelId` is set to null. On resume, the API collects from the very first submodel of the first AAS returned by the Plugin. -### 5.4 Why Store `PluginPageCursor` (Not `PluginNextCursor`) +### 4. Resume Algorithm (Conceptual) -| Stored Value | On Resume | Problem | -|---|---|---| -| `PluginNextCursor` | Gets products AFTER current batch | Products between `currentAasId` and end of batch are LOST | -| **`PluginPageCursor`** | **Re-fetches the same batch** | **All products in the batch are available; skip already-consumed ones** | +On receiving a non-null cursor: +1. Extract `SubmodelId` and `AasId` from the cursor. +2. Initialize the tracking state from `AasId`. +3. Call the Plugin with the tracking state as the exclusive cursor → receives the first relevant batch. +4. In the **first AAS of the batch**: + - If `SubmodelId` is set: scan this AAS to find `SubmodelId`, skip everything up to and including it, and begin collecting from the next submodel. + - If `SubmodelId` is null: collect from the very first submodel. +5. For all subsequent AAS in the same batch, and for further batches: collect all submodels from their start. +6. Each time an AAS is fully consumed, advance the tracking state to that AAS's ID. +7. When the page limit is reached: encode the output cursor as `{SubmodelId=last_collected, AasId=current_tracking_state}`. If the limit was reached exactly at an AAS boundary: encode as `{SubmodelId=null, AasId=that_AAS_ID}`. +8. When a batch is exhausted without reaching the page limit: request the next batch from the Plugin using the current tracking state as the exclusive cursor. -The trade-off: on resume, we re-fetch and re-scan some products we've already processed. But: -- The re-scan is bounded (one Plugin page, typically small). -- Correctness is guaranteed - no submodels are ever lost. -- No extra Plugin endpoints required. +The `SubmodelId` skip scan is always bounded to the **first AAS of the batch** - never a full dataset scan - because `AasId` guarantees the Plugin skips all previously fully consumed AAS before returning the batch. --- -## 6. Request/Response Examples - -### 6.1 Example: `limit=2` (The Tricky Case) - -**Page 1 - `GET /submodels?limit=2`** +## Sequence Diagram -``` -1. No cursor → pluginPageCursor = null, no skipping. -2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2], nextPluginCursor="C1" -3. Expand AAS-1 → [SM-1, SM-2, SM-3] -4. Collect SM-1. Count=1. -5. Collect SM-2. Count=2. Limit reached. -6. Cursor = Encode(pluginPageCursor=null, AasId="AAS-1", LastSM="SM-2") -``` +The following sequence diagram illustrates the end-to-end execution flow of `GET /submodels?limit=2` across two page requests, demonstrating how the tracking state advances when an AAS is fully consumed and how the two-field cursor drives the next Plugin request. The API endpoint handles pagination logic directly - no intermediate service layer is involved: -```json -{ - "result": [ - { "id": "SM-1" }, - { "id": "SM-2" } - ], - "paging_metadata": { "cursor": "fEFBUy0xfFNNLTI=" } -} -``` - ---- +```mermaid +sequenceDiagram + autonumber + actor Client + participant API as GET /submodels + participant Plugin as Plugin / AAS Registry -**Page 2 - `GET /submodels?limit=2&cursor=fEFBUy0xfFNNLTI=`** + Note over Client,Plugin: Page 1 Request (cursor = null) + Client->>API: GET /submodels?limit=2 -``` -1. Decode cursor → (pluginPageCursor=null, currentAasId="AAS-1", lastSM="SM-2") -2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2], nextPluginCursor="C1" - (Same call as Page 1 - re-fetches the same batch) -3. Skip products before AAS-1: none to skip. -4. Expand AAS-1 → [SM-1, SM-2, SM-3] -5. Skip submodels up to and including SM-2. -6. Collect SM-3. Count=1. -7. Move to AAS-2 (next product in SAME batch). -8. Expand AAS-2 → [SM-4, SM-5, SM-6] -9. Collect SM-4. Count=2. Limit reached. -10. Cursor = Encode(pluginPageCursor=null, AasId="AAS-2", LastSM="SM-4") -``` + Note over API: Tracking state initialized to null (AasId=null) + API->>Plugin: GetShells(cursor=null, limit=2) + Plugin-->>API: Shells: [AAS-1, AAS-2] -```json -{ - "result": [ - { "id": "SM-3" }, - { "id": "SM-4" } - ], - "paging_metadata": { "cursor": "fEFBUy0yfFNNLTQ=" } -} -``` + Note over API: Expand AAS-1 submodel refs -> [SM-1, SM-2, SM-3] + API->>Plugin: GetSubmodel("SM-1") + Plugin-->>API: Submodel("SM-1") (Count = 1) ---- + API->>Plugin: GetSubmodel("SM-2") + Plugin-->>API: Submodel("SM-2") (Count = 2 - Limit Reached!) -**Page 3 - `GET /submodels?limit=2&cursor=fEFBUy0yfFNNLTQ=`** + Note over API: AAS-1 not fully consumed. Tracking state stays null. + Note over API: Encode Cursor: SubmodelId="SM-2" | AasId=null + API-->>Client: 200 OK [SM-1, SM-2] + cursor="U00tMnw" -``` -1. Decode cursor → (pluginPageCursor=null, currentAasId="AAS-2", lastSM="SM-4") -2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2], nextPluginCursor="C1" -3. Skip products before AAS-2: skip AAS-1. -4. Expand AAS-2 → [SM-4, SM-5, SM-6] -5. Skip submodels up to and including SM-4. -6. Collect SM-5. Count=1. -7. Collect SM-6. Count=2. Limit reached. -8. Cursor = Encode(pluginPageCursor=null, AasId="AAS-2", LastSM="SM-6") -``` + Note over Client,Plugin: Page 2 Request (Resume from partially consumed AAS-1) + Client->>API: GET /submodels?limit=2&cursor=U00tMnw -```json -{ - "result": [ - { "id": "SM-5" }, - { "id": "SM-6" } - ], - "paging_metadata": { "cursor": "fEFBUy0yfFNNLTY=" } -} -``` + Note over API: Decode Cursor -> SubmodelId="SM-2", AasId=null + Note over API: Tracking state initialized to null (from AasId=null) + API->>Plugin: GetShells(cursor=null, limit=2) + Plugin-->>API: Shells: [AAS-1, AAS-2] ---- + Note over API: First AAS = AAS-1. Scan for SM-2 → skip SM-1 and SM-2 + API->>Plugin: GetSubmodel("SM-3") + Plugin-->>API: Submodel("SM-3") (Count = 1, AAS-1 complete) -**Page 4 - `GET /submodels?limit=2&cursor=fEFBUy0yfFNNLTY=`** + Note over API: AAS-1 fully consumed. Tracking state advances to AAS-1. + Note over API: Advance to AAS-2 in same batch -> [SM-4, SM-5, SM-6] + API->>Plugin: GetSubmodel("SM-4") + Plugin-->>API: Submodel("SM-4") (Count = 2 - Limit Reached!) -``` -1. Decode cursor → (pluginPageCursor=null, currentAasId="AAS-2", lastSM="SM-6") -2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2], nextPluginCursor="C1" -3. Skip products before AAS-2: skip AAS-1. -4. Expand AAS-2 → [SM-4, SM-5, SM-6] -5. Skip SM-4, SM-5, SM-6 (all skipped - product fully consumed). -6. No more products in this batch. -7. Move to next Plugin page: GetProducts(cursor="C1") → [AAS-3], nextPluginCursor=null -8. Expand AAS-3 → [SM-7, SM-8, SM-9] -9. Collect SM-7. Count=1. -10. Collect SM-8. Count=2. Limit reached. -11. Cursor = Encode(pluginPageCursor="C1", AasId="AAS-3", LastSM="SM-8") + Note over API: AAS-2 not fully consumed. Tracking state = AAS-1. + Note over API: Encode Cursor: SubmodelId="SM-4" | AasId="AAS-1" + API-->>Client: 200 OK [SM-3, SM-4] + cursor="U00tNHxBQVMtMQ" ``` -```json -{ - "result": [ - { "id": "SM-7" }, - { "id": "SM-8" } - ], - "paging_metadata": { "cursor": "QzF8QUFTLTN8U00tOA==" } -} -``` +On Page 3, the API calls `Plugin.GetShells(cursor="AAS-1")` - the Plugin returns `[AAS-2, AAS-3]` (all AAS strictly after `AAS-1`). The API scans `AAS-2` (the first AAS in the new batch) to find `SM-4`, skips it, and resumes collection from `SM-5`. --- -**Page 5 (Final) - `GET /submodels?limit=2&cursor=QzF8QUFTLTN8U00tOA==`** +## Step-by-Step Examples -``` -1. Decode cursor → (pluginPageCursor="C1", currentAasId="AAS-3", lastSM="SM-8") -2. Call Plugin: GetProducts(cursor="C1") → [AAS-3], nextPluginCursor=null -3. Skip products before AAS-3: none. -4. Expand AAS-3 → [SM-7, SM-8, SM-9] -5. Skip SM-7, SM-8. -6. Collect SM-9. Count=1. -7. No more products, no more plugin pages. -8. End of dataset → cursor = null. -``` +### Example 1: `limit=2` Across 5 Consecutive Pages (The Tricky Case) -```json -{ - "result": [ - { "id": "SM-9" } - ], - "paging_metadata": { "cursor": null } -} -``` - ---- +Dataset: +- `AAS-1` → `[SM-1, SM-2, SM-3]` +- `AAS-2` → `[SM-4, SM-5, SM-6]` +- `AAS-3` → `[SM-7, SM-8, SM-9]` -### 6.2 Example: `limit=4` - -**Page 1 - `GET /submodels?limit=4`** - -``` -1. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2, AAS-3], nextPluginCursor=null -2. Expand AAS-1 → [SM-1, SM-2, SM-3]. Collect all. Count=3. -3. Expand AAS-2 → [SM-4, SM-5, SM-6]. Collect SM-4. Count=4. Limit reached. -4. Cursor = Encode(pluginPageCursor=null, AasId="AAS-2", LastSM="SM-4") -``` +Plugin batch size: 2 products per request. +#### Page 1: `GET /submodels?limit=2` +- **Request:** `limit=2`, `cursor=null` +- **API Execution:** + 1. Tracking state initialized to null. + 2. Call `Plugin.GetShells(cursor=null)` → returns `[AAS-1, AAS-2]`. + 3. First AAS is `AAS-1`. `SubmodelId` is null (no cursor) → collect from the start. + 4. Collect `SM-1` (Count=1), `SM-2` (Count=2) → **Limit reached!** `AAS-1` is partial. + 5. Tracking state unchanged (null). No AAS was fully consumed. + 6. Encode cursor: `SubmodelId="SM-2"`, `AasId=null`. +- **Response:** ```json -{ - "result": [ - { "id": "SM-1" }, - { "id": "SM-2" }, - { "id": "SM-3" }, - { "id": "SM-4" } - ], - "paging_metadata": { "cursor": "fEFBUy0yfFNNLTQ=" } -} -``` - -**Page 2 - `GET /submodels?limit=4&cursor=fEFBUy0yfFNNLTQ=`** - -``` -1. Decode → (pluginPageCursor=null, currentAasId="AAS-2", lastSM="SM-4") -2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2, AAS-3], nextPluginCursor=null -3. Skip AAS-1 (before AAS-2). -4. Expand AAS-2 → [SM-4, SM-5, SM-6]. Skip SM-4. Collect SM-5, SM-6. Count=2. -5. Expand AAS-3 → [SM-7, SM-8, SM-9]. Collect SM-7, SM-8. Count=4. Limit reached. -6. Cursor = Encode(pluginPageCursor=null, AasId="AAS-3", LastSM="SM-8") -``` - +{ "paging_metadata": { "cursor": "U00tMnw" }, "result": [{ "id": "SM-1" }, { "id": "SM-2" }] } +``` + +#### Page 2: `GET /submodels?limit=2&cursor=U00tMnw` +- **Request:** `limit=2`, cursor decodes to `SubmodelId="SM-2"`, `AasId=null` +- **API Execution:** + 1. Tracking state initialized to null (from `AasId=null`). + 2. Call `Plugin.GetShells(cursor=null)` → returns `[AAS-1, AAS-2]`. + 3. First AAS is `AAS-1`. Scan for `SM-2` → skip `SM-1`, `SM-2`. Collect `SM-3` (Count=1). `AAS-1` complete. + 4. Tracking state advances to `AAS-1`. + 5. Advance to `AAS-2` in the same batch. Collect `SM-4` (Count=2) → **Limit reached!** `AAS-2` is partial. + 6. Tracking state = `AAS-1`. Encode cursor: `SubmodelId="SM-4"`, `AasId="AAS-1"`. +- **Response:** ```json -{ - "result": [ - { "id": "SM-5" }, - { "id": "SM-6" }, - { "id": "SM-7" }, - { "id": "SM-8" } - ], - "paging_metadata": { "cursor": "fEFBUy0zfFNNLTg=" } -} -``` - -**Page 3 (Final) - `GET /submodels?limit=4&cursor=fEFBUy0zfFNNLTg=`** - -``` -1. Decode → (pluginPageCursor=null, currentAasId="AAS-3", lastSM="SM-8") -2. Call Plugin: GetProducts(cursor=null) → [AAS-1, AAS-2, AAS-3] -3. Skip AAS-1, AAS-2. Find AAS-3. -4. Expand AAS-3 → [SM-7, SM-8, SM-9]. Skip SM-7, SM-8. Collect SM-9. Count=1. -5. No more products. Plugin exhausted. -6. cursor = null. -``` - +{ "paging_metadata": { "cursor": "U00tNHxBQVMtMQ" }, "result": [{ "id": "SM-3" }, { "id": "SM-4" }] } +``` + +#### Page 3: `GET /submodels?limit=2&cursor=U00tNHxBQVMtMQ` +- **Request:** `limit=2`, cursor decodes to `SubmodelId="SM-4"`, `AasId="AAS-1"` +- **API Execution:** + 1. Tracking state initialized to `AAS-1` (from `AasId="AAS-1"`). + 2. Call `Plugin.GetShells(cursor="AAS-1")` → returns `[AAS-2, AAS-3]` (all AAS strictly after `AAS-1`). + 3. First AAS is `AAS-2`. Scan for `SM-4` → skip `SM-4`. Collect `SM-5` (Count=1), `SM-6` (Count=2) → **Limit reached!** `AAS-2` is fully consumed at the limit boundary. + 4. Tracking state advances to `AAS-2`. Since `AAS-2` is fully consumed at the boundary, `SubmodelId=null`. + 5. Encode cursor: `SubmodelId=null`, `AasId="AAS-2"`. +- **Response:** ```json -{ - "result": [ - { "id": "SM-9" } - ], - "paging_metadata": { "cursor": null } -} +{ "paging_metadata": { "cursor": "fEFBUy0y" }, "result": [{ "id": "SM-5" }, { "id": "SM-6" }] } +``` + +#### Page 4: `GET /submodels?limit=2&cursor=fEFBUy0y` +- **Request:** `limit=2`, cursor decodes to `SubmodelId=null`, `AasId="AAS-2"` +- **API Execution:** + 1. Tracking state initialized to `AAS-2` (from `AasId="AAS-2"`). + 2. Call `Plugin.GetShells(cursor="AAS-2")` → returns `[AAS-3]` (all AAS strictly after `AAS-2`). + 3. First AAS is `AAS-3`. `SubmodelId` is null → collect from the start. + 4. Collect `SM-7` (Count=1), `SM-8` (Count=2) → **Limit reached!** `AAS-3` is partial. + 5. Tracking state unchanged (`AAS-2`). Encode cursor: `SubmodelId="SM-8"`, `AasId="AAS-2"`. +- **Response:** +```json +{ "paging_metadata": { "cursor": "U00tOHxBQVMtMg" }, "result": [{ "id": "SM-7" }, { "id": "SM-8" }] } +``` + +#### Page 5 (Final): `GET /submodels?limit=2&cursor=U00tOHxBQVMtMg` +- **Request:** `limit=2`, cursor decodes to `SubmodelId="SM-8"`, `AasId="AAS-2"` +- **API Execution:** + 1. Tracking state initialized to `AAS-2`. + 2. Call `Plugin.GetShells(cursor="AAS-2")` → returns `[AAS-3]`. + 3. First AAS is `AAS-3`. Scan for `SM-8` → skip `SM-7`, `SM-8`. Collect `SM-9` (Count=1). + 4. `AAS-3` exhausted. No more batches (`nextCursor=null`). Dataset exhausted. + 5. Return with `cursor=null`. +- **Response:** +```json +{ "paging_metadata": { "cursor": null }, "result": [{ "id": "SM-9" }] } ``` --- +### Example 2: `limit=4` Across 3 Pages -## 9. Sequence Diagram (limit=2) +#### Page 1: `GET /submodels?limit=4` +- **Execution:** Tracking state=null. Plugin(null) → [AAS-1, AAS-2]. Collect all 3 of AAS-1 (tracking advances to AAS-1), then collect SM-4 from AAS-2 (limit reached, AAS-2 partial). +- **Response Cursor:** `SubmodelId="SM-4"`, `AasId="AAS-1"` (encoded: `U00tNHxBQVMtMQ`) +- **Result:** `[SM-1, SM-2, SM-3, SM-4]` -```mermaid -sequenceDiagram - autonumber - actor Client - participant API as GET /submodels - participant Engine as Pagination Engine - participant Plugin as Plugin API - - Client->>API: GET /submodels?limit=2 - API->>Engine: Paginate(limit=2, cursor=null) - - Engine->>Plugin: GetProducts(cursor=null) - Plugin-->>Engine: [AAS-1, AAS-2], nextCursor="C1" - - Note over Engine: Expand AAS-1 → [SM-1, SM-2, SM-3] - Engine->>Plugin: GetSubmodelData(SM-1) - Engine->>Plugin: GetSubmodelData(SM-2) - Note over Engine: Collected: 2. Limit reached. - Note over Engine: Cursor = Encode(null, AAS-1, SM-2) - - Engine-->>API: [SM-1, SM-2], cursor=encoded - API-->>Client: 200 OK +#### Page 2: `GET /submodels?limit=4&cursor=U00tNHxBQVMtMQ` +- **Execution:** Tracking state=AAS-1. Plugin(AAS-1) → [AAS-2, AAS-3]. Scan AAS-2 for SM-4, skip it, collect SM-5, SM-6 (AAS-2 done, tracking → AAS-2). Advance to AAS-3, collect SM-7, SM-8 (Count=4, limit reached, AAS-3 partial). +- **Response Cursor:** `SubmodelId="SM-8"`, `AasId="AAS-2"` (encoded: `U00tOHxBQVMtMg`) +- **Result:** `[SM-5, SM-6, SM-7, SM-8]` - Client->>API: GET /submodels?limit=2&cursor=... - API->>Engine: Paginate(limit=2, cursor=...) +#### Page 3: `GET /submodels?limit=4&cursor=U00tOHxBQVMtMg` +- **Execution:** Tracking state=AAS-2. Plugin(AAS-2) → [AAS-3]. Scan AAS-3 for SM-8, skip SM-7, SM-8, collect SM-9 (Count=1). Dataset exhausted. +- **Response Cursor:** `null` +- **Result:** `[SM-9]` - Note over Engine: Decode → (pluginPageCursor=null, AAS-1, SM-2) - Engine->>Plugin: GetProducts(cursor=null) - Note over Engine: Re-fetches SAME batch - Plugin-->>Engine: [AAS-1, AAS-2], nextCursor="C1" - - Note over Engine: Find AAS-1, expand → [SM-1, SM-2, SM-3] - Note over Engine: Skip SM-1, SM-2 (anchor found) - Engine->>Plugin: GetSubmodelData(SM-3) - Note over Engine: Collected: 1. Need 1 more. - - Note over Engine: Move to AAS-2 (same batch!) - Note over Engine: Expand AAS-2 → [SM-4, SM-5, SM-6] - Engine->>Plugin: GetSubmodelData(SM-4) - Note over Engine: Collected: 2. Limit reached. - Note over Engine: Cursor = Encode(null, AAS-2, SM-4) - - Engine-->>API: [SM-3, SM-4], cursor=encoded - API-->>Client: 200 OK -``` - ---- \ No newline at end of file +--- From cd95786c023086f643d20ef48492c2678ff36d16 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Fri, 31 Jul 2026 17:49:42 +0530 Subject: [PATCH 13/30] Revert "Refactor SubmodelRepositoryService and related components for pagination support" This reverts commit 77a54fb471d90f3a63867c9c92a4582825889d5a. --- .../AasRepositoryServiceTests.cs | 6 +- .../Discovery/AssetIdSearchServiceTests.cs | 18 +- .../SubmodelPageCursorTests.cs | 48 ------ .../SubmodelRepositoryServiceTests.cs | 148 +--------------- .../Services/PluginDataHandlerTests.cs | 24 +-- .../Services/PluginDataProviderTests.cs | 16 +- .../AasRepository/AasRepositoryService.cs | 2 +- .../Discovery/AssetIdSearchService.cs | 4 +- .../Services/Plugin/IPluginDataHandler.cs | 4 +- .../Plugin/Providers/IPluginDataProvider.cs | 4 +- .../SubmodelRepository/SubmodelPageCursor.cs | 47 ------ .../SubmodelRepositoryService.cs | 158 +++++------------- .../Services/PluginDataHandler.cs | 4 +- .../Services/PluginDataProvider.cs | 4 +- 14 files changed, 86 insertions(+), 401 deletions(-) delete mode 100644 source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursorTests.cs delete mode 100644 source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursor.cs diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs index c3f91968..87bff847 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs @@ -328,8 +328,6 @@ public async Task GetShellsByFiltersAsync_WithIdShort_QueriesFilteredShellMetada .GetDataForShellsByAssetIdsAsync( manifests, Arg.Is(f => f != null && f.IdShort == targetIdShort), - Arg.Any(), - Arg.Any(), cancellationToken) .Returns(new ShellDescriptorsMetaData { @@ -349,7 +347,7 @@ public async Task GetShellsByFiltersAsync_WithIdShort_QueriesFilteredShellMetada Assert.Single(result.Result); Assert.Equal("aas-1", result.Result[0].Id); await _pluginDataHandler.Received(1) - .GetDataForShellsByAssetIdsAsync(manifests, Arg.Is(f => f != null && f.IdShort == targetIdShort), Arg.Any(), Arg.Any(), cancellationToken); + .GetDataForShellsByAssetIdsAsync(manifests, Arg.Is(f => f != null && f.IdShort == targetIdShort), cancellationToken); } [Fact] @@ -363,7 +361,7 @@ public async Task GetShellsByFiltersAsync_ShouldBuildShellsInParallelAndSkipFail var metadataItems = new List { new() { Id = "aas-1", SpecificAssetIds = [] }, - new() { Id = "aas-2", SpecificAssetIds = [] }, + new() { Id = "aas-2", SpecificAssetIds = [] }, new() { Id = "aas-3", SpecificAssetIds = [] } }; diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs index 8d34fc7d..d3634fd1 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Base; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasRepository; @@ -51,8 +51,6 @@ public async Task SearchShellsByAssetLinkAsync_ReturnsAasIds() _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), - Arg.Any(), - Arg.Any(), Arg.Any()) .Returns(metadata); @@ -85,8 +83,6 @@ public async Task SearchShellsByAssetLinkAsync_WithPagination_ReturnsPagedResult _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), - Arg.Any(), - Arg.Any(), Arg.Any()) .Returns(metadata); @@ -106,8 +102,6 @@ public async Task SearchShellsByAssetLinkAsync_WhenPluginTimeout_ThrowsPluginNot _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), - Arg.Any(), - Arg.Any(), Arg.Any()) .Throws(new RequestTimeoutException()); @@ -125,8 +119,6 @@ public async Task SearchShellsByAssetLinkAsync_WhenUnauthorized_ThrowsServiceUnA _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), - Arg.Any(), - Arg.Any(), Arg.Any()) .Throws(new AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException()); @@ -144,8 +136,6 @@ public async Task SearchShellsByAssetLinkAsync_WhenResponseParsingError_ThrowsIn _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), - Arg.Any(), - Arg.Any(), Arg.Any()) .Throws(new ResponseParsingException()); @@ -164,8 +154,6 @@ public async Task SearchShellsByAssetLinkAsync_WhenMultiPluginConflict_ThrowsInt _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), - Arg.Any(), - Arg.Any(), Arg.Any()) .Throws(new MultiPluginConflictException()); @@ -183,8 +171,6 @@ public async Task SearchShellsByAssetLinkAsync_WhenResourceNotFound_ThrowsIntern _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), - Arg.Any(), - Arg.Any(), Arg.Any()) .Throws(new ResourceNotFoundException()); @@ -214,8 +200,6 @@ public async Task SearchShellsByAssetLinkAsync_FiltersOutEmptyIds() _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), - Arg.Any(), - Arg.Any(), Arg.Any()) .Returns(metadata); diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursorTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursorTests.cs deleted file mode 100644 index 7612fbbf..00000000 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursorTests.cs +++ /dev/null @@ -1,48 +0,0 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; -using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.SubmodelRepository; - -namespace AAS.TwinEngine.DataEngine.UnitTests.ApplicationLogic.Services.SubmodelRepository; - -public class SubmodelPageCursorTests -{ - [Fact] - public void EncodeThenTryDecode_RoundTripsAllFields() - { - var cursor = new SubmodelPageCursor("PLUGIN-PAGE-2", "AAS-1", "SM-2"); - - var decoded = SubmodelPageCursor.TryDecode(cursor.Encode()); - - Assert.NotNull(decoded); - Assert.Equal("PLUGIN-PAGE-2", decoded!.PluginPageCursor); - Assert.Equal("AAS-1", decoded.CurrentAasId); - Assert.Equal("SM-2", decoded.LastSubmodelId); - } - - [Fact] - public void EncodeThenTryDecode_WithNullPluginPageCursor_RoundTripsAsNull() - { - var cursor = new SubmodelPageCursor(null, "AAS-1", "SM-2"); - - var decoded = SubmodelPageCursor.TryDecode(cursor.Encode()); - - Assert.NotNull(decoded); - Assert.Null(decoded!.PluginPageCursor); - Assert.Equal("AAS-1", decoded.CurrentAasId); - Assert.Equal("SM-2", decoded.LastSubmodelId); - } - - [Theory] - [InlineData(null)] - [InlineData("")] - [InlineData(" ")] - public void TryDecode_WithEmptyOrWhitespace_ReturnsNull(string? encodedCursor) => Assert.Null(SubmodelPageCursor.TryDecode(encodedCursor)); - - [Fact] - public void TryDecode_WithWrongFieldCount_ReturnsNull() - { - // Only two fields instead of the expected three. - var malformed = "AAS-1|SM-2".EncodeBase64Url(); - - Assert.Null(SubmodelPageCursor.TryDecode(malformed)); - } -} diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs index 70805336..7b840855 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasRepository; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin; @@ -245,33 +245,11 @@ public static SemanticBranchNode CreateSubmodelTreeNode(string value) return submodel; } - private static Reference SubmodelReference(string submodelId) => - new(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, submodelId)]); - - // Arranges the template + value pipeline so every collected submodel id yields one built submodel. - private void ArrangeSubmodelBuildPipeline() - { - _templateService - .GetFilteredSubmodelTemplateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(TestData.CreateSubmodel()); - - ArrangeValuePipeline(); - } - - private void ArrangeValuePipeline() - { - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(TestData.CreateFilledSubmodel()); - } - [Fact] public async Task GetAllSubmodelsAsync_ReturnsEmpty_WhenNoShellsFound() { _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [] }); var result = await _sut.GetAllSubmodelsAsync(null, null, null, null, CancellationToken.None); @@ -289,7 +267,7 @@ public async Task GetAllSubmodelsAsync_BuildsSubmodelForEachSubmodelId() var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _aasRepositoryTemplateService @@ -319,7 +297,7 @@ public async Task GetAllSubmodelsAsync_SkipsSubmodel_WhenFilteredTemplateReturns var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _aasRepositoryTemplateService @@ -341,7 +319,7 @@ public async Task GetAllSubmodelsAsync_SkipsShell_WhenSubmodelRefsFails() const string ShellId = "https://example.com/shells/001"; _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _aasRepositoryTemplateService @@ -361,7 +339,7 @@ public async Task GetAllSubmodelsAsync_DeduplicatesSubmodelIds_AcrossShells() var filledSubmodel = TestData.CreateFilledSubmodel(); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = @@ -400,8 +378,6 @@ public async Task GetAllSubmodelsAsync_FiltersShellsByIdShort_WhenIdShortProvide .GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Is(f => f != null && f.IdShort == IdShort), - Arg.Any(), - Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [] }); @@ -411,8 +387,6 @@ await _pluginDataHandler.Received(1) .GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Is(f => f != null && f.IdShort == IdShort), - Arg.Any(), - Arg.Any(), Arg.Any()); } @@ -428,7 +402,7 @@ public async Task GetAllSubmodelsAsync_WhenSemanticIdFilterProvided_LooksUpFilte var filledSubmodel = TestData.CreateFilledSubmodel(); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _templateService @@ -463,7 +437,7 @@ public async Task GetAllSubmodelsAsync_WhenSemanticIdNotFound_ThrowsSubmodelNotF var filter = new SubmodelSearchFilter { SemanticId = SemanticId }; _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = "https://example.com/shells/001" }] }); // GetFilteredSubmodelTemplateIdAsync returns null — semantic ID not found in any template @@ -484,7 +458,7 @@ public async Task GetAllSubmodelsAsync_SkipsShellsWithNullOrEmptyId() var filledSubmodel = TestData.CreateFilledSubmodel(); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = @@ -517,110 +491,6 @@ public async Task GetAllSubmodelsAsync_SkipsShellsWithNullOrEmptyId() await _aasRepositoryTemplateService.DidNotReceive().GetSubmodelRefByIdAsync(null!, Arg.Any()); } - [Fact] - public async Task GetAllSubmodelsAsync_WhenLimitReachedMidProduct_ReturnsResumeCursor() - { - const string AasId = "https://example.com/shells/001"; - const string SubmodelId1 = "https://example.com/submodels/SM-1"; - const string SubmodelId2 = "https://example.com/submodels/SM-2"; - const string SubmodelId3 = "https://example.com/submodels/SM-3"; - - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = AasId }] }); - - _aasRepositoryTemplateService - .GetSubmodelRefByIdAsync(AasId, Arg.Any()) - .Returns([SubmodelReference(SubmodelId1), SubmodelReference(SubmodelId2), SubmodelReference(SubmodelId3)]); - - ArrangeSubmodelBuildPipeline(); - - var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 2, cursor: null, CancellationToken.None); - - Assert.Equal(2, result.Result.Count); - - var nextCursor = SubmodelPageCursor.TryDecode(result.PagingMetaData?.Cursor); - Assert.NotNull(nextCursor); - Assert.Null(nextCursor!.PluginPageCursor); - Assert.Equal(AasId, nextCursor.CurrentAasId); - Assert.Equal(SubmodelId2, nextCursor.LastSubmodelId); - } - - [Fact] - public async Task GetAllSubmodelsAsync_WithResumeCursor_SkipsAlreadyDeliveredSubmodels() - { - const string AasId = "https://example.com/shells/001"; - const string SubmodelId1 = "https://example.com/submodels/SM-1"; - const string SubmodelId2 = "https://example.com/submodels/SM-2"; - const string SubmodelId3 = "https://example.com/submodels/SM-3"; - - var resumeCursor = new SubmodelPageCursor(null, AasId, SubmodelId2).Encode(); - - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = AasId }] }); - - _aasRepositoryTemplateService - .GetSubmodelRefByIdAsync(AasId, Arg.Any()) - .Returns([SubmodelReference(SubmodelId1), SubmodelReference(SubmodelId2), SubmodelReference(SubmodelId3)]); - - ArrangeSubmodelBuildPipeline(); - - var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 100, cursor: resumeCursor, CancellationToken.None); - - // Only the submodel after the resume point should remain, and the data is exhausted. - Assert.Single(result.Result); - Assert.Null(result.PagingMetaData?.Cursor); - await _templateService.Received(1).GetFilteredSubmodelTemplateAsync(SubmodelId3, (string?)null, Arg.Any(), Arg.Any()); - await _templateService.DidNotReceive().GetFilteredSubmodelTemplateAsync(SubmodelId1, (string?)null, Arg.Any(), Arg.Any()); - await _templateService.DidNotReceive().GetFilteredSubmodelTemplateAsync(SubmodelId2, (string?)null, Arg.Any(), Arg.Any()); - } - - [Fact] - public async Task GetAllSubmodelsAsync_WhenLimitSpansPluginPages_AdvancesPluginCursor() - { - const string AasId1 = "https://example.com/shells/001"; - const string AasId2 = "https://example.com/shells/002"; - const string NextPluginCursor = "PLUGIN-PAGE-2"; - const string SubmodelId1 = "https://example.com/submodels/SM-1"; - const string SubmodelId2 = "https://example.com/submodels/SM-2"; - const string SubmodelId3 = "https://example.com/submodels/SM-3"; - const string SubmodelId4 = "https://example.com/submodels/SM-4"; - - // First plugin page (null cursor) points to the next page. - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Is(c => c == null), Arg.Any()) - .Returns(new ShellDescriptorsMetaData - { - ShellDescriptors = [new ShellDescriptorMetaData { Id = AasId1 }], - PagingMetaData = new PagingMetaData { Cursor = NextPluginCursor } - }); - - // Second plugin page (advanced cursor) ends the data. - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Is(c => c == NextPluginCursor), Arg.Any()) - .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = AasId2 }] }); - - _aasRepositoryTemplateService - .GetSubmodelRefByIdAsync(AasId1, Arg.Any()) - .Returns([SubmodelReference(SubmodelId1), SubmodelReference(SubmodelId2)]); - _aasRepositoryTemplateService - .GetSubmodelRefByIdAsync(AasId2, Arg.Any()) - .Returns([SubmodelReference(SubmodelId3), SubmodelReference(SubmodelId4)]); - - ArrangeSubmodelBuildPipeline(); - - var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 3, cursor: null, CancellationToken.None); - - Assert.Equal(3, result.Result.Count); - - var nextCursor = SubmodelPageCursor.TryDecode(result.PagingMetaData?.Cursor); - Assert.NotNull(nextCursor); - Assert.Equal(NextPluginCursor, nextCursor!.PluginPageCursor); - Assert.Equal(AasId2, nextCursor.CurrentAasId); - Assert.Equal(SubmodelId3, nextCursor.LastSubmodelId); - } - [Fact] public async Task GetAllSubmodelElementsAsync_ReturnsAllElements_WhenSubmodelExists() { diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs index 940339ce..9c650eff 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs @@ -630,14 +630,14 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_ReturnsShellDescript }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new List { httpResponse.Content }); const string Json = """[{"name":"sn","value":"123"}]"""; var assetIds = JsonSerializer.Deserialize>(Json)!; var filter = new ShellSearchFilter { SpecificAssetIds = assetIds }; - var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); + var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None); Assert.NotNull(result); Assert.Single(result.ShellDescriptors); @@ -673,12 +673,12 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenNullDeserializat }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None)); + _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None)); } [Fact] @@ -710,12 +710,12 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenInvalidJson_Thro }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None)); + _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None)); } [Fact] @@ -757,11 +757,11 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_SetsHrefOnResults() }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; - var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); + var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None); Assert.Equal(2, result.ShellDescriptors.Count); Assert.All(result.ShellDescriptors, dto => Assert.StartsWith("https://www.mm-software.com/shells/", dto.Href)); @@ -786,7 +786,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenNoAvailablePlugi var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None)); + _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None)); } [Fact] @@ -829,17 +829,17 @@ public async Task GetDataForShellsByAssetIdsAsync_WithIdShort_PassesIdShort() const string targetIdShort = "Motor001"; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Is(v => v == null), Arg.Is(v => v == targetIdShort), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), null, targetIdShort, Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { IdShort = targetIdShort }; - var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); + var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None); Assert.NotNull(result); Assert.Single(result.ShellDescriptors); Assert.Equal("urn:aas:001", result.ShellDescriptors[0].Id); await _pluginDataProvider.Received(1) - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Is(v => v == null), Arg.Is(v => v == targetIdShort), Arg.Any(), Arg.Any(), Arg.Any()); + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), null, targetIdShort, Arg.Any()); } private const string AssetData = """ diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs index 89ee6b53..0af87a02 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Net.Http.Json; using System.Text; using System.Text.Json; @@ -391,7 +391,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_ReturnsContent() var assetIdsHeaderValue = """[{"name":"SerialNumber","value":"SN-4711"}]"""; - var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, assetIdsHeaderValue, null, null, null, CancellationToken.None); + var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, assetIdsHeaderValue, null, CancellationToken.None); Assert.NotNull(result); Assert.Single(result); @@ -431,7 +431,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WithIdShort_AddsIdSh const string idShort = "test-idshort-value"; - var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, null, idShort, null, null, CancellationToken.None); + var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, null, idShort, CancellationToken.None); Assert.NotNull(result); Assert.Single(result); @@ -461,7 +461,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenNotFound_ThrowsR var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); } [Fact] @@ -481,7 +481,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenUnauthorized_Thr var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); } [Fact] @@ -495,7 +495,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenTimeout_ThrowsRe var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); } [Fact] @@ -504,7 +504,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WithNullRequest_Skip var metadata = new List { null! }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); } [Fact] @@ -524,7 +524,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenForbidden_Throws var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); } private static List GetTestShellDescriptorDataList() diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs index fbe944c0..129b5a34 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs @@ -230,7 +230,7 @@ private void FillShellFromMetadata(IAssetAdministrationShell shell, ShellDescrip CancellationToken cancellationToken) { var metadata = await pluginDataHandler - .GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, filter, null, null, cancellationToken) + .GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, filter, cancellationToken) .ConfigureAwait(false); var allMetadata = metadata.ShellDescriptors? diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs index df111ab3..e6c17cfe 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Base; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; @@ -73,7 +73,7 @@ private async Task GetFilteredMetadataAsync(List GetDataForAssetInformationByIdAsync(IReadOnlyList pluginManifests, string id, CancellationToken cancellationToken); - Task GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, int? limit, string? cursor, CancellationToken cancellationToken); + Task GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, CancellationToken cancellationToken); } diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs index cc193f52..53b42af7 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.DomainModel.Plugin; +using AAS.TwinEngine.DataEngine.DomainModel.Plugin; namespace AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin.Providers; @@ -12,5 +12,5 @@ public interface IPluginDataProvider Task> GetDataForAssetInformationByIdAsync(IList pluginRequests, CancellationToken cancellationToken); - Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, int? limit, string? cursor, CancellationToken cancellationToken); + Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, CancellationToken cancellationToken); } diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursor.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursor.cs deleted file mode 100644 index dd6d206b..00000000 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelPageCursor.cs +++ /dev/null @@ -1,47 +0,0 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; - -namespace AAS.TwinEngine.DataEngine.ApplicationLogic.Services.SubmodelRepository; - -///

-/// Opaque cursor for the "Get All Submodels" flow. A single product (AAS) can hold multiple -/// submodels and may be only partially delivered within one page, so resuming requires more -/// state than a single submodel id. -/// -/// -/// The plugin cursor that fetched the batch the client is currently positioned in -/// (null for the first plugin page). On resume the same batch is re-fetched so products -/// that were returned alongside a partially consumed product are not lost. -/// -/// The AAS id of the partially consumed product to resume within. -/// The last submodel id already delivered inside . -public sealed record SubmodelPageCursor(string? PluginPageCursor, string CurrentAasId, string LastSubmodelId) -{ - // Logical layout: {PluginPageCursor}|{CurrentAasId}|{LastSubmodelId} - private const char Separator = '|'; - private const int FieldCount = 3; - - /// Encodes the cursor into the opaque Base64Url token returned to the client. - public string Encode() => - string.Join(Separator, PluginPageCursor, CurrentAasId, LastSubmodelId).EncodeBase64Url(); - - /// - /// Decodes a client supplied cursor. Returns null when the cursor is absent or does not - /// have the expected shape, which callers treat as "start from the first page". - /// - public static SubmodelPageCursor? TryDecode(string? encodedCursor) - { - if (string.IsNullOrWhiteSpace(encodedCursor)) - { - return null; - } - - var parts = encodedCursor.DecodeBase64Url().Split(Separator); - if (parts.Length != FieldCount) - { - return null; - } - - var pluginPageCursor = string.IsNullOrEmpty(parts[0]) ? null : parts[0]; - return new SubmodelPageCursor(pluginPageCursor, parts[1], parts[2]); - } -} diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 45d323bf..50f443f2 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -5,7 +5,6 @@ using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin; using AAS.TwinEngine.DataEngine.DomainModel.AasRegistry; using AAS.TwinEngine.DataEngine.DomainModel.AasRepository; -using AAS.TwinEngine.DataEngine.DomainModel.Shared; using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; @@ -27,9 +26,6 @@ public class SubmodelRepositoryService( IOptions templateManagementConfig) : ISubmodelRepositoryService { private readonly int _concurrentOperationsLimit = templateManagementConfig.Value.SubmodelTemplateRepository.ConcurrentOperationsLimit; - - // Default number of submodels per page when the client does not supply a limit. - private const int DefaultPageSize = 100; public async Task GetSubmodelAsync(string submodelId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) { return await ExecuteWithExceptionHandlingAsync(async () => @@ -66,136 +62,68 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte { return await ExecuteWithExceptionHandlingAsync(async () => { - var filteredTemplateId = await ResolveFilteredTemplateIdAsync(filter?.SemanticId, cancellationToken).ConfigureAwait(false); + var shellSearchFilter = new ShellSearchFilter + { + IdShort = filter?.IdShort + }; + + var shellMetadata = await pluginDataHandler.GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, shellSearchFilter, cancellationToken).ConfigureAwait(false); + var shellDescriptors = shellMetadata.ShellDescriptors ?? []; + + string? filteredTemplateId = null; + if (filter?.SemanticId is not null) + { + filteredTemplateId = await submodelTemplateService.GetFilteredSubmodelTemplateIdAsync(filter.SemanticId, cancellationToken).ConfigureAwait(false); + + if (filteredTemplateId is null) + { + throw new SubmodelNotFoundException(); + } + } + + var distinctSubmodelIds = await GetDistinctSubmodelIdsAsync(shellDescriptors, cancellationToken).ConfigureAwait(false); - var (submodelIds, nextCursor) = await CollectSubmodelIdPageAsync(filter?.IdShort, limit, cursor, cancellationToken).ConfigureAwait(false); + var (pagedIds, pagingMetaData) = PagingExtensions.GetPagedResult(distinctSubmodelIds, id => id, limit, cursor); - var submodels = await BuildSubmodelsAsync(submodelIds, filteredTemplateId, queryOptions, cancellationToken).ConfigureAwait(false); + var submodels = await BuildSubmodelsAsync(pagedIds, filteredTemplateId, queryOptions, cancellationToken).ConfigureAwait(false); return new SubmodelList { - PagingMetaData = new PagingMetaData { Cursor = nextCursor }, + PagingMetaData = pagingMetaData, Result = submodels }; }).ConfigureAwait(false); } - private async Task ResolveFilteredTemplateIdAsync(string? semanticId, CancellationToken cancellationToken) + private async Task> GetDistinctSubmodelIdsAsync(List shellDescriptors, CancellationToken cancellationToken) { - if (semanticId is null) - { - return null; - } - - return await submodelTemplateService.GetFilteredSubmodelTemplateIdAsync(semanticId, cancellationToken).ConfigureAwait(false) - ?? throw new SubmodelNotFoundException(); - } - - /// - /// Builds one page of submodel ids using the two-phase resume strategy: a partially consumed - /// product is finished first (Phase 1), then forward pagination continues through the next - /// products and plugin pages (Phase 2) until the page size is reached or the data is exhausted. - /// - private async Task<(IReadOnlyList SubmodelIds, string? NextCursor)> CollectSubmodelIdPageAsync( - string? idShort, int? limit, string? encodedCursor, CancellationToken cancellationToken) - { - var pageSize = limit ?? DefaultPageSize; - var resumePoint = SubmodelPageCursor.TryDecode(encodedCursor); - - var collected = new List(); - var seen = new HashSet(StringComparer.Ordinal); - - // Plugin cursor that produced the batch currently being scanned (null = first plugin page). - var pluginPageCursor = resumePoint?.PluginPageCursor; - // Resume coordinates; only meaningful for the first scanned batch. - var resumeAasId = resumePoint?.CurrentAasId; - var resumeAfterSubmodelId = resumePoint?.LastSubmodelId; - - while (true) - { - var batch = await FetchShellBatchAsync(idShort, pageSize, pluginPageCursor, cancellationToken).ConfigureAwait(false); - - foreach (var aasId in SkipProductsBefore(batch.ShellIds, resumeAasId)) + using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); + var tasks = shellDescriptors.Where(shell => !string.IsNullOrWhiteSpace(shell.Id)).Select(async shell => { - var submodelIds = await ExpandProductSubmodelIdsAsync(aasId, cancellationToken).ConfigureAwait(false); - - // On resume, drop the submodels that were already delivered from this product. - if (resumeAfterSubmodelId is not null) + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try { - submodelIds = SkipThrough(submodelIds, resumeAfterSubmodelId); - resumeAfterSubmodelId = null; + return await aasRepositoryTemplateService.GetSubmodelRefByIdAsync(shell.Id!, cancellationToken).ConfigureAwait(false); } - - foreach (var submodelId in submodelIds) + catch (ResourceNotFoundException ex) { - if (!seen.Add(submodelId)) - { - continue; - } - - collected.Add(submodelId); - if (collected.Count == pageSize) - { - var nextCursor = new SubmodelPageCursor(pluginPageCursor, aasId, submodelId).Encode(); - return (collected, nextCursor); - } + logger.LogWarning(ex, "Could not retrieve submodel refs for shell {ShellId}. Skipping shell.", shell.Id); + return []; } - } - - // The whole batch is consumed; resume state applies to the first batch only. - resumeAasId = null; - - if (string.IsNullOrEmpty(batch.NextPluginCursor)) - { - return (collected, null); - } - - pluginPageCursor = batch.NextPluginCursor; - } - } - - private async Task<(List ShellIds, string? NextPluginCursor)> FetchShellBatchAsync( - string? idShort, int pageSize, string? pluginPageCursor, CancellationToken cancellationToken) - { - var shellSearchFilter = new ShellSearchFilter { IdShort = idShort }; + finally + { + _ = semaphore.Release(); + } + }); - var metadata = await pluginDataHandler - .GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, shellSearchFilter, pageSize, pluginPageCursor, cancellationToken) - .ConfigureAwait(false); + var references = await Task.WhenAll(tasks).ConfigureAwait(false); - var shellIds = (metadata.ShellDescriptors ?? []) - .Select(shell => shell.Id) + return references + .SelectMany(x => x) + .Select(reference => reference.Keys.FirstOrDefault()?.Value) .Where(id => !string.IsNullOrWhiteSpace(id)) - .ToList(); - - return (shellIds!, metadata.PagingMetaData?.Cursor); - } - - private static IEnumerable SkipProductsBefore(IEnumerable shellIds, string? resumeAasId) => - resumeAasId is null ? shellIds : shellIds.SkipWhile(id => id != resumeAasId); - - private static List SkipThrough(List submodelIds, string lastDeliveredSubmodelId) - { - var index = submodelIds.IndexOf(lastDeliveredSubmodelId); - return index < 0 ? submodelIds : [.. submodelIds.Skip(index + 1)]; - } - - private async Task> ExpandProductSubmodelIdsAsync(string aasId, CancellationToken cancellationToken) - { - try - { - var references = await aasRepositoryTemplateService.GetSubmodelRefByIdAsync(aasId, cancellationToken).ConfigureAwait(false); - - return references - .Select(reference => reference.Keys.FirstOrDefault()?.Value) - .Where(id => !string.IsNullOrWhiteSpace(id)) - .ToList()!; - } - catch (ResourceNotFoundException ex) - { - logger.LogWarning(ex, "Could not retrieve submodel refs for shell {ShellId}. Skipping shell.", aasId); - return []; - } + .Distinct() + .ToList()!; } private async Task> BuildSubmodelsAsync(IEnumerable submodelIds, string? filteredTemplateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs index 0b45ba05..f8aa9ca8 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs @@ -207,7 +207,7 @@ public async Task GetDataForAssetInformationByIdAsync(IReadOnlyList

GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, int? limit, string? cursor, CancellationToken cancellationToken) + public async Task GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, CancellationToken cancellationToken) { using var activity = DataEngineTracing.StartSpan(DataEngineTracing.Spans.GetPluginMetadataShells); @@ -230,7 +230,7 @@ public async Task GetDataForShellsByAssetIdsAsync(IRea })) : null; - var response = await pluginDataProvider.GetDataForShellDescriptorsByAssetIdsAsync(pluginRequests, assetIdsHeaderValue, filter?.IdShort, limit, cursor, cancellationToken).ConfigureAwait(false); + var response = await pluginDataProvider.GetDataForShellDescriptorsByAssetIdsAsync(pluginRequests, assetIdsHeaderValue, filter?.IdShort, cancellationToken).ConfigureAwait(false); var result = new ShellDescriptorsMetaData(); diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs index 6a5b4737..e12c38a8 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs @@ -105,14 +105,14 @@ public Task> GetDataForShellDescriptorByIdAsync(IList> GetDataForAssetInformationByIdAsync(IList pluginRequests, CancellationToken cancellationToken) => GetAndProcessAsync(pluginRequests, AssetInformationEndpoint, cancellationToken); - public async Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, int? limit, string? cursor, CancellationToken cancellationToken) + public async Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, CancellationToken cancellationToken) { var result = new List(); var exceptions = new List(); foreach (var pluginRequest in pluginRequests) { - var url = BuildShellsUrl(limit, cursor); + var url = BuildUrl(ApiPaths.PluginMetadata, ShellsEndpoint); if (pluginRequest == null) { From 196fd343d66c9533c1650ee2a8a70ed37cc3a99b Mon Sep 17 00:00:00 2001 From: Hardi Shah Date: Fri, 31 Jul 2026 18:22:26 +0530 Subject: [PATCH 14/30] Refactor SubmodelRepositoryService and related components for pagination support Updated SubmodelRepositoryService to implement pagination for submodel retrieval, including cursor handling. Introduced SubmodelPageCursor to manage state across paginated requests. Modified IPluginDataHandler and IPluginDataProvider interfaces to include pagination parameters. Adjusted PluginDataHandler and PluginDataProvider implementations to support new pagination logic. Enhanced unit tests for SubmodelRepositoryService to cover new pagination scenarios. Added new tests for SubmodelPageCursor to ensure correct encoding and decoding behavior. Updated existing tests in PluginDataHandler and PluginDataProvider to accommodate changes in method signatures. --- .../AasRepositoryServiceTests.cs | 2 +- .../SubmodelRepositoryServiceTests.cs | 16 +- .../Services/PluginDataHandlerTests.cs | 26 +-- .../Services/PluginDataProviderTests.cs | 4 +- .../AasRepository/AasRepositoryService.cs | 2 +- .../Discovery/AssetIdSearchService.cs | 4 +- .../Services/Plugin/IPluginDataHandler.cs | 4 +- .../Plugin/Providers/IPluginDataProvider.cs | 4 +- .../SubmodelRepositoryService.cs | 180 ++++++++++++++---- .../SubmodelPaginationCursor.cs | 43 +++++ .../Services/PluginDataHandler.cs | 4 +- .../Services/PluginDataProvider.cs | 24 ++- 12 files changed, 243 insertions(+), 70 deletions(-) create mode 100644 source/AAS.TwinEngine.DataEngine/DomainModel/SubmodelRepository/SubmodelPaginationCursor.cs diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs index 87bff847..3ac04285 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasRepository; diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs index 7b840855..f8496344 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs @@ -249,7 +249,7 @@ public static SemanticBranchNode CreateSubmodelTreeNode(string value) public async Task GetAllSubmodelsAsync_ReturnsEmpty_WhenNoShellsFound() { _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [] }); var result = await _sut.GetAllSubmodelsAsync(null, null, null, null, CancellationToken.None); @@ -267,7 +267,7 @@ public async Task GetAllSubmodelsAsync_BuildsSubmodelForEachSubmodelId() var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _aasRepositoryTemplateService @@ -297,7 +297,7 @@ public async Task GetAllSubmodelsAsync_SkipsSubmodel_WhenFilteredTemplateReturns var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _aasRepositoryTemplateService @@ -319,7 +319,7 @@ public async Task GetAllSubmodelsAsync_SkipsShell_WhenSubmodelRefsFails() const string ShellId = "https://example.com/shells/001"; _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _aasRepositoryTemplateService @@ -339,7 +339,7 @@ public async Task GetAllSubmodelsAsync_DeduplicatesSubmodelIds_AcrossShells() var filledSubmodel = TestData.CreateFilledSubmodel(); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = @@ -402,7 +402,7 @@ public async Task GetAllSubmodelsAsync_WhenSemanticIdFilterProvided_LooksUpFilte var filledSubmodel = TestData.CreateFilledSubmodel(); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _templateService @@ -437,7 +437,7 @@ public async Task GetAllSubmodelsAsync_WhenSemanticIdNotFound_ThrowsSubmodelNotF var filter = new SubmodelSearchFilter { SemanticId = SemanticId }; _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = "https://example.com/shells/001" }] }); // GetFilteredSubmodelTemplateIdAsync returns null — semantic ID not found in any template @@ -458,7 +458,7 @@ public async Task GetAllSubmodelsAsync_SkipsShellsWithNullOrEmptyId() var filledSubmodel = TestData.CreateFilledSubmodel(); _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any()) + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs index 9c650eff..3b3fa84e 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Net.Http.Json; using System.Text; using System.Text.Json; @@ -630,14 +630,14 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_ReturnsShellDescript }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new List { httpResponse.Content }); const string Json = """[{"name":"sn","value":"123"}]"""; var assetIds = JsonSerializer.Deserialize>(Json)!; var filter = new ShellSearchFilter { SpecificAssetIds = assetIds }; - var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None); + var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); Assert.NotNull(result); Assert.Single(result.ShellDescriptors); @@ -673,12 +673,12 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenNullDeserializat }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None)); + _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None)); } [Fact] @@ -710,12 +710,12 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenInvalidJson_Thro }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None)); + _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None)); } [Fact] @@ -757,11 +757,11 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_SetsHrefOnResults() }; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; - var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None); + var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); Assert.Equal(2, result.ShellDescriptors.Count); Assert.All(result.ShellDescriptors, dto => Assert.StartsWith("https://www.mm-software.com/shells/", dto.Href)); @@ -786,7 +786,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenNoAvailablePlugi var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None)); + _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None)); } [Fact] @@ -829,17 +829,17 @@ public async Task GetDataForShellsByAssetIdsAsync_WithIdShort_PassesIdShort() const string targetIdShort = "Motor001"; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), null, targetIdShort, Arg.Any()) + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), null, targetIdShort, Arg.Any(), Arg.Any(), Arg.Any()) .Returns([httpResponse.Content]); var filter = new ShellSearchFilter { IdShort = targetIdShort }; - var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, CancellationToken.None); + var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); Assert.NotNull(result); Assert.Single(result.ShellDescriptors); Assert.Equal("urn:aas:001", result.ShellDescriptors[0].Id); await _pluginDataProvider.Received(1) - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), null, targetIdShort, Arg.Any()); + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), null, targetIdShort, Arg.Any(), Arg.Any(), Arg.Any()); } private const string AssetData = """ diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs index 0af87a02..9ce3754f 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs @@ -391,7 +391,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_ReturnsContent() var assetIdsHeaderValue = """[{"name":"SerialNumber","value":"SN-4711"}]"""; - var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, assetIdsHeaderValue, null, CancellationToken.None); + var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, assetIdsHeaderValue, null, null, null, CancellationToken.None); Assert.NotNull(result); Assert.Single(result); @@ -431,7 +431,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WithIdShort_AddsIdSh const string idShort = "test-idshort-value"; - var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, null, idShort, CancellationToken.None); + var result = await _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, null, idShort, null, null, CancellationToken.None); Assert.NotNull(result); Assert.Single(result); diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs index 129b5a34..fbe944c0 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs @@ -230,7 +230,7 @@ private void FillShellFromMetadata(IAssetAdministrationShell shell, ShellDescrip CancellationToken cancellationToken) { var metadata = await pluginDataHandler - .GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, filter, cancellationToken) + .GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, filter, null, null, cancellationToken) .ConfigureAwait(false); var allMetadata = metadata.ShellDescriptors? diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs index e6c17cfe..df111ab3 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Base; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; @@ -73,7 +73,7 @@ private async Task GetFilteredMetadataAsync(List GetDataForAssetInformationByIdAsync(IReadOnlyList pluginManifests, string id, CancellationToken cancellationToken); - Task GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, CancellationToken cancellationToken); + Task GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, int? limit, string? cursor, CancellationToken cancellationToken); } diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs index 53b42af7..cc193f52 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.DomainModel.Plugin; +using AAS.TwinEngine.DataEngine.DomainModel.Plugin; namespace AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin.Providers; @@ -12,5 +12,5 @@ public interface IPluginDataProvider Task> GetDataForAssetInformationByIdAsync(IList pluginRequests, CancellationToken cancellationToken); - Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, CancellationToken cancellationToken); + Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, int? limit, string? cursor, CancellationToken cancellationToken); } diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 50f443f2..0045c8bc 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -5,6 +5,7 @@ using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin; using AAS.TwinEngine.DataEngine.DomainModel.AasRegistry; using AAS.TwinEngine.DataEngine.DomainModel.AasRepository; +using AAS.TwinEngine.DataEngine.DomainModel.Shared; using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; @@ -62,14 +63,6 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte { return await ExecuteWithExceptionHandlingAsync(async () => { - var shellSearchFilter = new ShellSearchFilter - { - IdShort = filter?.IdShort - }; - - var shellMetadata = await pluginDataHandler.GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, shellSearchFilter, cancellationToken).ConfigureAwait(false); - var shellDescriptors = shellMetadata.ShellDescriptors ?? []; - string? filteredTemplateId = null; if (filter?.SemanticId is not null) { @@ -81,49 +74,166 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte } } - var distinctSubmodelIds = await GetDistinctSubmodelIdsAsync(shellDescriptors, cancellationToken).ConfigureAwait(false); + var shellSearchFilter = new ShellSearchFilter + { + IdShort = filter?.IdShort + }; - var (pagedIds, pagingMetaData) = PagingExtensions.GetPagedResult(distinctSubmodelIds, id => id, limit, cursor); + var pageSize = limit ?? 100; + var paginationResult = await CollectSubmodelPageAsync(shellSearchFilter, pageSize, cursor, cancellationToken).ConfigureAwait(false); - var submodels = await BuildSubmodelsAsync(pagedIds, filteredTemplateId, queryOptions, cancellationToken).ConfigureAwait(false); + var submodels = await BuildSubmodelsAsync(paginationResult.SubmodelIds, filteredTemplateId, queryOptions, cancellationToken).ConfigureAwait(false); return new SubmodelList { - PagingMetaData = pagingMetaData, + PagingMetaData = new PagingMetaData { Cursor = paginationResult.NextCursor }, Result = submodels }; }).ConfigureAwait(false); } - private async Task> GetDistinctSubmodelIdsAsync(List shellDescriptors, CancellationToken cancellationToken) + ///

+ /// Implements the two-field composite cursor pagination algorithm. + /// Iterates through plugin shell batches, expanding submodel refs per AAS, + /// and collects submodel IDs up to the requested page size. + /// + private async Task CollectSubmodelPageAsync( + ShellSearchFilter shellSearchFilter, + int pageSize, + string? encodedCursor, + CancellationToken cancellationToken) { - using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); - var tasks = shellDescriptors.Where(shell => !string.IsNullOrWhiteSpace(shell.Id)).Select(async shell => + var incomingCursor = SubmodelPaginationCursor.Decode(encodedCursor); + var state = new PaginationState(incomingCursor); + var pluginCursor = state.TrackingAasId; + + while (state.CollectedIds.Count < pageSize) + { + var shellMetadata = await pluginDataHandler.GetDataForShellsByAssetIdsAsync( + pluginManifestConflictHandler.Manifests, shellSearchFilter, pageSize, pluginCursor, cancellationToken).ConfigureAwait(false); + + var shellDescriptors = shellMetadata.ShellDescriptors? + .Where(s => !string.IsNullOrWhiteSpace(s.Id)) + .ToList() ?? []; + + if (shellDescriptors.Count == 0) { - await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - return await aasRepositoryTemplateService.GetSubmodelRefByIdAsync(shell.Id!, cancellationToken).ConfigureAwait(false); - } - catch (ResourceNotFoundException ex) - { - logger.LogWarning(ex, "Could not retrieve submodel refs for shell {ShellId}. Skipping shell.", shell.Id); - return []; - } - finally + break; + } + + var limitReached = await ProcessShellBatchAsync(shellDescriptors, pageSize, state, cancellationToken).ConfigureAwait(false); + + if (limitReached) + { + break; + } + + // Batch exhausted without reaching limit — check if plugin has more data + if (shellMetadata.PagingMetaData?.Cursor is null) + { + break; + } + + pluginCursor = state.TrackingAasId; + } + + var nextCursor = state.CollectedIds.Count >= pageSize + ? SubmodelPaginationCursor.Encode(state.LastCollectedSubmodelId, state.TrackingAasId) + : null; + + return new SubmodelPageResult(state.CollectedIds, nextCursor); + } + + private async Task ProcessShellBatchAsync( + List shellDescriptors, + int pageSize, + PaginationState state, + CancellationToken cancellationToken) + { + foreach (var shell in shellDescriptors) + { + var submodelIds = await GetSubmodelIdsForShellAsync(shell.Id!, cancellationToken).ConfigureAwait(false); + + if (submodelIds.Count == 0) + { + state.TrackingAasId = shell.Id; + state.IsFirstAasInResume = false; + continue; + } + + var startIndex = 0; + + // Skip-scan: only on the first AAS when resuming from a cursor + if (state.IsFirstAasInResume && state.SkipToSubmodelId is not null) + { + startIndex = submodelIds.IndexOf(state.SkipToSubmodelId) + 1; + state.IsFirstAasInResume = false; + state.SkipToSubmodelId = null; + } + else + { + state.IsFirstAasInResume = false; + } + + for (var i = startIndex; i < submodelIds.Count; i++) + { + state.CollectedIds.Add(submodelIds[i]); + state.LastCollectedSubmodelId = submodelIds[i]; + + if (state.CollectedIds.Count >= pageSize) { - _ = semaphore.Release(); + // If we consumed the last submodel of this AAS, advance tracking + if (i == submodelIds.Count - 1) + { + state.TrackingAasId = shell.Id; + state.LastCollectedSubmodelId = null; + } + + return true; } - }); + } - var references = await Task.WhenAll(tasks).ConfigureAwait(false); + // AAS fully consumed — advance tracking state + state.TrackingAasId = shell.Id; + } - return references - .SelectMany(x => x) - .Select(reference => reference.Keys.FirstOrDefault()?.Value) - .Where(id => !string.IsNullOrWhiteSpace(id)) - .Distinct() - .ToList()!; + return false; + } + + private async Task> GetSubmodelIdsForShellAsync(string shellId, CancellationToken cancellationToken) + { + try + { + var references = await aasRepositoryTemplateService.GetSubmodelRefByIdAsync(shellId, cancellationToken).ConfigureAwait(false); + + return references + .Select(r => r.Keys.FirstOrDefault()?.Value) + .Where(id => !string.IsNullOrWhiteSpace(id)) + .ToList()!; + } + catch (ResourceNotFoundException ex) + { + logger.LogWarning(ex, "Could not retrieve submodel refs for shell {ShellId}. Skipping shell.", shellId); + return []; + } + } + + private sealed record SubmodelPageResult(List SubmodelIds, string? NextCursor); + + private sealed class PaginationState + { + public List CollectedIds { get; } = []; + public string? TrackingAasId { get; set; } + public string? LastCollectedSubmodelId { get; set; } + public string? SkipToSubmodelId { get; set; } + public bool IsFirstAasInResume { get; set; } + + public PaginationState(SubmodelPaginationCursor? cursor) + { + TrackingAasId = cursor?.AasId; + SkipToSubmodelId = cursor?.SubmodelId; + IsFirstAasInResume = cursor is not null; + } } private async Task> BuildSubmodelsAsync(IEnumerable submodelIds, string? filteredTemplateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) diff --git a/source/AAS.TwinEngine.DataEngine/DomainModel/SubmodelRepository/SubmodelPaginationCursor.cs b/source/AAS.TwinEngine.DataEngine/DomainModel/SubmodelRepository/SubmodelPaginationCursor.cs new file mode 100644 index 00000000..0598585e --- /dev/null +++ b/source/AAS.TwinEngine.DataEngine/DomainModel/SubmodelRepository/SubmodelPaginationCursor.cs @@ -0,0 +1,43 @@ +using System.Text; + +using Microsoft.AspNetCore.WebUtilities; + +namespace AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; + +/// +/// Two-field composite cursor for submodel pagination. +/// Wire format: Base64Url("{SubmodelId}|{AasId}") +/// +public sealed record SubmodelPaginationCursor(string? SubmodelId, string? AasId) +{ + private const char Separator = '|'; + + public static SubmodelPaginationCursor? Decode(string? encodedCursor) + { + if (string.IsNullOrWhiteSpace(encodedCursor)) + { + return null; + } + + var decoded = Encoding.UTF8.GetString(WebEncoders.Base64UrlDecode(encodedCursor)); + var separatorIndex = decoded.IndexOf(Separator); + + if (separatorIndex < 0) + { + return null; + } + + var submodelId = decoded[..separatorIndex]; + var aasId = decoded[(separatorIndex + 1)..]; + + return new SubmodelPaginationCursor( + string.IsNullOrEmpty(submodelId) ? null : submodelId, + string.IsNullOrEmpty(aasId) ? null : aasId); + } + + public static string? Encode(string? submodelId, string? aasId) + { + var logical = $"{submodelId ?? string.Empty}{Separator}{aasId ?? string.Empty}"; + return WebEncoders.Base64UrlEncode(Encoding.UTF8.GetBytes(logical)); + } +} diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs index f8aa9ca8..0b45ba05 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs @@ -207,7 +207,7 @@ public async Task GetDataForAssetInformationByIdAsync(IReadOnlyList

GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, CancellationToken cancellationToken) + public async Task GetDataForShellsByAssetIdsAsync(IReadOnlyList pluginManifests, ShellSearchFilter? filter, int? limit, string? cursor, CancellationToken cancellationToken) { using var activity = DataEngineTracing.StartSpan(DataEngineTracing.Spans.GetPluginMetadataShells); @@ -230,7 +230,7 @@ public async Task GetDataForShellsByAssetIdsAsync(IRea })) : null; - var response = await pluginDataProvider.GetDataForShellDescriptorsByAssetIdsAsync(pluginRequests, assetIdsHeaderValue, filter?.IdShort, cancellationToken).ConfigureAwait(false); + var response = await pluginDataProvider.GetDataForShellDescriptorsByAssetIdsAsync(pluginRequests, assetIdsHeaderValue, filter?.IdShort, limit, cursor, cancellationToken).ConfigureAwait(false); var result = new ShellDescriptorsMetaData(); diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs index e12c38a8..fceb1d79 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs @@ -105,14 +105,14 @@ public Task> GetDataForShellDescriptorByIdAsync(IList> GetDataForAssetInformationByIdAsync(IList pluginRequests, CancellationToken cancellationToken) => GetAndProcessAsync(pluginRequests, AssetInformationEndpoint, cancellationToken); - public async Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, CancellationToken cancellationToken) + public async Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, int? limit, string? cursor, CancellationToken cancellationToken) { var result = new List(); var exceptions = new List(); foreach (var pluginRequest in pluginRequests) { - var url = BuildUrl(ApiPaths.PluginMetadata, ShellsEndpoint); + var url = BuildShellsByAssetIdsUrl(limit, cursor); if (pluginRequest == null) { @@ -268,6 +268,26 @@ private static string BuildShellsUrl(int? limit, string? cursor) : BaseUrl; } + private static string BuildShellsByAssetIdsUrl(int? limit, string? cursor) + { + const string BaseUrl = $"/{ApiPaths.PluginMetadata}/{ShellsEndpoint}"; + var queryParams = new Dictionary(); + + if (limit is > 0) + { + queryParams["limit"] = limit.Value.ToString(); + } + + if (!string.IsNullOrWhiteSpace(cursor)) + { + queryParams["cursor"] = cursor; + } + + return queryParams.Count > 0 + ? QueryHelpers.AddQueryString(BaseUrl, queryParams!) + : BaseUrl; + } + private static Exception HandleFailureResponse(System.Net.HttpStatusCode statusCode) => statusCode switch { From dd828fa7e2f17b75fc86bf1173be37a0a4e1d282 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Sat, 1 Aug 2026 12:48:21 +0530 Subject: [PATCH 15/30] chore: remove outdated Submodel Pagination Cursor Concept documentation --- Submodel-Pagination-Cursor-Concept.md | 311 -------------------------- 1 file changed, 311 deletions(-) delete mode 100644 Submodel-Pagination-Cursor-Concept.md diff --git a/Submodel-Pagination-Cursor-Concept.md b/Submodel-Pagination-Cursor-Concept.md deleted file mode 100644 index a6906b9f..00000000 --- a/Submodel-Pagination-Cursor-Concept.md +++ /dev/null @@ -1,311 +0,0 @@ -# Submodel Pagination Cursor Concept - Get All Submodels - -## Problem -The DataEngine's `GET /submodels` and `GET /submodel-descriptors` endpoints must paginate **submodels**, but the underlying Plugin APIs paginate **products (Asset Administration Shells / AAS IDs)**. In an Asset Administration Shell ecosystem, a single product (AAS) can contain multiple submodels (a `1-to-N` expansion relationship). - -When a client requests a page of submodels with a specific `limit` (e.g., `limit=2` or `limit=100`), a single product's submodel list may be only **partially consumed** when the page limit is reached. If the pagination engine uses standard single-field or exclusive cursor pagination (which moves the underlying Plugin cursor past the product), any remaining uncollected submodels in that product are permanently skipped on subsequent page requests. - -## Constraints -- **O(limit) Memory Footprint:** The pagination engine must operate with constant memory proportional to the requested page `limit` and batch size, never loading the full dataset into memory. -- **No Plugin API Modifications:** Underlying Plugins implement standard IDTA / BaSyx product-level cursor pagination (`limit` and `cursor`); Plugins do not support submodel-level cursors. -- **AAS-ID-Based Exclusive Plugin Cursor:** The Plugin cursor is AAS-ID-based and uses exclusive semantics - passing an AAS ID as cursor instructs the Plugin to return all AAS strictly after that ID in its deterministic ordering. -- **Deterministic Ordering:** Plugins return products in stable, deterministic order, and submodel references within a product maintain a deterministic order. -- **Zero Data Loss / Zero Duplication:** Traversing all pages from start to finish must yield every submodel exactly once under static conditions. -- **IDTA Compliance:** The pagination solution must remain fully compliant with the IDTA AAS Part 2 API specification. The DataEngine may introduce internal pagination logic and cursor management, but the externally exposed API behavior, request/response contracts must remain compliant with the IDTA specification. - -## Assumptions -- Each Asset Administration Shell (`IAssetAdministrationShell`) has a unique `ID` and a deterministic list of submodel references (`Submodels`). -- The API endpoint acts as the orchestrator over one or more external Plugin instances. -- Submodel data is collected on-demand after identifying candidate submodel IDs from shell templates. - ---- - -## Considered Alternatives - -### 1. Submodel ID Cursor with Unbounded Plugin Fetching (No Plugin Limit/Cursor) -- **Description:** Store only the last delivered `SubmodelId` in the client cursor token (`cursor = LastSubmodelId`). When fetching products from the Plugin (AAS Shell request), the DataEngine does **not** provide any `limit` or `cursor` to the Plugin, thereby fetching **all products (shells)** on every page request. During submodel expansion, the DataEngine iterates through all products and submodels from the beginning, skipping submodels until it matches the requested `SubmodelId` cursor, and then collects the next page of submodels up to `limit`. -- **Pros:** - - Very simple and compact cursor token (only a single `SubmodelId`). - - Correctness is maintained (no submodels are skipped or lost across page boundaries, as all products are inspected). - - No complex multi-field cursor coordinate tracking required. -- **Cons:** - - **High Upstream Overload & Network Waste:** Fetching *all* products from the Plugin on *every single page request* destroys scalability. If there are 100,000 products, every page request fetches all 100,000 products from the Plugin just to find where `SubmodelId` is located. - -### 2. Product ID Bulk Endpoint in Plugin with DataEngine Expansion -- **Description:** Introduce a new bulk endpoint in the Plugin contract to retrieve all product IDs (or a paginated list of all product IDs). The DataEngine would retrieve these product IDs, resolve the shell template for each ID to determine the associated submodels, and then query the Plugin for the specific submodel data. Under this approach, the client cursor only needs to store the last delivered `SubmodelId`. -- **Pros:** - - **Simple Cursor Token:** The pagination cursor remains simple (e.g., just the last delivered `SubmodelId` or index). -- **Cons:** - - **Plugin Contract Breaking Change:** Requires updating the Plugin API contract to support the new bulk product ID endpoint. - - **High Architectural Complexity:** Orchestrating product ID retrieval, resolving matching shell templates, and mapping them to submodels on the fly in the DataEngine introduces significant complexity and state tracking. - - **Association / Resolution Difficulties:** It is difficult to map a product ID directly to its associated shell template and submodels without fetching the entire shell, which negates the performance benefits. - - -### 3. Two-Field Composite Cursor (`SubmodelId` | `AasId`) (Chosen) -- **Description:** The cursor stores exactly two lightweight coordinates: the last submodel ID delivered to the client (`SubmodelId`) and the AAS ID that serves simultaneously as both the position anchor and the next plugin request cursor (`AasId`). Because the plugin cursor is AAS-ID-based with exclusive semantics, the engine can derive the exact plugin request cursor directly from the AAS ID stored in the DataEngine cursor. A single tracking state - initialized from the cursor's `AasId` field and advanced each time an AAS is fully consumed - determines which AAS to start from on every plugin request. This eliminates the need for a separate opaque plugin cursor field. -- **Pros:** - - **Guaranteed Correctness:** Eliminates data loss and duplication; fully supports `1-to-N` expansion. - - **Minimal Cursor Size:** Exactly 2 pipe-delimited string fields, smaller than a 3-field cursor. - - **Constant Memory (`O(limit)`):** Streams and collects only what is needed for the current page. - - **No Plugin API Changes Required:** Works seamlessly over AAS-ID-based exclusive cursor endpoints. - - **Bounded Skip Scan:** The `SubmodelId` skip scan is confined to the first AAS of the returned batch only, never scanning the full dataset. - ---- - -### Alternative Comparison Summary - -| Approach | Memory Footprint | Correctness (No Data Loss) | Cursor Token Size | Plugin API Changes Required | -|---|---|---|---|---| -| **1. Submodel ID Cursor (Unbounded Fetch)** | `O(N)` (Unbounded Fetch) | Yes | Small (`SubmodelId`) | No | -| **2. Product ID Bulk Endpoint** | `O(N)` (Unbounded Fetch of IDs) | Yes | Small (`SubmodelId`) | **Yes** | -| **3. Two-Field Composite Cursor (Chosen)** | `O(limit)` (Optimal) | **Yes** | **Smallest (2 fields)** | No | - ---- - -## Decision -We adopt **Option 3: Two-Field Composite Cursor (`SubmodelId` | `AasId`)** as the standard architectural pattern for all submodel and submodel-descriptor pagination. - ---- - -## Architectural Concept & Two-Field Composite Cursor - -### 1. The `1-to-N` Expansion Problem (`limit=2` Walkthrough) - -Consider an environment with 3 products (`AAS-1`, `AAS-2`, `AAS-3`), each containing 3 submodels: -``` -Product 1 (AAS-1) Product 2 (AAS-2) Product 3 (AAS-3) -├── SM-1 ├── SM-4 ├── SM-7 -├── SM-2 ├── SM-5 ├── SM-8 -└── SM-3 ├── SM-6 └── SM-9 -``` - -When a client executes `GET /submodels?limit=2`: -``` -Page 1: GET /submodels?limit=2 - - 1. API calls Plugin.GetShells(cursor=null) → returns [AAS-1, AAS-2] - 2. Expand AAS-1 → [SM-1, SM-2, SM-3] - 3. Collect SM-1 (Count=1), Collect SM-2 (Count=2). Page limit reached! - 4. AAS-1 is PARTIALLY consumed (SM-3 is still pending). -``` - -If the API were to advance the Plugin cursor past AAS-1 for Page 2: -``` -Page 2: GET /submodels?limit=2&cursor="AAS-1" (exclusive - starts after AAS-1) - - 1. API calls Plugin.GetShells(cursor="AAS-1") - 2. Plugin returns products AFTER AAS-1 → [AAS-2, AAS-3, ...] - 3. RESULT: SM-3 (from AAS-1) IS PERMANENTLY LOST! -``` - -The cursor must not advance the Plugin cursor until the current AAS is fully consumed. - -### 2. Logical Composition of the Two-Field Cursor - -The cursor stores two coordinates that together capture exact positioning across both the AAS layer and the Submodel layer: - -| Coordinate | Wire Field Position | Nullable | Description | -|---|---|---|---| -| `SubmodelId` | Index 0 (Prefix) | Yes (`null` when limit is reached exactly at an AAS boundary) | The unique identifier of the last submodel delivered to the client. On resume, the API scans only the **first AAS** of the returned batch to skip already-delivered submodels. | -| `AasId` | Index 1 (Suffix) | Yes (`null` on first page or when no AAS has been fully consumed) | The AAS ID that is passed directly to the Plugin as an exclusive cursor on resume. The Plugin returns all AAS strictly after this ID. Advances only when an AAS is fully consumed. | - -#### Wire Encoding Strategy -The logical cursor string is formatted as UTF-8 text separated by a pipe character (`|`), then encoded using standard `Base64Url` encoding (RFC 4648) without padding: -``` -Logical Structure: {SubmodelId}|{AasId} -Encoded Wire Token: Base64Url( UTF-8( "{SubmodelId}|{AasId}" ) ) -``` - -**Examples:** -- Mid-AAS cursor, no AAS yet fully consumed (re-fetch from plugin start): - - Logical: `SM-2|` (AasId is empty/null) - - Encoded: `U00tMnw` -- Mid-AAS cursor, AAS-1 was fully consumed (plugin starts after AAS-1): - - Logical: `SM-4|AAS-1` - - Encoded: `U00tNHxBQVMtMQ` -- Limit reached at AAS boundary, AAS-2 fully consumed (plugin starts after AAS-2): - - Logical: `|AAS-2` (SubmodelId is empty/null) - - Encoded: `fEFBUy0y` - -### 3. The AasId Tracking Logic - -A single position-tracking state drives the cursor throughout a page execution. It is: -- **Initialized** from the incoming cursor's `AasId` field (null at the very first request). -- **Advanced** each time an AAS within the current batch is fully consumed - it is updated to that AAS's ID. -- **Captured** as the `AasId` field of the output cursor when the page limit is reached. - -This guarantees the following invariant: the `AasId` in the output cursor always equals the ID of the **last fully consumed AAS at the moment the page limit was reached**, or null if no AAS was fully consumed since the current plugin batch started. On the next request, the API passes this `AasId` directly to the Plugin as an exclusive cursor, which returns all AAS strictly after that ID - precisely the position where collection should resume. - -When the page limit is reached **exactly at the boundary of an AAS** (i.e., the last submodel of that AAS is the last one collected): the AAS is considered fully consumed, the tracking state advances to that AAS's ID, and `SubmodelId` is set to null. On resume, the API collects from the very first submodel of the first AAS returned by the Plugin. - -### 4. Resume Algorithm (Conceptual) - -On receiving a non-null cursor: -1. Extract `SubmodelId` and `AasId` from the cursor. -2. Initialize the tracking state from `AasId`. -3. Call the Plugin with the tracking state as the exclusive cursor → receives the first relevant batch. -4. In the **first AAS of the batch**: - - If `SubmodelId` is set: scan this AAS to find `SubmodelId`, skip everything up to and including it, and begin collecting from the next submodel. - - If `SubmodelId` is null: collect from the very first submodel. -5. For all subsequent AAS in the same batch, and for further batches: collect all submodels from their start. -6. Each time an AAS is fully consumed, advance the tracking state to that AAS's ID. -7. When the page limit is reached: encode the output cursor as `{SubmodelId=last_collected, AasId=current_tracking_state}`. If the limit was reached exactly at an AAS boundary: encode as `{SubmodelId=null, AasId=that_AAS_ID}`. -8. When a batch is exhausted without reaching the page limit: request the next batch from the Plugin using the current tracking state as the exclusive cursor. - -The `SubmodelId` skip scan is always bounded to the **first AAS of the batch** - never a full dataset scan - because `AasId` guarantees the Plugin skips all previously fully consumed AAS before returning the batch. - ---- - -## Sequence Diagram - -The following sequence diagram illustrates the end-to-end execution flow of `GET /submodels?limit=2` across two page requests, demonstrating how the tracking state advances when an AAS is fully consumed and how the two-field cursor drives the next Plugin request. The API endpoint handles pagination logic directly - no intermediate service layer is involved: - -```mermaid -sequenceDiagram - autonumber - actor Client - participant API as GET /submodels - participant Plugin as Plugin / AAS Registry - - Note over Client,Plugin: Page 1 Request (cursor = null) - Client->>API: GET /submodels?limit=2 - - Note over API: Tracking state initialized to null (AasId=null) - API->>Plugin: GetShells(cursor=null, limit=2) - Plugin-->>API: Shells: [AAS-1, AAS-2] - - Note over API: Expand AAS-1 submodel refs -> [SM-1, SM-2, SM-3] - API->>Plugin: GetSubmodel("SM-1") - Plugin-->>API: Submodel("SM-1") (Count = 1) - - API->>Plugin: GetSubmodel("SM-2") - Plugin-->>API: Submodel("SM-2") (Count = 2 - Limit Reached!) - - Note over API: AAS-1 not fully consumed. Tracking state stays null. - Note over API: Encode Cursor: SubmodelId="SM-2" | AasId=null - API-->>Client: 200 OK [SM-1, SM-2] + cursor="U00tMnw" - - Note over Client,Plugin: Page 2 Request (Resume from partially consumed AAS-1) - Client->>API: GET /submodels?limit=2&cursor=U00tMnw - - Note over API: Decode Cursor -> SubmodelId="SM-2", AasId=null - Note over API: Tracking state initialized to null (from AasId=null) - API->>Plugin: GetShells(cursor=null, limit=2) - Plugin-->>API: Shells: [AAS-1, AAS-2] - - Note over API: First AAS = AAS-1. Scan for SM-2 → skip SM-1 and SM-2 - API->>Plugin: GetSubmodel("SM-3") - Plugin-->>API: Submodel("SM-3") (Count = 1, AAS-1 complete) - - Note over API: AAS-1 fully consumed. Tracking state advances to AAS-1. - Note over API: Advance to AAS-2 in same batch -> [SM-4, SM-5, SM-6] - API->>Plugin: GetSubmodel("SM-4") - Plugin-->>API: Submodel("SM-4") (Count = 2 - Limit Reached!) - - Note over API: AAS-2 not fully consumed. Tracking state = AAS-1. - Note over API: Encode Cursor: SubmodelId="SM-4" | AasId="AAS-1" - API-->>Client: 200 OK [SM-3, SM-4] + cursor="U00tNHxBQVMtMQ" -``` - -On Page 3, the API calls `Plugin.GetShells(cursor="AAS-1")` - the Plugin returns `[AAS-2, AAS-3]` (all AAS strictly after `AAS-1`). The API scans `AAS-2` (the first AAS in the new batch) to find `SM-4`, skips it, and resumes collection from `SM-5`. - ---- - -## Step-by-Step Examples - -### Example 1: `limit=2` Across 5 Consecutive Pages (The Tricky Case) - -Dataset: -- `AAS-1` → `[SM-1, SM-2, SM-3]` -- `AAS-2` → `[SM-4, SM-5, SM-6]` -- `AAS-3` → `[SM-7, SM-8, SM-9]` - -Plugin batch size: 2 products per request. - -#### Page 1: `GET /submodels?limit=2` -- **Request:** `limit=2`, `cursor=null` -- **API Execution:** - 1. Tracking state initialized to null. - 2. Call `Plugin.GetShells(cursor=null)` → returns `[AAS-1, AAS-2]`. - 3. First AAS is `AAS-1`. `SubmodelId` is null (no cursor) → collect from the start. - 4. Collect `SM-1` (Count=1), `SM-2` (Count=2) → **Limit reached!** `AAS-1` is partial. - 5. Tracking state unchanged (null). No AAS was fully consumed. - 6. Encode cursor: `SubmodelId="SM-2"`, `AasId=null`. -- **Response:** -```json -{ "paging_metadata": { "cursor": "U00tMnw" }, "result": [{ "id": "SM-1" }, { "id": "SM-2" }] } -``` - -#### Page 2: `GET /submodels?limit=2&cursor=U00tMnw` -- **Request:** `limit=2`, cursor decodes to `SubmodelId="SM-2"`, `AasId=null` -- **API Execution:** - 1. Tracking state initialized to null (from `AasId=null`). - 2. Call `Plugin.GetShells(cursor=null)` → returns `[AAS-1, AAS-2]`. - 3. First AAS is `AAS-1`. Scan for `SM-2` → skip `SM-1`, `SM-2`. Collect `SM-3` (Count=1). `AAS-1` complete. - 4. Tracking state advances to `AAS-1`. - 5. Advance to `AAS-2` in the same batch. Collect `SM-4` (Count=2) → **Limit reached!** `AAS-2` is partial. - 6. Tracking state = `AAS-1`. Encode cursor: `SubmodelId="SM-4"`, `AasId="AAS-1"`. -- **Response:** -```json -{ "paging_metadata": { "cursor": "U00tNHxBQVMtMQ" }, "result": [{ "id": "SM-3" }, { "id": "SM-4" }] } -``` - -#### Page 3: `GET /submodels?limit=2&cursor=U00tNHxBQVMtMQ` -- **Request:** `limit=2`, cursor decodes to `SubmodelId="SM-4"`, `AasId="AAS-1"` -- **API Execution:** - 1. Tracking state initialized to `AAS-1` (from `AasId="AAS-1"`). - 2. Call `Plugin.GetShells(cursor="AAS-1")` → returns `[AAS-2, AAS-3]` (all AAS strictly after `AAS-1`). - 3. First AAS is `AAS-2`. Scan for `SM-4` → skip `SM-4`. Collect `SM-5` (Count=1), `SM-6` (Count=2) → **Limit reached!** `AAS-2` is fully consumed at the limit boundary. - 4. Tracking state advances to `AAS-2`. Since `AAS-2` is fully consumed at the boundary, `SubmodelId=null`. - 5. Encode cursor: `SubmodelId=null`, `AasId="AAS-2"`. -- **Response:** -```json -{ "paging_metadata": { "cursor": "fEFBUy0y" }, "result": [{ "id": "SM-5" }, { "id": "SM-6" }] } -``` - -#### Page 4: `GET /submodels?limit=2&cursor=fEFBUy0y` -- **Request:** `limit=2`, cursor decodes to `SubmodelId=null`, `AasId="AAS-2"` -- **API Execution:** - 1. Tracking state initialized to `AAS-2` (from `AasId="AAS-2"`). - 2. Call `Plugin.GetShells(cursor="AAS-2")` → returns `[AAS-3]` (all AAS strictly after `AAS-2`). - 3. First AAS is `AAS-3`. `SubmodelId` is null → collect from the start. - 4. Collect `SM-7` (Count=1), `SM-8` (Count=2) → **Limit reached!** `AAS-3` is partial. - 5. Tracking state unchanged (`AAS-2`). Encode cursor: `SubmodelId="SM-8"`, `AasId="AAS-2"`. -- **Response:** -```json -{ "paging_metadata": { "cursor": "U00tOHxBQVMtMg" }, "result": [{ "id": "SM-7" }, { "id": "SM-8" }] } -``` - -#### Page 5 (Final): `GET /submodels?limit=2&cursor=U00tOHxBQVMtMg` -- **Request:** `limit=2`, cursor decodes to `SubmodelId="SM-8"`, `AasId="AAS-2"` -- **API Execution:** - 1. Tracking state initialized to `AAS-2`. - 2. Call `Plugin.GetShells(cursor="AAS-2")` → returns `[AAS-3]`. - 3. First AAS is `AAS-3`. Scan for `SM-8` → skip `SM-7`, `SM-8`. Collect `SM-9` (Count=1). - 4. `AAS-3` exhausted. No more batches (`nextCursor=null`). Dataset exhausted. - 5. Return with `cursor=null`. -- **Response:** -```json -{ "paging_metadata": { "cursor": null }, "result": [{ "id": "SM-9" }] } -``` - ---- - -### Example 2: `limit=4` Across 3 Pages - -#### Page 1: `GET /submodels?limit=4` -- **Execution:** Tracking state=null. Plugin(null) → [AAS-1, AAS-2]. Collect all 3 of AAS-1 (tracking advances to AAS-1), then collect SM-4 from AAS-2 (limit reached, AAS-2 partial). -- **Response Cursor:** `SubmodelId="SM-4"`, `AasId="AAS-1"` (encoded: `U00tNHxBQVMtMQ`) -- **Result:** `[SM-1, SM-2, SM-3, SM-4]` - -#### Page 2: `GET /submodels?limit=4&cursor=U00tNHxBQVMtMQ` -- **Execution:** Tracking state=AAS-1. Plugin(AAS-1) → [AAS-2, AAS-3]. Scan AAS-2 for SM-4, skip it, collect SM-5, SM-6 (AAS-2 done, tracking → AAS-2). Advance to AAS-3, collect SM-7, SM-8 (Count=4, limit reached, AAS-3 partial). -- **Response Cursor:** `SubmodelId="SM-8"`, `AasId="AAS-2"` (encoded: `U00tOHxBQVMtMg`) -- **Result:** `[SM-5, SM-6, SM-7, SM-8]` - -#### Page 3: `GET /submodels?limit=4&cursor=U00tOHxBQVMtMg` -- **Execution:** Tracking state=AAS-2. Plugin(AAS-2) → [AAS-3]. Scan AAS-3 for SM-8, skip SM-7, SM-8, collect SM-9 (Count=1). Dataset exhausted. -- **Response Cursor:** `null` -- **Result:** `[SM-9]` - ---- From 97d2cf737775f566c2bc633dcd1e7787a3e3494c Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Mon, 3 Aug 2026 10:21:30 +0530 Subject: [PATCH 16/30] refactor: enhance pagination logic and update submodel ID handling in SubmodelRepositoryService --- .../SubmodelRepositoryServiceTests.cs | 97 +++++++++++++++++++ .../SubmodelRepositoryService.cs | 13 +-- 2 files changed, 102 insertions(+), 8 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs index f8496344..4ff78d8c 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs @@ -491,6 +491,103 @@ public async Task GetAllSubmodelsAsync_SkipsShellsWithNullOrEmptyId() await _aasRepositoryTemplateService.DidNotReceive().GetSubmodelRefByIdAsync(null!, Arg.Any()); } + [Fact] + public async Task GetAllSubmodelsAsync_WhenLimitReachedAtAasBoundary_CursorSubmodelIdIsLastDeliveredAndAasIdIsConsumedAas() + { + // AAS-1 has 3 submodels, AAS-2 has 2 submodels. limit=5 hits the exact end of AAS-2. + // AasId in the cursor must be AAS-2 (the consumed AAS), not AAS-1 (the entry cursor), + // so Plugin(AAS-2 exclusive) on resume skips directly to the next AAS. + const string Shell1Id = "https://example.com/shells/aas-1"; + const string Shell2Id = "https://example.com/shells/aas-2"; + const string Sm1 = "https://example.com/submodels/sm-1"; + const string Sm2 = "https://example.com/submodels/sm-2"; + const string Sm3 = "https://example.com/submodels/sm-3"; + const string Sm4 = "https://example.com/submodels/sm-4"; + const string Sm5 = "https://example.com/submodels/sm-5"; + + _pluginDataHandler + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ShellDescriptorsMetaData + { + ShellDescriptors = [new ShellDescriptorMetaData { Id = Shell1Id }, new ShellDescriptorMetaData { Id = Shell2Id }] + }); + + _aasRepositoryTemplateService.GetSubmodelRefByIdAsync(Shell1Id, Arg.Any()) + .Returns([ + new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm1)]), + new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm2)]), + new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm3)]) + ]); + + _aasRepositoryTemplateService.GetSubmodelRefByIdAsync(Shell2Id, Arg.Any()) + .Returns([ + new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm4)]), + new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm5)]) + ]); + + _templateService + .GetFilteredSubmodelTemplateAsync(Arg.Any(), (string?)null, Arg.Any(), Arg.Any()) + .Returns(TestData.CreateSubmodel()); + _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); + _pluginDataHandler + .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); + _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()) + .Returns(TestData.CreateFilledSubmodel()); + + var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 5, null, CancellationToken.None); + + Assert.Equal(5, result.Result.Count); + Assert.NotNull(result.PagingMetaData?.Cursor); + + var decoded = SubmodelPaginationCursor.Decode(result.PagingMetaData!.Cursor!); + Assert.Equal(Sm5, decoded!.SubmodelId); // last delivered submodel + Assert.Equal(Shell2Id, decoded.AasId); // the consumed AAS, not the entry cursor + } + + [Fact] + public async Task GetAllSubmodelsAsync_WhenLimitReachedAtFirstAasBoundary_CursorAasIdIsConsumedAas() + { + // AAS-1 has exactly 2 submodels, limit=2. Limit hit at the end of AAS-1. + // AasId must be AAS-1 (the consumed AAS) so the next request starts from after AAS-1. + const string Shell1Id = "https://example.com/shells/aas-1"; + const string Sm1 = "https://example.com/submodels/sm-1"; + const string Sm2 = "https://example.com/submodels/sm-2"; + + _pluginDataHandler + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ShellDescriptorsMetaData + { + ShellDescriptors = [new ShellDescriptorMetaData { Id = Shell1Id }], + PagingMetaData = new PagingMetaData { Cursor = "next-page-token" } + }); + + _aasRepositoryTemplateService.GetSubmodelRefByIdAsync(Shell1Id, Arg.Any()) + .Returns([ + new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm1)]), + new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm2)]) + ]); + + _templateService + .GetFilteredSubmodelTemplateAsync(Arg.Any(), (string?)null, Arg.Any(), Arg.Any()) + .Returns(TestData.CreateSubmodel()); + _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); + _pluginDataHandler + .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); + _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()) + .Returns(TestData.CreateFilledSubmodel()); + + var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 2, null, CancellationToken.None); + + Assert.Equal(2, result.Result.Count); + Assert.NotNull(result.PagingMetaData?.Cursor); + + var decoded = SubmodelPaginationCursor.Decode(result.PagingMetaData!.Cursor!); + Assert.Equal(Sm2, decoded!.SubmodelId); // last delivered submodel + Assert.Equal(Shell1Id, decoded.AasId); // consumed AAS — next request starts after AAS-1 + } + [Fact] public async Task GetAllSubmodelElementsAsync_ReturnsAllElements_WhenSubmodelExists() { diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 0045c8bc..2204963f 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -40,8 +40,6 @@ public async Task GetSubmodelAsync(string submodelId, SubmodelQueryOp var submodelWithValues = await BuildSubmodelWithValuesAsync(submodelTemplate, submodelId, cancellationToken).ConfigureAwait(false); - submodelWithValues.Id = submodelId; - return submodelWithValues; }).ConfigureAwait(false); } @@ -110,7 +108,7 @@ private async Task CollectSubmodelPageAsync( while (state.CollectedIds.Count < pageSize) { var shellMetadata = await pluginDataHandler.GetDataForShellsByAssetIdsAsync( - pluginManifestConflictHandler.Manifests, shellSearchFilter, pageSize, pluginCursor, cancellationToken).ConfigureAwait(false); + pluginManifestConflictHandler.Manifests, shellSearchFilter, pageSize, Base64UrlExtensions.EncodeBase64Url(pluginCursor), cancellationToken).ConfigureAwait(false); var shellDescriptors = shellMetadata.ShellDescriptors? .Where(s => !string.IsNullOrWhiteSpace(s.Id)) @@ -182,18 +180,15 @@ private async Task ProcessShellBatchAsync( if (state.CollectedIds.Count >= pageSize) { - // If we consumed the last submodel of this AAS, advance tracking - if (i == submodelIds.Count - 1) + if (state.CollectedIds.Contains(submodelIds.Last())) { state.TrackingAasId = shell.Id; - state.LastCollectedSubmodelId = null; } return true; } } - // AAS fully consumed — advance tracking state state.TrackingAasId = shell.Id; } @@ -301,7 +296,9 @@ private async Task BuildSubmodelWithValuesAsync(ISubmodel template, s var values = await pluginDataHandler.TryGetValuesAsync(pluginManifests, semanticIds, submodelId, cancellationToken).ConfigureAwait(false); - return semanticIdHandler.FillOutTemplate(template, values); + var submodelWithValues = semanticIdHandler.FillOutTemplate(template, values); + submodelWithValues.Id = submodelId; + return submodelWithValues; } private static async Task ExecuteWithExceptionHandlingAsync(Func> action) From f3409a0eb121ecd80cda2d4f6b4a00a5a4e4d25a Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Mon, 3 Aug 2026 17:34:00 +0530 Subject: [PATCH 17/30] Implement two-field cursor-based pagination Refactored `SubmodelDescriptorService` to introduce a two-field composite cursor pagination algorithm for efficient data retrieval. Updated `GetAllSubmodelDescriptorsAsync` to use the new pagination logic and include cursor metadata. Added helper methods `CollectSubmodelDescriptorPageAsync`, `ProcessShellBatch`, and `GetSubmodelIdsForShell` to support the new pagination mechanism. Introduced `PaginationState` and `SubmodelDescriptorPageResult` to manage pagination state. Updated test cases in `SubmodelDescriptorServiceTests` to validate the new pagination logic, including a new test method `GetAllSubmodelDescriptorsAsync_ResumesCorrectly_WithTwoFieldCompositeCursor`. Refactored `CreateShell` to support customizable test data. Removed the obsolete `ExtractDistinctSubmodelIds` method and added necessary `using` directives for additional namespaces. --- .../SubmodelDescriptorControllerTests.cs | 6 +- .../SubmodelDescriptorServiceTests.cs | 53 +++++- .../SubmodelDescriptorService.cs | 151 ++++++++++++++++-- 3 files changed, 185 insertions(+), 25 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.ModuleTests/Api/Services/SubmodelRegistry/SubmodelDescriptorControllerTests.cs b/source/AAS.TwinEngine.DataEngine.ModuleTests/Api/Services/SubmodelRegistry/SubmodelDescriptorControllerTests.cs index a849232d..59ec0d99 100644 --- a/source/AAS.TwinEngine.DataEngine.ModuleTests/Api/Services/SubmodelRegistry/SubmodelDescriptorControllerTests.cs +++ b/source/AAS.TwinEngine.DataEngine.ModuleTests/Api/Services/SubmodelRegistry/SubmodelDescriptorControllerTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Net.Http.Json; using System.Text; using System.Text.Json.Nodes; @@ -61,7 +61,7 @@ public async Task GetAllSubmodelDescriptorsAsync_ReturnsOkAsync() CreateShell("Nameplate") ] }; - _ = _mockAasRepositoryService.GetShellsByFiltersAsync(null, null, null, Arg.Any()) + _ = _mockAasRepositoryService.GetShellsByFiltersAsync(null, Arg.Any(), Arg.Any(), Arg.Any()) .Returns(shells); _ = _mockSubmodelDescriptorProvider.GetDataForSubmodelDescriptorByIdAsync(Arg.Any(), Arg.Any()) @@ -95,7 +95,7 @@ public async Task GetAllSubmodelDescriptorsAsync_WithNotFound_Returns404Async() CreateShell("MissingSubmodel") ] }; - _ = _mockAasRepositoryService.GetShellsByFiltersAsync(null, null, null, Arg.Any()) + _ = _mockAasRepositoryService.GetShellsByFiltersAsync(null, Arg.Any(), Arg.Any(), Arg.Any()) .Returns(shells); _ = _mockSubmodelDescriptorProvider.GetDataForSubmodelDescriptorByIdAsync(Arg.Any(), Arg.Any()) .Throws(new ResourceNotFoundException()); diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorServiceTests.cs index c265ab2d..36d812c9 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorServiceTests.cs @@ -7,6 +7,7 @@ using AAS.TwinEngine.DataEngine.DomainModel.AasRepository; using AAS.TwinEngine.DataEngine.DomainModel.Shared; using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRegistry; +using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; using AasCore.Aas3_1; @@ -56,7 +57,7 @@ public async Task GetAllSubmodelDescriptorsAsync_ReturnsPagedDescriptorsDerivedF CreateShell("Nameplate") ] }; - _aasRepositoryService.GetShellsByFiltersAsync(null, null, null, Arg.Any()) + _aasRepositoryService.GetShellsByFiltersAsync(null, Arg.Any(), Arg.Any(), Arg.Any()) .Returns(shells); _submodelTemplateMappingProvider.GetTemplateId("ContactInformation").Returns("ContactInformation"); _submodelTemplateMappingProvider.GetTemplateId("Nameplate").Returns("Nameplate"); @@ -85,7 +86,7 @@ public async Task GetAllSubmodelDescriptorsAsync_SkipsDescriptorWhenSingleDescri CreateShell("MissingSubmodel") ] }; - _aasRepositoryService.GetShellsByFiltersAsync(null, null, null, Arg.Any()) + _aasRepositoryService.GetShellsByFiltersAsync(null, Arg.Any(), Arg.Any(), Arg.Any()) .Returns(shells); _submodelTemplateMappingProvider.GetTemplateId("ValidSubmodelId").Returns("ValidSubmodelId"); _submodelTemplateMappingProvider.GetTemplateId("MissingSubmodel").Returns("MissingSubmodel"); @@ -111,7 +112,7 @@ public async Task GetAllSubmodelDescriptorsAsync_ThrowsSubmodelDescriptorNotFoun CreateShell("MissingSubmodel2") ] }; - _aasRepositoryService.GetShellsByFiltersAsync(null, null, null, Arg.Any()) + _aasRepositoryService.GetShellsByFiltersAsync(null, Arg.Any(), Arg.Any(), Arg.Any()) .Returns(shells); _submodelTemplateMappingProvider.GetTemplateId("MissingSubmodel1").Returns("MissingSubmodel1"); _submodelTemplateMappingProvider.GetTemplateId("MissingSubmodel2").Returns("MissingSubmodel2"); @@ -131,7 +132,7 @@ public async Task GetAllSubmodelDescriptorsAsync_ReturnsEmptyResult_WhenShellsHa { Result = [] }; - _aasRepositoryService.GetShellsByFiltersAsync(null, null, null, Arg.Any()) + _aasRepositoryService.GetShellsByFiltersAsync(null, Arg.Any(), Arg.Any(), Arg.Any()) .Returns(shells); var result = await _sut.GetAllSubmodelDescriptorsAsync(5, null, CancellationToken.None); @@ -277,10 +278,50 @@ public async Task GetSubmodelDescriptorByIdAsync_ThrowsRegistryNotAvailable_When Assert.IsType(ex); } - private static AssetAdministrationShell CreateShell(string submodelId) + [Fact] + public async Task GetAllSubmodelDescriptorsAsync_ResumesCorrectly_WithTwoFieldCompositeCursor() + { + var shell1 = CreateShell("ContactInformation", "shell-1"); + var shell2 = CreateShell("Nameplate", "shell-2"); + + var shellsPage1 = new Shells { Result = [shell1, shell2] }; + var shellsPage2 = new Shells { Result = [shell2] }; + + _aasRepositoryService.GetShellsByFiltersAsync(null, 1, Arg.Is(s => s == null), Arg.Any()) + .Returns(shellsPage1); + _aasRepositoryService.GetShellsByFiltersAsync(null, 1, Arg.Is(s => s != null), Arg.Any()) + .Returns(shellsPage2); + + _submodelTemplateMappingProvider.GetTemplateId("ContactInformation").Returns("ContactInformation"); + _submodelTemplateMappingProvider.GetTemplateId("Nameplate").Returns("Nameplate"); + _provider.GetDataForSubmodelDescriptorByIdAsync("ContactInformation", Arg.Any()) + .Returns(new SubmodelDescriptor { Id = "ContactInformation" }); + _provider.GetDataForSubmodelDescriptorByIdAsync("Nameplate", Arg.Any()) + .Returns(new SubmodelDescriptor { Id = "Nameplate" }); + + var page1 = await _sut.GetAllSubmodelDescriptorsAsync(1, null, CancellationToken.None); + + Assert.NotNull(page1); + Assert.Single(page1.Result!); + Assert.Equal("ContactInformation", page1.Result![0].Id); + Assert.NotNull(page1.PagingMetaData?.Cursor); + + var decodedCursor = SubmodelPaginationCursor.Decode(page1.PagingMetaData.Cursor); + Assert.NotNull(decodedCursor); + Assert.Equal("ContactInformation", decodedCursor.SubmodelId); + Assert.Equal("shell-1", decodedCursor.AasId); + + var page2 = await _sut.GetAllSubmodelDescriptorsAsync(1, page1.PagingMetaData.Cursor, CancellationToken.None); + + Assert.NotNull(page2); + Assert.Single(page2.Result!); + Assert.Equal("Nameplate", page2.Result![0].Id); + } + + private static AssetAdministrationShell CreateShell(string submodelId, string shellId = "shell-id") { return new AssetAdministrationShell( - id: "shell-id", + id: shellId, assetInformation: new AssetInformation(assetKind: AssetKind.Instance, globalAssetId: null), submodels: [ diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs index 983a68ef..1f7a0e1c 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs @@ -1,4 +1,4 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; @@ -8,6 +8,7 @@ using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.SubmodelRegistry.Providers; using AAS.TwinEngine.DataEngine.DomainModel.Shared; using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRegistry; +using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; using Microsoft.Extensions.Options; @@ -29,9 +30,9 @@ public class SubmodelDescriptorService( public async Task GetAllSubmodelDescriptorsAsync(int? limit, string? cursor, CancellationToken cancellationToken) { - var shells = await aasRepositoryService.GetShellsByFiltersAsync(null, null, null, cancellationToken).ConfigureAwait(false); - - var submodelIds = ExtractDistinctSubmodelIds(shells.Result); + var pageSize = limit ?? 100; + var paginationResult = await CollectSubmodelDescriptorPageAsync(pageSize, cursor, cancellationToken).ConfigureAwait(false); + var submodelIds = paginationResult.SubmodelIds; using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); var descriptorTasks = submodelIds.Select(async submodelId => @@ -62,12 +63,10 @@ public async Task GetAllSubmodelDescriptorsAsync(int? limit throw new SubmodelDescriptorNotFoundException(); } - var (pagedItems, pagingMetaData) = PagingExtensions.GetPagedResult(allDescriptors, d => d.Id!, limit, cursor); - return new SubmodelDescriptors { - PagingMetaData = pagingMetaData, - Result = pagedItems + PagingMetaData = new PagingMetaData { Cursor = paginationResult.NextCursor }, + Result = allDescriptors }; } @@ -139,15 +138,135 @@ private static void SetHref(EndpointData endpoint, string href) endpoint.ProtocolInformation.Href = href; } - private static IList ExtractDistinctSubmodelIds(IList? shells) + ///

+ /// Implements the two-field composite cursor pagination algorithm. + /// Iterates through AAS shell batches, expanding submodel refs per AAS, + /// and collects submodel IDs up to the requested page size. + /// + private async Task CollectSubmodelDescriptorPageAsync( + int pageSize, + string? encodedCursor, + CancellationToken cancellationToken) + { + var incomingCursor = SubmodelPaginationCursor.Decode(encodedCursor); + var state = new PaginationState(incomingCursor); + var pluginCursor = state.TrackingAasId; + + while (state.CollectedIds.Count < pageSize) + { + var shellsResult = await aasRepositoryService.GetShellsByFiltersAsync( + null, pageSize, pluginCursor?.EncodeBase64Url(), cancellationToken).ConfigureAwait(false); + + var shellList = shellsResult?.Result? + .Where(s => !string.IsNullOrWhiteSpace(s.Id)) + .ToList() ?? []; + + if (shellList.Count == 0) + { + break; + } + + var limitReached = ProcessShellBatch(shellList, pageSize, state); + + if (limitReached) + { + break; + } + + // Batch exhausted without reaching limit — check if repository has more data + if (shellsResult.PagingMetaData?.Cursor is null) + { + break; + } + + pluginCursor = state.TrackingAasId; + } + + var nextCursor = state.CollectedIds.Count >= pageSize + ? SubmodelPaginationCursor.Encode(state.LastCollectedSubmodelId, state.TrackingAasId) + : null; + + return new SubmodelDescriptorPageResult(state.CollectedIds, nextCursor); + } + + private static bool ProcessShellBatch( + List shellList, + int pageSize, + PaginationState state) + { + foreach (var shell in shellList) + { + var submodelIds = GetSubmodelIdsForShell(shell); + + if (submodelIds.Count == 0) + { + state.TrackingAasId = shell.Id; + state.IsFirstAasInResume = false; + continue; + } + + var startIndex = 0; + + // Skip-scan: only on the first AAS when resuming from a cursor + if (state.IsFirstAasInResume && state.SkipToSubmodelId is not null) + { + startIndex = submodelIds.IndexOf(state.SkipToSubmodelId) + 1; + state.IsFirstAasInResume = false; + state.SkipToSubmodelId = null; + } + else + { + state.IsFirstAasInResume = false; + } + + for (var i = startIndex; i < submodelIds.Count; i++) + { + state.CollectedIds.Add(submodelIds[i]); + state.LastCollectedSubmodelId = submodelIds[i]; + + if (state.CollectedIds.Count >= pageSize) + { + if (state.CollectedIds.Contains(submodelIds.Last())) + { + state.TrackingAasId = shell.Id; + } + + return true; + } + } + + state.TrackingAasId = shell.Id; + } + + return false; + } + + private static List GetSubmodelIdsForShell(AasCore.Aas3_1.IAssetAdministrationShell shell) { - return shells? - .SelectMany(shell => shell.Submodels ?? []) - .SelectMany(reference => reference.Keys ?? []) - .Select(key => key.Value) - .Where(value => !string.IsNullOrWhiteSpace(value)) - .Distinct(StringComparer.OrdinalIgnoreCase) - .ToList() ?? []; + return shell.Submodels? + .SelectMany(reference => reference.Keys ?? []) + .Select(key => key.Value) + .Where(value => !string.IsNullOrWhiteSpace(value)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList() ?? []; + } + + private sealed record SubmodelDescriptorPageResult(List SubmodelIds, string? NextCursor); + + private sealed class PaginationState + { + public List CollectedIds { get; } = []; + public string? TrackingAasId { get; set; } + public string? LastCollectedSubmodelId { get; set; } + public string? SkipToSubmodelId { get; set; } + public bool IsFirstAasInResume { get; set; } + + public PaginationState(SubmodelPaginationCursor? cursor) + { + TrackingAasId = cursor?.AasId; + SkipToSubmodelId = cursor?.SubmodelId; + IsFirstAasInResume = cursor is not null; + } } private string GenerateHref(string encodedId) => $"{_baseUrl}{ApiPaths.Submodels}/{encodedId}"; From 7405a8b06e74607da1274cfcbe7cd4c4efdd05b5 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Tue, 4 Aug 2026 11:35:54 +0530 Subject: [PATCH 18/30] feat: implement SubmodelDescriptorService and SubmodelRepositoryService with pagination and associated test data fixtures --- Pegination-story.md | 123 ------------------ .../SubmodelDescriptorService.cs | 45 ++----- .../SubmodelRepositoryService.cs | 47 ++----- ...bmodelDescriptors_WithCursor_Expected.json | 45 +++---- ...odelDescriptors_WithLimitOne_Expected.json | 4 +- .../GetAllSubmodels_ByIdShort_Expected.json | 12 +- ...etAllSubmodels_By_SemanticId_Expected.json | 12 +- .../TestData/GetAllSubmodels_Expected.json | 24 ++-- 8 files changed, 57 insertions(+), 255 deletions(-) delete mode 100644 Pegination-story.md diff --git a/Pegination-story.md b/Pegination-story.md deleted file mode 100644 index 53f71216..00000000 --- a/Pegination-story.md +++ /dev/null @@ -1,123 +0,0 @@ -### User Story1 - -**As a** DataEngine API consumer -**I want to** retrieve Submodels and Submodel Descriptors using efficient and consistent pagination -**So that** large digital twin datasets can be processed without requiring DataEngine to load and generate all Submodels before applying pagination - ---- - -### Acceptance Criteria -- [ ] DataEngine shall avoid requesting all Shell metadata from the Plugin when a cursor or limit is provided by the consumer. -- [ ] Plugin metadata/shell endpoints shall support pagination using aasId as a cursor. -- [ ] DataEngine shall be able to continue pagination correctly when the client provides a submodelId cursor. -- [ ] The pagination implementation shall preserve the logical ordering of generated Submodels within a Shell Template. -- [ ] The implementation shall ensure that no Submodels are skipped or duplicated across pages. -- [ ] Solutions for cursor translation between submodelId and aasId shall be documented and evaluated. -- [ ] Edge cases involving partially processed Shells shall be handled consistently. -- [ ] The selected approach shall be documented before implementation begins. - ---- - -### Concept - -Current implementation of both: - -GET /submodels -GET /submodel-descriptors - -does not leverage pagination during metadata retrieval. -Current flow: - -1. DataEngine requests all metadata/shells from Plugin. -2. For every Shell: - -- Resolve Shell Template using ShellTemplateMappingRule. -- Request Shell Template from Template Registry. -- Generate Submodel IDs based on Shell Template configuration. - -3. After generating all Submodel IDs: - -- Apply existing logic used by: - - Get Submodel by Id - - Get Submodel Descriptor by Id - -4. Finally return paginated response. - -This approach works for small datasets but becomes expensive when thousands of products/shells exist. - -Desired Improvement -Introduce pagination as early as possible in the processing pipeline. -**Instead of:** - -All Shells - ↓ -Generate All Submodels - ↓ -Apply Pagination - -**Use:** - -Paginated Shell Retrieval - ↓ -Generate Required Submodels - ↓ -Apply Pagination again (bcz we don't know which shell have how many submodel template ) - ↓ -Return Page - -**Key Design Challenge** - -Plugin pagination is based on: `aasId` -while DataEngine pagination is exposed using: `submodelId` -A mechanism is required to map: `submodelId -> aasId` - -to determine from which Shell Plugin pagination should continue - ---- - -### Additional Notes - -**Important Edge Case** - -Assume: - -``` -Product1 - ├─ Nameplate - ├─ TechnicalData - ├─ ContactInformation - └─ CarbonFootprint - -Product2 - ├─ Nameplate - ├─ TechnicalData - ├─ ContactInformation - └─ CarbonFootprint -``` -Consumer requests: `limit=12cursor=` - -Naive implementation: - -1. Resolve cursor to Product1 AAS. -2. Send Product1 AAS as cursor to Plugin. -3. Plugin returns Product2. - -Result: `Returned Item = Product2 Nameplate` -Expected result: `Returned Item = Product1 ContactInformation` - -because pagination should continue within the same Shell before moving to the next Shell. - ---- - -1. This PBI should initially focus on design and architecture evaluation before implementation. -2. A technical proposal should be produced describing: - -- Cursor structure. -- Ordering guarantees. -- Plugin API changes. -- Backward compatibility considerations. -- Performance impact. - -3.Final implementation should be done only after the pagination concept has been agreed upon. -4. Solution must work consistently for both: GET /submodels , GET /submodel-descriptors -5. Consider scenarios where Multiple Shell Templates exist. \ No newline at end of file diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs index 1f7a0e1c..d07b5dce 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs @@ -1,4 +1,4 @@ -using System.Collections.Concurrent; +using System.Collections.Concurrent; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; @@ -138,15 +138,7 @@ private static void SetHref(EndpointData endpoint, string href) endpoint.ProtocolInformation.Href = href; } - /// - /// Implements the two-field composite cursor pagination algorithm. - /// Iterates through AAS shell batches, expanding submodel refs per AAS, - /// and collects submodel IDs up to the requested page size. - /// - private async Task CollectSubmodelDescriptorPageAsync( - int pageSize, - string? encodedCursor, - CancellationToken cancellationToken) + private async Task CollectSubmodelDescriptorPageAsync(int pageSize, string? encodedCursor, CancellationToken cancellationToken) { var incomingCursor = SubmodelPaginationCursor.Decode(encodedCursor); var state = new PaginationState(incomingCursor); @@ -154,12 +146,9 @@ private async Task CollectSubmodelDescriptorPageAs while (state.CollectedIds.Count < pageSize) { - var shellsResult = await aasRepositoryService.GetShellsByFiltersAsync( - null, pageSize, pluginCursor?.EncodeBase64Url(), cancellationToken).ConfigureAwait(false); + var shellsResult = await aasRepositoryService.GetShellsByFiltersAsync(null, pageSize, pluginCursor?.EncodeBase64Url(), cancellationToken).ConfigureAwait(false); - var shellList = shellsResult?.Result? - .Where(s => !string.IsNullOrWhiteSpace(s.Id)) - .ToList() ?? []; + var shellList = shellsResult?.Result?.Where(s => !string.IsNullOrWhiteSpace(s.Id)).ToList() ?? []; if (shellList.Count == 0) { @@ -173,7 +162,6 @@ private async Task CollectSubmodelDescriptorPageAs break; } - // Batch exhausted without reaching limit — check if repository has more data if (shellsResult.PagingMetaData?.Cursor is null) { break; @@ -182,17 +170,12 @@ private async Task CollectSubmodelDescriptorPageAs pluginCursor = state.TrackingAasId; } - var nextCursor = state.CollectedIds.Count >= pageSize - ? SubmodelPaginationCursor.Encode(state.LastCollectedSubmodelId, state.TrackingAasId) - : null; + var nextCursor = state.CollectedIds.Count >= pageSize ? SubmodelPaginationCursor.Encode(state.LastCollectedSubmodelId, state.TrackingAasId) : null; return new SubmodelDescriptorPageResult(state.CollectedIds, nextCursor); } - private static bool ProcessShellBatch( - List shellList, - int pageSize, - PaginationState state) + private static bool ProcessShellBatch(List shellList, int pageSize, PaginationState state) { foreach (var shell in shellList) { @@ -207,7 +190,6 @@ private static bool ProcessShellBatch( var startIndex = 0; - // Skip-scan: only on the first AAS when resuming from a cursor if (state.IsFirstAasInResume && state.SkipToSubmodelId is not null) { startIndex = submodelIds.IndexOf(state.SkipToSubmodelId) + 1; @@ -253,20 +235,13 @@ private static List GetSubmodelIdsForShell(AasCore.Aas3_1.IAssetAdminist private sealed record SubmodelDescriptorPageResult(List SubmodelIds, string? NextCursor); - private sealed class PaginationState + private sealed class PaginationState(SubmodelPaginationCursor? cursor) { public List CollectedIds { get; } = []; - public string? TrackingAasId { get; set; } + public string? TrackingAasId { get; set; } = cursor?.AasId; public string? LastCollectedSubmodelId { get; set; } - public string? SkipToSubmodelId { get; set; } - public bool IsFirstAasInResume { get; set; } - - public PaginationState(SubmodelPaginationCursor? cursor) - { - TrackingAasId = cursor?.AasId; - SkipToSubmodelId = cursor?.SubmodelId; - IsFirstAasInResume = cursor is not null; - } + public string? SkipToSubmodelId { get; set; } = cursor?.SubmodelId; + public bool IsFirstAasInResume { get; set; } = cursor is not null; } private string GenerateHref(string encodedId) => $"{_baseUrl}{ApiPaths.Submodels}/{encodedId}"; diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 2204963f..043f879d 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -90,16 +90,7 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte }).ConfigureAwait(false); } - /// - /// Implements the two-field composite cursor pagination algorithm. - /// Iterates through plugin shell batches, expanding submodel refs per AAS, - /// and collects submodel IDs up to the requested page size. - /// - private async Task CollectSubmodelPageAsync( - ShellSearchFilter shellSearchFilter, - int pageSize, - string? encodedCursor, - CancellationToken cancellationToken) + private async Task CollectSubmodelPageAsync(ShellSearchFilter shellSearchFilter, int pageSize, string? encodedCursor, CancellationToken cancellationToken) { var incomingCursor = SubmodelPaginationCursor.Decode(encodedCursor); var state = new PaginationState(incomingCursor); @@ -110,9 +101,7 @@ private async Task CollectSubmodelPageAsync( var shellMetadata = await pluginDataHandler.GetDataForShellsByAssetIdsAsync( pluginManifestConflictHandler.Manifests, shellSearchFilter, pageSize, Base64UrlExtensions.EncodeBase64Url(pluginCursor), cancellationToken).ConfigureAwait(false); - var shellDescriptors = shellMetadata.ShellDescriptors? - .Where(s => !string.IsNullOrWhiteSpace(s.Id)) - .ToList() ?? []; + var shellDescriptors = shellMetadata.ShellDescriptors?.Where(s => !string.IsNullOrWhiteSpace(s.Id)).ToList() ?? []; if (shellDescriptors.Count == 0) { @@ -126,7 +115,6 @@ private async Task CollectSubmodelPageAsync( break; } - // Batch exhausted without reaching limit — check if plugin has more data if (shellMetadata.PagingMetaData?.Cursor is null) { break; @@ -135,18 +123,12 @@ private async Task CollectSubmodelPageAsync( pluginCursor = state.TrackingAasId; } - var nextCursor = state.CollectedIds.Count >= pageSize - ? SubmodelPaginationCursor.Encode(state.LastCollectedSubmodelId, state.TrackingAasId) - : null; + var nextCursor = state.CollectedIds.Count >= pageSize ? SubmodelPaginationCursor.Encode(state.LastCollectedSubmodelId, state.TrackingAasId) : null; return new SubmodelPageResult(state.CollectedIds, nextCursor); } - private async Task ProcessShellBatchAsync( - List shellDescriptors, - int pageSize, - PaginationState state, - CancellationToken cancellationToken) + private async Task ProcessShellBatchAsync(List shellDescriptors, int pageSize, PaginationState state, CancellationToken cancellationToken) { foreach (var shell in shellDescriptors) { @@ -161,7 +143,6 @@ private async Task ProcessShellBatchAsync( var startIndex = 0; - // Skip-scan: only on the first AAS when resuming from a cursor if (state.IsFirstAasInResume && state.SkipToSubmodelId is not null) { startIndex = submodelIds.IndexOf(state.SkipToSubmodelId) + 1; @@ -201,10 +182,7 @@ private async Task> GetSubmodelIdsForShellAsync(string shellId, Can { var references = await aasRepositoryTemplateService.GetSubmodelRefByIdAsync(shellId, cancellationToken).ConfigureAwait(false); - return references - .Select(r => r.Keys.FirstOrDefault()?.Value) - .Where(id => !string.IsNullOrWhiteSpace(id)) - .ToList()!; + return references.Select(r => r.Keys.FirstOrDefault()?.Value).Where(id => !string.IsNullOrWhiteSpace(id)).ToList()!; } catch (ResourceNotFoundException ex) { @@ -215,20 +193,13 @@ private async Task> GetSubmodelIdsForShellAsync(string shellId, Can private sealed record SubmodelPageResult(List SubmodelIds, string? NextCursor); - private sealed class PaginationState + private sealed class PaginationState(SubmodelPaginationCursor? cursor) { public List CollectedIds { get; } = []; - public string? TrackingAasId { get; set; } + public string? TrackingAasId { get; set; } = cursor?.AasId; public string? LastCollectedSubmodelId { get; set; } - public string? SkipToSubmodelId { get; set; } - public bool IsFirstAasInResume { get; set; } - - public PaginationState(SubmodelPaginationCursor? cursor) - { - TrackingAasId = cursor?.AasId; - SkipToSubmodelId = cursor?.SubmodelId; - IsFirstAasInResume = cursor is not null; - } + public string? SkipToSubmodelId { get; set; } = cursor?.SubmodelId; + public bool IsFirstAasInResume { get; set; } = cursor is not null; } private async Task> BuildSubmodelsAsync(IEnumerable submodelIds, string? filteredTemplateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRegistry/TestData/GetAllSubmodelDescriptors_WithCursor_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRegistry/TestData/GetAllSubmodelDescriptors_WithCursor_Expected.json index d6917b63..8e6d3e3d 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRegistry/TestData/GetAllSubmodelDescriptors_WithCursor_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRegistry/TestData/GetAllSubmodelDescriptors_WithCursor_Expected.json @@ -1,33 +1,25 @@ { - "paging_metadata": { - "cursor": "aHR0cHM6Ly9tbS1zb2Z0d2FyZS5jb20vc3VibW9kZWwvMDAwLTAwMi9DdXN0b21TdWJtb2RlbA" - }, + "paging_metadata": { "cursor": "aHR0cHM6Ly9tbS1zb2Z0d2FyZS5jb20vc3VibW9kZWwvMDAwLTAwMS9Db250YWN0SW5mb3JtYXRpb258" }, "result": [ { "description": [ { "language": "en", - "text": "The Submodel defines a set meta data for the handover of documentation from the manufacturer to the operator for industrial equipment" + "text": "The Submodel HierarchicalStructures identified by its semanticId. The Submodel idShort can be picked freely." } ], "displayName": null, "extensions": null, - "administration": { - "embeddedDataSpecifications": null, - "version": "2", - "revision": "0", - "creator": null, - "templateId": "https://admin-shell.io/idta-02004-2-0" - }, - "idShort": "HandoverDocumentation", - "id": "https://mm-software.com/submodel/000-001/HandoverDocumentation", + "administration": null, + "idShort": "CustomSubmodel", + "id": "https://mm-software.com/submodel/000-001/CustomSubmodel", "semanticId": { - "type": "ModelReference", + "type": "ExternalReference", "referredSemanticId": null, "keys": [ { - "type": "Submodel", - "value": "0173-1#01-AHF578#003" + "type": "GlobalReference", + "value": "https://admin-shell.io/idta/CustomSubmodel/Submodel/Template/0/1" } ] }, @@ -36,7 +28,7 @@ { "interface": "SUBMODEL-3.0", "protocolInformation": { - "href": "http://localhost:8080/submodels/aHR0cHM6Ly9tbS1zb2Z0d2FyZS5jb20vc3VibW9kZWwvMDAwLTAwMS9IYW5kb3ZlckRvY3VtZW50YXRpb24", + "href": "http://localhost:8080/submodels/aHR0cHM6Ly9tbS1zb2Z0d2FyZS5jb20vc3VibW9kZWwvMDAwLTAwMS9DdXN0b21TdWJtb2RlbA", "endpointProtocol": "http", "endpointProtocolVersion": null, "subprotocol": null, @@ -48,24 +40,19 @@ ] }, { - "description": [ - { - "language": "en", - "text": "The Submodel HierarchicalStructures identified by its semanticId. The Submodel idShort can be picked freely." - } - ], + "description": null, "displayName": null, "extensions": null, "administration": null, - "idShort": "CustomSubmodel", - "id": "https://mm-software.com/submodel/000-002/CustomSubmodel", + "idShort": "ContactInformations", + "id": "https://mm-software.com/submodel/000-001/ContactInformation", "semanticId": { - "type": "ExternalReference", + "type": "ModelReference", "referredSemanticId": null, "keys": [ { - "type": "GlobalReference", - "value": "https://admin-shell.io/idta/CustomSubmodel/Submodel/Template/0/1" + "type": "Submodel", + "value": "https://admin-shell.io/zvei/nameplate/1/0/ContactInformations" } ] }, @@ -74,7 +61,7 @@ { "interface": "SUBMODEL-3.0", "protocolInformation": { - "href": "http://localhost:8080/submodels/aHR0cHM6Ly9tbS1zb2Z0d2FyZS5jb20vc3VibW9kZWwvMDAwLTAwMi9DdXN0b21TdWJtb2RlbA", + "href": "http://localhost:8080/submodels/aHR0cHM6Ly9tbS1zb2Z0d2FyZS5jb20vc3VibW9kZWwvMDAwLTAwMS9Db250YWN0SW5mb3JtYXRpb24", "endpointProtocol": "http", "endpointProtocolVersion": null, "subprotocol": null, diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRegistry/TestData/GetAllSubmodelDescriptors_WithLimitOne_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRegistry/TestData/GetAllSubmodelDescriptors_WithLimitOne_Expected.json index 6f2f0df2..5a5a92c2 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRegistry/TestData/GetAllSubmodelDescriptors_WithLimitOne_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRegistry/TestData/GetAllSubmodelDescriptors_WithLimitOne_Expected.json @@ -1,7 +1,5 @@ { - "paging_metadata": { - "cursor": "aHR0cHM6Ly9tbS1zb2Z0d2FyZS5jb20vc3VibW9kZWwvMDAwLTAwMS9DdXN0b21TdWJtb2RlbA" - }, + "paging_metadata": { "cursor": "aHR0cHM6Ly9tbS1zb2Z0d2FyZS5jb20vc3VibW9kZWwvMDAwLTAwMS9DdXN0b21TdWJtb2RlbHw" }, "result": [ { "description": [ diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_ByIdShort_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_ByIdShort_Expected.json index 11ef07bf..0808fd81 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_ByIdShort_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_ByIdShort_Expected.json @@ -1,7 +1,5 @@ { - "paging_metadata": { - "cursor": null - }, + "paging_metadata": { "cursor": null }, "result": [ { "idShort": "CustomSubmodel", @@ -11,7 +9,7 @@ "text": "The Submodel HierarchicalStructures identified by its semanticId. The Submodel idShort can be picked freely." } ], - "id": "https://admin-shell.io/idta/CustomSubmodel/Template/0/1", + "id": "https://mm-software.com/submodel/000-003/CustomSubmodel", "kind": "Instance", "semanticId": { "type": "ExternalReference", @@ -1608,7 +1606,7 @@ }, { "idShort": "ContactInformations", - "id": "https://admin-shell.io/idta/SubmodelTemplate/ContactInformation/1/0", + "id": "https://mm-software.com/submodel/000-003/ContactInformation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -3456,7 +3454,7 @@ "revision": "0", "templateId": "https://admin-shell.io/idta-02004-2-0" }, - "id": "https://admin-shell.io/idta/SubmodelTemplate/HandoverDocumentation/2/0", + "id": "https://mm-software.com/submodel/000-003/HandoverDocumentation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -10167,4 +10165,4 @@ "modelType": "Submodel" } ] -} \ No newline at end of file +} diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_By_SemanticId_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_By_SemanticId_Expected.json index 4b3edcd7..162103a8 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_By_SemanticId_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_By_SemanticId_Expected.json @@ -1,11 +1,9 @@ { - "paging_metadata": { - "cursor": null - }, + "paging_metadata": { "cursor": null }, "result": [ { "idShort": "ContactInformations", - "id": "https://admin-shell.io/idta/SubmodelTemplate/ContactInformation/1/0", + "id": "https://mm-software.com/submodel/000-001/ContactInformation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -1310,7 +1308,7 @@ }, { "idShort": "ContactInformations", - "id": "https://admin-shell.io/idta/SubmodelTemplate/ContactInformation/1/0", + "id": "https://mm-software.com/submodel/000-002/ContactInformation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -2503,7 +2501,7 @@ }, { "idShort": "ContactInformations", - "id": "https://admin-shell.io/idta/SubmodelTemplate/ContactInformation/1/0", + "id": "https://mm-software.com/submodel/000-003/ContactInformation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -4339,4 +4337,4 @@ "modelType": "Submodel" } ] -} \ No newline at end of file +} diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_Expected.json index 7181875b..67eedfa7 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/TestData/GetAllSubmodels_Expected.json @@ -1,7 +1,5 @@ { - "paging_metadata": { - "cursor": null - }, + "paging_metadata": { "cursor": null }, "result": [ { "idShort": "CustomSubmodel", @@ -11,7 +9,7 @@ "text": "The Submodel HierarchicalStructures identified by its semanticId. The Submodel idShort can be picked freely." } ], - "id": "https://admin-shell.io/idta/CustomSubmodel/Template/0/1", + "id": "https://mm-software.com/submodel/000-001/CustomSubmodel", "kind": "Instance", "semanticId": { "type": "ExternalReference", @@ -1608,7 +1606,7 @@ }, { "idShort": "ContactInformations", - "id": "https://admin-shell.io/idta/SubmodelTemplate/ContactInformation/1/0", + "id": "https://mm-software.com/submodel/000-001/ContactInformation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -2924,7 +2922,7 @@ "revision": "0", "templateId": "https://admin-shell.io/idta-02004-2-0" }, - "id": "https://admin-shell.io/idta/SubmodelTemplate/HandoverDocumentation/2/0", + "id": "https://mm-software.com/submodel/000-001/HandoverDocumentation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -9886,7 +9884,7 @@ "text": "The Submodel HierarchicalStructures identified by its semanticId. The Submodel idShort can be picked freely." } ], - "id": "https://admin-shell.io/idta/CustomSubmodel/Template/0/1", + "id": "https://mm-software.com/submodel/000-002/CustomSubmodel", "kind": "Instance", "semanticId": { "type": "ExternalReference", @@ -11483,7 +11481,7 @@ }, { "idShort": "ContactInformations", - "id": "https://admin-shell.io/idta/SubmodelTemplate/ContactInformation/1/0", + "id": "https://mm-software.com/submodel/000-002/ContactInformation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -12687,7 +12685,7 @@ "revision": "0", "templateId": "https://admin-shell.io/idta-02004-2-0" }, - "id": "https://admin-shell.io/idta/SubmodelTemplate/HandoverDocumentation/2/0", + "id": "https://mm-software.com/submodel/000-002/HandoverDocumentation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -16355,7 +16353,7 @@ "text": "The Submodel HierarchicalStructures identified by its semanticId. The Submodel idShort can be picked freely." } ], - "id": "https://admin-shell.io/idta/CustomSubmodel/Template/0/1", + "id": "https://mm-software.com/submodel/000-003/CustomSubmodel", "kind": "Instance", "semanticId": { "type": "ExternalReference", @@ -17952,7 +17950,7 @@ }, { "idShort": "ContactInformations", - "id": "https://admin-shell.io/idta/SubmodelTemplate/ContactInformation/1/0", + "id": "https://mm-software.com/submodel/000-003/ContactInformation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -19800,7 +19798,7 @@ "revision": "0", "templateId": "https://admin-shell.io/idta-02004-2-0" }, - "id": "https://admin-shell.io/idta/SubmodelTemplate/HandoverDocumentation/2/0", + "id": "https://mm-software.com/submodel/000-003/HandoverDocumentation", "kind": "Instance", "semanticId": { "type": "ModelReference", @@ -26511,4 +26509,4 @@ "modelType": "Submodel" } ] -} \ No newline at end of file +} From b22b66aa53f12a410d1e0c1e78e64b62a4d35b38 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Tue, 4 Aug 2026 12:27:51 +0530 Subject: [PATCH 19/30] feat: implement plugin data provider interface and handler for managing semantic and shell descriptor data --- .../Services/Plugin/Providers/IPluginDataProvider.cs | 2 +- .../Providers/PluginDataProvider/Services/PluginDataHandler.cs | 2 +- .../Providers/PluginDataProvider/Services/PluginDataProvider.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs index b91a8cba..52bf7cb0 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Plugin/Providers/IPluginDataProvider.cs @@ -12,5 +12,5 @@ public interface IPluginDataProvider Task> GetDataForAssetInformationByIdAsync(IList pluginRequests, CancellationToken cancellationToken); - Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, int? limit, string? cursor, CancellationToken cancellationToken); + Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, int? limit, string? cursor, CancellationToken cancellationToken); } diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs index 646e1864..266b9bcc 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs @@ -224,7 +224,7 @@ public async Task GetDataForShellsByAssetIdsAsync(IRea })) : null; - var response = await pluginDataProvider.GetDataForShellDescriptorsByAssetIdsAsync(pluginRequests, assetIdsHeaderValue, filter?.IdShort, limit, cursor, cancellationToken).ConfigureAwait(false); + var responses = await pluginDataProvider.GetDataForShellDescriptorsByAssetIdsAsync(pluginRequests, assetIdsHeaderValue, filter?.IdShort, limit, cursor, cancellationToken).ConfigureAwait(false); var result = new ShellDescriptorsMetaData(); diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs index 74cadaaa..3f829624 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs @@ -113,7 +113,7 @@ public Task> GetDataForShellDescriptorByIdAsync(IList> GetDataForAssetInformationByIdAsync(IList pluginRequests, CancellationToken cancellationToken) => GetAndProcessAsync(pluginRequests, AssetInformationEndpoint, cancellationToken); - public async Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, int? limit, string? cursor, CancellationToken cancellationToken) + public async Task> GetDataForShellDescriptorsByAssetIdsAsync(IList pluginRequests, string? assetIdsHeaderValue, string? idShortHeaderValue, int? limit, string? cursor, CancellationToken cancellationToken) { var result = new List(); var exceptions = new List(); From 78bc9056d34ac3672e94287f2633e8bd47a4475a Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Mon, 17 Aug 2026 02:11:08 +0530 Subject: [PATCH 20/30] feat: enhance pagination and data retrieval logic across services and tests --- .../AasRepositoryServiceTests.cs | 4 +- .../Discovery/AssetIdSearchServiceTests.cs | 26 +++- .../SubmodelDescriptorServiceTests.cs | 86 ++++++++++++ .../SubmodelRepositoryServiceTests.cs | 45 +------ .../Services/PluginDataHandlerTests.cs | 39 +----- .../Services/PluginDataProviderTests.cs | 12 +- .../SubmodelDescriptorService.cs | 22 +-- .../SubmodelRepositoryService.cs | 125 ++++++++++++------ .../Services/PluginDataProvider.cs | 15 +-- .../Config/TemplateManagementConfig.cs | 2 +- .../appsettings.development.json | 10 +- 11 files changed, 234 insertions(+), 152 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs index 3ac04285..e686b9ab 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/AasRepository/AasRepositoryServiceTests.cs @@ -328,6 +328,8 @@ public async Task GetShellsByFiltersAsync_WithIdShort_QueriesFilteredShellMetada .GetDataForShellsByAssetIdsAsync( manifests, Arg.Is(f => f != null && f.IdShort == targetIdShort), + Arg.Any(), + Arg.Any(), cancellationToken) .Returns(new ShellDescriptorsMetaData { @@ -347,7 +349,7 @@ public async Task GetShellsByFiltersAsync_WithIdShort_QueriesFilteredShellMetada Assert.Single(result.Result); Assert.Equal("aas-1", result.Result[0].Id); await _pluginDataHandler.Received(1) - .GetDataForShellsByAssetIdsAsync(manifests, Arg.Is(f => f != null && f.IdShort == targetIdShort), cancellationToken); + .GetDataForShellsByAssetIdsAsync(manifests, Arg.Is(f => f != null && f.IdShort == targetIdShort), Arg.Any(), Arg.Any(), cancellationToken); } [Fact] diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs index d3634fd1..33475d7a 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/Discovery/AssetIdSearchServiceTests.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Base; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasRepository; @@ -51,6 +51,8 @@ public async Task SearchShellsByAssetLinkAsync_ReturnsAasIds() _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Returns(metadata); @@ -83,6 +85,8 @@ public async Task SearchShellsByAssetLinkAsync_WithPagination_ReturnsPagedResult _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Returns(metadata); @@ -102,6 +106,8 @@ public async Task SearchShellsByAssetLinkAsync_WhenPluginTimeout_ThrowsPluginNot _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Throws(new RequestTimeoutException()); @@ -119,6 +125,8 @@ public async Task SearchShellsByAssetLinkAsync_WhenUnauthorized_ThrowsServiceUnA _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Throws(new AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException()); @@ -136,6 +144,8 @@ public async Task SearchShellsByAssetLinkAsync_WhenResponseParsingError_ThrowsIn _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Throws(new ResponseParsingException()); @@ -154,6 +164,8 @@ public async Task SearchShellsByAssetLinkAsync_WhenMultiPluginConflict_ThrowsInt _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Throws(new MultiPluginConflictException()); @@ -171,6 +183,8 @@ public async Task SearchShellsByAssetLinkAsync_WhenResourceNotFound_ThrowsIntern _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Throws(new ResourceNotFoundException()); @@ -200,14 +214,16 @@ public async Task SearchShellsByAssetLinkAsync_FiltersOutEmptyIds() _ = _pluginDataHandler.GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Any(), + Arg.Any(), + Arg.Any(), Arg.Any()) .Returns(metadata); - var result3 = await _sut.SearchShellsByAssetLinkAsync(assetLinks, null, null, CancellationToken.None); + var result = await _sut.SearchShellsByAssetLinkAsync(assetLinks, null, null, CancellationToken.None); - Assert.Equal(2, result3.Result!.Count); - Assert.Equal("urn:example:aas:001", result3.Result![0]); - Assert.Equal("urn:example:aas:002", result3.Result![1]); + Assert.Equal(2, result.Result!.Count); + Assert.Equal("urn:example:aas:001", result.Result![0]); + Assert.Equal("urn:example:aas:002", result.Result![1]); } [Fact] diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorServiceTests.cs index 36d812c9..86c4d500 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorServiceTests.cs @@ -318,6 +318,79 @@ public async Task GetAllSubmodelDescriptorsAsync_ResumesCorrectly_WithTwoFieldCo Assert.Equal("Nameplate", page2.Result![0].Id); } + [Fact] + public async Task GetAllSubmodelDescriptorsAsync_SkipsShellsWithNullOrEmptyId() + { + var validShell = CreateShell("Nameplate", "valid-shell"); + var nullIdShell = new AssetAdministrationShell( + id: null!, + assetInformation: new AssetInformation(assetKind: AssetKind.Instance, globalAssetId: null), + submodels: [new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, "ShouldBeSkipped")])]); + + var shells = new Shells { Result = [nullIdShell, validShell] }; + _aasRepositoryService.GetShellsByFiltersAsync(null, Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(shells); + _submodelTemplateMappingProvider.GetTemplateId("Nameplate").Returns("Nameplate"); + _provider.GetDataForSubmodelDescriptorByIdAsync("Nameplate", Arg.Any()) + .Returns(new SubmodelDescriptor { Id = "Nameplate" }); + + var result = await _sut.GetAllSubmodelDescriptorsAsync(5, null, CancellationToken.None); + + Assert.Single(result.Result!); + Assert.Equal("Nameplate", result.Result![0].Id); + } + + [Fact] + public async Task GetAllSubmodelDescriptorsAsync_WhenLimitReachedAtAasBoundary_CursorAasIdIsConsumedAas() + { + var shell1 = CreateShellWithMultipleSubmodels("shell-1", "sm-1", "sm-2", "sm-3"); + var shell2 = CreateShellWithMultipleSubmodels("shell-2", "sm-4", "sm-5"); + + var shells = new Shells { Result = [shell1, shell2] }; + _aasRepositoryService.GetShellsByFiltersAsync(null, Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(shells); + + _submodelTemplateMappingProvider.GetTemplateId(Arg.Any()).Returns(x => (string)x[0]); + _provider.GetDataForSubmodelDescriptorByIdAsync(Arg.Any(), Arg.Any()) + .Returns(x => new SubmodelDescriptor { Id = (string)x[0] }); + + var result = await _sut.GetAllSubmodelDescriptorsAsync(5, null, CancellationToken.None); + + Assert.Equal(5, result.Result!.Count); + Assert.NotNull(result.PagingMetaData?.Cursor); + + var decoded = SubmodelPaginationCursor.Decode(result.PagingMetaData!.Cursor!); + Assert.Equal("sm-5", decoded!.SubmodelId); + Assert.Equal("shell-2", decoded.AasId); + } + + [Fact] + public async Task GetAllSubmodelDescriptorsAsync_WhenLimitReachedAtFirstAasBoundary_CursorAasIdIsConsumedAas() + { + var shell1 = CreateShellWithMultipleSubmodels("shell-1", "sm-1", "sm-2"); + + var shells = new Shells + { + Result = [shell1], + PagingMetaData = new PagingMetaData { Cursor = "next-page-token" } + }; + _aasRepositoryService.GetShellsByFiltersAsync(null, Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(shells); + + _submodelTemplateMappingProvider.GetTemplateId(Arg.Any()).Returns(x => (string)x[0]); + _provider.GetDataForSubmodelDescriptorByIdAsync(Arg.Any(), Arg.Any()) + .Returns(x => new SubmodelDescriptor { Id = (string)x[0] }); + + var result = await _sut.GetAllSubmodelDescriptorsAsync(2, null, CancellationToken.None); + + Assert.Equal(2, result.Result!.Count); + Assert.NotNull(result.PagingMetaData?.Cursor); + + var decoded = SubmodelPaginationCursor.Decode(result.PagingMetaData!.Cursor!); + Assert.Equal("sm-2", decoded!.SubmodelId); + Assert.Equal("shell-1", decoded.AasId); + } + private static AssetAdministrationShell CreateShell(string submodelId, string shellId = "shell-id") { return new AssetAdministrationShell( @@ -331,4 +404,17 @@ private static AssetAdministrationShell CreateShell(string submodelId, string sh referredSemanticId: null) ]); } + + private static AssetAdministrationShell CreateShellWithMultipleSubmodels(string shellId, params string[] submodelIds) + { + var refs = submodelIds.Select(id => (IReference)new Reference( + type: ReferenceTypes.ModelReference, + keys: [new Key(KeyTypes.Submodel, id)], + referredSemanticId: null)).ToList(); + + return new AssetAdministrationShell( + id: shellId, + assetInformation: new AssetInformation(assetKind: AssetKind.Instance, globalAssetId: null), + submodels: refs); + } } diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs index 4ff78d8c..089cbeca 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs @@ -331,43 +331,6 @@ public async Task GetAllSubmodelsAsync_SkipsShell_WhenSubmodelRefsFails() Assert.Empty(result.Result); } - [Fact] - public async Task GetAllSubmodelsAsync_DeduplicatesSubmodelIds_AcrossShells() - { - const string SubmodelId1 = "https://example.com/submodels/Shared"; - var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); - var filledSubmodel = TestData.CreateFilledSubmodel(); - - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new ShellDescriptorsMetaData - { - ShellDescriptors = - [ - new ShellDescriptorMetaData { Id = "https://example.com/shells/001" }, - new ShellDescriptorMetaData { Id = "https://example.com/shells/002" } - ] - }); - - _aasRepositoryTemplateService - .GetSubmodelRefByIdAsync(Arg.Any(), Arg.Any()) - .Returns([submodelRef]); - - _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId1, (string?)null, Arg.Any(), Arg.Any()) - .Returns(TestData.CreateSubmodel()); - - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(filledSubmodel); - - var result = await _sut.GetAllSubmodelsAsync(null, null, null, null, CancellationToken.None); - - Assert.Single(result.Result); - } - [Fact] public async Task GetAllSubmodelsAsync_FiltersShellsByIdShort_WhenIdShortProvided() { @@ -378,6 +341,8 @@ public async Task GetAllSubmodelsAsync_FiltersShellsByIdShort_WhenIdShortProvide .GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Is(f => f != null && f.IdShort == IdShort), + Arg.Any(), + Arg.Any(), Arg.Any()) .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [] }); @@ -387,6 +352,8 @@ await _pluginDataHandler.Received(1) .GetDataForShellsByAssetIdsAsync( Arg.Any>(), Arg.Is(f => f != null && f.IdShort == IdShort), + Arg.Any(), + Arg.Any(), Arg.Any()); } @@ -526,7 +493,7 @@ public async Task GetAllSubmodelsAsync_WhenLimitReachedAtAasBoundary_CursorSubmo ]); _templateService - .GetFilteredSubmodelTemplateAsync(Arg.Any(), (string?)null, Arg.Any(), Arg.Any()) + .GetFilteredSubmodelTemplateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(TestData.CreateSubmodel()); _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); _pluginDataHandler @@ -569,7 +536,7 @@ public async Task GetAllSubmodelsAsync_WhenLimitReachedAtFirstAasBoundary_Cursor ]); _templateService - .GetFilteredSubmodelTemplateAsync(Arg.Any(), (string?)null, Arg.Any(), Arg.Any()) + .GetFilteredSubmodelTemplateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(TestData.CreateSubmodel()); _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); _pluginDataHandler diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs index 86eada3d..1d86d6a7 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs @@ -624,14 +624,9 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_ReturnsShellDescript } """; - var httpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(responseJson, Encoding.UTF8, "application/json") - }; - _pluginDataProvider .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new List { httpResponse.Content }); + .Returns(new List { responseJson }); const string Json = """[{"name":"sn","value":"123"}]"""; @@ -667,14 +662,9 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenNullDeserializat _pluginRequestBuilder.Build(Arg.Any>()) .Returns([new($"{HttpClientNames.PluginDataProviderPrefix}PluginA", "")]); - var httpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("null", Encoding.UTF8, "application/json") - }; - _pluginDataProvider .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns([httpResponse.Content]); + .Returns(new List { "null" }); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => @@ -704,14 +694,9 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenInvalidJson_Thro _pluginRequestBuilder.Build(Arg.Any>()) .Returns([new($"{HttpClientNames.PluginDataProviderPrefix}PluginA", "")]); - var httpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("invalid json!", Encoding.UTF8, "application/json") - }; - _pluginDataProvider .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns([httpResponse.Content]); + .Returns(new List { "invalid json!" }); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; await Assert.ThrowsAsync(() => @@ -751,14 +736,9 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_SetsHrefOnResults() } """; - var httpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(responseJson, Encoding.UTF8, "application/json") - }; - _pluginDataProvider .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns([httpResponse.Content]); + .Returns(new List { responseJson }); var filter = new ShellSearchFilter { SpecificAssetIds = [] }; var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); @@ -821,16 +801,11 @@ public async Task GetDataForShellsByAssetIdsAsync_WithIdShort_PassesIdShort() } """; - var httpResponse = new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(responseJson, Encoding.UTF8, "application/json") - }; - const string targetIdShort = "Motor001"; _pluginDataProvider - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), null, targetIdShort, Arg.Any(), Arg.Any(), Arg.Any()) - .Returns([httpResponse.Content]); + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Is(s => s == null), targetIdShort, Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new List { responseJson }); var filter = new ShellSearchFilter { IdShort = targetIdShort }; var result = await _sut.GetDataForShellsByAssetIdsAsync(manifests, filter, null, null, CancellationToken.None); @@ -839,7 +814,7 @@ public async Task GetDataForShellsByAssetIdsAsync_WithIdShort_PassesIdShort() Assert.Single(result.ShellDescriptors); Assert.Equal("urn:aas:001", result.ShellDescriptors[0].Id); await _pluginDataProvider.Received(1) - .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), null, targetIdShort, Arg.Any(), Arg.Any(), Arg.Any()); + .GetDataForShellDescriptorsByAssetIdsAsync(Arg.Any>(), Arg.Is(s => s == null), targetIdShort, Arg.Any(), Arg.Any(), Arg.Any()); } private const string AssetData = """ diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs index 1082e60f..825bf51b 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProviderTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Net.Http.Json; using System.Text; using System.Text.Json; @@ -461,7 +461,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenNotFound_ThrowsR var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); } [Fact] @@ -481,7 +481,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenUnauthorized_Thr var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); } [Fact] @@ -495,7 +495,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenTimeout_ThrowsRe var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); } [Fact] @@ -504,7 +504,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WithNullRequest_Skip var metadata = new List { null! }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); } [Fact] @@ -524,7 +524,7 @@ public async Task GetDataForShellDescriptorsByAssetIdsAsync_WhenForbidden_Throws var metadata = new List { new(httpClientName, "") }; await Assert.ThrowsAsync(() => - _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, CancellationToken.None)); + _sut.GetDataForShellDescriptorsByAssetIdsAsync(metadata, "[]", null, null, null, CancellationToken.None)); } private static List GetTestShellDescriptorDataList() diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs index d07b5dce..c2de3d85 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs @@ -1,6 +1,4 @@ -using System.Collections.Concurrent; - -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasEnvironment.Providers; @@ -184,21 +182,16 @@ private static bool ProcessShellBatch(List= pageSize) { - if (state.CollectedIds.Contains(submodelIds.Last())) + if (submodelIds[^1] == state.LastCollectedSubmodelId) { state.TrackingAasId = shell.Id; } @@ -240,8 +233,7 @@ private sealed class PaginationState(SubmodelPaginationCursor? cursor) public List CollectedIds { get; } = []; public string? TrackingAasId { get; set; } = cursor?.AasId; public string? LastCollectedSubmodelId { get; set; } - public string? SkipToSubmodelId { get; set; } = cursor?.SubmodelId; - public bool IsFirstAasInResume { get; set; } = cursor is not null; + public string? ResumeAfterSubmodelId { get; set; } = cursor?.SubmodelId; } private string GenerateHref(string encodedId) => $"{_baseUrl}{ApiPaths.Submodels}/{encodedId}"; diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 043f879d..0d983be7 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -93,7 +93,7 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte private async Task CollectSubmodelPageAsync(ShellSearchFilter shellSearchFilter, int pageSize, string? encodedCursor, CancellationToken cancellationToken) { var incomingCursor = SubmodelPaginationCursor.Decode(encodedCursor); - var state = new PaginationState(incomingCursor); + var state = new PaginationState(incomingCursor, pageSize); var pluginCursor = state.TrackingAasId; while (state.CollectedIds.Count < pageSize) @@ -101,9 +101,8 @@ private async Task CollectSubmodelPageAsync(ShellSearchFilte var shellMetadata = await pluginDataHandler.GetDataForShellsByAssetIdsAsync( pluginManifestConflictHandler.Manifests, shellSearchFilter, pageSize, Base64UrlExtensions.EncodeBase64Url(pluginCursor), cancellationToken).ConfigureAwait(false); - var shellDescriptors = shellMetadata.ShellDescriptors?.Where(s => !string.IsNullOrWhiteSpace(s.Id)).ToList() ?? []; - - if (shellDescriptors.Count == 0) + var shellDescriptors = shellMetadata.ShellDescriptors; + if (shellDescriptors is null || shellDescriptors.Count == 0) { break; } @@ -128,30 +127,48 @@ private async Task CollectSubmodelPageAsync(ShellSearchFilte return new SubmodelPageResult(state.CollectedIds, nextCursor); } - private async Task ProcessShellBatchAsync(List shellDescriptors, int pageSize, PaginationState state, CancellationToken cancellationToken) + private async Task ProcessShellBatchAsync(IReadOnlyList shellDescriptors, int pageSize, PaginationState state, CancellationToken cancellationToken) { - foreach (var shell in shellDescriptors) + var prefetchTasks = new Task>[shellDescriptors.Count]; + using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); + + for (var idx = 0; idx < shellDescriptors.Count; idx++) { - var submodelIds = await GetSubmodelIdsForShellAsync(shell.Id!, cancellationToken).ConfigureAwait(false); + var shellId = shellDescriptors[idx].Id; + if (string.IsNullOrWhiteSpace(shellId)) + { + prefetchTasks[idx] = Task.FromResult>([]); + continue; + } - if (submodelIds.Count == 0) + prefetchTasks[idx] = PrefetchSubmodelIdsAsync(shellId, semaphore, cancellationToken); + } + + var allSubmodelIds = await Task.WhenAll(prefetchTasks).ConfigureAwait(false); + + for (var idx = 0; idx < shellDescriptors.Count; idx++) + { + var shellId = shellDescriptors[idx].Id; + if (string.IsNullOrWhiteSpace(shellId)) { - state.TrackingAasId = shell.Id; - state.IsFirstAasInResume = false; continue; } - var startIndex = 0; + var submodelIds = allSubmodelIds[idx]; - if (state.IsFirstAasInResume && state.SkipToSubmodelId is not null) + if (submodelIds.Count == 0) { - startIndex = submodelIds.IndexOf(state.SkipToSubmodelId) + 1; - state.IsFirstAasInResume = false; - state.SkipToSubmodelId = null; + state.TrackingAasId = shellId; + state.ResumeAfterSubmodelId = null; + continue; } - else + + var startIndex = 0; + + if (state.ResumeAfterSubmodelId is not null) { - state.IsFirstAasInResume = false; + startIndex = submodelIds.IndexOf(state.ResumeAfterSubmodelId) + 1; + state.ResumeAfterSubmodelId = null; } for (var i = startIndex; i < submodelIds.Count; i++) @@ -161,21 +178,34 @@ private async Task ProcessShellBatchAsync(List sh if (state.CollectedIds.Count >= pageSize) { - if (state.CollectedIds.Contains(submodelIds.Last())) + if (submodelIds[^1] == state.LastCollectedSubmodelId) { - state.TrackingAasId = shell.Id; + state.TrackingAasId = shellId; } return true; } } - state.TrackingAasId = shell.Id; + state.TrackingAasId = shellId; } return false; } + private async Task> PrefetchSubmodelIdsAsync(string shellId, SemaphoreSlim semaphore, CancellationToken cancellationToken) + { + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + return await GetSubmodelIdsForShellAsync(shellId, cancellationToken).ConfigureAwait(false); + } + finally + { + _ = semaphore.Release(); + } + } + private async Task> GetSubmodelIdsForShellAsync(string shellId, CancellationToken cancellationToken) { try @@ -193,41 +223,56 @@ private async Task> GetSubmodelIdsForShellAsync(string shellId, Can private sealed record SubmodelPageResult(List SubmodelIds, string? NextCursor); - private sealed class PaginationState(SubmodelPaginationCursor? cursor) + private sealed class PaginationState(SubmodelPaginationCursor? cursor, int capacity) { - public List CollectedIds { get; } = []; + public List CollectedIds { get; } = new(capacity); public string? TrackingAasId { get; set; } = cursor?.AasId; public string? LastCollectedSubmodelId { get; set; } - public string? SkipToSubmodelId { get; set; } = cursor?.SubmodelId; - public bool IsFirstAasInResume { get; set; } = cursor is not null; + public string? ResumeAfterSubmodelId { get; set; } = cursor?.SubmodelId; } - private async Task> BuildSubmodelsAsync(IEnumerable submodelIds, string? filteredTemplateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) + private async Task> BuildSubmodelsAsync(List submodelIds, string? filteredTemplateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) { using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); - var tasks = submodelIds.Select(async submodelId => + var tasks = new Task[submodelIds.Count]; + + for (var i = 0; i < submodelIds.Count; i++) { - await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try + tasks[i] = BuildSingleSubmodelAsync(submodelIds[i], filteredTemplateId, queryOptions, semaphore, cancellationToken); + } + + var results = await Task.WhenAll(tasks).ConfigureAwait(false); + + var submodels = new List(results.Length); + foreach (var result in results) + { + if (result is not null) { - var template = await submodelTemplateService.GetFilteredSubmodelTemplateAsync(submodelId, filteredTemplateId, queryOptions, cancellationToken).ConfigureAwait(false); + submodels.Add(result); + } + } - if (template is null) - { - return null; - } + return submodels; + } - var submodel = await BuildSubmodelWithValuesAsync(template, submodelId, cancellationToken).ConfigureAwait(false); + private async Task BuildSingleSubmodelAsync(string submodelId, string? filteredTemplateId, SubmodelQueryOptions? queryOptions, SemaphoreSlim semaphore, CancellationToken cancellationToken) + { + await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var template = await submodelTemplateService.GetFilteredSubmodelTemplateAsync(submodelId, filteredTemplateId, queryOptions, cancellationToken).ConfigureAwait(false); - return submodel; - } - finally + if (template is null) { - _ = semaphore.Release(); + return null; } - }); - return [.. (await Task.WhenAll(tasks).ConfigureAwait(false)).OfType()]; + return await BuildSubmodelWithValuesAsync(template, submodelId, cancellationToken).ConfigureAwait(false); + } + finally + { + semaphore.Release(); + } } public async Task GetAllSubmodelElementsAsync(string submodelId, SubmodelQueryOptions? queryOptions, int? limit, string? cursor, CancellationToken cancellationToken) diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs index 3f829624..fe478e31 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs @@ -31,24 +31,24 @@ public async Task> GetDataForSemanticIdsAsync(IList(); - foreach (var pluginRequest in pluginRequests) + + var tasks = pluginRequests.Select(async pluginRequest => { using var httpClient = CreateClient(pluginRequest.HttpClientName); try { using var response = await httpClient.PostAsync(relativeUri, pluginRequest.JsonSchema, cancellationToken).ConfigureAwait(false); - var processedResponse = await ProcessResponseAsync(response, url, cancellationToken).ConfigureAwait(false); - result.Add(processedResponse); + return await ProcessResponseAsync(response, url, cancellationToken).ConfigureAwait(false); } catch (TaskCanceledException) { logger.LogError("Request timed out. Endpoint: {Url}", url); throw new RequestTimeoutException(); } - } + }); - return result; + var results = await Task.WhenAll(tasks).ConfigureAwait(false); + return results.ToList(); } public async Task> GetDataForAllShellDescriptorsAsync( @@ -121,7 +121,6 @@ public async Task> GetDataForShellDescriptorsByAssetIdsAsync(IList foreach (var pluginRequest in pluginRequests) { var url = BuildShellsByAssetIdsUrl(limit, cursor); - if (pluginRequest == null) { logger.LogWarning("Plugin request is null. Skipping request to {Url}", url); @@ -278,7 +277,7 @@ private static string BuildShellsUrl(int? limit, string? cursor) ? QueryHelpers.AddQueryString(BaseUrl, queryParams!) : BaseUrl; } - + private static string BuildShellsByAssetIdsUrl(int? limit, string? cursor) { const string BaseUrl = $"/{ApiPaths.PluginMetadata}/{ShellsEndpoint}"; diff --git a/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/TemplateManagementConfig.cs b/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/TemplateManagementConfig.cs index 8eb09ae2..1e34e966 100644 --- a/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/TemplateManagementConfig.cs +++ b/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/TemplateManagementConfig.cs @@ -39,6 +39,6 @@ public class ServiceInstance public Uri? BaseUrl { get; set; } public IList HeaderMappings { get; init; } = []; public string HealthEndpoint { get; set; } = string.Empty; - public int ConcurrentOperationsLimit { get; set; } = 10; + public int ConcurrentOperationsLimit { get; set; } = 100; public int LocalCacheExpirationInMinutes { get; set; } = 5; } diff --git a/source/AAS.TwinEngine.DataEngine/appsettings.development.json b/source/AAS.TwinEngine.DataEngine/appsettings.development.json index 1e17c008..b7c5b69b 100644 --- a/source/AAS.TwinEngine.DataEngine/appsettings.development.json +++ b/source/AAS.TwinEngine.DataEngine/appsettings.development.json @@ -183,7 +183,7 @@ "Name": "AasTemplateRepository", "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", - "ConcurrentOperationsLimit": 10, + "ConcurrentOperationsLimit": 100, "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { @@ -202,7 +202,7 @@ "Name": "SubmodelTemplateRepository", "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", - "ConcurrentOperationsLimit": 10, + "ConcurrentOperationsLimit": 100, "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { @@ -221,7 +221,7 @@ "Name": "ConceptDescriptionTemplateRepository", "baseUrl": "http://localhost:8081", "healthEndpoint": "/actuator/health", - "ConcurrentOperationsLimit": 10, + "ConcurrentOperationsLimit": 100, "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { @@ -240,7 +240,7 @@ "Name": "AasTemplateRegistry", "baseUrl": "http://localhost:8082", "healthEndpoint": "/actuator/health", - "ConcurrentOperationsLimit": 10, + "ConcurrentOperationsLimit": 100, "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { @@ -254,7 +254,7 @@ "Name": "SubmodelTemplateRegistry", "baseUrl": "http://localhost:8083", "healthEndpoint": "/actuator/health", - "ConcurrentOperationsLimit": 12, + "ConcurrentOperationsLimit": 100, "LocalCacheExpirationInMinutes": 5, "headerMappings": [ { From 5f84d229dd0b699f1522a1aa1b8ae8e9be749dc2 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Wed, 19 Aug 2026 10:50:13 +0530 Subject: [PATCH 21/30] feat: refactor pagination logic and introduce SubmodelPaginationState for improved data handling --- .../Services/Shared/SubmodelPagination.cs | 53 +++++++++++++++++++ .../SubmodelDescriptorService.cs | 53 +++---------------- .../SubmodelRepositoryService.cs | 50 +++-------------- 3 files changed, 66 insertions(+), 90 deletions(-) create mode 100644 source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Shared/SubmodelPagination.cs diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Shared/SubmodelPagination.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Shared/SubmodelPagination.cs new file mode 100644 index 00000000..7e677249 --- /dev/null +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Shared/SubmodelPagination.cs @@ -0,0 +1,53 @@ +using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; + +namespace AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Shared; + +internal sealed record SubmodelPageResult(List SubmodelIds, string? NextCursor); + +internal sealed class SubmodelPaginationState(SubmodelPaginationCursor? cursor, int capacity = 0) +{ + public List CollectedIds { get; } = capacity > 0 ? new(capacity) : []; + public string? TrackingAasId { get; set; } = cursor?.AasId; + public string? LastCollectedSubmodelId { get; set; } + public string? ResumeAfterSubmodelId { get; set; } = cursor?.SubmodelId; + + public bool CollectSubmodelIds(IList submodelIds, string shellId, int pageSize) + { + if (submodelIds.Count == 0) + { + TrackingAasId = shellId; + ResumeAfterSubmodelId = null; + return false; + } + + var startIndex = 0; + + if (ResumeAfterSubmodelId is not null) + { + startIndex = submodelIds.IndexOf(ResumeAfterSubmodelId) + 1; + ResumeAfterSubmodelId = null; + } + + for (var i = startIndex; i < submodelIds.Count; i++) + { + CollectedIds.Add(submodelIds[i]); + LastCollectedSubmodelId = submodelIds[i]; + + if (CollectedIds.Count >= pageSize) + { + if (submodelIds[^1] == LastCollectedSubmodelId) + { + TrackingAasId = shellId; + } + + return true; + } + } + + TrackingAasId = shellId; + return false; + } + + public string? BuildNextCursor(int pageSize) => + CollectedIds.Count >= pageSize ? SubmodelPaginationCursor.Encode(LastCollectedSubmodelId, TrackingAasId) : null; +} diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs index c2de3d85..94dd1124 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs @@ -3,6 +3,7 @@ using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasEnvironment.Providers; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasRepository; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Shared; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.SubmodelRegistry.Providers; using AAS.TwinEngine.DataEngine.DomainModel.Shared; using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRegistry; @@ -136,10 +137,10 @@ private static void SetHref(EndpointData endpoint, string href) endpoint.ProtocolInformation.Href = href; } - private async Task CollectSubmodelDescriptorPageAsync(int pageSize, string? encodedCursor, CancellationToken cancellationToken) + private async Task CollectSubmodelDescriptorPageAsync(int pageSize, string? encodedCursor, CancellationToken cancellationToken) { var incomingCursor = SubmodelPaginationCursor.Decode(encodedCursor); - var state = new PaginationState(incomingCursor); + var state = new SubmodelPaginationState(incomingCursor); var pluginCursor = state.TrackingAasId; while (state.CollectedIds.Count < pageSize) @@ -168,49 +169,19 @@ private async Task CollectSubmodelDescriptorPageAs pluginCursor = state.TrackingAasId; } - var nextCursor = state.CollectedIds.Count >= pageSize ? SubmodelPaginationCursor.Encode(state.LastCollectedSubmodelId, state.TrackingAasId) : null; - - return new SubmodelDescriptorPageResult(state.CollectedIds, nextCursor); + return new SubmodelPageResult(state.CollectedIds, state.BuildNextCursor(pageSize)); } - private static bool ProcessShellBatch(List shellList, int pageSize, PaginationState state) + private static bool ProcessShellBatch(List shellList, int pageSize, SubmodelPaginationState state) { foreach (var shell in shellList) { var submodelIds = GetSubmodelIdsForShell(shell); - if (submodelIds.Count == 0) - { - state.TrackingAasId = shell.Id; - state.ResumeAfterSubmodelId = null; - continue; - } - - var startIndex = 0; - - if (state.ResumeAfterSubmodelId is not null) + if (state.CollectSubmodelIds(submodelIds, shell.Id!, pageSize)) { - startIndex = submodelIds.IndexOf(state.ResumeAfterSubmodelId) + 1; - state.ResumeAfterSubmodelId = null; + return true; } - - for (var i = startIndex; i < submodelIds.Count; i++) - { - state.CollectedIds.Add(submodelIds[i]); - state.LastCollectedSubmodelId = submodelIds[i]; - - if (state.CollectedIds.Count >= pageSize) - { - if (submodelIds[^1] == state.LastCollectedSubmodelId) - { - state.TrackingAasId = shell.Id; - } - - return true; - } - } - - state.TrackingAasId = shell.Id; } return false; @@ -226,15 +197,5 @@ private static List GetSubmodelIdsForShell(AasCore.Aas3_1.IAssetAdminist .ToList() ?? []; } - private sealed record SubmodelDescriptorPageResult(List SubmodelIds, string? NextCursor); - - private sealed class PaginationState(SubmodelPaginationCursor? cursor) - { - public List CollectedIds { get; } = []; - public string? TrackingAasId { get; set; } = cursor?.AasId; - public string? LastCollectedSubmodelId { get; set; } - public string? ResumeAfterSubmodelId { get; set; } = cursor?.SubmodelId; - } - private string GenerateHref(string encodedId) => $"{_baseUrl}{ApiPaths.Submodels}/{encodedId}"; } diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 5d90c445..ec610e3a 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -3,6 +3,7 @@ using AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasRepository; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Plugin; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Shared; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Shared.Providers; using AAS.TwinEngine.DataEngine.DomainModel.AasRegistry; using AAS.TwinEngine.DataEngine.DomainModel.AasRepository; @@ -98,7 +99,7 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte private async Task CollectSubmodelPageAsync(ShellSearchFilter shellSearchFilter, int pageSize, string? encodedCursor, CancellationToken cancellationToken) { var incomingCursor = SubmodelPaginationCursor.Decode(encodedCursor); - var state = new PaginationState(incomingCursor, pageSize); + var state = new SubmodelPaginationState(incomingCursor, pageSize); var pluginCursor = state.TrackingAasId; while (state.CollectedIds.Count < pageSize) @@ -127,12 +128,10 @@ private async Task CollectSubmodelPageAsync(ShellSearchFilte pluginCursor = state.TrackingAasId; } - var nextCursor = state.CollectedIds.Count >= pageSize ? SubmodelPaginationCursor.Encode(state.LastCollectedSubmodelId, state.TrackingAasId) : null; - - return new SubmodelPageResult(state.CollectedIds, nextCursor); + return new SubmodelPageResult(state.CollectedIds, state.BuildNextCursor(pageSize)); } - private async Task ProcessShellBatchAsync(IReadOnlyList shellDescriptors, int pageSize, PaginationState state, CancellationToken cancellationToken) + private async Task ProcessShellBatchAsync(IReadOnlyList shellDescriptors, int pageSize, SubmodelPaginationState state, CancellationToken cancellationToken) { var prefetchTasks = new Task>[shellDescriptors.Count]; using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); @@ -161,38 +160,10 @@ private async Task ProcessShellBatchAsync(IReadOnlyList= pageSize) - { - if (submodelIds[^1] == state.LastCollectedSubmodelId) - { - state.TrackingAasId = shellId; - } - - return true; - } - } - - state.TrackingAasId = shellId; } return false; @@ -226,15 +197,6 @@ private async Task> GetSubmodelIdsForShellAsync(string shellId, Can } } - private sealed record SubmodelPageResult(List SubmodelIds, string? NextCursor); - - private sealed class PaginationState(SubmodelPaginationCursor? cursor, int capacity) - { - public List CollectedIds { get; } = new(capacity); - public string? TrackingAasId { get; set; } = cursor?.AasId; - public string? LastCollectedSubmodelId { get; set; } - public string? ResumeAfterSubmodelId { get; set; } = cursor?.SubmodelId; - } private async Task> BuildSubmodelsAsync(List submodelIds, string? filteredTemplateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) { From d4a5c3ac88ffb4de61a7bbc5f96588ddb8e959d7 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Wed, 19 Aug 2026 11:33:31 +0530 Subject: [PATCH 22/30] fix: update null handling in results and improve query parameter usage --- .../Services/AasRepository/AasRepositoryService.cs | 2 +- .../SubmodelRegistry/SubmodelDescriptorService.cs | 2 +- .../SubmodelRepository/SubmodelRepositoryService.cs | 10 ++-------- .../PluginDataProvider/Services/PluginDataProvider.cs | 4 ++-- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs index 5d1e6d37..95ea9685 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs @@ -373,7 +373,7 @@ private async Task> BuildShellsAsync(IEnumerable var results = await Task.WhenAll(tasks).ConfigureAwait(false); - return [.. results.Where(s => s is not null).Select(s => s!)]; + return [.. results.Where(s => s is not null).Select(s => s)]; } private static IList FilterByExternalSubjectId(IList shells, IList? filters) diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs index 94dd1124..152de5fb 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs @@ -178,7 +178,7 @@ private static bool ProcessShellBatch(List> GetSubmodelIdsForShellAsync(string shellId, Can { var references = await aasRepositoryTemplateService.GetSubmodelRefByIdAsync(shellId, cancellationToken).ConfigureAwait(false); - return references.Select(r => r.Keys.FirstOrDefault()?.Value).Where(id => !string.IsNullOrWhiteSpace(id)).ToList()!; + return references.Select(r => r.Keys.FirstOrDefault()?.Value).Where(id => !string.IsNullOrWhiteSpace(id)).ToList(); } catch (ResourceNotFoundException ex) { @@ -211,13 +211,7 @@ private async Task> BuildSubmodelsAsync(List submodelIds var results = await Task.WhenAll(tasks).ConfigureAwait(false); var submodels = new List(results.Length); - foreach (var result in results) - { - if (result is not null) - { - submodels.Add(result); - } - } + submodels.AddRange(results.Where(result => result is not null)); return submodels; } diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs index fe478e31..0eb5d897 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataProvider.cs @@ -277,7 +277,7 @@ private static string BuildShellsUrl(int? limit, string? cursor) ? QueryHelpers.AddQueryString(BaseUrl, queryParams!) : BaseUrl; } - + private static string BuildShellsByAssetIdsUrl(int? limit, string? cursor) { const string BaseUrl = $"/{ApiPaths.PluginMetadata}/{ShellsEndpoint}"; @@ -294,7 +294,7 @@ private static string BuildShellsByAssetIdsUrl(int? limit, string? cursor) } return queryParams.Count > 0 - ? QueryHelpers.AddQueryString(BaseUrl, queryParams!) + ? QueryHelpers.AddQueryString(BaseUrl, queryParams) : BaseUrl; } From 88e6d2e153e4d2abc07fd61be6d800dffce1ca13 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Wed, 19 Aug 2026 11:51:09 +0530 Subject: [PATCH 23/30] feat: enhance metadata retrieval by adding pagination parameters to GetFilteredMetadataAsync --- .../Services/AasRepository/AasRepositoryService.cs | 2 +- .../Services/Discovery/AssetIdSearchService.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs index 95ea9685..12245c0a 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/AasRepository/AasRepositoryService.cs @@ -328,7 +328,7 @@ private void FillShellFromMetadata(IAssetAdministrationShell shell, ShellDescrip CancellationToken cancellationToken) { var metadata = await pluginDataHandler - .GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, filter, null, null, cancellationToken) + .GetDataForShellsByAssetIdsAsync(pluginManifestConflictHandler.Manifests, filter, limit, cursor, cancellationToken) .ConfigureAwait(false); var allMetadata = metadata.ShellDescriptors? diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs index df111ab3..398c83d9 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Discovery/AssetIdSearchService.cs @@ -27,7 +27,7 @@ public async Task SearchShellsByAssetLinkAsync(IList !string.IsNullOrWhiteSpace(m.Id)) @@ -62,7 +62,7 @@ public async Task> GetSpecificAssetIdByAasIdentifierAsyn return specificAssetIds; } - private async Task GetFilteredMetadataAsync(List specificAssetIds, CancellationToken cancellationToken) + private async Task GetFilteredMetadataAsync(List specificAssetIds, int? limit, string? cursor, CancellationToken cancellationToken) { try { @@ -73,7 +73,7 @@ private async Task GetFilteredMetadataAsync(List Date: Thu, 20 Aug 2026 23:59:17 +0530 Subject: [PATCH 24/30] resolve semantic id filter issue --- .../SubmodelRepositoryServiceTests.cs | 734 +++++++++++------- .../SubmodelTemplateServiceTests.cs | 360 ++++++--- .../ISubmodelTemplateService.cs | 4 +- .../SubmodelRepositoryService.cs | 50 +- .../SubmodelTemplateService.cs | 15 +- 5 files changed, 728 insertions(+), 435 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs index 8ca469b4..89b284c8 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs @@ -32,9 +32,8 @@ public class SubmodelRepositoryServiceTests private readonly IPluginDataHandler _pluginDataHandler = Substitute.For(); private readonly IPluginManifestConflictHandler _pluginManifestConflictHandler = Substitute.For(); private readonly IAasRepositoryTemplateService _aasRepositoryTemplateService = Substitute.For(); - private readonly IFileContentProvider _fileAttachmentStreamProvider = Substitute.For(); + private readonly IFileContentProvider _fileContentProvider = Substitute.For(); private readonly ILogger _logger = Substitute.For>(); - private readonly IOptions _templateManagementOptions; private readonly SubmodelRepositoryService _sut; private const string SubmodelId = "NameplateSubmodel"; @@ -42,7 +41,7 @@ public class SubmodelRepositoryServiceTests public SubmodelRepositoryServiceTests() { - _templateManagementOptions = Options.Create(new TemplateManagementConfig + var templateManagementOptions = Options.Create(new TemplateManagementConfig { SubmodelTemplateRepository = new ServiceInstance { @@ -54,36 +53,21 @@ public SubmodelRepositoryServiceTests() _logger, _templateService, _aasRepositoryTemplateService, - _templateManagementOptions, + templateManagementOptions, _semanticIdHandler, _pluginDataHandler, _pluginManifestConflictHandler, - _fileAttachmentStreamProvider, + _fileContentProvider, Options.Create(new GeneralConfig { MaxFileAttachmentSizeBytes = 30 * 1024 * 1024 })); } + #region GetSubmodelAsync + [Fact] public async Task GetSubmodelAsync_ReturnsFilledSubmodel() { - var semanticId = TestData.CreateSubmodelTreeNode(); - var values = TestData.CreateSubmodelTreeNode(); var expected = TestData.CreateFilledSubmodel(); - - _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId, (string?)null, Arg.Any(), Arg.Any()) - .Returns(TestData.CreateSubmodel()); - _semanticIdHandler.Extract(Arg.Any()).Returns(semanticId); - - _pluginDataHandler - .TryGetValuesAsync( - Arg.Any>(), - Arg.Any(), - Arg.Any(), - Arg.Any()) - .Returns(Task.FromResult(values)); - - _semanticIdHandler.FillOutTemplate(Arg.Any(), values) - .Returns(expected); + ArrangeSubmodelBuild(SubmodelId, expected); var result = await _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None); @@ -94,34 +78,87 @@ public async Task GetSubmodelAsync_ReturnsFilledSubmodel() public async Task GetSubmodelAsync_WhenQueryOptionsProvided_PassesThemToTemplateService() { var queryOptions = new SubmodelQueryOptions("deep", "withBlobValue"); - var template = TestData.CreateSubmodel(); - - _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), queryOptions, Arg.Any()) - .Returns(template); - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) - .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(TestData.CreateFilledSubmodel()); + ArrangeSubmodelBuild(SubmodelId, TestData.CreateFilledSubmodel()); await _sut.GetSubmodelAsync(SubmodelId, queryOptions, CancellationToken.None); await _templateService.Received(1) - .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), queryOptions, Arg.Any()); + .GetFilteredSubmodelTemplateAsync(SubmodelId, queryOptions, Arg.Any()); } [Fact] public async Task GetSubmodelAsync_WhenTemplateReturnsNull_ThrowsSubmodelNotFoundException() { _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any(), Arg.Any()) + .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any()) .Returns((ISubmodel?)null); await Assert.ThrowsAsync(() => _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None)); } + [Fact] + public async Task GetSubmodelAsync_WhenResourceNotFound_ThrowsSubmodelNotFoundException() + { + _templateService + .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any()) + .ThrowsAsync(new ResourceNotFoundException()); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelAsync_WhenResponseParsingFails_ThrowsInternalDataProcessingException() + { + ArrangeSubmodelBuild_PluginThrows(new ResponseParsingException()); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelAsync_WhenRequestTimesOut_ThrowsPluginNotAvailableException() + { + ArrangeSubmodelBuild_PluginThrows(new RequestTimeoutException()); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelAsync_WhenUnauthorized_ThrowsServiceUnAuthorizedException() + { + ArrangeSubmodelBuild_PluginThrows(new AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException()); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelAsync_WhenMultiPluginConflict_ThrowsInternalDataProcessingException() + { + ArrangeSubmodelBuild_PluginThrows(new MultiPluginConflictException()); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelAsync_SetsSubmodelIdOnResult() + { + var filledSubmodel = TestData.CreateFilledSubmodel(); + ArrangeSubmodelBuild(SubmodelId, filledSubmodel); + + var result = await _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None); + + Assert.Equal(SubmodelId, result.Id); + } + + #endregion + + #region GetSubmodelElementAsync + [Fact] public async Task GetSubmodelElementAsync_ReturnsFilledSubmodelElement() { @@ -133,13 +170,7 @@ public async Task GetSubmodelElementAsync_ReturnsFilledSubmodelElement() .GetSubmodelTemplateAsync(SubmodelId, IdShortPath, Arg.Any(), Arg.Any()) .Returns(submodel); - var semanticTree = CreateSubmodelTreeNode(""); - _semanticIdHandler.Extract(submodel).Returns(semanticTree); - - _pluginDataHandler.TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()).Returns(CreateSubmodelTreeNode("Test John Doe")); - - _semanticIdHandler.FillOutTemplate(submodel, Arg.Any()) - .Returns(filledSubmodel); + ArrangeSemanticPipeline(submodel, filledSubmodel); _semanticIdHandler.Extract(filledSubmodel, IdShortPath).Returns(expected); var result = await _sut.GetSubmodelElementAsync(SubmodelId, IdShortPath, null, CancellationToken.None) as SubmodelElementCollection; @@ -160,13 +191,7 @@ public async Task GetSubmodelElementAsync_IdShortWithIndex_ReturnsFilledSubmodel .GetSubmodelTemplateAsync(SubmodelId, IdShortPathWithNestedElement, Arg.Any(), Arg.Any()) .Returns(submodel); - var semanticTree = CreateSubmodelTreeNode(""); - _semanticIdHandler.Extract(submodel).Returns(semanticTree); - - _pluginDataHandler.TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()).Returns(CreateSubmodelTreeNode("Test John Doe")); - - _semanticIdHandler.FillOutTemplate(submodel, Arg.Any()) - .Returns(filledSubmodel); + ArrangeSemanticPipeline(submodel, filledSubmodel); _semanticIdHandler.Extract(filledSubmodel, IdShortPathWithNestedElement).Returns(expected); var result = await _sut.GetSubmodelElementAsync(SubmodelId, IdShortPathWithNestedElement, null, CancellationToken.None) as Property; @@ -175,39 +200,6 @@ public async Task GetSubmodelElementAsync_IdShortWithIndex_ReturnsFilledSubmodel Assert.Equal(expected.Value, result?.Value); } - [Fact] - public async Task GetSubmodelAsync_WhenResourceNotFound_ThrowsPluginRequestFailedException() - { - _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any(), Arg.Any()) - .ThrowsAsync(new ResourceNotFoundException()); - - await Assert.ThrowsAsync(() => - _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None)); - } - - [Fact] - public async Task GetSubmodelAsync_WhenResponseParsingFails_ThrowsInternalDataProcessingException() - { - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) - .ThrowsAsync(new ResponseParsingException()); - - await Assert.ThrowsAsync(() => - _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None)); - } - - [Fact] - public async Task GetSubmodelAsync_WhenRequestTimesOut_ThrowsPluginNotAvailableException() - { - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) - .ThrowsAsync(new RequestTimeoutException()); - - await Assert.ThrowsAsync(() => - _sut.GetSubmodelAsync(SubmodelId, null, CancellationToken.None)); - } - [Fact] public async Task GetSubmodelElementAsync_WhenResourceNotFound_ThrowsSubmodelElementNotFoundException() { @@ -222,6 +214,10 @@ await Assert.ThrowsAsync(() => [Fact] public async Task GetSubmodelElementAsync_WhenResponseParsingFails_ThrowsInternalDataProcessingException() { + _templateService + .GetSubmodelTemplateAsync(SubmodelId, IdShortPath, Arg.Any(), Arg.Any()) + .Returns(TestData.CreateSubmodel()); + _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); _pluginDataHandler .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) .ThrowsAsync(new ResponseParsingException()); @@ -233,6 +229,10 @@ await Assert.ThrowsAsync(() => [Fact] public async Task GetSubmodelElementAsync_WhenRequestTimesOut_ThrowsPluginNotAvailableException() { + _templateService + .GetSubmodelTemplateAsync(SubmodelId, IdShortPath, Arg.Any(), Arg.Any()) + .Returns(TestData.CreateSubmodel()); + _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); _pluginDataHandler .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) .ThrowsAsync(new RequestTimeoutException()); @@ -241,15 +241,9 @@ await Assert.ThrowsAsync(() => _sut.GetSubmodelElementAsync(SubmodelId, IdShortPath, null, CancellationToken.None)); } - public static SemanticBranchNode CreateSubmodelTreeNode(string value) - { - var submodel = new SemanticBranchNode("http://example.com/idta/digital-nameplate/semantic-id", Cardinality.Unknown); - var contactInformation = new SemanticBranchNode("http://example.com/idta/digital-nameplate/contact-information", Cardinality.ZeroToMany); - var contactName = new SemanticLeafNode("http://example.com/idta/digital-nameplate/contact-name", value, DataType.String, Cardinality.One); - submodel.AddChild(contactInformation); - contactInformation.AddChild(contactName); - return submodel; - } + #endregion + + #region GetAllSubmodelsAsync [Fact] public async Task GetAllSubmodelsAsync_ReturnsEmpty_WhenNoShellsFound() @@ -268,66 +262,23 @@ public async Task GetAllSubmodelsAsync_BuildsSubmodelForEachSubmodelId() { const string ShellId = "https://example.com/shells/001"; const string SubmodelId1 = "https://example.com/submodels/Nameplate"; - var filledSubmodel = TestData.CreateFilledSubmodel(); - var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); - - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); - - _aasRepositoryTemplateService - .GetSubmodelRefByIdAsync(ShellId, Arg.Any()) - .Returns([submodelRef]); - - _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId1, Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(TestData.CreateSubmodel()); - - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(filledSubmodel); + ArrangeShellsResponse([new ShellDescriptorMetaData { Id = ShellId }]); + ArrangeSubmodelRefsForShell(ShellId, [SubmodelId1]); + ArrangeValidateSemanticIdFilter(SubmodelId1, true); + ArrangeSubmodelBuild(SubmodelId1, TestData.CreateFilledSubmodel()); var result = await _sut.GetAllSubmodelsAsync(null, null, null, null, CancellationToken.None); Assert.Single(result.Result); } - [Fact] - public async Task GetAllSubmodelsAsync_SkipsSubmodel_WhenFilteredTemplateReturnsNull() - { - const string ShellId = "https://example.com/shells/001"; - const string SubmodelId1 = "https://example.com/submodels/Nameplate"; - var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); - - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); - - _aasRepositoryTemplateService - .GetSubmodelRefByIdAsync(ShellId, Arg.Any()) - .Returns([submodelRef]); - - _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId1, Arg.Any(), Arg.Any(), Arg.Any()) - .Returns((ISubmodel?)null); - - var result = await _sut.GetAllSubmodelsAsync(null, null, null, null, CancellationToken.None); - - Assert.Empty(result.Result); - } - [Fact] public async Task GetAllSubmodelsAsync_SkipsShell_WhenSubmodelRefsFails() { const string ShellId = "https://example.com/shells/001"; - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); - + ArrangeShellsResponse([new ShellDescriptorMetaData { Id = ShellId }]); _aasRepositoryTemplateService .GetSubmodelRefByIdAsync(ShellId, Arg.Any()) .ThrowsAsync(new ResourceNotFoundException()); @@ -371,36 +322,20 @@ public async Task GetAllSubmodelsAsync_WhenSemanticIdFilterProvided_LooksUpFilte const string ShellId = "https://example.com/shells/001"; const string SubmodelId1 = "https://example.com/submodels/Nameplate"; var filter = new SubmodelSearchFilter { SemanticId = SemanticId }; - var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); - var filledSubmodel = TestData.CreateFilledSubmodel(); - - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = ShellId }] }); _templateService .GetFilteredSubmodelTemplateIdAsync(SemanticId, Arg.Any()) .Returns(FilteredTemplateId); - _aasRepositoryTemplateService - .GetSubmodelRefByIdAsync(ShellId, Arg.Any()) - .Returns([submodelRef]); - - _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId1, FilteredTemplateId, Arg.Any(), Arg.Any()) - .Returns(TestData.CreateSubmodel()); - - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(filledSubmodel); + ArrangeShellsResponse([new ShellDescriptorMetaData { Id = ShellId }]); + ArrangeSubmodelRefsForShell(ShellId, [SubmodelId1]); + _templateService.ValidateSemanticIdFilter(SubmodelId1, FilteredTemplateId).Returns(true); + ArrangeSubmodelBuild(SubmodelId1, TestData.CreateFilledSubmodel()); var result = await _sut.GetAllSubmodelsAsync(filter, null, null, null, CancellationToken.None); Assert.Single(result.Result); await _templateService.Received(1).GetFilteredSubmodelTemplateIdAsync(SemanticId, Arg.Any()); - await _templateService.Received(1).GetFilteredSubmodelTemplateAsync(SubmodelId1, FilteredTemplateId, Arg.Any(), Arg.Any()); } [Fact] @@ -409,11 +344,6 @@ public async Task GetAllSubmodelsAsync_WhenSemanticIdNotFound_ThrowsSubmodelNotF const string SemanticId = "https://example.com/unknown-semantic-id"; var filter = new SubmodelSearchFilter { SemanticId = SemanticId }; - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new ShellDescriptorsMetaData { ShellDescriptors = [new ShellDescriptorMetaData { Id = "https://example.com/shells/001" }] }); - - // GetFilteredSubmodelTemplateIdAsync returns null — semantic ID not found in any template _templateService .GetFilteredSubmodelTemplateIdAsync(SemanticId, Arg.Any()) .Returns((string?)null); @@ -423,53 +353,51 @@ await Assert.ThrowsAsync(() => } [Fact] - public async Task GetAllSubmodelsAsync_SkipsShellsWithEmptyOrWhitespaceId() + public async Task GetAllSubmodelsAsync_FiltersSubmodels_WhenValidateSemanticIdFilterReturnsFalse() { - const string ValidShellId = "https://example.com/shells/valid"; - const string SubmodelId1 = "https://example.com/submodels/Nameplate"; - var submodelRef = new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, SubmodelId1)]); - var filledSubmodel = TestData.CreateFilledSubmodel(); + const string SemanticId = "https://example.com/semanticId"; + const string FilteredTemplateId = "Nameplate"; + const string ShellId = "https://example.com/shells/001"; + const string SubmodelId1 = "https://example.com/submodels/sm-1"; + const string SubmodelId2 = "https://example.com/submodels/sm-2"; + var filter = new SubmodelSearchFilter { SemanticId = SemanticId }; - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new ShellDescriptorsMetaData - { - ShellDescriptors = - [ - new ShellDescriptorMetaData { Id = string.Empty }, // should be skipped - new ShellDescriptorMetaData { Id = " " }, // should be skipped - new ShellDescriptorMetaData { Id = ValidShellId } // should be included - ] - }); + _templateService.GetFilteredSubmodelTemplateIdAsync(SemanticId, Arg.Any()).Returns(FilteredTemplateId); + ArrangeShellsResponse([new ShellDescriptorMetaData { Id = ShellId }]); + ArrangeSubmodelRefsForShell(ShellId, [SubmodelId1, SubmodelId2]); + _templateService.ValidateSemanticIdFilter(SubmodelId1, FilteredTemplateId).Returns(true); + _templateService.ValidateSemanticIdFilter(SubmodelId2, FilteredTemplateId).Returns(false); + ArrangeSubmodelBuild(SubmodelId1, TestData.CreateFilledSubmodel()); - _aasRepositoryTemplateService - .GetSubmodelRefByIdAsync(ValidShellId, Arg.Any()) - .Returns([submodelRef]); + var result = await _sut.GetAllSubmodelsAsync(filter, null, null, null, CancellationToken.None); - _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId1, Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(TestData.CreateSubmodel()); + Assert.Single(result.Result); + } - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(filledSubmodel); + [Fact] + public async Task GetAllSubmodelsAsync_SkipsShellsWithEmptyOrWhitespaceId() + { + const string ValidShellId = "https://example.com/shells/valid"; + const string SubmodelId1 = "https://example.com/submodels/Nameplate"; + + ArrangeShellsResponse([ + new ShellDescriptorMetaData { Id = string.Empty }, + new ShellDescriptorMetaData { Id = " " }, + new ShellDescriptorMetaData { Id = ValidShellId } + ]); + ArrangeSubmodelRefsForShell(ValidShellId, [SubmodelId1]); + ArrangeValidateSemanticIdFilter(SubmodelId1, true); + ArrangeSubmodelBuild(SubmodelId1, TestData.CreateFilledSubmodel()); var result = await _sut.GetAllSubmodelsAsync(null, null, null, null, CancellationToken.None); Assert.Single(result.Result); - // Only the valid shell should have been queried for submodel refs await _aasRepositoryTemplateService.Received(1).GetSubmodelRefByIdAsync(ValidShellId, Arg.Any()); - await _aasRepositoryTemplateService.DidNotReceive().GetSubmodelRefByIdAsync(null!, Arg.Any()); } [Fact] - public async Task GetAllSubmodelsAsync_WhenLimitReachedAtAasBoundary_CursorSubmodelIdIsLastDeliveredAndAasIdIsConsumedAas() + public async Task GetAllSubmodelsAsync_WhenLimitReachedAtAasBoundary_CursorContainsLastSubmodelAndConsumedAas() { - // AAS-1 has 3 submodels, AAS-2 has 2 submodels. limit=5 hits the exact end of AAS-2. - // AasId in the cursor must be AAS-2 (the consumed AAS), not AAS-1 (the entry cursor), - // so Plugin(AAS-2 exclusive) on resume skips directly to the next AAS. const string Shell1Id = "https://example.com/shells/aas-1"; const string Shell2Id = "https://example.com/shells/aas-2"; const string Sm1 = "https://example.com/submodels/sm-1"; @@ -478,35 +406,14 @@ public async Task GetAllSubmodelsAsync_WhenLimitReachedAtAasBoundary_CursorSubmo const string Sm4 = "https://example.com/submodels/sm-4"; const string Sm5 = "https://example.com/submodels/sm-5"; - _pluginDataHandler - .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(new ShellDescriptorsMetaData - { - ShellDescriptors = [new ShellDescriptorMetaData { Id = Shell1Id }, new ShellDescriptorMetaData { Id = Shell2Id }] - }); - - _aasRepositoryTemplateService.GetSubmodelRefByIdAsync(Shell1Id, Arg.Any()) - .Returns([ - new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm1)]), - new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm2)]), - new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm3)]) - ]); - - _aasRepositoryTemplateService.GetSubmodelRefByIdAsync(Shell2Id, Arg.Any()) - .Returns([ - new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm4)]), - new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm5)]) - ]); - - _templateService - .GetFilteredSubmodelTemplateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(TestData.CreateSubmodel()); - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()) - .Returns(TestData.CreateFilledSubmodel()); + ArrangeShellsResponse([ + new ShellDescriptorMetaData { Id = Shell1Id }, + new ShellDescriptorMetaData { Id = Shell2Id } + ]); + ArrangeSubmodelRefsForShell(Shell1Id, [Sm1, Sm2, Sm3]); + ArrangeSubmodelRefsForShell(Shell2Id, [Sm4, Sm5]); + ArrangeValidateSemanticIdFilterForAll(true); + ArrangeSubmodelBuildForAny(TestData.CreateFilledSubmodel()); var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 5, null, CancellationToken.None); @@ -514,18 +421,17 @@ public async Task GetAllSubmodelsAsync_WhenLimitReachedAtAasBoundary_CursorSubmo Assert.NotNull(result.PagingMetaData?.Cursor); var decoded = SubmodelPaginationCursor.Decode(result.PagingMetaData!.Cursor!); - Assert.Equal(Sm5, decoded!.SubmodelId); // last delivered submodel - Assert.Equal(Shell2Id, decoded.AasId); // the consumed AAS, not the entry cursor + Assert.Equal(Sm5, decoded!.SubmodelId); + Assert.Equal(Shell2Id, decoded.AasId); } [Fact] - public async Task GetAllSubmodelsAsync_WhenLimitReachedAtFirstAasBoundary_CursorAasIdIsConsumedAas() + public async Task GetAllSubmodelsAsync_WhenLimitReachedMidAas_CursorAasIdIsCurrentShell() { - // AAS-1 has exactly 2 submodels, limit=2. Limit hit at the end of AAS-1. - // AasId must be AAS-1 (the consumed AAS) so the next request starts from after AAS-1. const string Shell1Id = "https://example.com/shells/aas-1"; const string Sm1 = "https://example.com/submodels/sm-1"; const string Sm2 = "https://example.com/submodels/sm-2"; + const string Sm3 = "https://example.com/submodels/sm-3"; _pluginDataHandler .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) @@ -534,22 +440,36 @@ public async Task GetAllSubmodelsAsync_WhenLimitReachedAtFirstAasBoundary_Cursor ShellDescriptors = [new ShellDescriptorMetaData { Id = Shell1Id }], PagingMetaData = new PagingMetaData { Cursor = "next-page-token" } }); + ArrangeSubmodelRefsForShell(Shell1Id, [Sm1, Sm2, Sm3]); + ArrangeValidateSemanticIdFilterForAll(true); + ArrangeSubmodelBuildForAny(TestData.CreateFilledSubmodel()); - _aasRepositoryTemplateService.GetSubmodelRefByIdAsync(Shell1Id, Arg.Any()) - .Returns([ - new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm1)]), - new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, Sm2)]) - ]); + var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 2, null, CancellationToken.None); + + Assert.Equal(2, result.Result.Count); + Assert.NotNull(result.PagingMetaData?.Cursor); + + var decoded = SubmodelPaginationCursor.Decode(result.PagingMetaData!.Cursor!); + Assert.Equal(Sm2, decoded!.SubmodelId); + } + + [Fact] + public async Task GetAllSubmodelsAsync_WhenLimitReachedAtExactAasBoundary_CursorAasIdIsConsumedAas() + { + const string Shell1Id = "https://example.com/shells/aas-1"; + const string Sm1 = "https://example.com/submodels/sm-1"; + const string Sm2 = "https://example.com/submodels/sm-2"; - _templateService - .GetFilteredSubmodelTemplateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(TestData.CreateSubmodel()); - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()) - .Returns(TestData.CreateFilledSubmodel()); + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ShellDescriptorsMetaData + { + ShellDescriptors = [new ShellDescriptorMetaData { Id = Shell1Id }], + PagingMetaData = new PagingMetaData { Cursor = "next-page-token" } + }); + ArrangeSubmodelRefsForShell(Shell1Id, [Sm1, Sm2]); + ArrangeValidateSemanticIdFilterForAll(true); + ArrangeSubmodelBuildForAny(TestData.CreateFilledSubmodel()); var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 2, null, CancellationToken.None); @@ -557,24 +477,72 @@ public async Task GetAllSubmodelsAsync_WhenLimitReachedAtFirstAasBoundary_Cursor Assert.NotNull(result.PagingMetaData?.Cursor); var decoded = SubmodelPaginationCursor.Decode(result.PagingMetaData!.Cursor!); - Assert.Equal(Sm2, decoded!.SubmodelId); // last delivered submodel - Assert.Equal(Shell1Id, decoded.AasId); // consumed AAS — next request starts after AAS-1 + Assert.Equal(Sm2, decoded!.SubmodelId); + Assert.Equal(Shell1Id, decoded.AasId); } [Fact] - public async Task GetAllSubmodelElementsAsync_ReturnsAllElements_WhenSubmodelExists() + public async Task GetAllSubmodelsAsync_NoCursor_WhenAllResultsFitInPage() { - var filledSubmodel = TestData.CreateFilledSubmodel(); + const string ShellId = "https://example.com/shells/001"; + const string SubmodelId1 = "https://example.com/submodels/sm-1"; - _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(TestData.CreateSubmodel()); + ArrangeShellsResponse([new ShellDescriptorMetaData { Id = ShellId }]); + ArrangeSubmodelRefsForShell(ShellId, [SubmodelId1]); + ArrangeValidateSemanticIdFilter(SubmodelId1, true); + ArrangeSubmodelBuild(SubmodelId1, TestData.CreateFilledSubmodel()); - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) - .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(filledSubmodel); + var result = await _sut.GetAllSubmodelsAsync(null, null, limit: 100, null, CancellationToken.None); + + Assert.Single(result.Result); + Assert.Null(result.PagingMetaData?.Cursor); + } + + [Fact] + public async Task GetAllSubmodelsAsync_DefaultsTo100_WhenLimitIsNull() + { + ArrangeShellsResponse([]); + + await _sut.GetAllSubmodelsAsync(null, null, null, null, CancellationToken.None); + + await _pluginDataHandler.Received(1).GetDataForShellsByAssetIdsAsync( + Arg.Any>(), + Arg.Any(), + 100, + Arg.Any(), + Arg.Any()); + } + + [Fact] + public async Task GetAllSubmodelsAsync_SkipsSubmodelRefsWithNullOrEmptyValues() + { + const string ShellId = "https://example.com/shells/001"; + const string ValidSubmodelId = "https://example.com/submodels/valid"; + + ArrangeShellsResponse([new ShellDescriptorMetaData { Id = ShellId }]); + _aasRepositoryTemplateService.GetSubmodelRefByIdAsync(ShellId, Arg.Any()) + .Returns(new List + { + new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, "")]), + new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, ValidSubmodelId)]) + }); + ArrangeValidateSemanticIdFilter(ValidSubmodelId, true); + ArrangeSubmodelBuild(ValidSubmodelId, TestData.CreateFilledSubmodel()); + + var result = await _sut.GetAllSubmodelsAsync(null, null, null, null, CancellationToken.None); + + Assert.Single(result.Result); + } + + #endregion + + #region GetAllSubmodelElementsAsync + + [Fact] + public async Task GetAllSubmodelElementsAsync_ReturnsAllElements_WhenSubmodelExists() + { + var filledSubmodel = TestData.CreateFilledSubmodel(); + ArrangeSubmodelBuild(SubmodelId, filledSubmodel); var result = await _sut.GetAllSubmodelElementsAsync(SubmodelId, null, null, null, CancellationToken.None); @@ -592,9 +560,8 @@ public async Task GetAllSubmodelElementsAsync_ReturnsEmptyList_WhenSubmodelHasNo submodelElements: []); _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any(), Arg.Any()) + .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any()) .Returns(emptySubmodel); - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); _pluginDataHandler .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) @@ -611,16 +578,7 @@ public async Task GetAllSubmodelElementsAsync_ReturnsEmptyList_WhenSubmodelHasNo public async Task GetAllSubmodelElementsAsync_ReturnsPagedResult_WhenLimitApplied() { var filledSubmodel = TestData.CreateFilledSubmodel(); - - _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(TestData.CreateSubmodel()); - - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _pluginDataHandler - .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) - .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(filledSubmodel); + ArrangeSubmodelBuild(SubmodelId, filledSubmodel); var result = await _sut.GetAllSubmodelElementsAsync(SubmodelId, null, limit: 2, cursor: null, CancellationToken.None); @@ -629,10 +587,21 @@ public async Task GetAllSubmodelElementsAsync_ReturnsPagedResult_WhenLimitApplie } [Fact] - public async Task GetAllSubmodelElementsAsync_WhenSubmodelNotFound_ThrowsSubmodelElementNotFoundException() + public async Task GetAllSubmodelElementsAsync_WhenTemplateNull_ThrowsSubmodelElementNotFoundException() { _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId, (string?)null, Arg.Any(), Arg.Any()) + .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any()) + .Returns((ISubmodel?)null); + + await Assert.ThrowsAsync(() => + _sut.GetAllSubmodelElementsAsync(SubmodelId, null, null, null, CancellationToken.None)); + } + + [Fact] + public async Task GetAllSubmodelElementsAsync_WhenResourceNotFound_ThrowsSubmodelElementNotFoundException() + { + _templateService + .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any()) .ThrowsAsync(new ResourceNotFoundException()); await Assert.ThrowsAsync(() => @@ -643,9 +612,8 @@ await Assert.ThrowsAsync(() => public async Task GetAllSubmodelElementsAsync_WhenResponseParsingFails_ThrowsInternalDataProcessingException() { _templateService - .GetFilteredSubmodelTemplateAsync(SubmodelId, (string?)null, Arg.Any(), Arg.Any()) + .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any()) .Returns(TestData.CreateSubmodel()); - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); _pluginDataHandler .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) @@ -655,31 +623,25 @@ await Assert.ThrowsAsync(() => _sut.GetAllSubmodelElementsAsync(SubmodelId, null, null, null, CancellationToken.None)); } - private void ArrangeAttachmentElement(string idShortPath, ISubmodelElement element) - { - var template = TestData.CreateSubmodelWithElement(element, idShortPath); - _templateService.GetSubmodelTemplateAsync(SubmodelId, idShortPath, Arg.Any(), Arg.Any()).Returns(template); - _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _pluginDataHandler.TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()).Returns(CreateSubmodelTreeNode("")); - _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(template); - _semanticIdHandler.Extract(Arg.Any(), idShortPath).Returns(element); - } + #endregion + + #region GetFileAttachmentAsync [Fact] public async Task GetFileAttachmentAsync_WhenElementIsFileWithHttpUrl_ReturnsStreamWithCorrectMetadata() { - const string IdShortPath = "Documents.ProductImage"; + const string fileIdShortPath = "Documents.ProductImage"; const string FileUrl = "https://fake-plugin.local/files/product.png"; const string FileContent = "binary-file-data"; var fileElement = new AasCore.Aas3_1.File(contentType: "image/png") { Value = FileUrl, IdShort = "ProductImage" }; - ArrangeAttachmentElement(IdShortPath, fileElement); + ArrangeAttachmentElement(fileIdShortPath, fileElement); var stream = new MemoryStream(Encoding.UTF8.GetBytes(FileContent)); var fileContentResponse = new FileContentResponse(stream); - _fileAttachmentStreamProvider.GetFileContentAsync(FileUrl, Arg.Any()).Returns(fileContentResponse); + _fileContentProvider.GetFileContentAsync(FileUrl, Arg.Any()).Returns(fileContentResponse); - var result = await _sut.GetFileAttachmentAsync(SubmodelId, IdShortPath, CancellationToken.None); + var result = await _sut.GetFileAttachmentAsync(SubmodelId, fileIdShortPath, CancellationToken.None); await using (result.Content) { var body = await new StreamReader(result.Content).ReadToEndAsync(); @@ -688,54 +650,226 @@ public async Task GetFileAttachmentAsync_WhenElementIsFileWithHttpUrl_ReturnsStr Assert.Contains("image/png", result.ContentType); } - await _fileAttachmentStreamProvider.Received(1).GetFileContentAsync(FileUrl, Arg.Any()); + await _fileContentProvider.Received(1).GetFileContentAsync(FileUrl, Arg.Any()); + } + + [Fact] + public async Task GetFileAttachmentAsync_WhenContentTypeIsEmpty_DefaultsToOctetStream() + { + const string fileIdShortPath = "Documents.ProductImage"; + const string FileUrl = "https://fake-plugin.local/files/data.bin"; + + var fileElement = new AasCore.Aas3_1.File(contentType: "") { Value = FileUrl, IdShort = "ProductImage" }; + ArrangeAttachmentElement(fileIdShortPath, fileElement); + + var stream = new MemoryStream([0x01, 0x02]); + _fileContentProvider.GetFileContentAsync(FileUrl, Arg.Any()).Returns(new FileContentResponse(stream)); + + var result = await _sut.GetFileAttachmentAsync(SubmodelId, fileIdShortPath, CancellationToken.None); + await using (result.Content) + { + Assert.Equal("application/octet-stream", result.ContentType); + } } [Fact] public async Task GetFileAttachmentAsync_WhenElementIsNotFile_ThrowsInvalidUserInputException() { - const string IdShortPath = "ManufacturerName"; + const string fileIdShortPath = "ManufacturerName"; var property = new Property(DataTypeDefXsd.String) { IdShort = "ManufacturerName" }; - ArrangeAttachmentElement(IdShortPath, property); - var ex = await Assert.ThrowsAsync(() => - _sut.GetFileAttachmentAsync(SubmodelId, IdShortPath, CancellationToken.None)); - Assert.Equal("Invalid User Input.", ex.Message); + ArrangeAttachmentElement(fileIdShortPath, property); + + await Assert.ThrowsAsync(() => + _sut.GetFileAttachmentAsync(SubmodelId, fileIdShortPath, CancellationToken.None)); } [Fact] public async Task GetFileAttachmentAsync_WhenSubmodelNotFound_ThrowsSubmodelElementNotFoundException() { - const string IdShortPath = "Documents.ProductImage"; + const string fileIdShortPath = "Documents.ProductImage"; _templateService - .GetSubmodelTemplateAsync(SubmodelId, IdShortPath, Arg.Any(), Arg.Any()) + .GetSubmodelTemplateAsync(SubmodelId, fileIdShortPath, Arg.Any(), Arg.Any()) .ThrowsAsync(new ResourceNotFoundException()); await Assert.ThrowsAsync(() => - _sut.GetFileAttachmentAsync(SubmodelId, IdShortPath, CancellationToken.None)); + _sut.GetFileAttachmentAsync(SubmodelId, fileIdShortPath, CancellationToken.None)); } [Fact] public async Task GetFileAttachmentAsync_WhenFileValueIsEmpty_ThrowsSubmodelElementNotFoundException() { - const string IdShortPath = "Documents.ProductImage"; + const string fileIdShortPath = "Documents.ProductImage"; var fileElement = new AasCore.Aas3_1.File(contentType: "image/png") { Value = "", IdShort = "ProductImage" }; - ArrangeAttachmentElement(IdShortPath, fileElement); + ArrangeAttachmentElement(fileIdShortPath, fileElement); + + await Assert.ThrowsAsync(() => + _sut.GetFileAttachmentAsync(SubmodelId, fileIdShortPath, CancellationToken.None)); + } + + [Fact] + public async Task GetFileAttachmentAsync_WhenFileValueIsNull_ThrowsSubmodelElementNotFoundException() + { + const string fileIdShortPath = "Documents.ProductImage"; + var fileElement = new AasCore.Aas3_1.File(contentType: "image/png") { Value = null, IdShort = "ProductImage" }; + ArrangeAttachmentElement(fileIdShortPath, fileElement); await Assert.ThrowsAsync(() => - _sut.GetFileAttachmentAsync(SubmodelId, IdShortPath, CancellationToken.None)); + _sut.GetFileAttachmentAsync(SubmodelId, fileIdShortPath, CancellationToken.None)); } [Fact] public async Task GetFileAttachmentAsync_WhenFileUrlIsNotHttpOrHttps_ThrowsInternalDataProcessingException() { - const string IdShortPath = "Documents.ProductImage"; + const string fileIdShortPath = "Documents.ProductImage"; const string FileUrl = "ftp://fake-plugin.local/files/product.png"; + var fileElement = new AasCore.Aas3_1.File(contentType: "image/png") { Value = FileUrl, IdShort = "ProductImage" }; + ArrangeAttachmentElement(fileIdShortPath, fileElement); + + await Assert.ThrowsAsync(() => + _sut.GetFileAttachmentAsync(SubmodelId, fileIdShortPath, CancellationToken.None)); + } + [Fact] + public async Task GetFileAttachmentAsync_WhenFileUrlIsRelative_ThrowsInternalDataProcessingException() + { + const string fileIdShortPath = "Documents.ProductImage"; + const string FileUrl = "/relative/path/file.png"; var fileElement = new AasCore.Aas3_1.File(contentType: "image/png") { Value = FileUrl, IdShort = "ProductImage" }; - ArrangeAttachmentElement(IdShortPath, fileElement); + ArrangeAttachmentElement(fileIdShortPath, fileElement); await Assert.ThrowsAsync(() => - _sut.GetFileAttachmentAsync(SubmodelId, IdShortPath, CancellationToken.None)); + _sut.GetFileAttachmentAsync(SubmodelId, fileIdShortPath, CancellationToken.None)); + } + + [Fact] + public async Task GetFileAttachmentAsync_ExtractsFileNameFromUrl() + { + const string fileIdShortPath = "Documents.ProductImage"; + const string FileUrl = "https://fake-plugin.local/files/my-document.pdf"; + + var fileElement = new AasCore.Aas3_1.File(contentType: "application/pdf") { Value = FileUrl, IdShort = "ProductImage" }; + ArrangeAttachmentElement(fileIdShortPath, fileElement); + + var stream = new MemoryStream([0x01]); + _fileContentProvider.GetFileContentAsync(FileUrl, Arg.Any()).Returns(new FileContentResponse(stream)); + + var result = await _sut.GetFileAttachmentAsync(SubmodelId, fileIdShortPath, CancellationToken.None); + await using (result.Content) + { + Assert.Equal("my-document.pdf", result.FileName); + } + } + + [Fact] + public async Task GetFileAttachmentAsync_UsesIdShortAsFileName_WhenUrlHasNoPath() + { + const string fileIdShortPath = "Documents.ProductImage"; + const string FileUrl = "https://fake-plugin.local/"; + + var fileElement = new AasCore.Aas3_1.File(contentType: "image/png") { Value = FileUrl, IdShort = "ProductImage" }; + ArrangeAttachmentElement(fileIdShortPath, fileElement); + + var stream = new MemoryStream([0x01]); + _fileContentProvider.GetFileContentAsync(FileUrl, Arg.Any()).Returns(new FileContentResponse(stream)); + + var result = await _sut.GetFileAttachmentAsync(SubmodelId, fileIdShortPath, CancellationToken.None); + await using (result.Content) + { + Assert.Equal("ProductImage", result.FileName); + } + } + + #endregion + + #region Helpers + + public static SemanticBranchNode CreateSubmodelTreeNode(string value) + { + var submodel = new SemanticBranchNode("http://example.com/idta/digital-nameplate/semantic-id", Cardinality.Unknown); + var contactInformation = new SemanticBranchNode("http://example.com/idta/digital-nameplate/contact-information", Cardinality.ZeroToMany); + var contactName = new SemanticLeafNode("http://example.com/idta/digital-nameplate/contact-name", value, DataType.String, Cardinality.One); + submodel.AddChild(contactInformation); + contactInformation.AddChild(contactName); + return submodel; + } + + private void ArrangeSubmodelBuild(string submodelId, Submodel filledSubmodel) + { + _templateService + .GetFilteredSubmodelTemplateAsync(submodelId, Arg.Any(), Arg.Any()) + .Returns(TestData.CreateSubmodel()); + _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); + _pluginDataHandler + .TryGetValuesAsync(Arg.Any>(), Arg.Any(), submodelId, Arg.Any()) + .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); + _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(filledSubmodel); } + + private void ArrangeSubmodelBuildForAny(Submodel filledSubmodel) + { + _templateService + .GetFilteredSubmodelTemplateAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(TestData.CreateSubmodel()); + _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); + _pluginDataHandler + .TryGetValuesAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(Task.FromResult(CreateSubmodelTreeNode("") as SemanticTreeNode)); + _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(filledSubmodel); + } + + private void ArrangeSubmodelBuild_PluginThrows(Exception exception) + { + _templateService + .GetFilteredSubmodelTemplateAsync(SubmodelId, Arg.Any(), Arg.Any()) + .Returns(TestData.CreateSubmodel()); + _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); + _pluginDataHandler + .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) + .ThrowsAsync(exception); + } + + private void ArrangeSemanticPipeline(ISubmodel template, Submodel filled) + { + _semanticIdHandler.Extract(template).Returns(CreateSubmodelTreeNode("")); + _pluginDataHandler + .TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()) + .Returns(CreateSubmodelTreeNode("Test John Doe")); + _semanticIdHandler.FillOutTemplate(template, Arg.Any()).Returns(filled); + } + + private void ArrangeShellsResponse(List descriptors) + { + _pluginDataHandler + .GetDataForShellsByAssetIdsAsync(Arg.Any>(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(new ShellDescriptorsMetaData { ShellDescriptors = descriptors }); + } + + private void ArrangeSubmodelRefsForShell(string shellId, List submodelIds) + { + var refs = submodelIds.Select(id => (IReference)new Reference(ReferenceTypes.ModelReference, [new Key(KeyTypes.Submodel, id)])).ToList(); + _aasRepositoryTemplateService.GetSubmodelRefByIdAsync(shellId, Arg.Any()).Returns(refs); + } + + private void ArrangeValidateSemanticIdFilter(string submodelId, bool result) + { + _templateService.ValidateSemanticIdFilter(submodelId, Arg.Any()).Returns(result); + } + + private void ArrangeValidateSemanticIdFilterForAll(bool result) + { + _templateService.ValidateSemanticIdFilter(Arg.Any(), Arg.Any()).Returns(result); + } + + private void ArrangeAttachmentElement(string idShortPath, ISubmodelElement element) + { + var template = TestData.CreateSubmodelWithElement(element, idShortPath); + _templateService.GetSubmodelTemplateAsync(SubmodelId, idShortPath, Arg.Any(), Arg.Any()).Returns(template); + _semanticIdHandler.Extract(Arg.Any()).Returns(CreateSubmodelTreeNode("")); + _pluginDataHandler.TryGetValuesAsync(Arg.Any>(), Arg.Any(), SubmodelId, Arg.Any()).Returns(CreateSubmodelTreeNode("")); + _semanticIdHandler.FillOutTemplate(Arg.Any(), Arg.Any()).Returns(template); + _semanticIdHandler.Extract(Arg.Any(), idShortPath).Returns(element); + } + + #endregion } diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelTemplateServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelTemplateServiceTests.cs index 11d29ca9..a3b741f2 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelTemplateServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelTemplateServiceTests.cs @@ -1,8 +1,10 @@ -using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; +using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Application; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Base; using AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.AasEnvironment.Providers; using AAS.TwinEngine.DataEngine.ApplicationLogic.Services.SubmodelRepository; +using AAS.TwinEngine.DataEngine.DomainModel.Shared; +using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; using AasCore.Aas3_1; @@ -27,22 +29,24 @@ public class SubmodelTemplateServiceTests public SubmodelTemplateServiceTests() => _sut = new SubmodelTemplateService(_templateProvider, _mappingProvider, _logger); + #region Constructor + [Fact] public void Constructor_ThrowsInvalidDependencyException_WhenTemplateProviderIsNull() { - ITemplateProvider? templateProvider = null; - - var ex = Assert.Throws(() => new SubmodelTemplateService(templateProvider!, _mappingProvider, _logger)); + Assert.Throws(() => new SubmodelTemplateService(null!, _mappingProvider, _logger)); } [Fact] public void Constructor_ThrowsInvalidDependencyException_WhenTemplateMappingProviderIsNull() { - ISubmodelTemplateMappingProvider? templateMappingProvider = null; - - var ex = Assert.Throws(() => new SubmodelTemplateService(_templateProvider, templateMappingProvider!, _logger)); + Assert.Throws(() => new SubmodelTemplateService(_templateProvider, null!, _logger)); } + #endregion + + #region GetSubmodelTemplateAsync (single param) + [Fact] public async Task GetSubmodelTemplateAsync_ReturnsSubmodel_WhenValidInput() { @@ -57,47 +61,88 @@ public async Task GetSubmodelTemplateAsync_ReturnsSubmodel_WhenValidInput() } [Fact] - public async Task GetSubmodelTemplateAsync_ThrowsBadRequestException_WhenSubmodelIdIsNull() + public async Task GetSubmodelTemplateAsync_ThrowsInternalDataProcessingException_WhenSubmodelIdIsNull() { - string? submodelId = null; + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(null!, CancellationToken.None)); + } - var exception = await Assert.ThrowsAsync(() => _sut.GetSubmodelTemplateAsync(submodelId!, CancellationToken.None)); - Assert.Equal("Internal Server Error.", exception.Message); + [Fact] + public async Task GetSubmodelTemplateAsync_ThrowsInternalDataProcessingException_WhenSubmodelIdIsEmpty() + { + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync("", CancellationToken.None)); } [Fact] - public async Task GetSubmodelTemplateAsync_ReturnsNull_WhenElementNotFound() + public async Task GetSubmodelTemplateAsync_ThrowsInternalDataProcessingException_WhenSubmodelIdIsWhitespace() + { + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(" ", CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelTemplateAsync_ThrowsSubmodelNotFoundException_WhenResourceNotFound() { - const string IdShortPath = "InvalidElement"; - var expectedSubmodel = TestData.CreateSubmodel(); _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .Returns(expectedSubmodel); + .ThrowsAsync(new ResourceNotFoundException()); - var exception = await Assert.ThrowsAsync(() => _sut.GetSubmodelTemplateAsync(SubmodelId, IdShortPath, null, CancellationToken.None)); - Assert.Equal("Submodel Element not found.", exception.Message); + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, CancellationToken.None)); } [Fact] - public async Task GetSubmodelTemplateAsync_ThrowsException_WhenSubmodelIdIsInvalid() + public async Task GetSubmodelTemplateAsync_ThrowsInternalDataProcessingException_WhenResponseParsingFails() { - string? submodelId = null; + _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); + _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) + .ThrowsAsync(new ResponseParsingException()); + + var exception = await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, CancellationToken.None)); - var exception = await Assert.ThrowsAsync(() => _sut.GetSubmodelTemplateAsync(submodelId!, "idShort", null, CancellationToken.None)); Assert.Equal("Internal Server Error.", exception.Message); } + [Fact] + public async Task GetSubmodelTemplateAsync_ThrowsTemplateRequestFailedException_WhenRequestTimesOut() + { + _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); + _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) + .ThrowsAsync(new RequestTimeoutException()); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelTemplateAsync_ThrowsRepositoryNotAvailableException_WhenServiceUnavailable() + { + _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); + _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) + .ThrowsAsync(new ServiceUnavailableException("http://fake-url")); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, CancellationToken.None)); + } + + #endregion + + #region GetSubmodelTemplateAsync (with idShortPath) + [Fact] public async Task GetSubmodelTemplateAsync_ReturnsElement_WhenSingleProperty() { - const string IdShortPath = "ManufacturerName"; + const string idShortPath = "ManufacturerName"; var expectedSubmodel = TestData.CreateSubmodel(); var expectedElement = TestData.CreateSubmodelWithoutExtraElements(); _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) .Returns(expectedSubmodel); - var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, IdShortPath, null, CancellationToken.None); + var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, idShortPath, null, CancellationToken.None); + Assert.Equal(GetSemanticId(expectedElement), GetSemanticId(result)); Assert.Equal(expectedElement.SubmodelElements!.Count, result.SubmodelElements!.Count); Assert.Single(expectedElement.SubmodelElements); @@ -106,51 +151,142 @@ public async Task GetSubmodelTemplateAsync_ReturnsElement_WhenSingleProperty() [Fact] public async Task GetSubmodelTemplateAsync_ReturnsCustomSubmodel_WhenNestedProperty() { - const string IdShortPath = "ContactInformation.ContactName"; + const string idShortPath = "ContactInformation.ContactName"; var expectedSubmodel = TestData.CreateSubmodel(); var expectedElement = TestData.CreateSubmodelWithoutExtraElementsNested(); _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) .Returns(expectedSubmodel); - var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, IdShortPath, null, CancellationToken.None); + var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, idShortPath, null, CancellationToken.None); Assert.Equal(GetSemanticId(expectedElement), GetSemanticId(result)); Assert.Equal(expectedElement.SubmodelElements!.Count, result.SubmodelElements!.Count); Assert.Single(expectedElement.SubmodelElements); } - private static string GetSemanticId(IHasSemantics hasSemantics) => hasSemantics.SemanticId?.Keys?.FirstOrDefault()?.Value ?? string.Empty; + [Fact] + public async Task GetSubmodelTemplateAsync_ThrowsSubmodelElementNotFoundException_WhenElementNotFound() + { + const string idShortPath = "InvalidElement"; + _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); + _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) + .Returns(TestData.CreateSubmodel()); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, idShortPath, null, CancellationToken.None)); + } [Fact] - public async Task GetSubmodelTemplateAsync_ReturnsSubmodelElementNotFoundException_WhenNotFindTheSubmodelElement() + public async Task GetSubmodelTemplateAsync_ThrowsSubmodelElementNotFoundException_WhenNestedPathInvalid() { - const string IdShortPath = "ContactInformation0.InvalidIdShort"; - var expectedSubmodel = TestData.CreateSubmodel(); + const string idShortPath = "ContactInformation0.InvalidIdShort"; _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .Returns(expectedSubmodel); + .Returns(TestData.CreateSubmodel()); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, idShortPath, null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelTemplateAsync_WithIdShortPath_ThrowsInternalDataProcessingException_WhenSubmodelIdIsEmpty() + { + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync("", "ContactInformation0", null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelTemplateAsync_WithIdShortPath_ThrowsInternalDataProcessingException_WhenSubmodelIdIsNull() + { + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(null!, "idShort", null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelTemplateAsync_ThrowsInvalidDependencyException_WhenIdShortPathIsEmpty() + { + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, "", null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelTemplateAsync_ThrowsInvalidDependencyException_WhenIdShortPathIsWhitespace() + { + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, " ", null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelTemplateAsync_WithIdShortPath_ThrowsSubmodelElementNotFoundException_WhenResourceNotFound() + { + _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); + _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) + .ThrowsAsync(new ResourceNotFoundException()); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, "SomePath", null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelTemplateAsync_WithIdShortPath_ThrowsInternalDataProcessingException_WhenResponseParsingFails() + { + _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); + _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, Arg.Any(), Arg.Any()) + .ThrowsAsync(new ResponseParsingException()); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, "SomePath", null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelTemplateAsync_WithIdShortPath_ThrowsTemplateRequestFailedException_WhenRequestTimesOut() + { + _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); + _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, Arg.Any(), Arg.Any()) + .ThrowsAsync(new RequestTimeoutException()); - var exception = await Assert.ThrowsAsync(() => _sut.GetSubmodelTemplateAsync(SubmodelId, IdShortPath, null, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, "SomePath", null, CancellationToken.None)); } [Fact] - public async Task GetSubmodelElementTemplateAsync_ThrowsBadRequestException_WhenSubmodelIdIsEmpty() + public async Task GetSubmodelTemplateAsync_WithIdShortPath_ThrowsRepositoryNotAvailableException_WhenServiceUnavailable() { - const string IdShortPath = "ContactInformation0"; + _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); + _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, Arg.Any(), Arg.Any()) + .ThrowsAsync(new ServiceUnavailableException("down")); + + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, "SomePath", null, CancellationToken.None)); + } + + [Fact] + public async Task GetSubmodelTemplateAsync_WithIdShortPath_PassesQueryOptionsToProvider() + { + const string idShortPath = "ManufacturerName"; + var queryOptions = new SubmodelQueryOptions("deep", "withBlobValue"); + _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); + _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, queryOptions, Arg.Any()) + .Returns(TestData.CreateSubmodel()); - var exception = await Assert.ThrowsAsync(() => _sut.GetSubmodelTemplateAsync("", IdShortPath, null, CancellationToken.None)); - Assert.IsType(exception); + await _sut.GetSubmodelTemplateAsync(SubmodelId, idShortPath, queryOptions, CancellationToken.None); + + await _templateProvider.Received(1).GetFilteredSubmodelTemplateAsync(TemplateId, queryOptions, Arg.Any()); } + #endregion + + #region GetSubmodelTemplateAsync (list index paths) + [Fact] public async Task GetSubmodelTemplateAsync_ReturnsSubmodel_WhenPathContainsListIndex() { var expectedSubmodel = TestData.CreateSubmodelWithModel3DList(); - var path = "Model3D[0].ModelDataFile"; + const string path = "Model3D[0].ModelDataFile"; _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .Returns(expectedSubmodel); + .Returns(expectedSubmodel); var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, path, null, CancellationToken.None); @@ -171,12 +307,12 @@ public async Task GetSubmodelTemplateAsync_ReturnsSubmodel_WhenPathContainsListI public async Task GetSubmodelTemplateAsync_WithListIndexPath_ReturnsSubmodelWithCorrectIndexedElement() { var expectedSubmodel = TestData.CreateSubmodelWithModel3DList(); - const string Path = "Model3D[0]"; + const string path = "Model3D[0]"; _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .Returns(expectedSubmodel); + .Returns(expectedSubmodel); - var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, Path, null, CancellationToken.None); + var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, path, null, CancellationToken.None); Assert.Equal(GetSemanticId(expectedSubmodel), GetSemanticId(result)); @@ -194,13 +330,13 @@ public async Task GetSubmodelTemplateAsync_WithListIndexPath_ReturnsSubmodelWith public async Task GetSubmodelTemplateAsync_Supports_UrlEncoded_ListIndex() { var submodel = TestData.CreateSubmodelWithModel3DList(); - const string Path = "Model3D%5B0%5D.ModelDataFile"; + const string path = "Model3D%5B0%5D.ModelDataFile"; _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .Returns(submodel); + .Returns(submodel); - var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, Path, null, CancellationToken.None); + var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, path, null, CancellationToken.None); Assert.NotNull(result); } @@ -209,14 +345,14 @@ public async Task GetSubmodelTemplateAsync_Supports_UrlEncoded_ListIndex() public async Task GetSubmodelTemplateAsync_Throws_When_ListIndex_IsNegative() { var submodel = TestData.CreateSubmodelWithModel3DList(); - const string Path = "Model3D[-1]"; + const string path = "Model3D[-1]"; _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .Returns(submodel); + .Returns(submodel); - await Assert.ThrowsAsync( - () => _sut.GetSubmodelTemplateAsync(SubmodelId, Path, null, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, path, null, CancellationToken.None)); } [Fact] @@ -224,12 +360,12 @@ public async Task GetSubmodelTemplateAsync_ReturnsSubmodel_WhenTypeValueListElem { var expectedSubmodel = TestData.CreateSubmodelWithModel3DList(); var submodel = TestData.CreateSubmodelWithModel3DList(); - const string Path = "Model3D[5].ModelDataFile"; + const string path = "Model3D[5].ModelDataFile"; _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .Returns(submodel); + .Returns(submodel); - var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, Path, null, CancellationToken.None); + var result = await _sut.GetSubmodelTemplateAsync(SubmodelId, path, null, CancellationToken.None); Assert.Equal(GetSemanticId(expectedSubmodel), GetSemanticId(result)); @@ -245,89 +381,85 @@ public async Task GetSubmodelTemplateAsync_ReturnsSubmodel_WhenTypeValueListElem } [Fact] - public async Task GetSubmodelTemplateAsync_ThrowsInternalDataProcessingException_WhenTypeValueListElementIsSubmodelProperty_AndListIndexExceedsAvailableElements() + public async Task GetSubmodelTemplateAsync_ThrowsInternalDataProcessingException_WhenTypeValueListElementIsProperty_AndListIndexExceedsAvailableElements() { var submodel = TestData.CreateSubmodelWithPropertyInsideList(); - const string Path = "listProperty[2]"; + const string path = "listProperty[2]"; _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .Returns(submodel); + .Returns(submodel); - await Assert.ThrowsAsync(() => _sut.GetSubmodelTemplateAsync(SubmodelId, Path, null, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, path, null, CancellationToken.None)); } [Fact] public async Task GetSubmodelTemplateAsync_ThrowsNotFoundException_WhenPathSegmentHasListElement_AndIsInvalid() { var submodel = TestData.CreateSubmodelWithModel3DList(); - const string Path = "Model3D[0].NonExistentFile"; + const string path = "Model3D[0].NonExistentFile"; _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .Returns(submodel); + .Returns(submodel); - await Assert.ThrowsAsync(() => _sut.GetSubmodelTemplateAsync(SubmodelId, Path, null, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.GetSubmodelTemplateAsync(SubmodelId, path, null, CancellationToken.None)); } + #endregion + + #region ValidateSemanticIdFilter + [Fact] - public async Task GetSubmodelTemplateAsync_ThrowsSubmodelNotFoundException_WhenResourceNotFound() + public async Task ValidateSemanticIdFilter_ReturnsTrue_WhenTemplateIdMatchesFilteredTemplateId() { _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); - _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .ThrowsAsync(new ResourceNotFoundException()); - await Assert.ThrowsAsync( - () => _sut.GetSubmodelTemplateAsync(SubmodelId, CancellationToken.None)); + var result = await _sut.ValidateSemanticIdFilter(SubmodelId, TemplateId); + + Assert.True(result); } [Fact] - public async Task GetSubmodelTemplateAsync_ThrowsInternalDataProcessingException_WhenResponseParsingFails() + public async Task ValidateSemanticIdFilter_ReturnsFalse_WhenTemplateIdDoesNotMatchFilteredTemplateId() { _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); - _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .ThrowsAsync(new ResponseParsingException()); - var exception = await Assert.ThrowsAsync( - () => _sut.GetSubmodelTemplateAsync(SubmodelId, CancellationToken.None)); + var result = await _sut.ValidateSemanticIdFilter(SubmodelId, "different-template-id"); - Assert.Equal("Internal Server Error.", exception.Message); + Assert.False(result); } [Fact] - public async Task GetSubmodelTemplateAsync_ThrowsTemplateRequestFailedException_WhenRequestTimesOut() + public async Task ValidateSemanticIdFilter_ReturnsFalse_WhenMappingThrowsResourceNotFoundException() { - _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); - _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .ThrowsAsync(new RequestTimeoutException()); + _mappingProvider.GetTemplateId(SubmodelId).Throws(new ResourceNotFoundException()); + + var result = await _sut.ValidateSemanticIdFilter(SubmodelId, TemplateId); - await Assert.ThrowsAsync( - () => _sut.GetSubmodelTemplateAsync(SubmodelId, CancellationToken.None)); + Assert.False(result); } [Fact] - public async Task GetSubmodelTemplateAsync_ThrowsRepositoryNotAvailableException_WhenServiceUnavailable() + public async Task ValidateSemanticIdFilter_ThrowsInternalDataProcessingException_WhenSubmodelIdIsNull() { - _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); - _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .ThrowsAsync(new ServiceUnavailableException("http://fake-url")); - - await Assert.ThrowsAsync( - () => _sut.GetSubmodelTemplateAsync(SubmodelId, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.ValidateSemanticIdFilter(null!, TemplateId)); } [Fact] - public async Task GetSubmodelTemplateAsync_WithIdShortPath_ThrowsSubmodelElementNotFoundException_WhenResourceNotFound() + public async Task ValidateSemanticIdFilter_ThrowsInternalDataProcessingException_WhenSubmodelIdIsEmpty() { - _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); - _templateProvider.GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) - .ThrowsAsync(new ResourceNotFoundException()); - - await Assert.ThrowsAsync( - () => _sut.GetSubmodelTemplateAsync(SubmodelId, "SomePath", null, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.ValidateSemanticIdFilter("", TemplateId)); } + + #endregion + #region GetFilteredSubmodelTemplateAsync [Fact] - public async Task GetFilteredSubmodelTemplateAsync_ReturnsTemplate_WhenTemplateIdMatches() + public async Task GetFilteredSubmodelTemplateAsync_ReturnsTemplate_WhenValid() { var expectedSubmodel = Substitute.For(); _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); @@ -335,29 +467,37 @@ public async Task GetFilteredSubmodelTemplateAsync_ReturnsTemplate_WhenTemplateI .GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) .Returns(expectedSubmodel); - var result = await _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, TemplateId, null, CancellationToken.None); + var result = await _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, null, CancellationToken.None); Assert.Equal(expectedSubmodel, result); } [Fact] - public async Task GetFilteredSubmodelTemplateAsync_ReturnsNull_WhenFilteredTemplateIdDoesNotMatch() + public async Task GetFilteredSubmodelTemplateAsync_PassesQueryOptionsToProvider() { + var queryOptions = new SubmodelQueryOptions("deep", "withBlobValue"); _mappingProvider.GetTemplateId(SubmodelId).Returns(TemplateId); + _templateProvider + .GetFilteredSubmodelTemplateAsync(TemplateId, queryOptions, Arg.Any()) + .Returns(Substitute.For()); - var result = await _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, "different-template-id", null, CancellationToken.None); + await _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, queryOptions, CancellationToken.None); - Assert.Null(result); + await _templateProvider.Received(1).GetFilteredSubmodelTemplateAsync(TemplateId, queryOptions, Arg.Any()); } [Fact] - public async Task GetFilteredSubmodelTemplateAsync_ReturnsNull_WhenMappingThrowsResourceNotFoundException() + public async Task GetFilteredSubmodelTemplateAsync_ThrowsInternalDataProcessingException_WhenSubmodelIdIsNull() { - _mappingProvider.GetTemplateId(SubmodelId).Throws(new ResourceNotFoundException()); - - var result = await _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, TemplateId, null, CancellationToken.None); + await Assert.ThrowsAsync(() => + _sut.GetFilteredSubmodelTemplateAsync(null!, null, CancellationToken.None)); + } - Assert.Null(result); + [Fact] + public async Task GetFilteredSubmodelTemplateAsync_ThrowsInternalDataProcessingException_WhenSubmodelIdIsEmpty() + { + await Assert.ThrowsAsync(() => + _sut.GetFilteredSubmodelTemplateAsync("", null, CancellationToken.None)); } [Fact] @@ -368,8 +508,8 @@ public async Task GetFilteredSubmodelTemplateAsync_ThrowsInternalDataProcessingE .GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) .ThrowsAsync(new ResponseParsingException()); - await Assert.ThrowsAsync( - () => _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, TemplateId, null, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, null, CancellationToken.None)); } [Fact] @@ -380,8 +520,8 @@ public async Task GetFilteredSubmodelTemplateAsync_ThrowsTemplateRequestFailedEx .GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) .ThrowsAsync(new RequestTimeoutException()); - await Assert.ThrowsAsync( - () => _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, TemplateId, null, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, null, CancellationToken.None)); } [Fact] @@ -392,8 +532,8 @@ public async Task GetFilteredSubmodelTemplateAsync_ThrowsRepositoryNotAvailableE .GetFilteredSubmodelTemplateAsync(TemplateId, null, Arg.Any()) .ThrowsAsync(new ServiceUnavailableException("down")); - await Assert.ThrowsAsync( - () => _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, TemplateId, null, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.GetFilteredSubmodelTemplateAsync(SubmodelId, null, CancellationToken.None)); } #endregion @@ -436,8 +576,8 @@ public async Task GetFilteredSubmodelTemplateIdAsync_ThrowsInternalDataProcessin .GetFilteredSubmodelTemplateBySemanticIdAsync(SemanticId, Arg.Any()) .ThrowsAsync(new ResponseParsingException()); - await Assert.ThrowsAsync( - () => _sut.GetFilteredSubmodelTemplateIdAsync(SemanticId, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.GetFilteredSubmodelTemplateIdAsync(SemanticId, CancellationToken.None)); } [Fact] @@ -448,8 +588,8 @@ public async Task GetFilteredSubmodelTemplateIdAsync_ThrowsTemplateRequestFailed .GetFilteredSubmodelTemplateBySemanticIdAsync(SemanticId, Arg.Any()) .ThrowsAsync(new RequestTimeoutException()); - await Assert.ThrowsAsync( - () => _sut.GetFilteredSubmodelTemplateIdAsync(SemanticId, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.GetFilteredSubmodelTemplateIdAsync(SemanticId, CancellationToken.None)); } [Fact] @@ -460,9 +600,11 @@ public async Task GetFilteredSubmodelTemplateIdAsync_ThrowsRepositoryNotAvailabl .GetFilteredSubmodelTemplateBySemanticIdAsync(SemanticId, Arg.Any()) .ThrowsAsync(new ServiceUnavailableException("down")); - await Assert.ThrowsAsync( - () => _sut.GetFilteredSubmodelTemplateIdAsync(SemanticId, CancellationToken.None)); + await Assert.ThrowsAsync(() => + _sut.GetFilteredSubmodelTemplateIdAsync(SemanticId, CancellationToken.None)); } #endregion + + private static string GetSemanticId(IHasSemantics hasSemantics) => hasSemantics.SemanticId?.Keys?.FirstOrDefault()?.Value ?? string.Empty; } diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/ISubmodelTemplateService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/ISubmodelTemplateService.cs index fa6f2855..87eccad3 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/ISubmodelTemplateService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/ISubmodelTemplateService.cs @@ -11,7 +11,9 @@ public interface ISubmodelTemplateService Task GetSubmodelTemplateAsync(string submodelId, string idShortPath, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken); - Task GetFilteredSubmodelTemplateAsync(string submodelId, string filteredTemplateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken); + Task GetFilteredSubmodelTemplateAsync(string submodelId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken); + + Task ValidateSemanticIdFilter(string submodelId, string filteredTemplateId); Task GetFilteredSubmodelTemplateIdAsync(string semanticId, CancellationToken cancellationToken); } diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 797bda52..988284ae 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -38,7 +38,7 @@ public async Task GetSubmodelAsync(string submodelId, SubmodelQueryOp { return await ExecuteWithExceptionHandlingAsync(async () => { - var submodelTemplate = await submodelTemplateService.GetFilteredSubmodelTemplateAsync(submodelId, null, queryOptions, cancellationToken).ConfigureAwait(false); + var submodelTemplate = await submodelTemplateService.GetFilteredSubmodelTemplateAsync(submodelId, queryOptions, cancellationToken).ConfigureAwait(false); if (submodelTemplate is null) { @@ -84,9 +84,9 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte }; var pageSize = limit ?? 100; - var paginationResult = await CollectSubmodelPageAsync(shellSearchFilter, pageSize, cursor, cancellationToken).ConfigureAwait(false); + var paginationResult = await CollectSubmodelPageAsync(shellSearchFilter, filteredTemplateId, pageSize, cursor, cancellationToken).ConfigureAwait(false); - var submodels = await BuildSubmodelsAsync(paginationResult.SubmodelIds, filteredTemplateId, queryOptions, cancellationToken).ConfigureAwait(false); + var submodels = await BuildSubmodelsAsync(paginationResult.SubmodelIds, queryOptions, cancellationToken).ConfigureAwait(false); return new SubmodelList { @@ -96,7 +96,7 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte }, ex => new SubmodelNotFoundException(ex)).ConfigureAwait(false); } - private async Task CollectSubmodelPageAsync(ShellSearchFilter shellSearchFilter, int pageSize, string? encodedCursor, CancellationToken cancellationToken) + private async Task CollectSubmodelPageAsync(ShellSearchFilter shellSearchFilter, string filteredTemplateId, int pageSize, string? encodedCursor, CancellationToken cancellationToken) { var incomingCursor = SubmodelPaginationCursor.Decode(encodedCursor); var state = new SubmodelPaginationState(incomingCursor, pageSize); @@ -113,7 +113,7 @@ private async Task CollectSubmodelPageAsync(ShellSearchFilte break; } - var limitReached = await ProcessShellBatchAsync(shellDescriptors, pageSize, state, cancellationToken).ConfigureAwait(false); + var limitReached = await ProcessShellBatchAsync(shellDescriptors, filteredTemplateId, pageSize, state, cancellationToken).ConfigureAwait(false); if (limitReached) { @@ -131,7 +131,7 @@ private async Task CollectSubmodelPageAsync(ShellSearchFilte return new SubmodelPageResult(state.CollectedIds, state.BuildNextCursor(pageSize)); } - private async Task ProcessShellBatchAsync(IReadOnlyList shellDescriptors, int pageSize, SubmodelPaginationState state, CancellationToken cancellationToken) + private async Task ProcessShellBatchAsync(IReadOnlyList shellDescriptors, string filteredTemplateId, int pageSize, SubmodelPaginationState state, CancellationToken cancellationToken) { var prefetchTasks = new Task>[shellDescriptors.Count]; using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); @@ -145,7 +145,7 @@ private async Task ProcessShellBatchAsync(IReadOnlyList ProcessShellBatchAsync(IReadOnlyList> PrefetchSubmodelIdsAsync(string shellId, SemaphoreSlim semaphore, CancellationToken cancellationToken) + private async Task> PrefetchSubmodelIdsAsync(string shellId, string filteredTemplateId, SemaphoreSlim semaphore, CancellationToken cancellationToken) { await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); try { - return await GetSubmodelIdsForShellAsync(shellId, cancellationToken).ConfigureAwait(false); + return await GetSubmodelIdsForShellAsync(shellId, filteredTemplateId, cancellationToken).ConfigureAwait(false); } finally { @@ -182,30 +182,41 @@ private async Task> PrefetchSubmodelIdsAsync(string shellId, Semaph } } - private async Task> GetSubmodelIdsForShellAsync(string shellId, CancellationToken cancellationToken) + private async Task> GetSubmodelIdsForShellAsync(string shellId, string filteredTemplateId, CancellationToken cancellationToken) { try { var references = await aasRepositoryTemplateService.GetSubmodelRefByIdAsync(shellId, cancellationToken).ConfigureAwait(false); - return references.Select(r => r.Keys.FirstOrDefault()?.Value).Where(id => !string.IsNullOrWhiteSpace(id)).ToList(); + var submodelIds = references.Select(reference => reference.Keys.FirstOrDefault()?.Value).Where(id => !string.IsNullOrWhiteSpace(id)).ToList(); + + var validationTasks = submodelIds.Select(async id => + new + { + Id = id!, + IsValid = await submodelTemplateService.ValidateSemanticIdFilter(id!, filteredTemplateId).ConfigureAwait(false) + }); + + var results = await Task.WhenAll(validationTasks).ConfigureAwait(false); + + return [.. results.Where(result => result.IsValid).Select(result => result.Id)]; } catch (ResourceNotFoundException ex) { logger.LogWarning(ex, "Could not retrieve submodel refs for shell {ShellId}. Skipping shell.", shellId); + return []; } } - - private async Task> BuildSubmodelsAsync(List submodelIds, string? filteredTemplateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) + private async Task> BuildSubmodelsAsync(List submodelIds, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) { using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); var tasks = new Task[submodelIds.Count]; for (var i = 0; i < submodelIds.Count; i++) { - tasks[i] = BuildSingleSubmodelAsync(submodelIds[i], filteredTemplateId, queryOptions, semaphore, cancellationToken); + tasks[i] = BuildSingleSubmodelAsync(submodelIds[i], queryOptions, semaphore, cancellationToken); } var results = await Task.WhenAll(tasks).ConfigureAwait(false); @@ -216,17 +227,12 @@ private async Task> BuildSubmodelsAsync(List submodelIds return submodels; } - private async Task BuildSingleSubmodelAsync(string submodelId, string? filteredTemplateId, SubmodelQueryOptions? queryOptions, SemaphoreSlim semaphore, CancellationToken cancellationToken) + private async Task BuildSingleSubmodelAsync(string submodelId, SubmodelQueryOptions? queryOptions, SemaphoreSlim semaphore, CancellationToken cancellationToken) { await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); try { - var template = await submodelTemplateService.GetFilteredSubmodelTemplateAsync(submodelId, filteredTemplateId, queryOptions, cancellationToken).ConfigureAwait(false); - - if (template is null) - { - return null; - } + var template = await submodelTemplateService.GetFilteredSubmodelTemplateAsync(submodelId, queryOptions, cancellationToken).ConfigureAwait(false); return await BuildSubmodelWithValuesAsync(template, submodelId, cancellationToken).ConfigureAwait(false); } @@ -240,7 +246,7 @@ public async Task GetAllSubmodelElementsAsync(string submo { return await ExecuteWithExceptionHandlingAsync(async () => { - var submodelTemplate = await submodelTemplateService.GetFilteredSubmodelTemplateAsync(submodelId, null, queryOptions, cancellationToken).ConfigureAwait(false); + var submodelTemplate = await submodelTemplateService.GetFilteredSubmodelTemplateAsync(submodelId, queryOptions, cancellationToken).ConfigureAwait(false); if (submodelTemplate is null) { diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelTemplateService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelTemplateService.cs index 8f1ba0fa..bd8df631 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelTemplateService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelTemplateService.cs @@ -83,7 +83,7 @@ public async Task GetSubmodelTemplateAsync(string submodelId, string throw new InvalidDependencyException(nameof(idShortPath)); } - public async Task GetFilteredSubmodelTemplateAsync(string submodelId, string filteredTemplateId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) + public async Task ValidateSemanticIdFilter(string submodelId, string filteredTemplateId) { ValidateSubmodelId(submodelId); @@ -94,16 +94,25 @@ public async Task GetSubmodelTemplateAsync(string submodelId, string if (filteredTemplateId is not null && templateId != filteredTemplateId) { - return null; + return false; } + return true; } catch (ResourceNotFoundException) { - return null; + return false; } + } + + public async Task GetFilteredSubmodelTemplateAsync(string submodelId, SubmodelQueryOptions? queryOptions, CancellationToken cancellationToken) + { + ValidateSubmodelId(submodelId); + string? templateId; try { + templateId = _submodelTemplateMappingProvider.GetTemplateId(submodelId); + return await _templateProvider.GetFilteredSubmodelTemplateAsync(templateId!, queryOptions, cancellationToken).ConfigureAwait(false); } catch (ResponseParsingException ex) From 0e9bbf944e565c4c209861067a04fe8aa20853fc Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Fri, 21 Aug 2026 00:03:27 +0530 Subject: [PATCH 25/30] refactor: clean up code formatting and remove unnecessary null-forgiving operators --- .../Api/SubmodelRepository/SubmodelRepositoryController.cs | 2 +- .../Services/SubmodelRepository/SubmodelRepositoryService.cs | 4 ++-- .../Http/Clients/Caching/CachedGetRequestClient.cs | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/SubmodelRepositoryController.cs b/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/SubmodelRepositoryController.cs index e4490393..f90fd7dd 100644 --- a/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/SubmodelRepositoryController.cs +++ b/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/SubmodelRepositoryController.cs @@ -56,7 +56,7 @@ public async Task> GetAllSubmodelsAsync( { logger.LogInformation("Get All Submodels"); - var request = new GetAllSubmodelsRequest{SemanticId = semanticId, IdShort = idShort, Limit = limit, Cursor = cursor, Level = level, Extent = extent}; + var request = new GetAllSubmodelsRequest { SemanticId = semanticId, IdShort = idShort, Limit = limit, Cursor = cursor, Level = level, Extent = extent }; var response = await submodelRepositoryHandler .GetAllSubmodels(request, cancellationToken) diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 988284ae..90bf070b 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -193,8 +193,8 @@ private async Task> GetSubmodelIdsForShellAsync(string shellId, str var validationTasks = submodelIds.Select(async id => new { - Id = id!, - IsValid = await submodelTemplateService.ValidateSemanticIdFilter(id!, filteredTemplateId).ConfigureAwait(false) + Id = id, + IsValid = await submodelTemplateService.ValidateSemanticIdFilter(id, filteredTemplateId).ConfigureAwait(false) }); var results = await Task.WhenAll(validationTasks).ConfigureAwait(false); diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs index 9c6a3c05..bc4c4df0 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Http/Clients/Caching/CachedGetRequestClient.cs @@ -9,9 +9,7 @@ using AAS.TwinEngine.DataEngine.Infrastructure.Logging; using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; -using Microsoft.AspNetCore.Http; using Microsoft.Extensions.Caching.Hybrid; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using UnauthorizedAccessException = AAS.TwinEngine.DataEngine.ApplicationLogic.Exceptions.Infrastructure.UnauthorizedAccessException; From fb7a142ba89f82ce59b60f3d6e63cf7715dbb63a Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Fri, 21 Aug 2026 13:20:34 +0530 Subject: [PATCH 26/30] refactor: update GetAllSubmodelsRequest to use constructor parameters and improve null handling in related tests --- .../Handler/SubmodelRepositoryHandlerTests.cs | 8 +++---- .../SubmodelRepositoryControllerTests.cs | 7 +------ .../Requests/GetAllSubmodelsRequest.cs | 21 +++++++------------ .../SubmodelRepositoryController.cs | 2 +- .../Services/Shared/SubmodelPagination.cs | 4 ++-- .../SubmodelRepositoryService.cs | 17 +++++++++------ 6 files changed, 26 insertions(+), 33 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Api/SubmodelRepository/Handler/SubmodelRepositoryHandlerTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Api/SubmodelRepository/Handler/SubmodelRepositoryHandlerTests.cs index 83b2b68b..131479c5 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Api/SubmodelRepository/Handler/SubmodelRepositoryHandlerTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Api/SubmodelRepository/Handler/SubmodelRepositoryHandlerTests.cs @@ -309,7 +309,7 @@ public async Task HandleSubmodelElement_NullOrEmptyIdShortPath_ThrowsInvalidUser [Fact] public async Task GetAllSubmodels_ReturnsSubmodelsDto_WhenServiceSucceeds() { - var request = new GetAllSubmodelsRequest(); + var request = new GetAllSubmodelsRequest(null, null, null, null, null, null); var submodelList = new SubmodelList { PagingMetaData = new DomainModel.Shared.PagingMetaData(), Result = [] }; _submodelRepository.GetAllSubmodelsAsync(Arg.Any(), Arg.Any(), null, null, Arg.Any()) .Returns(submodelList); @@ -323,7 +323,7 @@ public async Task GetAllSubmodels_ReturnsSubmodelsDto_WhenServiceSucceeds() [Fact] public async Task GetAllSubmodels_WithInvalidLimit_ThrowsInvalidUserInputException() { - var request = new GetAllSubmodelsRequest { Limit = 0 }; + var request = new GetAllSubmodelsRequest(null, null, 0, null, null, null); await Assert.ThrowsAsync(() => _sut.GetAllSubmodels(request, CancellationToken.None)); } @@ -333,7 +333,7 @@ public async Task GetAllSubmodels_BuildsFilterWithSemanticIdAndIdShort() { const string SemanticId = "https://example.com/semanticId"; const string IdShort = "Nameplate"; - var request = new GetAllSubmodelsRequest { SemanticId = SemanticId, IdShort = IdShort }; + var request = new GetAllSubmodelsRequest(SemanticId, IdShort, null, null, null, null); var submodelList = new SubmodelList { PagingMetaData = new DomainModel.Shared.PagingMetaData(), Result = [] }; _submodelRepository.GetAllSubmodelsAsync( Arg.Is(f => f.SemanticId == SemanticId && f.IdShort == IdShort), @@ -356,7 +356,7 @@ await _submodelRepository.Received(1).GetAllSubmodelsAsync( [Fact] public async Task GetAllSubmodels_WhenLevelAndExtentSet_BuildsQueryOptions() { - var request = new GetAllSubmodelsRequest { Level = Level.deep, Extent = Extent.withBlobValue }; + var request = new GetAllSubmodelsRequest(null, null, null, null, Level.deep, Extent.withBlobValue); var submodelList = new SubmodelList { PagingMetaData = new DomainModel.Shared.PagingMetaData(), Result = [] }; _submodelRepository.GetAllSubmodelsAsync( Arg.Any(), diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Api/SubmodelRepository/SubmodelRepositoryControllerTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Api/SubmodelRepository/SubmodelRepositoryControllerTests.cs index 0f3152a3..bc4c749b 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Api/SubmodelRepository/SubmodelRepositoryControllerTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Api/SubmodelRepository/SubmodelRepositoryControllerTests.cs @@ -141,12 +141,7 @@ public async Task GetAllSubmodelsAsync_PassesQueryParamsToHandler() var expectedDto = new SubmodelsDto { PagingMetaData = new AAS.TwinEngine.DataEngine.Api.Shared.PagingMetaDataDto(), Result = [] }; _handler.GetAllSubmodels(Arg.Any(), Arg.Any()) .Returns(expectedDto); - var request = new GetAllSubmodelsRequest - { - SemanticId = SemanticId, - IdShort = IdShort, - Limit = Limit - }; + var request = new GetAllSubmodelsRequest(SemanticId, IdShort, Limit, null, null, null); await _sut.GetAllSubmodelsAsync(SemanticId, IdShort, Limit, null, CancellationToken.None); diff --git a/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/Requests/GetAllSubmodelsRequest.cs b/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/Requests/GetAllSubmodelsRequest.cs index 50f04779..13caa939 100644 --- a/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/Requests/GetAllSubmodelsRequest.cs +++ b/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/Requests/GetAllSubmodelsRequest.cs @@ -1,19 +1,12 @@ namespace AAS.TwinEngine.DataEngine.Api.SubmodelRepository.Requests; -public record GetAllSubmodelsRequest -{ - public string? SemanticId { get; set; } - - public string? IdShort { get; set; } - - public int? Limit { get; set; } - - public string? Cursor { get; set; } - - public Level? Level { get; set; } - - public Extent? Extent { get; set; } -} +public record GetAllSubmodelsRequest( + string? SemanticId, + string? IdShort, + int? Limit, + string? Cursor, + Level? Level, + Extent? Extent); public enum Level { diff --git a/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/SubmodelRepositoryController.cs b/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/SubmodelRepositoryController.cs index f90fd7dd..f4d1fd1d 100644 --- a/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/SubmodelRepositoryController.cs +++ b/source/AAS.TwinEngine.DataEngine/Api/SubmodelRepository/SubmodelRepositoryController.cs @@ -56,7 +56,7 @@ public async Task> GetAllSubmodelsAsync( { logger.LogInformation("Get All Submodels"); - var request = new GetAllSubmodelsRequest { SemanticId = semanticId, IdShort = idShort, Limit = limit, Cursor = cursor, Level = level, Extent = extent }; + var request = new GetAllSubmodelsRequest(semanticId, idShort, limit, cursor, level, extent); var response = await submodelRepositoryHandler .GetAllSubmodels(request, cancellationToken) diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Shared/SubmodelPagination.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Shared/SubmodelPagination.cs index 7e677249..00c7a311 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Shared/SubmodelPagination.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/Shared/SubmodelPagination.cs @@ -1,4 +1,4 @@ -using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; +using AAS.TwinEngine.DataEngine.DomainModel.SubmodelRepository; namespace AAS.TwinEngine.DataEngine.ApplicationLogic.Services.Shared; @@ -11,7 +11,7 @@ internal sealed class SubmodelPaginationState(SubmodelPaginationCursor? cursor, public string? LastCollectedSubmodelId { get; set; } public string? ResumeAfterSubmodelId { get; set; } = cursor?.SubmodelId; - public bool CollectSubmodelIds(IList submodelIds, string shellId, int pageSize) + public bool CollectSubmodelIds(IList submodelIds, string shellId, int pageSize) { if (submodelIds.Count == 0) { diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 90bf070b..1c007176 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -96,7 +96,7 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte }, ex => new SubmodelNotFoundException(ex)).ConfigureAwait(false); } - private async Task CollectSubmodelPageAsync(ShellSearchFilter shellSearchFilter, string filteredTemplateId, int pageSize, string? encodedCursor, CancellationToken cancellationToken) + private async Task CollectSubmodelPageAsync(ShellSearchFilter shellSearchFilter, string? filteredTemplateId, int pageSize, string? encodedCursor, CancellationToken cancellationToken) { var incomingCursor = SubmodelPaginationCursor.Decode(encodedCursor); var state = new SubmodelPaginationState(incomingCursor, pageSize); @@ -131,9 +131,9 @@ private async Task CollectSubmodelPageAsync(ShellSearchFilte return new SubmodelPageResult(state.CollectedIds, state.BuildNextCursor(pageSize)); } - private async Task ProcessShellBatchAsync(IReadOnlyList shellDescriptors, string filteredTemplateId, int pageSize, SubmodelPaginationState state, CancellationToken cancellationToken) + private async Task ProcessShellBatchAsync(IReadOnlyList shellDescriptors, string? filteredTemplateId, int pageSize, SubmodelPaginationState state, CancellationToken cancellationToken) { - var prefetchTasks = new Task>[shellDescriptors.Count]; + var prefetchTasks = new Task>[shellDescriptors.Count]; using var semaphore = new SemaphoreSlim(_concurrentOperationsLimit, _concurrentOperationsLimit); for (var idx = 0; idx < shellDescriptors.Count; idx++) @@ -141,7 +141,7 @@ private async Task ProcessShellBatchAsync(IReadOnlyList>([]); + prefetchTasks[idx] = Task.FromResult>([]); continue; } @@ -169,7 +169,7 @@ private async Task ProcessShellBatchAsync(IReadOnlyList> PrefetchSubmodelIdsAsync(string shellId, string filteredTemplateId, SemaphoreSlim semaphore, CancellationToken cancellationToken) + private async Task> PrefetchSubmodelIdsAsync(string shellId, string? filteredTemplateId, SemaphoreSlim semaphore, CancellationToken cancellationToken) { await semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); try @@ -182,7 +182,7 @@ private async Task> PrefetchSubmodelIdsAsync(string shellId, string } } - private async Task> GetSubmodelIdsForShellAsync(string shellId, string filteredTemplateId, CancellationToken cancellationToken) + private async Task> GetSubmodelIdsForShellAsync(string shellId, string? filteredTemplateId, CancellationToken cancellationToken) { try { @@ -190,6 +190,11 @@ private async Task> GetSubmodelIdsForShellAsync(string shellId, str var submodelIds = references.Select(reference => reference.Keys.FirstOrDefault()?.Value).Where(id => !string.IsNullOrWhiteSpace(id)).ToList(); + if (string.IsNullOrWhiteSpace(filteredTemplateId)) + { + return submodelIds; + } + var validationTasks = submodelIds.Select(async id => new { From 703c91927c0f4a2a7730f4e7c5bdabaee5591ae5 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Wed, 26 Aug 2026 12:39:16 +0530 Subject: [PATCH 27/30] feat: implement default pagination limit in various services and handlers --- .../Services/PluginDataHandlerTests.cs | 10 +++++----- .../ApplicationLogic/Extensions/PagingExtensions.cs | 3 ++- .../SubmodelRegistry/SubmodelDescriptorService.cs | 2 +- .../SubmodelRepository/SubmodelRepositoryService.cs | 8 ++++++-- .../SubmodelRepository/SubmodelPaginationCursor.cs | 11 ++++++++++- .../PluginDataProvider/Services/PluginDataHandler.cs | 4 ++++ .../ServiceConfiguration/Config/GeneralConfig.cs | 1 + 7 files changed, 29 insertions(+), 10 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs index 1d86d6a7..89f903a5 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandlerTests.cs @@ -1,4 +1,4 @@ -using System.Net; +using System.Net; using System.Net.Http.Json; using System.Text; using System.Text.Json; @@ -188,7 +188,7 @@ public async Task GetDataForAllShellDescriptorsAsync_ReturnsListWithHrefSet() .Returns([new($"{HttpClientNames.PluginDataProviderPrefix}PluginA", "")]); _pluginDataProvider - .GetDataForAllShellDescriptorsAsync(null, null, Arg.Any>(), Arg.Any()) + .GetDataForAllShellDescriptorsAsync(Arg.Any(), null, Arg.Any>(), Arg.Any()) .Returns([json]); var result = await _sut.GetDataForAllShellDescriptorsAsync(null, null, manifests, CancellationToken.None); @@ -220,7 +220,7 @@ public async Task GetDataForAllShellDescriptorsAsync_Throws_WhenDeserializationF }; _pluginDataProvider - .GetDataForAllShellDescriptorsAsync(null, null, Arg.Any>(), Arg.Any()) + .GetDataForAllShellDescriptorsAsync(Arg.Any(), null, Arg.Any>(), Arg.Any()) .Returns(["null"]); await Assert.ThrowsAsync(() => @@ -263,7 +263,7 @@ public async Task GetDataForAllShellDescriptorsAsync_ThrowsAndLogsIdentifiers_Wh }; _pluginDataProvider - .GetDataForAllShellDescriptorsAsync(null, null, Arg.Any>(), Arg.Any()) + .GetDataForAllShellDescriptorsAsync(Arg.Any(), null, Arg.Any>(), Arg.Any()) .Returns([json]); await Assert.ThrowsAsync(() => @@ -315,7 +315,7 @@ public async Task GetDataForAllShellDescriptorsAsync_ThrowsAndLogsNullMarkers_Wh }; _pluginDataProvider - .GetDataForAllShellDescriptorsAsync(null, null, Arg.Any>(), Arg.Any()) + .GetDataForAllShellDescriptorsAsync(Arg.Any(), null, Arg.Any>(), Arg.Any()) .Returns([json]); await Assert.ThrowsAsync(() => diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Extensions/PagingExtensions.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Extensions/PagingExtensions.cs index 67e0af42..2bd58064 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Extensions/PagingExtensions.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Extensions/PagingExtensions.cs @@ -1,4 +1,5 @@ using AAS.TwinEngine.DataEngine.DomainModel.Shared; +using AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; namespace AAS.TwinEngine.DataEngine.ApplicationLogic.Extensions; @@ -17,7 +18,7 @@ public static (IList Items, PagingMetaData PagingMetaData) GetPagedResult( startIndex = allItems.ToList().FindIndex(item => getId(item) == lastId) + 1; } - var pageSize = limit ?? 100; + var pageSize = limit ?? GeneralConfig.DefaultPaginationLimit; var pagedItems = allItems.Skip(startIndex).Take(pageSize).ToList(); string? nextCursor = null; diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs index 152de5fb..40123816 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRegistry/SubmodelDescriptorService.cs @@ -29,7 +29,7 @@ public class SubmodelDescriptorService( public async Task GetAllSubmodelDescriptorsAsync(int? limit, string? cursor, CancellationToken cancellationToken) { - var pageSize = limit ?? 100; + var pageSize = limit ?? GeneralConfig.DefaultPaginationLimit; var paginationResult = await CollectSubmodelDescriptorPageAsync(pageSize, cursor, cancellationToken).ConfigureAwait(false); var submodelIds = paginationResult.SubmodelIds; diff --git a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs index 1c007176..bbbb86f0 100644 --- a/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs +++ b/source/AAS.TwinEngine.DataEngine/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryService.cs @@ -83,7 +83,7 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte IdShort = filter?.IdShort }; - var pageSize = limit ?? 100; + var pageSize = limit ?? GeneralConfig.DefaultPaginationLimit; var paginationResult = await CollectSubmodelPageAsync(shellSearchFilter, filteredTemplateId, pageSize, cursor, cancellationToken).ConfigureAwait(false); var submodels = await BuildSubmodelsAsync(paginationResult.SubmodelIds, queryOptions, cancellationToken).ConfigureAwait(false); @@ -99,6 +99,10 @@ public async Task GetAllSubmodelsAsync(SubmodelSearchFilter? filte private async Task CollectSubmodelPageAsync(ShellSearchFilter shellSearchFilter, string? filteredTemplateId, int pageSize, string? encodedCursor, CancellationToken cancellationToken) { var incomingCursor = SubmodelPaginationCursor.Decode(encodedCursor); + if (incomingCursor is null && !string.IsNullOrWhiteSpace(encodedCursor)) + { + throw new InvalidUserInputException(); + } var state = new SubmodelPaginationState(incomingCursor, pageSize); var pluginCursor = state.TrackingAasId; @@ -243,7 +247,7 @@ private async Task> BuildSubmodelsAsync(List submodelIds } finally { - semaphore.Release(); + _ = semaphore.Release(); } } diff --git a/source/AAS.TwinEngine.DataEngine/DomainModel/SubmodelRepository/SubmodelPaginationCursor.cs b/source/AAS.TwinEngine.DataEngine/DomainModel/SubmodelRepository/SubmodelPaginationCursor.cs index 0598585e..00dd33fa 100644 --- a/source/AAS.TwinEngine.DataEngine/DomainModel/SubmodelRepository/SubmodelPaginationCursor.cs +++ b/source/AAS.TwinEngine.DataEngine/DomainModel/SubmodelRepository/SubmodelPaginationCursor.cs @@ -19,7 +19,16 @@ public sealed record SubmodelPaginationCursor(string? SubmodelId, string? AasId) return null; } - var decoded = Encoding.UTF8.GetString(WebEncoders.Base64UrlDecode(encodedCursor)); + string decoded; + try + { + decoded = Encoding.UTF8.GetString(WebEncoders.Base64UrlDecode(encodedCursor)); + } + catch (FormatException) + { + return null; + } + var separatorIndex = decoded.IndexOf(Separator); if (separatorIndex < 0) diff --git a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs index 266b9bcc..d6203903 100644 --- a/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs +++ b/source/AAS.TwinEngine.DataEngine/Infrastructure/Providers/PluginDataProvider/Services/PluginDataHandler.cs @@ -74,6 +74,8 @@ public async Task GetDataForAllShellDescriptorsAsync(i { using var activity = DataEngineTracing.StartSpan(DataEngineTracing.Spans.GetPluginMetadataShells); + limit ??= GeneralConfig.DefaultPaginationLimit; + var availablePlugins = multiPluginDataHandler.GetAvailablePlugins(pluginManifests, c => c.HasShellDescriptor); var pluginRequests = pluginRequestBuilder.Build(availablePlugins); @@ -224,6 +226,8 @@ public async Task GetDataForShellsByAssetIdsAsync(IRea })) : null; + limit ??= GeneralConfig.DefaultPaginationLimit; + var responses = await pluginDataProvider.GetDataForShellDescriptorsByAssetIdsAsync(pluginRequests, assetIdsHeaderValue, filter?.IdShort, limit, cursor, cancellationToken).ConfigureAwait(false); var result = new ShellDescriptorsMetaData(); diff --git a/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/GeneralConfig.cs b/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/GeneralConfig.cs index b2d4f417..245dcf43 100644 --- a/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/GeneralConfig.cs +++ b/source/AAS.TwinEngine.DataEngine/ServiceConfiguration/Config/GeneralConfig.cs @@ -10,6 +10,7 @@ namespace AAS.TwinEngine.DataEngine.ServiceConfiguration.Config; public class GeneralConfig { public const string Section = "General"; + public const int DefaultPaginationLimit = 100; public ApiConfiguration ApiConfiguration { get; set; } = new(); public HeaderSanitizationOptions HeaderSanitization { get; set; } = new(); From 27dc822edf8af694c2aec3ef35d4ee294110b5fb Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Wed, 26 Aug 2026 14:49:39 +0530 Subject: [PATCH 28/30] fix: update cursor in paging metadata for GetAllShellDescriptors_Expected.json --- .../AasRegistry/TestData/GetAllShellDescriptors_Expected.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetAllShellDescriptors_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetAllShellDescriptors_Expected.json index 35d90ce2..f49e0ea2 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetAllShellDescriptors_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetAllShellDescriptors_Expected.json @@ -1,5 +1,5 @@ { - "paging_metadata": { "cursor": null }, + "paging_metadata": { "cursor": "aHR0cHM6Ly9tbS1zb2Z0d2FyZS5jb20vaWRzL2Fhcy8wMDAtMDAz" }, "result": [ { "description": null, From 58e184d64ddda7b1a53a8215ebb877ef5a1ac240 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Thu, 27 Aug 2026 10:54:13 +0530 Subject: [PATCH 29/30] fix: update idShort values from "M&M" to "MM" across various test files and mock data --- .../SubmodelRepository/SubmodelRepositoryServiceTests.cs | 2 +- .../TestData/GetAllShellDescriptors_Expected.json | 6 +++--- .../TestData/GetShellDescriptorById_Expected.json | 2 +- .../AasRepository/AasRepositoryTests.cs | 4 ++-- .../GetAllShells_ByAssetId_And_ByIdShort_Expected.json | 2 +- .../TestData/GetAllShells_ByAssetIds_Expected.json | 2 +- .../TestData/GetAllShells_ByIdShort_Expected.json | 2 +- .../TestData/GetAllShells_ByMultipleAssetIds_Expected.json | 2 +- .../TestData/GetAllShells_WithCursor_Expected.json | 2 +- .../TestData/GetAllShells_WithLimit_Expected.json | 2 +- .../SubmodelRepository/GetAllSubmodelsTests.cs | 2 +- .../SubmodelRepository/SubmodelTests.cs | 2 +- .../Data/mock-metadata.json | 6 +++--- .../Get Shells By Asset Id and IdShort - M&M03.bru | 4 ++-- .../Aas Repository/Get Shells By IdShort - M&M03.bru | 4 ++-- .../Submodel/Get All Submodels - by idShort.bru | 2 +- .../Example/apiCollection/collection.bru | 7 +------ .../Example/plugin/mock-metadata.json | 6 +++--- 18 files changed, 27 insertions(+), 32 deletions(-) diff --git a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs index 89b284c8..3fc8f7d7 100644 --- a/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs +++ b/source/AAS.TwinEngine.DataEngine.UnitTests/ApplicationLogic/Services/SubmodelRepository/SubmodelRepositoryServiceTests.cs @@ -291,7 +291,7 @@ public async Task GetAllSubmodelsAsync_SkipsShell_WhenSubmodelRefsFails() [Fact] public async Task GetAllSubmodelsAsync_FiltersShellsByIdShort_WhenIdShortProvided() { - const string IdShort = "M&M01"; + const string IdShort = "MM01"; var filter = new SubmodelSearchFilter { IdShort = IdShort }; _pluginDataHandler diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetAllShellDescriptors_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetAllShellDescriptors_Expected.json index f49e0ea2..06189776 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetAllShellDescriptors_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetAllShellDescriptors_Expected.json @@ -23,7 +23,7 @@ } ], "globalAssetId": "https://mm-software.com/ids/assets/000-001", - "idShort": "M&M01", + "idShort": "MM01", "id": "https://mm-software.com/ids/aas/000-001", "specificAssetIds": [ { @@ -99,7 +99,7 @@ } ], "globalAssetId": "https://mm-software.com/ids/assets/000-002", - "idShort": "M&M02", + "idShort": "MM02", "id": "https://mm-software.com/ids/aas/000-002", "specificAssetIds": [ { @@ -175,7 +175,7 @@ } ], "globalAssetId": "https://mm-software.com/ids/assets/000-003", - "idShort": "M&M03", + "idShort": "MM03", "id": "https://mm-software.com/ids/aas/000-003", "specificAssetIds": [ { diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetShellDescriptorById_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetShellDescriptorById_Expected.json index 0c2b2c30..42795fd9 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetShellDescriptorById_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRegistry/TestData/GetShellDescriptorById_Expected.json @@ -20,7 +20,7 @@ } ], "globalAssetId": "https://mm-software.com/ids/assets/000-001", - "idShort": "M&M01", + "idShort": "MM01", "id": "https://mm-software.com/ids/aas/000-001", "specificAssetIds": [ { diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/AasRepositoryTests.cs b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/AasRepositoryTests.cs index 3e1946e9..f4d54ecb 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/AasRepositoryTests.cs +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/AasRepositoryTests.cs @@ -51,7 +51,7 @@ public async Task GetAllShells_ByAssetIds() public async Task GetAllShells_ByIdShort() { // Arrange - var idShort = "M%26M03"; + var idShort = "MM03"; var url = $"/shells?idShort={idShort}"; // Act @@ -73,7 +73,7 @@ public async Task GetAllShells_ByAssetId_And_ByIdShort() { // Arrange var assetId = EncodeBase64Url("{\"name\":\"SerialNumber\",\"value\":\"SN-1111\"}"); - var idShort = "M%26M03"; + var idShort = "MM03"; var url = $"/shells?assetIds={assetId}&idShort={idShort}"; // Act diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByAssetId_And_ByIdShort_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByAssetId_And_ByIdShort_Expected.json index 220e6a01..2788bb00 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByAssetId_And_ByIdShort_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByAssetId_And_ByIdShort_Expected.json @@ -4,7 +4,7 @@ }, "result": [ { - "idShort": "M&M03", + "idShort": "MM03", "id": "https://mm-software.com/ids/aas/000-003", "assetInformation": { "assetKind": "Instance", diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByAssetIds_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByAssetIds_Expected.json index 220e6a01..2788bb00 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByAssetIds_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByAssetIds_Expected.json @@ -4,7 +4,7 @@ }, "result": [ { - "idShort": "M&M03", + "idShort": "MM03", "id": "https://mm-software.com/ids/aas/000-003", "assetInformation": { "assetKind": "Instance", diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByIdShort_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByIdShort_Expected.json index 220e6a01..2788bb00 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByIdShort_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByIdShort_Expected.json @@ -4,7 +4,7 @@ }, "result": [ { - "idShort": "M&M03", + "idShort": "MM03", "id": "https://mm-software.com/ids/aas/000-003", "assetInformation": { "assetKind": "Instance", diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByMultipleAssetIds_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByMultipleAssetIds_Expected.json index 4407fafe..1d9928f8 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByMultipleAssetIds_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_ByMultipleAssetIds_Expected.json @@ -4,7 +4,7 @@ }, "result": [ { - "idShort": "M&M02", + "idShort": "MM02", "id": "https://mm-software.com/ids/aas/000-002", "assetInformation": { "assetKind": "Instance", diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_WithCursor_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_WithCursor_Expected.json index 83d41d2b..044d4924 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_WithCursor_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_WithCursor_Expected.json @@ -4,7 +4,7 @@ }, "result": [ { - "idShort": "M&M02", + "idShort": "MM02", "id": "https://mm-software.com/ids/aas/000-002", "assetInformation": { "assetKind": "Instance", diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_WithLimit_Expected.json b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_WithLimit_Expected.json index 543b17bf..7c9ecf6e 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_WithLimit_Expected.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/AasRepository/TestData/GetAllShells_WithLimit_Expected.json @@ -4,7 +4,7 @@ }, "result": [ { - "idShort": "M&M01", + "idShort": "MM01", "id": "https://mm-software.com/ids/aas/000-001", "assetInformation": { "assetKind": "Instance", diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/GetAllSubmodelsTests.cs b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/GetAllSubmodelsTests.cs index bdb1575b..0d874cbf 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/GetAllSubmodelsTests.cs +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/GetAllSubmodelsTests.cs @@ -73,7 +73,7 @@ public async Task GetAllSubmodels_WithSemanticId_ShouldReturnFilteredResults() [Fact] public async Task GetAllSubmodels_WithIdShort_ShouldReturnMatchingSubmodels() { - var idShort = "M%26M01"; + var idShort = "MM01"; var response = await ApiContext.GetAsync($"/submodels/?idShort={idShort}"); AssertSuccessResponse(response); diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/SubmodelTests.cs b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/SubmodelTests.cs index e54c4a10..c47ead8d 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/SubmodelTests.cs +++ b/source/AAS.TwinEngine.Plugin.TestPlugin.PlaywrightTests/SubmodelRepository/SubmodelTests.cs @@ -52,7 +52,7 @@ public async Task GetAllSubmodels_By_SemanticId_should_return_success_and_conten public async Task GetAllSubmodels_ByIdShort() { // Arrange - var idShort = "M%26M03"; + var idShort = "MM03"; var url = $"/submodels?idShort={idShort}"; // Act diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin/Data/mock-metadata.json b/source/AAS.TwinEngine.Plugin.TestPlugin/Data/mock-metadata.json index 2b414d87..4ed342a3 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin/Data/mock-metadata.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin/Data/mock-metadata.json @@ -1,7 +1,7 @@ [ { "globalAssetId": "https://mm-software.com/ids/assets/000-001", - "idShort": "M&M01", + "idShort": "MM01", "id": "https://mm-software.com/ids/aas/000-001", "specificAssetIds": [ { @@ -22,7 +22,7 @@ }, { "globalAssetId": "https://mm-software.com/ids/assets/000-002", - "idShort": "M&M02", + "idShort": "MM02", "id": "https://mm-software.com/ids/aas/000-002", "specificAssetIds": [ { @@ -44,7 +44,7 @@ }, { "globalAssetId": "https://mm-software.com/ids/assets/000-003", - "idShort": "M&M03", + "idShort": "MM03", "id": "https://mm-software.com/ids/aas/000-003", "specificAssetIds": [ { diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Aas Repository/Get Shells By Asset Id and IdShort - M&M03.bru b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Aas Repository/Get Shells By Asset Id and IdShort - M&M03.bru index 51b26cdd..3c505e3d 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Aas Repository/Get Shells By Asset Id and IdShort - M&M03.bru +++ b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Aas Repository/Get Shells By Asset Id and IdShort - M&M03.bru @@ -1,5 +1,5 @@ meta { - name: Get Shells By Asset Id and IdShort - M&M03 + name: Get Shells By Asset Id and IdShort - MM03 type: http seq: 7 } @@ -23,6 +23,6 @@ settings { } docs { - Retrieves shells filtered by specific asset ID and idShort value "M&M03". + Retrieves shells filtered by specific asset ID and idShort value "MM03". The idShort value is URL-encoded in the collection pre-request script. } diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Aas Repository/Get Shells By IdShort - M&M03.bru b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Aas Repository/Get Shells By IdShort - M&M03.bru index 74826380..895285aa 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Aas Repository/Get Shells By IdShort - M&M03.bru +++ b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Aas Repository/Get Shells By IdShort - M&M03.bru @@ -1,5 +1,5 @@ meta { - name: Get Shells By IdShort - M&M03 + name: Get Shells By IdShort - MM03 type: http seq: 6 } @@ -22,6 +22,6 @@ settings { } docs { - Retrieves shells filtered by idShort value "M&M03" with no assetIds. + Retrieves shells filtered by idShort value "MM03" with no assetIds. The idShort value is URL-encoded in the collection pre-request script. } diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Submodel Repository/Submodel/Get All Submodels - by idShort.bru b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Submodel Repository/Submodel/Get All Submodels - by idShort.bru index 19586110..dde51337 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Submodel Repository/Submodel/Get All Submodels - by idShort.bru +++ b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Submodel Repository/Submodel/Get All Submodels - by idShort.bru @@ -21,4 +21,4 @@ params:query { settings { encodeUrl: false -} \ No newline at end of file +}​ \ No newline at end of file diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/collection.bru b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/collection.bru index a71cd421..8d8b6616 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/collection.bru +++ b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/collection.bru @@ -4,7 +4,7 @@ vars:pre-request { submodelIdentifierHandoverDocumentation: https://mm-software.com/submodel/000-001/HandoverDocumentation submodelIdentifierCustomSubmodel: https://mm-software.com/submodel/000-001/CustomSubmodel assetId: {"name":"SerialNumber","value":"SN-1111"} - idShort: M&M03 + idShort: MM03 } script:pre-request { @@ -25,10 +25,5 @@ script:pre-request { const encoded = b64EncodeUnicode(plain); bru.setVar(name, encoded); }); - - const rawIdShort = bru.getCollectionVar('idShort'); - if (rawIdShort) { - bru.setVar('idShort', encodeURIComponent(rawIdShort)); - } } diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/plugin/mock-metadata.json b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/plugin/mock-metadata.json index 2b414d87..4ed342a3 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/plugin/mock-metadata.json +++ b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/plugin/mock-metadata.json @@ -1,7 +1,7 @@ [ { "globalAssetId": "https://mm-software.com/ids/assets/000-001", - "idShort": "M&M01", + "idShort": "MM01", "id": "https://mm-software.com/ids/aas/000-001", "specificAssetIds": [ { @@ -22,7 +22,7 @@ }, { "globalAssetId": "https://mm-software.com/ids/assets/000-002", - "idShort": "M&M02", + "idShort": "MM02", "id": "https://mm-software.com/ids/aas/000-002", "specificAssetIds": [ { @@ -44,7 +44,7 @@ }, { "globalAssetId": "https://mm-software.com/ids/assets/000-003", - "idShort": "M&M03", + "idShort": "MM03", "id": "https://mm-software.com/ids/aas/000-003", "specificAssetIds": [ { From 7c262bc9575318178eff7439c6ba33cf1d6e0295 Mon Sep 17 00:00:00 2001 From: Kevalkumar Date: Thu, 27 Aug 2026 10:57:46 +0530 Subject: [PATCH 30/30] fix: add missing newline at the end of Get All Submodels - by idShort.bru --- .../Submodel/Get All Submodels - by idShort.bru | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Submodel Repository/Submodel/Get All Submodels - by idShort.bru b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Submodel Repository/Submodel/Get All Submodels - by idShort.bru index dde51337..19586110 100644 --- a/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Submodel Repository/Submodel/Get All Submodels - by idShort.bru +++ b/source/AAS.TwinEngine.Plugin.TestPlugin/Example/apiCollection/Submodel Repository/Submodel/Get All Submodels - by idShort.bru @@ -21,4 +21,4 @@ params:query { settings { encodeUrl: false -}​ \ No newline at end of file +} \ No newline at end of file