From ffed3b3f999900e59a87f71c1a4fb86505dc12ea Mon Sep 17 00:00:00 2001 From: kl3inIT Date: Sun, 2 Aug 2026 11:28:04 +0700 Subject: [PATCH 1/5] refactor(knowledge): publish retrieval adapter contracts --- ARCHITECTURE.md | 9 +- apps/api/build.gradle.kts | 1 + .../api/admin/AdminPermissionController.java | 8 +- .../api/RetrievalAdapterBoundaryTests.java | 47 + apps/worker/build.gradle.kts | 1 + .../worker/OrgMemoryWorkerApplication.java | 4 +- .../worker/RetrievalAdapterBoundaryTests.java | 34 + .../ConnectorContentEditIntegrationTests.java | 3 +- ...onnectorIdentityTrustIntegrationTests.java | 3 +- .../ConnectorPruningIntegrationTests.java | 3 +- ...ectorStagingIngestionIntegrationTests.java | 3 +- ...urceIngestionPipelineIntegrationTests.java | 3 +- .../AuthorizationResourceDirectory.java | 55 +- .../CanonicalHybridKnowledgeSearch.java | 403 +------ ...calHybridKnowledgeSearchConfiguration.java | 10 + .../retrieval/CitationContentService.java | 150 +-- ...DefaultAuthorizationResourceDirectory.java | 61 + ...DefaultCanonicalHybridKnowledgeSearch.java | 401 +++++++ .../DefaultCitationContentService.java | 155 +++ ...aultGraphRagKnowledgeRetrievalService.java | 1026 ++++++++++++++++ .../DefaultSourceContentService.java | 136 +++ .../retrieval/EmbeddingProfileRegistry.java | 84 +- ...aphRagKnowledgeRetrievalConfiguration.java | 4 +- .../GraphRagKnowledgeRetrievalService.java | 1029 +---------------- .../JdbcEmbeddingProfileRegistry.java | 89 ++ .../KnowledgeAssetAccessInspector.java | 19 + .../KnowledgeEvidenceScopeResolver.java | 5 +- .../retrieval/SourceContentService.java | 131 +-- .../knowledge/retrieval/package-info.java | 11 +- .../core/ModulithVerificationTests.java | 40 +- .../CanonicalHybridKnowledgeSearchTests.java | 4 +- .../CitationContentServiceTests.java | 2 +- .../EmbeddingProfileRegistryTests.java | 2 +- ...raphRagKnowledgeRetrievalServiceTests.java | 4 +- .../retrieval/SourceContentServiceTests.java | 2 +- .../design.md | 19 + .../plan.md | 45 +- docs/specs/domains/secure-graph-rag.md | 5 +- docs/specs/domains/secure-retrieval.md | 12 +- docs/tests/domains/secure-graph-rag.md | 6 +- docs/tests/domains/secure-retrieval.md | 3 +- 41 files changed, 2159 insertions(+), 1873 deletions(-) create mode 100644 apps/api/src/test/java/com/orgmemory/api/RetrievalAdapterBoundaryTests.java create mode 100644 apps/worker/src/test/java/com/orgmemory/worker/RetrievalAdapterBoundaryTests.java create mode 100644 core/src/main/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchConfiguration.java create mode 100644 core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultAuthorizationResourceDirectory.java create mode 100644 core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCanonicalHybridKnowledgeSearch.java create mode 100644 core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCitationContentService.java create mode 100644 core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultGraphRagKnowledgeRetrievalService.java create mode 100644 core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultSourceContentService.java create mode 100644 core/src/main/java/com/orgmemory/core/knowledge/retrieval/JdbcEmbeddingProfileRegistry.java create mode 100644 core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeAssetAccessInspector.java diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6da4e9cb9..aa4340729 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -169,9 +169,12 @@ evidence snapshot and exact current governing-evidence decision through the Retrieval-owned `GraphEvidenceVerifier`; Graph does not import Retrieval scope resolution, candidate, or store implementation types. Verified snapshots reject unknown Knowledge Spaces, and canonical evidence rechecks carry only the assets -authorized for the requested Space. Retrieval remains -explicitly open while its remaining sibling adapters are replaced by -intentional APIs. The +authorized for the requested Space. API and Worker inject interfaces for the +canonical/GraphRAG engines, citation/source opening, authorization inspection, +and embedding-profile resolution; full evidence-scope resolution plus the +default and JDBC implementations are package-private. Retrieval remains explicitly open while +its remaining concrete/persistence root types are internalized and its final +dependency allowlist is proven. The provider-neutral object-storage port is exposed as the `knowledge::storage` named interface. Leased database jobs carry ingestion work across processes. A specific Knowledge Asset diff --git a/apps/api/build.gradle.kts b/apps/api/build.gradle.kts index 7d7f7b19a..dc7a70750 100644 --- a/apps/api/build.gradle.kts +++ b/apps/api/build.gradle.kts @@ -33,6 +33,7 @@ dependencies { testImplementation("org.springframework.boot:spring-boot-starter-webmvc-test") testImplementation("org.springframework.security:spring-security-test") testImplementation("org.springframework.boot:spring-boot-testcontainers") + testImplementation("org.springframework.modulith:spring-modulith-starter-test") testImplementation("io.projectreactor:reactor-test") testImplementation("org.testcontainers:testcontainers-junit-jupiter") testImplementation("org.testcontainers:testcontainers-postgresql") diff --git a/apps/api/src/main/java/com/orgmemory/api/admin/AdminPermissionController.java b/apps/api/src/main/java/com/orgmemory/api/admin/AdminPermissionController.java index 351a2a61a..d3be47bdc 100644 --- a/apps/api/src/main/java/com/orgmemory/api/admin/AdminPermissionController.java +++ b/apps/api/src/main/java/com/orgmemory/api/admin/AdminPermissionController.java @@ -14,7 +14,7 @@ import com.orgmemory.core.organization.AppUser; import com.orgmemory.core.organization.CurrentActor; import com.orgmemory.core.knowledge.retrieval.AuthorizationResourceDirectory; -import com.orgmemory.core.knowledge.retrieval.KnowledgeEvidenceScopeResolver; +import com.orgmemory.core.knowledge.retrieval.KnowledgeAssetAccessInspector; import io.swagger.v3.oas.annotations.Operation; import java.time.Instant; import java.util.LinkedHashMap; @@ -64,7 +64,7 @@ class AdminPermissionController { private final AdminAccessGuard guard; private final AccessExplanationService explanations; private final AuthorizationResourceDirectory resources; - private final KnowledgeEvidenceScopeResolver evidenceScopes; + private final KnowledgeAssetAccessInspector evidenceScopes; private final KnowledgeAssetRepository assets; private final KnowledgeAssetVersionRepository versions; private final KnowledgeSpaceQuery spaces; @@ -73,7 +73,7 @@ class AdminPermissionController { AdminAccessGuard guard, AccessExplanationService explanations, AuthorizationResourceDirectory resources, - KnowledgeEvidenceScopeResolver evidenceScopes, + KnowledgeAssetAccessInspector evidenceScopes, KnowledgeAssetRepository assets, KnowledgeAssetVersionRepository versions, KnowledgeSpaceQuery spaces) { @@ -211,7 +211,7 @@ private ExplainAccessResponse canonicalContentResponse( contentState = AccessState.UNKNOWN; contentReason = "NOT_EVALUATED_RELATIONSHIP_NOT_ALLOWED"; } else { - KnowledgeEvidenceScopeResolver.AssetInspection content = evidenceScopes.inspectAsset( + KnowledgeAssetAccessInspector.AssetInspection content = evidenceScopes.inspectAsset( subject, assetId, relationship.policyVersion(), diff --git a/apps/api/src/test/java/com/orgmemory/api/RetrievalAdapterBoundaryTests.java b/apps/api/src/test/java/com/orgmemory/api/RetrievalAdapterBoundaryTests.java new file mode 100644 index 000000000..037a48160 --- /dev/null +++ b/apps/api/src/test/java/com/orgmemory/api/RetrievalAdapterBoundaryTests.java @@ -0,0 +1,47 @@ +package com.orgmemory.api; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.tngtech.archunit.core.importer.ClassFileImporter; +import com.tngtech.archunit.core.importer.ImportOption; +import java.util.Set; +import java.util.TreeSet; +import org.junit.jupiter.api.Test; + +class RetrievalAdapterBoundaryTests { + + @Test + void apiDependsOnlyOnIntentionalRetrievalContracts() { + var dependencies = new ClassFileImporter() + .withImportOption(ImportOption.Predefined.DO_NOT_INCLUDE_TESTS) + .importPackages("com.orgmemory.api") + .stream() + .flatMap(type -> type.getDirectDependenciesFromSelf().stream()) + .map(dependency -> dependency.getTargetClass().getName()) + .filter(name -> name.startsWith("com.orgmemory.core.knowledge.retrieval.")) + .collect(TreeSet::new, Set::add, Set::addAll); + + assertEquals( + Set.of( + "com.orgmemory.core.knowledge.retrieval.AuthorizationResourceDirectory", + "com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearch", + "com.orgmemory.core.knowledge.retrieval.CitationContent", + "com.orgmemory.core.knowledge.retrieval.CitationContentService", + "com.orgmemory.core.knowledge.retrieval.EmbeddingDistanceMetric", + "com.orgmemory.core.knowledge.retrieval.EmbeddingProfileRef", + "com.orgmemory.core.knowledge.retrieval.EmbeddingProfileRegistry", + "com.orgmemory.core.knowledge.retrieval.EmbeddingProfileSpec", + "com.orgmemory.core.knowledge.retrieval.GraphRagKnowledgeRetrievalService", + "com.orgmemory.core.knowledge.retrieval.GraphRagRetrievalPolicy", + "com.orgmemory.core.knowledge.retrieval.GraphRagRetrievalPolicy$RerankPolicy", + "com.orgmemory.core.knowledge.retrieval.KnowledgeAssetAccessInspector", + "com.orgmemory.core.knowledge.retrieval.KnowledgeAssetAccessInspector$AssetInspection", + "com.orgmemory.core.knowledge.retrieval.KnowledgeEmbeddingProperties", + "com.orgmemory.core.knowledge.retrieval.KnowledgeRetrievalProperties", + "com.orgmemory.core.knowledge.retrieval.QueryEmbedding", + "com.orgmemory.core.knowledge.retrieval.QueryEmbeddingPort", + "com.orgmemory.core.knowledge.retrieval.SourceContent", + "com.orgmemory.core.knowledge.retrieval.SourceContentService"), + dependencies); + } +} diff --git a/apps/worker/build.gradle.kts b/apps/worker/build.gradle.kts index 552e95cf7..575938603 100644 --- a/apps/worker/build.gradle.kts +++ b/apps/worker/build.gradle.kts @@ -20,6 +20,7 @@ dependencies { runtimeOnly("org.postgresql:postgresql") testImplementation("org.junit.jupiter:junit-jupiter") + testImplementation("org.springframework.modulith:spring-modulith-starter-test") testImplementation(libs.apache.poi.ooxml) testImplementation("org.springframework.boot:spring-boot-starter-webmvc-test") testImplementation("org.springframework.boot:spring-boot-testcontainers") diff --git a/apps/worker/src/main/java/com/orgmemory/worker/OrgMemoryWorkerApplication.java b/apps/worker/src/main/java/com/orgmemory/worker/OrgMemoryWorkerApplication.java index e1b47686f..3caf3dd86 100644 --- a/apps/worker/src/main/java/com/orgmemory/worker/OrgMemoryWorkerApplication.java +++ b/apps/worker/src/main/java/com/orgmemory/worker/OrgMemoryWorkerApplication.java @@ -2,7 +2,7 @@ import com.orgmemory.core.knowledge.sourceledger.SourceIngestionProperties; -import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearch; +import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearchConfiguration; import com.orgmemory.core.knowledge.graph.GraphProcessingProperties; import com.orgmemory.core.knowledge.graph.KnowledgeGraphExplorerConfiguration; import com.orgmemory.core.knowledge.retrieval.KnowledgeRetrievalProperties; @@ -51,7 +51,7 @@ excludeFilters = @ComponentScan.Filter( type = FilterType.ASSIGNABLE_TYPE, classes = { - CanonicalHybridKnowledgeSearch.class, + CanonicalHybridKnowledgeSearchConfiguration.class, KnowledgeGraphExplorerConfiguration.class })) public class OrgMemoryWorkerApplication { diff --git a/apps/worker/src/test/java/com/orgmemory/worker/RetrievalAdapterBoundaryTests.java b/apps/worker/src/test/java/com/orgmemory/worker/RetrievalAdapterBoundaryTests.java new file mode 100644 index 000000000..1d4f1d7cb --- /dev/null +++ b/apps/worker/src/test/java/com/orgmemory/worker/RetrievalAdapterBoundaryTests.java @@ -0,0 +1,34 @@ +package com.orgmemory.worker; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.tngtech.archunit.core.importer.ClassFileImporter; +import com.tngtech.archunit.core.importer.ImportOption; +import java.util.Set; +import java.util.TreeSet; +import org.junit.jupiter.api.Test; + +class RetrievalAdapterBoundaryTests { + + @Test + void workerDependsOnlyOnIntentionalRetrievalContracts() { + var dependencies = new ClassFileImporter() + .withImportOption(ImportOption.Predefined.DO_NOT_INCLUDE_TESTS) + .importPackages("com.orgmemory.worker") + .stream() + .flatMap(type -> type.getDirectDependenciesFromSelf().stream()) + .map(dependency -> dependency.getTargetClass().getName()) + .filter(name -> name.startsWith("com.orgmemory.core.knowledge.retrieval.")) + .collect(TreeSet::new, Set::add, Set::addAll); + + assertEquals( + Set.of( + "com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearchConfiguration", + "com.orgmemory.core.knowledge.retrieval.EmbeddingDistanceMetric", + "com.orgmemory.core.knowledge.retrieval.EmbeddingProfileRef", + "com.orgmemory.core.knowledge.retrieval.EmbeddingProfileRegistry", + "com.orgmemory.core.knowledge.retrieval.EmbeddingProfileSpec", + "com.orgmemory.core.knowledge.retrieval.KnowledgeRetrievalProperties"), + dependencies); + } +} diff --git a/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorContentEditIntegrationTests.java b/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorContentEditIntegrationTests.java index 286acb08a..d33959eaa 100644 --- a/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorContentEditIntegrationTests.java +++ b/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorContentEditIntegrationTests.java @@ -34,6 +34,7 @@ import com.orgmemory.core.knowledge.connector.ConnectorMembershipMember; import com.orgmemory.core.knowledge.connector.ConnectorPermissionItem; import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearch; +import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearchConfiguration; import com.orgmemory.core.knowledge.retrieval.KnowledgeRetrievalProperties; import com.orgmemory.core.knowledge.connector.ConnectorCaptureStatus; import com.orgmemory.core.knowledge.retrieval.QueryEmbeddingPort; @@ -88,7 +89,7 @@ "orgmemory.graph-rag.postgres.apache-age-mode=disabled", "orgmemory.connector.scheduling-enabled=false" }) -@Import(CanonicalHybridKnowledgeSearch.class) +@Import(CanonicalHybridKnowledgeSearchConfiguration.class) @EnableConfigurationProperties(KnowledgeRetrievalProperties.class) @Testcontainers @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) diff --git a/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorIdentityTrustIntegrationTests.java b/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorIdentityTrustIntegrationTests.java index 557d05e5c..3a9f8e05e 100644 --- a/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorIdentityTrustIntegrationTests.java +++ b/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorIdentityTrustIntegrationTests.java @@ -31,6 +31,7 @@ import com.orgmemory.core.knowledge.connector.ConnectorMembershipMember; import com.orgmemory.core.knowledge.connector.ConnectorPermissionItem; import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearch; +import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearchConfiguration; import com.orgmemory.core.knowledge.retrieval.KnowledgeRetrievalProperties; import com.orgmemory.core.knowledge.connector.ConnectorCaptureStatus; import com.orgmemory.core.knowledge.retrieval.QueryEmbeddingPort; @@ -84,7 +85,7 @@ "orgmemory.graph-rag.postgres.apache-age-mode=disabled", "orgmemory.connector.scheduling-enabled=false" }) -@Import(CanonicalHybridKnowledgeSearch.class) +@Import(CanonicalHybridKnowledgeSearchConfiguration.class) @EnableConfigurationProperties(KnowledgeRetrievalProperties.class) @Testcontainers @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) diff --git a/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorPruningIntegrationTests.java b/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorPruningIntegrationTests.java index 5915ecc56..71c441406 100644 --- a/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorPruningIntegrationTests.java +++ b/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorPruningIntegrationTests.java @@ -31,6 +31,7 @@ import com.orgmemory.core.knowledge.connector.ConnectorMembershipMember; import com.orgmemory.core.knowledge.connector.ConnectorPermissionItem; import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearch; +import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearchConfiguration; import com.orgmemory.core.knowledge.retrieval.KnowledgeRetrievalProperties; import com.orgmemory.core.knowledge.connector.ConnectorCaptureStatus; import com.orgmemory.core.knowledge.retrieval.QueryEmbeddingPort; @@ -81,7 +82,7 @@ "orgmemory.graph-rag.postgres.apache-age-mode=disabled", "orgmemory.connector.scheduling-enabled=false" }) -@Import(CanonicalHybridKnowledgeSearch.class) +@Import(CanonicalHybridKnowledgeSearchConfiguration.class) @EnableConfigurationProperties(KnowledgeRetrievalProperties.class) @Testcontainers @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) diff --git a/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorStagingIngestionIntegrationTests.java b/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorStagingIngestionIntegrationTests.java index cdc13f947..5309470ca 100644 --- a/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorStagingIngestionIntegrationTests.java +++ b/apps/worker/src/test/java/com/orgmemory/worker/connector/ConnectorStagingIngestionIntegrationTests.java @@ -22,6 +22,7 @@ import com.orgmemory.core.authorization.RelationshipTupleWriteResult; import com.orgmemory.core.authorization.ResourceRef; import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearch; +import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearchConfiguration; import com.orgmemory.core.knowledge.connector.ConnectorCrawlBatch; import com.orgmemory.core.knowledge.connector.ConnectorIngestionResult; import com.orgmemory.core.knowledge.connector.ConnectorIngestionService; @@ -76,7 +77,7 @@ "orgmemory.graph-rag.postgres.apache-age-mode=disabled", "orgmemory.connector.scheduling-enabled=false" }) -@Import(CanonicalHybridKnowledgeSearch.class) +@Import(CanonicalHybridKnowledgeSearchConfiguration.class) @EnableConfigurationProperties(KnowledgeRetrievalProperties.class) @Testcontainers @DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) diff --git a/apps/worker/src/test/java/com/orgmemory/worker/ingestion/SourceIngestionPipelineIntegrationTests.java b/apps/worker/src/test/java/com/orgmemory/worker/ingestion/SourceIngestionPipelineIntegrationTests.java index a964afa51..54c71a05a 100644 --- a/apps/worker/src/test/java/com/orgmemory/worker/ingestion/SourceIngestionPipelineIntegrationTests.java +++ b/apps/worker/src/test/java/com/orgmemory/worker/ingestion/SourceIngestionPipelineIntegrationTests.java @@ -33,6 +33,7 @@ import com.orgmemory.core.knowledge.retrieval.QueryEmbedding; import com.orgmemory.core.knowledge.retrieval.KnowledgeRetrievalProperties; import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearch; +import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearchConfiguration; import com.orgmemory.core.knowledge.storage.ObjectContent; import com.orgmemory.core.knowledge.storage.ObjectStoragePort; import com.orgmemory.core.knowledge.storage.ObjectWriteRequest; @@ -86,7 +87,7 @@ }) @Import({ SourceIngestionPipelineIntegrationTests.UploadTestConfiguration.class, - CanonicalHybridKnowledgeSearch.class + CanonicalHybridKnowledgeSearchConfiguration.class }) @EnableConfigurationProperties(KnowledgeRetrievalProperties.class) @Testcontainers diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/AuthorizationResourceDirectory.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/AuthorizationResourceDirectory.java index e9d0c623d..601e1f2d9 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/AuthorizationResourceDirectory.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/AuthorizationResourceDirectory.java @@ -1,60 +1,13 @@ package com.orgmemory.core.knowledge.retrieval; -import com.orgmemory.core.shared.error.KnowledgeResourceNotFoundException; - -import com.orgmemory.core.knowledge.asset.KnowledgeAssetRetrievalQuery; - import com.orgmemory.core.authorization.ResourceRef; -import com.orgmemory.core.knowledge.space.KnowledgeSpaceQuery; -import com.orgmemory.core.organization.OrganizationResourceQuery; -import java.util.Objects; import java.util.UUID; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; /** - * Resolves an administrator-supplied authorization resource against the - * canonical tenant directory before OpenFGA is queried. + * Adapter-facing query that validates an authorization resource against its + * canonical tenant-owned directory before policy evaluation. */ -@Service -public class AuthorizationResourceDirectory { - - private final OrganizationResourceQuery organizationResources; - private final KnowledgeSpaceQuery spaces; - private final KnowledgeAssetRetrievalQuery assets; - - AuthorizationResourceDirectory( - OrganizationResourceQuery organizationResources, - KnowledgeSpaceQuery spaces, - KnowledgeAssetRetrievalQuery assets) { - this.organizationResources = organizationResources; - this.spaces = spaces; - this.assets = assets; - } +public interface AuthorizationResourceDirectory { - @Transactional(readOnly = true) - public ResourceRef require( - UUID organizationId, - String resourceType, - UUID resourceId) { - Objects.requireNonNull(organizationId, "organizationId"); - Objects.requireNonNull(resourceId, "resourceId"); - String type = Objects.requireNonNull(resourceType, "resourceType").strip(); - boolean exists = switch (type) { - case "organization" -> - organizationId.equals(resourceId) - && organizationResources.organizationExists(organizationId); - case "organizational_unit" -> - organizationResources.departmentExists(organizationId, resourceId); - case "knowledge_space" -> - spaces.exists(organizationId, resourceId); - case "knowledge_asset" -> - assets.exists(organizationId, resourceId); - default -> false; - }; - if (!exists) { - throw new KnowledgeResourceNotFoundException(); - } - return ResourceRef.of(organizationId, type, resourceId); - } + ResourceRef require(UUID organizationId, String resourceType, UUID resourceId); } diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearch.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearch.java index 0359f48ea..665e4a4d4 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearch.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearch.java @@ -1,406 +1,7 @@ package com.orgmemory.core.knowledge.retrieval; -import com.orgmemory.core.knowledge.retrieval.QueryEmbedding; -import com.orgmemory.core.knowledge.retrieval.QueryEmbeddingPort; import com.orgmemory.core.knowledge.search.PermissionAwareKnowledgeSearch; -import com.orgmemory.core.knowledge.search.RetrievedKnowledgeEvidence; -import com.orgmemory.core.knowledge.search.SecureKnowledgeSearchResult; -import com.orgmemory.core.knowledge.sourceledger.SourceCitationUri; -import com.orgmemory.core.authorization.BatchAuthorizationQuery; -import com.orgmemory.core.authorization.PermissionKey; -import com.orgmemory.core.authorization.RelationshipAuthorizationSetPort; -import com.orgmemory.core.authorization.ResourceRef; -import com.orgmemory.core.organization.CurrentActor; -import com.orgmemory.core.permission.PermissionAuditCommand; -import com.orgmemory.core.permission.PermissionAuditDecision; -import com.orgmemory.core.permission.PermissionAuditService; -import com.orgmemory.core.shared.error.BusinessValidationException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -@Service -public class CanonicalHybridKnowledgeSearch - implements PermissionAwareKnowledgeSearch { - - static final String POLICY_VERSION = - KnowledgeSearchAuthorizationService.POLICY_VERSION; - private static final PermissionKey CAN_VIEW = PermissionKey.of("can_view"); - private static final String RESOURCE_TYPE = "knowledge_asset"; - private static final OpenFgaBatchRecheck.ReasonRule RESULT_REASON = - OpenFgaBatchRecheck.ReasonRule.resultReason(); - private static final OpenFgaBatchRecheck.ReasonRule MODEL_MISMATCH = - OpenFgaBatchRecheck.ReasonRule.fixed( - "AUTHORIZATION_MODEL_MISMATCH"); - private static final OpenFgaBatchRecheck.ReasonMapping RECHECK_REASONS = - new OpenFgaBatchRecheck.ReasonMapping( - RESULT_REASON, - RESULT_REASON, - MODEL_MISMATCH, - OpenFgaBatchRecheck.ReasonRule.fixed( - "OPENFGA_BATCH_INCOMPLETE"), - MODEL_MISMATCH, - OpenFgaBatchRecheck.ReasonRule.fixed( - "RELATIONSHIP_DENIED")); - private static final int RRF_RANK_CONSTANT = 60; - private static final int MAX_REQUEST_ID_LENGTH = 128; - - private final SecureKnowledgeRetrievalStore store; - private final KnowledgeEvidenceScopeResolver evidenceScopes; - private final KnowledgeSearchAuthorizationService searchAuthorization; - private final OpenFgaBatchRecheck batchRecheck; - private final QueryEmbeddingPort embeddings; - private final PermissionAuditService audit; - private final KnowledgeRetrievalProperties properties; - - CanonicalHybridKnowledgeSearch( - SecureKnowledgeRetrievalStore store, - KnowledgeEvidenceScopeResolver evidenceScopes, - KnowledgeSearchAuthorizationService searchAuthorization, - RelationshipAuthorizationSetPort authorization, - QueryEmbeddingPort embeddings, - PermissionAuditService audit, - KnowledgeRetrievalProperties properties) { - this.store = store; - this.evidenceScopes = evidenceScopes; - this.searchAuthorization = searchAuthorization; - this.batchRecheck = new OpenFgaBatchRecheck(authorization); - this.embeddings = embeddings; - this.audit = audit; - this.properties = properties; - } - - @Transactional(readOnly = true) - @Override - public SecureKnowledgeSearchResult search( - CurrentActor actor, - String query, - Integer requestedLimit, - String suppliedRequestId) { - String requestId = requestId(suppliedRequestId); - String normalizedQuery = normalizeQuery(query); - int limit = validateLimit(requestedLimit); - String entryModelId = - searchAuthorization.require( - actor, - requestId, - normalizedQuery); - ResolvedKnowledgeEvidenceScope evidenceScope; - try { - evidenceScope = evidenceScopes.resolve(actor, entryModelId); - } catch (KnowledgeEvidenceScopeUnavailableException unavailable) { - throw searchAuthorization.unavailable( - actor, - requestId, - normalizedQuery, - unavailable.reasonCode(), - unavailable.policyVersion()); - } - Set authorizedAssetIds = evidenceScope.allAssetIds(); - if (authorizedAssetIds.isEmpty()) { - audit.record(searchAudit( - actor, - requestId, - normalizedQuery, - PermissionAuditDecision.ALLOW, - "NO_AUTHORIZED_KNOWLEDGE_ASSETS", - evidenceScope.authorizationModelId())); - return new SecureKnowledgeSearchResult(requestId, List.of()); - } - - var scope = new SecureKnowledgeRetrievalStore.RetrievalScope( - actor.organizationId(), - actor.userId(), - evidenceScope.actorDepartmentId(), - evidenceScope.actorExecutive(), - authorizedAssetIds.stream().sorted().toList(), - evidenceScope.authorizationModelId(), - evidenceScope.evaluatedAt()); - int candidateLimit = Math.multiplyExact(limit, properties.candidateMultiplier()); - List lexical = store.lexical(scope, normalizedQuery, candidateLimit); - Optional queryEmbedding = embeddings.embed(actor.organizationId(), normalizedQuery); - List semantic = queryEmbedding - .map(embedding -> store.semantic(scope, embedding, candidateLimit)) - .orElseGet(List::of); - validateCandidateSet( - actor, - requestId, - normalizedQuery, - lexical, - authorizedAssetIds, - evidenceScope.authorizationModelId()); - validateCandidateSet( - actor, - requestId, - normalizedQuery, - semantic, - authorizedAssetIds, - evidenceScope.authorizationModelId()); - - List ranked = fuse(lexical, semantic).stream() - .limit(limit) - .toList(); - if (ranked.isEmpty()) { - audit.record(searchAudit( - actor, - requestId, - normalizedQuery, - PermissionAuditDecision.ALLOW, - "NO_ELIGIBLE_EVIDENCE", - evidenceScope.authorizationModelId())); - return new SecureKnowledgeSearchResult(requestId, List.of()); - } - - List rankedResources = ranked.stream() - .map(candidate -> ResourceRef.of( - actor.organizationId(), RESOURCE_TYPE, candidate.candidate().knowledgeAssetId())) - .distinct() - .toList(); - var authorizationRecheck = batchRecheck.recheck( - new BatchAuthorizationQuery( - actor.organizationId(), - actor.principal(), - CAN_VIEW, - rankedResources), - evidenceScope.authorizationModelId(), - OpenFgaBatchRecheck.ResultPolicy.FILTER_DENIED, - RECHECK_REASONS); - if (!authorizationRecheck.succeeded()) { - var failure = authorizationRecheck.failure(); - throw searchAuthorization.unavailable( - actor, - requestId, - normalizedQuery, - failure.reasonCode(), - failure.policyVersion()); - } - Set allowedAssetIds = authorizationRecheck.allowedResources().stream() - .map(resource -> UUID.fromString(resource.id())) - .collect(java.util.stream.Collectors.toCollection( - LinkedHashSet::new)); - - List allowed = ranked.stream() - .filter(candidate -> allowedAssetIds.contains(candidate.candidate().knowledgeAssetId())) - .toList(); - Map rechecked = store.recheck( - scope, - allowed.stream().map(candidate -> candidate.candidate().chunkId()).toList()) - .stream() - .collect(java.util.stream.Collectors.toMap( - SecureRetrievalCandidate::chunkId, - candidate -> candidate)); - - List evidence = new ArrayList<>(); - List auditCommands = new ArrayList<>(); - auditCommands.add(searchAudit( - actor, - requestId, - normalizedQuery, - PermissionAuditDecision.ALLOW, - "SECURE_RETRIEVAL_APPLIED", - evidenceScope.authorizationModelId())); - for (ScoredCandidate scored : allowed) { - SecureRetrievalCandidate canonical = rechecked.get(scored.candidate().chunkId()); - if (canonical == null - || !canonical.knowledgeAssetId().equals(scored.candidate().knowledgeAssetId())) { - auditCommands.add(evidenceAudit( - actor, - requestId, - normalizedQuery, - scored.candidate(), - PermissionAuditDecision.DENY, - "CANONICAL_RECHECK_DENIED")); - continue; - } - auditCommands.add(evidenceAudit( - actor, - requestId, - normalizedQuery, - canonical, - PermissionAuditDecision.ALLOW, - "VERIFIED_EVIDENCE")); - evidence.add(new RetrievedKnowledgeEvidence( - canonical.chunkId(), - canonical.knowledgeAssetId(), - canonical.sourceObjectId(), - canonical.sourceRevisionId(), - canonical.title(), - canonical.content(), - SourceCitationUri.safeForOutput(canonical.sourceUri()), - canonical.startPage(), - canonical.endPage(), - canonical.heading(), - scored.lexicalScore(), - scored.vectorScore(), - scored.relevanceScore(), - canonical.ingestionAclSnapshotId(), - canonical.currentAclSnapshotId(), - canonical.authorizationModelId(), - canonical.embeddingProfileId(), - canonical.projectionGeneration())); - } - audit.recordAll(auditCommands); - return new SecureKnowledgeSearchResult(requestId, evidence); - } - - private void validateCandidateSet( - CurrentActor actor, - String requestId, - String query, - List candidates, - Set authorizedAssetIds, - String policyVersion) { - boolean invalid = candidates.stream() - .anyMatch(candidate -> !actor.organizationId().equals(candidate.organizationId()) - || !authorizedAssetIds.contains(candidate.knowledgeAssetId())); - if (invalid) { - throw searchAuthorization.unavailable( - actor, - requestId, - query, - "RETRIEVAL_AUTHORIZATION_BOUNDARY_VIOLATION", - policyVersion); - } - } - - private List fuse( - List lexical, - List semantic) { - Map fused = new LinkedHashMap<>(); - addRanks(fused, lexical, true); - addRanks(fused, semantic, false); - return fused.values().stream() - .map(MutableScore::freeze) - .sorted(Comparator.comparingDouble(ScoredCandidate::relevanceScore).reversed() - .thenComparing(candidate -> candidate.candidate().chunkId())) - .toList(); - } - - private static void addRanks( - Map fused, - List candidates, - boolean lexical) { - Set countedAssets = new LinkedHashSet<>(); - int rank = 0; - for (SecureRetrievalCandidate candidate : candidates) { - if (!countedAssets.add(candidate.knowledgeAssetId())) { - continue; - } - MutableScore score = fused.computeIfAbsent(candidate.chunkId(), ignored -> new MutableScore(candidate)); - score.relevance += 1.0d / (RRF_RANK_CONSTANT + rank + 1); - if (lexical) { - score.lexical = candidate.score(); - } else { - score.vector = candidate.score(); - } - rank++; - } - } - - private PermissionAuditCommand searchAudit( - CurrentActor actor, - String requestId, - String query, - PermissionAuditDecision decision, - String reason, - String policyVersion) { - return searchAuthorization.command( - actor, - requestId, - query, - decision, - reason, - policyVersion); - } - - private PermissionAuditCommand evidenceAudit( - CurrentActor actor, - String requestId, - String query, - SecureRetrievalCandidate candidate, - PermissionAuditDecision decision, - String reason) { - return new PermissionAuditCommand( - actor.organizationId(), - actor.userId(), - "SEARCH", - "KNOWLEDGE_EVIDENCE", - candidate.chunkId().toString(), - decision, - reason, - POLICY_VERSION, - requestId, - query, - candidate.ingestionAclSnapshotId(), - candidate.currentAclSnapshotId(), - candidate.authorizationModelId(), - candidate.sourceRevisionId(), - candidate.chunkId(), - candidate.embeddingProfileId(), - candidate.projectionGeneration()); - } - - private String normalizeQuery(String query) { - if (query == null || query.isBlank()) { - throw new BusinessValidationException( - "knowledge-search.query-required", - "q is required"); - } - String normalized = query.strip(); - if (normalized.length() > properties.maximumQueryLength()) { - throw new BusinessValidationException( - "knowledge-search.query-invalid", - "q must not exceed " + properties.maximumQueryLength() + " characters"); - } - return normalized; - } - - private int validateLimit(Integer requestedLimit) { - int limit = requestedLimit == null ? Math.min(10, properties.maximumResults()) : requestedLimit; - if (limit < 1 || limit > properties.maximumResults()) { - throw new BusinessValidationException( - "knowledge-search.limit-invalid", - "limit must be between 1 and " + properties.maximumResults()); - } - return limit; - } - - private static String requestId(String requestId) { - if (requestId == null || requestId.isBlank()) { - return UUID.randomUUID().toString(); - } - String normalized = requestId.strip(); - return normalized.length() <= MAX_REQUEST_ID_LENGTH ? normalized : UUID.randomUUID().toString(); - } - - private static final class MutableScore { - private final SecureRetrievalCandidate candidate; - private double lexical; - private double vector; - private double relevance; - - private MutableScore(SecureRetrievalCandidate candidate) { - this.candidate = candidate; - } - - private ScoredCandidate freeze() { - return new ScoredCandidate(candidate, lexical, vector, relevance); - } - } - - private record ScoredCandidate( - SecureRetrievalCandidate candidate, - double lexicalScore, - double vectorScore, - double relevanceScore) { - } +/** Adapter-facing canonical hybrid retrieval engine. */ +public interface CanonicalHybridKnowledgeSearch extends PermissionAwareKnowledgeSearch { } diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchConfiguration.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchConfiguration.java new file mode 100644 index 000000000..905ecf16a --- /dev/null +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchConfiguration.java @@ -0,0 +1,10 @@ +package com.orgmemory.core.knowledge.retrieval; + +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +/** Registers the canonical query engine for interactive adapters that opt into it. */ +@Configuration(proxyBeanMethods = false) +@Import(DefaultCanonicalHybridKnowledgeSearch.class) +public class CanonicalHybridKnowledgeSearchConfiguration { +} diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CitationContentService.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CitationContentService.java index 5ca9a7174..e0d17faf6 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CitationContentService.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/CitationContentService.java @@ -1,154 +1,10 @@ package com.orgmemory.core.knowledge.retrieval; -import com.orgmemory.core.knowledge.sourceledger.SourceCitationEvidence; -import com.orgmemory.core.knowledge.sourceledger.SourceCitationEvidenceQuery; -import com.orgmemory.core.knowledge.sourceledger.SourceCitationEvidenceResult; - -import com.orgmemory.core.knowledge.storage.ObjectContent; -import com.orgmemory.core.knowledge.storage.ObjectStoragePort; import com.orgmemory.core.organization.CurrentActor; -import com.orgmemory.core.permission.PermissionAuditCommand; -import com.orgmemory.core.permission.PermissionAuditDecision; -import com.orgmemory.core.permission.PermissionAuditService; -import java.util.Objects; import java.util.UUID; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; - -/** - * Opens original source evidence through the same permission boundary used by - * retrieval. No storage URL or object key crosses the application boundary. - */ -@Service -public class CitationContentService { - - private final CanonicalEvidenceAuthorizationService authorization; - private final SourceCitationEvidenceQuery evidenceQuery; - private final ObjectStoragePort objects; - private final PermissionAuditService audit; - - CitationContentService( - CanonicalEvidenceAuthorizationService authorization, - SourceCitationEvidenceQuery evidenceQuery, - ObjectStoragePort objects, - PermissionAuditService audit) { - this.authorization = authorization; - this.evidenceQuery = evidenceQuery; - this.objects = objects; - this.audit = audit; - } - - @Transactional(readOnly = true) - public CitationContent open( - CurrentActor actor, - UUID chunkId, - String requestId) { - Objects.requireNonNull(actor, "actor"); - Objects.requireNonNull(chunkId, "chunkId"); - String normalizedRequestId = requestId == null || requestId.isBlank() - ? UUID.randomUUID().toString() - : requestId.strip(); - String auditQuery = "citation:" + chunkId; - CanonicalEvidenceAuthorizationService.Verification verified; - try { - verified = authorization.verify( - actor, - normalizedRequestId, - auditQuery, - java.util.List.of(chunkId)); - } catch (CanonicalEvidenceAuthorizationException denied) { - throw notFound( - actor, - chunkId, - normalizedRequestId, - denied.authorizationModelId(), - denied.reasonCode()); - } - SecureRetrievalCandidate currentCandidate = - verified.candidates().getFirst(); - String authorizationModelId = verified.authorizationModelId(); - - SourceCitationEvidenceResult result = evidenceQuery.findAvailable( - actor.organizationId(), - currentCandidate.sourceRevisionId(), - currentCandidate.knowledgeAssetId()); - SourceCitationEvidence evidence = switch (result) { - case SourceCitationEvidenceResult.Available available -> - available.evidence(); - case SourceCitationEvidenceResult.Unavailable unavailable -> { - String reason = switch (unavailable.reason()) { - case REVISION_NOT_CURRENT -> "CITATION_REVISION_NOT_CURRENT"; - case BLOB_NOT_AVAILABLE -> "CITATION_BLOB_NOT_AVAILABLE"; - }; - throw notFound( - actor, - chunkId, - normalizedRequestId, - authorizationModelId, - reason); - } - }; - - var content = objects.open(evidence.objectKey()); - if (!evidence.storedContentSha256().equals(content.metadata().sha256()) - || evidence.storedContentLength() - != content.metadata().contentLength()) { - closeQuietly(content); - throw new KnowledgeRetrievalUnavailableException( - "Citation evidence failed its integrity check"); - } - audit.record(new PermissionAuditCommand( - actor.organizationId(), - actor.userId(), - "READ_CITATION", - "KNOWLEDGE_CHUNK", - chunkId.toString(), - PermissionAuditDecision.ALLOW, - "AUTHORIZED_CITATION_CONTENT", - authorizationModelId, - normalizedRequestId, - null, - currentCandidate.ingestionAclSnapshotId(), - currentCandidate.currentAclSnapshotId(), - currentCandidate.authorizationModelId(), - currentCandidate.sourceRevisionId(), - currentCandidate.chunkId(), - currentCandidate.embeddingProfileId(), - currentCandidate.projectionGeneration())); - return new CitationContent( - chunkId, - evidence.fileName(), - evidence.mediaType(), - evidence.contentLength(), - evidence.contentSha256(), - content); - } - private static void closeQuietly(ObjectContent content) { - try { - content.close(); - } catch (java.io.IOException ignored) { - // The authorization or integrity failure is authoritative. - } - } +/** Opens one citation through canonical authorization without exposing object-storage details. */ +public interface CitationContentService { - private CitationNotFoundException notFound( - CurrentActor actor, - UUID chunkId, - String requestId, - String authorizationModelId, - String reason) { - audit.record(new PermissionAuditCommand( - actor.organizationId(), - actor.userId(), - "READ_CITATION", - "KNOWLEDGE_CHUNK", - chunkId.toString(), - PermissionAuditDecision.DENY, - reason, - authorizationModelId, - requestId, - null)); - return new CitationNotFoundException(); - } + CitationContent open(CurrentActor actor, UUID chunkId, String requestId); } diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultAuthorizationResourceDirectory.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultAuthorizationResourceDirectory.java new file mode 100644 index 000000000..6dd035605 --- /dev/null +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultAuthorizationResourceDirectory.java @@ -0,0 +1,61 @@ +package com.orgmemory.core.knowledge.retrieval; + +import com.orgmemory.core.shared.error.KnowledgeResourceNotFoundException; + +import com.orgmemory.core.knowledge.asset.KnowledgeAssetRetrievalQuery; + +import com.orgmemory.core.authorization.ResourceRef; +import com.orgmemory.core.knowledge.space.KnowledgeSpaceQuery; +import com.orgmemory.core.organization.OrganizationResourceQuery; +import java.util.Objects; +import java.util.UUID; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * Resolves an administrator-supplied authorization resource against the + * canonical tenant directory before OpenFGA is queried. + */ +@Service +class DefaultAuthorizationResourceDirectory implements AuthorizationResourceDirectory { + + private final OrganizationResourceQuery organizationResources; + private final KnowledgeSpaceQuery spaces; + private final KnowledgeAssetRetrievalQuery assets; + + DefaultAuthorizationResourceDirectory( + OrganizationResourceQuery organizationResources, + KnowledgeSpaceQuery spaces, + KnowledgeAssetRetrievalQuery assets) { + this.organizationResources = organizationResources; + this.spaces = spaces; + this.assets = assets; + } + + @Transactional(readOnly = true) + @Override + public ResourceRef require( + UUID organizationId, + String resourceType, + UUID resourceId) { + Objects.requireNonNull(organizationId, "organizationId"); + Objects.requireNonNull(resourceId, "resourceId"); + String type = Objects.requireNonNull(resourceType, "resourceType").strip(); + boolean exists = switch (type) { + case "organization" -> + organizationId.equals(resourceId) + && organizationResources.organizationExists(organizationId); + case "organizational_unit" -> + organizationResources.departmentExists(organizationId, resourceId); + case "knowledge_space" -> + spaces.exists(organizationId, resourceId); + case "knowledge_asset" -> + assets.exists(organizationId, resourceId); + default -> false; + }; + if (!exists) { + throw new KnowledgeResourceNotFoundException(); + } + return ResourceRef.of(organizationId, type, resourceId); + } +} diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCanonicalHybridKnowledgeSearch.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCanonicalHybridKnowledgeSearch.java new file mode 100644 index 000000000..291540a2a --- /dev/null +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCanonicalHybridKnowledgeSearch.java @@ -0,0 +1,401 @@ +package com.orgmemory.core.knowledge.retrieval; + +import com.orgmemory.core.knowledge.search.RetrievedKnowledgeEvidence; +import com.orgmemory.core.knowledge.search.SecureKnowledgeSearchResult; +import com.orgmemory.core.knowledge.sourceledger.SourceCitationUri; + +import com.orgmemory.core.authorization.BatchAuthorizationQuery; +import com.orgmemory.core.authorization.PermissionKey; +import com.orgmemory.core.authorization.RelationshipAuthorizationSetPort; +import com.orgmemory.core.authorization.ResourceRef; +import com.orgmemory.core.organization.CurrentActor; +import com.orgmemory.core.permission.PermissionAuditCommand; +import com.orgmemory.core.permission.PermissionAuditDecision; +import com.orgmemory.core.permission.PermissionAuditService; +import com.orgmemory.core.shared.error.BusinessValidationException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import org.springframework.transaction.annotation.Transactional; + +class DefaultCanonicalHybridKnowledgeSearch + implements CanonicalHybridKnowledgeSearch { + + static final String POLICY_VERSION = + KnowledgeSearchAuthorizationService.POLICY_VERSION; + private static final PermissionKey CAN_VIEW = PermissionKey.of("can_view"); + private static final String RESOURCE_TYPE = "knowledge_asset"; + private static final OpenFgaBatchRecheck.ReasonRule RESULT_REASON = + OpenFgaBatchRecheck.ReasonRule.resultReason(); + private static final OpenFgaBatchRecheck.ReasonRule MODEL_MISMATCH = + OpenFgaBatchRecheck.ReasonRule.fixed( + "AUTHORIZATION_MODEL_MISMATCH"); + private static final OpenFgaBatchRecheck.ReasonMapping RECHECK_REASONS = + new OpenFgaBatchRecheck.ReasonMapping( + RESULT_REASON, + RESULT_REASON, + MODEL_MISMATCH, + OpenFgaBatchRecheck.ReasonRule.fixed( + "OPENFGA_BATCH_INCOMPLETE"), + MODEL_MISMATCH, + OpenFgaBatchRecheck.ReasonRule.fixed( + "RELATIONSHIP_DENIED")); + private static final int RRF_RANK_CONSTANT = 60; + private static final int MAX_REQUEST_ID_LENGTH = 128; + + private final SecureKnowledgeRetrievalStore store; + private final KnowledgeEvidenceScopeResolver evidenceScopes; + private final KnowledgeSearchAuthorizationService searchAuthorization; + private final OpenFgaBatchRecheck batchRecheck; + private final QueryEmbeddingPort embeddings; + private final PermissionAuditService audit; + private final KnowledgeRetrievalProperties properties; + + DefaultCanonicalHybridKnowledgeSearch( + SecureKnowledgeRetrievalStore store, + KnowledgeEvidenceScopeResolver evidenceScopes, + KnowledgeSearchAuthorizationService searchAuthorization, + RelationshipAuthorizationSetPort authorization, + QueryEmbeddingPort embeddings, + PermissionAuditService audit, + KnowledgeRetrievalProperties properties) { + this.store = store; + this.evidenceScopes = evidenceScopes; + this.searchAuthorization = searchAuthorization; + this.batchRecheck = new OpenFgaBatchRecheck(authorization); + this.embeddings = embeddings; + this.audit = audit; + this.properties = properties; + } + + @Transactional(readOnly = true) + @Override + public SecureKnowledgeSearchResult search( + CurrentActor actor, + String query, + Integer requestedLimit, + String suppliedRequestId) { + String requestId = requestId(suppliedRequestId); + String normalizedQuery = normalizeQuery(query); + int limit = validateLimit(requestedLimit); + String entryModelId = + searchAuthorization.require( + actor, + requestId, + normalizedQuery); + ResolvedKnowledgeEvidenceScope evidenceScope; + try { + evidenceScope = evidenceScopes.resolve(actor, entryModelId); + } catch (KnowledgeEvidenceScopeUnavailableException unavailable) { + throw searchAuthorization.unavailable( + actor, + requestId, + normalizedQuery, + unavailable.reasonCode(), + unavailable.policyVersion()); + } + Set authorizedAssetIds = evidenceScope.allAssetIds(); + if (authorizedAssetIds.isEmpty()) { + audit.record(searchAudit( + actor, + requestId, + normalizedQuery, + PermissionAuditDecision.ALLOW, + "NO_AUTHORIZED_KNOWLEDGE_ASSETS", + evidenceScope.authorizationModelId())); + return new SecureKnowledgeSearchResult(requestId, List.of()); + } + + var scope = new SecureKnowledgeRetrievalStore.RetrievalScope( + actor.organizationId(), + actor.userId(), + evidenceScope.actorDepartmentId(), + evidenceScope.actorExecutive(), + authorizedAssetIds.stream().sorted().toList(), + evidenceScope.authorizationModelId(), + evidenceScope.evaluatedAt()); + int candidateLimit = Math.multiplyExact(limit, properties.candidateMultiplier()); + List lexical = store.lexical(scope, normalizedQuery, candidateLimit); + Optional queryEmbedding = embeddings.embed(actor.organizationId(), normalizedQuery); + List semantic = queryEmbedding + .map(embedding -> store.semantic(scope, embedding, candidateLimit)) + .orElseGet(List::of); + validateCandidateSet( + actor, + requestId, + normalizedQuery, + lexical, + authorizedAssetIds, + evidenceScope.authorizationModelId()); + validateCandidateSet( + actor, + requestId, + normalizedQuery, + semantic, + authorizedAssetIds, + evidenceScope.authorizationModelId()); + + List ranked = fuse(lexical, semantic).stream() + .limit(limit) + .toList(); + if (ranked.isEmpty()) { + audit.record(searchAudit( + actor, + requestId, + normalizedQuery, + PermissionAuditDecision.ALLOW, + "NO_ELIGIBLE_EVIDENCE", + evidenceScope.authorizationModelId())); + return new SecureKnowledgeSearchResult(requestId, List.of()); + } + + List rankedResources = ranked.stream() + .map(candidate -> ResourceRef.of( + actor.organizationId(), RESOURCE_TYPE, candidate.candidate().knowledgeAssetId())) + .distinct() + .toList(); + var authorizationRecheck = batchRecheck.recheck( + new BatchAuthorizationQuery( + actor.organizationId(), + actor.principal(), + CAN_VIEW, + rankedResources), + evidenceScope.authorizationModelId(), + OpenFgaBatchRecheck.ResultPolicy.FILTER_DENIED, + RECHECK_REASONS); + if (!authorizationRecheck.succeeded()) { + var failure = authorizationRecheck.failure(); + throw searchAuthorization.unavailable( + actor, + requestId, + normalizedQuery, + failure.reasonCode(), + failure.policyVersion()); + } + Set allowedAssetIds = authorizationRecheck.allowedResources().stream() + .map(resource -> UUID.fromString(resource.id())) + .collect(java.util.stream.Collectors.toCollection( + LinkedHashSet::new)); + + List allowed = ranked.stream() + .filter(candidate -> allowedAssetIds.contains(candidate.candidate().knowledgeAssetId())) + .toList(); + Map rechecked = store.recheck( + scope, + allowed.stream().map(candidate -> candidate.candidate().chunkId()).toList()) + .stream() + .collect(java.util.stream.Collectors.toMap( + SecureRetrievalCandidate::chunkId, + candidate -> candidate)); + + List evidence = new ArrayList<>(); + List auditCommands = new ArrayList<>(); + auditCommands.add(searchAudit( + actor, + requestId, + normalizedQuery, + PermissionAuditDecision.ALLOW, + "SECURE_RETRIEVAL_APPLIED", + evidenceScope.authorizationModelId())); + for (ScoredCandidate scored : allowed) { + SecureRetrievalCandidate canonical = rechecked.get(scored.candidate().chunkId()); + if (canonical == null + || !canonical.knowledgeAssetId().equals(scored.candidate().knowledgeAssetId())) { + auditCommands.add(evidenceAudit( + actor, + requestId, + normalizedQuery, + scored.candidate(), + PermissionAuditDecision.DENY, + "CANONICAL_RECHECK_DENIED")); + continue; + } + auditCommands.add(evidenceAudit( + actor, + requestId, + normalizedQuery, + canonical, + PermissionAuditDecision.ALLOW, + "VERIFIED_EVIDENCE")); + evidence.add(new RetrievedKnowledgeEvidence( + canonical.chunkId(), + canonical.knowledgeAssetId(), + canonical.sourceObjectId(), + canonical.sourceRevisionId(), + canonical.title(), + canonical.content(), + SourceCitationUri.safeForOutput(canonical.sourceUri()), + canonical.startPage(), + canonical.endPage(), + canonical.heading(), + scored.lexicalScore(), + scored.vectorScore(), + scored.relevanceScore(), + canonical.ingestionAclSnapshotId(), + canonical.currentAclSnapshotId(), + canonical.authorizationModelId(), + canonical.embeddingProfileId(), + canonical.projectionGeneration())); + } + audit.recordAll(auditCommands); + return new SecureKnowledgeSearchResult(requestId, evidence); + } + + private void validateCandidateSet( + CurrentActor actor, + String requestId, + String query, + List candidates, + Set authorizedAssetIds, + String policyVersion) { + boolean invalid = candidates.stream() + .anyMatch(candidate -> !actor.organizationId().equals(candidate.organizationId()) + || !authorizedAssetIds.contains(candidate.knowledgeAssetId())); + if (invalid) { + throw searchAuthorization.unavailable( + actor, + requestId, + query, + "RETRIEVAL_AUTHORIZATION_BOUNDARY_VIOLATION", + policyVersion); + } + } + + private List fuse( + List lexical, + List semantic) { + Map fused = new LinkedHashMap<>(); + addRanks(fused, lexical, true); + addRanks(fused, semantic, false); + return fused.values().stream() + .map(MutableScore::freeze) + .sorted(Comparator.comparingDouble(ScoredCandidate::relevanceScore).reversed() + .thenComparing(candidate -> candidate.candidate().chunkId())) + .toList(); + } + + private static void addRanks( + Map fused, + List candidates, + boolean lexical) { + Set countedAssets = new LinkedHashSet<>(); + int rank = 0; + for (SecureRetrievalCandidate candidate : candidates) { + if (!countedAssets.add(candidate.knowledgeAssetId())) { + continue; + } + MutableScore score = fused.computeIfAbsent(candidate.chunkId(), ignored -> new MutableScore(candidate)); + score.relevance += 1.0d / (RRF_RANK_CONSTANT + rank + 1); + if (lexical) { + score.lexical = candidate.score(); + } else { + score.vector = candidate.score(); + } + rank++; + } + } + + private PermissionAuditCommand searchAudit( + CurrentActor actor, + String requestId, + String query, + PermissionAuditDecision decision, + String reason, + String policyVersion) { + return searchAuthorization.command( + actor, + requestId, + query, + decision, + reason, + policyVersion); + } + + private PermissionAuditCommand evidenceAudit( + CurrentActor actor, + String requestId, + String query, + SecureRetrievalCandidate candidate, + PermissionAuditDecision decision, + String reason) { + return new PermissionAuditCommand( + actor.organizationId(), + actor.userId(), + "SEARCH", + "KNOWLEDGE_EVIDENCE", + candidate.chunkId().toString(), + decision, + reason, + POLICY_VERSION, + requestId, + query, + candidate.ingestionAclSnapshotId(), + candidate.currentAclSnapshotId(), + candidate.authorizationModelId(), + candidate.sourceRevisionId(), + candidate.chunkId(), + candidate.embeddingProfileId(), + candidate.projectionGeneration()); + } + + private String normalizeQuery(String query) { + if (query == null || query.isBlank()) { + throw new BusinessValidationException( + "knowledge-search.query-required", + "q is required"); + } + String normalized = query.strip(); + if (normalized.length() > properties.maximumQueryLength()) { + throw new BusinessValidationException( + "knowledge-search.query-invalid", + "q must not exceed " + properties.maximumQueryLength() + " characters"); + } + return normalized; + } + + private int validateLimit(Integer requestedLimit) { + int limit = requestedLimit == null ? Math.min(10, properties.maximumResults()) : requestedLimit; + if (limit < 1 || limit > properties.maximumResults()) { + throw new BusinessValidationException( + "knowledge-search.limit-invalid", + "limit must be between 1 and " + properties.maximumResults()); + } + return limit; + } + + private static String requestId(String requestId) { + if (requestId == null || requestId.isBlank()) { + return UUID.randomUUID().toString(); + } + String normalized = requestId.strip(); + return normalized.length() <= MAX_REQUEST_ID_LENGTH ? normalized : UUID.randomUUID().toString(); + } + + private static final class MutableScore { + private final SecureRetrievalCandidate candidate; + private double lexical; + private double vector; + private double relevance; + + private MutableScore(SecureRetrievalCandidate candidate) { + this.candidate = candidate; + } + + private ScoredCandidate freeze() { + return new ScoredCandidate(candidate, lexical, vector, relevance); + } + } + + private record ScoredCandidate( + SecureRetrievalCandidate candidate, + double lexicalScore, + double vectorScore, + double relevanceScore) { + } +} diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCitationContentService.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCitationContentService.java new file mode 100644 index 000000000..f904a3447 --- /dev/null +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCitationContentService.java @@ -0,0 +1,155 @@ +package com.orgmemory.core.knowledge.retrieval; + +import com.orgmemory.core.knowledge.sourceledger.SourceCitationEvidence; +import com.orgmemory.core.knowledge.sourceledger.SourceCitationEvidenceQuery; +import com.orgmemory.core.knowledge.sourceledger.SourceCitationEvidenceResult; + +import com.orgmemory.core.knowledge.storage.ObjectContent; +import com.orgmemory.core.knowledge.storage.ObjectStoragePort; +import com.orgmemory.core.organization.CurrentActor; +import com.orgmemory.core.permission.PermissionAuditCommand; +import com.orgmemory.core.permission.PermissionAuditDecision; +import com.orgmemory.core.permission.PermissionAuditService; +import java.util.Objects; +import java.util.UUID; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** + * Opens original source evidence through the same permission boundary used by + * retrieval. No storage URL or object key crosses the application boundary. + */ +@Service +class DefaultCitationContentService implements CitationContentService { + + private final CanonicalEvidenceAuthorizationService authorization; + private final SourceCitationEvidenceQuery evidenceQuery; + private final ObjectStoragePort objects; + private final PermissionAuditService audit; + + DefaultCitationContentService( + CanonicalEvidenceAuthorizationService authorization, + SourceCitationEvidenceQuery evidenceQuery, + ObjectStoragePort objects, + PermissionAuditService audit) { + this.authorization = authorization; + this.evidenceQuery = evidenceQuery; + this.objects = objects; + this.audit = audit; + } + + @Transactional(readOnly = true) + @Override + public CitationContent open( + CurrentActor actor, + UUID chunkId, + String requestId) { + Objects.requireNonNull(actor, "actor"); + Objects.requireNonNull(chunkId, "chunkId"); + String normalizedRequestId = requestId == null || requestId.isBlank() + ? UUID.randomUUID().toString() + : requestId.strip(); + String auditQuery = "citation:" + chunkId; + CanonicalEvidenceAuthorizationService.Verification verified; + try { + verified = authorization.verify( + actor, + normalizedRequestId, + auditQuery, + java.util.List.of(chunkId)); + } catch (CanonicalEvidenceAuthorizationException denied) { + throw notFound( + actor, + chunkId, + normalizedRequestId, + denied.authorizationModelId(), + denied.reasonCode()); + } + SecureRetrievalCandidate currentCandidate = + verified.candidates().getFirst(); + String authorizationModelId = verified.authorizationModelId(); + + SourceCitationEvidenceResult result = evidenceQuery.findAvailable( + actor.organizationId(), + currentCandidate.sourceRevisionId(), + currentCandidate.knowledgeAssetId()); + SourceCitationEvidence evidence = switch (result) { + case SourceCitationEvidenceResult.Available available -> + available.evidence(); + case SourceCitationEvidenceResult.Unavailable unavailable -> { + String reason = switch (unavailable.reason()) { + case REVISION_NOT_CURRENT -> "CITATION_REVISION_NOT_CURRENT"; + case BLOB_NOT_AVAILABLE -> "CITATION_BLOB_NOT_AVAILABLE"; + }; + throw notFound( + actor, + chunkId, + normalizedRequestId, + authorizationModelId, + reason); + } + }; + + var content = objects.open(evidence.objectKey()); + if (!evidence.storedContentSha256().equals(content.metadata().sha256()) + || evidence.storedContentLength() + != content.metadata().contentLength()) { + closeQuietly(content); + throw new KnowledgeRetrievalUnavailableException( + "Citation evidence failed its integrity check"); + } + audit.record(new PermissionAuditCommand( + actor.organizationId(), + actor.userId(), + "READ_CITATION", + "KNOWLEDGE_CHUNK", + chunkId.toString(), + PermissionAuditDecision.ALLOW, + "AUTHORIZED_CITATION_CONTENT", + authorizationModelId, + normalizedRequestId, + null, + currentCandidate.ingestionAclSnapshotId(), + currentCandidate.currentAclSnapshotId(), + currentCandidate.authorizationModelId(), + currentCandidate.sourceRevisionId(), + currentCandidate.chunkId(), + currentCandidate.embeddingProfileId(), + currentCandidate.projectionGeneration())); + return new CitationContent( + chunkId, + evidence.fileName(), + evidence.mediaType(), + evidence.contentLength(), + evidence.contentSha256(), + content); + } + + private static void closeQuietly(ObjectContent content) { + try { + content.close(); + } catch (java.io.IOException ignored) { + // The authorization or integrity failure is authoritative. + } + } + + private CitationNotFoundException notFound( + CurrentActor actor, + UUID chunkId, + String requestId, + String authorizationModelId, + String reason) { + audit.record(new PermissionAuditCommand( + actor.organizationId(), + actor.userId(), + "READ_CITATION", + "KNOWLEDGE_CHUNK", + chunkId.toString(), + PermissionAuditDecision.DENY, + reason, + authorizationModelId, + requestId, + null)); + return new CitationNotFoundException(); + } +} diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultGraphRagKnowledgeRetrievalService.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultGraphRagKnowledgeRetrievalService.java new file mode 100644 index 000000000..a66e5ec7a --- /dev/null +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultGraphRagKnowledgeRetrievalService.java @@ -0,0 +1,1026 @@ +package com.orgmemory.core.knowledge.retrieval; + +import com.orgmemory.core.ai.ChatGenerationRequest; +import com.orgmemory.core.authorization.BatchAuthorizationQuery; +import com.orgmemory.core.authorization.PermissionKey; +import com.orgmemory.core.authorization.RelationshipAuthorizationSetPort; +import com.orgmemory.core.authorization.ResourceRef; +import com.orgmemory.core.knowledge.asset.KnowledgeProjectionNamespaces; +import com.orgmemory.core.knowledge.search.RetrievedKnowledgeEvidence; +import com.orgmemory.core.knowledge.search.SecureKnowledgeSearchResult; +import com.orgmemory.core.knowledge.search.VerifiedKnowledgeGrounding; +import com.orgmemory.core.knowledge.sourceledger.SourceCitationUri; +import com.orgmemory.core.organization.CurrentActor; +import com.orgmemory.core.permission.PermissionAuditCommand; +import com.orgmemory.core.permission.PermissionAuditDecision; +import com.orgmemory.core.permission.PermissionAuditService; +import com.orgmemory.core.shared.error.BusinessErrorCategory; +import com.orgmemory.core.shared.error.BusinessException; +import com.orgmemory.core.shared.error.BusinessValidationException; +import com.orgmemory.graphrag.cache.CanonicalCacheKeyHasher; +import com.orgmemory.graphrag.model.EvidenceReference; +import com.orgmemory.graphrag.observability.GraphRagEventSink; +import com.orgmemory.graphrag.observability.GraphRagTaskDecorator; +import com.orgmemory.graphrag.query.ContextTokenUsage; +import com.orgmemory.graphrag.query.LightRagGrounding; +import com.orgmemory.graphrag.query.LightRagGroundingAssembler; +import com.orgmemory.graphrag.query.LightRagPreparedQuery; +import com.orgmemory.graphrag.query.LightRagQueryEngine; +import com.orgmemory.graphrag.query.LightRagQueryRequest; +import com.orgmemory.graphrag.query.LightRagQueryResult; +import com.orgmemory.graphrag.query.SecureContextBudget; +import com.orgmemory.graphrag.storage.ProjectionNamespace; +import com.orgmemory.graphrag.storage.ProjectionPublicationStore; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Permission-aware application shell around the framework-neutral LightRAG + * engine. OpenFGA ListObjects establishes the scope; BatchCheck and the + * canonical ledger recheck it before any selected evidence reaches the model. + */ +class DefaultGraphRagKnowledgeRetrievalService + implements GraphRagKnowledgeRetrievalService { + + private static final PermissionKey CAN_VIEW = PermissionKey.of("can_view"); + private static final String RESOURCE_TYPE = "knowledge_asset"; + private static final OpenFgaBatchRecheck.ReasonRule RESULT_REASON = + OpenFgaBatchRecheck.ReasonRule.resultReason(); + private static final OpenFgaBatchRecheck.ReasonRule FINAL_RECHECK_DENIED = + OpenFgaBatchRecheck.ReasonRule.fixed( + "FINAL_OPENFGA_RECHECK_DENIED"); + private static final OpenFgaBatchRecheck.ReasonMapping RECHECK_REASONS = + new OpenFgaBatchRecheck.ReasonMapping( + RESULT_REASON, + RESULT_REASON, + RESULT_REASON, + FINAL_RECHECK_DENIED, + FINAL_RECHECK_DENIED, + FINAL_RECHECK_DENIED); + private static final int MAX_REQUEST_ID_LENGTH = 128; + + private final KnowledgeSearchAuthorizationService searchAuthorization; + private final KnowledgeEvidenceScopeResolver evidenceScopes; + private final OpenFgaBatchRecheck batchRecheck; + private final SecureKnowledgeRetrievalStore canonicalEvidence; + private final EmbeddingProfileRegistry embeddingProfiles; + private final KnowledgeEmbeddingProperties embedding; + private final ProjectionPublicationStore publications; + private final LightRagQueryEngine engine; + private final GraphRagRetrievalPolicy policy; + private final PermissionAuditService audit; + private final KnowledgeRetrievalProperties retrievalProperties; + private final GraphRagEventSink events; + private final GraphRagTaskDecorator tasks; + + DefaultGraphRagKnowledgeRetrievalService( + KnowledgeSearchAuthorizationService searchAuthorization, + KnowledgeEvidenceScopeResolver evidenceScopes, + RelationshipAuthorizationSetPort authorization, + SecureKnowledgeRetrievalStore canonicalEvidence, + EmbeddingProfileRegistry embeddingProfiles, + KnowledgeEmbeddingProperties embedding, + ProjectionPublicationStore publications, + LightRagQueryEngine engine, + GraphRagRetrievalPolicy policy, + PermissionAuditService audit, + KnowledgeRetrievalProperties retrievalProperties, + GraphRagEventSink events) { + this( + searchAuthorization, + evidenceScopes, + authorization, + canonicalEvidence, + embeddingProfiles, + embedding, + publications, + engine, + policy, + audit, + retrievalProperties, + events, + GraphRagTaskDecorator.NONE); + } + + DefaultGraphRagKnowledgeRetrievalService( + KnowledgeSearchAuthorizationService searchAuthorization, + KnowledgeEvidenceScopeResolver evidenceScopes, + RelationshipAuthorizationSetPort authorization, + SecureKnowledgeRetrievalStore canonicalEvidence, + EmbeddingProfileRegistry embeddingProfiles, + KnowledgeEmbeddingProperties embedding, + ProjectionPublicationStore publications, + LightRagQueryEngine engine, + GraphRagRetrievalPolicy policy, + PermissionAuditService audit, + KnowledgeRetrievalProperties retrievalProperties, + GraphRagEventSink events, + GraphRagTaskDecorator tasks) { + this.searchAuthorization = searchAuthorization; + this.evidenceScopes = evidenceScopes; + this.batchRecheck = new OpenFgaBatchRecheck(authorization); + this.canonicalEvidence = canonicalEvidence; + this.embeddingProfiles = embeddingProfiles; + this.embedding = embedding; + this.publications = publications; + this.engine = engine; + this.policy = policy; + this.audit = audit; + this.retrievalProperties = retrievalProperties; + this.events = Objects.requireNonNull(events, "events"); + this.tasks = Objects.requireNonNull(tasks, "tasks"); + } + + @Override + public SecureKnowledgeSearchResult search( + CurrentActor actor, + String query, + Integer requestedLimit, + String suppliedRequestId) { + Objects.requireNonNull(actor, "actor"); + UUID operationId = UUID.randomUUID(); + long startedAt = System.nanoTime(); + try { + String requestId = requestId(suppliedRequestId); + String normalizedQuery = normalizeQuery(query); + int limit = validateLimit(requestedLimit); + long authorizationStartedAt = System.nanoTime(); + String authorizationModelId = searchAuthorization.require( + actor, + requestId, + normalizedQuery); + emitStage( + operationId, + actor.organizationId(), + GraphRagEventSink.Stage.AUTHORIZE, + authorizationStartedAt, + 1, + 1); + SecureKnowledgeSearchResult result = search( + actor, + normalizedQuery, + limit, + requestId, + authorizationModelId, + operationId, + 0); + emit( + operationId, + actor.organizationId(), + GraphRagEventSink.Outcome.SUCCEEDED, + startedAt, + result.evidence().size(), + null); + return result; + } catch (RuntimeException failure) { + emit( + operationId, + actor.organizationId(), + GraphRagEventSink.Outcome.FAILED, + startedAt, + 0, + failureCode(failure)); + throw failure; + } + } + + private void emit( + UUID operationId, + UUID organizationId, + GraphRagEventSink.Outcome outcome, + long startedAt, + int outputCount, + String failureCode) { + try { + events.emit(new GraphRagEventSink.GraphRagEvent( + operationId, + organizationId, + GraphRagEventSink.Stage.RETRIEVE, + outcome, + Duration.ofNanos(Math.max(0, System.nanoTime() - startedAt)), + 1, + outputCount, + null, + null, + null, + failureCode, + Instant.now())); + } catch (RuntimeException ignoredTelemetryFailure) { + // Telemetry must never become a retrieval availability dependency. + } + } + + private static String failureCode(RuntimeException failure) { + if (failure instanceof BusinessException business + && business.category() == BusinessErrorCategory.VALIDATION) { + return "invalid_request"; + } + if (failure instanceof KnowledgeRetrievalUnavailableException) { + return "retrieval_unavailable"; + } + return "retrieval_failed"; + } + + private SecureKnowledgeSearchResult search( + CurrentActor actor, + String query, + int limit, + String requestId, + String authorizationModelId, + UUID operationId, + int attempt) { + ResolvedKnowledgeEvidenceScope initial = + resolve( + actor, + authorizationModelId, + requestId, + query, + operationId); + if (initial.allAssetIds().isEmpty()) { + audit.record(searchAuthorization.command( + actor, + requestId, + query, + PermissionAuditDecision.ALLOW, + "NO_AUTHORIZED_KNOWLEDGE_ASSETS", + initial.authorizationModelId())); + return new SecureKnowledgeSearchResult(requestId, List.of()); + } + + EmbeddingProfileRef profile = embeddingProfiles + .find( + actor.organizationId(), + new EmbeddingProfileSpec( + embedding.provider(), + embedding.model(), + embedding.dimensions(), + EmbeddingDistanceMetric.COSINE)) + .orElseThrow(() -> searchAuthorization.unavailable( + actor, + requestId, + query, + "EMBEDDING_PROFILE_NOT_INDEXED", + initial.authorizationModelId())); + + LightRagQueryRequest.Options queryOptions = + policy.contextOptions(limit); + List spaceGroundings = + queryPublishedSpaces( + initial, + profile, + query, + queryOptions, + operationId); + if (spaceGroundings.isEmpty()) { + audit.record(searchAuthorization.command( + actor, + requestId, + query, + PermissionAuditDecision.ALLOW, + "NO_ELIGIBLE_EVIDENCE", + initial.authorizationModelId())); + return new SecureKnowledgeSearchResult(requestId, List.of()); + } + long consolidationStartedAt = System.nanoTime(); + LightRagGroundingAssembler.PreparedGrounding consolidated = + engine.consolidateGrounding( + query, + queryOptions, + spaceGroundings); + emitAssembledContext( + operationId, + actor.organizationId(), + consolidationStartedAt, + spaceGroundings.size(), + consolidated, + queryOptions.contextBudget()); + if (consolidated.grounding().empty() + || consolidated.grounding().chunks().isEmpty()) { + audit.record(searchAuthorization.command( + actor, + requestId, + query, + PermissionAuditDecision.ALLOW, + "NO_CITABLE_GROUNDING", + initial.authorizationModelId())); + return new SecureKnowledgeSearchResult(requestId, List.of()); + } + + ResolvedKnowledgeEvidenceScope current = + resolve( + actor, + authorizationModelId, + requestId, + query, + operationId); + if (!sameAuthorizationScope(initial, current)) { + return retryOrFail( + actor, + query, + limit, + requestId, + authorizationModelId, + operationId, + attempt, + "AUTHORIZATION_SCOPE_CHANGED"); + } + + List closure = + consolidated.grounding().evidenceClosure(); + if (closure.size() > policy.maximumEvidenceClosure()) { + throw searchAuthorization.unavailable( + actor, + requestId, + query, + "GROUNDING_EVIDENCE_CLOSURE_EXCEEDED", + current.authorizationModelId()); + } + long finalAuthorizationStartedAt = System.nanoTime(); + verifyOpenFga( + actor, + query, + requestId, + current.authorizationModelId(), + closure); + emitStage( + operationId, + actor.organizationId(), + GraphRagEventSink.Stage.AUTHORIZE, + finalAuthorizationStartedAt, + closure.size(), + closure.size()); + List verified = + recheckCanonical(current, closure); + if (!sameEvidence(closure, verified)) { + return retryOrFail( + actor, + query, + limit, + requestId, + authorizationModelId, + operationId, + attempt, + "CANONICAL_EVIDENCE_CHANGED"); + } + LightRagGroundingAssembler.PreparedGrounding rendered = + engine.renderGrounding( + query, + queryOptions, + consolidated.grounding()); + + Map canonicalByChunk = verified.stream() + .collect(Collectors.toMap( + SecureRetrievalCandidate::chunkId, + Function.identity())); + Map scoreByChunk = consolidated.grounding() + .chunks() + .stream() + .collect(Collectors.toMap( + LightRagGrounding.SelectedChunk::id, + LightRagGrounding.SelectedChunk::effectiveScore, + Math::max, + LinkedHashMap::new)); + List evidence = rendered.references() + .stream() + .map(reference -> toEvidence( + Objects.requireNonNull( + canonicalByChunk.get( + reference.evidence().chunkId()), + "verified citation evidence"), + scoreByChunk.getOrDefault( + reference.evidence().chunkId(), + 0.0))) + .toList(); + List auditCommands = new ArrayList<>(); + auditCommands.add(searchAuthorization.command( + actor, + requestId, + query, + PermissionAuditDecision.ALLOW, + "SECURE_GRAPH_RAG_RETRIEVAL_APPLIED", + current.authorizationModelId())); + for (LightRagGrounding.GroundingEvidence groundingEvidence : closure) { + SecureRetrievalCandidate canonical = canonicalByChunk.get( + groundingEvidence.evidence().chunkId()); + auditCommands.add(evidenceAudit( + actor, + requestId, + query, + canonical, + current.authorizationModelId(), + "VERIFIED_GRAPH_RAG_GROUNDING")); + } + audit.recordAll(auditCommands); + return new SecureKnowledgeSearchResult( + requestId, + evidence, + Optional.of(new VerifiedKnowledgeGrounding( + new ChatGenerationRequest( + rendered.systemPrompt(), + query), + evidence, + closure.size(), + rendered.inputTokens()))); + } + + private List queryPublishedSpaces( + ResolvedKnowledgeEvidenceScope scope, + EmbeddingProfileRef profile, + String query, + LightRagQueryRequest.Options options, + UUID operationId) { + if (scope.knowledgeSpaceIds().size() + > policy.maximumKnowledgeSpaces()) { + throw new KnowledgeRetrievalUnavailableException( + "Secure knowledge retrieval is temporarily unavailable"); + } + List requests = new ArrayList<>(); + for (UUID knowledgeSpaceId : + scope.knowledgeSpaceIds().stream().sorted().toList()) { + var evidenceScope = scope.forKnowledgeSpace(knowledgeSpaceId); + ProjectionNamespace namespace = namespace( + scope.organizationId(), + knowledgeSpaceId); + var snapshot = publications.current(namespace); + if (snapshot.isEmpty()) { + continue; + } + requests.add(new LightRagQueryRequest( + evidenceScope, + snapshot.orElseThrow(), + query, + options, + profile.id(), + profile.dimensions(), + null, + List.of())); + } + if (requests.isEmpty()) { + return List.of(); + } + if (requests.size() > 1 && policy.rerank().enabled()) { + throw new KnowledgeRetrievalUnavailableException( + "Secure knowledge retrieval is temporarily unavailable"); + } + + LightRagPreparedQuery prepared = engine.prepare(requests.getFirst()); + emitPreparedStage( + operationId, + scope.organizationId(), + GraphRagEventSink.Stage.PREPARE_QUERY, + prepared.keywordPlanningDuration(), + 1, + prepared.keywords().highLevel().size() + + prepared.keywords().lowLevel().size(), + prepared.keywordModelRouteFingerprint(), + prepared.keywordCacheStatus()); + emitPreparedStage( + operationId, + scope.organizationId(), + GraphRagEventSink.Stage.EMBED, + prepared.embeddingDuration(), + prepared.embeddingInputs().size(), + prepared.embeddingInputs().size(), + null, + null); + List groundings = new ArrayList<>(); + try (ExecutorService executor = + Executors.newVirtualThreadPerTaskExecutor()) { + for (int offset = 0; + offset < requests.size(); + offset += policy.maximumConcurrentSpaces()) { + int end = Math.min( + requests.size(), + offset + policy.maximumConcurrentSpaces()); + List> futures = + requests.subList(offset, end) + .stream() + .map(request -> executor.submit(tasks.decorate(() -> + queryPublishedSpace(request, prepared)))) + .toList(); + try { + for (Future future : futures) { + SnapshotQueryResult snapshotResult = future.get(); + emitSnapshotStage( + operationId, + scope.organizationId(), + snapshotResult.duration(), + snapshotResult.inputCount(), + snapshotResult.result() + .grounding() + .chunks() + .size(), + snapshotResult.namespace()); + emitRerank( + operationId, + scope.organizationId(), + snapshotResult.result()); + LightRagGrounding grounding = + snapshotResult.result().grounding(); + if (!grounding.empty()) { + groundings.add(grounding); + } + } + } catch (ExecutionException | InterruptedException + | RuntimeException failure) { + futures.forEach(future -> future.cancel(true)); + throw retrievalFailure(failure); + } + } + } + return List.copyOf(groundings); + } + + private SnapshotQueryResult queryPublishedSpace( + LightRagQueryRequest request, + LightRagPreparedQuery prepared) { + long startedAt = System.nanoTime(); + LightRagQueryResult result = + engine.executePrepared(request, prepared); + return new SnapshotQueryResult( + result, + Duration.ofNanos(Math.max( + 0, + System.nanoTime() - startedAt)), + request.scope().authorizedAssetIds().size(), + request.snapshot().namespace()); + } + + private static RuntimeException retrievalFailure(Exception failure) { + if (failure instanceof InterruptedException) { + Thread.currentThread().interrupt(); + return new KnowledgeRetrievalUnavailableException( + "Secure knowledge retrieval is temporarily unavailable"); + } + Throwable cause = failure instanceof ExecutionException + ? failure.getCause() + : failure; + if (cause instanceof RuntimeException runtime) { + return runtime; + } + if (cause instanceof Error error) { + throw error; + } + return new KnowledgeRetrievalUnavailableException( + "Secure knowledge retrieval is temporarily unavailable"); + } + + private ResolvedKnowledgeEvidenceScope resolve( + CurrentActor actor, + String authorizationModelId, + String requestId, + String query, + UUID operationId) { + long startedAt = System.nanoTime(); + try { + ResolvedKnowledgeEvidenceScope resolved = + evidenceScopes.resolve(actor, authorizationModelId); + emitStage( + operationId, + actor.organizationId(), + GraphRagEventSink.Stage.AUTHORIZE, + startedAt, + 1, + resolved.allAssetIds().size()); + return resolved; + } catch (KnowledgeEvidenceScopeUnavailableException unavailable) { + throw searchAuthorization.unavailable( + actor, + requestId, + query, + unavailable.reasonCode(), + unavailable.policyVersion()); + } + } + + private void emitStage( + UUID operationId, + UUID organizationId, + GraphRagEventSink.Stage stage, + long startedAt, + int inputCount, + int outputCount) { + try { + events.emit(new GraphRagEventSink.GraphRagEvent( + operationId, + organizationId, + stage, + GraphRagEventSink.Outcome.SUCCEEDED, + Duration.ofNanos(Math.max( + 0, + System.nanoTime() - startedAt)), + inputCount, + outputCount, + null, + null, + null, + null, + Instant.now())); + } catch (RuntimeException ignoredTelemetryFailure) { + // Telemetry must never become a retrieval availability dependency. + } + } + + /** + * Context assembly is the one retrieval stage whose cost is measured in + * tokens rather than items, and the only stage that can report how much of + * the retrieved context the budget refused to carry. Both numbers were being + * computed and discarded. + */ + private void emitAssembledContext( + UUID operationId, + UUID organizationId, + long startedAt, + int inputCount, + LightRagGroundingAssembler.PreparedGrounding prepared, + SecureContextBudget budget) { + LightRagGrounding grounding = prepared.grounding(); + ContextTokenUsage usage = grounding.tokenUsage(); + try { + events.emit(new GraphRagEventSink.GraphRagEvent( + operationId, + organizationId, + GraphRagEventSink.Stage.ASSEMBLE_CONTEXT, + GraphRagEventSink.Outcome.SUCCEEDED, + Duration.ofNanos(Math.max( + 0, + System.nanoTime() - startedAt)), + inputCount, + grounding.chunks().size(), + null, + null, + null, + null, + new GraphRagEventSink.TokenUsage( + prepared.inputTokens(), + usage.systemPromptTokens(), + usage.queryTokens(), + usage.entityTokens(), + usage.relationTokens(), + grounding.chunkTokens(), + budget.maximumInputTokens(), + prepared.droppedContributions()), + null, + Instant.now())); + } catch (RuntimeException ignoredTelemetryFailure) { + // Telemetry must never become a retrieval availability dependency. + } + } + + private void emitPreparedStage( + UUID operationId, + UUID organizationId, + GraphRagEventSink.Stage stage, + Duration duration, + int inputCount, + int outputCount, + String modelRouteFingerprint, + GraphRagEventSink.CacheStatus cacheStatus) { + try { + events.emit(new GraphRagEventSink.GraphRagEvent( + operationId, + organizationId, + stage, + GraphRagEventSink.Outcome.SUCCEEDED, + duration, + inputCount, + outputCount, + modelRouteFingerprint, + null, + cacheStatus, + null, + Instant.now())); + } catch (RuntimeException ignoredTelemetryFailure) { + // Telemetry must never become a retrieval availability dependency. + } + } + + private void emitSnapshotStage( + UUID operationId, + UUID organizationId, + Duration duration, + int inputCount, + int outputCount, + ProjectionNamespace namespace) { + String scopeFingerprint = CanonicalCacheKeyHasher.sha256( + "orgmemory.graph-rag.snapshot-scope.v1", + Map.of( + "organizationId", + namespace.organizationId().toString(), + "workspace", + namespace.workspace(), + "collection", + namespace.collection())); + try { + events.emit(new GraphRagEventSink.GraphRagEvent( + operationId, + organizationId, + GraphRagEventSink.Stage.RETRIEVE_SNAPSHOT, + GraphRagEventSink.Outcome.SUCCEEDED, + duration, + inputCount, + outputCount, + null, + scopeFingerprint, + null, + null, + Instant.now())); + } catch (RuntimeException ignoredTelemetryFailure) { + // Telemetry must never become a retrieval availability dependency. + } + } + + private record SnapshotQueryResult( + LightRagQueryResult result, + Duration duration, + int inputCount, + ProjectionNamespace namespace) { + + private SnapshotQueryResult { + Objects.requireNonNull(result, "result"); + Objects.requireNonNull(duration, "duration"); + if (duration.isNegative() || inputCount < 0) { + throw new IllegalArgumentException( + "snapshot query metrics must be non-negative"); + } + Objects.requireNonNull(namespace, "namespace"); + } + } + + private void verifyOpenFga( + CurrentActor actor, + String query, + String requestId, + String authorizationModelId, + List closure) { + List resources = closure.stream() + .map(candidate -> ResourceRef.of( + actor.organizationId(), + RESOURCE_TYPE, + candidate.evidence() + .knowledgeAssetId())) + .distinct() + .toList(); + var rechecked = batchRecheck.recheck( + new BatchAuthorizationQuery( + actor.organizationId(), + actor.principal(), + CAN_VIEW, + resources), + authorizationModelId, + OpenFgaBatchRecheck.ResultPolicy.REQUIRE_ALL_ALLOWED, + RECHECK_REASONS); + if (!rechecked.succeeded()) { + var failure = rechecked.failure(); + String failurePolicyVersion = switch (failure.kind()) { + case MISSING_DECISION, + DECISION_POLICY_MISMATCH, + DENIED -> authorizationModelId; + case UNRESOLVED, + DECISION_COUNT_MISMATCH, + OUTER_POLICY_MISMATCH -> failure.policyVersion(); + }; + throw searchAuthorization.unavailable( + actor, + requestId, + query, + failure.reasonCode(), + failurePolicyVersion); + } + } + + private List recheckCanonical( + ResolvedKnowledgeEvidenceScope scope, + List closure) { + return canonicalEvidence.recheck( + scope.toRetrievalScope(), + closure.stream() + .map(candidate -> candidate.evidence().chunkId()) + .toList()); + } + + private SecureKnowledgeSearchResult retryOrFail( + CurrentActor actor, + String query, + int limit, + String requestId, + String authorizationModelId, + UUID operationId, + int attempt, + String reason) { + if (attempt == 0) { + return search( + actor, + query, + limit, + requestId, + authorizationModelId, + operationId, + 1); + } + throw searchAuthorization.unavailable( + actor, + requestId, + query, + reason, + authorizationModelId); + } + + private void emitRerank( + UUID operationId, + UUID organizationId, + LightRagQueryResult result) { + if (!result.trace().rerankAttempted()) { + return; + } + GraphRagEventSink.Outcome outcome = result.trace().rerankFallback() + ? GraphRagEventSink.Outcome.FAILED + : GraphRagEventSink.Outcome.SUCCEEDED; + String failureCode = result.trace().rerankFallback() + ? "rerank_provider_fallback" + : null; + String routeFingerprint = CanonicalCacheKeyHasher.sha256( + "reranker-route", + Map.of("provider", policy.rerank().provider())); + try { + events.emit(new GraphRagEventSink.GraphRagEvent( + operationId, + organizationId, + GraphRagEventSink.Stage.RERANK, + outcome, + result.trace().rerankDuration(), + result.trace().chunkSignals().size(), + result.grounding().chunks().size(), + routeFingerprint, + null, + null, + failureCode, + Instant.now())); + } catch (RuntimeException ignoredTelemetryFailure) { + // Telemetry must never become a retrieval availability dependency. + } + } + + private static boolean sameAuthorizationScope( + ResolvedKnowledgeEvidenceScope left, + ResolvedKnowledgeEvidenceScope right) { + if (!left.authorizationModelId() + .equals(right.authorizationModelId())) { + return false; + } + Set spaces = new LinkedHashSet<>( + left.knowledgeSpaceIds()); + spaces.addAll(right.knowledgeSpaceIds()); + return spaces.stream().allMatch(space -> left + .forKnowledgeSpace(space) + .authorizationFingerprint() + .equals(right.forKnowledgeSpace(space) + .authorizationFingerprint())); + } + + private static boolean sameEvidence( + List closure, + List verified) { + Map byChunk = verified.stream() + .collect(Collectors.toMap( + SecureRetrievalCandidate::chunkId, + Function.identity(), + (left, right) -> left, + LinkedHashMap::new)); + if (byChunk.size() != closure.size()) { + return false; + } + return closure.stream().allMatch(candidate -> { + EvidenceReference reference = + candidate.evidence(); + SecureRetrievalCandidate canonical = + byChunk.get(reference.chunkId()); + return canonical != null + && reference.organizationId() + .equals(canonical.organizationId()) + && reference.knowledgeAssetId() + .equals(canonical.knowledgeAssetId()) + && reference.sourceRevisionId() + .equals(canonical.sourceRevisionId()) + && reference.aclSnapshotId() + .equals(canonical.ingestionAclSnapshotId()) + && candidate.projectionGeneration() + == canonical.projectionGeneration(); + }); + } + + private static RetrievedKnowledgeEvidence toEvidence( + SecureRetrievalCandidate candidate, + double score) { + return new RetrievedKnowledgeEvidence( + candidate.chunkId(), + candidate.knowledgeAssetId(), + candidate.sourceObjectId(), + candidate.sourceRevisionId(), + candidate.title(), + candidate.content(), + SourceCitationUri.safeForOutput(candidate.sourceUri()), + candidate.startPage(), + candidate.endPage(), + candidate.heading(), + 0.0, + score, + score, + candidate.ingestionAclSnapshotId(), + candidate.currentAclSnapshotId(), + candidate.authorizationModelId(), + candidate.embeddingProfileId(), + candidate.projectionGeneration()); + } + + private static PermissionAuditCommand evidenceAudit( + CurrentActor actor, + String requestId, + String query, + SecureRetrievalCandidate candidate, + String authorizationModelId, + String reason) { + return new PermissionAuditCommand( + actor.organizationId(), + actor.userId(), + "SEARCH", + "KNOWLEDGE_EVIDENCE", + candidate.chunkId().toString(), + PermissionAuditDecision.ALLOW, + reason, + authorizationModelId, + requestId, + query, + candidate.ingestionAclSnapshotId(), + candidate.currentAclSnapshotId(), + candidate.authorizationModelId(), + candidate.sourceRevisionId(), + candidate.chunkId(), + candidate.embeddingProfileId(), + candidate.projectionGeneration()); + } + + private String normalizeQuery(String query) { + if (query == null || query.isBlank()) { + throw new BusinessValidationException( + "knowledge-search.query-required", + "q is required"); + } + String normalized = query.strip(); + if (normalized.length() + > retrievalProperties.maximumQueryLength()) { + throw new BusinessValidationException( + "knowledge-search.query-invalid", + "q must not exceed " + + retrievalProperties.maximumQueryLength() + + " characters"); + } + return normalized; + } + + private int validateLimit(Integer requestedLimit) { + int limit = requestedLimit == null + ? Math.min(10, retrievalProperties.maximumResults()) + : requestedLimit; + if (limit < 1 + || limit > retrievalProperties.maximumResults()) { + throw new BusinessValidationException( + "knowledge-search.limit-invalid", + "limit must be between 1 and " + + retrievalProperties.maximumResults()); + } + return limit; + } + + private static String requestId(String requestId) { + if (requestId == null || requestId.isBlank()) { + return UUID.randomUUID().toString(); + } + String normalized = requestId.strip(); + return normalized.length() <= MAX_REQUEST_ID_LENGTH + ? normalized + : UUID.randomUUID().toString(); + } + + private static ProjectionNamespace namespace( + UUID organizationId, + UUID knowledgeSpaceId) { + return KnowledgeProjectionNamespaces.forSpace(organizationId, knowledgeSpaceId); + } + +} diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultSourceContentService.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultSourceContentService.java new file mode 100644 index 000000000..7d894a283 --- /dev/null +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultSourceContentService.java @@ -0,0 +1,136 @@ +package com.orgmemory.core.knowledge.retrieval; + +import com.orgmemory.core.knowledge.sourceledger.SourceDocumentEvidenceQuery; +import com.orgmemory.core.knowledge.storage.ObjectContent; +import com.orgmemory.core.knowledge.storage.ObjectStoragePort; +import com.orgmemory.core.organization.CurrentActor; +import com.orgmemory.core.permission.PermissionAuditCommand; +import com.orgmemory.core.permission.PermissionAuditDecision; +import com.orgmemory.core.permission.PermissionAuditService; +import com.orgmemory.core.shared.error.KnowledgeResourceNotFoundException; +import java.util.Objects; +import java.util.UUID; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** Opens a current document through the canonical evidence authorization scope. */ +@Service +class DefaultSourceContentService implements SourceContentService { + + private final KnowledgeEvidenceScopeResolver authorization; + private final SourceDocumentEvidenceQuery evidenceQuery; + private final ObjectStoragePort objects; + private final PermissionAuditService audit; + + DefaultSourceContentService( + KnowledgeEvidenceScopeResolver authorization, + SourceDocumentEvidenceQuery evidenceQuery, + ObjectStoragePort objects, + PermissionAuditService audit) { + this.authorization = authorization; + this.evidenceQuery = evidenceQuery; + this.objects = objects; + this.audit = audit; + } + + @Transactional(readOnly = true) + @Override + public SourceContent open(CurrentActor actor, UUID sourceId, String requestId) { + Objects.requireNonNull(actor, "actor"); + Objects.requireNonNull(sourceId, "sourceId"); + String normalizedRequestId = requestId == null || requestId.isBlank() + ? UUID.randomUUID().toString() + : requestId.strip(); + var scope = authorization.resolve(actor, null); + var document = evidenceQuery + .findAvailable(actor.organizationId(), sourceId) + .orElseThrow(() -> notFound( + actor, + sourceId, + normalizedRequestId, + scope.authorizationModelId(), + "SOURCE_NOT_CURRENT")); + if (!scope.allAssetIds().contains(document.knowledgeAssetId())) { + throw notFound( + actor, + sourceId, + normalizedRequestId, + scope.authorizationModelId(), + "SOURCE_NOT_AUTHORIZED"); + } + var evidence = document.evidence(); + ObjectContent content = objects.open(evidence.objectKey()); + if (!evidence.storedContentSha256().equals(evidence.contentSha256()) + || evidence.storedContentLength() != evidence.contentLength() + || !evidence.storedContentSha256().equals(content.metadata().sha256()) + || evidence.storedContentLength() != content.metadata().contentLength()) { + closeQuietly(content); + audit.record(new PermissionAuditCommand( + actor.organizationId(), + actor.userId(), + "READ_SOURCE_CONTENT", + "SOURCE_OBJECT", + sourceId.toString(), + PermissionAuditDecision.DENY, + "SOURCE_BLOB_INTEGRITY_FAILED", + scope.authorizationModelId(), + normalizedRequestId, + null)); + throw new KnowledgeRetrievalUnavailableException( + "Source evidence failed its integrity check"); + } + audit.record(new PermissionAuditCommand( + actor.organizationId(), + actor.userId(), + "READ_SOURCE_CONTENT", + "SOURCE_OBJECT", + sourceId.toString(), + PermissionAuditDecision.ALLOW, + "AUTHORIZED_SOURCE_CONTENT", + scope.authorizationModelId(), + normalizedRequestId, + null, + null, + null, + scope.authorizationModelId(), + document.sourceRevisionId(), + null, + document.embeddingProfileId(), + null)); + return new SourceContent( + sourceId, + evidence.fileName(), + evidence.mediaType(), + evidence.contentLength(), + evidence.contentSha256(), + content); + } + + private KnowledgeResourceNotFoundException notFound( + CurrentActor actor, + UUID sourceId, + String requestId, + String policyVersion, + String reason) { + audit.record(new PermissionAuditCommand( + actor.organizationId(), + actor.userId(), + "READ_SOURCE_CONTENT", + "SOURCE_OBJECT", + sourceId.toString(), + PermissionAuditDecision.DENY, + reason, + policyVersion, + requestId, + null)); + return new KnowledgeResourceNotFoundException(); + } + + private static void closeQuietly(ObjectContent content) { + try { + content.close(); + } catch (java.io.IOException ignored) { + // Integrity failure is authoritative. + } + } +} diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/EmbeddingProfileRegistry.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/EmbeddingProfileRegistry.java index 2f808aff7..bab6df992 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/EmbeddingProfileRegistry.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/EmbeddingProfileRegistry.java @@ -1,85 +1,19 @@ package com.orgmemory.core.knowledge.retrieval; -import java.nio.charset.StandardCharsets; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; import java.util.Optional; import java.util.UUID; -import org.springframework.jdbc.core.simple.JdbcClient; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -@Service -public class EmbeddingProfileRegistry { +/** + * Adapter-facing registry for resolving immutable embedding profiles without + * exposing their persistence implementation. + */ +public interface EmbeddingProfileRegistry { - private final EmbeddingProfileRepository profiles; - private final JdbcClient jdbc; + EmbeddingProfileRef resolve(UUID organizationId, EmbeddingProfileSpec spec); - EmbeddingProfileRegistry(EmbeddingProfileRepository profiles, JdbcClient jdbc) { - this.profiles = profiles; - this.jdbc = jdbc; - } + EmbeddingProfileRef get(UUID organizationId, UUID profileId); - @Transactional - public EmbeddingProfileRef resolve(UUID organizationId, EmbeddingProfileSpec spec) { - if (organizationId == null || spec == null) { - throw new IllegalArgumentException("organization and embedding profile are required"); - } - String profileKey = spec.profileKey(); - if (profileKey.length() > 255) { - throw new IllegalArgumentException("embedding profile key is too long"); - } - UUID id = UUID.nameUUIDFromBytes( - (organizationId + ":" + profileKey).getBytes(StandardCharsets.UTF_8)); - jdbc.sql(""" - INSERT INTO embedding_profiles ( - id, organization_id, profile_key, provider, model, - dimensions, distance_metric, created_at - ) VALUES ( - :id, :organizationId, :profileKey, :provider, :model, - :dimensions, :distanceMetric, :createdAt - ) - ON CONFLICT (organization_id, profile_key) DO NOTHING - """) - .param("id", id) - .param("organizationId", organizationId) - .param("profileKey", profileKey) - .param("provider", spec.provider()) - .param("model", spec.model()) - .param("dimensions", spec.dimensions()) - .param("distanceMetric", spec.distanceMetric().name()) - .param("createdAt", OffsetDateTime.now(ZoneOffset.UTC)) - .update(); - EmbeddingProfileRef resolved = profiles.findByOrganizationIdAndProfileKey(organizationId, profileKey) - .orElseThrow(() -> new IllegalStateException("embedding profile registration failed")) - .toRef(); - if (!resolved.provider().equals(spec.provider()) - || !resolved.model().equals(spec.model()) - || resolved.dimensions() != spec.dimensions() - || resolved.distanceMetric() != spec.distanceMetric()) { - throw new IllegalStateException("embedding profile key is already bound to different settings"); - } - return resolved; - } + Optional findById(UUID organizationId, UUID profileId); - @Transactional(readOnly = true) - public EmbeddingProfileRef get(UUID organizationId, UUID profileId) { - return findById(organizationId, profileId) - .orElseThrow(() -> new IllegalStateException("embedding profile was not found")); - } - - @Transactional(readOnly = true) - public Optional findById(UUID organizationId, UUID profileId) { - return profiles.findByIdAndOrganizationId(profileId, organizationId) - .map(EmbeddingProfile::toRef); - } - - @Transactional(readOnly = true) - public Optional find(UUID organizationId, EmbeddingProfileSpec spec) { - if (organizationId == null || spec == null) { - throw new IllegalArgumentException("organization and embedding profile are required"); - } - return profiles.findByOrganizationIdAndProfileKey(organizationId, spec.profileKey()) - .map(EmbeddingProfile::toRef); - } + Optional find(UUID organizationId, EmbeddingProfileSpec spec); } diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalConfiguration.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalConfiguration.java index 13b47fef0..5dcf3eb37 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalConfiguration.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalConfiguration.java @@ -1,7 +1,5 @@ package com.orgmemory.core.knowledge.retrieval; -import com.orgmemory.core.knowledge.retrieval.EmbeddingProfileRegistry; -import com.orgmemory.core.knowledge.retrieval.KnowledgeEmbeddingProperties; import com.orgmemory.core.authorization.RelationshipAuthorizationSetPort; import com.orgmemory.core.permission.PermissionAuditService; import com.orgmemory.graphrag.observability.GraphRagEventSink; @@ -32,7 +30,7 @@ GraphRagKnowledgeRetrievalService graphRagKnowledgeRetrievalService( KnowledgeRetrievalProperties retrievalProperties, ObjectProvider eventSinks, ObjectProvider taskDecorators) { - return new GraphRagKnowledgeRetrievalService( + return new DefaultGraphRagKnowledgeRetrievalService( searchAuthorization, evidenceScopes, authorization, diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalService.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalService.java index ba8e6a1d5..22c8543fc 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalService.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalService.java @@ -1,1032 +1,7 @@ package com.orgmemory.core.knowledge.retrieval; -import com.orgmemory.core.ai.ChatGenerationRequest; -import com.orgmemory.core.authorization.BatchAuthorizationQuery; -import com.orgmemory.core.authorization.PermissionKey; -import com.orgmemory.core.authorization.RelationshipAuthorizationSetPort; -import com.orgmemory.core.authorization.ResourceRef; -import com.orgmemory.core.knowledge.retrieval.EmbeddingDistanceMetric; -import com.orgmemory.core.knowledge.retrieval.EmbeddingProfileRef; -import com.orgmemory.core.knowledge.retrieval.EmbeddingProfileRegistry; -import com.orgmemory.core.knowledge.retrieval.EmbeddingProfileSpec; -import com.orgmemory.core.knowledge.retrieval.KnowledgeEmbeddingProperties; -import com.orgmemory.core.knowledge.asset.KnowledgeProjectionNamespaces; import com.orgmemory.core.knowledge.search.PermissionAwareKnowledgeSearch; -import com.orgmemory.core.knowledge.search.RetrievedKnowledgeEvidence; -import com.orgmemory.core.knowledge.search.SecureKnowledgeSearchResult; -import com.orgmemory.core.knowledge.search.VerifiedKnowledgeGrounding; -import com.orgmemory.core.knowledge.sourceledger.SourceCitationUri; -import com.orgmemory.core.organization.CurrentActor; -import com.orgmemory.core.permission.PermissionAuditCommand; -import com.orgmemory.core.permission.PermissionAuditDecision; -import com.orgmemory.core.permission.PermissionAuditService; -import com.orgmemory.core.shared.error.BusinessErrorCategory; -import com.orgmemory.core.shared.error.BusinessException; -import com.orgmemory.core.shared.error.BusinessValidationException; -import com.orgmemory.graphrag.cache.CanonicalCacheKeyHasher; -import com.orgmemory.graphrag.model.EvidenceReference; -import com.orgmemory.graphrag.observability.GraphRagEventSink; -import com.orgmemory.graphrag.observability.GraphRagTaskDecorator; -import com.orgmemory.graphrag.query.ContextTokenUsage; -import com.orgmemory.graphrag.query.LightRagGrounding; -import com.orgmemory.graphrag.query.LightRagGroundingAssembler; -import com.orgmemory.graphrag.query.LightRagPreparedQuery; -import com.orgmemory.graphrag.query.LightRagQueryEngine; -import com.orgmemory.graphrag.query.LightRagQueryRequest; -import com.orgmemory.graphrag.query.LightRagQueryResult; -import com.orgmemory.graphrag.query.SecureContextBudget; -import com.orgmemory.graphrag.storage.ProjectionNamespace; -import com.orgmemory.graphrag.storage.ProjectionPublicationStore; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.function.Function; -import java.util.stream.Collectors; - -/** - * Permission-aware application shell around the framework-neutral LightRAG - * engine. OpenFGA ListObjects establishes the scope; BatchCheck and the - * canonical ledger recheck it before any selected evidence reaches the model. - */ -public class GraphRagKnowledgeRetrievalService - implements PermissionAwareKnowledgeSearch { - - private static final PermissionKey CAN_VIEW = PermissionKey.of("can_view"); - private static final String RESOURCE_TYPE = "knowledge_asset"; - private static final OpenFgaBatchRecheck.ReasonRule RESULT_REASON = - OpenFgaBatchRecheck.ReasonRule.resultReason(); - private static final OpenFgaBatchRecheck.ReasonRule FINAL_RECHECK_DENIED = - OpenFgaBatchRecheck.ReasonRule.fixed( - "FINAL_OPENFGA_RECHECK_DENIED"); - private static final OpenFgaBatchRecheck.ReasonMapping RECHECK_REASONS = - new OpenFgaBatchRecheck.ReasonMapping( - RESULT_REASON, - RESULT_REASON, - RESULT_REASON, - FINAL_RECHECK_DENIED, - FINAL_RECHECK_DENIED, - FINAL_RECHECK_DENIED); - private static final int MAX_REQUEST_ID_LENGTH = 128; - - private final KnowledgeSearchAuthorizationService searchAuthorization; - private final KnowledgeEvidenceScopeResolver evidenceScopes; - private final OpenFgaBatchRecheck batchRecheck; - private final SecureKnowledgeRetrievalStore canonicalEvidence; - private final EmbeddingProfileRegistry embeddingProfiles; - private final KnowledgeEmbeddingProperties embedding; - private final ProjectionPublicationStore publications; - private final LightRagQueryEngine engine; - private final GraphRagRetrievalPolicy policy; - private final PermissionAuditService audit; - private final KnowledgeRetrievalProperties retrievalProperties; - private final GraphRagEventSink events; - private final GraphRagTaskDecorator tasks; - - public GraphRagKnowledgeRetrievalService( - KnowledgeSearchAuthorizationService searchAuthorization, - KnowledgeEvidenceScopeResolver evidenceScopes, - RelationshipAuthorizationSetPort authorization, - SecureKnowledgeRetrievalStore canonicalEvidence, - EmbeddingProfileRegistry embeddingProfiles, - KnowledgeEmbeddingProperties embedding, - ProjectionPublicationStore publications, - LightRagQueryEngine engine, - GraphRagRetrievalPolicy policy, - PermissionAuditService audit, - KnowledgeRetrievalProperties retrievalProperties, - GraphRagEventSink events) { - this( - searchAuthorization, - evidenceScopes, - authorization, - canonicalEvidence, - embeddingProfiles, - embedding, - publications, - engine, - policy, - audit, - retrievalProperties, - events, - GraphRagTaskDecorator.NONE); - } - - public GraphRagKnowledgeRetrievalService( - KnowledgeSearchAuthorizationService searchAuthorization, - KnowledgeEvidenceScopeResolver evidenceScopes, - RelationshipAuthorizationSetPort authorization, - SecureKnowledgeRetrievalStore canonicalEvidence, - EmbeddingProfileRegistry embeddingProfiles, - KnowledgeEmbeddingProperties embedding, - ProjectionPublicationStore publications, - LightRagQueryEngine engine, - GraphRagRetrievalPolicy policy, - PermissionAuditService audit, - KnowledgeRetrievalProperties retrievalProperties, - GraphRagEventSink events, - GraphRagTaskDecorator tasks) { - this.searchAuthorization = searchAuthorization; - this.evidenceScopes = evidenceScopes; - this.batchRecheck = new OpenFgaBatchRecheck(authorization); - this.canonicalEvidence = canonicalEvidence; - this.embeddingProfiles = embeddingProfiles; - this.embedding = embedding; - this.publications = publications; - this.engine = engine; - this.policy = policy; - this.audit = audit; - this.retrievalProperties = retrievalProperties; - this.events = Objects.requireNonNull(events, "events"); - this.tasks = Objects.requireNonNull(tasks, "tasks"); - } - - @Override - public SecureKnowledgeSearchResult search( - CurrentActor actor, - String query, - Integer requestedLimit, - String suppliedRequestId) { - Objects.requireNonNull(actor, "actor"); - UUID operationId = UUID.randomUUID(); - long startedAt = System.nanoTime(); - try { - String requestId = requestId(suppliedRequestId); - String normalizedQuery = normalizeQuery(query); - int limit = validateLimit(requestedLimit); - long authorizationStartedAt = System.nanoTime(); - String authorizationModelId = searchAuthorization.require( - actor, - requestId, - normalizedQuery); - emitStage( - operationId, - actor.organizationId(), - GraphRagEventSink.Stage.AUTHORIZE, - authorizationStartedAt, - 1, - 1); - SecureKnowledgeSearchResult result = search( - actor, - normalizedQuery, - limit, - requestId, - authorizationModelId, - operationId, - 0); - emit( - operationId, - actor.organizationId(), - GraphRagEventSink.Outcome.SUCCEEDED, - startedAt, - result.evidence().size(), - null); - return result; - } catch (RuntimeException failure) { - emit( - operationId, - actor.organizationId(), - GraphRagEventSink.Outcome.FAILED, - startedAt, - 0, - failureCode(failure)); - throw failure; - } - } - - private void emit( - UUID operationId, - UUID organizationId, - GraphRagEventSink.Outcome outcome, - long startedAt, - int outputCount, - String failureCode) { - try { - events.emit(new GraphRagEventSink.GraphRagEvent( - operationId, - organizationId, - GraphRagEventSink.Stage.RETRIEVE, - outcome, - Duration.ofNanos(Math.max(0, System.nanoTime() - startedAt)), - 1, - outputCount, - null, - null, - null, - failureCode, - Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } - } - - private static String failureCode(RuntimeException failure) { - if (failure instanceof BusinessException business - && business.category() == BusinessErrorCategory.VALIDATION) { - return "invalid_request"; - } - if (failure instanceof KnowledgeRetrievalUnavailableException) { - return "retrieval_unavailable"; - } - return "retrieval_failed"; - } - - private SecureKnowledgeSearchResult search( - CurrentActor actor, - String query, - int limit, - String requestId, - String authorizationModelId, - UUID operationId, - int attempt) { - ResolvedKnowledgeEvidenceScope initial = - resolve( - actor, - authorizationModelId, - requestId, - query, - operationId); - if (initial.allAssetIds().isEmpty()) { - audit.record(searchAuthorization.command( - actor, - requestId, - query, - PermissionAuditDecision.ALLOW, - "NO_AUTHORIZED_KNOWLEDGE_ASSETS", - initial.authorizationModelId())); - return new SecureKnowledgeSearchResult(requestId, List.of()); - } - - EmbeddingProfileRef profile = embeddingProfiles - .find( - actor.organizationId(), - new EmbeddingProfileSpec( - embedding.provider(), - embedding.model(), - embedding.dimensions(), - EmbeddingDistanceMetric.COSINE)) - .orElseThrow(() -> searchAuthorization.unavailable( - actor, - requestId, - query, - "EMBEDDING_PROFILE_NOT_INDEXED", - initial.authorizationModelId())); - - LightRagQueryRequest.Options queryOptions = - policy.contextOptions(limit); - List spaceGroundings = - queryPublishedSpaces( - initial, - profile, - query, - queryOptions, - operationId); - if (spaceGroundings.isEmpty()) { - audit.record(searchAuthorization.command( - actor, - requestId, - query, - PermissionAuditDecision.ALLOW, - "NO_ELIGIBLE_EVIDENCE", - initial.authorizationModelId())); - return new SecureKnowledgeSearchResult(requestId, List.of()); - } - long consolidationStartedAt = System.nanoTime(); - LightRagGroundingAssembler.PreparedGrounding consolidated = - engine.consolidateGrounding( - query, - queryOptions, - spaceGroundings); - emitAssembledContext( - operationId, - actor.organizationId(), - consolidationStartedAt, - spaceGroundings.size(), - consolidated, - queryOptions.contextBudget()); - if (consolidated.grounding().empty() - || consolidated.grounding().chunks().isEmpty()) { - audit.record(searchAuthorization.command( - actor, - requestId, - query, - PermissionAuditDecision.ALLOW, - "NO_CITABLE_GROUNDING", - initial.authorizationModelId())); - return new SecureKnowledgeSearchResult(requestId, List.of()); - } - - ResolvedKnowledgeEvidenceScope current = - resolve( - actor, - authorizationModelId, - requestId, - query, - operationId); - if (!sameAuthorizationScope(initial, current)) { - return retryOrFail( - actor, - query, - limit, - requestId, - authorizationModelId, - operationId, - attempt, - "AUTHORIZATION_SCOPE_CHANGED"); - } - - List closure = - consolidated.grounding().evidenceClosure(); - if (closure.size() > policy.maximumEvidenceClosure()) { - throw searchAuthorization.unavailable( - actor, - requestId, - query, - "GROUNDING_EVIDENCE_CLOSURE_EXCEEDED", - current.authorizationModelId()); - } - long finalAuthorizationStartedAt = System.nanoTime(); - verifyOpenFga( - actor, - query, - requestId, - current.authorizationModelId(), - closure); - emitStage( - operationId, - actor.organizationId(), - GraphRagEventSink.Stage.AUTHORIZE, - finalAuthorizationStartedAt, - closure.size(), - closure.size()); - List verified = - recheckCanonical(current, closure); - if (!sameEvidence(closure, verified)) { - return retryOrFail( - actor, - query, - limit, - requestId, - authorizationModelId, - operationId, - attempt, - "CANONICAL_EVIDENCE_CHANGED"); - } - LightRagGroundingAssembler.PreparedGrounding rendered = - engine.renderGrounding( - query, - queryOptions, - consolidated.grounding()); - - Map canonicalByChunk = verified.stream() - .collect(Collectors.toMap( - SecureRetrievalCandidate::chunkId, - Function.identity())); - Map scoreByChunk = consolidated.grounding() - .chunks() - .stream() - .collect(Collectors.toMap( - LightRagGrounding.SelectedChunk::id, - LightRagGrounding.SelectedChunk::effectiveScore, - Math::max, - LinkedHashMap::new)); - List evidence = rendered.references() - .stream() - .map(reference -> toEvidence( - Objects.requireNonNull( - canonicalByChunk.get( - reference.evidence().chunkId()), - "verified citation evidence"), - scoreByChunk.getOrDefault( - reference.evidence().chunkId(), - 0.0))) - .toList(); - List auditCommands = new ArrayList<>(); - auditCommands.add(searchAuthorization.command( - actor, - requestId, - query, - PermissionAuditDecision.ALLOW, - "SECURE_GRAPH_RAG_RETRIEVAL_APPLIED", - current.authorizationModelId())); - for (LightRagGrounding.GroundingEvidence groundingEvidence : closure) { - SecureRetrievalCandidate canonical = canonicalByChunk.get( - groundingEvidence.evidence().chunkId()); - auditCommands.add(evidenceAudit( - actor, - requestId, - query, - canonical, - current.authorizationModelId(), - "VERIFIED_GRAPH_RAG_GROUNDING")); - } - audit.recordAll(auditCommands); - return new SecureKnowledgeSearchResult( - requestId, - evidence, - Optional.of(new VerifiedKnowledgeGrounding( - new ChatGenerationRequest( - rendered.systemPrompt(), - query), - evidence, - closure.size(), - rendered.inputTokens()))); - } - - private List queryPublishedSpaces( - ResolvedKnowledgeEvidenceScope scope, - EmbeddingProfileRef profile, - String query, - LightRagQueryRequest.Options options, - UUID operationId) { - if (scope.knowledgeSpaceIds().size() - > policy.maximumKnowledgeSpaces()) { - throw new KnowledgeRetrievalUnavailableException( - "Secure knowledge retrieval is temporarily unavailable"); - } - List requests = new ArrayList<>(); - for (UUID knowledgeSpaceId : - scope.knowledgeSpaceIds().stream().sorted().toList()) { - var evidenceScope = scope.forKnowledgeSpace(knowledgeSpaceId); - ProjectionNamespace namespace = namespace( - scope.organizationId(), - knowledgeSpaceId); - var snapshot = publications.current(namespace); - if (snapshot.isEmpty()) { - continue; - } - requests.add(new LightRagQueryRequest( - evidenceScope, - snapshot.orElseThrow(), - query, - options, - profile.id(), - profile.dimensions(), - null, - List.of())); - } - if (requests.isEmpty()) { - return List.of(); - } - if (requests.size() > 1 && policy.rerank().enabled()) { - throw new KnowledgeRetrievalUnavailableException( - "Secure knowledge retrieval is temporarily unavailable"); - } - - LightRagPreparedQuery prepared = engine.prepare(requests.getFirst()); - emitPreparedStage( - operationId, - scope.organizationId(), - GraphRagEventSink.Stage.PREPARE_QUERY, - prepared.keywordPlanningDuration(), - 1, - prepared.keywords().highLevel().size() - + prepared.keywords().lowLevel().size(), - prepared.keywordModelRouteFingerprint(), - prepared.keywordCacheStatus()); - emitPreparedStage( - operationId, - scope.organizationId(), - GraphRagEventSink.Stage.EMBED, - prepared.embeddingDuration(), - prepared.embeddingInputs().size(), - prepared.embeddingInputs().size(), - null, - null); - List groundings = new ArrayList<>(); - try (ExecutorService executor = - Executors.newVirtualThreadPerTaskExecutor()) { - for (int offset = 0; - offset < requests.size(); - offset += policy.maximumConcurrentSpaces()) { - int end = Math.min( - requests.size(), - offset + policy.maximumConcurrentSpaces()); - List> futures = - requests.subList(offset, end) - .stream() - .map(request -> executor.submit(tasks.decorate(() -> - queryPublishedSpace(request, prepared)))) - .toList(); - try { - for (Future future : futures) { - SnapshotQueryResult snapshotResult = future.get(); - emitSnapshotStage( - operationId, - scope.organizationId(), - snapshotResult.duration(), - snapshotResult.inputCount(), - snapshotResult.result() - .grounding() - .chunks() - .size(), - snapshotResult.namespace()); - emitRerank( - operationId, - scope.organizationId(), - snapshotResult.result()); - LightRagGrounding grounding = - snapshotResult.result().grounding(); - if (!grounding.empty()) { - groundings.add(grounding); - } - } - } catch (ExecutionException | InterruptedException - | RuntimeException failure) { - futures.forEach(future -> future.cancel(true)); - throw retrievalFailure(failure); - } - } - } - return List.copyOf(groundings); - } - - private SnapshotQueryResult queryPublishedSpace( - LightRagQueryRequest request, - LightRagPreparedQuery prepared) { - long startedAt = System.nanoTime(); - LightRagQueryResult result = - engine.executePrepared(request, prepared); - return new SnapshotQueryResult( - result, - Duration.ofNanos(Math.max( - 0, - System.nanoTime() - startedAt)), - request.scope().authorizedAssetIds().size(), - request.snapshot().namespace()); - } - - private static RuntimeException retrievalFailure(Exception failure) { - if (failure instanceof InterruptedException) { - Thread.currentThread().interrupt(); - return new KnowledgeRetrievalUnavailableException( - "Secure knowledge retrieval is temporarily unavailable"); - } - Throwable cause = failure instanceof ExecutionException - ? failure.getCause() - : failure; - if (cause instanceof RuntimeException runtime) { - return runtime; - } - if (cause instanceof Error error) { - throw error; - } - return new KnowledgeRetrievalUnavailableException( - "Secure knowledge retrieval is temporarily unavailable"); - } - - private ResolvedKnowledgeEvidenceScope resolve( - CurrentActor actor, - String authorizationModelId, - String requestId, - String query, - UUID operationId) { - long startedAt = System.nanoTime(); - try { - ResolvedKnowledgeEvidenceScope resolved = - evidenceScopes.resolve(actor, authorizationModelId); - emitStage( - operationId, - actor.organizationId(), - GraphRagEventSink.Stage.AUTHORIZE, - startedAt, - 1, - resolved.allAssetIds().size()); - return resolved; - } catch (KnowledgeEvidenceScopeUnavailableException unavailable) { - throw searchAuthorization.unavailable( - actor, - requestId, - query, - unavailable.reasonCode(), - unavailable.policyVersion()); - } - } - - private void emitStage( - UUID operationId, - UUID organizationId, - GraphRagEventSink.Stage stage, - long startedAt, - int inputCount, - int outputCount) { - try { - events.emit(new GraphRagEventSink.GraphRagEvent( - operationId, - organizationId, - stage, - GraphRagEventSink.Outcome.SUCCEEDED, - Duration.ofNanos(Math.max( - 0, - System.nanoTime() - startedAt)), - inputCount, - outputCount, - null, - null, - null, - null, - Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } - } - - /** - * Context assembly is the one retrieval stage whose cost is measured in - * tokens rather than items, and the only stage that can report how much of - * the retrieved context the budget refused to carry. Both numbers were being - * computed and discarded. - */ - private void emitAssembledContext( - UUID operationId, - UUID organizationId, - long startedAt, - int inputCount, - LightRagGroundingAssembler.PreparedGrounding prepared, - SecureContextBudget budget) { - LightRagGrounding grounding = prepared.grounding(); - ContextTokenUsage usage = grounding.tokenUsage(); - try { - events.emit(new GraphRagEventSink.GraphRagEvent( - operationId, - organizationId, - GraphRagEventSink.Stage.ASSEMBLE_CONTEXT, - GraphRagEventSink.Outcome.SUCCEEDED, - Duration.ofNanos(Math.max( - 0, - System.nanoTime() - startedAt)), - inputCount, - grounding.chunks().size(), - null, - null, - null, - null, - new GraphRagEventSink.TokenUsage( - prepared.inputTokens(), - usage.systemPromptTokens(), - usage.queryTokens(), - usage.entityTokens(), - usage.relationTokens(), - grounding.chunkTokens(), - budget.maximumInputTokens(), - prepared.droppedContributions()), - null, - Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } - } - - private void emitPreparedStage( - UUID operationId, - UUID organizationId, - GraphRagEventSink.Stage stage, - Duration duration, - int inputCount, - int outputCount, - String modelRouteFingerprint, - GraphRagEventSink.CacheStatus cacheStatus) { - try { - events.emit(new GraphRagEventSink.GraphRagEvent( - operationId, - organizationId, - stage, - GraphRagEventSink.Outcome.SUCCEEDED, - duration, - inputCount, - outputCount, - modelRouteFingerprint, - null, - cacheStatus, - null, - Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } - } - - private void emitSnapshotStage( - UUID operationId, - UUID organizationId, - Duration duration, - int inputCount, - int outputCount, - ProjectionNamespace namespace) { - String scopeFingerprint = CanonicalCacheKeyHasher.sha256( - "orgmemory.graph-rag.snapshot-scope.v1", - Map.of( - "organizationId", - namespace.organizationId().toString(), - "workspace", - namespace.workspace(), - "collection", - namespace.collection())); - try { - events.emit(new GraphRagEventSink.GraphRagEvent( - operationId, - organizationId, - GraphRagEventSink.Stage.RETRIEVE_SNAPSHOT, - GraphRagEventSink.Outcome.SUCCEEDED, - duration, - inputCount, - outputCount, - null, - scopeFingerprint, - null, - null, - Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } - } - - private record SnapshotQueryResult( - LightRagQueryResult result, - Duration duration, - int inputCount, - ProjectionNamespace namespace) { - - private SnapshotQueryResult { - Objects.requireNonNull(result, "result"); - Objects.requireNonNull(duration, "duration"); - if (duration.isNegative() || inputCount < 0) { - throw new IllegalArgumentException( - "snapshot query metrics must be non-negative"); - } - Objects.requireNonNull(namespace, "namespace"); - } - } - - private void verifyOpenFga( - CurrentActor actor, - String query, - String requestId, - String authorizationModelId, - List closure) { - List resources = closure.stream() - .map(candidate -> ResourceRef.of( - actor.organizationId(), - RESOURCE_TYPE, - candidate.evidence() - .knowledgeAssetId())) - .distinct() - .toList(); - var rechecked = batchRecheck.recheck( - new BatchAuthorizationQuery( - actor.organizationId(), - actor.principal(), - CAN_VIEW, - resources), - authorizationModelId, - OpenFgaBatchRecheck.ResultPolicy.REQUIRE_ALL_ALLOWED, - RECHECK_REASONS); - if (!rechecked.succeeded()) { - var failure = rechecked.failure(); - String failurePolicyVersion = switch (failure.kind()) { - case MISSING_DECISION, - DECISION_POLICY_MISMATCH, - DENIED -> authorizationModelId; - case UNRESOLVED, - DECISION_COUNT_MISMATCH, - OUTER_POLICY_MISMATCH -> failure.policyVersion(); - }; - throw searchAuthorization.unavailable( - actor, - requestId, - query, - failure.reasonCode(), - failurePolicyVersion); - } - } - - private List recheckCanonical( - ResolvedKnowledgeEvidenceScope scope, - List closure) { - return canonicalEvidence.recheck( - scope.toRetrievalScope(), - closure.stream() - .map(candidate -> candidate.evidence().chunkId()) - .toList()); - } - - private SecureKnowledgeSearchResult retryOrFail( - CurrentActor actor, - String query, - int limit, - String requestId, - String authorizationModelId, - UUID operationId, - int attempt, - String reason) { - if (attempt == 0) { - return search( - actor, - query, - limit, - requestId, - authorizationModelId, - operationId, - 1); - } - throw searchAuthorization.unavailable( - actor, - requestId, - query, - reason, - authorizationModelId); - } - - private void emitRerank( - UUID operationId, - UUID organizationId, - LightRagQueryResult result) { - if (!result.trace().rerankAttempted()) { - return; - } - GraphRagEventSink.Outcome outcome = result.trace().rerankFallback() - ? GraphRagEventSink.Outcome.FAILED - : GraphRagEventSink.Outcome.SUCCEEDED; - String failureCode = result.trace().rerankFallback() - ? "rerank_provider_fallback" - : null; - String routeFingerprint = CanonicalCacheKeyHasher.sha256( - "reranker-route", - Map.of("provider", policy.rerank().provider())); - try { - events.emit(new GraphRagEventSink.GraphRagEvent( - operationId, - organizationId, - GraphRagEventSink.Stage.RERANK, - outcome, - result.trace().rerankDuration(), - result.trace().chunkSignals().size(), - result.grounding().chunks().size(), - routeFingerprint, - null, - null, - failureCode, - Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } - } - - private static boolean sameAuthorizationScope( - ResolvedKnowledgeEvidenceScope left, - ResolvedKnowledgeEvidenceScope right) { - if (!left.authorizationModelId() - .equals(right.authorizationModelId())) { - return false; - } - Set spaces = new LinkedHashSet<>( - left.knowledgeSpaceIds()); - spaces.addAll(right.knowledgeSpaceIds()); - return spaces.stream().allMatch(space -> left - .forKnowledgeSpace(space) - .authorizationFingerprint() - .equals(right.forKnowledgeSpace(space) - .authorizationFingerprint())); - } - - private static boolean sameEvidence( - List closure, - List verified) { - Map byChunk = verified.stream() - .collect(Collectors.toMap( - SecureRetrievalCandidate::chunkId, - Function.identity(), - (left, right) -> left, - LinkedHashMap::new)); - if (byChunk.size() != closure.size()) { - return false; - } - return closure.stream().allMatch(candidate -> { - EvidenceReference reference = - candidate.evidence(); - SecureRetrievalCandidate canonical = - byChunk.get(reference.chunkId()); - return canonical != null - && reference.organizationId() - .equals(canonical.organizationId()) - && reference.knowledgeAssetId() - .equals(canonical.knowledgeAssetId()) - && reference.sourceRevisionId() - .equals(canonical.sourceRevisionId()) - && reference.aclSnapshotId() - .equals(canonical.ingestionAclSnapshotId()) - && candidate.projectionGeneration() - == canonical.projectionGeneration(); - }); - } - - private static RetrievedKnowledgeEvidence toEvidence( - SecureRetrievalCandidate candidate, - double score) { - return new RetrievedKnowledgeEvidence( - candidate.chunkId(), - candidate.knowledgeAssetId(), - candidate.sourceObjectId(), - candidate.sourceRevisionId(), - candidate.title(), - candidate.content(), - SourceCitationUri.safeForOutput(candidate.sourceUri()), - candidate.startPage(), - candidate.endPage(), - candidate.heading(), - 0.0, - score, - score, - candidate.ingestionAclSnapshotId(), - candidate.currentAclSnapshotId(), - candidate.authorizationModelId(), - candidate.embeddingProfileId(), - candidate.projectionGeneration()); - } - - private static PermissionAuditCommand evidenceAudit( - CurrentActor actor, - String requestId, - String query, - SecureRetrievalCandidate candidate, - String authorizationModelId, - String reason) { - return new PermissionAuditCommand( - actor.organizationId(), - actor.userId(), - "SEARCH", - "KNOWLEDGE_EVIDENCE", - candidate.chunkId().toString(), - PermissionAuditDecision.ALLOW, - reason, - authorizationModelId, - requestId, - query, - candidate.ingestionAclSnapshotId(), - candidate.currentAclSnapshotId(), - candidate.authorizationModelId(), - candidate.sourceRevisionId(), - candidate.chunkId(), - candidate.embeddingProfileId(), - candidate.projectionGeneration()); - } - - private String normalizeQuery(String query) { - if (query == null || query.isBlank()) { - throw new BusinessValidationException( - "knowledge-search.query-required", - "q is required"); - } - String normalized = query.strip(); - if (normalized.length() - > retrievalProperties.maximumQueryLength()) { - throw new BusinessValidationException( - "knowledge-search.query-invalid", - "q must not exceed " - + retrievalProperties.maximumQueryLength() - + " characters"); - } - return normalized; - } - - private int validateLimit(Integer requestedLimit) { - int limit = requestedLimit == null - ? Math.min(10, retrievalProperties.maximumResults()) - : requestedLimit; - if (limit < 1 - || limit > retrievalProperties.maximumResults()) { - throw new BusinessValidationException( - "knowledge-search.limit-invalid", - "limit must be between 1 and " - + retrievalProperties.maximumResults()); - } - return limit; - } - - private static String requestId(String requestId) { - if (requestId == null || requestId.isBlank()) { - return UUID.randomUUID().toString(); - } - String normalized = requestId.strip(); - return normalized.length() <= MAX_REQUEST_ID_LENGTH - ? normalized - : UUID.randomUUID().toString(); - } - - private static ProjectionNamespace namespace( - UUID organizationId, - UUID knowledgeSpaceId) { - return KnowledgeProjectionNamespaces.forSpace(organizationId, knowledgeSpaceId); - } +/** Adapter-facing GraphRAG retrieval engine, present only when its runtime is configured. */ +public interface GraphRagKnowledgeRetrievalService extends PermissionAwareKnowledgeSearch { } diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/JdbcEmbeddingProfileRegistry.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/JdbcEmbeddingProfileRegistry.java new file mode 100644 index 000000000..3c5fb68a8 --- /dev/null +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/JdbcEmbeddingProfileRegistry.java @@ -0,0 +1,89 @@ +package com.orgmemory.core.knowledge.retrieval; + +import java.nio.charset.StandardCharsets; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.Optional; +import java.util.UUID; +import org.springframework.jdbc.core.simple.JdbcClient; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +@Service +class JdbcEmbeddingProfileRegistry implements EmbeddingProfileRegistry { + + private final EmbeddingProfileRepository profiles; + private final JdbcClient jdbc; + + JdbcEmbeddingProfileRegistry(EmbeddingProfileRepository profiles, JdbcClient jdbc) { + this.profiles = profiles; + this.jdbc = jdbc; + } + + @Transactional + @Override + public EmbeddingProfileRef resolve(UUID organizationId, EmbeddingProfileSpec spec) { + if (organizationId == null || spec == null) { + throw new IllegalArgumentException("organization and embedding profile are required"); + } + String profileKey = spec.profileKey(); + if (profileKey.length() > 255) { + throw new IllegalArgumentException("embedding profile key is too long"); + } + UUID id = UUID.nameUUIDFromBytes( + (organizationId + ":" + profileKey).getBytes(StandardCharsets.UTF_8)); + jdbc.sql(""" + INSERT INTO embedding_profiles ( + id, organization_id, profile_key, provider, model, + dimensions, distance_metric, created_at + ) VALUES ( + :id, :organizationId, :profileKey, :provider, :model, + :dimensions, :distanceMetric, :createdAt + ) + ON CONFLICT (organization_id, profile_key) DO NOTHING + """) + .param("id", id) + .param("organizationId", organizationId) + .param("profileKey", profileKey) + .param("provider", spec.provider()) + .param("model", spec.model()) + .param("dimensions", spec.dimensions()) + .param("distanceMetric", spec.distanceMetric().name()) + .param("createdAt", OffsetDateTime.now(ZoneOffset.UTC)) + .update(); + EmbeddingProfileRef resolved = profiles.findByOrganizationIdAndProfileKey(organizationId, profileKey) + .orElseThrow(() -> new IllegalStateException("embedding profile registration failed")) + .toRef(); + if (!resolved.provider().equals(spec.provider()) + || !resolved.model().equals(spec.model()) + || resolved.dimensions() != spec.dimensions() + || resolved.distanceMetric() != spec.distanceMetric()) { + throw new IllegalStateException("embedding profile key is already bound to different settings"); + } + return resolved; + } + + @Transactional(readOnly = true) + @Override + public EmbeddingProfileRef get(UUID organizationId, UUID profileId) { + return findById(organizationId, profileId) + .orElseThrow(() -> new IllegalStateException("embedding profile was not found")); + } + + @Transactional(readOnly = true) + @Override + public Optional findById(UUID organizationId, UUID profileId) { + return profiles.findByIdAndOrganizationId(profileId, organizationId) + .map(EmbeddingProfile::toRef); + } + + @Transactional(readOnly = true) + @Override + public Optional find(UUID organizationId, EmbeddingProfileSpec spec) { + if (organizationId == null || spec == null) { + throw new IllegalArgumentException("organization and embedding profile are required"); + } + return profiles.findByOrganizationIdAndProfileKey(organizationId, spec.profileKey()) + .map(EmbeddingProfile::toRef); + } +} diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeAssetAccessInspector.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeAssetAccessInspector.java new file mode 100644 index 000000000..58c8d4b29 --- /dev/null +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeAssetAccessInspector.java @@ -0,0 +1,19 @@ +package com.orgmemory.core.knowledge.retrieval; + +import com.orgmemory.core.authorization.AccessState; +import com.orgmemory.core.organization.CurrentActor; +import java.time.Instant; +import java.util.UUID; + +/** Bounded adapter query for one already relationship-authorized Knowledge Asset. */ +public interface KnowledgeAssetAccessInspector { + + AssetInspection inspectAsset( + CurrentActor actor, + UUID assetId, + String authorizationModelId, + Instant evaluatedAt); + + record AssetInspection(AccessState state, String reasonCode) { + } +} diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolver.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolver.java index 8bee2d506..cf0d981be 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolver.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolver.java @@ -32,7 +32,7 @@ * permission-aware retrieval, graph and citation use cases. */ @Service -public class KnowledgeEvidenceScopeResolver { +class KnowledgeEvidenceScopeResolver implements KnowledgeAssetAccessInspector { private static final PermissionKey CAN_VIEW = PermissionKey.of("can_view"); private static final String RESOURCE_TYPE = "knowledge_asset"; @@ -173,6 +173,7 @@ public ResolvedKnowledgeEvidenceScope resolve( * relationship-authorized asset inspected by an audit viewer. */ @Transactional(readOnly = true) + @Override public AssetInspection inspectAsset( CurrentActor actor, UUID assetId, @@ -264,6 +265,4 @@ private static KnowledgeEvidenceScopeUnavailableException unavailable( policyVersion); } - public record AssetInspection(AccessState state, String reasonCode) { - } } diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/SourceContentService.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/SourceContentService.java index 2ed83a673..574efc4de 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/SourceContentService.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/SourceContentService.java @@ -1,135 +1,10 @@ package com.orgmemory.core.knowledge.retrieval; -import com.orgmemory.core.knowledge.sourceledger.SourceDocumentEvidenceQuery; -import com.orgmemory.core.knowledge.storage.ObjectContent; -import com.orgmemory.core.knowledge.storage.ObjectStoragePort; import com.orgmemory.core.organization.CurrentActor; -import com.orgmemory.core.permission.PermissionAuditCommand; -import com.orgmemory.core.permission.PermissionAuditDecision; -import com.orgmemory.core.permission.PermissionAuditService; -import com.orgmemory.core.shared.error.KnowledgeResourceNotFoundException; -import java.util.Objects; import java.util.UUID; -import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -/** Opens a current document through the canonical evidence authorization scope. */ -@Service -public class SourceContentService { +/** Opens one current source through canonical authorization without exposing storage keys. */ +public interface SourceContentService { - private final KnowledgeEvidenceScopeResolver authorization; - private final SourceDocumentEvidenceQuery evidenceQuery; - private final ObjectStoragePort objects; - private final PermissionAuditService audit; - - SourceContentService( - KnowledgeEvidenceScopeResolver authorization, - SourceDocumentEvidenceQuery evidenceQuery, - ObjectStoragePort objects, - PermissionAuditService audit) { - this.authorization = authorization; - this.evidenceQuery = evidenceQuery; - this.objects = objects; - this.audit = audit; - } - - @Transactional(readOnly = true) - public SourceContent open(CurrentActor actor, UUID sourceId, String requestId) { - Objects.requireNonNull(actor, "actor"); - Objects.requireNonNull(sourceId, "sourceId"); - String normalizedRequestId = requestId == null || requestId.isBlank() - ? UUID.randomUUID().toString() - : requestId.strip(); - var scope = authorization.resolve(actor, null); - var document = evidenceQuery - .findAvailable(actor.organizationId(), sourceId) - .orElseThrow(() -> notFound( - actor, - sourceId, - normalizedRequestId, - scope.authorizationModelId(), - "SOURCE_NOT_CURRENT")); - if (!scope.allAssetIds().contains(document.knowledgeAssetId())) { - throw notFound( - actor, - sourceId, - normalizedRequestId, - scope.authorizationModelId(), - "SOURCE_NOT_AUTHORIZED"); - } - var evidence = document.evidence(); - ObjectContent content = objects.open(evidence.objectKey()); - if (!evidence.storedContentSha256().equals(evidence.contentSha256()) - || evidence.storedContentLength() != evidence.contentLength() - || !evidence.storedContentSha256().equals(content.metadata().sha256()) - || evidence.storedContentLength() != content.metadata().contentLength()) { - closeQuietly(content); - audit.record(new PermissionAuditCommand( - actor.organizationId(), - actor.userId(), - "READ_SOURCE_CONTENT", - "SOURCE_OBJECT", - sourceId.toString(), - PermissionAuditDecision.DENY, - "SOURCE_BLOB_INTEGRITY_FAILED", - scope.authorizationModelId(), - normalizedRequestId, - null)); - throw new KnowledgeRetrievalUnavailableException( - "Source evidence failed its integrity check"); - } - audit.record(new PermissionAuditCommand( - actor.organizationId(), - actor.userId(), - "READ_SOURCE_CONTENT", - "SOURCE_OBJECT", - sourceId.toString(), - PermissionAuditDecision.ALLOW, - "AUTHORIZED_SOURCE_CONTENT", - scope.authorizationModelId(), - normalizedRequestId, - null, - null, - null, - scope.authorizationModelId(), - document.sourceRevisionId(), - null, - document.embeddingProfileId(), - null)); - return new SourceContent( - sourceId, - evidence.fileName(), - evidence.mediaType(), - evidence.contentLength(), - evidence.contentSha256(), - content); - } - - private KnowledgeResourceNotFoundException notFound( - CurrentActor actor, - UUID sourceId, - String requestId, - String policyVersion, - String reason) { - audit.record(new PermissionAuditCommand( - actor.organizationId(), - actor.userId(), - "READ_SOURCE_CONTENT", - "SOURCE_OBJECT", - sourceId.toString(), - PermissionAuditDecision.DENY, - reason, - policyVersion, - requestId, - null)); - return new KnowledgeResourceNotFoundException(); - } - - private static void closeQuietly(ObjectContent content) { - try { - content.close(); - } catch (java.io.IOException ignored) { - // Integrity failure is authoritative. - } - } + SourceContent open(CurrentActor actor, UUID sourceId, String requestId); } diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/package-info.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/package-info.java index 94f81cd40..4b01db452 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/package-info.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/package-info.java @@ -8,10 +8,13 @@ * consumed here one way. Top-level search consumers cross the parent-owned * {@code knowledge::search} interface instead of this implementation package. Graph exploration, * export, and curation consume a Retrieval-owned canonical evidence verifier and immutable verified - * snapshot instead of scope resolution or retrieval-store implementation types. The module remains - * open while its remaining sibling-module adapters are replaced by intentional interfaces during - * the Knowledge module-closing phase. Asset, Organization, and Source Ledger citation reads already - * cross owner-defined queries. + * snapshot instead of scope resolution or retrieval-store implementation types. API and Worker + * adapters inject Retrieval interfaces for canonical/GraphRAG search, citation/source opening, + * authorization inspection, and embedding-profile resolution; their default/JDBC implementations + * are package-private. Full evidence-scope resolution remains internal. The module remains open + * only while the rest of its concrete + * and persistence root types are internalized and its final allowlist is proven. Asset, + * Organization, and Source Ledger citation reads already cross owner-defined queries. */ @org.springframework.modulith.ApplicationModule( type = org.springframework.modulith.ApplicationModule.Type.OPEN) diff --git a/core/src/test/java/com/orgmemory/core/ModulithVerificationTests.java b/core/src/test/java/com/orgmemory/core/ModulithVerificationTests.java index 910107f5e..07504d75a 100644 --- a/core/src/test/java/com/orgmemory/core/ModulithVerificationTests.java +++ b/core/src/test/java/com/orgmemory/core/ModulithVerificationTests.java @@ -7,9 +7,17 @@ import com.orgmemory.core.knowledge.catalog.KnowledgeCatalogEntry; import com.orgmemory.core.knowledge.catalog.KnowledgeCatalogQuery; +import com.orgmemory.core.knowledge.retrieval.AuthorizationResourceDirectory; +import com.orgmemory.core.knowledge.retrieval.CanonicalHybridKnowledgeSearch; +import com.orgmemory.core.knowledge.retrieval.CitationContentService; +import com.orgmemory.core.knowledge.retrieval.EmbeddingProfileRegistry; +import com.orgmemory.core.knowledge.retrieval.GraphRagKnowledgeRetrievalService; +import com.orgmemory.core.knowledge.retrieval.KnowledgeAssetAccessInspector; +import com.orgmemory.core.knowledge.retrieval.SourceContentService; import com.orgmemory.core.knowledge.storage.ObjectStoragePort; import com.tngtech.archunit.core.importer.ClassFileImporter; import com.tngtech.archunit.core.importer.ImportOption; +import java.lang.reflect.Modifier; import java.util.Set; import java.util.TreeSet; import org.junit.jupiter.api.Test; @@ -25,6 +33,28 @@ void modulesAreWellFormed() { modules.verify(); } + @Test + void retrievalAdapterContractsAreInterfaces() throws ClassNotFoundException { + assertTrue(AuthorizationResourceDirectory.class.isInterface()); + assertTrue(CanonicalHybridKnowledgeSearch.class.isInterface()); + assertTrue(CitationContentService.class.isInterface()); + assertTrue(EmbeddingProfileRegistry.class.isInterface()); + assertTrue(GraphRagKnowledgeRetrievalService.class.isInterface()); + assertTrue(KnowledgeAssetAccessInspector.class.isInterface()); + assertTrue(SourceContentService.class.isInterface()); + + for (String implementation : Set.of( + "com.orgmemory.core.knowledge.retrieval.DefaultAuthorizationResourceDirectory", + "com.orgmemory.core.knowledge.retrieval.DefaultCanonicalHybridKnowledgeSearch", + "com.orgmemory.core.knowledge.retrieval.DefaultCitationContentService", + "com.orgmemory.core.knowledge.retrieval.DefaultGraphRagKnowledgeRetrievalService", + "com.orgmemory.core.knowledge.retrieval.KnowledgeEvidenceScopeResolver", + "com.orgmemory.core.knowledge.retrieval.DefaultSourceContentService", + "com.orgmemory.core.knowledge.retrieval.JdbcEmbeddingProfileRegistry")) { + assertFalse(Modifier.isPublic(Class.forName(implementation).getModifiers())); + } + } + @Test void knowledgeRootPackageContainsNoDomainTypes() { var rootTypes = new ClassFileImporter() @@ -767,7 +797,7 @@ void knowledgeAssetConsumerSurfaceDoesNotGainNewTypes() { assertEquals( Set.of( - "com.orgmemory.core.knowledge.retrieval.AuthorizationResourceDirectory", + "com.orgmemory.core.knowledge.retrieval.DefaultAuthorizationResourceDirectory", "com.orgmemory.core.knowledge.graph.GraphIndexingCoordinator", "com.orgmemory.core.knowledge.graph.GraphIndexJobQueue", "com.orgmemory.core.knowledge.graph.GraphIndexLifecycleService", @@ -775,7 +805,7 @@ void knowledgeAssetConsumerSurfaceDoesNotGainNewTypes() { "com.orgmemory.core.knowledge.graph.KnowledgeGraphExportService", "com.orgmemory.core.knowledge.retrieval.KnowledgeCatalogService", "com.orgmemory.core.knowledge.retrieval.KnowledgeEvidenceScopeResolver", - "com.orgmemory.core.knowledge.retrieval.GraphRagKnowledgeRetrievalService", + "com.orgmemory.core.knowledge.retrieval.DefaultGraphRagKnowledgeRetrievalService", "com.orgmemory.core.knowledge.retrieval.SecureKnowledgeRetrievalStore", "com.orgmemory.core.knowledge.graph.KnowledgeGraphCurationService", "com.orgmemory.core.knowledge.connector.ConnectorReconciler", @@ -832,7 +862,7 @@ void retrievalAssetReadsUseOnlyTheOwnerQuery() { assertEquals( Set.of( - "com.orgmemory.core.knowledge.retrieval.AuthorizationResourceDirectory", + "com.orgmemory.core.knowledge.retrieval.DefaultAuthorizationResourceDirectory", "com.orgmemory.core.knowledge.retrieval.KnowledgeCatalogService", "com.orgmemory.core.knowledge.retrieval.KnowledgeEvidenceScopeResolver"), consumers); @@ -882,7 +912,7 @@ void retrievalOrganizationReadsUseOnlyOwnerQueries() { assertEquals( Set.of( - "com.orgmemory.core.knowledge.retrieval.AuthorizationResourceDirectory", + "com.orgmemory.core.knowledge.retrieval.DefaultAuthorizationResourceDirectory", "com.orgmemory.core.knowledge.retrieval.KnowledgeEvidenceScopeResolver", "com.orgmemory.core.knowledge.retrieval.SecureSourceVisibilityAdapter"), consumers); @@ -924,7 +954,7 @@ void citationContentUsesTheSourceLedgerOwnerQuery() { .collect(TreeSet::new, Set::add, Set::addAll); assertEquals( - Set.of("com.orgmemory.core.knowledge.retrieval.CitationContentService"), + Set.of("com.orgmemory.core.knowledge.retrieval.DefaultCitationContentService"), consumers); } diff --git a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchTests.java b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchTests.java index 169bfdec3..a6b3dccc3 100644 --- a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchTests.java +++ b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchTests.java @@ -73,7 +73,7 @@ void setUp() { Instant.parse("2026-07-24T00:00:00Z"), Map.of(SPACE_ID, Set.of(assetId)), Map.of(SPACE_ID, 1L))); - service = new CanonicalHybridKnowledgeSearch( + service = new DefaultCanonicalHybridKnowledgeSearch( store, evidenceScopes, new KnowledgeSearchAuthorizationService( @@ -336,7 +336,7 @@ private ResourceRef resource(UUID knowledgeAssetId) { } private CanonicalHybridKnowledgeSearch freshService() { - return new CanonicalHybridKnowledgeSearch( + return new DefaultCanonicalHybridKnowledgeSearch( store, evidenceScopes, new KnowledgeSearchAuthorizationService( diff --git a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CitationContentServiceTests.java b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CitationContentServiceTests.java index ecbe2ecb7..9f41c49a8 100644 --- a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CitationContentServiceTests.java +++ b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CitationContentServiceTests.java @@ -177,7 +177,7 @@ private static final class Fixture { private final PermissionAuditService audit = mock(PermissionAuditService.class); private final CitationContentService service = - new CitationContentService( + new DefaultCitationContentService( authorization, evidenceQuery, objects, diff --git a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/EmbeddingProfileRegistryTests.java b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/EmbeddingProfileRegistryTests.java index 4f4c8a54f..08ca70708 100644 --- a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/EmbeddingProfileRegistryTests.java +++ b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/EmbeddingProfileRegistryTests.java @@ -17,7 +17,7 @@ class EmbeddingProfileRegistryTests { private final EmbeddingProfileRepository profiles = mock(EmbeddingProfileRepository.class); private final EmbeddingProfileRegistry registry = - new EmbeddingProfileRegistry(profiles, mock(JdbcClient.class)); + new JdbcEmbeddingProfileRegistry(profiles, mock(JdbcClient.class)); @Test void findsAnImmutableProfileByTenantAndId() { diff --git a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalServiceTests.java b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalServiceTests.java index ae6d71e7f..2bafa5cba 100644 --- a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalServiceTests.java +++ b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalServiceTests.java @@ -318,7 +318,7 @@ void revocationBetweenRetrievalAndCitationCausesAFullRetryWithoutEgress() { new NeverRecheckedStore(); GraphRagEventSink events = mock(GraphRagEventSink.class); - var service = new GraphRagKnowledgeRetrievalService( + var service = new DefaultGraphRagKnowledgeRetrievalService( new KnowledgeSearchAuthorizationService(entry, audit), scopes, finalAuthorization, @@ -941,7 +941,7 @@ private static GraphRagKnowledgeRetrievalService service( "text-embedding-3-large", 1536, EmbeddingDistanceMetric.COSINE))); - return new GraphRagKnowledgeRetrievalService( + return new DefaultGraphRagKnowledgeRetrievalService( new KnowledgeSearchAuthorizationService(entry, audit), scopes, finalAuthorization, diff --git a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/SourceContentServiceTests.java b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/SourceContentServiceTests.java index 8b190ba06..64958eb94 100644 --- a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/SourceContentServiceTests.java +++ b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/SourceContentServiceTests.java @@ -42,7 +42,7 @@ class SourceContentServiceTests { private final SourceDocumentEvidenceQuery evidenceQuery = mock(SourceDocumentEvidenceQuery.class); private final ObjectStoragePort objects = mock(ObjectStoragePort.class); private final PermissionAuditService audit = mock(PermissionAuditService.class); - private final SourceContentService service = new SourceContentService( + private final SourceContentService service = new DefaultSourceContentService( authorization, evidenceQuery, objects, audit); @Test diff --git a/docs/increments/active/2026-07-31-spring-modulith-package-refactor/design.md b/docs/increments/active/2026-07-31-spring-modulith-package-refactor/design.md index b557c76ee..6eebda894 100644 --- a/docs/increments/active/2026-07-31-spring-modulith-package-refactor/design.md +++ b/docs/increments/active/2026-07-31-spring-modulith-package-refactor/design.md @@ -230,6 +230,25 @@ queries and intentional adapter interfaces in separate code PRs below the documented security read model; this increment claims Java/domain/API closure, not datastore autonomy. +## Retrieval Adapter Boundary + +The API and Worker retain engine selection and provider wiring, but they inject +Retrieval contracts rather than implementation classes. The existing canonical +hybrid, GraphRAG, citation/source opening, authorization-resource, bounded +single-Asset inspection, and embedding-registry capabilities become interfaces +with unchanged method shapes. Full evidence-scope resolution stays +package-private so its internal scope model is not laundered into the API. The +default or JDBC implementations use distinct package-private types. A public canonical-engine configuration is the explicit +opt-in used by the API and by Worker integration tests; the production Worker +excludes that configuration because it does not serve interactive queries. + +This is the adapter-interface slice already required by the independent +Retrieval closure verdict, not a new policy or ownership decision. Query and +embedding properties/value types remain intentional adapter configuration +contracts. Retrieval stays open until the remaining root-package persistence +and concrete types are internalized and the exact final dependency allowlist is +verified. + See [retrieval-closure-challenge-verdict.md](retrieval-closure-challenge-verdict.md) for reviewer availability, exact ownership, the counterattack, blocking diff --git a/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md b/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md index b2c98f42f..b2bdadb37 100644 --- a/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md +++ b/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md @@ -1215,7 +1215,7 @@ direct audit against the architecture verdict found no defect, review, inline comment, or review thread. Both the PR head `815640cd` and merge commit are ancestors of current `origin/main`. -## Current Pull Request Gates +## Fortieth Pull Request Evidence - Retrieval owns one `GraphEvidenceVerifier` contract and immutable `VerifiedGraphEvidenceScope`; its package-private implementation alone may use @@ -1284,3 +1284,46 @@ After a second main sync at `8bf800c6` brought the governed document-action Retrieval changes, the Graph/verifier classes plus full Modulith slice passed again in 51s. Documentation passed for 506 Markdown files and 8 mirrored domain pairs, and all 41 release-policy tests passed on Node 24.15.0. + +PR #263 merged as `7772104d9733b6cb8361693cce42b3521f8a37f1` after all +required CI checks passed. CodeRabbit's fail-closed findings were fixed at head +`0a0f0eaaaf9a512762bccf10009ca66332f4e10d`; all five inline threads were +answered and resolved. Both the reviewed head and merge commit are ancestors of +current `origin/main`. + +## Current Pull Request Gates + +- Canonical hybrid search, GraphRAG search, citation/source content, + authorization-resource lookup, bounded Asset inspection, and embedding + profile resolution are adapter-facing interfaces rather than concrete types. +- Full evidence-scope resolution remains package-private and its internal scope + value does not leak through the API inspection contract. +- Their default/JDBC implementations are distinct package-private classes, so + API and Worker cannot import them. Existing method shapes and domain values + remain unchanged. +- The API selects canonical or GraphRAG through those interfaces. The production + Worker excludes the explicit canonical-query configuration, while Worker + integration tests opt into that same configuration when exercising real + search behavior. +- A failing-first structural test proves the seven adapter contracts are interfaces + and their seven implementation types are non-public. +- Focused engine, content, scope, registry, API configuration/controller, and + Worker integration tests pass; full Core/API/Worker and terminating repository + gates follow before the PR is opened. +- This is a code PR below 100 changed files. Retrieval remains open only for + root implementation/persistence internalization and its exact final closure. + +Local verification started with the seven-contract interface guard failing on +the unchanged concrete classes in 28s. Core/API/Worker main and test compilation +then passed in 27s. Focused engine, content, scope, registry, Modulith, API +controller/configuration, API context, external-principal, admin-inspector, and +Worker PostgreSQL integration slices passed. The combined full Core/API/Worker +run completed 142 test classes with zero failures in about 5m43s. The docs +operating-model check passed across 506 Markdown files and 8 mirrored domain +pairs; all 41 release-policy tests passed on Node 24.15.0; and the terminating +repository-wide `clean test` initially passed 99 tasks in 1m09s. Exact API and +Worker ArchUnit dependency-surface guards were then added and passed in 57s; a +fresh terminating `clean test` including those guards passed 108 tasks in +5m18s. The mechanical audit found 41 changed paths, no migration, no empty file, +no external import of Retrieval implementation/scope/store/candidate types, and +a clean whitespace diff. diff --git a/docs/specs/domains/secure-graph-rag.md b/docs/specs/domains/secure-graph-rag.md index 6b48a076b..7c2b92507 100644 --- a/docs/specs/domains/secure-graph-rag.md +++ b/docs/specs/domains/secure-graph-rag.md @@ -8,7 +8,7 @@ payload-boundary configuration this document states — `apps/api/src/main/resources/application*.yml` and `apps/worker/src/main/resources/application*.yml`. -Reconciled: `2026-08-02-spring-modulith-package-refactor (f2cf3c67)`. +Reconciled: `2026-08-02-spring-modulith-package-refactor (7772104d)`. ## Current Contract @@ -29,6 +29,9 @@ Reconciled: `2026-08-02-spring-modulith-package-refactor (f2cf3c67)`. reject an unverified Knowledge Space, and the canonical recheck is narrowed to the assets authorized for that exact Space. - `SECURE_MIX` is the product default. Strategy selection remains internal. +- API engine selection consumes the GraphRAG engine interface; its concrete + Retrieval implementation is package-private and constructed by Retrieval's + conditional runtime configuration. - Query results preserve structured entity, relation, and chunk selections. Entity and relation descriptions retain their individual chunk evidence; they are never reduced to an authorization-free merged string. diff --git a/docs/specs/domains/secure-retrieval.md b/docs/specs/domains/secure-retrieval.md index 3b453906d..e4f3a161a 100644 --- a/docs/specs/domains/secure-retrieval.md +++ b/docs/specs/domains/secure-retrieval.md @@ -5,7 +5,7 @@ Source: `core/src/main/java/com/orgmemory/core/knowledge`, `apps/api/src/main/java/com/orgmemory/api/knowledge`, and `integrations/authorization-openfga`. -Reconciled: `2026-08-02-spring-modulith-package-refactor (f2cf3c67)`. +Reconciled: `2026-08-02-spring-modulith-package-refactor (7772104d)`. ## Current Behavior @@ -29,9 +29,13 @@ its request snapshot and is bounded by the configured two-minute turn timeout. Parent Knowledge exposes the permission-aware query, immutable evidence, secure result, and verified grounding through the exact `knowledge::search` -named interface. Assistant and Asset Registry cross that interface; the open -Retrieval nested module retains the concrete engines, authorization sequence, -ranking, and persistence while its remaining adapter seams are closed. +named interface. Assistant and Asset Registry cross that interface. API and +Worker inject Retrieval interfaces for engine selection, citation/source +opening, bounded single-Asset authorization inspection, and embedding-profile +resolution. Full evidence-scope resolution and the default/JDBC implementations +are package-private. The +open Retrieval nested module retains authorization, ranking, and persistence +while its remaining root implementation types are internalized before closure. Asset existence, active authorization-scope, and current catalog reads cross one Asset-owned query that keeps tenant and lifecycle predicates behind the closed Asset module; Retrieval imports neither Asset repository. diff --git a/docs/tests/domains/secure-graph-rag.md b/docs/tests/domains/secure-graph-rag.md index bf8e0a9d5..c474d3ff7 100644 --- a/docs/tests/domains/secure-graph-rag.md +++ b/docs/tests/domains/secure-graph-rag.md @@ -9,7 +9,7 @@ Source: `components/graph-rag-core/src/test`, `core/src/test/java/com/orgmemory/core/knowledge`, and `apps/web/test/e2e`. -Reconciled: `2026-08-02-spring-modulith-package-refactor (f2cf3c67)`. +Reconciled: `2026-08-02-spring-modulith-package-refactor (7772104d)`. ## Automated @@ -84,6 +84,10 @@ Reconciled: `2026-08-02-spring-modulith-package-refactor (f2cf3c67)`. fallback telemetry. Multi-space tests additionally prove one preparation, bounded concurrent snapshot execution, deterministic collection, fail-closed multi-space reranking and hashed snapshot-stage telemetry. +- The Modulith adapter-contract guard proves both GraphRAG and canonical engine + surfaces are interfaces and their default implementations are package-private; + the API and Worker exact dependency-surface guards prevent concrete imports, + and API Assistant configuration tests retain engine-selection behavior. - Keyword-cache tests prove exact hit/miss isolation across organization, language, query strategy, route and query, plus trusted-keyword bypass. - OpenTelemetry adapter tests prove the closed payload-free attribute set, diff --git a/docs/tests/domains/secure-retrieval.md b/docs/tests/domains/secure-retrieval.md index 06af480dd..66b34d4ca 100644 --- a/docs/tests/domains/secure-retrieval.md +++ b/docs/tests/domains/secure-retrieval.md @@ -5,7 +5,7 @@ Source: `core/src/test/java/com/orgmemory/core/knowledge`, `apps/api/src/test/java/com/orgmemory/api/knowledge`, and `integrations/authorization-openfga/src/test`. -Reconciled: `2026-08-02-spring-modulith-package-refactor (f2cf3c67)`. +Reconciled: `2026-08-02-spring-modulith-package-refactor (7772104d)`. Primary evidence: `apps/api/src/test/java/com/orgmemory/api/knowledge/KnowledgeRetrievalIntegrationTests.java` and `core/src/test/java/com/orgmemory/core/permission/KnowledgePermissionPolicyTests.java`. @@ -28,6 +28,7 @@ Primary evidence: `apps/api/src/test/java/com/orgmemory/api/knowledge/KnowledgeR | Retrieval reloads active persisted department and Executive facts through Organization-owned queries and imports no Organization persistence or role types | `JpaKnowledgeAccessSubjectQueryTests`, `JpaOrganizationResourceQueryTests`, `ModulithVerificationTests#retrievalDoesNotDependOnOrganizationPersistenceOrRoleTypes`, `#retrievalOrganizationReadsUseOnlyOwnerQueries`, `SecureSourceVisibilityAdapterTests`, `KnowledgeRetrievalIntegrationTests` | | Citation opening consumes Source Ledger-owned immutable evidence without revision/blob persistence leakage, preserves typed unavailable audit reasons, and closes integrity-mismatched content before an allow audit | `SourceCitationEvidenceQueryTests`, `CitationContentServiceTests`, `ModulithVerificationTests#retrievalDoesNotDependOnSourceLedgerCitationPersistenceOrStatusTypes`, `#citationContentUsesTheSourceLedgerOwnerQuery` | | Graph exploration, export, and curation consume only Retrieval's verifier and immutable snapshot; unknown Spaces fail closed, rechecks carry only the requested Space's assets, and the verifier accepts only one exact current governing-evidence candidate | `CanonicalGraphEvidenceVerifierTests`, `KnowledgeGraphExplorerServiceTests`, `KnowledgeGraphExportServiceTests`, `KnowledgeGraphCurationServiceTests`, `ModulithVerificationTests#graphConsumesOnlyRetrievalGraphContracts` | +| API and Worker adapter contracts are interfaces, bounded Asset inspection does not expose the full evidence scope, default/JDBC implementations cannot be imported, and each deployable pins its exact Retrieval dependency surface | `ModulithVerificationTests#retrievalAdapterContractsAreInterfaces`, `apps/api/.../RetrievalAdapterBoundaryTests`, `apps/worker/.../RetrievalAdapterBoundaryTests`, focused engine/content/scope/registry tests, API controller/configuration tests, Worker connector and ingestion integration tests | Request-boundary missing control role/incomplete actor returns `403`; generic resource `404` does not claim otherwise. Provider-backed evaluation, From 8ca89908b44ad3168dc96fdd7aff4870756a445a Mon Sep 17 00:00:00 2001 From: kl3inIT Date: Sun, 2 Aug 2026 11:29:43 +0700 Subject: [PATCH 2/5] docs(increment): record retrieval adapter gates --- .../2026-07-31-spring-modulith-package-refactor/plan.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md b/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md index b2bdadb37..ab0cca73f 100644 --- a/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md +++ b/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md @@ -1327,3 +1327,9 @@ fresh terminating `clean test` including those guards passed 108 tasks in 5m18s. The mechanical audit found 41 changed paths, no migration, no empty file, no external import of Retrieval implementation/scope/store/candidate types, and a clean whitespace diff. + +After merging current `origin/main` at `0b5b0cfd`, the full Modulith verifier, +both exact deployable dependency guards, and API engine-selection tests passed +again in 48s. The documentation check still passed across 506 Markdown files +and 8 mirrored domain pairs, and all 41 release-policy tests passed again on +Node 24.15.0. From d4495b456c4e63ed50a01657e875806dd3d685ac Mon Sep 17 00:00:00 2001 From: kl3inIT Date: Sun, 2 Aug 2026 11:58:28 +0700 Subject: [PATCH 3/5] fix(retrieval): harden adapter review boundaries --- ...DefaultCanonicalHybridKnowledgeSearch.java | 3 +- .../DefaultCitationContentService.java | 18 ++++++ ...aultGraphRagKnowledgeRetrievalService.java | 42 +++++--------- .../KnowledgeAssetAccessInspector.java | 2 +- .../KnowledgeEvidenceScopeResolver.java | 42 ++++++++++++-- .../KnowledgeSearchAuthorizationService.java | 22 ++++++- .../CanonicalHybridKnowledgeSearchTests.java | 14 +++-- .../CitationContentServiceTests.java | 11 +++- ...raphRagKnowledgeRetrievalServiceTests.java | 1 + .../KnowledgeEvidenceScopeResolverTests.java | 57 +++++++++++++++++++ 10 files changed, 169 insertions(+), 43 deletions(-) diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCanonicalHybridKnowledgeSearch.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCanonicalHybridKnowledgeSearch.java index 291540a2a..501cd6e14 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCanonicalHybridKnowledgeSearch.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCanonicalHybridKnowledgeSearch.java @@ -98,7 +98,8 @@ public SecureKnowledgeSearchResult search( requestId, normalizedQuery, unavailable.reasonCode(), - unavailable.policyVersion()); + unavailable.policyVersion(), + unavailable); } Set authorizedAssetIds = evidenceScope.allAssetIds(); if (authorizedAssetIds.isEmpty()) { diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCitationContentService.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCitationContentService.java index f904a3447..0da42101c 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCitationContentService.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultCitationContentService.java @@ -95,6 +95,24 @@ public CitationContent open( || evidence.storedContentLength() != content.metadata().contentLength()) { closeQuietly(content); + audit.record(new PermissionAuditCommand( + actor.organizationId(), + actor.userId(), + "READ_CITATION", + "KNOWLEDGE_CHUNK", + chunkId.toString(), + PermissionAuditDecision.DENY, + "CITATION_BLOB_INTEGRITY_FAILED", + authorizationModelId, + normalizedRequestId, + null, + currentCandidate.ingestionAclSnapshotId(), + currentCandidate.currentAclSnapshotId(), + currentCandidate.authorizationModelId(), + currentCandidate.sourceRevisionId(), + currentCandidate.chunkId(), + currentCandidate.embeddingProfileId(), + currentCandidate.projectionGeneration())); throw new KnowledgeRetrievalUnavailableException( "Citation evidence failed its integrity check"); } diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultGraphRagKnowledgeRetrievalService.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultGraphRagKnowledgeRetrievalService.java index a66e5ec7a..118035e28 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultGraphRagKnowledgeRetrievalService.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/DefaultGraphRagKnowledgeRetrievalService.java @@ -206,8 +206,7 @@ private void emit( long startedAt, int outputCount, String failureCode) { - try { - events.emit(new GraphRagEventSink.GraphRagEvent( + safeEmit(new GraphRagEventSink.GraphRagEvent( operationId, organizationId, GraphRagEventSink.Stage.RETRIEVE, @@ -220,9 +219,6 @@ private void emit( null, failureCode, Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } } private static String failureCode(RuntimeException failure) { @@ -386,7 +382,9 @@ private SecureKnowledgeSearchResult search( Map canonicalByChunk = verified.stream() .collect(Collectors.toMap( SecureRetrievalCandidate::chunkId, - Function.identity())); + Function.identity(), + (left, right) -> left, + LinkedHashMap::new)); Map scoreByChunk = consolidated.grounding() .chunks() .stream() @@ -615,8 +613,7 @@ private void emitStage( long startedAt, int inputCount, int outputCount) { - try { - events.emit(new GraphRagEventSink.GraphRagEvent( + safeEmit(new GraphRagEventSink.GraphRagEvent( operationId, organizationId, stage, @@ -631,9 +628,6 @@ private void emitStage( null, null, Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } } /** @@ -651,8 +645,7 @@ private void emitAssembledContext( SecureContextBudget budget) { LightRagGrounding grounding = prepared.grounding(); ContextTokenUsage usage = grounding.tokenUsage(); - try { - events.emit(new GraphRagEventSink.GraphRagEvent( + safeEmit(new GraphRagEventSink.GraphRagEvent( operationId, organizationId, GraphRagEventSink.Stage.ASSEMBLE_CONTEXT, @@ -677,9 +670,6 @@ private void emitAssembledContext( prepared.droppedContributions()), null, Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } } private void emitPreparedStage( @@ -691,8 +681,7 @@ private void emitPreparedStage( int outputCount, String modelRouteFingerprint, GraphRagEventSink.CacheStatus cacheStatus) { - try { - events.emit(new GraphRagEventSink.GraphRagEvent( + safeEmit(new GraphRagEventSink.GraphRagEvent( operationId, organizationId, stage, @@ -705,9 +694,6 @@ private void emitPreparedStage( cacheStatus, null, Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } } private void emitSnapshotStage( @@ -726,8 +712,7 @@ private void emitSnapshotStage( namespace.workspace(), "collection", namespace.collection())); - try { - events.emit(new GraphRagEventSink.GraphRagEvent( + safeEmit(new GraphRagEventSink.GraphRagEvent( operationId, organizationId, GraphRagEventSink.Stage.RETRIEVE_SNAPSHOT, @@ -740,9 +725,6 @@ private void emitSnapshotStage( null, null, Instant.now())); - } catch (RuntimeException ignoredTelemetryFailure) { - // Telemetry must never become a retrieval availability dependency. - } } private record SnapshotQueryResult( @@ -857,8 +839,7 @@ private void emitRerank( String routeFingerprint = CanonicalCacheKeyHasher.sha256( "reranker-route", Map.of("provider", policy.rerank().provider())); - try { - events.emit(new GraphRagEventSink.GraphRagEvent( + safeEmit(new GraphRagEventSink.GraphRagEvent( operationId, organizationId, GraphRagEventSink.Stage.RERANK, @@ -871,6 +852,11 @@ private void emitRerank( null, failureCode, Instant.now())); + } + + private void safeEmit(GraphRagEventSink.GraphRagEvent event) { + try { + events.emit(event); } catch (RuntimeException ignoredTelemetryFailure) { // Telemetry must never become a retrieval availability dependency. } diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeAssetAccessInspector.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeAssetAccessInspector.java index 58c8d4b29..39f92494b 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeAssetAccessInspector.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeAssetAccessInspector.java @@ -5,7 +5,7 @@ import java.time.Instant; import java.util.UUID; -/** Bounded adapter query for one already relationship-authorized Knowledge Asset. */ +/** Bounded adapter query that rechecks relationship and canonical access for one Knowledge Asset. */ public interface KnowledgeAssetAccessInspector { AssetInspection inspectAsset( diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolver.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolver.java index cf0d981be..8a21633e7 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolver.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolver.java @@ -6,6 +6,7 @@ import com.orgmemory.core.knowledge.acl.SourceAclQuery; import com.orgmemory.core.authorization.AuthorizedResourceQuery; import com.orgmemory.core.authorization.AccessState; +import com.orgmemory.core.authorization.BatchAuthorizationQuery; import com.orgmemory.core.authorization.PermissionKey; import com.orgmemory.core.authorization.RelationshipAuthorizationSetPort; import com.orgmemory.core.authorization.ResourceRef; @@ -168,10 +169,7 @@ public ResolvedKnowledgeEvidenceScope resolve( visibleBySpace); } - /** - * Reuses the canonical retrieval eligibility query for one bounded, already - * relationship-authorized asset inspected by an audit viewer. - */ + /** Rechecks relationship authorization before canonical retrieval eligibility. */ @Transactional(readOnly = true) @Override public AssetInspection inspectAsset( @@ -188,6 +186,42 @@ public AssetInspection inspectAsset( if (subject == null) { return new AssetInspection(AccessState.DENIED, "SUBJECT_INACTIVE"); } + ResourceRef resource = ResourceRef.of( + actor.organizationId(), + RESOURCE_TYPE, + assetId); + var checked = authorization.batchCheck(new BatchAuthorizationQuery( + actor.organizationId(), + actor.principal(), + CAN_VIEW, + List.of(resource))); + if (!checked.resolved()) { + return new AssetInspection(AccessState.UNKNOWN, checked.reasonCode()); + } + if (checked.decisions().size() != 1) { + return new AssetInspection( + AccessState.UNKNOWN, + "RELATIONSHIP_DECISION_INCOMPLETE"); + } + if (!Objects.equals(authorizationModelId, checked.policyVersion())) { + return new AssetInspection( + AccessState.UNKNOWN, + "AUTHORIZATION_MODEL_MISMATCH"); + } + var decision = checked.decisions().get(resource); + if (decision == null) { + return new AssetInspection( + AccessState.UNKNOWN, + "RELATIONSHIP_DECISION_INCOMPLETE"); + } + if (!Objects.equals(authorizationModelId, decision.policyVersion())) { + return new AssetInspection( + AccessState.UNKNOWN, + "AUTHORIZATION_MODEL_MISMATCH"); + } + if (!decision.allowed()) { + return new AssetInspection(AccessState.DENIED, decision.reasonCode()); + } var scope = new SecureKnowledgeRetrievalStore.RetrievalScope( actor.organizationId(), actor.userId(), diff --git a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeSearchAuthorizationService.java b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeSearchAuthorizationService.java index 2f2af20a4..bca65bb03 100644 --- a/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeSearchAuthorizationService.java +++ b/core/src/main/java/com/orgmemory/core/knowledge/retrieval/KnowledgeSearchAuthorizationService.java @@ -72,6 +72,22 @@ KnowledgeRetrievalUnavailableException unavailable( String query, String reason, String policyVersion) { + return unavailable( + actor, + requestId, + query, + reason, + policyVersion, + null); + } + + KnowledgeRetrievalUnavailableException unavailable( + CurrentActor actor, + String requestId, + String query, + String reason, + String policyVersion, + Throwable cause) { audit.record(command( actor, requestId, @@ -79,8 +95,10 @@ KnowledgeRetrievalUnavailableException unavailable( PermissionAuditDecision.DENY, reason, policyVersion)); - return new KnowledgeRetrievalUnavailableException( - "Secure knowledge retrieval is temporarily unavailable"); + String message = "Secure knowledge retrieval is temporarily unavailable"; + return cause == null + ? new KnowledgeRetrievalUnavailableException(message) + : new KnowledgeRetrievalUnavailableException(message, cause); } PermissionAuditCommand command( diff --git a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchTests.java b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchTests.java index a6b3dccc3..a66679eda 100644 --- a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchTests.java +++ b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CanonicalHybridKnowledgeSearchTests.java @@ -1,6 +1,7 @@ package com.orgmemory.core.knowledge.retrieval; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; @@ -98,13 +99,16 @@ void explicitEntryDenialStopsBeforeListingObjects() { @Test void providerOutageFailsClosed() { - when(evidenceScopes.resolve(actor, MODEL_ID)).thenThrow( - new KnowledgeEvidenceScopeUnavailableException( - "OPENFGA_TIMEOUT", - MODEL_ID)); + var cause = new KnowledgeEvidenceScopeUnavailableException( + "OPENFGA_TIMEOUT", + MODEL_ID); + when(evidenceScopes.resolve(actor, MODEL_ID)).thenThrow(cause); - assertThrows(KnowledgeRetrievalUnavailableException.class, + var mapped = assertThrows( + KnowledgeRetrievalUnavailableException.class, () -> service.search(actor, "leave policy", 10, "request-2")); + + assertSame(cause, mapped.getCause()); } @Test diff --git a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CitationContentServiceTests.java b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CitationContentServiceTests.java index 9f41c49a8..200a61561 100644 --- a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CitationContentServiceTests.java +++ b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/CitationContentServiceTests.java @@ -100,7 +100,7 @@ void unavailableBlobRetainsItsOpaqueAuditReason() { } @Test - void storageIntegrityMismatchClosesContentBeforeAllowAudit() throws Exception { + void storageIntegrityMismatchClosesContentAndRecordsDenyAudit() throws Exception { Fixture fixture = new Fixture(); fixture.authorizeCitation(); fixture.citationEvidence(); @@ -121,7 +121,14 @@ void storageIntegrityMismatchClosesContentBeforeAllowAudit() throws Exception { () -> fixture.service.open(ACTOR, CHUNK_ID, "request-1")); verify(stream).close(); - verify(fixture.audit, never()).record(any()); + ArgumentCaptor audit = + ArgumentCaptor.forClass( + com.orgmemory.core.permission.PermissionAuditCommand.class); + verify(fixture.audit).record(audit.capture()); + assertEquals( + com.orgmemory.core.permission.PermissionAuditDecision.DENY, + audit.getValue().decision()); + assertEquals("CITATION_BLOB_INTEGRITY_FAILED", audit.getValue().reasonCode()); } private static void unavailableEvidenceRetainsItsOpaqueAuditReason( diff --git a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalServiceTests.java b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalServiceTests.java index 2bafa5cba..ddc06d387 100644 --- a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalServiceTests.java +++ b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/GraphRagKnowledgeRetrievalServiceTests.java @@ -410,6 +410,7 @@ void verifiesTheCompleteGraphGroundingBeforeCreatingTheModelInput() { MODEL_ID)); RecordingRecheckedStore canonical = new RecordingRecheckedStore(List.of( + candidate(ENTITY_CHUNK_ID), candidate(ENTITY_CHUNK_ID), candidate(RELATION_CHUNK_ID), candidate(CHUNK_ID))); diff --git a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolverTests.java b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolverTests.java index 91c47bd5b..b86f2a488 100644 --- a/core/src/test/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolverTests.java +++ b/core/src/test/java/com/orgmemory/core/knowledge/retrieval/KnowledgeEvidenceScopeResolverTests.java @@ -7,9 +7,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; +import com.orgmemory.core.authorization.AuthorizationDecision; import com.orgmemory.core.authorization.AuthorizedResourceSetResult; +import com.orgmemory.core.authorization.BatchAuthorizationResult; import com.orgmemory.core.authorization.RelationshipAuthorizationSetPort; import com.orgmemory.core.authorization.ResourceRef; import com.orgmemory.core.organization.CurrentActor; @@ -37,6 +40,60 @@ class KnowledgeEvidenceScopeResolverTests { UUID.fromString("71000000-0000-0000-0000-000000000004"); private static final String MODEL_ID = "model-v1"; + @Test + void assetInspectionRequiresRelationshipAuthorizationBeforeCanonicalVisibility() { + KnowledgeAccessSubjectQuery subjects = mock(KnowledgeAccessSubjectQuery.class); + RelationshipAuthorizationSetPort authorization = + mock(RelationshipAuthorizationSetPort.class); + SecureKnowledgeRetrievalStore canonical = + mock(SecureKnowledgeRetrievalStore.class); + @SuppressWarnings("unchecked") + ObjectProvider clocks = mock(ObjectProvider.class); + CurrentActor actor = new CurrentActor( + USER_ID, + ORGANIZATION_ID, + null, + "User", + "user@example.test"); + ResourceRef asset = ResourceRef.of( + ORGANIZATION_ID, + "knowledge_asset", + ASSET_ID); + when(subjects.findActive(ORGANIZATION_ID, USER_ID)) + .thenReturn(Optional.of(new KnowledgeAccessSubject( + USER_ID, + ORGANIZATION_ID, + null, + false))); + when(authorization.batchCheck(any())).thenReturn( + BatchAuthorizationResult.resolved( + java.util.Map.of( + asset, + AuthorizationDecision.deny( + "RELATIONSHIP_DENIED", + MODEL_ID)), + MODEL_ID)); + + var resolver = new KnowledgeEvidenceScopeResolver( + subjects, + authorization, + mock(KnowledgeAssetRetrievalQuery.class), + mock(SourceAclQuery.class), + canonical, + new KnowledgeRetrievalProperties(null, null, null, null), + clocks); + + KnowledgeAssetAccessInspector.AssetInspection result = resolver.inspectAsset( + actor, + ASSET_ID, + MODEL_ID, + Instant.parse("2026-08-02T00:00:00Z")); + + assertEquals(com.orgmemory.core.authorization.AccessState.DENIED, result.state()); + assertEquals("RELATIONSHIP_DENIED", result.reasonCode()); + verifyNoInteractions(canonical); + } + @Test void administratorUsesOpenFgaAssetVisibilityWithoutImplicitExecutiveAccess() { KnowledgeAccessSubjectQuery subjects = mock(KnowledgeAccessSubjectQuery.class); From cfb62e3cc8daf01fccb32fa7bf1e921ac26e5f94 Mon Sep 17 00:00:00 2001 From: kl3inIT Date: Sun, 2 Aug 2026 11:59:17 +0700 Subject: [PATCH 4/5] docs(increment): record adapter review hardening --- .../plan.md | 15 +++++++++++++++ docs/specs/domains/secure-graph-rag.md | 4 +++- docs/specs/domains/secure-retrieval.md | 9 +++++++-- docs/tests/domains/secure-graph-rag.md | 7 ++++--- docs/tests/domains/secure-retrieval.md | 6 +++--- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md b/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md index ab0cca73f..3524b64a3 100644 --- a/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md +++ b/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md @@ -1333,3 +1333,18 @@ both exact deployable dependency guards, and API engine-selection tests passed again in 48s. The documentation check still passed across 506 Markdown files and 8 mirrored domain pairs, and all 41 release-policy tests passed again on Node 24.15.0. + +CodeRabbit raised six inline findings. Five valid findings are fixed: evidence- +scope unavailability retains its cause, citation integrity mismatch records a +deny audit, duplicate canonical chunk rows collapse deterministically, all six +GraphRAG telemetry emitters share one fail-safe guard, and the bounded Asset +inspector independently rechecks relationship authorization plus model identity +before canonical SQL. Four characterization tests failed first on the unchanged +implementation and passed after the fixes. The focused Retrieval tests, full +Modulith verifier, API admin integration, and exact API/Worker dependency guards +then passed sequentially. The remaining UPSERT suggestion is rejected because +the repository uses PostgreSQL's default Read Committed isolation: the +`ON CONFLICT DO NOTHING` command may observe a concurrent uniqueness conflict, +and the following repository `SELECT` starts a new command snapshot that sees +the committed row; a no-op update would add writes and lock/trigger semantics +without closing a real visibility gap. diff --git a/docs/specs/domains/secure-graph-rag.md b/docs/specs/domains/secure-graph-rag.md index 7c2b92507..9e0d1031f 100644 --- a/docs/specs/domains/secure-graph-rag.md +++ b/docs/specs/domains/secure-graph-rag.md @@ -8,7 +8,7 @@ payload-boundary configuration this document states — `apps/api/src/main/resources/application*.yml` and `apps/worker/src/main/resources/application*.yml`. -Reconciled: `2026-08-02-spring-modulith-package-refactor (7772104d)`. +Reconciled: `2026-08-02-spring-modulith-package-refactor (d4495b45)`. ## Current Contract @@ -174,6 +174,8 @@ Reconciled: `2026-08-02-spring-modulith-package-refactor (7772104d)`. - The complete selected entity/relation/chunk evidence closure is BatchChecked and re-read from the canonical ledger after ranking. Scope, OpenFGA model, ACL snapshot, source revision, and projection generation must still match. + Duplicate canonical rows for one chunk collapse deterministically to the + first verified candidate before evidence and audit assembly. That verified closure is the request authorization snapshot; the same pure-Java renderer creates the model prompt and citation numbering, and answer tokens stream without replaying the full authorization pipeline after diff --git a/docs/specs/domains/secure-retrieval.md b/docs/specs/domains/secure-retrieval.md index e4f3a161a..d6362adf6 100644 --- a/docs/specs/domains/secure-retrieval.md +++ b/docs/specs/domains/secure-retrieval.md @@ -5,7 +5,7 @@ Source: `core/src/main/java/com/orgmemory/core/knowledge`, `apps/api/src/main/java/com/orgmemory/api/knowledge`, and `integrations/authorization-openfga`. -Reconciled: `2026-08-02-spring-modulith-package-refactor (7772104d)`. +Reconciled: `2026-08-02-spring-modulith-package-refactor (d4495b45)`. ## Current Behavior @@ -33,7 +33,9 @@ named interface. Assistant and Asset Registry cross that interface. API and Worker inject Retrieval interfaces for engine selection, citation/source opening, bounded single-Asset authorization inspection, and embedding-profile resolution. Full evidence-scope resolution and the default/JDBC implementations -are package-private. The +are package-private. The bounded inspector independently rechecks the Asset's +relationship decision and authorization-model identity before canonical SQL, +so it does not rely on a caller-supplied authorization precondition. The open Retrieval nested module retains authorization, ranking, and persistence while its remaining root implementation types are internalized before closure. Asset existence, active authorization-scope, and current catalog reads cross @@ -66,6 +68,9 @@ revision plus a validated blob, and returns only immutable response and storage integrity metadata. Retrieval imports no Source Revision/Evidence Blob entity, repository, or lifecycle enum. Missing revision and unavailable blob outcomes remain distinct audit reasons even though both map to the same opaque `404`. +An object-storage length or digest mismatch closes the stream, records a +`CITATION_BLOB_INTEGRITY_FAILED` deny audit, and returns unavailable without an +allow audit. Control-plane roles (`ADMIN`, `REVIEWER`, `CONTRIBUTOR`, `VIEWER`) are separate from knowledge roles (`EMPLOYEE`, `MANAGER`, `DIRECTOR`, `EXECUTIVE`). Admin does diff --git a/docs/tests/domains/secure-graph-rag.md b/docs/tests/domains/secure-graph-rag.md index c474d3ff7..3ae5107af 100644 --- a/docs/tests/domains/secure-graph-rag.md +++ b/docs/tests/domains/secure-graph-rag.md @@ -9,7 +9,7 @@ Source: `components/graph-rag-core/src/test`, `core/src/test/java/com/orgmemory/core/knowledge`, and `apps/web/test/e2e`. -Reconciled: `2026-08-02-spring-modulith-package-refactor (7772104d)`. +Reconciled: `2026-08-02-spring-modulith-package-refactor (d4495b45)`. ## Automated @@ -80,8 +80,9 @@ Reconciled: `2026-08-02-spring-modulith-package-refactor (7772104d)`. executions of one prepared query. - Application tests prove entity/relation/chunk closure BatchCheck plus canonical recheck, authorization-model mismatch denial before rendering, - request-scope revocation retry, exact Assistant handoff, and sanitized rerank - fallback telemetry. Multi-space tests additionally prove one preparation, + deterministic duplicate-chunk collapse, request-scope revocation retry, + exact Assistant handoff, and sanitized rerank fallback telemetry. Multi-space + tests additionally prove one preparation, bounded concurrent snapshot execution, deterministic collection, fail-closed multi-space reranking and hashed snapshot-stage telemetry. - The Modulith adapter-contract guard proves both GraphRAG and canonical engine diff --git a/docs/tests/domains/secure-retrieval.md b/docs/tests/domains/secure-retrieval.md index 66b34d4ca..9f15bbff0 100644 --- a/docs/tests/domains/secure-retrieval.md +++ b/docs/tests/domains/secure-retrieval.md @@ -5,7 +5,7 @@ Source: `core/src/test/java/com/orgmemory/core/knowledge`, `apps/api/src/test/java/com/orgmemory/api/knowledge`, and `integrations/authorization-openfga/src/test`. -Reconciled: `2026-08-02-spring-modulith-package-refactor (7772104d)`. +Reconciled: `2026-08-02-spring-modulith-package-refactor (d4495b45)`. Primary evidence: `apps/api/src/test/java/com/orgmemory/api/knowledge/KnowledgeRetrievalIntegrationTests.java` and `core/src/test/java/com/orgmemory/core/permission/KnowledgePermissionPolicyTests.java`. @@ -23,10 +23,10 @@ Primary evidence: `apps/api/src/test/java/com/orgmemory/api/knowledge/KnowledgeR | OpenFGA model mismatch cannot reach the renderer | `GraphRagKnowledgeRetrievalServiceTests#authorizationModelMismatchCannotReachTheVerifiedRenderer` | | Authorization scope changing during retrieval retries without egress | `GraphRagKnowledgeRetrievalServiceTests#revocationBetweenRetrievalAndCitationCausesAFullRetryWithoutEgress` | | Parent Knowledge exposes only the exact four-type `knowledge::search` contract, and Assistant plus Asset Registry do not import Retrieval implementation | `ModulithVerificationTests#searchIsAnExactExplicitKnowledgeInterface`, `#topLevelSearchConsumersUseOnlyTheParentSearchInterface`, `#assistantAndAssetRegistryDoNotDependOnRetrievalImplementation` | -| One-asset admin inspection uses canonical retrieval eligibility after relationship allowance | `PermissionsAdminIntegrationTests#effectiveContentAccessSeparatesRelationshipGrantFromCanonicalDenial` | +| One-asset admin inspection independently requires relationship allowance before canonical retrieval eligibility | `KnowledgeEvidenceScopeResolverTests#assetInspectionRequiresRelationshipAuthorizationBeforeCanonicalVisibility`, `PermissionsAdminIntegrationTests#effectiveContentAccessSeparatesRelationshipGrantFromCanonicalDenial` | | Retrieval crosses one Asset-owned query for existence, active authorization scopes, and current catalog projection without importing Asset repositories | `JpaKnowledgeAssetRetrievalQueryTests`, `ModulithVerificationTests#retrievalDoesNotDependOnAssetRepositories`, `#retrievalAssetReadsUseOnlyTheOwnerQuery`, `ExternalPrincipalRetrievalIntegrationTests`, `PermissionsAdminIntegrationTests` | | Retrieval reloads active persisted department and Executive facts through Organization-owned queries and imports no Organization persistence or role types | `JpaKnowledgeAccessSubjectQueryTests`, `JpaOrganizationResourceQueryTests`, `ModulithVerificationTests#retrievalDoesNotDependOnOrganizationPersistenceOrRoleTypes`, `#retrievalOrganizationReadsUseOnlyOwnerQueries`, `SecureSourceVisibilityAdapterTests`, `KnowledgeRetrievalIntegrationTests` | -| Citation opening consumes Source Ledger-owned immutable evidence without revision/blob persistence leakage, preserves typed unavailable audit reasons, and closes integrity-mismatched content before an allow audit | `SourceCitationEvidenceQueryTests`, `CitationContentServiceTests`, `ModulithVerificationTests#retrievalDoesNotDependOnSourceLedgerCitationPersistenceOrStatusTypes`, `#citationContentUsesTheSourceLedgerOwnerQuery` | +| Citation opening consumes Source Ledger-owned immutable evidence without revision/blob persistence leakage, preserves typed unavailable audit reasons, and closes integrity-mismatched content with a deny audit before any allow audit | `SourceCitationEvidenceQueryTests`, `CitationContentServiceTests#storageIntegrityMismatchClosesContentAndRecordsDenyAudit`, `ModulithVerificationTests#retrievalDoesNotDependOnSourceLedgerCitationPersistenceOrStatusTypes`, `#citationContentUsesTheSourceLedgerOwnerQuery` | | Graph exploration, export, and curation consume only Retrieval's verifier and immutable snapshot; unknown Spaces fail closed, rechecks carry only the requested Space's assets, and the verifier accepts only one exact current governing-evidence candidate | `CanonicalGraphEvidenceVerifierTests`, `KnowledgeGraphExplorerServiceTests`, `KnowledgeGraphExportServiceTests`, `KnowledgeGraphCurationServiceTests`, `ModulithVerificationTests#graphConsumesOnlyRetrievalGraphContracts` | | API and Worker adapter contracts are interfaces, bounded Asset inspection does not expose the full evidence scope, default/JDBC implementations cannot be imported, and each deployable pins its exact Retrieval dependency surface | `ModulithVerificationTests#retrievalAdapterContractsAreInterfaces`, `apps/api/.../RetrievalAdapterBoundaryTests`, `apps/worker/.../RetrievalAdapterBoundaryTests`, focused engine/content/scope/registry tests, API controller/configuration tests, Worker connector and ingestion integration tests | From 45518b1b940cbcb90e8b4611d06e4c456d75225a Mon Sep 17 00:00:00 2001 From: kl3inIT Date: Sun, 2 Aug 2026 12:05:47 +0700 Subject: [PATCH 5/5] docs(increment): record terminating adapter gates --- .../2026-07-31-spring-modulith-package-refactor/plan.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md b/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md index 3524b64a3..e09590aa4 100644 --- a/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md +++ b/docs/increments/active/2026-07-31-spring-modulith-package-refactor/plan.md @@ -1348,3 +1348,9 @@ the repository uses PostgreSQL's default Read Committed isolation: the and the following repository `SELECT` starts a new command snapshot that sees the committed row; a no-op update would add writes and lock/trigger semantics without closing a real visibility gap. + +After the review-fix commits, the terminating sequential repository-wide +`clean test` passed all 99 tasks in 4m44s. The documentation operating-model +check passed for 506 Markdown files and 8 mirrored domain pairs, and all 41 +release-policy tests passed again on exact Node 24.15.0. The PR diff remains 43 +changed paths.